/* * MinIO Cloud Storage, (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package lock import ( "context" "encoding/xml" "errors" "fmt" "io" "net/http" "strings" "sync" "time" "github.com/beevik/ntp" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/env" ) // RetMode - object retention mode. type RetMode string const ( // RetGovernance - governance mode. RetGovernance RetMode = "GOVERNANCE" // RetCompliance - compliance mode. RetCompliance RetMode = "COMPLIANCE" ) // Valid - returns if retention mode is valid func (r RetMode) Valid() bool { switch r { case RetGovernance, RetCompliance: return true } return false } func parseRetMode(modeStr string) (mode RetMode) { switch strings.ToUpper(modeStr) { case "GOVERNANCE": mode = RetGovernance case "COMPLIANCE": mode = RetCompliance } return mode } // LegalHoldStatus - object legal hold status. type LegalHoldStatus string const ( // LegalHoldOn - legal hold is on. LegalHoldOn LegalHoldStatus = "ON" // LegalHoldOff - legal hold is off. LegalHoldOff LegalHoldStatus = "OFF" ) // Valid - returns true if legal hold status has valid values func (l LegalHoldStatus) Valid() bool { switch l { case LegalHoldOn, LegalHoldOff: return true } return false } func parseLegalHoldStatus(holdStr string) (st LegalHoldStatus) { switch strings.ToUpper(holdStr) { case "ON": st = LegalHoldOn case "OFF": st = LegalHoldOff } return st } // Bypass retention governance header. const ( AmzObjectLockBypassRetGovernance = "X-Amz-Bypass-Governance-Retention" AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" AmzObjectLockMode = "X-Amz-Object-Lock-Mode" AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" ) var ( // ErrMalformedBucketObjectConfig -indicates that the bucket object lock config is malformed ErrMalformedBucketObjectConfig = errors.New("invalid bucket object lock config") // ErrInvalidRetentionDate - indicates that retention date needs to be in ISO 8601 format ErrInvalidRetentionDate = errors.New("date must be provided in ISO 8601 format") // ErrPastObjectLockRetainDate - indicates that retention date must be in the future ErrPastObjectLockRetainDate = errors.New("the retain until date must be in the future") // ErrUnknownWORMModeDirective - indicates that the retention mode is invalid ErrUnknownWORMModeDirective = errors.New("unknown WORM mode directive") // ErrObjectLockMissingContentMD5 - indicates missing Content-MD5 header for put object requests with locking ErrObjectLockMissingContentMD5 = errors.New("content-MD5 HTTP header is required for Put Object requests with Object Lock parameters") // ErrObjectLockInvalidHeaders indicates that object lock headers are missing ErrObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied") // ErrMalformedXML - generic error indicating malformed XML ErrMalformedXML = errors.New("the XML you provided was not well-formed or did not validate against our published schema") ) const ( ntpServerEnv = "MINIO_NTP_SERVER" ) var ( ntpServer = env.Get(ntpServerEnv, "") ) // UTCNowNTP - is similar in functionality to UTCNow() // but only used when we do not wish to rely on system // time. func UTCNowNTP() (time.Time, error) { // ntp server is disabled if ntpServer == "" { return time.Now().UTC(), nil } return ntp.Time(ntpServer) } // Retention - bucket level retention configuration. type Retention struct { Mode RetMode Validity time.Duration } // IsEmpty - returns whether retention is empty or not. func (r Retention) IsEmpty() bool { return !r.Mode.Valid() || r.Validity == 0 } // Retain - check whether given date is retainable by validity time. func (r Retention) Retain(created time.Time) bool { t, err := UTCNowNTP() if err != nil { logger.LogIf(context.Background(), err) // Retain return true } return created.Add(r.Validity).After(t) } // BucketObjectLockConfig - map of bucket and retention configuration. type BucketObjectLockConfig struct { sync.RWMutex retentionMap map[string]*Retention } // Set - set retention configuration. func (config *BucketObjectLockConfig) Set(bucketName string, retention *Retention) { config.Lock() config.retentionMap[bucketName] = retention config.Unlock() } // Get - Get retention configuration. func (config *BucketObjectLockConfig) Get(bucketName string) (r *Retention, ok bool) { config.RLock() defer config.RUnlock() r, ok = config.retentionMap[bucketName] return r, ok } // Remove - removes retention configuration. func (config *BucketObjectLockConfig) Remove(bucketName string) { config.Lock() delete(config.retentionMap, bucketName) config.Unlock() } // NewBucketObjectLockConfig returns initialized BucketObjectLockConfig func NewBucketObjectLockConfig() *BucketObjectLockConfig { return &BucketObjectLockConfig{ retentionMap: make(map[string]*Retention), } } // DefaultRetention - default retention configuration. type DefaultRetention struct { XMLName xml.Name `xml:"DefaultRetention"` Mode RetMode `xml:"Mode"` Days *uint64 `xml:"Days"` Years *uint64 `xml:"Years"` } // Maximum support retention days and years supported by AWS S3. const ( // This tested by using `mc lock` command maximumRetentionDays = 36500 maximumRetentionYears = 100 ) // UnmarshalXML - decodes XML data. func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // Make subtype to avoid recursive UnmarshalXML(). type defaultRetention DefaultRetention retention := defaultRetention{} if err := d.DecodeElement(&retention, &start); err != nil { return err } switch retention.Mode { case RetGovernance, RetCompliance: default: return fmt.Errorf("unknown retention mode %v", retention.Mode) } if retention.Days == nil && retention.Years == nil { return fmt.Errorf("either Days or Years must be specified") } if retention.Days != nil && retention.Years != nil { return fmt.Errorf("either Days or Years must be specified, not both") } if retention.Days != nil { if *retention.Days == 0 { return fmt.Errorf("Default retention period must be a positive integer value for 'Days'") } if *retention.Days > maximumRetentionDays { return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days) } } else if *retention.Years == 0 { return fmt.Errorf("Default retention period must be a positive integer value for 'Years'") } else if *retention.Years > maximumRetentionYears { return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years) } *dr = DefaultRetention(retention) return nil } // Config - object lock configuration specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html type Config struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"ObjectLockConfiguration"` ObjectLockEnabled string `xml:"ObjectLockEnabled"` Rule *struct { DefaultRetention DefaultRetention `xml:"DefaultRetention"` } `xml:"Rule,omitempty"` } // UnmarshalXML - decodes XML data. func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // Make subtype to avoid recursive UnmarshalXML(). type objectLockConfig Config parsedConfig := objectLockConfig{} if err := d.DecodeElement(&parsedConfig, &start); err != nil { return err } if parsedConfig.ObjectLockEnabled != "Enabled" { return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element") } *config = Config(parsedConfig) return nil } // ToRetention - convert to Retention type. func (config *Config) ToRetention() (r *Retention) { r = &Retention{} if config.Rule != nil { r.Mode = config.Rule.DefaultRetention.Mode t, err := UTCNowNTP() if err != nil { logger.LogIf(context.Background(), err) // Do not change any configuration // upon NTP failure. return r } if config.Rule.DefaultRetention.Days != nil { r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t) } else { r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t) } } return r } // Maximum 4KiB size per object lock config. const maxObjectLockConfigSize = 1 << 12 // ParseObjectLockConfig parses ObjectLockConfig from xml func ParseObjectLockConfig(reader io.Reader) (*Config, error) { config := Config{} if err := xml.NewDecoder(io.LimitReader(reader, maxObjectLockConfigSize)).Decode(&config); err != nil { return nil, err } return &config, nil } // NewObjectLockConfig returns a initialized lock.Config struct func NewObjectLockConfig() *Config { return &Config{ ObjectLockEnabled: "Enabled", } } // RetentionDate is a embedded type containing time.Time to unmarshal // Date in Retention type RetentionDate struct { time.Time } // UnmarshalXML parses date from Retention and validates date format func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { var dateStr string err := d.DecodeElement(&dateStr, &startElement) if err != nil { return err } // While AWS documentation mentions that the date specified // must be present in ISO 8601 format, in reality they allow // users to provide RFC 3339 compliant dates. retDate, err := time.Parse(time.RFC3339, dateStr) if err != nil { return ErrInvalidRetentionDate } *rDate = RetentionDate{retDate} return nil } // MarshalXML encodes expiration date if it is non-zero and encodes // empty string otherwise func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { if *rDate == (RetentionDate{time.Time{}}) { return nil } return e.EncodeElement(rDate.Format(time.RFC3339), startElement) } // ObjectRetention specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html type ObjectRetention struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"Retention"` Mode RetMode `xml:"Mode,omitempty"` RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"` } // Maximum 4KiB size per object retention config. const maxObjectRetentionSize = 1 << 12 // ParseObjectRetention constructs ObjectRetention struct from xml input func ParseObjectRetention(reader io.Reader) (*ObjectRetention, error) { ret := ObjectRetention{} if err := xml.NewDecoder(io.LimitReader(reader, maxObjectRetentionSize)).Decode(&ret); err != nil { return nil, err } if !ret.Mode.Valid() { return &ret, ErrUnknownWORMModeDirective } t, err := UTCNowNTP() if err != nil { logger.LogIf(context.Background(), err) return &ret, ErrPastObjectLockRetainDate } if ret.RetainUntilDate.Before(t) { return &ret, ErrPastObjectLockRetainDate } return &ret, nil } // IsObjectLockRetentionRequested returns true if object lock retention headers are set. func IsObjectLockRetentionRequested(h http.Header) bool { if _, ok := h[AmzObjectLockMode]; ok { return true } if _, ok := h[AmzObjectLockRetainUntilDate]; ok { return true } return false } // IsObjectLockLegalHoldRequested returns true if object lock legal hold header is set. func IsObjectLockLegalHoldRequested(h http.Header) bool { _, ok := h[AmzObjectLockLegalHold] return ok } // IsObjectLockGovernanceBypassSet returns true if object lock governance bypass header is set. func IsObjectLockGovernanceBypassSet(h http.Header) bool { return strings.ToLower(h.Get(AmzObjectLockBypassRetGovernance)) == "true" } // IsObjectLockRequested returns true if legal hold or object lock retention headers are requested. func IsObjectLockRequested(h http.Header) bool { return IsObjectLockLegalHoldRequested(h) || IsObjectLockRetentionRequested(h) } // ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionDate, err error) { retMode := h.Get(AmzObjectLockMode) dateStr := h.Get(AmzObjectLockRetainUntilDate) if len(retMode) == 0 || len(dateStr) == 0 { return rmode, r, ErrObjectLockInvalidHeaders } rmode = parseRetMode(retMode) if !rmode.Valid() { return rmode, r, ErrUnknownWORMModeDirective } var retDate time.Time // While AWS documentation mentions that the date specified // must be present in ISO 8601 format, in reality they allow // users to provide RFC 3339 compliant dates. retDate, err = time.Parse(time.RFC3339, dateStr) if err != nil { return rmode, r, ErrInvalidRetentionDate } t, err := UTCNowNTP() if err != nil { logger.LogIf(context.Background(), err) return rmode, r, ErrPastObjectLockRetainDate } if retDate.Before(t) { return rmode, r, ErrPastObjectLockRetainDate } return rmode, RetentionDate{retDate}, nil } // GetObjectRetentionMeta constructs ObjectRetention from metadata func GetObjectRetentionMeta(meta map[string]string) ObjectRetention { var mode RetMode var retainTill RetentionDate var modeStr, tillStr string ok := false modeStr, ok = meta[strings.ToLower(AmzObjectLockMode)] if !ok { modeStr, ok = meta[AmzObjectLockMode] } if ok { mode = parseRetMode(modeStr) } tillStr, ok = meta[strings.ToLower(AmzObjectLockRetainUntilDate)] if !ok { tillStr, ok = meta[AmzObjectLockRetainUntilDate] } if ok { if t, e := time.Parse(time.RFC3339, tillStr); e == nil { retainTill = RetentionDate{t.UTC()} } } return ObjectRetention{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Mode: mode, RetainUntilDate: retainTill} } // GetObjectLegalHoldMeta constructs ObjectLegalHold from metadata func GetObjectLegalHoldMeta(meta map[string]string) ObjectLegalHold { holdStr, ok := meta[strings.ToLower(AmzObjectLockLegalHold)] if !ok { holdStr, ok = meta[AmzObjectLockLegalHold] } if ok { return ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: parseLegalHoldStatus(holdStr)} } return ObjectLegalHold{} } // ParseObjectLockLegalHoldHeaders parses request headers to construct ObjectLegalHold func ParseObjectLockLegalHoldHeaders(h http.Header) (lhold ObjectLegalHold, err error) { holdStatus, ok := h[AmzObjectLockLegalHold] if ok { lh := parseLegalHoldStatus(holdStatus[0]) if !lh.Valid() { return lhold, ErrUnknownWORMModeDirective } lhold = ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: lh} } return lhold, nil } // ObjectLegalHold specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html type ObjectLegalHold struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"LegalHold"` Status LegalHoldStatus `xml:"Status,omitempty"` } // IsEmpty returns true if struct is empty func (l *ObjectLegalHold) IsEmpty() bool { return !l.Status.Valid() } // ParseObjectLegalHold decodes the XML into ObjectLegalHold func ParseObjectLegalHold(reader io.Reader) (hold *ObjectLegalHold, err error) { hold = &ObjectLegalHold{} if err = xml.NewDecoder(reader).Decode(hold); err != nil { return } if !hold.Status.Valid() { return nil, ErrMalformedXML } return } // FilterObjectLockMetadata filters object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set. func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filterLegalHold bool) map[string]string { // Copy on write dst := metadata var copied bool delKey := func(key string) { if _, ok := metadata[key]; !ok { return } if !copied { dst = make(map[string]string, len(metadata)) for k, v := range metadata { dst[k] = v } copied = true } delete(dst, key) } legalHold := GetObjectLegalHoldMeta(metadata) if !legalHold.Status.Valid() || filterLegalHold { delKey(AmzObjectLockLegalHold) } ret := GetObjectRetentionMeta(metadata) if !ret.Mode.Valid() || filterRetention { delKey(AmzObjectLockMode) delKey(AmzObjectLockRetainUntilDate) return dst } return dst }