From af88772a789af323bffc8ffb1cceb965f326dcf9 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Thu, 10 Sep 2020 02:11:24 +0100 Subject: [PATCH] lifecycle: NoncurrentVersionExpiration considers noncurrent version age (#10444) From https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions ``` When specifying the number of days in the NoncurrentVersionTransition and NoncurrentVersionExpiration actions in a Lifecycle configuration, note the following: It is the number of days from when the version of the object becomes noncurrent (that is, when the object is overwritten or deleted), that Amazon S3 will perform the action on the specified object or objects. Amazon S3 calculates the time by adding the number of days specified in the rule to the time when the new successor version of the object is created and rounding the resulting time to the next day midnight UTC. For example, in your bucket, suppose that you have a current version of an object that was created at 1/1/2014 10:30 AM UTC. If the new version of the object that replaces the current version is created at 1/15/2014 10:30 AM UTC, and you specify 3 days in a transition rule, the transition date of the object is calculated as 1/19/2014 00:00 UTC. ``` --- cmd/data-crawler.go | 35 +++++++++++++++++-------------- cmd/xl-storage-format-v2.go | 16 ++++++++++++++ cmd/xl-storage.go | 13 +++++++++--- pkg/bucket/lifecycle/lifecycle.go | 22 ++++++++++--------- 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/cmd/data-crawler.go b/cmd/data-crawler.go index 8dd421ea0..86fc96a0d 100644 --- a/cmd/data-crawler.go +++ b/cmd/data-crawler.go @@ -602,8 +602,9 @@ func (i *crawlItem) transformMetaDir() { // actionMeta contains information used to apply actions. type actionMeta struct { - oi ObjectInfo - numVersions int // The number of versions of this object + oi ObjectInfo + successorModTime time.Time // The modtime of the successor version + numVersions int // The number of versions of this object } // applyActions will apply lifecycle checks on to a scanned item. @@ -636,13 +637,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action versionID := meta.oi.VersionID action := i.lifeCycle.ComputeAction( lifecycle.ObjectOpts{ - Name: i.objectPath(), - UserTags: meta.oi.UserTags, - ModTime: meta.oi.ModTime, - VersionID: meta.oi.VersionID, - DeleteMarker: meta.oi.DeleteMarker, - IsLatest: meta.oi.IsLatest, - NumVersions: meta.numVersions, + Name: i.objectPath(), + UserTags: meta.oi.UserTags, + ModTime: meta.oi.ModTime, + VersionID: meta.oi.VersionID, + DeleteMarker: meta.oi.DeleteMarker, + IsLatest: meta.oi.IsLatest, + NumVersions: meta.numVersions, + SuccessorModTime: meta.successorModTime, }) if i.debug { logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action) @@ -679,13 +681,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action // Recalculate action. action = i.lifeCycle.ComputeAction( lifecycle.ObjectOpts{ - Name: i.objectPath(), - UserTags: obj.UserTags, - ModTime: obj.ModTime, - VersionID: obj.VersionID, - DeleteMarker: obj.DeleteMarker, - IsLatest: obj.IsLatest, - NumVersions: meta.numVersions, + Name: i.objectPath(), + UserTags: obj.UserTags, + ModTime: obj.ModTime, + VersionID: obj.VersionID, + DeleteMarker: obj.DeleteMarker, + IsLatest: obj.IsLatest, + NumVersions: meta.numVersions, + SuccessorModTime: meta.successorModTime, }) if i.debug { logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action) diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 59d54504e..beee45b7e 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -20,6 +20,7 @@ import ( "bytes" "errors" "fmt" + "sort" "strings" "time" @@ -524,6 +525,20 @@ func (z xlMetaV2) TotalSize() int64 { return total } +type versionsSorter []FileInfo + +func (v versionsSorter) Len() int { return len(v) } +func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] } +func (v versionsSorter) Less(i, j int) bool { + if v[i].IsLatest { + return true + } + if v[j].IsLatest { + return false + } + return v[i].ModTime.After(v[j].ModTime) +} + // ListVersions lists current versions, and current deleted // versions returns error for unexpected entries. func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTime time.Time, err error) { @@ -562,6 +577,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim break } + sort.Sort(versionsSorter(versions)) return versions, latestModTime, nil } diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 2459d6488..256349ba2 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -380,11 +380,18 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac } var totalSize int64 - for _, version := range fivs.Versions { + var numVersions = len(fivs.Versions) + + for i, version := range fivs.Versions { + var successorModTime time.Time + if i > 0 { + successorModTime = fivs.Versions[i-1].ModTime + } oi := version.ToObjectInfo(item.bucket, item.objectPath()) size := item.applyActions(ctx, objAPI, actionMeta{ - numVersions: len(fivs.Versions), - oi: oi, + numVersions: numVersions, + successorModTime: successorModTime, + oi: oi, }) if !version.Deleted { totalSize += size diff --git a/pkg/bucket/lifecycle/lifecycle.go b/pkg/bucket/lifecycle/lifecycle.go index 807e633f4..16a41d87b 100644 --- a/pkg/bucket/lifecycle/lifecycle.go +++ b/pkg/bucket/lifecycle/lifecycle.go @@ -176,13 +176,14 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule { // ObjectOpts provides information to deduce the lifecycle actions // which can be triggered on the resultant object. type ObjectOpts struct { - Name string - UserTags string - ModTime time.Time - VersionID string - IsLatest bool - DeleteMarker bool - NumVersions int + Name string + UserTags string + ModTime time.Time + VersionID string + IsLatest bool + DeleteMarker bool + NumVersions int + SuccessorModTime time.Time } // ComputeAction returns the action to perform by evaluating all lifecycle rules @@ -203,9 +204,10 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action { } if !rule.NoncurrentVersionExpiration.IsDaysNull() { - if obj.VersionID != "" && !obj.IsLatest { - // Non current versions should be deleted. - if time.Now().After(expectedExpiryTime(obj.ModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) { + if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() { + // Non current versions should be deleted if their age exceeds non current days configuration + // https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions + if time.Now().After(expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) { return DeleteVersionAction } }