lifecycle: NoncurrentVersionExpiration considers noncurrent version age (#10444)

From https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions

```
When specifying the number of days in the NoncurrentVersionTransition
and NoncurrentVersionExpiration actions in a Lifecycle configuration,
note the following:

It is the number of days from when the version of the object becomes
noncurrent (that is, when the object is overwritten or deleted), that
Amazon S3 will perform the action on the specified object or objects.

Amazon S3 calculates the time by adding the number of days specified in
the rule to the time when the new successor version of the object is
created and rounding the resulting time to the next day midnight UTC.
For example, in your bucket, suppose that you have a current version of
an object that was created at 1/1/2014 10:30 AM UTC. If the new version
of the object that replaces the current version is created at 1/15/2014
10:30 AM UTC, and you specify 3 days in a transition rule, the
transition date of the object is calculated as 1/19/2014 00:00 UTC.
```
master
Anis Elleuch 4 years ago committed by GitHub
parent 1dce6918c2
commit af88772a78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 35
      cmd/data-crawler.go
  2. 16
      cmd/xl-storage-format-v2.go
  3. 13
      cmd/xl-storage.go
  4. 22
      pkg/bucket/lifecycle/lifecycle.go

@ -602,8 +602,9 @@ func (i *crawlItem) transformMetaDir() {
// actionMeta contains information used to apply actions.
type actionMeta struct {
oi ObjectInfo
numVersions int // The number of versions of this object
oi ObjectInfo
successorModTime time.Time // The modtime of the successor version
numVersions int // The number of versions of this object
}
// applyActions will apply lifecycle checks on to a scanned item.
@ -636,13 +637,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
versionID := meta.oi.VersionID
action := i.lifeCycle.ComputeAction(
lifecycle.ObjectOpts{
Name: i.objectPath(),
UserTags: meta.oi.UserTags,
ModTime: meta.oi.ModTime,
VersionID: meta.oi.VersionID,
DeleteMarker: meta.oi.DeleteMarker,
IsLatest: meta.oi.IsLatest,
NumVersions: meta.numVersions,
Name: i.objectPath(),
UserTags: meta.oi.UserTags,
ModTime: meta.oi.ModTime,
VersionID: meta.oi.VersionID,
DeleteMarker: meta.oi.DeleteMarker,
IsLatest: meta.oi.IsLatest,
NumVersions: meta.numVersions,
SuccessorModTime: meta.successorModTime,
})
if i.debug {
logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
@ -679,13 +681,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
// Recalculate action.
action = i.lifeCycle.ComputeAction(
lifecycle.ObjectOpts{
Name: i.objectPath(),
UserTags: obj.UserTags,
ModTime: obj.ModTime,
VersionID: obj.VersionID,
DeleteMarker: obj.DeleteMarker,
IsLatest: obj.IsLatest,
NumVersions: meta.numVersions,
Name: i.objectPath(),
UserTags: obj.UserTags,
ModTime: obj.ModTime,
VersionID: obj.VersionID,
DeleteMarker: obj.DeleteMarker,
IsLatest: obj.IsLatest,
NumVersions: meta.numVersions,
SuccessorModTime: meta.successorModTime,
})
if i.debug {
logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)

@ -20,6 +20,7 @@ import (
"bytes"
"errors"
"fmt"
"sort"
"strings"
"time"
@ -524,6 +525,20 @@ func (z xlMetaV2) TotalSize() int64 {
return total
}
type versionsSorter []FileInfo
func (v versionsSorter) Len() int { return len(v) }
func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v versionsSorter) Less(i, j int) bool {
if v[i].IsLatest {
return true
}
if v[j].IsLatest {
return false
}
return v[i].ModTime.After(v[j].ModTime)
}
// ListVersions lists current versions, and current deleted
// versions returns error for unexpected entries.
func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTime time.Time, err error) {
@ -562,6 +577,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
break
}
sort.Sort(versionsSorter(versions))
return versions, latestModTime, nil
}

@ -380,11 +380,18 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
}
var totalSize int64
for _, version := range fivs.Versions {
var numVersions = len(fivs.Versions)
for i, version := range fivs.Versions {
var successorModTime time.Time
if i > 0 {
successorModTime = fivs.Versions[i-1].ModTime
}
oi := version.ToObjectInfo(item.bucket, item.objectPath())
size := item.applyActions(ctx, objAPI, actionMeta{
numVersions: len(fivs.Versions),
oi: oi,
numVersions: numVersions,
successorModTime: successorModTime,
oi: oi,
})
if !version.Deleted {
totalSize += size

@ -176,13 +176,14 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
// ObjectOpts provides information to deduce the lifecycle actions
// which can be triggered on the resultant object.
type ObjectOpts struct {
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
}
// ComputeAction returns the action to perform by evaluating all lifecycle rules
@ -203,9 +204,10 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
}
if !rule.NoncurrentVersionExpiration.IsDaysNull() {
if obj.VersionID != "" && !obj.IsLatest {
// Non current versions should be deleted.
if time.Now().After(expectedExpiryTime(obj.ModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
// Non current versions should be deleted if their age exceeds non current days configuration
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
if time.Now().After(expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
return DeleteVersionAction
}
}

Loading…
Cancel
Save