From cdf4815a6b6ea39c8afa5bfc5f1609f49d2af2fb Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Thu, 21 May 2020 22:12:52 +0100 Subject: [PATCH] Add x-amz-expiration header in some S3 responses (#9667) x-amz-expiration is described in the S3 specification as a header which indicates if the object in question will expire any time in the future. --- cmd/http/headers.go | 3 +++ cmd/object-handlers-common.go | 14 ++++++++++ cmd/object-handlers.go | 12 +++++++++ pkg/bucket/lifecycle/lifecycle.go | 36 +++++++++++++++++++++----- pkg/bucket/lifecycle/lifecycle_test.go | 29 +++++++++++++++++++++ 5 files changed, 87 insertions(+), 7 deletions(-) diff --git a/cmd/http/headers.go b/cmd/http/headers.go index 26585e0a0..982a9ae4d 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -80,6 +80,9 @@ const ( // Multipart parts count AmzMpPartsCount = "x-amz-mp-parts-count" + // Object date/time of expiration + AmzExpiration = "x-amz-expiration" + // Dummy putBucketACL AmzACL = "x-amz-acl" diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 04f6fb089..eb055797d 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -18,6 +18,7 @@ package cmd import ( "context" + "fmt" "net/http" "regexp" "time" @@ -252,6 +253,19 @@ func isETagEqual(left, right string) bool { return canonicalizeETag(left) == canonicalizeETag(right) } +// setAmzExpirationHeader sets x-amz-expiration header with expiry time +// after analyzing the current bucket lifecycle rules if any. +func setAmzExpirationHeader(w http.ResponseWriter, bucket string, objInfo ObjectInfo) { + if lc, err := globalLifecycleSys.Get(bucket); err == nil { + ruleID, expiryTime := lc.PredictExpiryTime(objInfo.Name, objInfo.UserTags) + if !expiryTime.IsZero() { + w.Header()[xhttp.AmzExpiration] = []string{ + fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID), + } + } + } +} + // deleteObject is a convenient wrapper to delete an object, this // is a common function to be called from object handlers and // web handlers. diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 997d0e8ad..8644bd9ac 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -427,6 +427,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req } setHeadGetRespHeaders(w, r.URL.Query()) + setAmzExpirationHeader(w, bucket, objInfo) statusCodeWritten := false httpWriter := ioutil.WriteOnClose(w) @@ -606,6 +607,9 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re // Set any additional requested response headers. setHeadGetRespHeaders(w, r.URL.Query()) + // Set the expiration header + setAmzExpirationHeader(w, bucket, objInfo) + // Successful response. if rs != nil { w.WriteHeader(http.StatusPartialContent) @@ -1165,6 +1169,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } + setAmzExpirationHeader(w, dstBucket, objInfo) + response := generateCopyObjectResponse(getDecryptedETag(r.Header, objInfo, false), objInfo.ModTime) encodedSuccessResponse := encodeResponse(response) @@ -1476,10 +1482,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } } } + // We must not use the http.Header().Set method here because some (broken) // clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive). // Therefore, we have to set the ETag directly as map entry. w.Header()[xhttp.ETag] = []string{`"` + etag + `"`} + + setAmzExpirationHeader(w, bucket, objInfo) + writeSuccessResponseHeadersOnly(w) // Notify object created event. @@ -2526,6 +2536,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite // Set etag. w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""} + setAmzExpirationHeader(w, bucket, objInfo) + // Write success response. writeSuccessResponseXML(w, encodedSuccessResponse) diff --git a/pkg/bucket/lifecycle/lifecycle.go b/pkg/bucket/lifecycle/lifecycle.go index 8ed208a9f..8dd0c550b 100644 --- a/pkg/bucket/lifecycle/lifecycle.go +++ b/pkg/bucket/lifecycle/lifecycle.go @@ -95,9 +95,9 @@ func (lc Lifecycle) Validate() error { // FilterRuleActions returns the expiration and transition from the object name // after evaluating all rules. -func (lc Lifecycle) FilterRuleActions(objName, objTags string) (Expiration, Transition) { +func (lc Lifecycle) FilterRuleActions(objName, objTags string) (string, Expiration, Transition) { if objName == "" { - return Expiration{}, Transition{} + return "", Expiration{}, Transition{} } for _, rule := range lc.Rules { if rule.Status == Disabled { @@ -107,14 +107,14 @@ func (lc Lifecycle) FilterRuleActions(objName, objTags string) (Expiration, Tran if strings.HasPrefix(objName, rule.Prefix()) { if tags != "" { if strings.Contains(objTags, tags) { - return rule.Expiration, Transition{} + return rule.ID, rule.Expiration, Transition{} } } else { - return rule.Expiration, Transition{} + return rule.ID, rule.Expiration, Transition{} } } } - return Expiration{}, Transition{} + return "", Expiration{}, Transition{} } // ComputeAction returns the action to perform by evaluating all lifecycle rules @@ -124,16 +124,38 @@ func (lc Lifecycle) ComputeAction(objName, objTags string, modTime time.Time) Ac if modTime.IsZero() { return action } - exp, _ := lc.FilterRuleActions(objName, objTags) + _, exp, _ := lc.FilterRuleActions(objName, objTags) if !exp.IsDateNull() { if time.Now().After(exp.Date.Time) { action = DeleteAction } } if !exp.IsDaysNull() { - if time.Now().After(modTime.Add(time.Duration(exp.Days) * 24 * time.Hour)) { + if time.Now().After(expectedExpiryTime(modTime, exp.Days)) { action = DeleteAction } } return action } + +// expectedExpiryTime calculates the expiry date/time based on a object modtime. +// The expected expiry time is always a midnight time following the the object +// modification time plus the number of expiration days. +// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should +// expire in 1 day, then the expected expiry time is `Fri, 23 May 2020 00:00:00 GMT` +func expectedExpiryTime(modTime time.Time, days ExpirationDays) time.Time { + t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour) + return t.Truncate(24 * time.Hour) +} + +// PredictExpiryTime returns the expiry date/time of a given object +func (lc Lifecycle) PredictExpiryTime(objName, objTags string) (string, time.Time) { + ruleID, exp, _ := lc.FilterRuleActions(objName, objTags) + if !exp.IsDateNull() { + return ruleID, exp.Date.Time + } + if !exp.IsDaysNull() { + return ruleID, expectedExpiryTime(time.Now(), exp.Days) + } + return "", time.Time{} +} diff --git a/pkg/bucket/lifecycle/lifecycle_test.go b/pkg/bucket/lifecycle/lifecycle_test.go index b695620a6..e87928bf0 100644 --- a/pkg/bucket/lifecycle/lifecycle_test.go +++ b/pkg/bucket/lifecycle/lifecycle_test.go @@ -168,6 +168,35 @@ func TestMarshalLifecycleConfig(t *testing.T) { } } +func TestExpectedExpiryTime(t *testing.T) { + testCases := []struct { + modTime time.Time + days ExpirationDays + expected time.Time + }{ + { + time.Date(2020, time.March, 15, 10, 10, 10, 0, time.UTC), + 4, + time.Date(2020, time.March, 20, 0, 0, 0, 0, time.UTC), + }, + { + time.Date(2020, time.March, 15, 0, 0, 0, 0, time.UTC), + 1, + time.Date(2020, time.March, 17, 0, 0, 0, 0, time.UTC), + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + got := expectedExpiryTime(tc.modTime, tc.days) + if got != tc.expected { + t.Fatalf("Expected %v to be equal to %v", got, tc.expected) + } + }) + } + +} + func TestComputeActions(t *testing.T) { testCases := []struct { inputConfig string