fix: reply back user-metadata in lower case form (#9697)

some clients such as veeam expect the x-amz-meta to
be sent in lower cased form, while this does indeed
defeats the HTTP protocol contract it is harder to
change these applications, while these applications
get fixed appropriately in future.

x-amz-meta is usually sent in lowercased form
by AWS S3 and some applications like veeam
incorrectly end up relying on the case sensitivity
of the HTTP headers.

Bonus fixes

 - Fix the iso8601 time format to keep it same as
   AWS S3 response
 - Increase maxObjectList to 50,000 and use
   maxDeleteList as 10,000 whenever multi-object
   deletes are needed.
master
Harshavardhana 5 years ago committed by GitHub
parent 6e0575a53d
commit 7ea026ff1d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      cmd/admin-heal-ops.go
  2. 21
      cmd/api-headers.go
  3. 26
      cmd/api-response.go
  4. 2
      cmd/bucket-quota.go
  5. 2
      cmd/daily-lifecycle-ops.go
  6. 4
      cmd/disk-cache.go
  7. 2
      cmd/handler-utils.go
  8. 2
      cmd/http/headers.go
  9. 7
      cmd/object-handlers.go
  10. 21
      cmd/post-policy_test.go
  11. 2
      cmd/server_test.go
  12. 11
      cmd/web-handlers.go
  13. 8
      cmd/xl-zones.go
  14. 2
      mint/run/core/aws-sdk-php/quick-tests.php
  15. 2
      pkg/event/event.go

@ -221,8 +221,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
// Check if new heal sequence to be started overlaps with any // Check if new heal sequence to be started overlaps with any
// existing, running sequence // existing, running sequence
for k, hSeq := range ahs.healSeqMap { for k, hSeq := range ahs.healSeqMap {
if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) || if !hSeq.hasEnded() && (HasPrefix(k, h.path) || HasPrefix(h.path, k)) {
strings.HasPrefix(h.path, k)) {
errMsg = "The provided heal sequence path overlaps with an existing " + errMsg = "The provided heal sequence path overlaps with an existing " +
fmt.Sprintf("heal path: %s", k) fmt.Sprintf("heal path: %s", k)

@ -83,7 +83,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
} }
if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 { if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts))) w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))}
} }
if objInfo.ContentType != "" { if objInfo.ContentType != "" {
@ -106,18 +106,29 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
// Set tag count if object has tags // Set tag count if object has tags
tags, _ := url.ParseQuery(objInfo.UserTags) tags, _ := url.ParseQuery(objInfo.UserTags)
tagCount := len(tags) tagCount := len(tags)
if tagCount != 0 { if tagCount > 0 {
w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
} }
// Set all other user defined metadata. // Set all other user defined metadata.
for k, v := range objInfo.UserDefined { for k, v := range objInfo.UserDefined {
if HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(k, ReservedMetadataPrefix) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
// values to client. // values to client.
continue continue
} }
w.Header().Set(k, v) var isSet bool
for _, userMetadataPrefix := range userMetadataKeyPrefixes {
if !strings.HasPrefix(k, userMetadataPrefix) {
continue
}
w.Header()[strings.ToLower(k)] = []string{v}
isSet = true
break
}
if !isSet {
w.Header().Set(k, v)
}
} }
totalObjectSize, err := objInfo.GetActualSize() totalObjectSize, err := objInfo.GetActualSize()

@ -33,8 +33,10 @@ import (
) )
const ( const (
timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
maxObjectList = 10000 // Limit number of objects in a listObjectsResponse. iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
maxObjectList = 50000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
maxDeleteList = 10000 // Limit number of objects deleted in a delete call.
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
maxPartsList = 10000 // Limit number of parts in a listPartsResponse. maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
) )
@ -400,7 +402,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
for _, bucket := range buckets { for _, bucket := range buckets {
var listbucket = Bucket{} var listbucket = Bucket{}
listbucket.Name = bucket.Name listbucket.Name = bucket.Name
listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong) listbucket.CreationDate = bucket.Created.UTC().Format(iso8601TimeFormat)
listbuckets = append(listbuckets, listbucket) listbuckets = append(listbuckets, listbucket)
} }
@ -424,7 +426,7 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
continue continue
} }
content.Key = s3EncodeName(object.Name, encodingType) content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
if object.ETag != "" { if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\"" content.ETag = "\"" + object.ETag + "\""
} }
@ -440,9 +442,9 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
content.IsLatest = true content.IsLatest = true
versions = append(versions, content) versions = append(versions, content)
} }
data.Name = bucket data.Name = bucket
data.Versions = versions data.Versions = versions
data.EncodingType = encodingType data.EncodingType = encodingType
data.Prefix = s3EncodeName(prefix, encodingType) data.Prefix = s3EncodeName(prefix, encodingType)
data.KeyMarker = s3EncodeName(marker, encodingType) data.KeyMarker = s3EncodeName(marker, encodingType)
@ -475,7 +477,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
continue continue
} }
content.Key = s3EncodeName(object.Name, encodingType) content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
if object.ETag != "" { if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\"" content.ETag = "\"" + object.ETag + "\""
} }
@ -525,7 +527,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
continue continue
} }
content.Key = s3EncodeName(object.Name, encodingType) content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
if object.ETag != "" { if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\"" content.ETag = "\"" + object.ETag + "\""
} }
@ -539,7 +541,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
if metadata { if metadata {
content.UserMetadata = make(StringMap) content.UserMetadata = make(StringMap)
for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) { for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
if HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(k, ReservedMetadataPrefix) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
// values to client. // values to client.
continue continue
@ -574,7 +576,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
return CopyObjectResponse{ return CopyObjectResponse{
ETag: "\"" + etag + "\"", ETag: "\"" + etag + "\"",
LastModified: lastModified.UTC().Format(timeFormatAMZLong), LastModified: lastModified.UTC().Format(iso8601TimeFormat),
} }
} }
@ -582,7 +584,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse { func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
return CopyObjectPartResponse{ return CopyObjectPartResponse{
ETag: "\"" + etag + "\"", ETag: "\"" + etag + "\"",
LastModified: lastModified.UTC().Format(timeFormatAMZLong), LastModified: lastModified.UTC().Format(iso8601TimeFormat),
} }
} }
@ -627,7 +629,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
newPart.PartNumber = part.PartNumber newPart.PartNumber = part.PartNumber
newPart.ETag = "\"" + part.ETag + "\"" newPart.ETag = "\"" + part.ETag + "\""
newPart.Size = part.Size newPart.Size = part.Size
newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZLong) newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
listPartsResponse.Parts[index] = newPart listPartsResponse.Parts[index] = newPart
} }
return listPartsResponse return listPartsResponse
@ -657,7 +659,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
newUpload := Upload{} newUpload := Upload{}
newUpload.UploadID = upload.UploadID newUpload.UploadID = upload.UploadID
newUpload.Key = s3EncodeName(upload.Object, encodingType) newUpload.Key = s3EncodeName(upload.Object, encodingType)
newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong) newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
listMultipartUploadsResponse.Uploads[index] = newUpload listMultipartUploadsResponse.Uploads[index] = newUpload
} }
return listMultipartUploadsResponse return listMultipartUploadsResponse

@ -197,7 +197,7 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
numKeys := len(scorer.fileNames()) numKeys := len(scorer.fileNames())
for i, key := range scorer.fileNames() { for i, key := range scorer.fileNames() {
objects = append(objects, key) objects = append(objects, key)
if len(objects) < maxObjectList && (i < numKeys-1) { if len(objects) < maxDeleteList && (i < numKeys-1) {
// skip deletion until maxObjectList or end of slice // skip deletion until maxObjectList or end of slice
continue continue
} }

@ -85,7 +85,7 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
for { for {
var objects []string var objects []string
for obj := range objInfoCh { for obj := range objInfoCh {
if len(objects) == maxObjectList { if len(objects) == maxDeleteList {
// Reached maximum delete requests, attempt a delete for now. // Reached maximum delete requests, attempt a delete for now.
break break
} }

@ -99,14 +99,14 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
bkMeta := make(map[string]string) bkMeta := make(map[string]string)
cacheMeta := make(map[string]string) cacheMeta := make(map[string]string)
for k, v := range bkObjectInfo.UserDefined { for k, v := range bkObjectInfo.UserDefined {
if HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(k, ReservedMetadataPrefix) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
continue continue
} }
bkMeta[http.CanonicalHeaderKey(k)] = v bkMeta[http.CanonicalHeaderKey(k)] = v
} }
for k, v := range cacheObjInfo.UserDefined { for k, v := range cacheObjInfo.UserDefined {
if HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(k, ReservedMetadataPrefix) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
continue continue
} }

@ -104,6 +104,8 @@ func isDirectiveReplace(value string) bool {
var userMetadataKeyPrefixes = []string{ var userMetadataKeyPrefixes = []string{
"X-Amz-Meta-", "X-Amz-Meta-",
"X-Minio-Meta-", "X-Minio-Meta-",
"x-amz-meta-",
"x-minio-meta-",
} }
// extractMetadata extracts metadata from HTTP header and HTTP queryString. // extractMetadata extracts metadata from HTTP header and HTTP queryString.

@ -58,7 +58,7 @@ const (
// S3 object tagging // S3 object tagging
AmzObjectTagging = "X-Amz-Tagging" AmzObjectTagging = "X-Amz-Tagging"
AmzTagCount = "X-Amz-Tagging-Count" AmzTagCount = "x-amz-tagging-count"
AmzTagDirective = "X-Amz-Tagging-Directive" AmzTagDirective = "X-Amz-Tagging-Directive"
// S3 extensions // S3 extensions

@ -1072,7 +1072,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
// If x-amz-tagging-directive header is REPLACE, get passed tags. // If x-amz-tagging-directive header is REPLACE, get passed tags.
if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
objTags = r.Header.Get(xhttp.AmzObjectTagging) objTags = r.Header.Get(xhttp.AmzObjectTagging)
srcInfo.UserDefined[xhttp.AmzTagDirective] = replaceDirective
if _, err := tags.ParseObjectTags(objTags); err != nil { if _, err := tags.ParseObjectTags(objTags); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
@ -1096,7 +1095,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms) retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode.Valid() { if s3Err == ErrNone && retentionMode.Valid() {
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339) srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
} }
if s3Err == ErrNone && legalHold.Status.Valid() { if s3Err == ErrNone && legalHold.Status.Valid() {
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
@ -1429,7 +1428,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode.Valid() { if s3Err == ErrNone && retentionMode.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339) metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
} }
if s3Err == ErrNone && legalHold.Status.Valid() { if s3Err == ErrNone && legalHold.Status.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
@ -1605,7 +1604,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode.Valid() { if s3Err == ErrNone && retentionMode.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339) metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
} }
if s3Err == ErrNone && legalHold.Status.Valid() { if s3Err == ErrNone && legalHold.Status.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)

@ -33,14 +33,13 @@ import (
) )
const ( const (
expirationDateFormat = "2006-01-02T15:04:05.999Z" iso8601DateFormat = "20060102T150405Z"
iso8601DateFormat = "20060102T150405Z"
) )
func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte { func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte {
t := UTCNow() t := UTCNow()
// Add the expiration date. // Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat)) expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed. // Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed. // Add the key condition, only accept keys equal to the one passed.
@ -71,7 +70,7 @@ func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey stri
func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte { func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte {
t := UTCNow() t := UTCNow()
// Add the expiration date. // Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat)) expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed. // Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed. // Add the key condition, only accept keys equal to the one passed.
@ -98,7 +97,7 @@ func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration t
// newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches. // newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches.
func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte { func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte {
// Add the expiration date. // Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat)) expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
// Add the bucket condition, only accept buckets equal to the one passed. // Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed. // Add the key condition, only accept keys equal to the one passed.
@ -264,7 +263,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
expectedRespStatus: http.StatusNoContent, expectedRespStatus: http.StatusNoContent,
accessKey: credentials.AccessKey, accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey, secretKey: credentials.SecretKey,
dates: []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`, policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`,
}, },
// Corrupted Base 64 result // Corrupted Base 64 result
@ -274,7 +273,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
expectedRespStatus: http.StatusBadRequest, expectedRespStatus: http.StatusBadRequest,
accessKey: credentials.AccessKey, accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey, secretKey: credentials.SecretKey,
dates: []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`,
corruptedBase64: true, corruptedBase64: true,
}, },
@ -285,7 +284,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
expectedRespStatus: http.StatusBadRequest, expectedRespStatus: http.StatusBadRequest,
accessKey: credentials.AccessKey, accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey, secretKey: credentials.SecretKey,
dates: []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`,
corruptedMultipart: true, corruptedMultipart: true,
}, },
@ -307,7 +306,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
expectedRespStatus: http.StatusForbidden, expectedRespStatus: http.StatusForbidden,
accessKey: credentials.AccessKey, accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey, secretKey: credentials.SecretKey,
dates: []interface{}{curTime.Add(-1 * time.Minute * 5).Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, dates: []interface{}{curTime.Add(-1 * time.Minute * 5).Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`,
}, },
// Corrupted policy document // Corrupted policy document
@ -317,7 +316,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
expectedRespStatus: http.StatusForbidden, expectedRespStatus: http.StatusForbidden,
accessKey: credentials.AccessKey, accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey, secretKey: credentials.SecretKey,
dates: []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
policy: `{"3/aws4_request"]]}`, policy: `{"3/aws4_request"]]}`,
}, },
} }
@ -460,7 +459,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
dates := []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)} dates := []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}
policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}` policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`
// Generate the final policy document // Generate the final policy document

@ -1236,7 +1236,7 @@ func (s *TestSuiteCommon) TestListBuckets(c *check) {
c.Assert(createdBucket.Name != "", true) c.Assert(createdBucket.Name != "", true)
// Parse the bucket modtime // Parse the bucket modtime
creationTime, err := time.Parse(timeFormatAMZLong, createdBucket.CreationDate) creationTime, err := time.Parse(iso8601TimeFormat, createdBucket.CreationDate)
c.Assert(err, nil) c.Assert(err, nil)
// Check if bucket modtime is consistent (not less than current time and not late more than 5 minutes) // Check if bucket modtime is consistent (not less than current time and not late more than 5 minutes)

@ -409,8 +409,8 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
nextMarker := "" nextMarker := ""
// Fetch all the objects // Fetch all the objects
for { for {
result, err := core.ListObjects(args.BucketName, args.Prefix, nextMarker, SlashSeparator, // Let listObjects reply back the maximum from server implementation
maxObjectList) result, err := core.ListObjects(args.BucketName, args.Prefix, nextMarker, SlashSeparator, 0)
if err != nil { if err != nil {
return toJSONError(ctx, err, args.BucketName) return toJSONError(ctx, err, args.BucketName)
} }
@ -524,7 +524,8 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
nextMarker := "" nextMarker := ""
// Fetch all the objects // Fetch all the objects
for { for {
lo, err := listObjects(ctx, args.BucketName, args.Prefix, nextMarker, SlashSeparator, maxObjectList) // Limit browser to 1000 batches to be more responsive, scrolling friendly.
lo, err := listObjects(ctx, args.BucketName, args.Prefix, nextMarker, SlashSeparator, 1000)
if err != nil { if err != nil {
return &json2.Error{Message: err.Error()} return &json2.Error{Message: err.Error()}
} }
@ -761,7 +762,7 @@ next:
for { for {
var objects []string var objects []string
for obj := range objInfoCh { for obj := range objInfoCh {
if len(objects) == maxObjectList { if len(objects) == maxDeleteList {
// Reached maximum delete requests, attempt a delete for now. // Reached maximum delete requests, attempt a delete for now.
break break
} }
@ -1122,7 +1123,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode != "" { if s3Err == ErrNone && retentionMode != "" {
opts.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode) opts.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode)
opts.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(time.RFC3339) opts.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(iso8601TimeFormat)
} }
if s3Err == ErrNone && legalHold.Status != "" { if s3Err == ErrNone && legalHold.Status != "" {
opts.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status) opts.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status)

@ -1046,7 +1046,7 @@ func (z *xlZones) PutObjectPart(ctx context.Context, bucket, object, uploadID st
return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
} }
for _, zone := range z.zones { for _, zone := range z.zones {
result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxObjectList) result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
if err != nil { if err != nil {
return PartInfo{}, err return PartInfo{}, err
} }
@ -1078,7 +1078,7 @@ func (z *xlZones) ListObjectParts(ctx context.Context, bucket, object, uploadID
return z.zones[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts) return z.zones[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
} }
for _, zone := range z.zones { for _, zone := range z.zones {
result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxObjectList) result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
if err != nil { if err != nil {
return ListPartsInfo{}, err return ListPartsInfo{}, err
} }
@ -1110,7 +1110,7 @@ func (z *xlZones) AbortMultipartUpload(ctx context.Context, bucket, object, uplo
} }
for _, zone := range z.zones { for _, zone := range z.zones {
result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxObjectList) result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
if err != nil { if err != nil {
return err return err
} }
@ -1157,7 +1157,7 @@ func (z *xlZones) CompleteMultipartUpload(ctx context.Context, bucket, object, u
} }
for _, zone := range z.zones { for _, zone := range z.zones {
result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxObjectList) result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
if err != nil { if err != nil {
return objInfo, err return objInfo, err
} }

@ -33,7 +33,7 @@ const HTTP_NOCONTENT = "204";
const HTTP_BADREQUEST = "400"; const HTTP_BADREQUEST = "400";
const HTTP_NOTIMPLEMENTED = "501"; const HTTP_NOTIMPLEMENTED = "501";
const HTTP_INTERNAL_ERROR = "500"; const HTTP_INTERNAL_ERROR = "500";
const TEST_METADATA = ['Param_1' => 'val-1']; const TEST_METADATA = ['param_1' => 'val-1'];
/** /**
* ClientConfig abstracts configuration details to connect to a * ClientConfig abstracts configuration details to connect to a

@ -24,7 +24,7 @@ const (
AccessFormat = "access" AccessFormat = "access"
// AMZTimeFormat - event time format. // AMZTimeFormat - event time format.
AMZTimeFormat = "2006-01-02T15:04:05Z" AMZTimeFormat = "2006-01-02T15:04:05.000Z"
) )
// Identity represents access key who caused the event. // Identity represents access key who caused the event.

Loading…
Cancel
Save