From 61c17c89335cccf1a687807068f090eaca817d9c Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Mon, 20 Jan 2020 22:15:59 +0530 Subject: [PATCH] Add ObjectTagging Support (#8754) This PR adds support for AWS S3 ObjectTagging API as explained here https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html --- cmd/api-errors.go | 13 ++ cmd/api-headers.go | 9 ++ cmd/api-router.go | 6 +- cmd/dummy-handlers.go | 58 +-------- cmd/fs-v1-metadata.go | 5 + cmd/fs-v1.go | 54 ++++++++ cmd/gateway-unsupported.go | 19 +++ cmd/generic-handlers.go | 5 +- cmd/handler-utils.go | 55 +++++--- cmd/http/headers.go | 5 + cmd/object-api-datatypes.go | 3 + cmd/object-api-interface.go | 6 + cmd/object-api-utils.go | 4 +- cmd/object-handlers.go | 159 ++++++++++++++++++++++-- cmd/xl-sets.go | 16 +++ cmd/xl-v1-metadata.go | 4 + cmd/xl-v1-object.go | 60 ++++++++- cmd/xl-zones.go | 61 +++++++++ docs/minio-limits.md | 1 - mint/run/core/aws-sdk-go/quick-tests.go | 138 ++++++++++++++++++++ pkg/iam/policy/action.go | 17 +++ pkg/policy/action.go | 14 +++ pkg/tagging/error.go | 44 +++++++ pkg/tagging/tag.go | 62 +++++++++ pkg/tagging/tagging.go | 113 +++++++++++++++++ pkg/tagging/tagset.go | 41 ++++++ 26 files changed, 886 insertions(+), 86 deletions(-) create mode 100644 pkg/tagging/error.go create mode 100644 pkg/tagging/tag.go create mode 100644 pkg/tagging/tagging.go create mode 100644 pkg/tagging/tagset.go diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 689475531..fc316642e 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -36,6 +36,7 @@ import ( "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/objectlock" "github.com/minio/minio/pkg/policy" + "github.com/minio/minio/pkg/tagging" ) // APIError structure @@ -150,6 +151,7 @@ const ( ErrPastObjectLockRetainDate ErrUnknownWORMModeDirective ErrObjectLockInvalidHeaders + ErrInvalidTagDirective // Add new error codes here. // SSE-S3 related API errors @@ -830,6 +832,11 @@ var errorCodes = errorCodeMap{ Description: "Your metadata headers exceed the maximum allowed metadata size.", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidTagDirective: { + Code: "InvalidArgument", + Description: "Unknown tag directive.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidEncryptionMethod: { Code: "InvalidRequest", Description: "The encryption method specified is not supported", @@ -1780,6 +1787,12 @@ func toAPIError(ctx context.Context, err error) APIError { // their internal error types. This code is only // useful with gateway implementations. switch e := err.(type) { + case tagging.Error: + apiErr = APIError{ + Code: "InvalidTag", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } case policy.Error: apiErr = APIError{ Code: "MalformedPolicy", diff --git a/cmd/api-headers.go b/cmd/api-headers.go index 5f47d562e..24c044150 100644 --- a/cmd/api-headers.go +++ b/cmd/api-headers.go @@ -22,6 +22,7 @@ import ( "encoding/xml" "fmt" "net/http" + "net/url" "strconv" "time" @@ -95,6 +96,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String()) w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String()) } + + // Set tag count if object has tags + tags, _ := url.ParseQuery(objInfo.UserTags) + tagCount := len(tags) + if tagCount != 0 { + w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) + } + // Set all other user defined metadata. for k, v := range objInfo.UserDefined { if HasPrefix(k, ReservedMetadataPrefix) { diff --git a/cmd/api-router.go b/cmd/api-router.go index da4170600..78d831b82 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -105,8 +105,12 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}") // GetObjectACL - this is a dummy call. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "") - // GetObjectTagging - this is a dummy call. + // GetObjectTagging bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "") + // PutObjectTagging + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler))).Queries("tagging", "") + // DeleteObjectTagging + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler))).Queries("tagging", "") // SelectObjectContent bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2") // GetObjectRetention diff --git a/cmd/dummy-handlers.go b/cmd/dummy-handlers.go index 383d8d41d..28ae3d4b9 100644 --- a/cmd/dummy-handlers.go +++ b/cmd/dummy-handlers.go @@ -22,26 +22,13 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio/pkg/policy" + "github.com/minio/minio/pkg/tagging" ) // Data types used for returning dummy tagging XML. // These variables shouldn't be used elsewhere. // They are only defined to be used in this file alone. -type tagging struct { - XMLName xml.Name `xml:"Tagging"` - TagSet tagSet `xml:"TagSet"` -} - -type tagSet struct { - Tag []tagElem `xml:"Tag"` -} - -type tagElem struct { - Key string `xml:"Key"` - Value string `xml:"Value"` -} - // GetBucketWebsite - GET bucket website, a dummy api func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { writeSuccessResponseHeadersOnly(w) @@ -171,47 +158,8 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h return } - tags := &tagging{} - tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{}) - - if err := xml.NewEncoder(w).Encode(tags); err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) - return - } - - w.(http.Flusher).Flush() -} - -// GetObjectTaggingHandler - GET object tagging, a dummy api -func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "GetObjectTagging") - - vars := mux.Vars(r) - bucket := vars["bucket"] - object := vars["object"] - - objAPI := api.ObjectAPI() - if objAPI == nil { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) - return - } - - // Allow getObjectTagging if policy action is set, since this is a dummy call - // we are simply re-purposing the bucketPolicyAction. - if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) - return - } - - // Validate if object exists, before proceeding further... - _, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) - return - } - - tags := &tagging{} - tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{}) + tags := &tagging.Tagging{} + tags.TagSet.Tags = append(tags.TagSet.Tags, tagging.Tag{}) if err := xml.NewEncoder(w).Encode(tags); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index 83edd1849..b1e108ae8 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -182,9 +182,14 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo objInfo.Expires = t.UTC() } } + + // Add user tags to the object info + objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging] + // etag/md5Sum has already been extracted. We need to // remove to avoid it from appearing as part of // response headers. e.g, X-Minio-* or X-Amz-*. + // Tags have also been extracted, we remove that as well. objInfo.UserDefined = cleanMetadata(m.Meta) // All the parts per object. diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 9c6fa013b..6428799d8 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -37,6 +37,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v6/pkg/s3utils" "github.com/minio/minio/cmd/config" + xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lifecycle" "github.com/minio/minio/pkg/lock" @@ -44,6 +45,7 @@ import ( "github.com/minio/minio/pkg/mimedb" "github.com/minio/minio/pkg/mountinfo" "github.com/minio/minio/pkg/policy" + "github.com/minio/minio/pkg/tagging" ) // Default etag is used for pre-existing objects. @@ -1235,6 +1237,58 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo) } +// GetObjectTag - get object tags from an existing object +func (fs *FSObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) { + oi, err := fs.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if err != nil { + return tagging.Tagging{}, err + } + + tags, err := tagging.FromString(oi.UserTags) + if err != nil { + return tagging.Tagging{}, err + } + + return tags, nil +} + +// PutObjectTag - replace or add tags to an existing object +func (fs *FSObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error { + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) + fsMeta := fsMetaV1{} + wlk, err := fs.rwPool.Write(fsMetaPath) + if err != nil { + logger.LogIf(ctx, err) + return toObjectErr(err, bucket, object) + } + // This close will allow for locks to be synchronized on `fs.json`. + defer wlk.Close() + + // Read objects' metadata in `fs.json`. + if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil { + // For any error to read fsMeta, set default ETag and proceed. + fsMeta = fs.defaultFsJSON(object) + } + + // clean fsMeta.Meta of tag key, before updating the new tags + delete(fsMeta.Meta, xhttp.AmzObjectTagging) + + // Do not update for empty tags + if tags != "" { + fsMeta.Meta[xhttp.AmzObjectTagging] = tags + } + + if _, err = fsMeta.WriteTo(wlk); err != nil { + return toObjectErr(err, bucket, object) + } + return nil +} + +// DeleteObjectTag - delete object tags from an existing object +func (fs *FSObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error { + return fs.PutObjectTag(ctx, bucket, object, "") +} + // ReloadFormat - no-op for fs, Valid only for XL. func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error { logger.LogIf(ctx, NotImplemented{}) diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index e722d7708..fc80784e4 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -24,6 +24,7 @@ import ( "github.com/minio/minio/pkg/lifecycle" "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/policy" + "github.com/minio/minio/pkg/tagging" ) // GatewayLocker implements custom NeNSLock implementation @@ -173,6 +174,24 @@ func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) { return &Metrics{}, NotImplemented{} } +// PutObjectTag - not implemented. +func (a GatewayUnsupported) PutObjectTag(ctx context.Context, bucket, object string, tags string) error { + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} +} + +// GetObjectTag - not implemented. +func (a GatewayUnsupported) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) { + logger.LogIf(ctx, NotImplemented{}) + return tagging.Tagging{}, NotImplemented{} +} + +// DeleteObjectTag - not implemented. +func (a GatewayUnsupported) DeleteObjectTag(ctx context.Context, bucket, object string) error { + logger.LogIf(ctx, NotImplemented{}) + return NotImplemented{} +} + // IsNotificationSupported returns whether bucket notification is applicable for this layer. func (a GatewayUnsupported) IsNotificationSupported() bool { return false diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 49a82ef06..66649fe46 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -467,8 +467,8 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool { // Checks requests for not implemented Object resources func ignoreNotImplementedObjectResources(req *http.Request) bool { for name := range req.URL.Query() { - // Enable GetObjectACL and GetObjectTagging dummy calls specifically. - if (name == "acl" || name == "tagging") && req.Method == http.MethodGet { + // Enable GetObjectACL dummy call specifically. + if name == "acl" && req.Method == http.MethodGet { return false } if notimplementedObjectResourceNames[name] { @@ -497,7 +497,6 @@ var notimplementedBucketResourceNames = map[string]bool{ var notimplementedObjectResourceNames = map[string]bool{ "acl": true, "restore": true, - "tagging": true, "torrent": true, } diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index 09ea0950b..4bcd7e0cf 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -34,6 +34,12 @@ import ( "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/madmin" + "github.com/minio/minio/pkg/tagging" +) + +const ( + copyDirective = "COPY" + replaceDirective = "REPLACE" ) // Parses location constraint from the incoming reader. @@ -69,30 +75,27 @@ var supportedHeaders = []string{ "content-encoding", "content-disposition", xhttp.AmzStorageClass, + xhttp.AmzObjectTagging, "expires", // Add more supported headers here. } -// isMetadataDirectiveValid - check if metadata-directive is valid. -func isMetadataDirectiveValid(h http.Header) bool { - _, ok := h[http.CanonicalHeaderKey(xhttp.AmzMetadataDirective)] - if ok { - // Check atleast set metadata-directive is valid. - return (isMetadataCopy(h) || isMetadataReplace(h)) - } - // By default if x-amz-metadata-directive is not we - // treat it as 'COPY' this function returns true. - return true +// isDirectiveValid - check if tagging-directive is valid. +func isDirectiveValid(v string) bool { + // Check if set metadata-directive is valid. + return isDirectiveCopy(v) || isDirectiveReplace(v) } -// Check if the metadata COPY is requested. -func isMetadataCopy(h http.Header) bool { - return h.Get(xhttp.AmzMetadataDirective) == "COPY" +// Check if the directive COPY is requested. +func isDirectiveCopy(value string) bool { + // By default if directive is not set we + // treat it as 'COPY' this function returns true. + return value == copyDirective || value == "" } -// Check if the metadata REPLACE is requested. -func isMetadataReplace(h http.Header) bool { - return h.Get(xhttp.AmzMetadataDirective) == "REPLACE" +// Check if the directive REPLACE is requested. +func isDirectiveReplace(value string) bool { + return value == replaceDirective } // Splits an incoming path into bucket and object components. @@ -174,6 +177,26 @@ func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[st return nil } +// extractTags extracts tag key and value from given http header. It then +// - Parses the input format X-Amz-Tagging:"Key1=Value1&Key2=Value2" into a map[string]string +// with entries in the format X-Amg-Tag-Key1:Value1, X-Amz-Tag-Key2:Value2 +// - Validates the tags +// - Returns the Tag in original string format "Key1=Value1&Key2=Value2" +func extractTags(ctx context.Context, tags string) (string, error) { + // Check if the metadata has tagging related header + if tags != "" { + tagging, err := tagging.FromString(tags) + if err != nil { + return "", err + } + if err := tagging.Validate(); err != nil { + return "", err + } + return tagging.String(), nil + } + return "", nil +} + // The Query string for the redirect URL the client is // redirected on successful upload. func getRedirectPostRawQuery(objInfo ObjectInfo) string { diff --git a/cmd/http/headers.go b/cmd/http/headers.go index 86326f8c8..704d153ad 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -56,6 +56,11 @@ const ( // S3 storage class AmzStorageClass = "x-amz-storage-class" + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "X-Amz-Tag-Count" + AmzTagDirective = "X-Amz-Tagging-Directive" + // S3 extensions AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index 0a40f7aef..a06f3e8ca 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -154,6 +154,9 @@ type ObjectInfo struct { // User-Defined metadata UserDefined map[string]string + // User-Defined object tags + UserTags string + // List of individual parts, maximum size of upto 10,000 Parts []ObjectPartInfo `json:"-"` diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 61b5e5c04..3ae2fbebc 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -25,6 +25,7 @@ import ( "github.com/minio/minio/pkg/lifecycle" "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/policy" + "github.com/minio/minio/pkg/tagging" ) // CheckCopyPreconditionFn returns true if copy precondition check failed. @@ -125,4 +126,9 @@ type ObjectLayer interface { // Check Readiness IsReady(ctx context.Context) bool + + // ObjectTagging operations + PutObjectTag(context.Context, string, string, string) error + GetObjectTag(context.Context, string, string) (tagging.Tagging, error) + DeleteObjectTag(context.Context, string, string) error } diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 048673ccc..9388eb698 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -239,8 +239,8 @@ func getCompleteMultipartMD5(parts []CompletePart) string { func cleanMetadata(metadata map[string]string) map[string]string { // Remove STANDARD StorageClass metadata = removeStandardStorageClass(metadata) - // Clean meta etag keys 'md5Sum', 'etag', "expires". - return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires") + // Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging". + return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging) } // Filter X-Amz-Storage-Class field only if it is set to STANDARD. diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index b356a6bda..d49c1e8c7 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -50,6 +50,7 @@ import ( "github.com/minio/minio/pkg/ioutil" "github.com/minio/minio/pkg/policy" "github.com/minio/minio/pkg/s3select" + "github.com/minio/minio/pkg/tagging" sha256 "github.com/minio/sha256-simd" "github.com/minio/sio" ) @@ -604,13 +605,13 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m // if x-amz-metadata-directive says REPLACE then // we extract metadata from the input headers. - if isMetadataReplace(r.Header) { + if isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) { return extractMetadata(ctx, r) } // if x-amz-metadata-directive says COPY then we // return the default metadata. - if isMetadataCopy(r.Header) { + if isDirectiveCopy(r.Header.Get(xhttp.AmzMetadataDirective)) { return defaultMeta, nil } @@ -618,6 +619,24 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m return defaultMeta, nil } +// Extract tags relevant for an CopyObject operation based on conditional +// header values specified in X-Amz-Tagging-Directive. +func getCpObjTagsFromHeader(ctx context.Context, r *http.Request, tags string) (string, error) { + // if x-amz-tagging-directive says REPLACE then + // we extract tags from the input headers. + if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { + if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" { + return extractTags(ctx, tags) + } + // Copy is default behavior if x-amz-tagging-directive is set, but x-amz-tagging is + // is not set + return tags, nil + } + + // Copy is default behavior if x-amz-tagging-directive is not set. + return tags, nil +} + // Returns a minio-go Client configured to access remote host described by destDNSRecord // Applicable only in a federated deployment var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { @@ -737,11 +756,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // Check if metadata directive is valid. - if !isMetadataDirectiveValid(r.Header) { + if !isDirectiveValid(r.Header.Get(xhttp.AmzMetadataDirective)) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMetadataDirective), r.URL, guessIsBrowserReq(r)) return } + // check if tag directive is valid + if !isDirectiveValid(r.Header.Get(xhttp.AmzTagDirective)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidTagDirective), r.URL, guessIsBrowserReq(r)) + return + } + // This request header needs to be set prior to setting ObjectOptions if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) @@ -795,7 +820,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re defer gr.Close() srcInfo := gr.ObjInfo - /// maximum Upload size for object in a single CopyObject operation. + // maximum Upload size for object in a single CopyObject operation. if isMaxObjectSize(srcInfo.Size) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r)) return @@ -968,6 +993,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } + + tags, err := getCpObjTagsFromHeader(ctx, r, srcInfo.UserTags) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + if tags != "" { + srcInfo.UserDefined[xhttp.AmzObjectTagging] = tags + } + getObjectInfo := objectAPI.GetObjectInfo if api.CacheAPI() != nil { getObjectInfo = api.CacheAPI().GetObjectInfo @@ -1002,12 +1038,13 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Ensure that metadata does not contain sensitive information crypto.RemoveSensitiveEntries(srcInfo.UserDefined) - // Check if x-amz-metadata-directive was not set to REPLACE and source, - // desination are same objects. Apply this restriction also when + // Check if x-amz-metadata-directive or x-amz-tagging-directive was not set to REPLACE and source, + // destination are same objects. Apply this restriction also when // metadataOnly is true indicating that we are not overwriting the object. // if encryption is enabled we do not need explicit "REPLACE" metadata to // be enabled as well - this is to allow for key-rotation. - if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) { + if !isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) && !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) && + srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) { // If x-amz-metadata-directive is not set to REPLACE then we need // to error out if source and destination are same. writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL, guessIsBrowserReq(r)) @@ -1155,6 +1192,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } + if tags := r.Header.Get(http.CanonicalHeaderKey(xhttp.AmzObjectTagging)); tags != "" { + metadata[xhttp.AmzObjectTagging], err = extractTags(ctx, tags) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + } + if rAuthType == authTypeStreamingSigned { if contentEncoding, ok := metadata["content-encoding"]; ok { contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) @@ -2774,3 +2819,103 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r Host: handlers.GetSourceIP(r), }) } + +// GetObjectTaggingHandler - GET object tagging +func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectTagging") + defer logger.AuditLog(w, r, "GetObjectTagging", mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) + return + } + + // Allow getObjectTagging if policy action is set. + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectTaggingAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return + } + + // Get object tags + tags, err := objAPI.GetObjectTag(ctx, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + writeSuccessResponseXML(w, encodeResponse(tags)) +} + +// PutObjectTaggingHandler - PUT object tagging +func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectTagging") + defer logger.AuditLog(w, r, "PutObjectTagging", mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) + return + } + + // Allow putObjectTagging if policy action is set + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectTaggingAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return + } + + tagging, err := tagging.ParseTagging(io.LimitReader(r.Body, r.ContentLength)) + + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + // Put object tags + err = objAPI.PutObjectTag(ctx, bucket, object, tagging.String()) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + writeSuccessResponseHeadersOnly(w) +} + +// DeleteObjectTaggingHandler - DELETE object tagging +func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteObjectTagging") + defer logger.AuditLog(w, r, "DeleteObjectTagging", mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) + return + } + + // Allow deleteObjectTagging if policy action is set + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectTaggingAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return + } + + // Delete object tags + err := objAPI.DeleteObjectTag(ctx, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + writeSuccessResponseHeadersOnly(w) +} diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index 7d70a43ea..90c0ded45 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -35,6 +35,7 @@ import ( "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/policy" "github.com/minio/minio/pkg/sync/errgroup" + "github.com/minio/minio/pkg/tagging" ) // setsStorageAPI is encapsulated type for Close() @@ -1661,6 +1662,21 @@ func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, de return s.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys, true) } +// PutObjectTag - replace or add tags to an existing object +func (s *xlSets) PutObjectTag(ctx context.Context, bucket, object string, tags string) error { + return s.getHashedSet(object).PutObjectTag(ctx, bucket, object, tags) +} + +// DeleteObjectTag - delete object tags from an existing object +func (s *xlSets) DeleteObjectTag(ctx context.Context, bucket, object string) error { + return s.getHashedSet(object).DeleteObjectTag(ctx, bucket, object) +} + +// GetObjectTag - get object tags from an existing object +func (s *xlSets) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) { + return s.getHashedSet(object).GetObjectTag(ctx, bucket, object) +} + // GetMetrics - no op func (s *xlSets) GetMetrics(ctx context.Context) (*Metrics, error) { logger.LogIf(ctx, NotImplemented{}) diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go index 5d4651092..2cefe5cd0 100644 --- a/cmd/xl-v1-metadata.go +++ b/cmd/xl-v1-metadata.go @@ -245,9 +245,13 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo { // Extract etag from metadata. objInfo.ETag = extractETag(m.Meta) + // Add user tags to the object info + objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging] + // etag/md5Sum has already been extracted. We need to // remove to avoid it from appearing as part of // response headers. e.g, X-Minio-* or X-Amz-*. + // Tags have also been extracted, we remove that as well. objInfo.UserDefined = cleanMetadata(m.Meta) // All the parts per object. diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index 6daa6923a..ed0ae577c 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -27,6 +27,7 @@ import ( "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/mimedb" "github.com/minio/minio/pkg/sync/errgroup" + "github.com/minio/minio/pkg/tagging" ) // list all errors which can be ignored in object operations. @@ -969,7 +970,7 @@ func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat return listObjectsV2Info, err } -// Send the successul but partial upload, however ignore +// Send the successful but partial upload, however ignore // if the channel is blocked by other items. func (xl xlObjects) addPartialUpload(bucket, key string) { select { @@ -977,3 +978,60 @@ func (xl xlObjects) addPartialUpload(bucket, key string) { default: } } + +// PutObjectTag - replace or add tags to an existing object +func (xl xlObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error { + disks := xl.getDisks() + + // Read metadata associated with the object from all disks. + metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object) + + _, writeQuorum, err := objectQuorumFromMeta(ctx, xl, metaArr, errs) + if err != nil { + return err + } + + for i, xlMeta := range metaArr { + // clean xlMeta.Meta of tag key, before updating the new tags + delete(xlMeta.Meta, xhttp.AmzObjectTagging) + // Don't update for empty tags + if tags != "" { + xlMeta.Meta[xhttp.AmzObjectTagging] = tags + } + metaArr[i].Meta = xlMeta.Meta + } + + tempObj := mustGetUUID() + + // Write unique `xl.json` for each disk. + if disks, err = writeUniqueXLMetadata(ctx, disks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil { + return toObjectErr(err, bucket, object) + } + + // Atomically rename `xl.json` from tmp location to destination for each disk. + if _, err = renameXLMetadata(ctx, disks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil { + return toObjectErr(err, bucket, object) + } + + return nil +} + +// DeleteObjectTag - delete object tags from an existing object +func (xl xlObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error { + return xl.PutObjectTag(ctx, bucket, object, "") +} + +// GetObjectTag - get object tags from an existing object +func (xl xlObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) { + // GetObjectInfo will return tag value as well + oi, err := xl.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if err != nil { + return tagging.Tagging{}, err + } + + tags, err := tagging.FromString(oi.UserTags) + if err != nil { + return tagging.Tagging{}, err + } + return tags, nil +} diff --git a/cmd/xl-zones.go b/cmd/xl-zones.go index ac8259867..db913b936 100644 --- a/cmd/xl-zones.go +++ b/cmd/xl-zones.go @@ -31,6 +31,7 @@ import ( "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/policy" "github.com/minio/minio/pkg/sync/errgroup" + "github.com/minio/minio/pkg/tagging" ) type xlZones struct { @@ -1374,3 +1375,63 @@ func (z *xlZones) GetMetrics(ctx context.Context) (*Metrics, error) { func (z *xlZones) IsReady(ctx context.Context) bool { return z.zones[0].IsReady(ctx) } + +// PutObjectTag - replace or add tags to an existing object +func (z *xlZones) PutObjectTag(ctx context.Context, bucket, object string, tags string) error { + if z.SingleZone() { + return z.zones[0].PutObjectTag(ctx, bucket, object, tags) + } + for _, zone := range z.zones { + err := zone.PutObjectTag(ctx, bucket, object, tags) + if err != nil { + if isErrBucketNotFound(err) { + continue + } + return err + } + return nil + } + return BucketNotFound{ + Bucket: bucket, + } +} + +// DeleteObjectTag - delete object tags from an existing object +func (z *xlZones) DeleteObjectTag(ctx context.Context, bucket, object string) error { + if z.SingleZone() { + return z.zones[0].DeleteObjectTag(ctx, bucket, object) + } + for _, zone := range z.zones { + err := zone.DeleteObjectTag(ctx, bucket, object) + if err != nil { + if isErrBucketNotFound(err) { + continue + } + return err + } + return nil + } + return BucketNotFound{ + Bucket: bucket, + } +} + +// GetObjectTag - get object tags from an existing object +func (z *xlZones) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) { + if z.SingleZone() { + return z.zones[0].GetObjectTag(ctx, bucket, object) + } + for _, zone := range z.zones { + tags, err := zone.GetObjectTag(ctx, bucket, object) + if err != nil { + if isErrBucketNotFound(err) { + continue + } + return tags, err + } + return tags, nil + } + return tagging.Tagging{}, BucketNotFound{ + Bucket: bucket, + } +} diff --git a/docs/minio-limits.md b/docs/minio-limits.md index bec203963..677d133cc 100644 --- a/docs/minio-limits.md +++ b/docs/minio-limits.md @@ -51,7 +51,6 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If - ObjectACL (Use [bucket policies](https://docs.min.io/docs/minio-client-complete-guide#policy) instead) - ObjectTorrent - ObjectVersions -- ObjectTagging ### Object name restrictions on MinIO Object names that contain characters `^*|\/&";` are unsupported on Windows and other file systems which do not support filenames with these characters. Note that this list is not exhaustive, and depends on the maintainers of the filesystem itself. diff --git a/mint/run/core/aws-sdk-go/quick-tests.go b/mint/run/core/aws-sdk-go/quick-tests.go index c2147d51c..d79b741cc 100644 --- a/mint/run/core/aws-sdk-go/quick-tests.go +++ b/mint/run/core/aws-sdk-go/quick-tests.go @@ -28,10 +28,12 @@ import ( "math/rand" "net/http" "os" + "reflect" "strings" "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -132,6 +134,50 @@ func randString(n int, src rand.Source, prefix string) string { return prefix + string(b[0:30-len(prefix)]) } +func isObjectTaggingImplemented(s3Client *s3.S3) bool { + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") + object := randString(60, rand.NewSource(time.Now().UnixNano()), "") + startTime := time.Now() + function := "isObjectTaggingImplemented" + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + } + defer cleanup(s3Client, bucket, object, function, args, startTime, true) + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() + return false + } + + _, err = s3Client.PutObject(&s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("testfile")), + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() + return false + } + + _, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NotImplemented" { + return false + } + } + } + return true +} + func cleanup(s3Client *s3.S3, bucket string, object string, function string, args map[string]interface{}, startTime time.Time, deleteBucket bool) { @@ -474,6 +520,95 @@ func testSelectObject(s3Client *s3.S3) { successLogger(function, args, startTime).Info() } +func testObjectTagging(s3Client *s3.S3) { + startTime := time.Now() + function := "testObjectTagging" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") + object := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() + return + } + defer cleanup(s3Client, bucket, object, function, args, startTime, true) + + taginput := "Tag1=Value1" + tagInputSet := []*s3.Tag{ + { + Key: aws.String("Tag1"), + Value: aws.String("Value1"), + }, + } + _, err = s3Client.PutObject(&s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("testfile")), + Bucket: aws.String(bucket), + Key: aws.String(object), + Tagging: &taginput, + }) + + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() + return + } + + tagop, err := s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() + return + } + } + if !reflect.DeepEqual(tagop.TagSet, tagInputSet) { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObject Tag input did not match with GetObjectTagging output %v", nil), nil).Fatal() + return + } + + taginputSet1 := []*s3.Tag{ + { + Key: aws.String("Key4"), + Value: aws.String("Value4"), + }, + } + _, err = s3Client.PutObjectTagging(&s3.PutObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + Tagging: &s3.Tagging{ + TagSet: taginputSet1, + }, + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() + return + } + } + + tagop, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() + return + } + } + if !reflect.DeepEqual(tagop.TagSet, taginputSet1) { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging input did not match with GetObjectTagging output %v", nil), nil).Fatal() + return + } +} + func main() { endpoint := os.Getenv("SERVER_ENDPOINT") accessKey := os.Getenv("ACCESS_KEY") @@ -508,4 +643,7 @@ func main() { testPresignedPutInvalidHash(s3Client) testListObjects(s3Client) testSelectObject(s3Client) + if isObjectTaggingImplemented(s3Client) { + testObjectTagging(s3Client) + } } diff --git a/pkg/iam/policy/action.go b/pkg/iam/policy/action.go index 31ff755fe..4cbeb4ed1 100644 --- a/pkg/iam/policy/action.go +++ b/pkg/iam/policy/action.go @@ -114,6 +114,15 @@ const ( // PutBucketObjectLockConfigurationAction - PutBucketObjectLockConfiguration Rest API action PutBucketObjectLockConfigurationAction = "s3:PutBucketObjectLockConfiguration" + // GetObjectTaggingAction - Get Object Tags API action + GetObjectTaggingAction = "s3:GetObjectTagging" + + // PutObjectTaggingAction - Put Object Tags API action + PutObjectTaggingAction = "s3:PutObjectTagging" + + // DeleteObjectTaggingAction - Delete Object Tags API action + DeleteObjectTaggingAction = "s3:DeleteObjectTagging" + // AllActions - all API actions AllActions = "s3:*" ) @@ -149,6 +158,9 @@ var supportedActions = map[Action]struct{}{ GetBucketObjectLockConfigurationAction: {}, BypassGovernanceModeAction: {}, BypassGovernanceRetentionAction: {}, + GetObjectTaggingAction: {}, + PutObjectTaggingAction: {}, + DeleteObjectTaggingAction: {}, } // isObjectAction - returns whether action is object type or not. @@ -164,6 +176,8 @@ func (action Action) isObjectAction() bool { return true case PutObjectLegalHoldAction, GetObjectLegalHoldAction: return true + case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction: + return true } return false @@ -283,4 +297,7 @@ var actionConditionKeyMap = map[Action]condition.KeySet{ BypassGovernanceRetentionAction: condition.NewKeySet(condition.CommonKeys...), GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...), PutBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...), + PutObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), + GetObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), + DeleteObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), } diff --git a/pkg/policy/action.go b/pkg/policy/action.go index c57b05b97..bc4983481 100644 --- a/pkg/policy/action.go +++ b/pkg/policy/action.go @@ -106,6 +106,13 @@ const ( GetBucketObjectLockConfigurationAction = "s3:GetBucketObjectLockConfiguration" // PutBucketObjectLockConfigurationAction - PutObjectLockConfiguration Rest API action PutBucketObjectLockConfigurationAction = "s3:PutBucketObjectLockConfiguration" + + // GetObjectTaggingAction - Get Object Tags API action + GetObjectTaggingAction = "s3:GetObjectTagging" + // PutObjectTaggingAction - Put Object Tags API action + PutObjectTaggingAction = "s3:PutObjectTagging" + // DeleteObjectTaggingAction - Delete Object Tags API action + DeleteObjectTaggingAction = "s3:DeleteObjectTagging" ) // isObjectAction - returns whether action is object type or not. @@ -121,6 +128,8 @@ func (action Action) isObjectAction() bool { return true case BypassGovernanceModeAction, BypassGovernanceRetentionAction: return true + case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction: + return true } return false @@ -153,6 +162,8 @@ func (action Action) IsValid() bool { return true case PutBucketObjectLockConfigurationAction, GetBucketObjectLockConfigurationAction: return true + case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction: + return true } return false @@ -243,4 +254,7 @@ var actionConditionKeyMap = map[Action]condition.KeySet{ GetObjectLegalHoldAction: condition.NewKeySet(condition.CommonKeys...), GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...), PutBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...), + PutObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), + GetObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), + DeleteObjectTaggingAction: condition.NewKeySet(condition.CommonKeys...), } diff --git a/pkg/tagging/error.go b/pkg/tagging/error.go new file mode 100644 index 000000000..42d0b75e0 --- /dev/null +++ b/pkg/tagging/error.go @@ -0,0 +1,44 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tagging + +import ( + "fmt" +) + +// Error is the generic type for any error happening during tag +// parsing. +type Error struct { + err error +} + +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type tagging.Error +func Errorf(format string, a ...interface{}) error { + return Error{err: fmt.Errorf(format, a...)} +} + +// Unwrap the internal error. +func (e Error) Unwrap() error { return e.err } + +// Error 'error' compatible method. +func (e Error) Error() string { + if e.err == nil { + return "tagging: cause " + } + return e.err.Error() +} diff --git a/pkg/tagging/tag.go b/pkg/tagging/tag.go new file mode 100644 index 000000000..5c59851de --- /dev/null +++ b/pkg/tagging/tag.go @@ -0,0 +1,62 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tagging + +import ( + "encoding/xml" + "unicode/utf8" +) + +// Tag - single tag +type Tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +// Validate - validates the tag element +func (t Tag) Validate() error { + if err := t.validateKey(); err != nil { + return err + } + if err := t.validateValue(); err != nil { + return err + } + return nil +} + +// validateKey - checks if key is valid or not. +func (t Tag) validateKey() error { + // cannot be longer than maxTagKeyLength characters + if utf8.RuneCountInString(t.Key) > maxTagKeyLength { + return ErrInvalidTagKey + } + // cannot be empty + if len(t.Key) == 0 { + return ErrInvalidTagKey + } + return nil +} + +// validateValue - checks if value is valid or not. +func (t Tag) validateValue() error { + // cannot be longer than maxTagValueLength characters + if utf8.RuneCountInString(t.Value) > maxTagValueLength { + return ErrInvalidTagValue + } + return nil +} diff --git a/pkg/tagging/tagging.go b/pkg/tagging/tagging.go new file mode 100644 index 000000000..9fc683ecd --- /dev/null +++ b/pkg/tagging/tagging.go @@ -0,0 +1,113 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tagging + +import ( + "bytes" + "encoding/xml" + "io" + "net/url" +) + +// S3 API limits for tags +// Ref: https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html +const ( + maxTags = 10 + maxTagKeyLength = 128 + maxTagValueLength = 256 +) + +// errors returned by tagging package +var ( + ErrTooManyTags = Errorf("Cannot have more than 10 object tags") + ErrInvalidTagKey = Errorf("The TagKey you have provided is invalid") + ErrInvalidTagValue = Errorf("The TagValue you have provided is invalid") + ErrInvalidTag = Errorf("Cannot provide multiple Tags with the same key") +) + +// Tagging - object tagging interface +type Tagging struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + TagSet TagSet `xml:"TagSet"` +} + +// Validate - validates the tagging configuration +func (t Tagging) Validate() error { + // Tagging can't have more than 10 tags + if len(t.TagSet.Tags) > maxTags { + return ErrTooManyTags + } + // Validate all the rules in the tagging config + for _, ts := range t.TagSet.Tags { + if t.TagSet.ContainsDuplicate(ts.Key) { + return ErrInvalidTag + } + if err := ts.Validate(); err != nil { + return err + } + } + return nil +} + +// String - returns a string in format "tag1=value1&tag2=value2" with all the +// tags in this Tagging Struct +func (t Tagging) String() string { + var buf bytes.Buffer + for _, tag := range t.TagSet.Tags { + if buf.Len() > 0 { + buf.WriteString("&") + } + buf.WriteString(tag.Key + "=") + buf.WriteString(tag.Value) + } + return buf.String() +} + +// FromString - returns a Tagging struct when given a string in format +// "tag1=value1&tag2=value2" +func FromString(tagStr string) (Tagging, error) { + tags, err := url.ParseQuery(tagStr) + if err != nil { + return Tagging{}, err + } + var idx = 0 + parsedTags := make([]Tag, len(tags)) + for k := range tags { + parsedTags[idx].Key = k + parsedTags[idx].Value = tags.Get(k) + idx++ + } + return Tagging{ + TagSet: TagSet{ + Tags: parsedTags, + }, + }, nil +} + +// ParseTagging - parses incoming xml data in given reader +// into Tagging interface. After parsing, also validates the +// parsed fields based on S3 API constraints. +func ParseTagging(reader io.Reader) (*Tagging, error) { + var t Tagging + if err := xml.NewDecoder(reader).Decode(&t); err != nil { + return nil, err + } + if err := t.Validate(); err != nil { + return nil, err + } + return &t, nil +} diff --git a/pkg/tagging/tagset.go b/pkg/tagging/tagset.go new file mode 100644 index 000000000..135890a30 --- /dev/null +++ b/pkg/tagging/tagset.go @@ -0,0 +1,41 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tagging + +import ( + "encoding/xml" +) + +// TagSet - Set of tags under Tagging +type TagSet struct { + XMLName xml.Name `xml:"TagSet"` + Tags []Tag `xml:"Tag"` +} + +// ContainsDuplicate - returns true if duplicate keys are present in TagSet +func (t TagSet) ContainsDuplicate(key string) bool { + var found bool + for _, tag := range t.Tags { + if tag.Key == key { + if found { + return true + } + found = true + } + } + return false +}