From 01e9adc4b3b38571d54d2a61fd002d5d69a83835 Mon Sep 17 00:00:00 2001 From: Remco Verhoef Date: Thu, 4 May 2017 20:03:56 -0700 Subject: [PATCH] Implement anonymous uploads, fixes #4250 (#4259) --- cmd/gateway-azure-anonymous.go | 7 ++ cmd/gateway-handlers.go | 129 +++++++++++++++++++++++++++++++++ cmd/gateway-router.go | 3 + cmd/gateway-s3-anonymous.go | 31 ++++++++ 4 files changed, 170 insertions(+) diff --git a/cmd/gateway-azure-anonymous.go b/cmd/gateway-azure-anonymous.go index 820905157..69dcc1aa5 100644 --- a/cmd/gateway-azure-anonymous.go +++ b/cmd/gateway-azure-anonymous.go @@ -57,6 +57,13 @@ func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, e return bucketInfo, nil } +// AnonPutObject - SendPUT request without authentication. +// This is needed when clients send PUT requests on objects that can be uploaded without auth. +func (a AzureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) { + // azure doesn't support anonymous put + return ObjectInfo{}, traceError(NotImplemented{}) +} + // AnonGetObject - SendGET request without authentication. // This is needed when clients send GET requests on objects that can be downloaded without auth. func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) { diff --git a/cmd/gateway-handlers.go b/cmd/gateway-handlers.go index 28ed69e23..269a3e31c 100644 --- a/cmd/gateway-handlers.go +++ b/cmd/gateway-handlers.go @@ -20,7 +20,9 @@ import ( "io" "io/ioutil" "net/http" + "strconv" + "encoding/hex" "encoding/json" "encoding/xml" @@ -151,6 +153,133 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re } } +// PutObjectHandler - PUT Object +// ---------- +// This implementation of the PUT operation adds an object to a bucket. +func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, ErrServerNotInitialized, r.URL) + return + } + + // X-Amz-Copy-Source shouldn't be set for this call. + if _, ok := r.Header["X-Amz-Copy-Source"]; ok { + writeErrorResponse(w, ErrInvalidCopySource, r.URL) + return + } + + var object, bucket string + vars := router.Vars(r) + bucket = vars["bucket"] + object = vars["object"] + + // Get Content-Md5 sent by client and verify if valid + md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) + if err != nil { + errorIf(err, "Unable to validate content-md5 format.") + writeErrorResponse(w, ErrInvalidDigest, r.URL) + return + } + + /// if Content-Length is unknown/missing, deny the request + size := r.ContentLength + reqAuthType := getRequestAuthType(r) + if reqAuthType == authTypeStreamingSigned { + sizeStr := r.Header.Get("x-amz-decoded-content-length") + size, err = strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr) + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + } + if size == -1 { + writeErrorResponse(w, ErrMissingContentLength, r.URL) + return + } + + /// maximum Upload size for objects in a single operation + if isMaxObjectSize(size) { + writeErrorResponse(w, ErrEntityTooLarge, r.URL) + return + } + + // Extract metadata to be saved from incoming HTTP header. + metadata := extractMetadataFromHeader(r.Header) + if reqAuthType == authTypeStreamingSigned { + if contentEncoding, ok := metadata["content-encoding"]; ok { + contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) + if contentEncoding != "" { + // Make sure to trim and save the content-encoding + // parameter for a streaming signature which is set + // to a custom value for example: "aws-chunked,gzip". + metadata["content-encoding"] = contentEncoding + } else { + // Trimmed content encoding is empty when the header + // value is set to "aws-chunked" only. + + // Make sure to delete the content-encoding parameter + // for a streaming signature which is set to value + // for example: "aws-chunked" + delete(metadata, "content-encoding") + } + } + } + + // Make sure we hex encode md5sum here. + metadata["md5Sum"] = hex.EncodeToString(md5Bytes) + + sha256sum := "" + + // Lock the object. + objectLock := globalNSMutex.NewNSLock(bucket, object) + objectLock.Lock() + defer objectLock.Unlock() + + var objInfo ObjectInfo + switch reqAuthType { + default: + // For all unknown auth types return error. + writeErrorResponse(w, ErrAccessDenied, r.URL) + return + case authTypeAnonymous: + // Create anonymous object. + objInfo, err = objectAPI.AnonPutObject(bucket, object, size, r.Body, metadata, sha256sum) + case authTypeStreamingSigned: + // Initialize stream signature verifier. + reader, s3Error := newSignV4ChunkedReader(r) + if s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, s3Error, r.URL) + return + } + objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata, sha256sum) + case authTypeSignedV2, authTypePresignedV2: + s3Error := isReqAuthenticatedV2(r) + if s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, s3Error, r.URL) + return + } + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum) + case authTypePresigned, authTypeSigned: + if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, s3Error, r.URL) + return + } + if !skipContentSha256Cksum(r) { + sha256sum = r.Header.Get("X-Amz-Content-Sha256") + } + // Create object. + objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum) + } + + w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"") + writeSuccessResponseHeadersOnly(w) +} + // HeadObjectHandler - HEAD Object // ----------- // The HEAD operation retrieves metadata from an object without returning the object itself. diff --git a/cmd/gateway-router.go b/cmd/gateway-router.go index 1cd3589b1..b1d16f310 100644 --- a/cmd/gateway-router.go +++ b/cmd/gateway-router.go @@ -31,6 +31,9 @@ type GatewayLayer interface { AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) + + AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) + SetBucketPolicies(string, policy.BucketAccessPolicy) error GetBucketPolicies(string) (policy.BucketAccessPolicy, error) DeleteBucketPolicies(string) error diff --git a/cmd/gateway-s3-anonymous.go b/cmd/gateway-s3-anonymous.go index bcaf6036f..d4fc4e31a 100644 --- a/cmd/gateway-s3-anonymous.go +++ b/cmd/gateway-s3-anonymous.go @@ -17,11 +17,42 @@ package cmd import ( + "encoding/hex" "io" minio "github.com/minio/minio-go" ) +// AnonPutObject creates a new object anonymously with the incoming data, +func (l *s3Gateway) AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) { + var sha256sumBytes []byte + + var err error + if sha256sum != "" { + sha256sumBytes, err = hex.DecodeString(sha256sum) + if err != nil { + return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object) + } + } + + var md5sumBytes []byte + md5sum := metadata["md5Sum"] + if md5sum != "" { + md5sumBytes, err = hex.DecodeString(md5sum) + if err != nil { + return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object) + } + delete(metadata, "md5Sum") + } + + oi, err := l.anonClient.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata)) + if err != nil { + return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object) + } + + return fromMinioClientObjectInfo(bucket, oi), nil +} + // AnonGetObject - Get object anonymously func (l *s3Gateway) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error { r := minio.NewGetReqHeaders()