From 9dca46e1564d62aba62303a7d125e55018c5f3c9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 12 Mar 2016 16:08:15 -0800 Subject: [PATCH] signature: Use a layered approach for signature verification. Signature calculation has now moved out from being a package to top-level as a layered mechanism. In case of payload calculation with body, go-routines are initiated to simultaneously write and calculate shasum. Errors are sent over the writer so that the lower layer removes the temporary files properly. --- auth-handler.go | 50 +++- bucket-handlers.go | 130 +++------ bucket-policy-handlers.go | 66 +++-- generic-handlers.go | 10 +- object-handlers.go | 213 ++++++++++----- pkg/fs/api_suite_test.go | 94 +++---- pkg/fs/fs-bucket-listobjects_test.go | 15 +- pkg/fs/fs-errors.go | 31 +-- pkg/fs/fs-multipart.go | 96 +++---- pkg/fs/fs-object.go | 66 ++--- pkg/fs/fs-object_test.go | 9 +- pkg/xl/acl.go | 2 + pkg/xl/bucket.go | 2 + pkg/xl/common.go | 2 + pkg/xl/config.go | 2 + pkg/xl/definitions.go | 2 + pkg/xl/doc.go | 3 + pkg/xl/encoder.go | 2 + pkg/xl/errors.go | 2 + pkg/xl/heal.go | 2 + pkg/xl/interfaces.go | 2 + pkg/xl/management.go | 2 + pkg/xl/multipart.go | 2 + pkg/xl/node.go | 2 + pkg/xl/xl-v1.go | 2 + pkg/xl/xl-v1_test.go | 2 + pkg/xl/xl-v2.go | 2 + pkg/xl/xl-v2_test.go | 2 + routers.go | 10 - server_fs_test.go | 36 ++- jwt.go => signature-jwt.go | 2 + .../v4-errors.go => signature-v4-errors.go | 7 +- .../v4-parser.go => signature-v4-parser.go | 47 ++-- ...yform.go => signature-v4-postpolicyform.go | 6 +- .../v4-utils.go => signature-v4-utils.go | 10 +- .../v4-signature.go => signature-v4.go | 250 +++++++++--------- signature_utils_test.go | 104 -------- typed-errors.go | 3 + utils.go | 17 +- 39 files changed, 570 insertions(+), 737 deletions(-) create mode 100644 pkg/xl/doc.go rename jwt.go => signature-jwt.go (98%) rename pkg/s3/signature4/v4-errors.go => signature-v4-errors.go (90%) rename pkg/s3/signature4/v4-parser.go => signature-v4-parser.go (78%) rename pkg/s3/signature4/v4-postpolicyform.go => signature-v4-postpolicyform.go (97%) rename pkg/s3/signature4/v4-utils.go => signature-v4-utils.go (94%) rename pkg/s3/signature4/v4-signature.go => signature-v4.go (53%) delete mode 100644 signature_utils_test.go diff --git a/auth-handler.go b/auth-handler.go index 81e04758d..5d6eb5974 100644 --- a/auth-handler.go +++ b/auth-handler.go @@ -17,17 +17,16 @@ package main import ( + "bytes" + "crypto/md5" "crypto/sha256" + "encoding/base64" "encoding/hex" + "io/ioutil" "net/http" "strings" - "github.com/minio/minio/pkg/s3/signature4" -) - -const ( - signV4Algorithm = "AWS4-HMAC-SHA256" - jwtAlgorithm = "Bearer" + "github.com/minio/minio/pkg/probe" ) // Verify if request has JWT. @@ -97,12 +96,41 @@ func getRequestAuthType(r *http.Request) authType { return authTypeUnknown } +// sum256 calculate sha256 sum for an input byte array +func sum256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// sumMD5 calculate md5 sum for an input byte array +func sumMD5(data []byte) []byte { + hash := md5.New() + hash.Write(data) + return hash.Sum(nil) +} + // Verify if request has valid AWS Signature Version '4'. -func isReqAuthenticated(sign *signature4.Sign, r *http.Request) (s3Error APIErrorCode) { - auth := sign.SetHTTPRequestToVerify(r) +func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) { + if r == nil { + errorIf(probe.NewError(errInvalidArgument), "HTTP request cannot be empty.", nil) + return ErrInternalError + } + payload, e := ioutil.ReadAll(r.Body) + if e != nil { + errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) + return ErrInternalError + } + // Verify Content-Md5, if payload is set. + if r.Header.Get("Content-Md5") != "" { + if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) { + return ErrBadDigest + } + } + // Populate back the payload. + r.Body = ioutil.NopCloser(bytes.NewReader(payload)) if isRequestSignatureV4(r) { - dummyPayload := sha256.Sum256([]byte("")) - ok, err := auth.DoesSignatureMatch(hex.EncodeToString(dummyPayload[:])) + ok, err := doesSignatureMatch(hex.EncodeToString(sum256(payload)), r) if err != nil { errorIf(err.Trace(), "Signature verification failed.", nil) return ErrInternalError @@ -112,7 +140,7 @@ func isReqAuthenticated(sign *signature4.Sign, r *http.Request) (s3Error APIErro } return ErrNone } else if isRequestPresignedSignatureV4(r) { - ok, err := auth.DoesPresignedSignatureMatch() + ok, err := doesPresignedSignatureMatch(r) if err != nil { errorIf(err.Trace(), "Presigned signature verification failed.", nil) return ErrInternalError diff --git a/bucket-handlers.go b/bucket-handlers.go index 708a3ce15..16494417f 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -18,9 +18,6 @@ package main import ( "bytes" - "crypto/md5" - "encoding/base64" - "encoding/hex" "encoding/xml" "io" "io/ioutil" @@ -30,10 +27,8 @@ import ( "strings" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/crypto/sha256" "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/s3/signature4" ) // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html @@ -94,7 +89,7 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re return } case authTypeSigned, authTypePresigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -151,7 +146,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http return } case authTypePresigned, authTypeSigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -208,7 +203,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) return } case authTypeSigned, authTypePresigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -262,7 +257,7 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return case authTypeSigned, authTypePresigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -288,6 +283,24 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt vars := mux.Vars(r) bucket := vars["bucket"] + switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return + case authTypeAnonymous: + // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html + if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } + } + // Content-Length is required and should be non-zero // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html if r.ContentLength <= 0 { @@ -302,9 +315,6 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt return } - // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) - // Allocate incoming content length bytes. deleteXMLBytes := make([]byte, r.ContentLength) @@ -316,52 +326,6 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt return } - switch getRequestAuthType(r) { - default: - // For all unknown auth types return error. - writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) - return - case authTypeAnonymous: - // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html - if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone { - writeErrorResponse(w, r, s3Error, r.URL.Path) - return - } - case authTypePresigned: - // Check if request is presigned. - ok, err := auth.DoesPresignedSignatureMatch() - if err != nil { - errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - case authTypeSigned: - // Check if request is signed. - sha := sha256.New() - mdSh := md5.New() - sha.Write(deleteXMLBytes) - mdSh.Write(deleteXMLBytes) - ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil))) - if err != nil { - errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - // Verify content md5. - if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) { - writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - return - } - } - // Unmarshal list of keys to be deleted. deleteObjects := &DeleteObjectsRequest{} if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil { @@ -431,41 +395,14 @@ func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { bucket := vars["bucket"] // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) switch getRequestAuthType(r) { default: // For all unknown auth types return error. writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return - case authTypePresigned: - ok, err := auth.DoesPresignedSignatureMatch() - if err != nil { - errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - case authTypeSigned: - // Verify signature for the incoming body if any. - locationBytes, e := ioutil.ReadAll(r.Body) - if e != nil { - errorIf(probe.NewError(e), "MakeBucket failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) - return - } - sh := sha256.New() - sh.Write(locationBytes) - ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil))) - if err != nil { - errorIf(err.Trace(), "MakeBucket failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) return } } @@ -539,11 +476,8 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req object := formValues["Key"] var ok bool - // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) - // Verify policy signature. - ok, err = auth.DoesPolicySignatureMatch(formValues) + ok, err = doesPolicySignatureMatch(formValues) if err != nil { errorIf(err.Trace(), "Unable to verify signature.", nil) writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) @@ -553,12 +487,12 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) return } - if err = signature4.ApplyPolicyCond(formValues); err != nil { - errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil) + if err = checkPostPolicy(formValues); err != nil { + errorIf(err.Trace(), "Invalid request, policy doesn't match.", nil) writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path) return } - objectInfo, err := api.Filesystem.CreateObject(bucket, object, "", -1, fileBody, nil) + objectInfo, err := api.Filesystem.CreateObject(bucket, object, -1, fileBody, nil) if err != nil { errorIf(err.Trace(), "CreateObject failed.", nil) switch err.ToGoError().(type) { @@ -572,8 +506,6 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) case fs.IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.InvalidDigest: - writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } @@ -601,7 +533,7 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return case authTypePresigned, authTypeSigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -634,7 +566,7 @@ func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return case authTypePresigned, authTypeSigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } diff --git a/bucket-policy-handlers.go b/bucket-policy-handlers.go index f59244d20..4ea72dabe 100644 --- a/bucket-policy-handlers.go +++ b/bucket-policy-handlers.go @@ -18,8 +18,6 @@ package main import ( "bytes" - "crypto/sha256" - "encoding/hex" "io" "io/ioutil" "net/http" @@ -134,6 +132,18 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ vars := mux.Vars(r) bucket := vars["bucket"] + switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } + } + // If Content-Length is unknown or zero, deny the // request. PutBucketPolicy always needs a Content-Length if // incoming request is not chunked. @@ -173,34 +183,6 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ return } - // Set http request for signature verification. - auth := api.Signature.SetHTTPRequestToVerify(r) - if isRequestPresignedSignatureV4(r) { - ok, err := auth.DoesPresignedSignatureMatch() - if err != nil { - errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - } else if isRequestSignatureV4(r) { - sh := sha256.New() - sh.Write(bucketPolicyBuf) - ok, err := api.Signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil))) - if err != nil { - errorIf(err.Trace(string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil) - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) - return - } - } - // Save bucket policy. err := writeBucketPolicy(bucket, bucketPolicyBuf) if err != nil { @@ -224,10 +206,16 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R vars := mux.Vars(r) bucket := vars["bucket"] - // Validate incoming signature. - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { - writeErrorResponse(w, r, s3Error, r.URL.Path) + switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } } // Delete bucket access policy. @@ -255,10 +243,16 @@ func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Requ vars := mux.Vars(r) bucket := vars["bucket"] - // Validate incoming signature. - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { - writeErrorResponse(w, r, s3Error, r.URL.Path) + switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } } // Read bucket access policy. diff --git a/generic-handlers.go b/generic-handlers.go index 9f8f0ece9..a00d9f9ba 100644 --- a/generic-handlers.go +++ b/generic-handlers.go @@ -28,11 +28,6 @@ import ( "github.com/rs/cors" ) -const ( - iso8601Format = "20060102T150405Z" - privateBucket = "/minio" -) - // HandlerFunc - useful to chain different middleware http.Handler type HandlerFunc func(http.Handler) http.Handler @@ -51,6 +46,11 @@ type redirectHandler struct { locationPrefix string } +// Private bucket. +const ( + privateBucket = "/minio" +) + func setBrowserRedirectHandler(h http.Handler) http.Handler { return redirectHandler{handler: h, locationPrefix: privateBucket} } diff --git a/object-handlers.go b/object-handlers.go index 97173ecb2..ec5542e4f 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -17,7 +17,10 @@ package main import ( + "crypto/sha256" + "encoding/hex" "io" + "io/ioutil" "net/http" "net/url" "strconv" @@ -73,7 +76,7 @@ func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { return } case authTypePresigned, authTypeSigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -230,9 +233,16 @@ func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) bucket = vars["bucket"] object = vars["object"] - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { - writeErrorResponse(w, r, s3Error, r.URL.Path) + switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) return + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) + return + } } objectInfo, err := api.Filesystem.GetObjectInfo(bucket, object) @@ -292,7 +302,7 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) return } case authTypePresigned, authTypeSigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -366,6 +376,17 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) return } + var md5Bytes []byte + if objectInfo.MD5Sum != "" { + var e error + md5Bytes, e = hex.DecodeString(objectInfo.MD5Sum) + if e != nil { + errorIf(probe.NewError(e), "Decoding md5 failed.", nil) + writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) + return + } + } + // Initialize a pipe for data pipe line. reader, writer := io.Pipe() @@ -378,13 +399,11 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) } }() - // Verify md5sum. - expectedMD5Sum := objectInfo.MD5Sum // Size of object. size := objectInfo.Size // Create the object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, expectedMD5Sum, size, reader, nil) + objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) if err != nil { errorIf(err.Trace(), "CreateObject failed.", nil) switch err.ToGoError().(type) { @@ -398,8 +417,6 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) case fs.IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.InvalidDigest: - writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) case fs.ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: @@ -522,8 +539,9 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { object := vars["object"] // Get Content-Md5 sent by client and verify if valid - md5 := r.Header.Get("Content-Md5") - if !isValidMD5(md5) { + md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) + if err != nil { + errorIf(err.Trace(r.Header.Get("Content-Md5")), "Decoding md5 failed.", nil) writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) return } @@ -539,10 +557,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } - // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) var objectInfo fs.ObjectInfo - var err *probe.Error switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -555,11 +570,11 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } // Create anonymous object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, md5, size, r.Body, nil) + objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) case authTypePresigned: // For presigned requests verify them right here. var ok bool - ok, err = auth.DoesPresignedSignatureMatch() + ok, err = doesPresignedSignatureMatch(r) if err != nil { errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) @@ -570,14 +585,46 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } // Create presigned object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, md5, size, r.Body, nil) + objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) case authTypeSigned: + // Initialize a pipe for data pipe line. + reader, writer := io.Pipe() + + // Start writing in a routine. + go func() { + shaWriter := sha256.New() + multiWriter := io.MultiWriter(shaWriter, writer) + if _, e := io.CopyN(multiWriter, r.Body, size); e != nil { + errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) + writer.CloseWithError(e) + return + } + shaPayload := shaWriter.Sum(nil) + ok, serr := doesSignatureMatch(hex.EncodeToString(shaPayload), r) + if serr != nil { + errorIf(serr.Trace(), "Signature verification failed.", nil) + writer.CloseWithError(probe.WrapError(serr)) + return + } + if !ok { + writer.CloseWithError(errSignatureMismatch) + return + } + writer.Close() + }() + // Create object. - objectInfo, err = api.Filesystem.CreateObject(bucket, object, md5, size, r.Body, &auth) + objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) } if err != nil { errorIf(err.Trace(), "CreateObject failed.", nil) - switch err.ToGoError().(type) { + e := err.ToGoError() + // Verify if the underlying error is signature mismatch. + if e == errSignatureMismatch { + writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) + return + } + switch e.(type) { case fs.RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) case fs.BucketNotFound: @@ -586,12 +633,8 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case fs.BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.SignDoesNotMatch: - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) case fs.IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.InvalidDigest: - writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) case fs.ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: @@ -615,14 +658,18 @@ func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.R object = vars["object"] switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } - default: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -663,8 +710,9 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques object := vars["object"] // get Content-Md5 sent by client and verify if valid - md5 := r.Header.Get("Content-Md5") - if !isValidMD5(md5) { + md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) + if err != nil { + errorIf(err.Trace(r.Header.Get("Content-Md5")), "Decoding md5 failed.", nil) writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) return } @@ -691,11 +739,12 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques return } - // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) var partMD5 string - var err *probe.Error switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html if s3Error := enforceBucketPolicy("s3:PutObject", bucket, r.URL); s3Error != ErrNone { @@ -704,11 +753,11 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques } // No need to verify signature, anonymous request access is // already allowed. - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, md5, partID, size, r.Body, nil) + partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) case authTypePresigned: // For presigned requests verify right here. var ok bool - ok, err = auth.DoesPresignedSignatureMatch() + ok, err = doesPresignedSignatureMatch(r) if err != nil { errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) @@ -718,25 +767,52 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) return } - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, md5, partID, size, r.Body, nil) - default: - partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, md5, partID, size, r.Body, &auth) + partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) + case authTypeSigned: + // Initialize a pipe for data pipe line. + reader, writer := io.Pipe() + + // Start writing in a routine. + go func() { + shaWriter := sha256.New() + multiWriter := io.MultiWriter(shaWriter, writer) + if _, e := io.CopyN(multiWriter, r.Body, size); e != nil { + errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) + writer.CloseWithError(e) + return + } + shaPayload := shaWriter.Sum(nil) + ok, serr := doesSignatureMatch(hex.EncodeToString(shaPayload), r) + if serr != nil { + errorIf(serr.Trace(), "Signature verification failed.", nil) + writer.CloseWithError(probe.WrapError(serr)) + return + } + if !ok { + writer.CloseWithError(errSignatureMismatch) + return + } + writer.Close() + }() + partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, reader, md5Bytes) } if err != nil { errorIf(err.Trace(), "CreateObjectPart failed.", nil) - switch err.ToGoError().(type) { + e := err.ToGoError() + // Verify if the underlying error is signature mismatch. + if e == errSignatureMismatch { + writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) + return + } + switch e.(type) { case fs.RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) case fs.InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) case fs.BadDigest: writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) - case fs.SignDoesNotMatch: - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) case fs.IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) - case fs.InvalidDigest: - writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } @@ -755,14 +831,18 @@ func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http object := vars["object"] switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html if s3Error := enforceBucketPolicy("s3:AbortMultipartUpload", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } - default: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -798,14 +878,18 @@ func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Requ object := vars["object"] switch getRequestAuthType(r) { + default: + // For all unknown auth types return error. + writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path) + return case authTypeAnonymous: // http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html if s3Error := enforceBucketPolicy("s3:ListMultipartUploadParts", bucket, r.URL); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } - default: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -860,9 +944,6 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h // Extract object resources. objectResourcesMetadata := getObjectResources(r.URL.Query()) - // Set http request for signature. - auth := api.Signature.SetHTTPRequestToVerify(r) - var objectInfo fs.ObjectInfo var err *probe.Error switch getRequestAuthType(r) { @@ -876,26 +957,27 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h writeErrorResponse(w, r, s3Error, r.URL.Path) return } + completePartBytes, e := ioutil.ReadAll(r.Body) + if e != nil { + errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) + return + } // Complete multipart upload anonymous. - objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, r.Body, nil) - case authTypePresigned: - // For presigned requests verify right here. - var ok bool - ok, err = auth.DoesPresignedSignatureMatch() - if err != nil { - errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) + objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes) + case authTypePresigned, authTypeSigned: + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { + writeErrorResponse(w, r, s3Error, r.URL.Path) return } - if !ok { - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) + completePartBytes, e := ioutil.ReadAll(r.Body) + if e != nil { + errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } // Complete multipart upload presigned. - objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, r.Body, nil) - case authTypeSigned: - // Complete multipart upload. - objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, r.Body, &auth) + objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes) } if err != nil { errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil) @@ -914,8 +996,6 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path) case fs.InvalidPartOrder: writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path) - case fs.SignDoesNotMatch: - writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) case fs.IncompleteBody: writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) case fs.MalformedXML: @@ -925,12 +1005,12 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h } return } - // get object location. + // Get object location. location := getLocation(r) // Generate complete multipart response. response := generateCompleteMultpartUploadResponse(bucket, object, location, objectInfo.MD5Sum) encodedSuccessResponse := encodeResponse(response) - // write headers + // Write headers. setCommonHeaders(w) // write success response. writeSuccessResponse(w, encodedSuccessResponse) @@ -956,7 +1036,7 @@ func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request return } case authTypeSigned, authTypePresigned: - if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone { + if s3Error := isReqAuthenticated(r); s3Error != ErrNone { writeErrorResponse(w, r, s3Error, r.URL.Path) return } @@ -976,6 +1056,7 @@ func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } + return } writeSuccessNoContent(w) } diff --git a/pkg/fs/api_suite_test.go b/pkg/fs/api_suite_test.go index ecf2864fd..aa4156c56 100644 --- a/pkg/fs/api_suite_test.go +++ b/pkg/fs/api_suite_test.go @@ -19,7 +19,6 @@ package fs import ( "bytes" "crypto/md5" - "encoding/base64" "encoding/hex" "encoding/xml" "math/rand" @@ -61,7 +60,7 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) { c.Assert(err, check.IsNil) completedParts := CompleteMultipartUpload{} - completedParts.Part = make([]CompletePart, 0) + //completedParts.Part = make([]CompletePart, 10) for i := 1; i <= 10; i++ { randomPerm := rand.Perm(10) randomString := "" @@ -71,19 +70,17 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) { hasher := md5.New() hasher.Write([]byte(randomString)) - expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil)) + expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) - var calculatedmd5sum string - calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)), - bytes.NewBufferString(randomString), nil) + var calculatedMD5sum string + calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) - c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex) - completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum}) + c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) + completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedMD5sum}) } completedPartsBytes, e := xml.Marshal(completedParts) c.Assert(e, check.IsNil) - objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil) + objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedPartsBytes) c.Assert(err, check.IsNil) c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10") } @@ -105,15 +102,13 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) { hasher := md5.New() hasher.Write([]byte(randomString)) - expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil)) + expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) - var calculatedmd5sum string - calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)), - bytes.NewBufferString(randomString), nil) + var calculatedMD5sum string + calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) - c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex) - parts[i] = calculatedmd5sum + c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) + parts[i] = expectedMD5Sumhex } err = fs.AbortMultipartUpload("bucket", "key", uploadID) c.Assert(err, check.IsNil) @@ -133,14 +128,14 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) { hasher := md5.New() hasher.Write([]byte(randomString)) - expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil)) + expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) key := "obj" + strconv.Itoa(i) objects[key] = []byte(randomString) - objectInfo, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil) + var objectInfo ObjectInfo + objectInfo, err = fs.CreateObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) c.Assert(err, check.IsNil) - c.Assert(objectInfo.MD5Sum, check.Equals, expectedmd5Sumhex) + c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex) } for key, value := range objects { @@ -165,7 +160,7 @@ func testPaging(c *check.C, create func() Filesystem) { // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) @@ -175,7 +170,7 @@ func testPaging(c *check.C, create func() Filesystem) { // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "obj", "", "", 5) c.Assert(err, check.IsNil) @@ -184,9 +179,9 @@ func testPaging(c *check.C, create func() Filesystem) { } // check paging with prefix at end returns less objects { - _, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = fs.CreateObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = fs.CreateObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) @@ -206,9 +201,9 @@ func testPaging(c *check.C, create func() Filesystem) { // check delimited results with delimiter and prefix { - _, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = fs.CreateObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) @@ -260,18 +255,11 @@ func testObjectOverwriteWorks(c *check.C, create func() Filesystem) { err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - hasher1 := md5.New() - hasher1.Write([]byte("one")) - md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil)) - md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil)) - objectInfo, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil) + _, err = fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.IsNil) - c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum) + // c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum) - hasher2 := md5.New() - hasher2.Write([]byte("three")) - md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil)) - _, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil) + _, err = fs.CreateObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer @@ -283,7 +271,7 @@ func testObjectOverwriteWorks(c *check.C, create func() Filesystem) { func testNonExistantBucketOperations(c *check.C, create func() Filesystem) { fs := create() - _, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err := fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.Not(check.IsNil)) } @@ -300,13 +288,8 @@ func testPutObjectInSubdir(c *check.C, create func() Filesystem) { err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - hasher := md5.New() - hasher.Write([]byte("hello world")) - md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - md5Sum1hex := hex.EncodeToString(hasher.Sum(nil)) - objectInfo, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) - c.Assert(objectInfo.MD5Sum, check.Equals, md5Sum1hex) var bytesBuffer bytes.Buffer length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0) @@ -396,7 +379,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) var byteBuffer bytes.Buffer @@ -431,26 +414,9 @@ func testDefaultContentType(c *check.C, create func() Filesystem) { err := fs.MakeBucket("bucket") c.Assert(err, check.IsNil) - // test empty - _, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil) + // Test empty + _, err = fs.CreateObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) metadata, err := fs.GetObjectInfo("bucket", "one") c.Assert(err, check.IsNil) c.Assert(metadata.ContentType, check.Equals, "application/octet-stream") } - -func testContentMD5Set(c *check.C, create func() Filesystem) { - fs := create() - err := fs.MakeBucket("bucket") - c.Assert(err, check.IsNil) - - // test md5 invalid - badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA" - calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil) - c.Assert(err, check.Not(check.IsNil)) - c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum) - - goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=" - calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil) - c.Assert(err, check.IsNil) - c.Assert(calculatedmd5sum, check.Equals, goodmd5sum) -} diff --git a/pkg/fs/fs-bucket-listobjects_test.go b/pkg/fs/fs-bucket-listobjects_test.go index 8f9719882..9d69693f3 100644 --- a/pkg/fs/fs-bucket-listobjects_test.go +++ b/pkg/fs/fs-bucket-listobjects_test.go @@ -27,7 +27,6 @@ import ( ) func TestListObjects(t *testing.T) { - // Make a temporary directory to use as the filesystem. directory, e := ioutil.TempDir("", "minio-list-object-test") if e != nil { @@ -58,36 +57,36 @@ func TestListObjects(t *testing.T) { } defer os.Remove(tmpfile.Name()) // clean up - _, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", "", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) + _, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) if err != nil { t.Fatal(e) } - _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", "", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } - _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 2; i++ { key := "newPrefix" + strconv.Itoa(i) - _, err = fs.CreateObject("test-bucket-list-object", key, "", int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } } - _, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", "", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) + _, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 3; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.CreateObject("test-bucket-list-object", key, "", int64(len(key)), bytes.NewBufferString(key), nil) + _, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } @@ -592,7 +591,7 @@ func BenchmarkListObjects(b *testing.B) { for i := 0; i < 20000; i++ { key := "obj" + strconv.Itoa(i) - _, err = filesystem.CreateObject("ls-benchmark-bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) + _, err = filesystem.CreateObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { b.Fatal(err) } diff --git a/pkg/fs/fs-errors.go b/pkg/fs/fs-errors.go index a360bc702..d3c7313a1 100644 --- a/pkg/fs/fs-errors.go +++ b/pkg/fs/fs-errors.go @@ -18,13 +18,6 @@ package fs import "fmt" -// SignDoesNotMatch - signature does not match. -type SignDoesNotMatch struct{} - -func (e SignDoesNotMatch) Error() string { - return "Signature does not match." -} - // InvalidArgument invalid argument type InvalidArgument struct{} @@ -131,15 +124,14 @@ func (e InvalidDisksArgument) Error() string { return "Invalid number of disks per node" } -// BadDigest bad md5sum +// BadDigest - Content-MD5 you specified did not match what we received. type BadDigest struct { - MD5 string - Bucket string - Object string + ExpectedMD5 string + CalculatedMD5 string } func (e BadDigest) Error() string { - return "Bad digest" + return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5 } // InternalError - generic internal error @@ -183,13 +175,6 @@ type ImplementationError struct { Err error } -// DigestError - Generic MD5 error -type DigestError struct { - Bucket string - Key string - MD5 string -} - /// Bucket related errors // BucketNameInvalid - bucketname provided is invalid @@ -200,9 +185,6 @@ type BucketNameInvalid GenericBucketError // ObjectNameInvalid - object name provided is invalid type ObjectNameInvalid GenericObjectError -// InvalidDigest - md5 in request header invalid -type InvalidDigest DigestError - // Return string an error formatted as the given text func (e ImplementationError) Error() string { error := "" @@ -258,11 +240,6 @@ func (e BackendCorrupted) Error() string { return "Backend corrupted: " + e.Path } -// Return string an error formatted as the given text -func (e InvalidDigest) Error() string { - return "MD5 provided " + e.MD5 + " is invalid" -} - // OperationNotPermitted - operation not permitted type OperationNotPermitted struct { Op string diff --git a/pkg/fs/fs-multipart.go b/pkg/fs/fs-multipart.go index 277cb1164..8ccb2d721 100644 --- a/pkg/fs/fs-multipart.go +++ b/pkg/fs/fs-multipart.go @@ -24,7 +24,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -34,12 +33,10 @@ import ( "time" "github.com/minio/minio/pkg/atomic" - "github.com/minio/minio/pkg/crypto/sha256" "github.com/minio/minio/pkg/crypto/sha512" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/mimedb" "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/s3/signature4" ) // isValidUploadID - is upload id. @@ -322,7 +319,7 @@ func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } // CreateObjectPart - create a part in a multipart session -func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *signature4.Sign) (string, *probe.Error) { +func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Bytes []byte) (string, *probe.Error) { di, err := disk.GetInfo(fs.path) if err != nil { return "", probe.NewError(err) @@ -355,71 +352,58 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) } - if expectedMD5Sum != "" { - var expectedMD5SumBytes []byte - expectedMD5SumBytes, err = base64.StdEncoding.DecodeString(expectedMD5Sum) - if err != nil { - // Pro-actively close the connection - return "", probe.NewError(InvalidDigest{MD5: expectedMD5Sum}) - } - expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) - } - bucket = fs.denormalizeBucket(bucket) bucketPath := filepath.Join(fs.path, bucket) - if _, err = os.Stat(bucketPath); err != nil { + if _, e := os.Stat(bucketPath); e != nil { // Check bucket exists. - if os.IsNotExist(err) { + if os.IsNotExist(e) { return "", probe.NewError(BucketNotFound{Bucket: bucket}) } - return "", probe.NewError(err) + return "", probe.NewError(e) + } + + // md5Hex representation. + var md5Hex string + if len(md5Bytes) != 0 { + md5Hex = hex.EncodeToString(md5Bytes) } objectPath := filepath.Join(bucketPath, object) partPathPrefix := objectPath + uploadID - partPath := partPathPrefix + expectedMD5Sum + fmt.Sprintf("$%d-$multiparts", partID) + partPath := partPathPrefix + md5Hex + fmt.Sprintf("$%d-$multiparts", partID) partFile, e := atomic.FileCreateWithPrefix(partPath, "$multiparts") if e != nil { return "", probe.NewError(e) } + defer partFile.Close() - md5Hasher := md5.New() - sha256Hasher := sha256.New() - partWriter := io.MultiWriter(partFile, md5Hasher, sha256Hasher) - if _, e = io.CopyN(partWriter, data, size); e != nil { + // Initialize md5 writer. + md5Writer := md5.New() + + // Create a multiwriter. + multiWriter := io.MultiWriter(md5Writer, partFile) + + if _, e = io.CopyN(multiWriter, data, size); e != nil { partFile.CloseAndPurge() return "", probe.NewError(e) } - md5sum := hex.EncodeToString(md5Hasher.Sum(nil)) - // Verify if the written object is equal to what is expected, only - // if it is requested as such. - if expectedMD5Sum != "" { - if !isMD5SumEqual(expectedMD5Sum, md5sum) { - partFile.CloseAndPurge() - return "", probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object}) + // Finalize new md5. + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + if len(md5Bytes) != 0 { + if newMD5Hex != md5Hex { + return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) } } - if signature != nil { - ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256Hasher.Sum(nil))) - if err != nil { - partFile.CloseAndPurge() - return "", err.Trace() - } - if !ok { - partFile.CloseAndPurge() - return "", probe.NewError(SignDoesNotMatch{}) - } - } - partFile.Close() - fi, e := os.Stat(partPath) + // Stat the file to get the latest information. + fi, e := os.Stat(partFile.Name()) if e != nil { return "", probe.NewError(e) } partMetadata := PartMetadata{} - partMetadata.ETag = md5sum partMetadata.PartNumber = partID + partMetadata.ETag = newMD5Hex partMetadata.Size = fi.Size() partMetadata.LastModified = fi.ModTime() @@ -450,13 +434,11 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s if err := saveMultipartsSession(*fs.multiparts); err != nil { return "", err.Trace(partPathPrefix) } - - // Return etag. - return partMetadata.ETag, nil + return newMD5Hex, nil } // CompleteMultipartUpload - complete a multipart upload and persist the data -func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *signature4.Sign) (ObjectInfo, *probe.Error) { +func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, completeMultipartBytes []byte) (ObjectInfo, *probe.Error) { // Check bucket name is valid. if !IsValidBucketName(bucket) { return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) @@ -488,26 +470,8 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da return ObjectInfo{}, probe.NewError(e) } - partBytes, e := ioutil.ReadAll(data) - if e != nil { - objectWriter.CloseAndPurge() - return ObjectInfo{}, probe.NewError(e) - } - if signature != nil { - sh := sha256.New() - sh.Write(partBytes) - ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil))) - if err != nil { - objectWriter.CloseAndPurge() - return ObjectInfo{}, err.Trace() - } - if !ok { - objectWriter.CloseAndPurge() - return ObjectInfo{}, probe.NewError(SignDoesNotMatch{}) - } - } completeMultipartUpload := &CompleteMultipartUpload{} - if e = xml.Unmarshal(partBytes, completeMultipartUpload); e != nil { + if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil { objectWriter.CloseAndPurge() return ObjectInfo{}, probe.NewError(MalformedXML{}) } diff --git a/pkg/fs/fs-object.go b/pkg/fs/fs-object.go index 7803af74f..57ef1599c 100644 --- a/pkg/fs/fs-object.go +++ b/pkg/fs/fs-object.go @@ -18,22 +18,19 @@ package fs import ( "bytes" + "crypto/md5" "io" "os" "path/filepath" "strings" - "crypto/md5" - "encoding/base64" "encoding/hex" "runtime" "github.com/minio/minio/pkg/atomic" - "github.com/minio/minio/pkg/crypto/sha256" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/mimedb" "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/s3/signature4" ) /// Object Operations @@ -200,7 +197,7 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool { } // CreateObject - create an object. -func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, sig *signature4.Sign) (ObjectInfo, *probe.Error) { +func (fs Filesystem) CreateObject(bucket string, object string, size int64, data io.Reader, md5Bytes []byte) (ObjectInfo, *probe.Error) { di, e := disk.GetInfo(fs.path) if e != nil { return ObjectInfo{}, probe.NewError(e) @@ -234,18 +231,15 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in // Get object path. objectPath := filepath.Join(bucketPath, object) - if expectedMD5Sum != "" { - var expectedMD5SumBytes []byte - expectedMD5SumBytes, e = base64.StdEncoding.DecodeString(expectedMD5Sum) - if e != nil { - // Pro-actively close the connection. - return ObjectInfo{}, probe.NewError(InvalidDigest{MD5: expectedMD5Sum}) - } - expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) + + // md5Hex representation. + var md5Hex string + if len(md5Bytes) != 0 { + md5Hex = hex.EncodeToString(md5Bytes) } // Write object. - file, e := atomic.FileCreateWithPrefix(objectPath, expectedMD5Sum+"$tmpobject") + file, e := atomic.FileCreateWithPrefix(objectPath, md5Hex+"$tmpobject") if e != nil { switch e := e.(type) { case *os.PathError: @@ -259,52 +253,40 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in return ObjectInfo{}, probe.NewError(e) } } + defer file.Close() - // Instantiate checksum hashers and create a multiwriter. - md5Hasher := md5.New() - sha256Hasher := sha256.New() - objectWriter := io.MultiWriter(file, md5Hasher, sha256Hasher) + // Initialize md5 writer. + md5Writer := md5.New() + // Instantiate a new multi writer. + multiWriter := io.MultiWriter(md5Writer, file) + + // Instantiate checksum hashers and create a multiwriter. if size > 0 { - if _, e = io.CopyN(objectWriter, data, size); e != nil { + if _, e = io.CopyN(multiWriter, data, size); e != nil { file.CloseAndPurge() return ObjectInfo{}, probe.NewError(e) } } else { - if _, e = io.Copy(objectWriter, data); e != nil { + if _, e = io.Copy(multiWriter, data); e != nil { file.CloseAndPurge() return ObjectInfo{}, probe.NewError(e) } } - md5Sum := hex.EncodeToString(md5Hasher.Sum(nil)) - // Verify if the written object is equal to what is expected, only - // if it is requested as such. - if expectedMD5Sum != "" { - if !isMD5SumEqual(expectedMD5Sum, md5Sum) { - file.CloseAndPurge() - return ObjectInfo{}, probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object}) - } - } - sha256Sum := hex.EncodeToString(sha256Hasher.Sum(nil)) - if sig != nil { - ok, err := sig.DoesSignatureMatch(sha256Sum) - if err != nil { - file.CloseAndPurge() - return ObjectInfo{}, err.Trace() - } - if !ok { - file.CloseAndPurge() - return ObjectInfo{}, probe.NewError(SignDoesNotMatch{}) + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + if len(md5Bytes) != 0 { + if newMD5Hex != md5Hex { + return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex}) } } - file.Close() // Set stat again to get the latest metadata. - st, e := os.Stat(objectPath) + st, e := os.Stat(file.Name()) if e != nil { return ObjectInfo{}, probe.NewError(e) } + contentType := "application/octet-stream" if objectExt := filepath.Ext(objectPath); objectExt != "" { content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] @@ -317,8 +299,8 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in Name: object, ModifiedTime: st.ModTime(), Size: st.Size(), + MD5Sum: newMD5Hex, ContentType: contentType, - MD5Sum: md5Sum, } return newObject, nil } diff --git a/pkg/fs/fs-object_test.go b/pkg/fs/fs-object_test.go index ca48acb6d..27c6a44ce 100644 --- a/pkg/fs/fs-object_test.go +++ b/pkg/fs/fs-object_test.go @@ -19,7 +19,6 @@ package fs import ( "bytes" "crypto/md5" - "encoding/base64" "fmt" "io/ioutil" "os" @@ -47,7 +46,7 @@ func TestGetObjectInfo(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", "", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + _, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { t.Fatal(err) } @@ -138,7 +137,7 @@ func TestGetObjectInfoCore(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", "", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + _, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { t.Fatal(err) } @@ -228,16 +227,14 @@ func BenchmarkGetObject(b *testing.B) { text := "Jack and Jill went up the hill / To fetch a pail of water." hasher := md5.New() hasher.Write([]byte(text)) - sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) for i := 0; i < 10; i++ { - _, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), sum, int64(len(text)), bytes.NewBufferString(text), nil) + _, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), hasher.Sum(nil)) if err != nil { b.Fatal(err) } } var w bytes.Buffer - b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/pkg/xl/acl.go b/pkg/xl/acl.go index 796b5954a..c385755ba 100644 --- a/pkg/xl/acl.go +++ b/pkg/xl/acl.go @@ -1,3 +1,5 @@ +// +build ignore + package xl // BucketACL - bucket level access control diff --git a/pkg/xl/bucket.go b/pkg/xl/bucket.go index 2e84b0aab..2b2dbd485 100644 --- a/pkg/xl/bucket.go +++ b/pkg/xl/bucket.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/common.go b/pkg/xl/common.go index 7bda6202a..04d8c9dad 100644 --- a/pkg/xl/common.go +++ b/pkg/xl/common.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/config.go b/pkg/xl/config.go index d9ff4a5f7..f28d5b344 100644 --- a/pkg/xl/config.go +++ b/pkg/xl/config.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/definitions.go b/pkg/xl/definitions.go index b1474b450..9783c619b 100644 --- a/pkg/xl/definitions.go +++ b/pkg/xl/definitions.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/doc.go b/pkg/xl/doc.go new file mode 100644 index 000000000..71587ee37 --- /dev/null +++ b/pkg/xl/doc.go @@ -0,0 +1,3 @@ +// +build ignore + +package xl diff --git a/pkg/xl/encoder.go b/pkg/xl/encoder.go index f82e04031..53567a5ab 100644 --- a/pkg/xl/encoder.go +++ b/pkg/xl/encoder.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/errors.go b/pkg/xl/errors.go index 75e59477c..fc27bb47d 100644 --- a/pkg/xl/errors.go +++ b/pkg/xl/errors.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/heal.go b/pkg/xl/heal.go index 6af30ffd1..da5593600 100644 --- a/pkg/xl/heal.go +++ b/pkg/xl/heal.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/interfaces.go b/pkg/xl/interfaces.go index 79cc9822b..104900bd3 100644 --- a/pkg/xl/interfaces.go +++ b/pkg/xl/interfaces.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/management.go b/pkg/xl/management.go index 3f89c95d0..2b5da25ce 100644 --- a/pkg/xl/management.go +++ b/pkg/xl/management.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/multipart.go b/pkg/xl/multipart.go index df03567b1..c168a3e19 100644 --- a/pkg/xl/multipart.go +++ b/pkg/xl/multipart.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/node.go b/pkg/xl/node.go index a029a15e7..ce79b93e9 100644 --- a/pkg/xl/node.go +++ b/pkg/xl/node.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/xl-v1.go b/pkg/xl/xl-v1.go index 723f6d5b4..8cbad166c 100644 --- a/pkg/xl/xl-v1.go +++ b/pkg/xl/xl-v1.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/xl-v1_test.go b/pkg/xl/xl-v1_test.go index 0a91a315a..4b93ac5a7 100644 --- a/pkg/xl/xl-v1_test.go +++ b/pkg/xl/xl-v1_test.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/xl-v2.go b/pkg/xl/xl-v2.go index 7d6a40ba5..5c57df904 100644 --- a/pkg/xl/xl-v2.go +++ b/pkg/xl/xl-v2.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/pkg/xl/xl-v2_test.go b/pkg/xl/xl-v2_test.go index b82f54fcd..cda78ae30 100644 --- a/pkg/xl/xl-v2_test.go +++ b/pkg/xl/xl-v2_test.go @@ -1,3 +1,5 @@ +// +build ignore + /* * Minio Cloud Storage, (C) 2015 Minio, Inc. * diff --git a/routers.go b/routers.go index 26207022d..86a5fa453 100644 --- a/routers.go +++ b/routers.go @@ -29,7 +29,6 @@ import ( "github.com/minio/minio-go" "github.com/minio/minio/pkg/fs" "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/s3/signature4" "github.com/minio/miniobrowser" ) @@ -37,8 +36,6 @@ import ( type storageAPI struct { // Filesystem instance. Filesystem fs.Filesystem - // Signature instance. - Signature *signature4.Sign } // webAPI container for Web API. @@ -163,19 +160,12 @@ func configureServerHandler(filesystem fs.Filesystem) http.Handler { // Access credentials. cred := serverConfig.GetCredential() - // Server region. - region := serverConfig.GetRegion() - // Server addr. addr := serverConfig.GetAddr() - sign, err := signature4.New(cred.AccessKeyID, cred.SecretAccessKey, region) - fatalIf(err.Trace(cred.AccessKeyID, cred.SecretAccessKey, region), "Initializing signature version '4' failed.", nil) - // Initialize API. api := storageAPI{ Filesystem: filesystem, - Signature: sign, } // Split host port. diff --git a/server_fs_test.go b/server_fs_test.go index 7fce0aa49..23134a22c 100644 --- a/server_fs_test.go +++ b/server_fs_test.go @@ -38,10 +38,6 @@ import ( . "gopkg.in/check.v1" ) -const ( - yyyymmdd = "20060102" -) - type MyAPIFSCacheSuite struct { root string req *http.Request @@ -154,30 +150,32 @@ func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int6 req.Header.Set("x-amz-date", t.Format(iso8601Format)) - // add Content-Length + // Add Content-Length req.ContentLength = contentLength - // add body + // Save for subsequent use + var hashedPayload string switch { case body == nil: - req.Body = nil + hashedPayload = hex.EncodeToString(sum256([]byte{})) default: - req.Body = ioutil.NopCloser(body) - } - - // save for subsequent use - hash := func() string { - switch { - case body == nil: - return hex.EncodeToString(sum256([]byte{})) - default: - sum256Bytes, _ := sum256Reader(body) - return hex.EncodeToString(sum256Bytes) + payloadBytes, e := ioutil.ReadAll(body) + if e != nil { + return nil, e } + hashedPayload = hex.EncodeToString(sum256(payloadBytes)) + md5base64 := base64.StdEncoding.EncodeToString(sumMD5(payloadBytes)) + req.Header.Set("Content-Md5", md5base64) } - hashedPayload := hash() req.Header.Set("x-amz-content-sha256", hashedPayload) + // Seek back to beginning. + if body != nil { + body.Seek(0, 0) + // Add body + req.Body = ioutil.NopCloser(body) + } + var headers []string vals := make(map[string][]string) for k, vv := range req.Header { diff --git a/jwt.go b/signature-jwt.go similarity index 98% rename from jwt.go rename to signature-jwt.go index 2a80d5eab..7d7336921 100644 --- a/jwt.go +++ b/signature-jwt.go @@ -25,6 +25,8 @@ import ( "golang.org/x/crypto/bcrypt" ) +const jwtAlgorithm = "Bearer" + // JWT - jwt auth backend type JWT struct { credential diff --git a/pkg/s3/signature4/v4-errors.go b/signature-v4-errors.go similarity index 90% rename from pkg/s3/signature4/v4-errors.go rename to signature-v4-errors.go index cbc8792df..dce0bd391 100644 --- a/pkg/s3/signature4/v4-errors.go +++ b/signature-v4-errors.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package signature4 +package main import ( "fmt" @@ -48,10 +48,7 @@ var ( ErrMalformedExpires = errFactory() ErrAuthHeaderEmpty = errFactory() ErrUnsuppSignAlgo = errFactory() - ErrMissingExpiresQuery = errFactory() ErrExpiredPresignRequest = errFactory() - ErrSignDoesNotMath = errFactory() - ErrInvalidAccessKeyID = errFactory() - ErrInvalidSecretKey = errFactory() ErrRegionISEmpty = errFactory() + ErrInvalidAccessKey = errFactory() ) diff --git a/pkg/s3/signature4/v4-parser.go b/signature-v4-parser.go similarity index 78% rename from pkg/s3/signature4/v4-parser.go rename to signature-v4-parser.go index 0674b5807..e4c7eab64 100644 --- a/pkg/s3/signature4/v4-parser.go +++ b/signature-v4-parser.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package signature4 +package main import ( "net/url" @@ -24,11 +24,11 @@ import ( "github.com/minio/minio/pkg/probe" ) -// credential data type represents structured form of Credential +// credentialHeader data type represents structured form of Credential // string from authorization header. -type credential struct { - accessKeyID string - scope struct { +type credentialHeader struct { + accessKey string + scope struct { date time.Time region string service string @@ -36,40 +36,41 @@ type credential struct { } } -// parse credential string into its structured form. -func parseCredential(credElement string) (credential, *probe.Error) { +// parse credentialHeader string into its structured form. +func parseCredentialHeader(credElement string) (credentialHeader, *probe.Error) { creds := strings.Split(strings.TrimSpace(credElement), "=") if len(creds) != 2 { - return credential{}, ErrMissingFields("Credential tag has missing fields.", credElement).Trace(credElement) + return credentialHeader{}, ErrMissingFields("Credential tag has missing fields.", credElement).Trace(credElement) } if creds[0] != "Credential" { - return credential{}, ErrMissingCredTag("Missing credentials tag.", credElement).Trace(credElement) + return credentialHeader{}, ErrMissingCredTag("Missing credentials tag.", credElement).Trace(credElement) } credElements := strings.Split(strings.TrimSpace(creds[1]), "/") if len(credElements) != 5 { - return credential{}, ErrCredMalformed("Credential values malformed.", credElement).Trace(credElement) + return credentialHeader{}, ErrCredMalformed("Credential values malformed.", credElement).Trace(credElement) } if !isValidAccessKey.MatchString(credElements[0]) { - return credential{}, ErrInvalidAccessKeyID("Invalid access key id.", credElement).Trace(credElement) + return credentialHeader{}, ErrInvalidAccessKey("Invalid access key id.", credElement).Trace(credElement) } - cred := credential{ - accessKeyID: credElements[0], + // Save access key id. + cred := credentialHeader{ + accessKey: credElements[0], } var e error cred.scope.date, e = time.Parse(yyyymmdd, credElements[1]) if e != nil { - return credential{}, ErrInvalidDateFormat("Invalid date format.", credElement).Trace(credElement) + return credentialHeader{}, ErrInvalidDateFormat("Invalid date format.", credElement).Trace(credElement) } if credElements[2] == "" { - return credential{}, ErrRegionISEmpty("Region is empty.", credElement).Trace(credElement) + return credentialHeader{}, ErrRegionISEmpty("Region is empty.", credElement).Trace(credElement) } cred.scope.region = credElements[2] if credElements[3] != "s3" { - return credential{}, ErrInvalidService("Invalid service detected.", credElement).Trace(credElement) + return credentialHeader{}, ErrInvalidService("Invalid service detected.", credElement).Trace(credElement) } cred.scope.service = credElements[3] if credElements[4] != "aws4_request" { - return credential{}, ErrInvalidRequestVersion("Invalid request version detected.", credElement).Trace(credElement) + return credentialHeader{}, ErrInvalidRequestVersion("Invalid request version detected.", credElement).Trace(credElement) } cred.scope.request = credElements[4] return cred, nil @@ -103,7 +104,7 @@ func parseSignedHeaders(signedHdrElement string) ([]string, *probe.Error) { // signValues data type represents structured form of AWS Signature V4 header. type signValues struct { - Credential credential + Credential credentialHeader SignedHeaders []string Signature string } @@ -118,7 +119,7 @@ type preSignValues struct { // Parses signature version '4' query string of the following form. // // querystring = X-Amz-Algorithm=algorithm -// querystring += &X-Amz-Credential= urlencode(access_key_ID + '/' + credential_scope) +// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope) // querystring += &X-Amz-Date=date // querystring += &X-Amz-Expires=timeout interval // querystring += &X-Amz-SignedHeaders=signed_headers @@ -135,7 +136,7 @@ func parsePreSignV4(query url.Values) (preSignValues, *probe.Error) { var err *probe.Error // Save credential. - preSignV4Values.Credential, err = parseCredential("Credential=" + query.Get("X-Amz-Credential")) + preSignV4Values.Credential, err = parseCredentialHeader("Credential=" + query.Get("X-Amz-Credential")) if err != nil { return preSignValues{}, err.Trace(query.Get("X-Amz-Credential")) } @@ -171,8 +172,8 @@ func parsePreSignV4(query url.Values) (preSignValues, *probe.Error) { // Parses signature version '4' header of the following form. // -// Authorization: algorithm Credential=access key ID/credential scope, \ -// SignedHeaders=SignedHeaders, Signature=signature +// Authorization: algorithm Credential=accessKeyID/credScope, \ +// SignedHeaders=signedHeaders, Signature=signature // func parseSignV4(v4Auth string) (signValues, *probe.Error) { // Replace all spaced strings, some clients can send spaced @@ -200,7 +201,7 @@ func parseSignV4(v4Auth string) (signValues, *probe.Error) { var err *probe.Error // Save credentail values. - signV4Values.Credential, err = parseCredential(authFields[0]) + signV4Values.Credential, err = parseCredentialHeader(authFields[0]) if err != nil { return signValues{}, err.Trace(v4Auth) } diff --git a/pkg/s3/signature4/v4-postpolicyform.go b/signature-v4-postpolicyform.go similarity index 97% rename from pkg/s3/signature4/v4-postpolicyform.go rename to signature-v4-postpolicyform.go index 47247a433..9cd5c4dad 100644 --- a/pkg/s3/signature4/v4-postpolicyform.go +++ b/signature-v4-postpolicyform.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package signature4 +package main import ( "encoding/base64" @@ -158,8 +158,8 @@ func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { return parsedPolicy, nil } -// ApplyPolicyCond - apply policy conditions and validate input values. -func ApplyPolicyCond(formValues map[string]string) *probe.Error { +// checkPostPolicy - apply policy conditions and validate input values. +func checkPostPolicy(formValues map[string]string) *probe.Error { if formValues["X-Amz-Algorithm"] != signV4Algorithm { return ErrUnsuppSignAlgo("Unsupported signature algorithm in policy form data.", formValues["X-Amz-Algorithm"]).Trace(formValues["X-Amz-Algorithm"]) } diff --git a/pkg/s3/signature4/v4-utils.go b/signature-v4-utils.go similarity index 94% rename from pkg/s3/signature4/v4-utils.go rename to signature-v4-utils.go index 270c1f2a3..b01011b8c 100644 --- a/pkg/s3/signature4/v4-utils.go +++ b/signature-v4-utils.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package signature4 +package main import ( "crypto/hmac" @@ -27,14 +27,6 @@ import ( "github.com/minio/minio/pkg/crypto/sha256" ) -/// helpers - -// isValidSecretKey - validate secret key. -var isValidSecretKey = regexp.MustCompile("^.{40}$") - -// isValidAccessKey - validate access key. -var isValidAccessKey = regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$") - // isValidRegion - verify if incoming region value is valid with configured Region. func isValidRegion(reqRegion string, confRegion string) bool { if confRegion == "" || confRegion == "US" { diff --git a/pkg/s3/signature4/v4-signature.go b/signature-v4.go similarity index 53% rename from pkg/s3/signature4/v4-signature.go rename to signature-v4.go index 8647794b9..7ca6544f4 100644 --- a/pkg/s3/signature4/v4-signature.go +++ b/signature-v4.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -// Package signature4 implements helper functions to validate AWS +// This file implements helper functions to validate AWS // Signature Version '4' authorization header. // // This package provides comprehensive helpers for following signature @@ -22,7 +22,7 @@ // - Based on Authorization header. // - Based on Query parameters. // - Based on Form POST policy. -package signature4 +package main import ( "bytes" @@ -38,15 +38,6 @@ import ( "github.com/minio/minio/pkg/probe" ) -// Sign - local variables -type Sign struct { - accessKeyID string - secretAccessKey string - region string - httpRequest *http.Request - extractedSignedHeaders http.Header -} - // AWS Signature Version '4' constants. const ( signV4Algorithm = "AWS4-HMAC-SHA256" @@ -54,37 +45,8 @@ const ( yyyymmdd = "20060102" ) -// New - initialize a new authorization checkes. -func New(accessKeyID, secretAccessKey, region string) (*Sign, *probe.Error) { - if !isValidAccessKey.MatchString(accessKeyID) { - return nil, ErrInvalidAccessKeyID("Invalid access key id.", accessKeyID).Trace(accessKeyID) - } - if !isValidSecretKey.MatchString(secretAccessKey) { - return nil, ErrInvalidAccessKeyID("Invalid secret key.", secretAccessKey).Trace(secretAccessKey) - } - if region == "" { - return nil, ErrRegionISEmpty("Region is empty.").Trace() - } - signature := &Sign{ - accessKeyID: accessKeyID, - secretAccessKey: secretAccessKey, - region: region, - } - return signature, nil -} - -// SetHTTPRequestToVerify - sets the http request which needs to be verified. -func (s *Sign) SetHTTPRequestToVerify(r *http.Request) Sign { - // Do not set http request if its 'nil'. - if r == nil { - return *s - } - s.httpRequest = r - return *s -} - // getCanonicalHeaders generate a list of request headers with their values -func (s Sign) getCanonicalHeaders(signedHeaders http.Header) string { +func getCanonicalHeaders(signedHeaders http.Header, host string) string { var headers []string vals := make(http.Header) for k, vv := range signedHeaders { @@ -100,7 +62,7 @@ func (s Sign) getCanonicalHeaders(signedHeaders http.Header) string { buf.WriteByte(':') switch { case k == "host": - buf.WriteString(s.httpRequest.Host) + buf.WriteString(host) fallthrough default: for idx, v := range vals[k] { @@ -116,7 +78,7 @@ func (s Sign) getCanonicalHeaders(signedHeaders http.Header) string { } // getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names -func (s Sign) getSignedHeaders(signedHeaders http.Header) string { +func getSignedHeaders(signedHeaders http.Header) string { var headers []string for k := range signedHeaders { headers = append(headers, strings.ToLower(k)) @@ -136,18 +98,17 @@ func (s Sign) getSignedHeaders(signedHeaders http.Header) string { // \n // // -func (s *Sign) getCanonicalRequest() string { - payload := s.httpRequest.Header.Get(http.CanonicalHeaderKey("x-amz-content-sha256")) - s.httpRequest.URL.RawQuery = strings.Replace(s.httpRequest.URL.Query().Encode(), "+", "%20", -1) - encodedPath := getURLEncodedName(s.httpRequest.URL.Path) +func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, urlPath, method, host string) string { + rawQuery := strings.Replace(queryStr, "+", "%20", -1) + encodedPath := getURLEncodedName(urlPath) // Convert any space strings back to "+". encodedPath = strings.Replace(encodedPath, "+", "%20", -1) canonicalRequest := strings.Join([]string{ - s.httpRequest.Method, + method, encodedPath, - s.httpRequest.URL.RawQuery, - s.getCanonicalHeaders(s.extractedSignedHeaders), - s.getSignedHeaders(s.extractedSignedHeaders), + rawQuery, + getCanonicalHeaders(extractedSignedHeaders, host), + getSignedHeaders(extractedSignedHeaders), payload, }, "\n") return canonicalRequest @@ -163,27 +124,27 @@ func (s *Sign) getCanonicalRequest() string { // \n // // -func (s Sign) getPresignedCanonicalRequest(presignedQuery string) string { +func getPresignCanonicalRequest(extractedSignedHeaders http.Header, presignedQuery, urlPath, method, host string) string { rawQuery := strings.Replace(presignedQuery, "+", "%20", -1) - encodedPath := getURLEncodedName(s.httpRequest.URL.Path) + encodedPath := getURLEncodedName(urlPath) // Convert any space strings back to "+". encodedPath = strings.Replace(encodedPath, "+", "%20", -1) canonicalRequest := strings.Join([]string{ - s.httpRequest.Method, + method, encodedPath, rawQuery, - s.getCanonicalHeaders(s.extractedSignedHeaders), - s.getSignedHeaders(s.extractedSignedHeaders), + getCanonicalHeaders(extractedSignedHeaders, host), + getSignedHeaders(extractedSignedHeaders), "UNSIGNED-PAYLOAD", }, "\n") return canonicalRequest } // getScope generate a string of a specific date, an AWS region, and a service. -func (s Sign) getScope(t time.Time) string { +func getScope(t time.Time, region string) string { scope := strings.Join([]string{ t.Format(yyyymmdd), - s.region, + region, "s3", "aws4_request", }, "/") @@ -191,92 +152,105 @@ func (s Sign) getScope(t time.Time) string { } // getStringToSign a string based on selected query values. -func (s Sign) getStringToSign(canonicalRequest string, t time.Time) string { +func getStringToSign(canonicalRequest string, t time.Time, region string) string { stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n" - stringToSign = stringToSign + s.getScope(t) + "\n" + stringToSign = stringToSign + getScope(t, region) + "\n" canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest)) stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:]) return stringToSign } // getSigningKey hmac seed to calculate final signature. -func (s Sign) getSigningKey(t time.Time) []byte { - secret := s.secretAccessKey - date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) - region := sumHMAC(date, []byte(s.region)) - service := sumHMAC(region, []byte("s3")) +func getSigningKey(secretKey string, t time.Time, region string) []byte { + date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd))) + regionBytes := sumHMAC(date, []byte(region)) + service := sumHMAC(regionBytes, []byte("s3")) signingKey := sumHMAC(service, []byte("aws4_request")) return signingKey } // getSignature final signature in hexadecimal form. -func (s Sign) getSignature(signingKey []byte, stringToSign string) string { +func getSignature(signingKey []byte, stringToSign string) string { return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) } -// DoesPolicySignatureMatch - Verify query headers with post policy +// doesPolicySignatureMatch - Verify query headers with post policy // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // returns true if matches, false otherwise. if error is not nil then it is always false -func (s *Sign) DoesPolicySignatureMatch(formValues map[string]string) (bool, *probe.Error) { +func doesPolicySignatureMatch(formValues map[string]string) (bool, *probe.Error) { + // Access credentials. + cred := serverConfig.GetCredential() + + // Server region. + region := serverConfig.GetRegion() + // Parse credential tag. - credential, err := parseCredential("Credential=" + formValues["X-Amz-Credential"]) + credHeader, err := parseCredentialHeader("Credential=" + formValues["X-Amz-Credential"]) if err != nil { return false, err.Trace(formValues["X-Amz-Credential"]) } // Verify if the access key id matches. - if credential.accessKeyID != s.accessKeyID { - return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", credential.accessKeyID).Trace(credential.accessKeyID) + if credHeader.accessKey != cred.AccessKeyID { + return false, ErrInvalidAccessKey("Access key id does not match with our records.", credHeader.accessKey).Trace(credHeader.accessKey) } // Verify if the region is valid. - reqRegion := credential.scope.region - if !isValidRegion(reqRegion, s.region) { - return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion) + if !isValidRegion(credHeader.scope.region, region) { + return false, ErrInvalidRegion("Requested region is not recognized.", credHeader.scope.region).Trace(credHeader.scope.region) } - // Save region. - s.region = reqRegion - // Parse date string. t, e := time.Parse(iso8601Format, formValues["X-Amz-Date"]) if e != nil { return false, probe.NewError(e) } - signingKey := s.getSigningKey(t) - newSignature := s.getSignature(signingKey, formValues["Policy"]) + + // Get signing key. + signingKey := getSigningKey(cred.SecretAccessKey, t, region) + + // Get signature. + newSignature := getSignature(signingKey, formValues["Policy"]) + + // Verify signature. if newSignature != formValues["X-Amz-Signature"] { return false, nil } return true, nil } -// DoesPresignedSignatureMatch - Verify query headers with presigned signature +// doesPresignedSignatureMatch - Verify query headers with presigned signature // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html // returns true if matches, false otherwise. if error is not nil then it is always false -func (s *Sign) DoesPresignedSignatureMatch() (bool, *probe.Error) { +func doesPresignedSignatureMatch(r *http.Request) (bool, *probe.Error) { + // Access credentials. + cred := serverConfig.GetCredential() + + // Server region. + region := serverConfig.GetRegion() + + // Copy request + req := *r + // Parse request query string. - preSignValues, err := parsePreSignV4(s.httpRequest.URL.Query()) + preSignValues, err := parsePreSignV4(req.URL.Query()) if err != nil { - return false, err.Trace(s.httpRequest.URL.String()) + return false, err.Trace(req.URL.String()) } // Verify if the access key id matches. - if preSignValues.Credential.accessKeyID != s.accessKeyID { - return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", preSignValues.Credential.accessKeyID).Trace(preSignValues.Credential.accessKeyID) + if preSignValues.Credential.accessKey != cred.AccessKeyID { + return false, ErrInvalidAccessKey("Access key id does not match with our records.", preSignValues.Credential.accessKey).Trace(preSignValues.Credential.accessKey) } // Verify if region is valid. - reqRegion := preSignValues.Credential.scope.region - if !isValidRegion(reqRegion, s.region) { - return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion) + sRegion := preSignValues.Credential.scope.region + if !isValidRegion(sRegion, region) { + return false, ErrInvalidRegion("Requested region is not recognized.", sRegion).Trace(sRegion) } - // Save region. - s.region = reqRegion - // Extract all the signed headers along with its values. - s.extractedSignedHeaders = extractSignedHeaders(preSignValues.SignedHeaders, s.httpRequest.Header) + extractedSignedHeaders := extractSignedHeaders(preSignValues.SignedHeaders, req.Header) // Construct new query. query := make(url.Values) @@ -293,11 +267,11 @@ func (s *Sign) DoesPresignedSignatureMatch() (bool, *probe.Error) { // Construct the query. query.Set("X-Amz-Date", t.Format(iso8601Format)) query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds)) - query.Set("X-Amz-SignedHeaders", s.getSignedHeaders(s.extractedSignedHeaders)) - query.Set("X-Amz-Credential", s.accessKeyID+"/"+s.getScope(t)) + query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders)) + query.Set("X-Amz-Credential", cred.AccessKeyID+"/"+getScope(t, region)) // Save other headers available in the request parameters. - for k, v := range s.httpRequest.URL.Query() { + for k, v := range req.URL.Query() { if strings.HasPrefix(strings.ToLower(k), "x-amz") { continue } @@ -308,35 +282,58 @@ func (s *Sign) DoesPresignedSignatureMatch() (bool, *probe.Error) { encodedQuery := query.Encode() // Verify if date query is same. - if s.httpRequest.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") { + if req.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") { return false, nil } // Verify if expires query is same. - if s.httpRequest.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") { + if req.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") { return false, nil } // Verify if signed headers query is same. - if s.httpRequest.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") { + if req.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") { return false, nil } // Verify if credential query is same. - if s.httpRequest.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") { + if req.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") { return false, nil } - // Verify finally if signature is same. - newSignature := s.getSignature(s.getSigningKey(t), s.getStringToSign(s.getPresignedCanonicalRequest(encodedQuery), t)) - if s.httpRequest.URL.Query().Get("X-Amz-Signature") != newSignature { + + /// Verify finally if signature is same. + + // Get canonical request. + presignedCanonicalReq := getPresignCanonicalRequest(extractedSignedHeaders, encodedQuery, req.URL.Path, req.Method, req.Host) + + // Get string to sign from canonical request. + presignedStringToSign := getStringToSign(presignedCanonicalReq, t, region) + + // Get hmac presigned signing key. + presignedSigningKey := getSigningKey(cred.SecretAccessKey, t, region) + + // Get new signature. + newSignature := getSignature(presignedSigningKey, presignedStringToSign) + + // Verify signature. + if req.URL.Query().Get("X-Amz-Signature") != newSignature { return false, nil } return true, nil } -// DoesSignatureMatch - Verify authorization header with calculated header in accordance with +// doesSignatureMatch - Verify authorization header with calculated header in accordance with // - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html // returns true if matches, false otherwise. if error is not nil then it is always false -func (s *Sign) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { +func doesSignatureMatch(hashedPayload string, r *http.Request) (bool, *probe.Error) { + // Access credentials. + cred := serverConfig.GetCredential() + + // Server region. + region := serverConfig.GetRegion() + + // Copy request. + req := *r + // Save authorization header. - v4Auth := s.httpRequest.Header.Get("Authorization") + v4Auth := req.Header.Get("Authorization") // Parse signature version '4' header. signV4Values, err := parseSignV4(v4Auth) @@ -345,29 +342,23 @@ func (s *Sign) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { } // Extract all the signed headers along with its values. - s.extractedSignedHeaders = extractSignedHeaders(signV4Values.SignedHeaders, s.httpRequest.Header) + extractedSignedHeaders := extractSignedHeaders(signV4Values.SignedHeaders, req.Header) // Verify if the access key id matches. - if signV4Values.Credential.accessKeyID != s.accessKeyID { - return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", signV4Values.Credential.accessKeyID).Trace(signV4Values.Credential.accessKeyID) + if signV4Values.Credential.accessKey != cred.AccessKeyID { + return false, ErrInvalidAccessKey("Access key id does not match with our records.", signV4Values.Credential.accessKey).Trace(signV4Values.Credential.accessKey) } // Verify if region is valid. - reqRegion := signV4Values.Credential.scope.region - if !isValidRegion(reqRegion, s.region) { - return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion) + sRegion := signV4Values.Credential.scope.region + if !isValidRegion(sRegion, region) { + return false, ErrInvalidRegion("Requested region is not recognized.", sRegion).Trace(sRegion) } - // Save region. - s.region = reqRegion - - // Set input payload. - s.httpRequest.Header.Set("X-Amz-Content-Sha256", hashedPayload) - // Extract date, if not present throw error. var date string - if date = s.httpRequest.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" { - if date = s.httpRequest.Header.Get("Date"); date == "" { + if date = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" { + if date = r.Header.Get("Date"); date == "" { return false, ErrMissingDateHeader("Date header is missing from the request.").Trace() } } @@ -377,11 +368,20 @@ func (s *Sign) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { return false, probe.NewError(e) } - // Signature version '4'. - canonicalRequest := s.getCanonicalRequest() - stringToSign := s.getStringToSign(canonicalRequest, t) - signingKey := s.getSigningKey(t) - newSignature := s.getSignature(signingKey, stringToSign) + // Query string. + queryStr := req.URL.Query().Encode() + + // Get canonical request. + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method, req.Host) + + // Get string to sign from canonical request. + stringToSign := getStringToSign(canonicalRequest, t, region) + + // Get hmac signing key. + signingKey := getSigningKey(cred.SecretAccessKey, t, region) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) // Verify if signature match. if newSignature != signV4Values.Signature { diff --git a/signature_utils_test.go b/signature_utils_test.go deleted file mode 100644 index 8d747b5e3..000000000 --- a/signature_utils_test.go +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "crypto/hmac" - "encoding/hex" - "io" - "regexp" - "strings" - "unicode/utf8" - - "github.com/minio/minio/pkg/crypto/sha256" -) - -// sum256Reader calculate sha256 sum for an input read seeker -func sum256Reader(reader io.ReadSeeker) ([]byte, error) { - h := sha256.New() - var err error - - start, _ := reader.Seek(0, 1) - defer reader.Seek(start, 0) - - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - h.Write(byteBuffer) - } - - if err != io.EOF { - return nil, err - } - - return h.Sum(nil), nil -} - -// sum256 calculate sha256 sum for an input byte array -func sum256(data []byte) []byte { - hash := sha256.New() - hash.Write(data) - return hash.Sum(nil) -} - -// sumHMAC calculate hmac between two input byte array -func sumHMAC(key []byte, data []byte) []byte { - hash := hmac.New(sha256.New, key) - hash.Write(data) - return hash.Sum(nil) -} - -// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences -// -// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 -// non english characters cannot be parsed due to the nature in which url.Encode() is written -// -// This function on the other hand is a direct replacement for url.Encode() technique to support -// pretty much every UTF-8 character. -func getURLEncodedName(name string) string { - // if object matches reserved string, no need to encode them - reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") - if reservedNames.MatchString(name) { - return name - } - var encodedName string - for _, s := range name { - if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) - encodedName = encodedName + string(s) - continue - } - switch s { - case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedName = encodedName + string(s) - continue - default: - len := utf8.RuneLen(s) - if len < 0 { - return name - } - u := make([]byte, len) - utf8.EncodeRune(u, s) - for _, r := range u { - hex := hex.EncodeToString([]byte{r}) - encodedName = encodedName + "%" + strings.ToUpper(hex) - } - } - } - return encodedName -} diff --git a/typed-errors.go b/typed-errors.go index a7f88840d..64ae986fe 100644 --- a/typed-errors.go +++ b/typed-errors.go @@ -23,3 +23,6 @@ var errSyslogNotSupported = errors.New("Syslog logger not supported on windows") // errInvalidArgument means that input argument is invalid. var errInvalidArgument = errors.New("Invalid arguments specified") + +// errSignatureMismatch means signature did not match. +var errSignatureMismatch = errors.New("Signature does not match") diff --git a/utils.go b/utils.go index c478008a1..b473fa35f 100644 --- a/utils.go +++ b/utils.go @@ -19,18 +19,17 @@ package main import ( "encoding/base64" "strings" + + "github.com/minio/minio/pkg/probe" ) -// isValidMD5 - verify if valid md5 -func isValidMD5(md5 string) bool { - if md5 == "" { - return true - } - _, err := base64.StdEncoding.DecodeString(strings.TrimSpace(md5)) - if err != nil { - return false +// checkValidMD5 - verify if valid md5, returns md5 in bytes. +func checkValidMD5(md5 string) ([]byte, *probe.Error) { + md5Bytes, e := base64.StdEncoding.DecodeString(strings.TrimSpace(md5)) + if e != nil { + return nil, probe.NewError(e) } - return true + return md5Bytes, nil } /// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html