From b62cd8ed84eb29cab0b667e07e671f33c1465760 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 27 Mar 2017 17:02:04 -0700 Subject: [PATCH] sign/streaming: Content-Encoding is not set in newer aws-java-sdks (#3986) We can't use Content-Encoding to verify if `aws-chunked` is set or not. Just use 'streaming' signature header instead. While this is considered mandatory, on the contrary aws-sdk-java doesn't set this value http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html ``` Set the value to aws-chunked. ``` We will relax it and behave appropriately. Also this PR supports saving custom encoding after trimming off the `aws-chunked` parameter. Fixes #3983 --- cmd/auth-handler.go | 1 - cmd/handler-utils.go | 17 +++++++++++++++++ cmd/object-handlers.go | 21 +++++++++++++++++---- cmd/object-handlers_test.go | 31 ++++++++++++++++++++++++++++--- cmd/test-utils_test.go | 19 +++++++++++++++++++ 5 files changed, 81 insertions(+), 8 deletions(-) diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index a979ae961..27b6a1408 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -64,7 +64,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool { // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. func isRequestSignStreamingV4(r *http.Request) bool { return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 && - r.Header.Get("content-encoding") == streamingContentEncoding && r.Method == httpPUT } diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index 19319f6a4..e8435b1bd 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -158,6 +158,23 @@ func extractReqParams(r *http.Request) map[string]string { } } +// Trims away `aws-chunked` from the content-encoding header if present. +// Streaming signature clients can have custom content-encoding such as +// `aws-chunked,gzip` here we need to only save `gzip`. +// For more refer http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string) { + if contentEnc == "" { + return contentEnc + } + var newEncs []string + for _, enc := range strings.Split(contentEnc, ",") { + if enc != streamingContentEncoding { + newEncs = append(newEncs, enc) + } + } + return strings.Join(newEncs, ",") +} + // extractMetadataFromForm extracts metadata from Post Form. func extractMetadataFromForm(formValues http.Header) map[string]string { return extractMetadataFromHeader(formValues) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index e33fe680b..b3d230cb3 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -445,10 +445,23 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Extract metadata to be saved from incoming HTTP header. metadata := extractMetadataFromHeader(r.Header) if rAuthType == authTypeStreamingSigned { - // Make sure to delete the content-encoding parameter - // for a streaming signature which is set to value - // "aws-chunked" - delete(metadata, "content-encoding") + if contentEncoding, ok := metadata["content-encoding"]; ok { + contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) + if contentEncoding != "" { + // Make sure to trim and save the content-encoding + // parameter for a streaming signature which is set + // to a custom value for example: "aws-chunked,gzip". + metadata["content-encoding"] = contentEncoding + } else { + // Trimmed content encoding is empty when the header + // value is set to "aws-chunked" only. + + // Make sure to delete the content-encoding parameter + // for a streaming signature which is set to value + // for example: "aws-chunked" + delete(metadata, "content-encoding") + } + } } // Make sure we hex encode md5sum here. diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index feda6e29b..cc75c5bfa 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -459,6 +459,8 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam shouldPass bool removeAuthHeader bool fault streamFault + // Custom content encoding. + contentEncoding string }{ // Test case - 1. // Fetching the entire object and validating its contents. @@ -543,7 +545,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam expectedRespStatus: http.StatusOK, accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, - shouldPass: false, + shouldPass: true, }, // Test case - 7 // Chunk with malformed encoding. @@ -621,6 +623,21 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam shouldPass: false, fault: tooBigDecodedLength, }, + // Test case - 12 + // Set custom content encoding should succeed and save the encoding properly. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 100 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + contentEncoding: "aws-chunked,gzip", + }, } // Iterating over the cases, fetching the object validating the response. for i, testCase := range testCases { @@ -634,11 +651,16 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey) - } else { + } else if testCase.contentEncoding == "" { req, err = newTestStreamingSignedRequest("PUT", getPutObjectURL("", testCase.bucketName, testCase.objectName), int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey) + } else if testCase.contentEncoding != "" { + req, err = newTestStreamingSignedCustomEncodingRequest("PUT", + getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), + testCase.accessKey, testCase.secretKey, testCase.contentEncoding) } if err != nil { t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i+1, err) @@ -681,7 +703,6 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) continue } - objInfo, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) @@ -689,6 +710,10 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam if objInfo.ContentEncoding == streamingContentEncoding { t.Fatalf("Test %d: %s: ContentEncoding is set to \"aws-chunked\" which is unexpected", i+1, instanceType) } + expectedContentEncoding := trimAwsChunkedContentEncoding(testCase.contentEncoding) + if expectedContentEncoding != objInfo.ContentEncoding { + t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding) + } buffer := new(bytes.Buffer) err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer) if err != nil { diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 782107fc0..b529e58e2 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -828,6 +828,25 @@ func newTestStreamingSignedBadChunkDateRequest(method, urlStr string, contentLen return req, err } +func newTestStreamingSignedCustomEncodingRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey, contentEncoding string) (*http.Request, error) { + req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body) + if err != nil { + return nil, err + } + + // Set custom encoding. + req.Header.Set("content-encoding", contentEncoding) + + currTime := UTCNow() + signature, err := signStreamingRequest(req, accessKey, secretKey, currTime) + if err != nil { + return nil, err + } + + req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime) + return req, err +} + // Returns new HTTP request object signed with streaming signature v4. func newTestStreamingSignedRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) { req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)