sign/streaming: Content-Encoding is not set in newer aws-java-sdks (#3986)

We can't use Content-Encoding to verify if `aws-chunked` is set
or not. Just use 'streaming' signature header instead.

While this is considered mandatory, on the contrary aws-sdk-java
doesn't set this value

http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html

```
Set the value to aws-chunked.
```

We will relax it and behave appropriately. Also this PR supports
saving custom encoding after trimming off the `aws-chunked`
parameter.

Fixes #3983

cmd/auth-handler.go

@@ -64,7 +64,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
 	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
-		r.Header.Get("content-encoding") == streamingContentEncoding &&
 		r.Method == httpPUT
 }
 

cmd/handler-utils.go

@@ -158,6 +158,23 @@ func extractReqParams(r *http.Request) map[string]string {
 	}
 }
 
+// Trims away `aws-chunked` from the content-encoding header if present.
+// Streaming signature clients can have custom content-encoding such as
+// `aws-chunked,gzip` here we need to only save `gzip`.
+// For more refer http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string) {
+	if contentEnc == "" {
+		return contentEnc
+	}
+	var newEncs []string
+	for _, enc := range strings.Split(contentEnc, ",") {
+		if enc != streamingContentEncoding {
+			newEncs = append(newEncs, enc)
+		}
+	}
+	return strings.Join(newEncs, ",")
+}
+
 // extractMetadataFromForm extracts metadata from Post Form.
 func extractMetadataFromForm(formValues http.Header) map[string]string {
 	return extractMetadataFromHeader(formValues)

cmd/object-handlers.go

@@ -445,11 +445,24 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	// Extract metadata to be saved from incoming HTTP header.
 	metadata := extractMetadataFromHeader(r.Header)
 	if rAuthType == authTypeStreamingSigned {
+		if contentEncoding, ok := metadata["content-encoding"]; ok {
+			contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
+			if contentEncoding != "" {
+				// Make sure to trim and save the content-encoding
+				// parameter for a streaming signature which is set
+				// to a custom value for example: "aws-chunked,gzip".
+				metadata["content-encoding"] = contentEncoding
+			} else {
+				// Trimmed content encoding is empty when the header
+				// value is set to "aws-chunked" only.
+
 				// Make sure to delete the content-encoding parameter
 				// for a streaming signature which is set to value
-		// "aws-chunked"
+				// for example: "aws-chunked"
 				delete(metadata, "content-encoding")
 			}
+		}
+	}
 
 	// Make sure we hex encode md5sum here.
 	metadata["md5Sum"] = hex.EncodeToString(md5Bytes)

cmd/object-handlers_test.go

@@ -459,6 +459,8 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 		shouldPass       bool
 		removeAuthHeader bool
 		fault            streamFault
+		// Custom content encoding.
+		contentEncoding string
 	}{
 		// Test case - 1.
 		// Fetching the entire object and validating its contents.
@@ -543,7 +545,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			expectedRespStatus: http.StatusOK,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			shouldPass:         false,
+			shouldPass:         true,
 		},
 		// Test case - 7
 		// Chunk with malformed encoding.
@@ -621,6 +623,21 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			shouldPass:         false,
 			fault:              tooBigDecodedLength,
 		},
+		// Test case - 12
+		// Set custom content encoding should succeed and save the encoding properly.
+		{
+			bucketName:         bucketName,
+			objectName:         objectName,
+			data:               bytesData,
+			dataLen:            len(bytesData),
+			chunkSize:          100 * humanize.KiByte,
+			expectedContent:    []byte{},
+			expectedRespStatus: http.StatusOK,
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			shouldPass:         true,
+			contentEncoding:    "aws-chunked,gzip",
+		},
 	}
 	// Iterating over the cases, fetching the object validating the response.
 	for i, testCase := range testCases {
@@ -634,11 +651,16 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 				int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
 				testCase.accessKey, testCase.secretKey)
 
-		} else {
+		} else if testCase.contentEncoding == "" {
 			req, err = newTestStreamingSignedRequest("PUT",
 				getPutObjectURL("", testCase.bucketName, testCase.objectName),
 				int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
 				testCase.accessKey, testCase.secretKey)
+		} else if testCase.contentEncoding != "" {
+			req, err = newTestStreamingSignedCustomEncodingRequest("PUT",
+				getPutObjectURL("", testCase.bucketName, testCase.objectName),
+				int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data),
+				testCase.accessKey, testCase.secretKey, testCase.contentEncoding)
 		}
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for Put Object: <ERROR> %v", i+1, err)
@@ -681,7 +703,6 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 				t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent))
 				continue
 			}
-
 			objInfo, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
@@ -689,6 +710,10 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			if objInfo.ContentEncoding == streamingContentEncoding {
 				t.Fatalf("Test %d: %s: ContentEncoding is set to \"aws-chunked\" which is unexpected", i+1, instanceType)
 			}
+			expectedContentEncoding := trimAwsChunkedContentEncoding(testCase.contentEncoding)
+			if expectedContentEncoding != objInfo.ContentEncoding {
+				t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding)
+			}
 			buffer := new(bytes.Buffer)
 			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer)
 			if err != nil {

cmd/test-utils_test.go

@@ -828,6 +828,25 @@ func newTestStreamingSignedBadChunkDateRequest(method, urlStr string, contentLen
 	return req, err
 }
 
+func newTestStreamingSignedCustomEncodingRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey, contentEncoding string) (*http.Request, error) {
+	req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set custom encoding.
+	req.Header.Set("content-encoding", contentEncoding)
+
+	currTime := UTCNow()
+	signature, err := signStreamingRequest(req, accessKey, secretKey, currTime)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime)
+	return req, err
+}
+
 // Returns new HTTP request object signed with streaming signature v4.
 func newTestStreamingSignedRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) {
 	req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)