API: add writePartTooSmallErrorResponse to extend standard error responses. (#2005)

This function is added to extend the standard error responses.
Which is needed in some cases for example CompleteMultipartUpload
should respond with ErrPartTooSmall error when parts uploaded are
lesser than 5MB (i.e minimum allowed size per part).

Fixes #1536

api-response-multipart.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// File carries any specific responses constructed/necessary in
+// multipart operations.
+package main
+
+import "net/http"
+
+// writeErrorResponsePartTooSmall - function is used specifically to
+// construct a proper error response during CompleteMultipartUpload
+// when one of the parts is < 5MB.
+// The requirement comes due to the fact that generic ErrorResponse
+// XML doesn't carry the additional fields required to send this
+// error. So we construct a new type which lies well within the scope
+// of this function.
+func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
+	// Represents additional fields necessary for ErrPartTooSmall S3 error.
+	type completeMultipartAPIError struct {
+		// Proposed size represents uploaded size of the part.
+		ProposedSize int64
+		// Minimum size allowed epresents the minimum size allowed per
+		// part. Defaults to 5MB.
+		MinSizeAllowed int64
+		// Part number of the part which is incorrect.
+		PartNumber int
+		// ETag of the part which is incorrect.
+		PartETag string
+		// Other default XML error responses.
+		APIErrorResponse
+	}
+	// Generate complete multipart error response.
+	errorResponse := getAPIErrorResponse(getAPIError(toAPIErrorCode(err)), r.URL.Path)
+	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
+	encodedErrorResponse := encodeResponse(cmpErrResp)
+	// Write error body
+	w.Write(encodedErrorResponse)
+	w.(http.Flusher).Flush()
+}
+
+// Add any other multipart specific responses here.

api-response.go

@@ -506,7 +506,7 @@ func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIE
 }
 
 func writeErrorResponseNoHeader(w http.ResponseWriter, req *http.Request, error APIError, resource string) {
-	// generate error response
+	// Generate error response.
 	errorResponse := getAPIErrorResponse(error, resource)
 	encodedErrorResponse := encodeResponse(errorResponse)
 	// HEAD should have no body, do not attempt to write to it

fs-v1-multipart.go

@@ -504,7 +504,11 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		}
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-			return "", PartTooSmall{}
+			return "", PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   fsMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			}
 		}
 		// Construct part suffix.
 		partSuffix := fmt.Sprintf("object%d", part.PartNumber)

object-api-multipart_test.go

@@ -1841,7 +1841,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t *
 		// Test case with non existent object name (Test number 14).
 		{bucketNames[0], "my-object", uploadIDs[0], []completePart{{ETag: "abcd", PartNumber: 1}}, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
 		// Testing for Part being too small (Test number 15).
-		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[1].parts, "", PartTooSmall{}, false},
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[1].parts, "", PartTooSmall{PartNumber: 1}, false},
 		// TestCase with invalid Part Number (Test number 16).
 		// Should error with Invalid Part .
 		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[2].parts, "", InvalidPart{}, false},

object-errors.go

@@ -251,8 +251,12 @@ func (e InvalidPartOrder) Error() string {
 }
 
 // PartTooSmall - error if part size is less than 5MB.
-type PartTooSmall struct{}
+type PartTooSmall struct {
+	PartSize   int64
+	PartNumber int
+	PartETag   string
+}
 
 func (e PartTooSmall) Error() string {
-	return "Part size should be atleast 5MB"
+	return fmt.Sprintf("Part size for %d should be atleast 5MB", e.PartNumber)
 }

object-handlers.go

@@ -986,7 +986,14 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	if err != nil {
 		errorIf(err, "Unable to complete multipart upload.")
+		switch oErr := err.(type) {
+		case PartTooSmall:
+			// Write part too small error.
+			writePartSmallErrorResponse(w, r, oErr)
+		default:
+			// Handle all other generic issues.
 			writeErrorResponseNoHeader(w, r, getAPIError(toAPIErrorCode(err)), r.URL.Path)
+		}
 		return
 	}
 

server_xl_test.go

@@ -2002,6 +2002,114 @@ func (s *MyAPIXLSuite) TestObjectMultipartListError(c *C) {
 	verifyError(c, response4, "InvalidArgument", "Argument maxParts must be an integer between 1 and 10000.", http.StatusBadRequest)
 }
 
+// TestMultipartErrorEntityTooSmall - initiates a new multipart upload,
+// uploads 2 parts of size less than 5MB, upon complete multipart upload
+// validates EntityTooSmall error returned by the operation.
+func (s *MyAPIXLSuite) TestMultipartErrorEntityTooSmall(c *C) {
+	// generate a random bucket name.
+	bucketName := getRandomBucketName()
+	// HTTP request to create the bucket.
+	request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
+		0, nil, s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+
+	client := http.Client{}
+	// execute the HTTP request to create bucket.
+	response, err := client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response.StatusCode, Equals, 200)
+
+	objectName := "test-multipart-object"
+	// construct HTTP request to initiate a NewMultipart upload.
+	request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName),
+		0, nil, s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+
+	client = http.Client{}
+	// execute the HTTP request initiating the new multipart upload.
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	// expecting the response status code to be http.StatusOK(200 OK).
+	c.Assert(response.StatusCode, Equals, http.StatusOK)
+	// parse the response body and obtain the new upload ID.
+	decoder := xml.NewDecoder(response.Body)
+	newResponse := &InitiateMultipartUploadResponse{}
+
+	err = decoder.Decode(newResponse)
+	c.Assert(err, IsNil)
+	c.Assert(len(newResponse.UploadID) > 0, Equals, true)
+	// uploadID to be used for rest of the multipart operations on the object.
+	uploadID := newResponse.UploadID
+
+	// content for the part to be uploaded.
+	// Create a byte array of 4MB.
+	data := bytes.Repeat([]byte("0123456789abcdef"), 4*1024*1024/16)
+	// calculate md5Sum of the data.
+	hasher := md5.New()
+	hasher.Write(data)
+	md5Sum := hasher.Sum(nil)
+
+	buffer1 := bytes.NewReader(data)
+	// HTTP request for the part to be uploaded.
+	request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"),
+		int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey)
+	// set the Content-Md5 header to the base64 encoding the md5Sum of the content.
+	request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
+	c.Assert(err, IsNil)
+
+	client = http.Client{}
+	// execute the HTTP request to upload the first part.
+	response1, err := client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response1.StatusCode, Equals, http.StatusOK)
+
+	// content for the second part to be uploaded will be same as first part.
+	hasher = md5.New()
+	hasher.Write(data)
+	// calculate md5Sum of the data.
+	md5Sum = hasher.Sum(nil)
+
+	buffer2 := bytes.NewReader(data)
+	// HTTP request for the second part to be uploaded.
+	request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"),
+		int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey)
+	// set the Content-Md5 header to the base64 encoding the md5Sum of the content.
+	request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
+	c.Assert(err, IsNil)
+
+	client = http.Client{}
+	// execute the HTTP request to upload the second part.
+	response2, err := client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response2.StatusCode, Equals, http.StatusOK)
+
+	// Complete multipart upload
+	completeUploads := &completeMultipartUpload{
+		Parts: []completePart{
+			{
+				PartNumber: 1,
+				ETag:       response1.Header.Get("ETag"),
+			},
+			{
+				PartNumber: 2,
+				ETag:       response2.Header.Get("ETag"),
+			},
+		},
+	}
+
+	completeBytes, err := xml.Marshal(completeUploads)
+	c.Assert(err, IsNil)
+	// Indicating that all parts are uploaded and initiating completeMultipartUpload.
+	request, err = newTestRequest("POST", getCompleteMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), int64(len(completeBytes)), bytes.NewReader(completeBytes), s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+
+	// Execute the complete multipart request.
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	// verify whether complete multipart was successfull.
+	verifyError(c, response, "EntityTooSmall", "Your proposed upload is smaller than the minimum allowed object size.", http.StatusOK)
+}
+
 // TestObjectMultipart - Initiates a NewMultipart upload, uploads 2 parts,
 // completes the multipart upload and validates the status of the operation.
 func (s *MyAPIXLSuite) TestObjectMultipart(c *C) {

xl-v1-multipart.go

@@ -594,7 +594,11 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload
 
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentXLMeta.Parts[partIdx].Size) {
-			return "", PartTooSmall{}
+			return "", PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   currentXLMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			}
 		}
 
 		// Last part could have been uploaded as 0bytes, do not need