PutObjectPartHandler unit-tests (#2810)

cmd/object-handlers_test.go

@@ -21,14 +21,28 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"encoding/xml"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strconv"
 	"sync"
 	"testing"
 )
 
+// Type to capture different modifications to API request to simulate failure cases.
+type Fault int
+
+const (
+	None Fault = iota
+	MissingContentLength
+	TooBigObject
+	TooBigDecodedLength
+	BadSignature
+	BadMD5
+)
+
 // Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup.
 func TestAPIGetOjectHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testAPIGetOjectHandler, []string{"GetObject"})
@@ -945,3 +959,343 @@ func testAPIDeleteOjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		}
 	}
 }
+
+func testAPIPutObjectHandlerV2(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t TestErrHandler) {
+	testObject := "testobject"
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
+			instanceType, bucketName, testObject, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+
+	// Get uploadID of the mulitpart upload initiated.
+	var mpartResp InitiateMultipartUploadResponse
+	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
+	if err != nil {
+		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
+
+	}
+	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
+	if err != nil {
+		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
+	}
+
+	rec = httptest.NewRecorder()
+	req, err = newTestSignedRequestV2("PUT",
+		getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
+		int64(len("hello")), bytes.NewReader([]byte("hello")), credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
+			instanceType, bucketName, testObject, err)
+	}
+	signatureMismatchErr := getAPIError(ErrSignatureDoesNotMatch)
+	// Reset date field in header to make signature V2 fail.
+	req.Header.Set("x-amz-date", "")
+	apiRouter.ServeHTTP(rec, req)
+	errBytes, err := ioutil.ReadAll(rec.Result().Body)
+	if err != nil {
+		t.Fatalf("Test %d %s Failed to read error response from upload part request %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, err)
+	}
+	var errXML APIErrorResponse
+	err = xml.Unmarshal(errBytes, &errXML)
+	if err != nil {
+		t.Fatalf("Test %d %s Failed to unmarshal error response from upload part request %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, err)
+	}
+	if errXML.Code != signatureMismatchErr.Code {
+		t.Errorf("Test %d %s expected to fail with error %s, but received %s", 1, instanceType,
+			signatureMismatchErr.Code, errXML.Code)
+	}
+}
+
+func TestAPIPutObjectHandlerV2(t *testing.T) {
+	ExecObjectLayerAPITest(t, testAPIPutObjectHandlerV2, []string{"NewMultipart", "PutObjectPart"})
+}
+
+func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t TestErrHandler) {
+	testObject := "testobject"
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
+			instanceType, bucketName, testObject, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+
+	// Get uploadID of the mulitpart upload initiated.
+	var mpartResp InitiateMultipartUploadResponse
+	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
+	if err != nil {
+		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
+
+	}
+	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
+	if err != nil {
+		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
+	}
+
+	noAPIErr := APIError{}
+	missingDateHeaderErr := getAPIError(ErrMissingDateHeader)
+	internalErr := getAPIError(ErrInternalError)
+	testCases := []struct {
+		fault       Fault
+		expectedErr APIError
+	}{
+		{BadSignature, missingDateHeaderErr},
+		{None, noAPIErr},
+		{TooBigDecodedLength, internalErr},
+	}
+
+	for i, test := range testCases {
+		rec = httptest.NewRecorder()
+		req, err = newTestStreamingSignedRequest("PUT",
+			getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
+			5, 1, bytes.NewReader([]byte("hello")), credentials.AccessKeyID, credentials.SecretAccessKey)
+
+		switch test.fault {
+		case BadSignature:
+			// Reset date field in header to make streaming signature fail.
+			req.Header.Set("x-amz-date", "")
+		case TooBigDecodedLength:
+			// Set decoded length to a large value out of int64 range to simulate parse failure.
+			req.Header.Set("x-amz-decoded-content-length", "9999999999999999999999")
+		}
+		apiRouter.ServeHTTP(rec, req)
+
+		if test.expectedErr != noAPIErr {
+			errBytes, err := ioutil.ReadAll(rec.Result().Body)
+			if err != nil {
+				t.Fatalf("Test %d %s Failed to read error response from upload part request %s/%s: <ERROR> %v",
+					i+1, instanceType, bucketName, testObject, err)
+			}
+			var errXML APIErrorResponse
+			err = xml.Unmarshal(errBytes, &errXML)
+			if err != nil {
+				t.Fatalf("Test %d %s Failed to unmarshal error response from upload part request %s/%s: <ERROR> %v",
+					i+1, instanceType, bucketName, testObject, err)
+			}
+			if test.expectedErr.Code != errXML.Code {
+				t.Errorf("Test %d %s expected to fail with error %s, but received %s", i+1, instanceType,
+					test.expectedErr.Code, errXML.Code)
+			}
+		} else {
+			if rec.Code != http.StatusOK {
+				t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d",
+					i+1, instanceType, rec.Code)
+
+			}
+		}
+	}
+}
+
+func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {
+	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
+}
+
+func testAPIPutObjectPartHandlerAnon(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t TestErrHandler) {
+	// Initialize bucket policies for anonymous request test
+	err := initBucketPolicies(obj)
+	if err != nil {
+		t.Fatalf("Failed to initialize bucket policies: <ERROR> %v", err)
+	}
+
+	testObject := "testobject"
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
+			instanceType, bucketName, testObject, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+
+	// Get uploadID of the mulitpart upload initiated.
+	var mpartResp InitiateMultipartUploadResponse
+	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
+	if err != nil {
+		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
+	}
+	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
+	if err != nil {
+		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
+
+	}
+
+	accessDeniedErr := getAPIError(ErrAccessDenied)
+	anonRec := httptest.NewRecorder()
+	anonReq, aErr := newTestRequest("PUT",
+		getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
+		int64(len("hello")), bytes.NewReader([]byte("hello")))
+	if aErr != nil {
+		t.Fatalf("Test %d %s Failed to create a signed request to upload part for %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, aErr)
+	}
+	apiRouter.ServeHTTP(anonRec, anonReq)
+
+	anonErrBytes, err := ioutil.ReadAll(anonRec.Result().Body)
+	if err != nil {
+		t.Fatalf("Test %d %s Failed to read error response from upload part request %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, err)
+	}
+	var anonErrXML APIErrorResponse
+	err = xml.Unmarshal(anonErrBytes, &anonErrXML)
+	if err != nil {
+		t.Fatalf("Test %d %s Failed to unmarshal error response from upload part request %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, err)
+	}
+	if accessDeniedErr.Code != anonErrXML.Code {
+		t.Errorf("Test %d %s expected to fail with error %s, but received %s", 1, instanceType,
+			accessDeniedErr.Code, anonErrXML.Code)
+	}
+
+	// Set write only policy on bucket to allow anonymous PutObjectPart API
+	// request to go through.
+	writeOnlyPolicy := bucketPolicy{
+		Version:    "1.0",
+		Statements: []policyStatement{getWriteOnlyObjectStatement(bucketName, "")},
+	}
+	globalBucketPolicies.SetBucketPolicy(bucketName, &writeOnlyPolicy)
+
+	anonRec = httptest.NewRecorder()
+	anonReq, aErr = newTestRequest("PUT",
+		getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
+		int64(len("hello")), bytes.NewReader([]byte("hello")))
+	if aErr != nil {
+		t.Fatalf("Test %d %s Failed to create a signed request to upload part for %s/%s: <ERROR> %v",
+			1, instanceType, bucketName, testObject, aErr)
+	}
+	apiRouter.ServeHTTP(anonRec, anonReq)
+	if anonRec.Code != http.StatusOK {
+		t.Errorf("Test %d %s expected PutObject Part with authAnonymous type to succeed but failed with "+
+			"HTTP status code %d", 1, instanceType, anonRec.Code)
+	}
+}
+
+func TestAPIPutObjectPartHandlerAnon(t *testing.T) {
+	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerAnon, []string{"PutObjectPart", "NewMultipart"})
+}
+
+func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t TestErrHandler) {
+	// Initiate Multipart upload for testing PutObjectPartHandler.
+	testObject := "testobject"
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
+			instanceType, bucketName, testObject, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+
+	// Get uploadID of the mulitpart upload initiated.
+	var mpartResp InitiateMultipartUploadResponse
+	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
+	if err != nil {
+		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
+
+	}
+	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
+	if err != nil {
+		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
+	}
+
+	NoAPIErr := APIError{}
+	MissingContent := getAPIError(ErrMissingContentLength)
+	EntityTooLarge := getAPIError(ErrEntityTooLarge)
+	BadSigning := getAPIError(ErrContentSHA256Mismatch)
+	BadChecksum := getAPIError(ErrInvalidDigest)
+	InvalidPart := getAPIError(ErrInvalidPart)
+	InvalidMaxParts := getAPIError(ErrInvalidMaxParts)
+	// SignatureMismatch for various signing types
+	testCases := []struct {
+		objectName       string
+		reader           io.ReadSeeker
+		partNumber       string
+		fault            Fault
+		expectedAPIError APIError
+	}{
+		// Success case
+		{testObject, bytes.NewReader([]byte("hello")), "1", None, NoAPIErr},
+		{testObject, bytes.NewReader([]byte("hello")), "9999999999999999999", None, InvalidPart},
+		{testObject, bytes.NewReader([]byte("hello")), strconv.Itoa(maxPartID + 1), None, InvalidMaxParts},
+		{testObject, bytes.NewReader([]byte("hello")), "1", MissingContentLength, MissingContent},
+		{testObject, bytes.NewReader([]byte("hello")), "1", TooBigObject, EntityTooLarge},
+		{testObject, bytes.NewReader([]byte("hello")), "1", BadSignature, BadSigning},
+		{testObject, bytes.NewReader([]byte("hello")), "1", BadMD5, BadChecksum},
+	}
+
+	for i, test := range testCases {
+		tRec := httptest.NewRecorder()
+		tReq, tErr := newTestSignedRequestV4("PUT",
+			getPutObjectPartURL("", bucketName, test.objectName, mpartResp.UploadID, test.partNumber),
+			0, test.reader, credentials.AccessKeyID, credentials.SecretAccessKey)
+		if tErr != nil {
+			t.Fatalf("Test %d %s Failed to create a signed request to upload part for %s/%s: <ERROR> %v", i+1, instanceType,
+				bucketName, test.objectName, tErr)
+		}
+		switch test.fault {
+		case MissingContentLength:
+			tReq.ContentLength = -1
+		case TooBigObject:
+			tReq.ContentLength = maxObjectSize + 1
+		case BadSignature:
+			tReq.Header.Set("x-amz-content-sha256", "somethingElse")
+		case BadMD5:
+			tReq.Header.Set("Content-MD5", "badmd5")
+		}
+		apiRouter.ServeHTTP(tRec, tReq)
+		if test.expectedAPIError != NoAPIErr {
+			errBytes, err := ioutil.ReadAll(tRec.Result().Body)
+			if err != nil {
+				t.Fatalf("Test %d %s Failed to read error response from upload part request %s/%s: <ERROR> %v",
+					i+1, instanceType, bucketName, test.objectName, err)
+			}
+			var errXML APIErrorResponse
+			err = xml.Unmarshal(errBytes, &errXML)
+			if err != nil {
+				t.Fatalf("Test %d %s Failed to unmarshal error response from upload part request %s/%s: <ERROR> %v",
+					i+1, instanceType, bucketName, test.objectName, err)
+			}
+			if test.expectedAPIError.Code != errXML.Code {
+				t.Errorf("Test %d %s expected to fail with error %s, but received %s", i+1, instanceType,
+					test.expectedAPIError.Code, errXML.Code)
+			}
+		}
+	}
+}
+
+func TestAPIPutObjectPartHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart", "NewMultipart"})
+}
+
+func TestPutObjectPartNilObjAPI(t *testing.T) {
+	configDir, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Failed to create a test config: %v", err)
+	}
+	defer removeAll(configDir)
+
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("PUT",
+		getPutObjectPartURL("", "testbucket", "testobject", "uploadId1", "1"),
+		-1, bytes.NewReader([]byte("hello")), "abcd1", "abcd123")
+	if err != nil {
+		t.Fatal("Failed to create a signed UploadPart request.")
+	}
+	// Setup the 'nil' objectAPI router.
+	nilAPIRouter := initTestNilObjAPIEndPoints([]string{"PutObjectPart"})
+	nilAPIRouter.ServeHTTP(rec, req)
+	serverNotInitializedErr := getAPIError(ErrServerNotInitialized).HTTPStatusCode
+	if rec.Code != serverNotInitializedErr {
+		t.Errorf("Test expected to fail with %d, but failed with %d", serverNotInitializedErr, rec.Code)
+	}
+}

cmd/test-utils_test.go

@@ -972,6 +972,13 @@ func getPutObjectURL(endPoint, bucketName, objectName string) string {
 	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
 }
 
+func getPutObjectPartURL(endPoint, bucketName, objectName, uploadID, partNumber string) string {
+	queryValues := url.Values{}
+	queryValues.Set("uploadId", uploadID)
+	queryValues.Set("partNumber", partNumber)
+	return makeTestTargetURL(endPoint, bucketName, objectName, queryValues)
+}
+
 // return URL for fetching object from the bucket.
 func getGetObjectURL(endPoint, bucketName, objectName string) string {
 	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
@@ -1345,29 +1352,9 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
 	defer removeRoots(erasureDisks)
 }
 
-// Takes in XL/FS object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler.
+// addAPIFunc helper function to add API functions identified by name to the routers.
-// Need isolated registration of API end points while writing unit tests for end points.
+func addAPIFunc(muxRouter *router.Router, apiRouter *router.Router, bucket *router.Router,
-// All the API end points are registered only for the default case.
+	api objectAPIHandlers, apiFunction string) {
-func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler {
-	// initialize a new mux router.
-	// goriilla/mux is the library used to register all the routes and handle them.
-	muxRouter := router.NewRouter()
-	// All object storage operations are registered as HTTP handlers on `objectAPIHandlers`.
-	// When the handlers get a HTTP request they use the underlyting ObjectLayer to perform operations.
-	objLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	objLayerMutex.Unlock()
-
-	api := objectAPIHandlers{
-		ObjectAPI: newObjectLayerFn,
-	}
-
-	// API Router.
-	apiRouter := muxRouter.NewRoute().PathPrefix("/").Subrouter()
-	// Bucket router.
-	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
-	// Iterate the list of API functions requested for and register them in mux HTTP handler.
-	for _, apiFunction := range apiFunctions {
 	switch apiFunction {
 	// Register ListBuckets	handler.
 	case "ListBuckets":
@@ -1403,6 +1390,9 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 	case "NewMultipart":
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
 
+	// Register PutObjectPart handler.
+	case "PutObjectPart":
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 	// Register ListMultipartUploads handler.
 	case "ListMultipartUploads":
 		bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
@@ -1425,6 +1415,58 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 		break
 	}
 }
+
+// Returns a http.Handler capable of routing API requests to handlers corresponding to apiFunctions,
+// with ObjectAPI set to nil.
+func initTestNilObjAPIEndPoints(apiFunctions []string) http.Handler {
+	muxRouter := router.NewRouter()
+	// All object storage operations are registered as HTTP handlers on `objectAPIHandlers`.
+	// When the handlers get a HTTP request they use the underlyting ObjectLayer to perform operations.
+	nilAPI := objectAPIHandlers{
+		ObjectAPI: func() ObjectLayer {
+			objLayerMutex.Lock()
+			defer objLayerMutex.Unlock()
+			globalObjectAPI = nil
+			return globalObjectAPI
+		},
+	}
+
+	// API Router.
+	apiRouter := muxRouter.NewRoute().PathPrefix("/").Subrouter()
+	// Bucket router.
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+	// Iterate the list of API functions requested for and register them in mux HTTP handler.
+	for _, apiFunction := range apiFunctions {
+		addAPIFunc(muxRouter, apiRouter, bucket, nilAPI, apiFunction)
+	}
+	return muxRouter
+}
+
+// Takes in XL/FS object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler.
+// Need isolated registration of API end points while writing unit tests for end points.
+// All the API end points are registered only for the default case.
+func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler {
+	// initialize a new mux router.
+	// goriilla/mux is the library used to register all the routes and handle them.
+	muxRouter := router.NewRouter()
+	// All object storage operations are registered as HTTP handlers on `objectAPIHandlers`.
+	// When the handlers get a HTTP request they use the underlyting ObjectLayer to perform operations.
+	objLayerMutex.Lock()
+	globalObjectAPI = objLayer
+	objLayerMutex.Unlock()
+
+	api := objectAPIHandlers{
+		ObjectAPI: newObjectLayerFn,
+	}
+
+	// API Router.
+	apiRouter := muxRouter.NewRoute().PathPrefix("/").Subrouter()
+	// Bucket router.
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+	// Iterate the list of API functions requested for and register them in mux HTTP handler.
+	for _, apiFunction := range apiFunctions {
+		addAPIFunc(muxRouter, apiRouter, bucket, api, apiFunction)
+	}
 	return muxRouter
 }