tests: Add tests for ListMultipartUploads, DeleteMultipleObjects. (#2649)

Additionally adds PostPolicyHandler tests.

cmd/bucket-handlers_test.go

@@ -93,10 +93,10 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType string, t TestEr
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		// construct HTTP request for PUT bucket policy endpoint.
+		// construct HTTP request for Get bucket location.
 		req, err := newTestSignedRequest("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
 		if err != nil {
-			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
 		// Call the ServeHTTP to execute the handler.
@@ -186,10 +186,10 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		// construct HTTP request for PUT bucket policy endpoint.
+		// construct HTTP request for HEAD bucket.
 		req, err := newTestSignedRequest("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
 		if err != nil {
-			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
 		// Call the ServeHTTP to execute the handler.
@@ -199,3 +199,104 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 		}
 	}
 }
+
+// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup.
+func TestListMultipartUploadsHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testListMultipartUploads)
+}
+
+// testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
+func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	initBucketPolicies(obj)
+
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"ListMultipartUploads"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before initiating NewMultipartUpload.
+	err = obj.MakeBucket(bucketName)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
+	// and success responses.
+	testCases := []struct {
+		// Inputs to ListMultipartUploads.
+		bucket             string
+		prefix             string
+		keyMarker          string
+		uploadIDMarker     string
+		delimiter          string
+		maxUploads         string
+		expectedRespStatus int
+		shouldPass         bool
+	}{
+		// 1 - invalid bucket name.
+		{".test", "", "", "", "", "0", http.StatusBadRequest, false},
+		// 2 - bucket not found.
+		{"volatile-bucket-1", "", "", "", "", "0", http.StatusNotFound, false},
+		// 3 - invalid delimiter.
+		{bucketName, "", "", "", "-", "0", http.StatusBadRequest, false},
+		// 4 - invalid prefix and marker combination.
+		{bucketName, "asia", "europe-object", "", "", "0", http.StatusNotImplemented, false},
+		// 5 - invalid upload id and marker combination.
+		{bucketName, "asia", "asia/europe/", "abc", "", "0", http.StatusBadRequest, false},
+		// 6 - invalid max upload id.
+		{bucketName, "", "", "", "", "-1", http.StatusBadRequest, false},
+		// 7 - good case delimiter.
+		{bucketName, "", "", "", "/", "100", http.StatusOK, true},
+		// 8 - good case without delimiter.
+		{bucketName, "", "", "", "", "100", http.StatusOK, true},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+
+		// construct HTTP request for List multipart uploads endpoint.
+		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
+		req, gerr := newTestSignedRequest("GET", u, 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+		if gerr != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+	}
+
+	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+	rec := httptest.NewRecorder()
+
+	// construct HTTP request for List multipart uploads endpoint.
+	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
+	req, err := newTestSignedRequest("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	if err != nil {
+		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
+	}
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(rec, req)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code)
+	}
+}

cmd/fs-v1_test.go

@@ -93,7 +93,7 @@ func TestFSShutdown(t *testing.T) {
 	for i := 1; i <= 5; i++ {
 		naughty := newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil)
 		fs.storage = naughty
-		if err := fs.Shutdown(); err != errFaultyDisk {
+		if err := fs.Shutdown(); errorCause(err) != errFaultyDisk {
 			t.Fatal(i, ", Got unexpected fs shutdown error: ", err)
 		}
 	}

cmd/post-policy_test.go

@@ -0,0 +1,201 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+const (
+	expirationDateFormat = "2006-01-02T15:04:05.999Z"
+	iso8601DateFormat    = "20060102T150405Z"
+)
+
+// newPostPolicyBytes - creates a bare bones postpolicy string with key and bucket matches.
+func newPostPolicyBytes(credential, bucketName, objectKey string, expiration time.Time) []byte {
+	t := time.Now().UTC()
+	// Add the expiration date.
+	expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat))
+	// Add the bucket condition, only accept buckets equal to the one passed.
+	bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
+	// Add the key condition, only accept keys equal to the one passed.
+	keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s"]`, objectKey)
+	// Add the algorithm condition, only accept AWS SignV4 Sha256.
+	algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
+	// Add the date condition, only accept the current date.
+	dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
+	// Add the credential string, only accept the credential passed.
+	credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
+
+	// Combine all conditions into one string.
+	conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr)
+	retStr := "{"
+	retStr = retStr + expirationStr + ","
+	retStr = retStr + conditionStr
+	retStr = retStr + "}"
+
+	return []byte(retStr)
+}
+
+// Wrapper for calling TestPostPolicyHandlerHandler tests for both XL multiple disks and single node setup.
+func TestPostPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testPostPolicyHandler)
+}
+
+// testPostPolicyHandler - Tests validate post policy handler uploading objects.
+func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"})
+
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before initiating NewMultipartUpload.
+	err = obj.MakeBucket(bucketName)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
+	// and success responses.
+	testCases := []struct {
+		objectName         string
+		data               []byte
+		expectedRespStatus int
+		shouldPass         bool
+	}{
+		// Success case.
+		{
+			objectName:         "test",
+			data:               []byte("Hello, World"),
+			expectedRespStatus: http.StatusNoContent,
+			shouldPass:         true,
+		},
+		// Bad case.
+		{
+			objectName:         "test",
+			data:               []byte("Hello, World"),
+			expectedRespStatus: http.StatusBadRequest,
+			shouldPass:         false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass)
+		if perr != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+	}
+
+}
+
+// postPresignSignatureV4 - presigned signature for PostPolicy requests.
+func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
+	// Get signining key.
+	signingkey := getSigningKey(secretAccessKey, t, location)
+	// Calculate signature.
+	signature := getSignature(signingkey, policyBase64)
+	return signature
+}
+
+func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) {
+	// Keep time.
+	t := time.Now().UTC()
+	// Expire the request five minutes from now.
+	expirationTime := t.Add(time.Minute * 5)
+	// Get the user credential.
+	credentials := serverConfig.GetCredential()
+	credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t)
+	// Create a new post policy.
+	policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime)
+	// Only need the encoding.
+	encodedPolicy := base64.StdEncoding.EncodeToString(policy)
+
+	formData := make(map[string]string)
+	if shouldPass {
+		// Presign with V4 signature based on the policy.
+		signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion())
+
+		formData = map[string]string{
+			"bucket":           bucketName,
+			"key":              objectName,
+			"x-amz-credential": credStr,
+			"policy":           encodedPolicy,
+			"x-amz-signature":  signature,
+			"x-amz-date":       t.Format(iso8601DateFormat),
+			"x-amz-algorithm":  "AWS4-HMAC-SHA256",
+		}
+	}
+
+	// Create the multipart form.
+	var buf bytes.Buffer
+	w := multipart.NewWriter(&buf)
+
+	// Set the normal formData
+	for k, v := range formData {
+		w.WriteField(k, v)
+	}
+	// Set the File formData
+	writer, err := w.CreateFormFile("file", "s3verify/post/object")
+	if err != nil {
+		// return nil, err
+		return nil, err
+	}
+	writer.Write(objData)
+	// Close before creating the new request.
+	w.Close()
+
+	// Set the body equal to the created policy.
+	reader := bytes.NewReader(buf.Bytes())
+
+	req, err := http.NewRequest("POST", makeTestTargetURL(endPoint, bucketName, objectName, nil), reader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set form content-type.
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	return req, nil
+}

cmd/server_test.go

@@ -309,6 +309,7 @@ func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) {
 
 }
 
+// Test deletes multple objects and verifies server resonse.
 func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
 	// generate a random bucket name.
 	bucketName := getRandomBucketName()
@@ -347,18 +348,11 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
 			ObjectName: objName,
 		})
 	}
-	// Append a non-existent object for which the response should be marked
-	// as deleted.
-	delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{
-		ObjectName: fmt.Sprintf("%d/%s", 10, objectName),
-	})
-
 	// Marshal delete request.
 	deleteReqBytes, err := xml.Marshal(delObjReq)
 	c.Assert(err, IsNil)
 
-	// object name was "prefix/myobject", an attempt to delelte "prefix"
+	// Delete list of objects.
-	// Should not delete "prefix/myobject"
 	request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName),
 		int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey)
 	c.Assert(err, IsNil)
@@ -372,11 +366,31 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
 	c.Assert(err, IsNil)
 	err = xml.Unmarshal(delRespBytes, &deleteResp)
 	c.Assert(err, IsNil)
-	for i := 0; i <= 10; i++ {
+	for i := 0; i < 10; i++ {
 		// All the objects should be under deleted list (including non-existent object)
 		c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i])
 	}
 	c.Assert(len(deleteResp.Errors), Equals, 0)
+
+	// Attempt second time results should be same, NoSuchKey for objects not found
+	// shouldn't be set.
+	request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName),
+		int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+	client = http.Client{}
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response.StatusCode, Equals, http.StatusOK)
+
+	deleteResp = DeleteObjectsResponse{}
+	delRespBytes, err = ioutil.ReadAll(response.Body)
+	c.Assert(err, IsNil)
+	err = xml.Unmarshal(delRespBytes, &deleteResp)
+	c.Assert(err, IsNil)
+	for i := 0; i < 10; i++ {
+		c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i])
+	}
+	c.Assert(len(deleteResp.Errors), Equals, 0)
 }
 
 // Tests delete object responses and success.

cmd/test-utils_test.go

@@ -577,6 +577,11 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
 	return nil
 }
 
+// getCredential generate a credential string.
+func getCredential(accessKeyID, location string, t time.Time) string {
+	return accessKeyID + "/" + getScope(t, location)
+}
+
 // Returns new HTTP request object.
 func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
 	if method == "" {
@@ -1046,14 +1051,26 @@ func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID strin
 	return makeTestTargetURL(endPoint, bucketName, objectName, queryValue)
 }
 
-// return URL for a new multipart upload.
+// return URL for a listing pending multipart uploads.
 func getListMultipartURL(endPoint, bucketName string) string {
 	queryValue := url.Values{}
 	queryValue.Set("uploads", "")
 	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
 }
 
-// return URL for a new multipart upload.
+// return URL for listing pending multipart uploads with parameters.
+func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string {
+	queryValue := url.Values{}
+	queryValue.Set("uploads", "")
+	queryValue.Set("prefix", prefix)
+	queryValue.Set("delimiter", delimiter)
+	queryValue.Set("key-marker", keyMarker)
+	queryValue.Set("upload-id-marker", uploadIDMarker)
+	queryValue.Set("max-uploads", maxUploads)
+	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
+}
+
+// return URL for a listing parts on a given upload id.
 func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts string) string {
 	queryValues := url.Values{}
 	queryValues.Set("uploadId", uploadID)
@@ -1289,8 +1306,12 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 		// Register GetBucketLocation handler.
 		case "GetBucketLocation":
 			bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+		// Register HeadBucket handler.
 		case "HeadBucket":
 			bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+		// Register ListMultipartUploads handler.
+		case "ListMultipartUploads":
+			bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
 		// Register all api endpoints by default.
 		default:
 			registerAPIRouter(muxRouter, api)