tests: Add ListBucketHandler tests. (#2701)

part-3 final fix for #2412

cmd/bucket-handlers_test.go

@@ -300,3 +300,74 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t Tes
 		t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code)
 	}
 }
+
+// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup.
+func TestListBucketsHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testListBuckets)
+}
+
+// testListBucketsHandler - Tests validate listing of buckets.
+func testListBucketsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"ListBuckets"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before initiating NewMultipartUpload.
+	err = obj.MakeBucket(bucketName)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	testCases := []struct {
+		bucketName         string
+		accessKey          string
+		secretKey          string
+		expectedRespStatus int
+	}{
+		// Validate a good case request succeeds.
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Validate a bad case request fails with http.StatusForbidden.
+		{
+			bucketName:         bucketName,
+			accessKey:          "",
+			secretKey:          "",
+			expectedRespStatus: http.StatusForbidden,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		req, lerr := newTestSignedRequest("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		if lerr != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+	}
+}

cmd/erasure-readfile_test.go

@@ -222,7 +222,7 @@ func TestErasureReadUtils(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	objLayer, err := getXLObjectLayer(disks)
+	objLayer, err := getXLObjectLayer(disks, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/event-notifier_test.go

@@ -99,7 +99,7 @@ func TestInitEventNotifier(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for XL backend. ", err)
 	}
-	xl, err := getXLObjectLayer(disks)
+	xl, err := getXLObjectLayer(disks, nil)
 	if err != nil {
 		t.Fatal("Unable to initialize XL backend.", err)
 	}

cmd/format-config-v1_test.go

@@ -268,7 +268,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Create an instance of xl backend.
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -300,7 +300,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Create an instance of xl backend.
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -584,7 +584,7 @@ func TestInitFormatXLErrors(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Create an instance of xl backend.
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -680,7 +680,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 	}
 
 	// Create an instance of xl backend.
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -705,7 +705,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -730,7 +730,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -753,7 +753,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -779,7 +779,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// Everything is fine, should return nil
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -796,7 +796,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// Disks 0..15 are nil
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -815,7 +815,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// One disk returns Faulty Disk
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -836,7 +836,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// One disk is not found, heal corrupted disks should return nil
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -853,7 +853,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// Remove format.json of all disks
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -874,7 +874,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 
 	// Corrupted format json in one disk
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -899,7 +899,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// Everything is fine, should return nil
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -915,7 +915,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// Disks 0..15 are nil
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -934,7 +934,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// One disk returns Faulty Disk
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -955,7 +955,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// One disk is not found, heal corrupted disks should return nil
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -972,7 +972,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// Remove format.json of all disks
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -993,7 +993,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 
 	// Remove format.json of all disks
-	obj, err = getXLObjectLayer(fsDirs)
+	obj, err = getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/post-policy_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
@@ -80,6 +81,8 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 	// remove the root folder after the test ends.
 	defer removeAll(rootPath)
 
+	credentials := serverConfig.GetCredential()
+
 	// bucketnames[0].
 	// objectNames[0].
 	// uploadIds [0].
@@ -96,31 +99,50 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 		objectName         string
 		data               []byte
 		expectedRespStatus int
-		shouldPass         bool
+		accessKey          string
+		secretKey          string
+		malformedBody      bool
 	}{
 		// Success case.
 		{
 			objectName:         "test",
 			data:               []byte("Hello, World"),
 			expectedRespStatus: http.StatusNoContent,
-			shouldPass:         true,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			malformedBody:      false,
 		},
-		// Bad case.
+		// Bad case invalid request.
 		{
 			objectName:         "test",
 			data:               []byte("Hello, World"),
 			expectedRespStatus: http.StatusBadRequest,
-			shouldPass:         false,
+			accessKey:          "",
+			secretKey:          "",
+			malformedBody:      false,
+		},
+		// Bad case malformed input.
+		{
+			objectName:         "test",
+			data:               []byte("Hello, World"),
+			expectedRespStatus: http.StatusBadRequest,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			malformedBody:      true,
 		},
 	}
 
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass)
+		req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey)
 		if perr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr)
 		}
+		if testCase.malformedBody {
+			// Change the request body.
+			req.Body = ioutil.NopCloser(bytes.NewReader([]byte("Hello,")))
+		}
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(rec, req)
@@ -128,7 +150,6 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 	}
-
 }
 
 // postPresignSignatureV4 - presigned signature for PostPolicy requests.
@@ -140,33 +161,29 @@ func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l
 	return signature
 }
 
-func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) {
+func newPostRequest(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
 	// Keep time.
 	t := time.Now().UTC()
 	// Expire the request five minutes from now.
 	expirationTime := t.Add(time.Minute * 5)
 	// Get the user credential.
-	credentials := serverConfig.GetCredential()
-	credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t)
+	credStr := getCredential(accessKey, serverConfig.GetRegion(), t)
 	// Create a new post policy.
 	policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime)
 	// Only need the encoding.
 	encodedPolicy := base64.StdEncoding.EncodeToString(policy)
 
-	formData := make(map[string]string)
-	if shouldPass {
-		// Presign with V4 signature based on the policy.
-		signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion())
-
-		formData = map[string]string{
-			"bucket":           bucketName,
-			"key":              objectName,
-			"x-amz-credential": credStr,
-			"policy":           encodedPolicy,
-			"x-amz-signature":  signature,
-			"x-amz-date":       t.Format(iso8601DateFormat),
-			"x-amz-algorithm":  "AWS4-HMAC-SHA256",
-		}
+	// Presign with V4 signature based on the policy.
+	signature := postPresignSignatureV4(encodedPolicy, t, secretKey, serverConfig.GetRegion())
+
+	formData := map[string]string{
+		"bucket":           bucketName,
+		"key":              objectName,
+		"x-amz-credential": credStr,
+		"policy":           encodedPolicy,
+		"x-amz-signature":  signature,
+		"x-amz-date":       t.Format(iso8601DateFormat),
+		"x-amz-algorithm":  "AWS4-HMAC-SHA256",
 	}
 
 	// Create the multipart form.

cmd/prepare-storage.go

@@ -211,6 +211,7 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error)
 	for index, disk := range disks {
 		// Check if disk is ignored.
 		if disksSet.Contains(disk) {
+			// Set this situation as disk not found.
 			storageDisks[index] = nil
 			continue
 		}
@@ -222,6 +223,6 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error)
 		}
 		storageDisks[index] = storage
 	}
-
+	// Start wait loop retrying formatting disks.
 	return retryFormattingDisks(disks, storageDisks)
 }

cmd/test-utils_test.go

@@ -68,7 +68,7 @@ func prepareXL() (ObjectLayer, []string, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	obj, err := getXLObjectLayer(fsDirs)
+	obj, err := getXLObjectLayer(fsDirs, nil)
 	if err != nil {
 		removeRoots(fsDirs)
 		return nil, nil, err
@@ -498,6 +498,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
 	}
 	sort.Strings(headers)
 
+	region := serverConfig.GetRegion()
+
 	// Get canonical headers.
 	var buf bytes.Buffer
 	for _, k := range headers {
@@ -549,7 +551,7 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
 	// Get scope.
 	scope := strings.Join([]string{
 		currTime.Format(yyyymmdd),
-		"us-east-1",
+		region,
 		"s3",
 		"aws4_request",
 	}, "/")
@@ -559,8 +561,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
 	stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
 
 	date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
-	region := sumHMAC(date, []byte("us-east-1"))
-	service := sumHMAC(region, []byte("s3"))
+	regionHMAC := sumHMAC(date, []byte(region))
+	service := sumHMAC(regionHMAC, []byte("s3"))
 	signingKey := sumHMAC(service, []byte("aws4_request"))
 
 	signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
@@ -731,7 +733,7 @@ func makeTestBackend(disks []string, instanceType string) (ObjectLayer, error) {
 		return objLayer, err
 
 	case "XL":
-		objectLayer, err := getXLObjectLayer(disks)
+		objectLayer, err := getXLObjectLayer(disks, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -1107,12 +1109,12 @@ func getRandomDisks(N int) ([]string, error) {
 }
 
 // getXLObjectLayer - Instantiates XL object layer and returns it.
-func getXLObjectLayer(erasureDisks []string) (ObjectLayer, error) {
-	err := formatDisks(erasureDisks, nil)
+func getXLObjectLayer(erasureDisks []string, ignoredDisks []string) (ObjectLayer, error) {
+	err := formatDisks(erasureDisks, ignoredDisks)
 	if err != nil {
 		return nil, err
 	}
-	objLayer, err := newXLObjects(erasureDisks, nil)
+	objLayer, err := newXLObjects(erasureDisks, ignoredDisks)
 	if err != nil {
 		return nil, err
 	}
@@ -1260,7 +1262,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
 	if err != nil {
 		t.Fatalf("Initialization of disks for XL setup: %s", err)
 	}
-	objLayer, err := getXLObjectLayer(erasureDisks)
+	objLayer, err := getXLObjectLayer(erasureDisks, nil)
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
 	}
@@ -1288,6 +1290,9 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 	// Iterate the list of API functions requested for and register them in mux HTTP handler.
 	for _, apiFunction := range apiFunctions {
 		switch apiFunction {
+		// Register ListBuckets	handler.
+		case "ListBuckets":
+			apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
 		// Register GetObject handler.
 		case "GetObject`":
 			bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)

cmd/xl-v1.go

@@ -200,6 +200,10 @@ func (d byDiskTotal) Less(i, j int) bool {
 func (xl xlObjects) StorageInfo() StorageInfo {
 	var disksInfo []disk.Info
 	for _, storageDisk := range xl.storageDisks {
+		if storageDisk == nil {
+			// Storage disk is empty, perhaps ignored disk or not available.
+			continue
+		}
 		info, err := storageDisk.DiskInfo()
 		if err != nil {
 			errorIf(err, "Unable to fetch disk info for %#v", storageDisk)

cmd/xl-v1_test.go

@@ -115,6 +115,25 @@ func TestStorageInfo(t *testing.T) {
 	if disks16Info.Total <= 0 {
 		t.Fatalf("Diskinfo total values should be greater 0")
 	}
+
+	objLayer, err = newXLObjects(fsDirs, fsDirs[:4])
+	if err != nil {
+		t.Fatalf("Unable to initialize 'XL' object layer with ignored disks %s.", fsDirs[:4])
+	}
+
+	// Get storage info first attempt.
+	disks16Info = objLayer.StorageInfo()
+
+	// This test assumes homogenity between all disks,
+	// i.e if we loose one disk the effective storage
+	// usage values is assumed to decrease. If we have
+	// heterogenous environment this is not true all the time.
+	if disks16Info.Free <= 0 {
+		t.Fatalf("Diskinfo total free values should be greater 0")
+	}
+	if disks16Info.Total <= 0 {
+		t.Fatalf("Diskinfo total values should be greater 0")
+	}
 }
 
 // TestNewXL - tests initialization of all input disks