From 0b4bbe6d9eff030d6f09e2e42eb229c848f0b7bb Mon Sep 17 00:00:00 2001 From: karthic rao Date: Sat, 7 May 2016 00:27:04 +0530 Subject: [PATCH] Adding XL Object layer validation for existing unit tests of single node (#1499) object layer. Adding isBucketExist check for GetObjectInfo in the XL layer. --- object-api-getobjectinfo_test.go | 37 ++++++------- object-api-listobjects_test.go | 49 ++++++++--------- object-api-multipart_test.go | 90 ++++++++++++------------------- test-utils.go | 91 ++++++++++++++++++++++++++++++++ xl-objects.go | 6 +++ 5 files changed, 167 insertions(+), 106 deletions(-) create mode 100644 test-utils.go diff --git a/object-api-getobjectinfo_test.go b/object-api-getobjectinfo_test.go index cccf8ebf1..83b1a3263 100644 --- a/object-api-getobjectinfo_test.go +++ b/object-api-getobjectinfo_test.go @@ -27,28 +27,21 @@ import ( "testing" ) -// Testing GetObjectInfo(). +// Wrapper for calling GetObjectInfo tests for both XL muliple disks and single node setup. func TestGetObjectInfo(t *testing.T) { - directory, err := ioutil.TempDir("", "minio-get-objinfo-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(directory) - - // Create the obj. - obj, err := newFSObjects(directory) - if err != nil { - t.Fatal(err) - } + ExecObjectLayerTest(t, testGetObjectInfo) +} +// Testing GetObjectInfo(). +func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) { // This bucket is used for testing getObjectInfo operations. - err = obj.MakeBucket("test-getobjectinfo") + err := obj.MakeBucket("test-getobjectinfo") if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } _, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } resultCases := []ObjectInfo{ // ObjectInfo -1. @@ -88,31 +81,31 @@ func TestGetObjectInfo(t *testing.T) { for i, testCase := range testCases { result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName) if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if testCase.err.Error() != err.Error() { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) } } // Test passes as expected, but the output values are verified for correctness here. if err == nil && testCase.shouldPass { if testCase.result.Bucket != result.Bucket { - t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket) + t.Fatalf("Test %d: %s: Expected Bucket name to be '%s', but found '%s' instead", i+1, instanceType, testCase.result.Bucket, result.Bucket) } if testCase.result.Name != result.Name { - t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name) + t.Errorf("Test %d: %s: Expected Object name to be %s, but instead found it to be %s", i+1, instanceType, testCase.result.Name, result.Name) } if testCase.result.ContentType != result.ContentType { - t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType) + t.Errorf("Test %d: %s: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.ContentType, result.ContentType) } if testCase.result.IsDir != result.IsDir { - t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir) + t.Errorf("Test %d: %s: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, instanceType,testCase.result.IsDir, result.IsDir) } } } diff --git a/object-api-listobjects_test.go b/object-api-listobjects_test.go index 04836bf6b..350d4324f 100644 --- a/object-api-listobjects_test.go +++ b/object-api-listobjects_test.go @@ -26,70 +26,64 @@ import ( "testing" ) +// Wrapper for calling ListObjects tests for both XL muliple disks and single node setup. func TestListObjects(t *testing.T) { - // Make a temporary directory to use as the obj. - directory, err := ioutil.TempDir("", "minio-list-object-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(directory) + ExecObjectLayerTest(t, testListObjects) +} - // Create the obj. - obj, err := newFSObjects(directory) - if err != nil { - t.Fatal(err) - } +// Unit test for ListObjects in general. +func testListObjects(obj ObjectLayer, instanceType string, t *testing.T) { // This bucket is used for testing ListObject operations. - err = obj.MakeBucket("test-bucket-list-object") + err := obj.MakeBucket("test-bucket-list-object") if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } // Will not store any objects in this bucket, // Its to test ListObjects on an empty bucket. err = obj.MakeBucket("empty-bucket") if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } tmpfile, err := ioutil.TempFile("", "simple-file.txt") if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } defer os.Remove(tmpfile.Name()) // clean up _, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } _, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } _, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } for i := 0; i < 2; i++ { key := "newPrefix" + strconv.Itoa(i) _, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } } _, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } for i := 0; i < 3; i++ { key := "obj" + strconv.Itoa(i) _, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } } @@ -532,16 +526,15 @@ func TestListObjects(t *testing.T) { for i, testCase := range testCases { result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys) if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } if err == nil && !testCase.shouldPass { - t.Log(result) - t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if !strings.Contains(err.Error(), testCase.err.Error()) { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) } } // Since there are cases for which ListObjects fails, this is @@ -554,15 +547,15 @@ func TestListObjects(t *testing.T) { // otherwise it may lead to index out of range error in // assertion following this. if len(testCase.result.Objects) != len(result.Objects) { - t.Fatalf("Test %d: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, len(testCase.result.Objects), len(result.Objects)) + t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects)) } for j := 0; j < len(testCase.result.Objects); j++ { if testCase.result.Objects[j].Name != result.Objects[j].Name { - t.Errorf("Test %d: Expected object name to be \"%s\", but found \"%s\" instead", i+1, testCase.result.Objects[j].Name, result.Objects[j].Name) + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) } } if testCase.result.IsTruncated != result.IsTruncated { - t.Errorf("Test %d: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, testCase.result.IsTruncated, result.IsTruncated) + t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated) } } diff --git a/object-api-multipart_test.go b/object-api-multipart_test.go index 708921cc0..4a107d526 100644 --- a/object-api-multipart_test.go +++ b/object-api-multipart_test.go @@ -19,24 +19,16 @@ package main import ( "bytes" "fmt" - "io/ioutil" - "os" "testing" ) -// Tests validate creation of new multipart upload instance. +// Wrapper for calling NewMultipartUpload tests for both XL muliple disks and single node setup. func TestObjectNewMultipartUpload(t *testing.T) { - directory, err := ioutil.TempDir("", "minio-multipart-1-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(directory) + ExecObjectLayerTest(t, testObjectNewMultipartUpload) +} - // Initialize fs object layer. - obj, err := newFSObjects(directory) - if err != nil { - t.Fatal(err) - } +// Tests validate creation of new multipart upload instance. +func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t *testing.T) { bucket := "minio-bucket" object := "minio-object" @@ -45,107 +37,93 @@ func TestObjectNewMultipartUpload(t *testing.T) { // opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist. uploadID, err := obj.NewMultipartUpload(bucket, object) if err == nil { - t.Fatalf("Expected to fail since the NewMultipartUpload is intialized on a non-existant bucket.") + t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existant bucket.", instanceType) } if errMsg != err.Error() { - t.Errorf("Expected to fail with Error \"%s\", but instead found \"%s\".", errMsg, err.Error()) + t.Errorf("%s, Expected to fail with Error \"%s\", but instead found \"%s\".", instanceType, errMsg, err.Error()) } // Create bucket before intiating NewMultipartUpload. err = obj.MakeBucket(bucket) if err != nil { // failed to create newbucket, abort. - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } uploadID, err = obj.NewMultipartUpload(bucket, object) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } err = obj.AbortMultipartUpload(bucket, object, uploadID) if err != nil { switch err.(type) { case InvalidUploadID: - t.Fatalf("New Multipart upload failed to create uuid file.") + t.Fatalf("%s: New Multipart upload failed to create uuid file.", instanceType) default: t.Fatalf(err.Error()) } } } -// Tests validates the validator for existence of uploadID. +// Wrapper for calling isUploadIDExists tests for both XL muliple disks and single node setup. func TestObjectAPIIsUploadIDExists(t *testing.T) { - directory, err := ioutil.TempDir("", "minio-multipart-2-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(directory) - - // Initialize fs object layer. - obj, err := newFSObjects(directory) - if err != nil { - t.Fatal(err) - } + ExecObjectLayerTest(t, testObjectAPIIsUploadIDExists) +} +// Tests validates the validator for existence of uploadID. +func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t *testing.T) { bucket := "minio-bucket" object := "minio-object" // Create bucket before intiating NewMultipartUpload. - err = obj.MakeBucket(bucket) + err := obj.MakeBucket(bucket) if err != nil { // Failed to create newbucket, abort. - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } _, err = obj.NewMultipartUpload(bucket, object) if err != nil { - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } err = obj.AbortMultipartUpload(bucket, object, "abc") switch err.(type) { case InvalidUploadID: default: - t.Fatal("Expected uploadIDPath to exist.") + t.Fatalf("%s: Expected uploadIDPath to exist.", instanceType) } } -// Tests validate correctness of PutObjectPart. +// Wrapper for calling PutObjectPart tests for both XL muliple disks and single node setup. func TestObjectAPIPutObjectPart(t *testing.T) { - // Generating cases for which the PutObjectPart fails. - directory, err := ioutil.TempDir("", "minio-multipart-3-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(directory) - - // Initializing fs object layer. - obj, err := newFSObjects(directory) - if err != nil { - t.Fatal(err) - } + ExecObjectLayerTest(t, testObjectAPIPutObjectPart) +} +// Tests validate correctness of PutObjectPart. +func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing.T) { + // Generating cases for which the PutObjectPart fails. bucket := "minio-bucket" object := "minio-object" // Create bucket before intiating NewMultipartUpload. - err = obj.MakeBucket(bucket) + err := obj.MakeBucket(bucket) if err != nil { // Failed to create newbucket, abort. - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } // Initiate Multipart Upload on the above created bucket. uploadID, err := obj.NewMultipartUpload(bucket, object) if err != nil { // Failed to create NewMultipartUpload, abort. - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } // Creating a dummy bucket for tests. err = obj.MakeBucket("unused-bucket") if err != nil { // Failed to create newbucket, abort. - t.Fatal(err) + t.Fatalf("%s : %s", instanceType, err.Error()) } failCases := []struct { @@ -220,23 +198,23 @@ func TestObjectAPIPutObjectPart(t *testing.T) { // All are test cases above are expected to fail. if actualErr != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s.", i+1, actualErr.Error()) + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) } if actualErr == nil && !testCase.shouldPass { - t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead.", i+1, testCase.expectedError.Error()) + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) } // Failed as expected, but does it fail for the expected reason. if actualErr != nil && !testCase.shouldPass { if testCase.expectedError.Error() != actualErr.Error() { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, - testCase.expectedError.Error(), actualErr.Error()) + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, + instanceType, testCase.expectedError.Error(), actualErr.Error()) } } // Test passes as expected, but the output values are verified for correctness here. if actualErr == nil && testCase.shouldPass { // Asserting whether the md5 output is correct. if testCase.inputMd5 != actualMd5Hex { - t.Errorf("Test %d: Calculated Md5 different from the actual one %s.", i+1, actualMd5Hex) + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex) } } } diff --git a/test-utils.go b/test-utils.go new file mode 100644 index 000000000..94970ff37 --- /dev/null +++ b/test-utils.go @@ -0,0 +1,91 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io/ioutil" + "os" + "testing" +) + +const ( + // singleNodeTestStr is the string which is used as notation for Single node ObjectLayer in the unit tests. + singleNodeTestStr string = "SingleNode" + // xLTestStr is the string which is used as notation for XL ObjectLayer in the unit tests. + xLTestStr string = "XL" +) + +// ExecObjectLayerTest - executes object layer tests. +// Creates single node and XL ObjectLayer instance and runs test for both the layers. +func ExecObjectLayerTest(t *testing.T, objTest func(obj ObjectLayer, instanceType string, t *testing.T)) { + + // getXLObjectLayer - Instantiates XL object layer and returns it. + getXLObjectLayer := func() (ObjectLayer, []string, error) { + var nDisks = 16 // Maximum disks. + var erasureDisks []string + for i := 0; i < nDisks; i++ { + path, err := ioutil.TempDir(os.TempDir(), "minio-") + if err != nil { + return nil, nil, err + } + erasureDisks = append(erasureDisks, path) + } + objLayer, err := newXLObjects(erasureDisks...) + if err != nil { + return nil, nil, err + } + return objLayer, erasureDisks, nil + } + + // getSingleNodeObjectLayer - Instantiates single node object layer and retuns it. + getSingleNodeObjectLayer := func() (ObjectLayer, string, error) { + // Make a temporary directory to use as the obj. + fsDir, err := ioutil.TempDir("", "minio-") + if err != nil { + return nil, "", err + } + + // Create the obj. + objLayer, err := newFSObjects(fsDir) + if err != nil { + return nil, "", err + } + return objLayer, fsDir, nil + } + + // removeRoots - Cleans up initialized directories during tests. + removeRoots := func(roots []string) { + for _, root := range roots { + os.RemoveAll(root) + } + } + + objLayer, fsDir, err := getSingleNodeObjectLayer() + if err != nil { + t.Fatalf("Initialization of object layer failed for single node setup: %s", err.Error()) + } + // Executing the object layer tests for single node setup. + objTest(objLayer, singleNodeTestStr, t) + initNSLock() + objLayer, fsDirs, err := getXLObjectLayer() + if err != nil { + t.Fatalf("Initialization of object layer failed for XL setup: %s", err.Error()) + } + // Executing the object layer tests for XL. + objTest(objLayer, xLTestStr, t) + defer removeRoots(append(fsDirs, fsDir)) +} diff --git a/xl-objects.go b/xl-objects.go index d0b7b9b0c..9adfd661b 100644 --- a/xl-objects.go +++ b/xl-objects.go @@ -159,6 +159,12 @@ func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { if !IsValidObjectName(object) { return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object} } + // Check whether the bucket exists. + if isExist, err := isBucketExist(xl.storage, bucket); err != nil { + return ObjectInfo{}, err + } else if !isExist { + return ObjectInfo{}, BucketNotFound{Bucket: bucket} + } fi, err := xl.storage.StatFile(bucket, object) if err != nil { if err != errFileNotFound {