From 10feb1af3f0a719a62bbe841cb14b11570e1caf2 Mon Sep 17 00:00:00 2001 From: karthic rao Date: Wed, 17 Aug 2016 07:54:23 +0530 Subject: [PATCH] tests: object handlers: Unit tests for Get and Copy Object handlers (#2451) --- object-handlers_test.go | 334 ++++++++++++++++++++++++++++++++++++++++ server_test.go | 44 ++++-- test-utils_test.go | 22 ++- 3 files changed, 382 insertions(+), 18 deletions(-) create mode 100644 object-handlers_test.go diff --git a/object-handlers_test.go b/object-handlers_test.go new file mode 100644 index 000000000..1e385688f --- /dev/null +++ b/object-handlers_test.go @@ -0,0 +1,334 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +// Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup. +func TestAPIGetOjectHandler(t *testing.T) { + ExecObjectLayerTest(t, testAPIGetOjectHandler) +} + +func testAPIGetOjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + + // get random bucket name. + bucketName := getRandomBucketName() + objectName := "test-object" + // Create bucket. + err := obj.MakeBucket(bucketName) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + // Register the API end points with XL/FS object layer. + // Registering only the GetObject handler. + apiRouter := initTestAPIEndPoints(obj, []string{"GetObject"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + + // set of byte data for PutObject. + // object has to be inserted before running tests for GetObject. + // this is required even to assert the GetObject data, + // since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical. + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * 1024 * 1024)}, + } + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + byteRange string // range of bytes to be fetched from GetObject. + // expected output. + expectedContent []byte // expected response body. + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Fetching the entire object and validating its contents. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "", + expectedContent: bytesData[0].byteData, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Case with non-existent object name. + { + bucketName: bucketName, + objectName: "abcd", + byteRange: "", + expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"))), + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Requesting from range 10-100. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=10-100", + expectedContent: bytesData[0].byteData[10:101], + expectedRespStatus: http.StatusPartialContent, + }, + // Test case - 4. + // Test case with invalid range. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=-0", + expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName))), + expectedRespStatus: http.StatusRequestedRangeNotSatisfiable, + }, + // Test case - 5. + // Test case with byte range exceeding the object size. + // Expected to read till end of the object. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=10-1000000000000000", + expectedContent: bytesData[0].byteData[10:], + expectedRespStatus: http.StatusPartialContent, + }, + } + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err := newTestSignedRequest("GET", getGetObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey) + + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Get Object: %v", i+1, err) + } + if testCase.byteRange != "" { + req.Header.Add("Range", testCase.byteRange) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + } + // read the response body. + actualContent, err := ioutil.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + // Verify whether the bucket obtained object is same as the one inserted. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) + } + } +} + +// Wrapper for calling Copy Object API handler tests for both XL multiple disks and single node setup. +func TestAPICopyObjectHandler(t *testing.T) { + ExecObjectLayerTest(t, testAPICopyObjectHandler) +} + +func testAPICopyObjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + // get random bucket name. + bucketName := getRandomBucketName() + objectName := "test-object" + // Create bucket. + err := obj.MakeBucket(bucketName) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + // Register the API end points with XL/FS object layer. + // Registering only the Copy Object handler. + apiRouter := initTestAPIEndPoints(obj, []string{"CopyObject"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + err = initEventNotifier(obj) + if err != nil { + t.Fatalf("Initializing event notifiers failed") + } + + credentials := serverConfig.GetCredential() + + // set of byte data for PutObject. + // object has to be inserted before running tests for Copy Object. + // this is required even to assert the copied object, + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * 1024 * 1024)}, + } + + buffers := []*bytes.Buffer{ + new(bytes.Buffer), + new(bytes.Buffer), + } + + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + // test cases with inputs and expected result for Copy Object. + testCases := []struct { + bucketName string + newObjectName string // name of the newly copied object. + copySourceHeader string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL. + // expected output. + expectedRespStatus int + }{ + // Test case - 1. + { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape("/" + bucketName + "/" + objectName), + expectedRespStatus: http.StatusOK, + }, + + // Test case - 2. + // Test case with invalid source object. + { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape("/"), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 3. + // Test case with new object name is same as object to be copied. + { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape("/" + bucketName + "/" + objectName), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 4. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`. + // Expecting the response status code to http.StatusNotFound (404). + { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape("/" + bucketName + "/" + "non-existent-object"), + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 5. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.PutObject`. + // Expecting the response status code to http.StatusNotFound (404). + { + bucketName: "non-existent-destination-bucket", + newObjectName: objectName, + copySourceHeader: url.QueryEscape("/" + bucketName + "/" + objectName), + expectedRespStatus: http.StatusNotFound, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for copy object. + req, err := newTestSignedRequest("PUT", getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), + 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey) + + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i+1, err) + } + // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. + if testCase.copySourceHeader != "" { + req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + if rec.Code == http.StatusOK { + // See if the new object is formed. + // testing whether the copy was successful. + err = obj.GetObject(testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0]) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType) + } + buffers[0].Reset() + } + } +} diff --git a/server_test.go b/server_test.go index 76a6a61ff..e81fe8eb2 100644 --- a/server_test.go +++ b/server_test.go @@ -536,24 +536,35 @@ func (s *TestSuiteCommon) TestObjectGet(c *C) { c.Assert(err, IsNil) // assert the HTTP response status code. c.Assert(response.StatusCode, Equals, http.StatusOK) + // concurrently reading the object, safety check for races. + var wg sync.WaitGroup + for i := 0; i < ConcurrencyLevel; i++ { + wg.Add(1) + go func() { + defer wg.Done() + // HTTP request to create the bucket. + // create HTTP request to fetch the object. + getRequest, err := newTestSignedRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey) + c.Assert(err, IsNil) - // create HTTP request to fetch the object. - request, err = newTestSignedRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) + reqClient := http.Client{} + // execute the http request to fetch the object. + getResponse, err := reqClient.Do(getRequest) + c.Assert(err, IsNil) + defer getResponse.Body.Close() + // assert the http response status code. + c.Assert(getResponse.StatusCode, Equals, http.StatusOK) - client = http.Client{} - // execute the http request to fetch the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) + // extract response body content. + responseBody, err := ioutil.ReadAll(getResponse.Body) + c.Assert(err, IsNil) + // assert the HTTP response body content with the expected content. + c.Assert(responseBody, DeepEquals, []byte("hello world")) + }() - // extract response body content. - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // assert the HTTP response body content with the expected content. - c.Assert(responseBody, DeepEquals, []byte("hello world")) + } + wg.Wait() } // TestMultipleObjects - Validates upload and fetching of multiple object into the bucket. @@ -783,7 +794,8 @@ func (s *TestSuiteCommon) TestCopyObject(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) objectName2 := "testObject2" - // creating HTTP request for uploading the object. + // Unlike the actual PUT object request, the request to Copy Object doesn't contain request body, + // empty body with the "X-Amz-Copy-Source" header pointing to the object to copies it in the backend. request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName2), 0, nil) c.Assert(err, IsNil) // setting the "X-Amz-Copy-Source" to allow copying the content of previously uploaded object. diff --git a/test-utils_test.go b/test-utils_test.go index f75128155..5099b8d73 100644 --- a/test-utils_test.go +++ b/test-utils_test.go @@ -623,11 +623,16 @@ func getDeleteObjectURL(endPoint, bucketName, objectName string) string { return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) } -// return URL for HEAD o nthe object. +// return URL for HEAD on the object. func getHeadObjectURL(endPoint, bucketName, objectName string) string { return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) } +// return url to be used while copying the object. +func getCopyObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + // return URL for inserting bucket notification. func getPutNotificationURL(endPoint, bucketName string) string { queryValue := url.Values{} @@ -769,6 +774,12 @@ func getXLObjectLayer() (ObjectLayer, []string, error) { if err != nil { return nil, nil, err } + // Disabling the cache for integration tests. + // Should use the object layer tests for validating cache. + if xl, ok := objLayer.(xlObjects); ok { + xl.objCacheEnabled = false + } + return objLayer, erasureDisks, nil } @@ -876,11 +887,18 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand // Iterate the list of API functions requested for and register them in mux HTTP handler. for _, apiFunction := range apiFunctions { switch apiFunction { + // Register GetObject handler. + case "GetObject`": + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler) + // Register GetObject handler. + case "CopyObject`": + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler) + // Register PutBucket Policy handler. case "PutBucketPolicy": bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "") - // Register Delete bucket HTTP policy handler. + // Register Delete bucket HTTP policy handler. case "DeleteBucketPolicy": bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")