From dba7c94571718c510de3ef6e605d6f5caee55671 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 14 May 2015 17:07:30 -0700 Subject: [PATCH] Add tests for abortMultipartUpload, listobjectparts, listmultipartuploads --- pkg/api/api_bucket_handlers.go | 8 +- pkg/api/api_object_handlers.go | 24 ++- pkg/api/api_test.go | 268 ++++++++++++++++++++++++++++++++- 3 files changed, 287 insertions(+), 13 deletions(-) diff --git a/pkg/api/api_bucket_handlers.go b/pkg/api/api_bucket_handlers.go index 4454f2062..1a9d358f3 100644 --- a/pkg/api/api_bucket_handlers.go +++ b/pkg/api/api_bucket_handlers.go @@ -70,10 +70,6 @@ func (server *minioAPI) isValidOp(w http.ResponseWriter, req *http.Request, acce // func (server *minioAPI) listMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) { acceptsContentType := getContentType(req) - // verify if bucket allows this operation - if !server.isValidOp(w, req, acceptsContentType) { - return - } resources := getBucketMultipartResources(req.URL.Query()) if resources.MaxUploads == 0 { @@ -98,9 +94,9 @@ func (server *minioAPI) listMultipartUploadsHandler(w http.ResponseWriter, req * // write body w.Write(encodedSuccessResponse) } - case drivers.ObjectNotFound: + case drivers.BucketNotFound: { - writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path) + writeErrorResponse(w, req, NoSuchBucket, acceptsContentType, req.URL.Path) } default: { diff --git a/pkg/api/api_object_handlers.go b/pkg/api/api_object_handlers.go index d6260ab6e..fc105ac84 100644 --- a/pkg/api/api_object_handlers.go +++ b/pkg/api/api_object_handlers.go @@ -215,6 +215,11 @@ func (server *minioAPI) newMultipartUploadHandler(w http.ResponseWriter, req *ht return } + if !isRequestUploads(req.URL.Query()) { + writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path) + return + } + var object, bucket string vars := mux.Vars(req) bucket = vars["bucket"] @@ -330,6 +335,10 @@ func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Re func (server *minioAPI) abortMultipartUploadHandler(w http.ResponseWriter, req *http.Request) { acceptsContentType := getContentType(req) + // handle ACL's here at bucket level + if !server.isValidOp(w, req, acceptsContentType) { + return + } vars := mux.Vars(req) bucket := vars["bucket"] @@ -352,15 +361,20 @@ func (server *minioAPI) abortMultipartUploadHandler(w http.ResponseWriter, req * func (server *minioAPI) listObjectPartsHandler(w http.ResponseWriter, req *http.Request) { acceptsContentType := getContentType(req) + // handle ACL's here at bucket level + if !server.isValidOp(w, req, acceptsContentType) { + return + } - vars := mux.Vars(req) - bucket := vars["bucket"] - object := vars["object"] objectResourcesMetadata := getObjectResources(req.URL.Query()) if objectResourcesMetadata.MaxParts == 0 { objectResourcesMetadata.MaxParts = maxPartsList } + vars := mux.Vars(req) + bucket := vars["bucket"] + object := vars["object"] + objectResourcesMetadata, err := server.driver.ListObjectParts(bucket, object, objectResourcesMetadata) switch err := iodine.ToError(err).(type) { case nil: @@ -383,6 +397,10 @@ func (server *minioAPI) listObjectPartsHandler(w http.ResponseWriter, req *http. func (server *minioAPI) completeMultipartUploadHandler(w http.ResponseWriter, req *http.Request) { acceptsContentType := getContentType(req) + // handle ACL's here at bucket level + if !server.isValidOp(w, req, acceptsContentType) { + return + } decoder := xml.NewDecoder(req.Body) parts := &CompleteMultipartUpload{} diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index fa353f7cf..569eaf774 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -1306,13 +1306,14 @@ func (s *MySuite) TestGetObjectRangeErrors(c *C) { verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) } -func (s *MySuite) TestPutMultipart(c *C) { +func (s *MySuite) TestObjectMultipartAbort(c *C) { switch driver := s.Driver.(type) { case *mocks.Driver: { driver.AssertExpectations(c) } default: + // Donut doesn't have multipart support yet { if reflect.TypeOf(driver).String() == "*memory.memoryDriver" { @@ -1369,7 +1370,7 @@ func (s *MySuite) TestPutMultipart(c *C) { c.Assert(err, IsNil) c.Assert(response1.StatusCode, Equals, http.StatusOK) - // // put part two + // put part two typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) @@ -1379,9 +1380,267 @@ func (s *MySuite) TestPutMultipart(c *C) { response2, err := client.Do(request) c.Assert(err, IsNil) c.Assert(response2.StatusCode, Equals, http.StatusOK) - // - // complete multipart upload + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("AbortMultipartUpload", "foo", "object", "uploadid").Return(nil).Once() + request, err = http.NewRequest("DELETE", testServer.URL+"/foo/object?uploadId="+uploadID, nil) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response3, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response3.StatusCode, Equals, http.StatusNoContent) +} + +func (s *MySuite) TestBuckeMultipartList(c *C) { + switch driver := s.Driver.(type) { + case *mocks.Driver: + { + driver.AssertExpectations(c) + } + default: + // Donut doesn't have multipart support yet + { + if reflect.TypeOf(driver).String() == "*memory.memoryDriver" { + + } else { + return + } + } + } + driver := s.Driver + typedDriver := s.MockDriver + featureflags.Enable(featureflags.MultipartPutObject) + + httpHandler := HTTPHandler(driver) + testServer := httptest.NewServer(httpHandler) + defer testServer.Close() + client := http.Client{} + + // create bucket + typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() + request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, 200) + + // Initiate multipart upload + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() + request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err = client.Do(request) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResult{} + + err = decoder.Decode(newResponse) + c.Assert(err, IsNil) + c.Assert(len(newResponse.UploadID) > 0, Equals, true) + uploadID := newResponse.UploadID + + // put part one + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response1, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response1.StatusCode, Equals, http.StatusOK) + + // put part two + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response2, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response2.StatusCode, Equals, http.StatusOK) + + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("ListMultipartUploads", "foo", mock.Anything).Return(drivers.BucketMultipartResourcesMetadata{}, nil).Once() + request, err = http.NewRequest("GET", testServer.URL+"/foo?uploads", nil) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response3, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response3.StatusCode, Equals, http.StatusOK) +} + +func (s *MySuite) TestObjectMultipartList(c *C) { + switch driver := s.Driver.(type) { + case *mocks.Driver: + { + driver.AssertExpectations(c) + } + default: + // Donut doesn't have multipart support yet + { + if reflect.TypeOf(driver).String() == "*memory.memoryDriver" { + + } else { + return + } + } + } + driver := s.Driver + typedDriver := s.MockDriver + featureflags.Enable(featureflags.MultipartPutObject) + + httpHandler := HTTPHandler(driver) + testServer := httptest.NewServer(httpHandler) + defer testServer.Close() + client := http.Client{} + + // create bucket + typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() + request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, 200) + + // Initiate multipart upload + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() + request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err = client.Do(request) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResult{} + + err = decoder.Decode(newResponse) + c.Assert(err, IsNil) + c.Assert(len(newResponse.UploadID) > 0, Equals, true) + uploadID := newResponse.UploadID + + // put part one + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response1, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response1.StatusCode, Equals, http.StatusOK) + + // put part two + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response2, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response2.StatusCode, Equals, http.StatusOK) + + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("ListObjectParts", "foo", "object", mock.Anything).Return(drivers.ObjectResourcesMetadata{}, nil).Once() + request, err = http.NewRequest("GET", testServer.URL+"/foo/object?uploadId="+uploadID, nil) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response3, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response3.StatusCode, Equals, http.StatusOK) + +} + +func (s *MySuite) TestObjectMultipart(c *C) { + switch driver := s.Driver.(type) { + case *mocks.Driver: + { + driver.AssertExpectations(c) + } + default: + // Donut doesn't have multipart support yet + { + if reflect.TypeOf(driver).String() == "*memory.memoryDriver" { + + } else { + return + } + } + } + driver := s.Driver + typedDriver := s.MockDriver + featureflags.Enable(featureflags.MultipartPutObject) + + httpHandler := HTTPHandler(driver) + testServer := httptest.NewServer(httpHandler) + defer testServer.Close() + client := http.Client{} + + // create bucket + typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() + request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, 200) + + // Initiate multipart upload + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() + request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response, err = client.Do(request) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResult{} + + err = decoder.Decode(newResponse) + c.Assert(err, IsNil) + c.Assert(len(newResponse.UploadID) > 0, Equals, true) + uploadID := newResponse.UploadID + + // put part one + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response1, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response1.StatusCode, Equals, http.StatusOK) + + // put part two + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() + typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() + request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) + c.Assert(err, IsNil) + setDummyAuthHeader(request) + + response2, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response2.StatusCode, Equals, http.StatusOK) + + // complete multipart upload completeUploads := &CompleteMultipartUpload{ Part: []Part{ { @@ -1399,6 +1658,7 @@ func (s *MySuite) TestPutMultipart(c *C) { encoder := xml.NewEncoder(&completeBuffer) encoder.Encode(completeUploads) + typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() typedDriver.On("CompleteMultipartUpload", "foo", "object", "uploadid", mock.Anything).Return("etag", nil).Once() request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploadId="+uploadID, &completeBuffer) c.Assert(err, IsNil)