From aed62788d998399607fa80ed2100876a2c3b2de5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 5 Mar 2016 16:43:48 -0800 Subject: [PATCH] api: Implement multiple objects Delete api - fixes #956 This API takes input XML input in following form. ``` true Key Key ... ``` and responds the list of successful deletes, list of errors for all the deleted objects. ``` sample1.txt sample2.txt AccessDenied Access Denied ``` --- api-datatypes.go | 30 +++++++++ api-errors.go | 10 ++- api-response.go | 28 +++++++++ bucket-handlers.go | 139 ++++++++++++++++++++++++++++++++++++++++- generic-handlers.go | 1 - object-handlers.go | 10 +-- pkg/fs/fs-multipart.go | 2 +- routers.go | 2 + server_fs_test.go | 4 +- 9 files changed, 214 insertions(+), 12 deletions(-) create mode 100644 api-datatypes.go diff --git a/api-datatypes.go b/api-datatypes.go new file mode 100644 index 000000000..958b3af33 --- /dev/null +++ b/api-datatypes.go @@ -0,0 +1,30 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +// ObjectIdentifier carries key name for the object to delete. +type ObjectIdentifier struct { + ObjectName string `xml:"Key"` +} + +// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted. +type DeleteObjectsRequest struct { + // Element to enable quiet mode for the request + Quiet bool + // List of objects to be deleted + Objects []ObjectIdentifier `xml:"Object"` +} diff --git a/api-errors.go b/api-errors.go index 787e67d3d..a052666ba 100644 --- a/api-errors.go +++ b/api-errors.go @@ -62,6 +62,7 @@ const ( InvalidCopyDest MalformedXML MissingContentLength + MissingContentMD5 MissingRequestBodyError NoSuchBucket NoSuchKey @@ -125,7 +126,7 @@ var errorCodeResponse = map[int]APIError{ }, BadDigest: { Code: "BadDigest", - Description: "The Content-MD5 you specified did not match what we received.", + Description: "The Content-Md5 you specified did not match what we received.", HTTPStatusCode: http.StatusBadRequest, }, BucketAlreadyExists: { @@ -165,7 +166,7 @@ var errorCodeResponse = map[int]APIError{ }, InvalidDigest: { Code: "InvalidDigest", - Description: "The Content-MD5 you specified is not valid.", + Description: "The Content-Md5 you specified is not valid.", HTTPStatusCode: http.StatusBadRequest, }, InvalidRange: { @@ -183,6 +184,11 @@ var errorCodeResponse = map[int]APIError{ Description: "You must provide the Content-Length HTTP header.", HTTPStatusCode: http.StatusLengthRequired, }, + MissingContentMD5: { + Code: "MissingContentMD5", + Description: "Missing required header for this request: Content-Md5.", + HTTPStatusCode: http.StatusBadRequest, + }, MissingRequestBodyError: { Code: "MissingRequestBodyError", Description: "Request body is empty.", diff --git a/api-response.go b/api-response.go index 9114b550a..5d5453493 100644 --- a/api-response.go +++ b/api-response.go @@ -218,6 +218,24 @@ type CompleteMultipartUploadResponse struct { ETag string } +// DeleteError structure. +type DeleteError struct { + Code string + Message string + Key string +} + +// DeleteObjectsResponse container for multiple object deletes. +type DeleteObjectsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"` + + // Collection of all deleted objects + DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"` + + // Collection of errors deleting certain objects. + Errors []DeleteError `xml:"Error,omitempty"` +} + // getLocation get URL location. func getLocation(r *http.Request) string { return r.URL.Path @@ -414,6 +432,16 @@ func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMulti return listMultipartUploadsResponse } +// generate multi objects delete response. +func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse { + deleteResp := DeleteObjectsResponse{} + if !quiet { + deleteResp.DeletedObjects = deletedObjects + } + deleteResp.Errors = errs + return deleteResp +} + // writeSuccessResponse write success headers and response if any. func writeSuccessResponse(w http.ResponseWriter, response []byte) { setCommonHeaders(w) diff --git a/bucket-handlers.go b/bucket-handlers.go index 3bcdb3f87..b91f91549 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,10 @@ package main import ( "bytes" + "crypto/md5" + "encoding/base64" "encoding/hex" + "encoding/xml" "io" "io/ioutil" "mime/multipart" @@ -212,6 +215,140 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) writeErrorResponse(w, r, InternalError, r.URL.Path) } +// DeleteMultipleObjectsHandler - deletes multiple objects. +func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + + if isRequestRequiresACLCheck(r) { + writeErrorResponse(w, r, AccessDenied, r.URL.Path) + return + } + + // Content-Length is required and should be non-zero + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if r.ContentLength <= 0 { + writeErrorResponse(w, r, MissingContentLength, r.URL.Path) + return + } + + // Content-Md5 is requied should be set + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if _, ok := r.Header["Content-Md5"]; !ok { + writeErrorResponse(w, r, MissingContentMD5, r.URL.Path) + return + } + + // Set http request for signature. + auth := api.Signature.SetHTTPRequestToVerify(r) + + // Allocate incoming content length bytes. + deleteXMLBytes := make([]byte, r.ContentLength) + + // Read incoming body XML bytes. + _, e := io.ReadFull(r.Body, deleteXMLBytes) + if e != nil { + errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil) + writeErrorResponse(w, r, InternalError, r.URL.Path) + return + } + + // Check if request is presigned. + if isRequestPresignedSignatureV4(r) { + ok, err := auth.DoesPresignedSignatureMatch() + if err != nil { + errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil) + writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path) + return + } + if !ok { + writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path) + return + } + } else if isRequestSignatureV4(r) { + // Check if request is signed. + sha := sha256.New() + mdSh := md5.New() + sha.Write(deleteXMLBytes) + mdSh.Write(deleteXMLBytes) + ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil))) + if err != nil { + errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil) + writeErrorResponse(w, r, InternalError, r.URL.Path) + return + } + if !ok { + writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path) + return + } + // Verify content md5. + if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) { + writeErrorResponse(w, r, BadDigest, r.URL.Path) + return + } + } + + // Unmarshal list of keys to be deleted. + deleteObjects := &DeleteObjectsRequest{} + if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil { + writeErrorResponse(w, r, MalformedXML, r.URL.Path) + return + } + + var deleteErrors []DeleteError + var deletedObjects []ObjectIdentifier + // Loop through all the objects and delete them sequentially. + for _, object := range deleteObjects.Objects { + err := api.Filesystem.DeleteObject(bucket, object.ObjectName) + if err == nil { + deletedObjects = append(deletedObjects, ObjectIdentifier{ + ObjectName: object.ObjectName, + }) + } else { + errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil) + switch err.ToGoError().(type) { + case fs.BucketNameInvalid: + deleteErrors = append(deleteErrors, DeleteError{ + Code: errorCodeResponse[InvalidBucketName].Code, + Message: errorCodeResponse[InvalidBucketName].Description, + Key: object.ObjectName, + }) + case fs.BucketNotFound: + deleteErrors = append(deleteErrors, DeleteError{ + Code: errorCodeResponse[NoSuchBucket].Code, + Message: errorCodeResponse[NoSuchBucket].Description, + Key: object.ObjectName, + }) + case fs.ObjectNotFound: + deleteErrors = append(deleteErrors, DeleteError{ + Code: errorCodeResponse[NoSuchKey].Code, + Message: errorCodeResponse[NoSuchKey].Description, + Key: object.ObjectName, + }) + case fs.ObjectNameInvalid: + deleteErrors = append(deleteErrors, DeleteError{ + Code: errorCodeResponse[NoSuchKey].Code, + Message: errorCodeResponse[NoSuchKey].Description, + Key: object.ObjectName, + }) + default: + deleteErrors = append(deleteErrors, DeleteError{ + Code: errorCodeResponse[InternalError].Code, + Message: errorCodeResponse[InternalError].Description, + Key: object.ObjectName, + }) + } + } + } + // Generate response + response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors) + encodedSuccessResponse := encodeResponse(response) + // Write headers + setCommonHeaders(w) + // Write success response. + writeSuccessResponse(w, encodedSuccessResponse) +} + // PutBucketHandler - PUT Bucket // ---------- // This implementation of the PUT operation creates a new bucket for authenticated request diff --git a/generic-handlers.go b/generic-handlers.go index b063187b2..2d7bd63ff 100644 --- a/generic-handlers.go +++ b/generic-handlers.go @@ -289,7 +289,6 @@ var notimplementedBucketResourceNames = map[string]bool{ "requestPayment": true, "versioning": true, "website": true, - "delete": true, } // List of not implemented object queries diff --git a/object-handlers.go b/object-handlers.go index 61257280d..12b3706d3 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -420,8 +420,8 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } } - // get Content-MD5 sent by client and verify if valid - md5 := r.Header.Get("Content-MD5") + // get Content-Md5 sent by client and verify if valid + md5 := r.Header.Get("Content-Md5") if !isValidMD5(md5) { writeErrorResponse(w, r, InvalidDigest, r.URL.Path) return @@ -554,8 +554,8 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques } } - // get Content-MD5 sent by client and verify if valid - md5 := r.Header.Get("Content-MD5") + // get Content-Md5 sent by client and verify if valid + md5 := r.Header.Get("Content-Md5") if !isValidMD5(md5) { writeErrorResponse(w, r, InvalidDigest, r.URL.Path) return @@ -811,7 +811,7 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h /// Delete storageAPI -// DeleteObjectHandler - Delete object +// DeleteObjectHandler - delete an object func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/pkg/fs/fs-multipart.go b/pkg/fs/fs-multipart.go index 290eea22e..4ac9856ec 100644 --- a/pkg/fs/fs-multipart.go +++ b/pkg/fs/fs-multipart.go @@ -202,7 +202,7 @@ func saveParts(partPathPrefix string, mw io.Writer, parts []CompletePart) *probe if !os.IsNotExist(e) { return probe.NewError(e) } - // Some clients do not set Content-MD5, so we would have + // Some clients do not set Content-Md5, so we would have // created part files without 'ETag' in them. partFile, e = os.OpenFile(partPathPrefix+fmt.Sprintf("$%d-$multiparts", part.PartNumber), os.O_RDONLY, 0600) if e != nil { diff --git a/routers.go b/routers.go index fc83ee9e4..d0a2f5bb0 100644 --- a/routers.go +++ b/routers.go @@ -149,6 +149,8 @@ func registerAPIHandlers(mux *router.Router, a storageAPI, w *webAPI) { bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler) // HeadBucket bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler) + // DeleteMultipleObjects + bucket.Methods("POST").HandlerFunc(a.DeleteMultipleObjectsHandler) // PostPolicy bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler) // DeleteBucket diff --git a/server_fs_test.go b/server_fs_test.go index bcb8f6ec0..5263e61f9 100644 --- a/server_fs_test.go +++ b/server_fs_test.go @@ -1158,7 +1158,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) { buffer1 := bytes.NewReader([]byte("hello world")) request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=1", int64(buffer1.Len()), buffer1) - request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum)) + request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) c.Assert(err, IsNil) client = http.Client{} @@ -1168,7 +1168,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) { buffer2 := bytes.NewReader([]byte("hello world")) request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=2", int64(buffer2.Len()), buffer2) - request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum)) + request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) c.Assert(err, IsNil) client = http.Client{}