Merge pull request #1195 from harshavardhana/delete-objects

api: Implement multiple objects Delete api - fixes #956

api-datatypes.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+// ObjectIdentifier carries key name for the object to delete.
+type ObjectIdentifier struct {
+	ObjectName string `xml:"Key"`
+}
+
+// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
+type DeleteObjectsRequest struct {
+	// Element to enable quiet mode for the request
+	Quiet bool
+	// List of objects to be deleted
+	Objects []ObjectIdentifier `xml:"Object"`
+}

api-errors.go

@@ -62,6 +62,7 @@ const (
 	InvalidCopyDest
 	MalformedXML
 	MissingContentLength
+	MissingContentMD5
 	MissingRequestBodyError
 	NoSuchBucket
 	NoSuchKey
@@ -125,7 +126,7 @@ var errorCodeResponse = map[int]APIError{
 	},
 	BadDigest: {
 		Code:           "BadDigest",
-		Description:    "The Content-MD5 you specified did not match what we received.",
+		Description:    "The Content-Md5 you specified did not match what we received.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	BucketAlreadyExists: {
@@ -165,7 +166,7 @@ var errorCodeResponse = map[int]APIError{
 	},
 	InvalidDigest: {
 		Code:           "InvalidDigest",
-		Description:    "The Content-MD5 you specified is not valid.",
+		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	InvalidRange: {
@@ -183,6 +184,11 @@ var errorCodeResponse = map[int]APIError{
 		Description:    "You must provide the Content-Length HTTP header.",
 		HTTPStatusCode: http.StatusLengthRequired,
 	},
+	MissingContentMD5: {
+		Code:           "MissingContentMD5",
+		Description:    "Missing required header for this request: Content-Md5.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	MissingRequestBodyError: {
 		Code:           "MissingRequestBodyError",
 		Description:    "Request body is empty.",

api-response.go

@@ -218,6 +218,24 @@ type CompleteMultipartUploadResponse struct {
 	ETag     string
 }
 
+// DeleteError structure.
+type DeleteError struct {
+	Code    string
+	Message string
+	Key     string
+}
+
+// DeleteObjectsResponse container for multiple object deletes.
+type DeleteObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
+
+	// Collection of all deleted objects
+	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+
+	// Collection of errors deleting certain objects.
+	Errors []DeleteError `xml:"Error,omitempty"`
+}
+
 // getLocation get URL location.
 func getLocation(r *http.Request) string {
 	return r.URL.Path
@@ -414,6 +432,16 @@ func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMulti
 	return listMultipartUploadsResponse
 }
 
+// generate multi objects delete response.
+func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+	deleteResp := DeleteObjectsResponse{}
+	if !quiet {
+		deleteResp.DeletedObjects = deletedObjects
+	}
+	deleteResp.Errors = errs
+	return deleteResp
+}
+
 // writeSuccessResponse write success headers and response if any.
 func writeSuccessResponse(w http.ResponseWriter, response []byte) {
 	setCommonHeaders(w)

bucket-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,10 @@ package main
 
 import (
 	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/xml"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
@@ -212,6 +215,140 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
 	writeErrorResponse(w, r, InternalError, r.URL.Path)
 }
 
+// DeleteMultipleObjectsHandler - deletes multiple objects.
+func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(r) {
+		writeErrorResponse(w, r, AccessDenied, r.URL.Path)
+		return
+	}
+
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(w, r, MissingContentLength, r.URL.Path)
+		return
+	}
+
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(w, r, MissingContentMD5, r.URL.Path)
+		return
+	}
+
+	// Set http request for signature.
+	auth := api.Signature.SetHTTPRequestToVerify(r)
+
+	// Allocate incoming content length bytes.
+	deleteXMLBytes := make([]byte, r.ContentLength)
+
+	// Read incoming body XML bytes.
+	_, e := io.ReadFull(r.Body, deleteXMLBytes)
+	if e != nil {
+		errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil)
+		writeErrorResponse(w, r, InternalError, r.URL.Path)
+		return
+	}
+
+	// Check if request is presigned.
+	if isRequestPresignedSignatureV4(r) {
+		ok, err := auth.DoesPresignedSignatureMatch()
+		if err != nil {
+			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
+			writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+	} else if isRequestSignatureV4(r) {
+		// Check if request is signed.
+		sha := sha256.New()
+		mdSh := md5.New()
+		sha.Write(deleteXMLBytes)
+		mdSh.Write(deleteXMLBytes)
+		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil)))
+		if err != nil {
+			errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil)
+			writeErrorResponse(w, r, InternalError, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		// Verify content md5.
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) {
+			writeErrorResponse(w, r, BadDigest, r.URL.Path)
+			return
+		}
+	}
+
+	// Unmarshal list of keys to be deleted.
+	deleteObjects := &DeleteObjectsRequest{}
+	if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
+		writeErrorResponse(w, r, MalformedXML, r.URL.Path)
+		return
+	}
+
+	var deleteErrors []DeleteError
+	var deletedObjects []ObjectIdentifier
+	// Loop through all the objects and delete them sequentially.
+	for _, object := range deleteObjects.Objects {
+		err := api.Filesystem.DeleteObject(bucket, object.ObjectName)
+		if err == nil {
+			deletedObjects = append(deletedObjects, ObjectIdentifier{
+				ObjectName: object.ObjectName,
+			})
+		} else {
+			errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
+			switch err.ToGoError().(type) {
+			case fs.BucketNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[InvalidBucketName].Code,
+					Message: errorCodeResponse[InvalidBucketName].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.BucketNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[NoSuchBucket].Code,
+					Message: errorCodeResponse[NoSuchBucket].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.ObjectNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[NoSuchKey].Code,
+					Message: errorCodeResponse[NoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.ObjectNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[NoSuchKey].Code,
+					Message: errorCodeResponse[NoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			default:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[InternalError].Code,
+					Message: errorCodeResponse[InternalError].Description,
+					Key:     object.ObjectName,
+				})
+			}
+		}
+	}
+	// Generate response
+	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
 // PutBucketHandler - PUT Bucket
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request

generic-handlers.go

@@ -289,7 +289,6 @@ var notimplementedBucketResourceNames = map[string]bool{
 	"requestPayment": true,
 	"versioning":     true,
 	"website":        true,
-	"delete":         true,
 }
 
 // List of not implemented object queries

object-handlers.go

@@ -420,8 +420,8 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// get Content-MD5 sent by client and verify if valid
-	md5 := r.Header.Get("Content-MD5")
+	// get Content-Md5 sent by client and verify if valid
+	md5 := r.Header.Get("Content-Md5")
 	if !isValidMD5(md5) {
 		writeErrorResponse(w, r, InvalidDigest, r.URL.Path)
 		return
@@ -554,8 +554,8 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	// get Content-MD5 sent by client and verify if valid
-	md5 := r.Header.Get("Content-MD5")
+	// get Content-Md5 sent by client and verify if valid
+	md5 := r.Header.Get("Content-Md5")
 	if !isValidMD5(md5) {
 		writeErrorResponse(w, r, InvalidDigest, r.URL.Path)
 		return
@@ -811,7 +811,7 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h
 
 /// Delete storageAPI
 
-// DeleteObjectHandler - Delete object
+// DeleteObjectHandler - delete an object
 func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

pkg/fs/fs-multipart.go

@@ -202,7 +202,7 @@ func saveParts(partPathPrefix string, mw io.Writer, parts []CompletePart) *probe
 			if !os.IsNotExist(e) {
 				return probe.NewError(e)
 			}
-			// Some clients do not set Content-MD5, so we would have
+			// Some clients do not set Content-Md5, so we would have
 			// created part files without 'ETag' in them.
 			partFile, e = os.OpenFile(partPathPrefix+fmt.Sprintf("$%d-$multiparts", part.PartNumber), os.O_RDONLY, 0600)
 			if e != nil {

routers.go

@@ -149,6 +149,8 @@ func registerAPIHandlers(mux *router.Router, a storageAPI, w *webAPI) {
 	bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler)
 	// HeadBucket
 	bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(a.DeleteMultipleObjectsHandler)
 	// PostPolicy
 	bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler)
 	// DeleteBucket

server_fs_test.go

@@ -1158,7 +1158,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) {
 
 	buffer1 := bytes.NewReader([]byte("hello world"))
 	request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=1", int64(buffer1.Len()), buffer1)
-	request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum))
+	request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
 	c.Assert(err, IsNil)
 
 	client = http.Client{}
@@ -1168,7 +1168,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) {
 
 	buffer2 := bytes.NewReader([]byte("hello world"))
 	request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts/object?uploadId="+uploadID+"&partNumber=2", int64(buffer2.Len()), buffer2)
-	request.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Sum))
+	request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum))
 	c.Assert(err, IsNil)
 
 	client = http.Client{}