diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 816263c3a..166f30940 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "net/http" "net/url" + "path" "strconv" "time" ) @@ -49,6 +50,7 @@ const ( mgmtDryRun mgmtQueryKey = "dry-run" mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker" mgmtMaxUploads mgmtQueryKey = "max-uploads" + mgmtUploadID mgmtQueryKey = "upload-id" ) // ServerVersion - server version @@ -654,6 +656,75 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt writeSuccessResponseHeadersOnly(w) } +// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run +// - x-minio-operation = upload +// - bucket, object and upload-id are mandatory query parameters +// Heal a given upload, if present. +func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) { + // Get object layer instance. + objLayer := newObjectLayerFn() + if objLayer == nil { + writeErrorResponse(w, ErrServerNotInitialized, r.URL) + return + } + + // Validate request signature. + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, adminAPIErr, r.URL) + return + } + + vars := r.URL.Query() + bucket := vars.Get(string(mgmtBucket)) + object := vars.Get(string(mgmtObject)) + uploadID := vars.Get(string(mgmtUploadID)) + uploadObj := path.Join(bucket, object, uploadID) + + // Validate bucket and object names as supplied via query + // parameters. + if err := checkBucketAndObjectNames(bucket, object); err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Validate the bucket and object w.r.t backend representation + // of an upload. + if err := checkBucketAndObjectNames(minioMetaMultipartBucket, + uploadObj); err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Check if upload exists. + if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket, + uploadObj); err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // if dry-run is set in query params then perform validations + // and return success. + if isDryRun(vars) { + writeSuccessResponseHeadersOnly(w) + return + } + + //We are able to use HealObject for healing an upload since an + //ongoing upload has the same backend representation as an + //object. The 'object' corresponding to a given bucket, + //object and uploadID is + //.minio.sys/multipart/bucket/object/uploadID. + err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj) + if err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Return 200 on success. + writeSuccessResponseHeadersOnly(w) +} + // HealFormatHandler - POST /?heal&dry-run // - x-minio-operation = format // - bucket and object are both mandatory query parameters diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 187c7c0a6..099129bf5 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -1048,6 +1049,160 @@ func TestHealObjectHandler(t *testing.T) { t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) } } + +} + +// buildAdminRequest - helper function to build an admin API request. +func buildAdminRequest(queryVal url.Values, opHdr, method string, + contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) { + req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker) + if err != nil { + return nil, traceError(err) + } + + req.Header.Set(minioAdminOpHeader, opHdr) + + cred := serverConfig.GetCredential() + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + return nil, traceError(err) + } + + return req, nil +} + +// mkHealUploadQuery - helper to build HealUploadHandler query string. +func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values { + queryVal := url.Values{} + queryVal.Set(string(mgmtBucket), bucket) + queryVal.Set(string(mgmtObject), object) + queryVal.Set(string(mgmtUploadID), uploadID) + queryVal.Set("heal", "") + queryVal.Set(string(mgmtDryRun), dryRun) + return queryVal +} + +// TestHealUploadHandler - test for HealUploadHandler. +func TestHealUploadHandler(t *testing.T) { + adminTestBed, err := prepareAdminXLTestBed() + if err != nil { + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") + } + defer adminTestBed.TearDown() + + // Create an object myobject under bucket mybucket. + bucketName := "mybucket" + objName := "myobject" + err = adminTestBed.objLayer.MakeBucket(bucketName) + if err != nil { + t.Fatalf("Failed to make bucket %s - %v", bucketName, err) + } + + // Create a new multipart upload. + uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil) + if err != nil { + t.Fatalf("Failed to create a new multipart upload %s/%s - %v", + bucketName, objName, err) + } + + // Upload a part. + partID := 1 + _, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID, + partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "") + if err != nil { + t.Fatalf("Failed to upload part %d of %s/%s - %v", partID, + bucketName, objName, err) + } + + testCases := []struct { + bucket string + object string + dryrun string + statusCode int + }{ + // 1. Valid test case. + { + bucket: bucketName, + object: objName, + statusCode: http.StatusOK, + }, + // 2. Invalid bucket name. + { + bucket: `invalid\\Bucket`, + object: "myobject", + statusCode: http.StatusBadRequest, + }, + // 3. Bucket not found. + { + bucket: "bucketnotfound", + object: "myobject", + statusCode: http.StatusNotFound, + }, + // 4. Invalid object name. + { + bucket: bucketName, + object: `invalid\\Object`, + statusCode: http.StatusBadRequest, + }, + // 5. Object not found. + { + bucket: bucketName, + object: "objectnotfound", + statusCode: http.StatusNotFound, + }, + // 6. Valid test case with dry-run. + { + bucket: bucketName, + object: objName, + dryrun: "yes", + statusCode: http.StatusOK, + }, + } + for i, test := range testCases { + // Prepare query params. + queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun) + req, err := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil) + if err != nil { + t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err) + } + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if test.statusCode != rec.Code { + t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) + } + } + + sample := testCases[0] + // Modify authorization header after signing to test signature + // mismatch handling. + queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun) + req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil) + if err != nil { + t.Fatalf("Failed to construct heal object request - %v", err) + } + + // Set x-amz-date to a date different than time of signing. + req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format)) + + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code) + } + + // Set objectAPI to nil to test Server not initialized case. + resetGlobalObjectAPI() + queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun) + req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil) + if err != nil { + t.Fatalf("Failed to construct heal object request - %v", err) + } + + rec = httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code) + } } // TestHealFormatHandler - test for HealFormatHandler. @@ -1061,21 +1216,11 @@ func TestHealFormatHandler(t *testing.T) { // Prepare query params for heal-format mgmt REST API. queryVal := url.Values{} queryVal.Set("heal", "") - req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) + req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil) if err != nil { t.Fatalf("Failed to construct heal object request - %v", err) } - // Set x-minio-operation header to format. - req.Header.Set(minioAdminOpHeader, "format") - - // Sign the request using signature v4. - cred := serverConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Failed to sign heal object request - %v", err) - } - rec := httptest.NewRecorder() adminTestBed.mux.ServeHTTP(rec, req) if rec.Code != http.StatusOK { @@ -1105,21 +1250,11 @@ func TestGetConfigHandler(t *testing.T) { queryVal := url.Values{} queryVal.Set("config", "") - req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil) + req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil) if err != nil { t.Fatalf("Failed to construct get-config object request - %v", err) } - // Set x-minio-operation header to get. - req.Header.Set(minioAdminOpHeader, "get") - - // Sign the request using signature v4. - cred := serverConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Failed to sign heal object request - %v", err) - } - rec := httptest.NewRecorder() adminTestBed.mux.ServeHTTP(rec, req) if rec.Code != http.StatusOK { @@ -1154,21 +1289,12 @@ func TestSetConfigHandler(t *testing.T) { queryVal := url.Values{} queryVal.Set("config", "") - req, err := newTestRequest("PUT", "/?"+queryVal.Encode(), int64(len(configJSON)), bytes.NewReader(configJSON)) + req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)), + bytes.NewReader(configJSON)) if err != nil { t.Fatalf("Failed to construct get-config object request - %v", err) } - // Set x-minio-operation header to set. - req.Header.Set(minioAdminOpHeader, "set") - - // Sign the request using signature v4. - cred := serverConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Failed to sign heal object request - %v", err) - } - rec := httptest.NewRecorder() adminTestBed.mux.ServeHTTP(rec, req) if rec.Code != http.StatusOK { @@ -1288,9 +1414,9 @@ func TestWriteSetConfigResponse(t *testing.T) { } } -// mkUploadsHealQuery - helper function to construct query values for +// mkListUploadsHealQuery - helper function to construct query values for // listUploadsHeal. -func mkUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values { +func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values { queryVal := make(url.Values) queryVal.Set("heal", "") @@ -1401,19 +1527,13 @@ func TestListHealUploadsHandler(t *testing.T) { } for i, test := range testCases { - queryVal := mkUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys) + queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys) - req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil) + req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil) if err != nil { t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err) } - req.Header.Set(minioAdminOpHeader, "list-uploads") - cred := serverConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Test %d - Failed to sign list uploads needing heal request - %v", i+1, err) - } rec := httptest.NewRecorder() adminTestBed.mux.ServeHTTP(rec, req) if test.statusCode != rec.Code { diff --git a/cmd/admin-router.go b/cmd/admin-router.go index b65c4eda2..63ebab1ee 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -64,6 +64,8 @@ func registerAdminRouter(mux *router.Router) { adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler) // Heal Format. adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler) + // Heal Uploads. + adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler) /// Config operations diff --git a/cmd/xl-v1-healing.go b/cmd/xl-v1-healing.go index e8d51a7e4..c2a833775 100644 --- a/cmd/xl-v1-healing.go +++ b/cmd/xl-v1-healing.go @@ -477,10 +477,6 @@ func healObject(storageDisks []StorageAPI, bucket string, object string, quorum // and later the disk comes back up again, heal on the object // should delete it. func (xl xlObjects) HealObject(bucket, object string) error { - if err := checkGetObjArgs(bucket, object); err != nil { - return err - } - // Lock the object before healing. objectLock := globalNSMutex.NewNSLock(bucket, object) objectLock.RLock() diff --git a/cmd/xl-v1-healing_test.go b/cmd/xl-v1-healing_test.go index 1acbd9909..e84b0af3e 100644 --- a/cmd/xl-v1-healing_test.go +++ b/cmd/xl-v1-healing_test.go @@ -17,7 +17,9 @@ package cmd import ( + "bytes" "fmt" + "path/filepath" "testing" ) @@ -486,3 +488,73 @@ func TestListBucketsHeal(t *testing.T) { t.Fatalf("Name of missing bucket is incorrect, expected: %s, found: %s", corruptedBucketName, buckets[0].Name) } } + +// Tests healing of object. +func TestHealObjectXL(t *testing.T) { + root, err := newTestConfig(globalMinioDefaultRegion) + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + endpoints, err := parseStorageEndpoints(fsDirs) + if err != nil { + t.Fatal(err) + } + + // Everything is fine, should return nil + obj, _, err := initObjectLayer(endpoints) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + data := []byte("hello") + err = obj.MakeBucket(bucket) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + _, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil, "") + if err != nil { + t.Fatalf("Failed to put an object - %v", err) + } + + // Remove the object backend files from the first disk. + xl := obj.(*xlObjects) + firstDisk := xl.storageDisks[0] + err = firstDisk.DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile)) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + err = obj.HealObject(bucket, object) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + _, err = firstDisk.StatFile(bucket, filepath.Join(object, xlMetaJSONFile)) + if err != nil { + t.Errorf("Expected xl.json file to be present but stat failed - %v", err) + } + + // Nil more than half the disks, to remove write quorum. + for i := 0; i <= len(xl.storageDisks)/2; i++ { + xl.storageDisks[i] = nil + } + + // Try healing now, expect to receive errDiskNotFound. + err = obj.HealObject(bucket, object) + if errorCause(err) != errDiskNotFound { + t.Errorf("Expected %v but received %v", errDiskNotFound, err) + } +} diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md index 7d5ba2a5e..9c8e352f1 100644 --- a/pkg/madmin/API.md +++ b/pkg/madmin/API.md @@ -44,6 +44,7 @@ func main() { | | |[`HealObject`](#HealObject)||| | | |[`HealFormat`](#HealFormat)||| | | |[`ListUploadsHeal`](#ListUploadsHeal)||| +| | |[`HealUpload`](#HealUpload)||| ## 1. Constructor @@ -278,6 +279,62 @@ __Example__ log.Println("successfully healed storage format on available disks.") ``` + +### ListUploadsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan UploadInfo, error) +List ongoing multipart uploads that need healing. + +| Param | Type | Description | +|---|---|---| +|`ui.Key` | _string_ | Name of the object being uploaded | +|`ui.UploadID` | _string_ | UploadID of the ongoing multipart upload | +|`ui.HealUploadInfo.Status` | _healStatus_| One of `Healthy`, `CanHeal`, `Corrupted`, `QuorumUnavailable`| +|`ui.Err`| _error_ | non-nil if fetching fetching healing information failed | + +__Example__ + +``` go + + // Set true if recursive listing is needed. + isRecursive := true + // List objects that need healing for a given bucket and + // prefix. + healUploadsCh, err := madmClnt.ListUploadsHeal(bucket, prefix, isRecursive, doneCh) + if err != nil { + log.Fatalln("Failed to get list of uploads to be healed: ", err) + } + + for upload := range healUploadsCh { + if upload.Err != nil { + log.Println("upload listing error: ", upload.Err) + } + + if upload.HealUploadInfo != nil { + switch healInfo := *upload.HealUploadInfo; healInfo.Status { + case madmin.CanHeal: + fmt.Println(upload.Key, " can be healed.") + case madmin.QuorumUnavailable: + fmt.Println(upload.Key, " can't be healed until quorum is available.") + case madmin.Corrupted: + fmt.Println(upload.Key, " can't be healed, not enough information.") + } + } + } + +``` + + +### HealUpload(bucket, object, uploadID string, isDryRun bool) error +If upload is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the upload is not healed, but heal upload request is validated by the server. e.g, if the upload exists, if upload name is valid etc. + +``` go + isDryRun = false + err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun) + if err != nil { + log.Fatalln(err) + } + + log.Println("successfully healed mybucket/myobject/myuploadID") +``` ## 5. Config operations @@ -354,46 +411,3 @@ __Example__ } log.Println("SetConfig: ", string(buf.Bytes())) ``` - - -### ListUploadsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan UploadInfo, error) -List ongoing multipart uploads that need healing. - -| Param | Type | Description | -|---|---|---| -|`ui.Key` | _string_ | Name of the object being uploaded | -|`ui.UploadID` | _string_ | UploadID of the ongoing multipart upload | -|`ui.HealUploadInfo.Status` | _healStatus_| One of `Healthy`, `CanHeal`, `Corrupted`, `QuorumUnavailable`| -|`ui.Err`| _error_ | non-nil if fetching fetching healing information failed | - -__Example__ - -``` go - - // Set true if recursive listing is needed. - isRecursive := true - // List objects that need healing for a given bucket and - // prefix. - healUploadsCh, err := madmClnt.ListUploadsHeal(bucket, prefix, isRecursive, doneCh) - if err != nil { - log.Fatalln("Failed to get list of uploads to be healed: ", err) - } - - for upload := range healUploadsCh { - if upload.Err != nil { - log.Println("upload listing error: ", upload.Err) - } - - if upload.HealUploadInfo != nil { - switch healInfo := *upload.HealUploadInfo; healInfo.Status { - case madmin.CanHeal: - fmt.Println(upload.Key, " can be healed.") - case madmin.QuorumUnavailable: - fmt.Println(upload.Key, " can't be healed until quorum is available.") - case madmin.Corrupted: - fmt.Println(upload.Key, " can't be healed, not enough information.") - } - } - } - -``` diff --git a/pkg/madmin/examples/heal-upload.go b/pkg/madmin/examples/heal-upload.go new file mode 100644 index 000000000..1b6f3f502 --- /dev/null +++ b/pkg/madmin/examples/heal-upload.go @@ -0,0 +1,57 @@ +// +build ignore + +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "log" + + "github.com/minio/minio/pkg/madmin" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise. + // New returns an Minio Admin client object. + madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Heal upload mybucket/myobject/uploadID - dry run. + isDryRun := true + err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun) + if err != nil { + log.Fatalln(err) + } + + // Heal upload mybucket/myobject/uploadID - this time for real. + isDryRun = false + err = madmClnt.HealUpload("mybucket", "myobject", "myuploadID", isDryRun) + if err != nil { + log.Fatalln(err) + } + + log.Println("successfully healed mybucket/myobject/myuploadID") +} diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go index 86042157c..3cda00b77 100644 --- a/pkg/madmin/heal-commands.go +++ b/pkg/madmin/heal-commands.go @@ -231,6 +231,7 @@ const ( healDryRun healQueryKey = "dry-run" healUploadIDMarker healQueryKey = "upload-id-marker" healMaxUpload healQueryKey = "max-uploads" + healUploadID healQueryKey = "upload-id" ) // mkHealQueryVal - helper function to construct heal REST API query params. @@ -432,6 +433,43 @@ func (adm *AdminClient) HealBucket(bucket string, dryrun bool) error { return nil } +// HealUpload - Heal the given upload. +func (adm *AdminClient) HealUpload(bucket, object, uploadID string, dryrun bool) error { + // Construct query params. + queryVal := url.Values{} + queryVal.Set("heal", "") + queryVal.Set(string(healBucket), bucket) + queryVal.Set(string(healObject), object) + queryVal.Set(string(healUploadID), uploadID) + if dryrun { + queryVal.Set(string(healDryRun), "") + } + + hdrs := make(http.Header) + hdrs.Set(minioAdminOpHeader, "upload") + + reqData := requestData{ + queryValues: queryVal, + customHeaders: hdrs, + } + + // Execute POST on + // /?heal&bucket=mybucket&object=myobject&upload-id=uploadID + // to heal an upload. + resp, err := adm.executeMethod("POST", reqData) + + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp) + } + + return nil +} + // HealObject - Heal the given object. func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error { // Construct query params.