From a337ea4d11eb2ae87144a76cdefd6f77ae7303a4 Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Mon, 22 Jan 2018 14:54:55 -0800 Subject: [PATCH] Move admin APIs to new path and add redesigned heal APIs (#5351) - Changes related to moving admin APIs - admin APIs now have an endpoint under /minio/admin - admin APIs are now versioned - a new API to server the version is added at "GET /minio/admin/version" and all API operations have the path prefix /minio/admin/v1/ - new service stop API added - credentials change API is moved to /minio/admin/v1/config/credential - credentials change API and configuration get/set API now require TLS so that credentials are protected - all API requests now receive JSON - heal APIs are disabled as they will be changed substantially - Heal API changes Heal API is now provided at a single endpoint with the ability for a client to start a heal sequence on all the data in the server, a single bucket, or under a prefix within a bucket. When a heal sequence is started, the server returns a unique token that needs to be used for subsequent 'status' requests to fetch heal results. On each status request from the client, the server returns heal result records that it has accumulated since the previous status request. The server accumulates upto 1000 records and pauses healing further objects until the client requests for status. If the client does not request any further records for a long time, the server aborts the heal sequence automatically. A heal result record is returned for each entity healed on the server, such as system metadata, object metadata, buckets and objects, and has information about the before and after states on each disk. A client may request to force restart a heal sequence - this causes the running heal sequence to be aborted at the next safe spot and starts a new heal sequence. --- cmd/admin-handlers.go | 765 ++++++++++--------------- cmd/admin-handlers_test.go | 831 +++++++++++----------------- cmd/admin-heal-ops.go | 683 +++++++++++++++++++++++ cmd/admin-router.go | 54 +- cmd/admin-rpc-client.go | 50 +- cmd/admin-rpc-server.go | 24 +- cmd/admin-rpc-server_test.go | 32 +- cmd/api-errors.go | 56 +- cmd/api-headers.go | 9 + cmd/api-response.go | 53 +- cmd/endpoint.go | 9 + cmd/format-xl.go | 64 +-- cmd/format-xl_test.go | 16 +- cmd/fs-v1.go | 13 +- cmd/fs-v1_test.go | 2 +- cmd/gateway-unsupported.go | 13 +- cmd/generic-handlers.go | 19 +- cmd/object-api-datatypes.go | 30 +- cmd/object-api-input-checks.go | 3 - cmd/object-api-interface.go | 5 +- cmd/routers.go | 6 +- cmd/server-main.go | 3 + cmd/test-utils_test.go | 13 + cmd/xl-v1-healing-common.go | 164 +----- cmd/xl-v1-healing-common_test.go | 72 +-- cmd/xl-v1-healing.go | 491 +++++++++------- cmd/xl-v1-healing_test.go | 39 +- cmd/xl-v1-list-objects-heal.go | 32 +- cmd/xl-v1-list-objects-heal_test.go | 18 +- cmd/xl-v1-object_test.go | 6 +- cmd/xl-v1-utils.go | 28 +- pkg/madmin/API.md | 214 +++---- pkg/madmin/api-error-response.go | 6 +- pkg/madmin/api.go | 15 +- pkg/madmin/config-commands.go | 58 +- pkg/madmin/constants.go | 3 - pkg/madmin/generic-commands.go | 48 +- pkg/madmin/heal-commands.go | 509 ++++------------- pkg/madmin/info-commands.go | 9 +- pkg/madmin/lock-commands.go | 42 +- pkg/madmin/service-commands.go | 86 +-- pkg/madmin/utils.go | 8 +- pkg/madmin/version-commands.go | 54 ++ 43 files changed, 2375 insertions(+), 2280 deletions(-) create mode 100644 cmd/admin-heal-ops.go create mode 100644 pkg/madmin/version-commands.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index b51b885ec..42474e723 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -19,21 +19,19 @@ package cmd import ( "bytes" "encoding/json" - "encoding/xml" "fmt" "io" - "io/ioutil" "net/http" "net/url" - "strconv" "sync" "time" + "github.com/gorilla/mux" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/madmin" ) const ( - minioAdminOpHeader = "X-Minio-Operation" minioConfigTmpFormat = "config-%s.json" maxConfigJSONSize = 256 * 1024 // 256KiB @@ -44,54 +42,67 @@ type mgmtQueryKey string // Only valid query params for mgmt admin APIs. const ( - mgmtBucket mgmtQueryKey = "bucket" - mgmtObject mgmtQueryKey = "object" - mgmtPrefix mgmtQueryKey = "prefix" - mgmtLockDuration mgmtQueryKey = "duration" - mgmtDelimiter mgmtQueryKey = "delimiter" - mgmtMarker mgmtQueryKey = "marker" - mgmtMaxKey mgmtQueryKey = "max-key" - mgmtDryRun mgmtQueryKey = "dry-run" + mgmtBucket mgmtQueryKey = "bucket" + mgmtPrefix mgmtQueryKey = "prefix" + mgmtLockOlderThan mgmtQueryKey = "older-than" + mgmtClientToken mgmtQueryKey = "clientToken" + mgmtForceStart mgmtQueryKey = "forceStart" ) -// ServerVersion - server version -type ServerVersion struct { - Version string `json:"version"` - CommitID string `json:"commitID"` -} +var ( + // This struct literal represents the Admin API version that + // the server uses. + adminAPIVersionInfo = madmin.AdminAPIVersionInfo{"1"} +) + +// VersionHandler - GET /minio/admin/version +// ----------- +// Returns Administration API version +func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) { + + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, adminAPIErr, r.URL) + return + } + + jsonBytes, err := json.Marshal(adminAPIVersionInfo) + if err != nil { + writeErrorResponseJSON(w, ErrInternalError, r.URL) + errorIf(err, "Failed to marshal Admin API Version to JSON.") + return + } -// ServerStatus - contains the response of service status API -type ServerStatus struct { - ServerVersion ServerVersion `json:"serverVersion"` - Uptime time.Duration `json:"uptime"` + writeSuccessResponseJSON(w, jsonBytes) } -// ServiceStatusHandler - GET /?service -// HTTP header x-minio-operation: status +// ServiceStatusHandler - GET /minio/admin/v1/service // ---------- -// Fetches server status information like total disk space available -// to use, online disks, offline disks and quorum threshold. -func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) { +// Returns server version and uptime. +func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) { adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } // Fetch server version - serverVersion := ServerVersion{Version: Version, CommitID: CommitID} + serverVersion := madmin.ServerVersion{ + Version: Version, + CommitID: CommitID, + } // Fetch uptimes from all peers. This may fail to due to lack // of read-quorum availability. uptime, err := getPeerUptimes(globalAdminPeers) if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) + writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL) errorIf(err, "Possibly failed to get uptime from majority of servers.") return } // Create API response - serverStatus := ServerStatus{ + serverStatus := madmin.ServiceStatus{ ServerVersion: serverVersion, Uptime: uptime, } @@ -99,7 +110,7 @@ func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r * // Marshal API response jsonBytes, err := json.Marshal(serverStatus) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to marshal storage info into json.") return } @@ -108,93 +119,42 @@ func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r * writeSuccessResponseJSON(w, jsonBytes) } -// ServiceRestartHandler - POST /?service -// HTTP header x-minio-operation: restart +// ServiceStopNRestartHandler - POST /minio/admin/v1/service +// Body: {"action": } // ---------- -// Restarts minio server gracefully. In a distributed setup, restarts -// all the servers in the cluster. -func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r *http.Request) { +// Restarts/Stops minio server gracefully. In a distributed setup, +// restarts all the servers in the cluster. +func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) { adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Reply to the client before restarting minio server. - writeSuccessResponseHeadersOnly(w) - - sendServiceCmd(globalAdminPeers, serviceRestart) -} - -// setCredsReq request -type setCredsReq struct { - Username string `xml:"username"` - Password string `xml:"password"` -} - -// ServiceCredsHandler - POST /?service -// HTTP header x-minio-operation: creds -// ---------- -// Update credentials in a minio server. In a distributed setup, update all the servers -// in the cluster. -func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter, r *http.Request) { - // Authenticate request - adminAPIErr := checkAdminRequestAuthType(r, "") - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Avoid setting new credentials when they are already passed - // by the environment. - if globalIsEnvCreds { - writeErrorResponse(w, ErrMethodNotAllowed, r.URL) - return - } - - // Load request body - inputData, err := ioutil.ReadAll(r.Body) - if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } - // Unmarshal request body - var req setCredsReq - err = xml.Unmarshal(inputData, &req) + var sa madmin.ServiceAction + err := json.NewDecoder(r.Body).Decode(&sa) if err != nil { - errorIf(err, "Cannot unmarshal credentials request") - writeErrorResponse(w, ErrMalformedXML, r.URL) + errorIf(err, "Error parsing body JSON") + writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL) return } - creds, err := auth.CreateCredentials(req.Username, req.Password) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) + var serviceSig serviceSignal + switch sa.Action { + case madmin.ServiceActionValueRestart: + serviceSig = serviceRestart + case madmin.ServiceActionValueStop: + serviceSig = serviceStop + default: + writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL) + errorIf(err, "Invalid service action received") return } - // Notify all other Minio peers to update credentials - updateErrs := updateCredsOnPeers(creds) - for peer, err := range updateErrs { - errorIf(err, "Unable to update credentials on peer %s.", peer) - } - - // Update local credentials in memory. - prevCred := globalServerConfig.SetCredential(creds) - - // Save credentials to config file - if err = globalServerConfig.Save(); err != nil { - // Save the current creds when failed to update. - globalServerConfig.SetCredential(prevCred) - - errorIf(err, "Unable to update the config with new credentials.") - writeErrorResponse(w, ErrInternalError, r.URL) - return - } + // Reply to the client before restarting minio server. + writeSuccessResponseHeadersOnly(w) - // At this stage, the operation is successful, return 200 OK - w.WriteHeader(http.StatusOK) + sendServiceCmd(globalAdminPeers, serviceSig) } // ServerProperties holds some server information such as, version, region @@ -252,14 +212,14 @@ type ServerInfo struct { Data *ServerInfoData `json:"data"` } -// ServerInfoHandler - GET /?info +// ServerInfoHandler - GET /minio/admin/v1/info // ---------- // Get server information -func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) { +func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) { // Authenticate request adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } @@ -295,7 +255,7 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt // Marshal API response jsonBytes, err := json.Marshal(reply) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to marshal storage info into json.") return } @@ -305,11 +265,14 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt writeSuccessResponseJSON(w, jsonBytes) } -// validateLockQueryParams - Validates query params for list/clear locks management APIs. -func validateLockQueryParams(vars url.Values) (string, string, time.Duration, APIErrorCode) { +// validateLockQueryParams - Validates query params for list/clear +// locks management APIs. +func validateLockQueryParams(vars url.Values) (string, string, time.Duration, + APIErrorCode) { + bucket := vars.Get(string(mgmtBucket)) prefix := vars.Get(string(mgmtPrefix)) - durationStr := vars.Get(string(mgmtLockDuration)) + olderThanStr := vars.Get(string(mgmtLockOlderThan)) // N B empty bucket name is invalid if !IsValidBucketName(bucket) { @@ -322,10 +285,10 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration, AP // If older-than parameter was empty then set it to 0s to list // all locks older than now. - if durationStr == "" { - durationStr = "0s" + if olderThanStr == "" { + olderThanStr = "0s" } - duration, err := time.ParseDuration(durationStr) + duration, err := time.ParseDuration(olderThanStr) if err != nil { errorIf(err, "Failed to parse duration passed as query value.") return "", "", time.Duration(0), ErrInvalidDuration @@ -334,31 +297,32 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration, AP return bucket, prefix, duration, ErrNone } -// ListLocksHandler - GET /?lock&bucket=mybucket&prefix=myprefix&duration=duration +// ListLocksHandler - GET /minio/admin/v1/locks?bucket=mybucket&prefix=myprefix&older-than=10s // - bucket is a mandatory query parameter // - prefix and older-than are optional query parameters -// HTTP header x-minio-operation: list // --------- // Lists locks held on a given bucket, prefix and duration it was held for. -func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) { +func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) { + adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } vars := r.URL.Query() bucket, prefix, duration, adminAPIErr := validateLockQueryParams(vars) if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } // Fetch lock information of locks matching bucket/prefix that // are available for longer than duration. - volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, duration) + volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, + duration) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to fetch lock information from remote nodes.") return } @@ -366,7 +330,7 @@ func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http // Marshal list of locks as json. jsonBytes, err := json.Marshal(volLocks) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to marshal lock information into json.") return } @@ -376,31 +340,32 @@ func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http writeSuccessResponseJSON(w, jsonBytes) } -// ClearLocksHandler - POST /?lock&bucket=mybucket&prefix=myprefix&duration=duration +// ClearLocksHandler - DELETE /minio/admin/v1/locks?bucket=mybucket&prefix=myprefix&duration=duration // - bucket is a mandatory query parameter // - prefix and older-than are optional query parameters -// HTTP header x-minio-operation: clear // --------- // Clear locks held on a given bucket, prefix and duration it was held for. -func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) { - adminAPIErr := checkAdminRequestAuthType(r, "") +func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) { + + adminAPIErr := checkRequestAuthType(r, "", "", "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } vars := r.URL.Query() bucket, prefix, duration, adminAPIErr := validateLockQueryParams(vars) if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } // Fetch lock information of locks matching bucket/prefix that // are held for longer than duration. - volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, duration) + volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix, + duration) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to fetch lock information from remote nodes.") return } @@ -408,7 +373,7 @@ func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *htt // Marshal list of locks as json. jsonBytes, err := json.Marshal(volLocks) if err != nil { - writeErrorResponse(w, ErrInternalError, r.URL) + writeErrorResponseJSON(w, ErrInternalError, r.URL) errorIf(err, "Failed to marshal lock information into json.") return } @@ -418,378 +383,204 @@ func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *htt writeSuccessResponseJSON(w, jsonBytes) } -// extractListObjectsHealQuery - Validates query params for heal objects list management API. -func extractListObjectsHealQuery(vars url.Values) (string, string, string, string, int, APIErrorCode) { - bucket := vars.Get(string(mgmtBucket)) - prefix := vars.Get(string(mgmtPrefix)) - marker := vars.Get(string(mgmtMarker)) - delimiter := vars.Get(string(mgmtDelimiter)) - maxKeyStr := vars.Get(string(mgmtMaxKey)) - - // N B empty bucket name is invalid - if !IsValidBucketName(bucket) { - return "", "", "", "", 0, ErrInvalidBucketName - } +// extractHealInitParams - Validates params for heal init API. +func extractHealInitParams(r *http.Request) (bucket, objPrefix string, + hs madmin.HealOpts, clientToken string, forceStart bool, + err APIErrorCode) { - // empty prefix is valid. - if !IsValidObjectPrefix(prefix) { - return "", "", "", "", 0, ErrInvalidObjectName - } + vars := mux.Vars(r) + bucket = vars[string(mgmtBucket)] + objPrefix = vars[string(mgmtPrefix)] - // check if maxKey is a valid integer, if present. - var maxKey int - var err error - if maxKeyStr != "" { - if maxKey, err = strconv.Atoi(maxKeyStr); err != nil { - return "", "", "", "", 0, ErrInvalidMaxKeys + if bucket == "" { + if objPrefix != "" { + // Bucket is required if object-prefix is given + err = ErrHealMissingBucket + return } - } - - // Validate prefix, marker, delimiter and maxKey. - apiErr := validateListObjectsArgs(prefix, marker, delimiter, maxKey) - if apiErr != ErrNone { - return "", "", "", "", 0, apiErr - } - - return bucket, prefix, marker, delimiter, maxKey, ErrNone -} - -// ListObjectsHealHandler - GET /?heal&bucket=mybucket&prefix=myprefix&marker=mymarker&delimiter=&mydelimiter&maxKey=1000 -// - bucket is mandatory query parameter -// - rest are optional query parameters -// List upto maxKey objects that need healing in a given bucket matching the given prefix. -func (adminAPI adminAPIHandlers) ListObjectsHealHandler(w http.ResponseWriter, r *http.Request) { - // Get object layer instance. - objLayer := newObjectLayerFn() - if objLayer == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) - return - } - - // Validate request signature. - adminAPIErr := checkAdminRequestAuthType(r, "") - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Check if this setup has an erasure coded backend. - if !globalIsXL { - writeErrorResponse(w, ErrHealNotImplemented, r.URL) - return - } - - // Validate query params. - vars := r.URL.Query() - bucket, prefix, marker, delimiter, maxKey, adminAPIErr := extractListObjectsHealQuery(vars) - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Get the list objects to be healed. - objectInfos, err := objLayer.ListObjectsHeal(bucket, prefix, marker, delimiter, maxKey) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - listResponse := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKey, objectInfos) - // Write success response. - writeSuccessResponseXML(w, encodeResponse(listResponse)) -} - -// ListBucketsHealHandler - GET /?heal -func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r *http.Request) { - // Get object layer instance. - objLayer := newObjectLayerFn() - if objLayer == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) - return - } - - // Validate request signature. - adminAPIErr := checkAdminRequestAuthType(r, "") - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Check if this setup has an erasure coded backend. - if !globalIsXL { - writeErrorResponse(w, ErrHealNotImplemented, r.URL) - return - } - - // Get the list buckets to be healed. - bucketsInfo, err := objLayer.ListBucketsHeal() - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - listResponse := generateListBucketsResponse(bucketsInfo) - // Write success response. - writeSuccessResponseXML(w, encodeResponse(listResponse)) -} - -// HealBucketHandler - POST /?heal&bucket=mybucket&dry-run -// - x-minio-operation = bucket -// - bucket is mandatory query parameter -// Heal a given bucket, if present. -func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) { - // Get object layer instance. - objLayer := newObjectLayerFn() - if objLayer == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) + } else if !IsValidBucketName(bucket) { + err = ErrInvalidBucketName return } - // Validate request signature. - adminAPIErr := checkAdminRequestAuthType(r, "") - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Check if this setup has an erasure coded backend. - if !globalIsXL { - writeErrorResponse(w, ErrHealNotImplemented, r.URL) + // empty prefix is valid. + if !IsValidObjectPrefix(objPrefix) { + err = ErrInvalidObjectName return } - // Validate bucket name and check if it exists. - vars := r.URL.Query() - bucket := vars.Get(string(mgmtBucket)) - if err := checkBucketExist(bucket, objLayer); err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return + qParms := r.URL.Query() + if len(qParms[string(mgmtClientToken)]) > 0 { + clientToken = qParms[string(mgmtClientToken)][0] } - - // if dry-run is present in query-params, then only perform validations and return success. - if isDryRun(vars) { - writeSuccessResponseHeadersOnly(w) - return + if _, ok := qParms[string(mgmtForceStart)]; ok { + forceStart = true } - // Heal the given bucket. - err := objLayer.HealBucket(bucket) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - // Return 200 on success. - writeSuccessResponseHeadersOnly(w) -} - -// isDryRun - returns true if dry-run query param was set and false otherwise. -// otherwise. -func isDryRun(qval url.Values) bool { - if _, dryRun := qval[string(mgmtDryRun)]; dryRun { - return true - } - return false -} - -// healResult - represents result of a heal operation like -// heal-object, heal-upload. -type healResult struct { - State healState `json:"state"` -} - -// healState - different states of heal operation -type healState int - -const ( - // healNone - none of the disks healed - healNone healState = iota - // healPartial - some disks were healed, others were offline - healPartial - // healOK - all disks were healed - healOK -) - -// newHealResult - returns healResult given number of disks healed and -// number of disks offline -func newHealResult(numHealedDisks, numOfflineDisks int) healResult { - var state healState - switch { - case numHealedDisks == 0: - state = healNone - - case numOfflineDisks > 0: - state = healPartial - - default: - state = healOK + // ignore body if clientToken is provided + if clientToken == "" { + jerr := json.NewDecoder(r.Body).Decode(&hs) + if jerr != nil { + errorIf(jerr, "Error parsing body JSON") + err = ErrRequestBodyParse + return + } } - return healResult{State: state} + err = ErrNone + return } -// HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject&dry-run -// - x-minio-operation = object -// - bucket and object are both mandatory query parameters -// Heal a given object, if present. -func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) { +// HealHandler - POST /minio/admin/v1/heal/ +// ----------- +// Start heal processing and return heal status items. +// +// On a successful heal sequence start, a unique client token is +// returned. Subsequent requests to this endpoint providing the client +// token will receive heal status records from the running heal +// sequence. +// +// If no client token is provided, and a heal sequence is in progress +// an error is returned with information about the running heal +// sequence. However, if the force-start flag is provided, the server +// aborts the running heal sequence and starts a new one. +func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) { // Get object layer instance. objLayer := newObjectLayerFn() if objLayer == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) + writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL) return } // Validate request signature. adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } // Check if this setup has an erasure coded backend. if !globalIsXL { - writeErrorResponse(w, ErrHealNotImplemented, r.URL) - return - } - - vars := r.URL.Query() - bucket := vars.Get(string(mgmtBucket)) - object := vars.Get(string(mgmtObject)) - - // Validate bucket and object names. - if err := checkBucketAndObjectNames(bucket, object); err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - // Check if object exists. - if _, err := objLayer.GetObjectInfo(bucket, object); err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - // if dry-run is set in query params then perform validations - // and return success. - if isDryRun(vars) { - writeSuccessResponseHeadersOnly(w) - return - } - - numOfflineDisks, numHealedDisks, err := objLayer.HealObject(bucket, object) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks)) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - // Return 200 on success. - writeSuccessResponseJSON(w, jsonBytes) -} - -// HealFormatHandler - POST /?heal&dry-run -// - x-minio-operation = format -// - bucket and object are both mandatory query parameters -// Heal a given object, if present. -func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) { - // Get current object layer instance. - objectAPI := newObjectLayerFn() - if objectAPI == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) - return - } - - // Validate request signature. - adminAPIErr := checkAdminRequestAuthType(r, "") - if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) - return - } - - // Check if this setup is an erasure code backend, since - // heal-format is only applicable to single node XL and - // distributed XL setup. - if !globalIsXL { - writeErrorResponse(w, ErrHealNotImplemented, r.URL) - return - } - - // if dry-run is set in query-params, return success as - // validations are successful so far. - vars := r.URL.Query() - if isDryRun(vars) { - writeSuccessResponseHeadersOnly(w) + writeErrorResponseJSON(w, ErrHealNotImplemented, r.URL) return } - // Create a new set of storage instances to heal format.json. - bootstrapDisks, err := initStorageDisks(globalEndpoints) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return - } - - // Wrap into retrying disks - retryingDisks := initRetryableStorageDisks(bootstrapDisks, - time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold) - - // Heal format.json on available storage. - err = healFormatXL(retryingDisks) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) + bucket, objPrefix, hs, clientToken, forceStart, apiErr := extractHealInitParams(r) + if apiErr != ErrNone { + writeErrorResponseJSON(w, apiErr, r.URL) return } - // Instantiate new object layer with newly formatted storage. - newObjectAPI, err := newXLObjects(retryingDisks) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - return + // Helper function to fetch client address - we use the + // X-forwarded-for header if one is present. + getClientAddress := func() string { + addr := r.RemoteAddr + fwdFor := r.Header.Get("X-Forwarded-For") + if fwdFor == "" { + return addr + } + return fwdFor + } + + type healResp struct { + respBytes []byte + errCode APIErrorCode + errBody string + } + + // Define a closure to start sending whitespace to client + // after 10s unless a response item comes in + keepConnLive := func(w http.ResponseWriter, respCh chan healResp) { + ticker := time.NewTicker(time.Second * 10) + started := false + forLoop: + for { + select { + case <-ticker.C: + if !started { + // Start writing response to client + started = true + setCommonHeaders(w) + w.Header().Set("Content-Type", string(mimeJSON)) + // Set 200 OK status + w.WriteHeader(200) + } + // Send whitespace and keep connection open + w.Write([]byte("\n\r")) + w.(http.Flusher).Flush() + case hr := <-respCh: + switch { + case hr.errCode == ErrNone: + writeSuccessResponseJSON(w, hr.respBytes) + case hr.errBody == "": + writeErrorResponseJSON(w, hr.errCode, r.URL) + default: + writeCustomErrorResponseJSON(w, hr.errCode, hr.errBody, r.URL) + } + break forLoop + } + } + ticker.Stop() + } + + // find number of disks in the setup + info := objLayer.StorageInfo() + numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks + + if clientToken == "" { + // Not a status request + nh := newHealSequence(bucket, objPrefix, getClientAddress(), + numDisks, hs, forceStart) + + respCh := make(chan healResp) + go func() { + respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh) + hr := healResp{respBytes, errCode, errMsg} + respCh <- hr + }() + + // Due to the force-starting functionality, the Launch + // call above can take a long time - to keep the + // connection alive, we start sending whitespace + keepConnLive(w, respCh) + } else { + // Since clientToken is given, fetch heal status from running + // heal sequence. + path := bucket + "/" + objPrefix + respBytes, errCode := globalAllHealState.PopHealStatusJSON( + path, clientToken) + if errCode != ErrNone { + writeErrorResponseJSON(w, errCode, r.URL) + } else { + writeSuccessResponseJSON(w, respBytes) + } } - - // Set object layer with newly formatted storage to globalObjectAPI. - globalObjLayerMutex.Lock() - globalObjectAPI = newObjectAPI - globalObjLayerMutex.Unlock() - - // Shutdown storage belonging to old object layer instance. - objectAPI.Shutdown() - - // Inform peers to reinitialize storage with newly formatted storage. - reInitPeerDisks(globalAdminPeers) - - // Return 200 on success. - writeSuccessResponseHeadersOnly(w) + return } -// GetConfigHandler - GET /?config -// - x-minio-operation = get +// GetConfigHandler - GET /minio/admin/v1/config // Get config.json of this minio setup. -func (adminAPI adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) { +func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) { + // Validate request signature. adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } // check if objectLayer is initialized, if not return. if newObjectLayerFn() == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) + writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL) return } - // Get config.json from all nodes. In a single node setup, it - // returns local config.json. + // Get config.json - in distributed mode, the configuration + // occurring on a quorum of the servers is returned. configBytes, err := getPeerConfig(globalAdminPeers) if err != nil { errorIf(err, "Failed to get config from peers") - writeErrorResponse(w, toAdminAPIErrCode(err), r.URL) + writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL) return } @@ -819,8 +610,11 @@ type setConfigResult struct { Status bool `json:"status"` } -// writeSetConfigResponse - writes setConfigResult value as json depending on the status. -func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers, errs []error, status bool, reqURL *url.URL) { +// writeSetConfigResponse - writes setConfigResult value as json +// depending on the status. +func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers, + errs []error, status bool, reqURL *url.URL) { + var nodeResults []nodeSummary // Build nodeResults based on error values received during // set-config operation. @@ -846,7 +640,7 @@ func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers, errs []erro enc.SetEscapeHTML(false) jsonErr := enc.Encode(result) if jsonErr != nil { - writeErrorResponse(w, toAPIErrorCode(jsonErr), reqURL) + writeErrorResponseJSON(w, toAPIErrorCode(jsonErr), reqURL) return } @@ -854,20 +648,20 @@ func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers, errs []erro return } -// SetConfigHandler - PUT /?config -// - x-minio-operation = set -func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) { +// SetConfigHandler - PUT /minio/admin/v1/config +func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) { + // Get current object layer instance. objectAPI := newObjectLayerFn() if objectAPI == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) + writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL) return } // Validate request signature. adminAPIErr := checkAdminRequestAuthType(r, "") if adminAPIErr != ErrNone { - writeErrorResponse(w, adminAPIErr, r.URL) + writeErrorResponseJSON(w, adminAPIErr, r.URL) return } @@ -909,7 +703,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http creds := globalServerConfig.GetCredential() if config.Credential.AccessKey != creds.AccessKey || config.Credential.SecretKey != creds.SecretKey { - writeErrorResponse(w, ErrAdminCredentialsMismatch, r.URL) + writeErrorResponseJSON(w, ErrAdminCredentialsMismatch, r.URL) return } } @@ -931,7 +725,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http // operations. configLock := globalNSMutex.NewNSLock(minioReservedBucket, minioConfigFile) if configLock.GetLock(globalObjectTimeout) != nil { - writeErrorResponse(w, ErrOperationTimedOut, r.URL) + writeErrorResponseJSON(w, ErrOperationTimedOut, r.URL) return } defer configLock.Unlock() @@ -953,3 +747,56 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http // Restart all node for the modified config to take effect. sendServiceCmd(globalAdminPeers, serviceRestart) } + +// ConfigCredsHandler - POST /minio/admin/v1/config/credential +// ---------- +// Update credentials in a minio server. In a distributed setup, +// update all the servers in the cluster. +func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter, + r *http.Request) { + + // Authenticate request + adminAPIErr := checkAdminRequestAuthType(r, "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, adminAPIErr, r.URL) + return + } + + // Avoid setting new credentials when they are already passed + // by the environment. + if globalIsEnvCreds { + writeErrorResponse(w, ErrMethodNotAllowed, r.URL) + return + } + + // Decode request body + var req madmin.SetCredsReq + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + errorIf(err, "Error parsing body JSON") + writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL) + return + } + + creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey) + if err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Notify all other Minio peers to update credentials + updateErrs := updateCredsOnPeers(creds) + for peer, err := range updateErrs { + errorIf(err, "Unable to update credentials on peer %s.", peer) + } + + // Update local credentials in memory. + globalServerConfig.SetCredential(creds) + if err = globalServerConfig.Save(); err != nil { + writeErrorResponse(w, ErrInternalError, r.URL) + return + } + + // At this stage, the operation is successful, return 200 OK + w.WriteHeader(http.StatusOK) +} diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 7abff1dbb..7a8010817 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -19,7 +19,6 @@ package cmd import ( "bytes" "encoding/json" - "encoding/xml" "fmt" "io" "io/ioutil" @@ -29,10 +28,12 @@ import ( "os" "strings" "testing" + "time" router "github.com/gorilla/mux" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/pkg/madmin" ) var ( @@ -164,13 +165,17 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) { globalEndpoints = mustGetNewEndpointList(xlDirs...) - // Set globalIsXL to indicate that the setup uses an erasure code backend. + // Set globalIsXL to indicate that the setup uses an erasure + // code backend. globalIsXL = true // initialize NSLock. isDistXL := false initNSLock(isDistXL) + // Init global heal state + initAllHealState(globalIsXL) + // Setup admin mgmt REST API handlers. adminRouter := router.NewRouter() registerAdminRouter(adminRouter) @@ -191,6 +196,60 @@ func (atb *adminXLTestBed) TearDown() { resetTestGlobals() } +func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) { + // Create an object myobject under bucket mybucket. + bucketName := "mybucket" + err := atb.objLayer.MakeBucketWithLocation(bucketName, "") + if err != nil { + t.Fatalf("Failed to make bucket %s - %v", bucketName, + err) + } + + // create some objects + { + objName := "myobject" + for i := 0; i < 10; i++ { + objectName := fmt.Sprintf("%s-%d", objName, i) + _, err = atb.objLayer.PutObject(bucketName, objectName, + mustGetHashReader(t, bytes.NewReader([]byte("hello")), + int64(len("hello")), "", ""), nil) + if err != nil { + t.Fatalf("Failed to create %s - %v", objectName, + err) + } + } + } + + // create a multipart upload (incomplete) + { + objName := "mpObject" + uploadID, err := atb.objLayer.NewMultipartUpload(bucketName, + objName, nil) + if err != nil { + t.Fatalf("mp new error: %v", err) + } + + _, err = atb.objLayer.PutObjectPart(bucketName, objName, + uploadID, 3, mustGetHashReader(t, bytes.NewReader( + []byte("hello")), int64(len("hello")), "", "")) + if err != nil { + t.Fatalf("mp put error: %v", err) + } + + } +} + +func (atb *adminXLTestBed) CleanupHealTestData(t *testing.T) { + bucketName := "mybucket" + objName := "myobject" + for i := 0; i < 10; i++ { + atb.objLayer.DeleteObject(bucketName, + fmt.Sprintf("%s-%d", objName, i)) + } + + atb.objLayer.DeleteBucket(bucketName) +} + // initTestObjLayer - Helper function to initialize an XL-based object // layer and set globalObjectAPI. func initTestXLObjLayer() (ObjectLayer, []string, error) { @@ -205,6 +264,41 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) { return objLayer, xlDirs, nil } +func TestAdminVersionHandler(t *testing.T) { + adminTestBed, err := prepareAdminXLTestBed() + if err != nil { + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") + } + defer adminTestBed.TearDown() + + req, err := newTestRequest("GET", "/minio/admin/version", 0, nil) + if err != nil { + t.Fatalf("Failed to construct request - %v", err) + } + cred := globalServerConfig.GetCredential() + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + t.Fatalf("Failed to sign request - %v", err) + } + + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if http.StatusOK != rec.Code { + t.Errorf("Unexpected status code - got %d but expected %d", + rec.Code, http.StatusOK) + } + + var result madmin.AdminAPIVersionInfo + err = json.NewDecoder(rec.Body).Decode(&result) + if err != nil { + t.Errorf("json parse err: %v", err) + } + + if result != adminAPIVersionInfo { + t.Errorf("unexpected version: %v", result) + } +} + // cmdType - Represents different service subcomands like status, stop // and restart. type cmdType int @@ -212,6 +306,7 @@ type cmdType int const ( statusCmd cmdType = iota restartCmd + stopCmd setCreds ) @@ -222,6 +317,8 @@ func (c cmdType) String() string { return "status" case restartCmd: return "restart" + case stopCmd: + return "stop" case setCreds: return "set-credentials" } @@ -236,12 +333,26 @@ func (c cmdType) apiMethod() string { return "GET" case restartCmd: return "POST" - case setCreds: + case stopCmd: return "POST" + case setCreds: + return "PUT" } return "GET" } +// apiEndpoint - Return endpoint for each admin REST API mapped to a +// command here. +func (c cmdType) apiEndpoint() string { + switch c { + case statusCmd, restartCmd, stopCmd: + return "/minio/admin/v1/service" + case setCreds: + return "/minio/admin/v1/config/credential" + } + return "" +} + // toServiceSignal - Helper function that translates a given cmdType // value to its corresponding serviceSignal value. func (c cmdType) toServiceSignal() serviceSignal { @@ -250,10 +361,22 @@ func (c cmdType) toServiceSignal() serviceSignal { return serviceStatus case restartCmd: return serviceRestart + case stopCmd: + return serviceStop } return serviceStatus } +func (c cmdType) toServiceActionValue() madmin.ServiceActionValue { + switch c { + case restartCmd: + return madmin.ServiceActionValueRestart + case stopCmd: + return madmin.ServiceActionValueStop + } + return madmin.ServiceActionValueStop +} + // testServiceSignalReceiver - Helper function that simulates a // go-routine waiting on service signal. func testServiceSignalReceiver(cmd cmdType, t *testing.T) { @@ -267,17 +390,14 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) { // getServiceCmdRequest - Constructs a management REST API request for service // subcommands for a given cmdType value. func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) { - req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil) + req, err := newTestRequest(cmd.apiMethod(), cmd.apiEndpoint(), 0, nil) if err != nil { return nil, err } // Set body req.Body = ioutil.NopCloser(bytes.NewReader(body)) - - // minioAdminOpHeader is to identify the request as a - // management REST API request. - req.Header.Set(minioAdminOpHeader, cmd.String()) + // Set sha-sum header req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body)) // management REST API uses signature V4 for authentication. @@ -309,7 +429,12 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) { go testServiceSignalReceiver(cmd, t) } credentials := globalServerConfig.GetCredential() - var body []byte + + body, err := json.Marshal(madmin.ServiceAction{ + cmd.toServiceActionValue()}) + if err != nil { + t.Fatalf("JSONify error: %v", err) + } req, err := getServiceCmdRequest(cmd, credentials, body) if err != nil { @@ -320,10 +445,10 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) { adminTestBed.mux.ServeHTTP(rec, req) if cmd == statusCmd { - expectedInfo := ServerStatus{ - ServerVersion: ServerVersion{Version: Version, CommitID: CommitID}, + expectedInfo := madmin.ServiceStatus{ + ServerVersion: madmin.ServerVersion{Version: Version, CommitID: CommitID}, } - receivedInfo := ServerStatus{} + receivedInfo := madmin.ServiceStatus{} if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil { t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr) } @@ -364,11 +489,10 @@ func TestServiceSetCreds(t *testing.T) { initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1")) credentials := globalServerConfig.GetCredential() - var body []byte testCases := []struct { - Username string - Password string + AccessKey string + SecretKey string EnvKeysSet bool ExpectedStatusCode int }{ @@ -378,19 +502,21 @@ func TestServiceSetCreds(t *testing.T) { {"minio", "minio", true, http.StatusMethodNotAllowed}, // Good keys set from the env {"minio", "minio123", true, http.StatusMethodNotAllowed}, - // Successful operation should be the last one to do not change server credentials during tests. + // Successful operation should be the last one to + // not change server credentials during tests. {"minio", "minio123", false, http.StatusOK}, } for i, testCase := range testCases { // Set or unset environement keys - if !testCase.EnvKeysSet { - globalIsEnvCreds = false - } else { - globalIsEnvCreds = true - } + globalIsEnvCreds = testCase.EnvKeysSet // Construct setCreds request body - body, _ = xml.Marshal(setCredsReq{Username: testCase.Username, Password: testCase.Password}) + body, err := json.Marshal(madmin.SetCredsReq{ + AccessKey: testCase.AccessKey, + SecretKey: testCase.SecretKey}) + if err != nil { + t.Fatalf("JSONify err: %v", err) + } // Construct setCreds request req, err := getServiceCmdRequest(setCreds, credentials, body) if err != nil { @@ -413,11 +539,11 @@ func TestServiceSetCreds(t *testing.T) { // If we got 200 OK, check if new credentials are really set if rec.Code == http.StatusOK { cred := globalServerConfig.GetCredential() - if cred.AccessKey != testCase.Username { - t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.Username, cred.AccessKey) + if cred.AccessKey != testCase.AccessKey { + t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.AccessKey, cred.AccessKey) } - if cred.SecretKey != testCase.Password { - t.Errorf("Test %d: Wrong secret key, expected = %s, found = %s", i+1, testCase.Password, cred.SecretKey) + if cred.SecretKey != testCase.SecretKey { + t.Errorf("Test %d: Wrong secret key, expected = %s, found = %s", i+1, testCase.SecretKey, cred.SecretKey) } } } @@ -426,10 +552,9 @@ func TestServiceSetCreds(t *testing.T) { // mkLockQueryVal - helper function to build lock query param. func mkLockQueryVal(bucket, prefix, durationStr string) url.Values { qVal := url.Values{} - qVal.Set("lock", "") qVal.Set(string(mgmtBucket), bucket) qVal.Set(string(mgmtPrefix), prefix) - qVal.Set(string(mgmtLockDuration), durationStr) + qVal.Set(string(mgmtLockOlderThan), durationStr) return qVal } @@ -483,11 +608,10 @@ func TestListLocksHandler(t *testing.T) { for i, test := range testCases { queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration) - req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil) + req, err := newTestRequest("GET", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil) if err != nil { t.Fatalf("Test %d - Failed to construct list locks request - %v", i+1, err) } - req.Header.Set(minioAdminOpHeader, "list") cred := globalServerConfig.GetCredential() err = signRequestV4(req, cred.AccessKey, cred.SecretKey) @@ -551,11 +675,10 @@ func TestClearLocksHandler(t *testing.T) { for i, test := range testCases { queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration) - req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) + req, err := newTestRequest("DELETE", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil) if err != nil { t.Fatalf("Test %d - Failed to construct clear locks request - %v", i+1, err) } - req.Header.Set(minioAdminOpHeader, "clear") cred := globalServerConfig.GetCredential() err = signRequestV4(req, cred.AccessKey, cred.SecretKey) @@ -613,433 +736,17 @@ func TestValidateLockQueryParams(t *testing.T) { } } -// mkListObjectsQueryStr - helper to build ListObjectsHeal query string. -func mkListObjectsQueryVal(bucket, prefix, marker, delimiter, maxKeyStr string) url.Values { - qVal := url.Values{} - qVal.Set("heal", "") - qVal.Set(string(mgmtBucket), bucket) - qVal.Set(string(mgmtPrefix), prefix) - qVal.Set(string(mgmtMarker), marker) - qVal.Set(string(mgmtDelimiter), delimiter) - qVal.Set(string(mgmtMaxKey), maxKeyStr) - return qVal -} - -// TestValidateHealQueryParams - Test for query param validation helper function for heal APIs. -func TestValidateHealQueryParams(t *testing.T) { - testCases := []struct { - bucket string - prefix string - marker string - delimiter string - maxKeys string - apiErr APIErrorCode - }{ - // 1. Valid params. - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - apiErr: ErrNone, - }, - // 2. Valid params with meta bucket. - { - bucket: minioMetaBucket, - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - apiErr: ErrNone, - }, - // 3. Valid params with empty prefix. - { - bucket: "mybucket", - prefix: "", - marker: "", - delimiter: "/", - maxKeys: "10", - apiErr: ErrNone, - }, - // 4. Invalid params with invalid bucket. - { - bucket: `invalid\\Bucket`, - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - apiErr: ErrInvalidBucketName, - }, - // 5. Invalid params with invalid prefix. - { - bucket: "mybucket", - prefix: `invalid\\Prefix`, - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - apiErr: ErrInvalidObjectName, - }, - // 6. Invalid params with invalid maxKeys. - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "-1", - apiErr: ErrInvalidMaxKeys, - }, - // 7. Invalid params with unsupported prefix marker combination. - { - bucket: "mybucket", - prefix: "prefix", - marker: "notmatchingmarker", - delimiter: "/", - maxKeys: "10", - apiErr: ErrInvalidPrefixMarker, - }, - // 8. Invalid params with unsupported delimiter. - { - bucket: "mybucket", - prefix: "prefix", - marker: "notmatchingmarker", - delimiter: "unsupported", - maxKeys: "10", - apiErr: ErrNotImplemented, - }, - // 9. Invalid params with invalid max Keys - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "999999999999999999999999999", - apiErr: ErrInvalidMaxKeys, - }, - } - for i, test := range testCases { - vars := mkListObjectsQueryVal(test.bucket, test.prefix, test.marker, test.delimiter, test.maxKeys) - _, _, _, _, _, actualErr := extractListObjectsHealQuery(vars) - if actualErr != test.apiErr { - t.Errorf("Test %d - Expected %v but received %v", - i+1, getAPIError(test.apiErr), getAPIError(actualErr)) - } - } -} - -// TestListObjectsHeal - Test for ListObjectsHealHandler. -func TestListObjectsHealHandler(t *testing.T) { - adminTestBed, err := prepareAdminXLTestBed() - if err != nil { - t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") - } - defer adminTestBed.TearDown() - - err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "") - if err != nil { - t.Fatalf("Failed to make bucket - %v", err) - } - - // Delete bucket after running all test cases. - defer adminTestBed.objLayer.DeleteBucket("mybucket") - - testCases := []struct { - bucket string - prefix string - marker string - delimiter string - maxKeys string - statusCode int - }{ - // 1. Valid params. - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - statusCode: http.StatusOK, - }, - // 2. Valid params with meta bucket. - { - bucket: minioMetaBucket, - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - statusCode: http.StatusOK, - }, - // 3. Valid params with empty prefix. - { - bucket: "mybucket", - prefix: "", - marker: "", - delimiter: "/", - maxKeys: "10", - statusCode: http.StatusOK, - }, - // 4. Invalid params with invalid bucket. - { - bucket: `invalid\\Bucket`, - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - statusCode: getAPIError(ErrInvalidBucketName).HTTPStatusCode, - }, - // 5. Invalid params with invalid prefix. - { - bucket: "mybucket", - prefix: `invalid\\Prefix`, - marker: "prefix11", - delimiter: "/", - maxKeys: "10", - statusCode: getAPIError(ErrInvalidObjectName).HTTPStatusCode, - }, - // 6. Invalid params with invalid maxKeys. - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "-1", - statusCode: getAPIError(ErrInvalidMaxKeys).HTTPStatusCode, - }, - // 7. Invalid params with unsupported prefix marker combination. - { - bucket: "mybucket", - prefix: "prefix", - marker: "notmatchingmarker", - delimiter: "/", - maxKeys: "10", - statusCode: getAPIError(ErrInvalidPrefixMarker).HTTPStatusCode, - }, - // 8. Invalid params with unsupported delimiter. - { - bucket: "mybucket", - prefix: "prefix", - marker: "notmatchingmarker", - delimiter: "unsupported", - maxKeys: "10", - statusCode: getAPIError(ErrNotImplemented).HTTPStatusCode, - }, - // 9. Invalid params with invalid max Keys - { - bucket: "mybucket", - prefix: "prefix", - marker: "prefix11", - delimiter: "/", - maxKeys: "999999999999999999999999999", - statusCode: getAPIError(ErrInvalidMaxKeys).HTTPStatusCode, - }, - } - - for i, test := range testCases { - queryVal := mkListObjectsQueryVal(test.bucket, test.prefix, test.marker, test.delimiter, test.maxKeys) - req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil) - if err != nil { - t.Fatalf("Test %d - Failed to construct list objects needing heal request - %v", i+1, err) - } - req.Header.Set(minioAdminOpHeader, "list-objects") - - cred := globalServerConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err) - } - rec := httptest.NewRecorder() - adminTestBed.mux.ServeHTTP(rec, req) - if test.statusCode != rec.Code { - t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) - } - } -} - -// TestHealBucketHandler - Test for HealBucketHandler. -func TestHealBucketHandler(t *testing.T) { - adminTestBed, err := prepareAdminXLTestBed() - if err != nil { - t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") - } - defer adminTestBed.TearDown() - - err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "") - if err != nil { - t.Fatalf("Failed to make bucket - %v", err) - } - - // Delete bucket after running all test cases. - defer adminTestBed.objLayer.DeleteBucket("mybucket") - - testCases := []struct { - bucket string - statusCode int - dryrun string - }{ - // 1. Valid test case. - { - bucket: "mybucket", - statusCode: http.StatusOK, - }, - // 2. Invalid bucket name. - { - bucket: `invalid\\Bucket`, - statusCode: http.StatusBadRequest, - }, - // 3. Bucket not found. - { - bucket: "bucketnotfound", - statusCode: http.StatusNotFound, - }, - // 4. Valid test case with dry-run. - { - bucket: "mybucket", - statusCode: http.StatusOK, - dryrun: "yes", - }, - } - for i, test := range testCases { - // Prepare query params. - queryVal := url.Values{} - queryVal.Set(string(mgmtBucket), test.bucket) - queryVal.Set("heal", "") - queryVal.Set(string(mgmtDryRun), test.dryrun) - - req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) - if err != nil { - t.Fatalf("Test %d - Failed to construct heal bucket request - %v", - i+1, err) - } - - req.Header.Set(minioAdminOpHeader, "bucket") - - cred := globalServerConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Test %d - Failed to sign heal bucket request - %v", - i+1, err) - } - rec := httptest.NewRecorder() - adminTestBed.mux.ServeHTTP(rec, req) - if test.statusCode != rec.Code { - t.Errorf("Test %d - Expected HTTP status code %d but received %d", - i+1, test.statusCode, rec.Code) - } - - } -} - -// TestHealObjectHandler - Test for HealObjectHandler. -func TestHealObjectHandler(t *testing.T) { - adminTestBed, err := prepareAdminXLTestBed() - if err != nil { - t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") - } - defer adminTestBed.TearDown() - - // Create an object myobject under bucket mybucket. - bucketName := "mybucket" - objName := "myobject" - err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "") - if err != nil { - t.Fatalf("Failed to make bucket %s - %v", bucketName, err) - } - - _, err = adminTestBed.objLayer.PutObject(bucketName, objName, - mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil) - if err != nil { - t.Fatalf("Failed to create %s - %v", objName, err) - } - - // Delete bucket and object after running all test cases. - defer func(objLayer ObjectLayer, bucketName, objName string) { - objLayer.DeleteObject(bucketName, objName) - objLayer.DeleteBucket(bucketName) - }(adminTestBed.objLayer, bucketName, objName) - - testCases := []struct { - bucket string - object string - dryrun string - statusCode int - }{ - // 1. Valid test case. - { - bucket: bucketName, - object: objName, - statusCode: http.StatusOK, - }, - // 2. Invalid bucket name. - { - bucket: `invalid\\Bucket`, - object: "myobject", - statusCode: http.StatusBadRequest, - }, - // 3. Bucket not found. - { - bucket: "bucketnotfound", - object: "myobject", - statusCode: http.StatusNotFound, - }, - // 4. Invalid object name. - { - bucket: bucketName, - object: `invalid\\Object`, - statusCode: http.StatusBadRequest, - }, - // 5. Object not found. - { - bucket: bucketName, - object: "objectnotfound", - statusCode: http.StatusNotFound, - }, - // 6. Valid test case with dry-run. - { - bucket: bucketName, - object: objName, - dryrun: "yes", - statusCode: http.StatusOK, - }, - } - for i, test := range testCases { - // Prepare query params. - queryVal := url.Values{} - queryVal.Set(string(mgmtBucket), test.bucket) - queryVal.Set(string(mgmtObject), test.object) - queryVal.Set("heal", "") - queryVal.Set(string(mgmtDryRun), test.dryrun) - - req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) - if err != nil { - t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err) - } - - req.Header.Set(minioAdminOpHeader, "object") - - cred := globalServerConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err) - } - rec := httptest.NewRecorder() - adminTestBed.mux.ServeHTTP(rec, req) - if test.statusCode != rec.Code { - t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) - } - } - -} - // buildAdminRequest - helper function to build an admin API request. -func buildAdminRequest(queryVal url.Values, opHdr, method string, +func buildAdminRequest(queryVal url.Values, method, path string, contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) { - req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker) + + req, err := newTestRequest(method, + "/minio/admin/v1"+path+"?"+queryVal.Encode(), + contentLength, bodySeeker) if err != nil { return nil, errors.Trace(err) } - req.Header.Set(minioAdminOpHeader, opHdr) - cred := globalServerConfig.GetCredential() err = signRequestV4(req, cred.AccessKey, cred.SecretKey) if err != nil { @@ -1049,29 +756,6 @@ func buildAdminRequest(queryVal url.Values, opHdr, method string, return req, nil } -// TestHealFormatHandler - test for HealFormatHandler. -func TestHealFormatHandler(t *testing.T) { - adminTestBed, err := prepareAdminXLTestBed() - if err != nil { - t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") - } - defer adminTestBed.TearDown() - - // Prepare query params for heal-format mgmt REST API. - queryVal := url.Values{} - queryVal.Set("heal", "") - req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil) - if err != nil { - t.Fatalf("Failed to construct heal object request - %v", err) - } - - rec := httptest.NewRecorder() - adminTestBed.mux.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - t.Errorf("Expected to succeed but failed with %d", rec.Code) - } -} - // TestGetConfigHandler - test for GetConfigHandler. func TestGetConfigHandler(t *testing.T) { adminTestBed, err := prepareAdminXLTestBed() @@ -1088,7 +772,7 @@ func TestGetConfigHandler(t *testing.T) { queryVal := url.Values{} queryVal.Set("config", "") - req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil) + req, err := buildAdminRequest(queryVal, http.MethodGet, "/config", 0, nil) if err != nil { t.Fatalf("Failed to construct get-config object request - %v", err) } @@ -1121,8 +805,8 @@ func TestSetConfigHandler(t *testing.T) { queryVal := url.Values{} queryVal.Set("config", "") - req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)), - bytes.NewReader(configJSON)) + req, err := buildAdminRequest(queryVal, http.MethodPut, "/config", + int64(len(configJSON)), bytes.NewReader(configJSON)) if err != nil { t.Fatalf("Failed to construct set-config object request - %v", err) } @@ -1147,8 +831,8 @@ func TestSetConfigHandler(t *testing.T) { { // Make a large enough config string invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1)) - req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(invalidCfg)), - bytes.NewReader(invalidCfg)) + req, err := buildAdminRequest(queryVal, http.MethodPut, "/config", + int64(len(invalidCfg)), bytes.NewReader(invalidCfg)) if err != nil { t.Fatalf("Failed to construct set-config object request - %v", err) } @@ -1166,8 +850,8 @@ func TestSetConfigHandler(t *testing.T) { // error. { invalidCfg := append(configJSON[:len(configJSON)-1], []byte(`, "version": "15"}`)...) - req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(invalidCfg)), - bytes.NewReader(invalidCfg)) + req, err := buildAdminRequest(queryVal, http.MethodPut, "/config", + int64(len(invalidCfg)), bytes.NewReader(invalidCfg)) if err != nil { t.Fatalf("Failed to construct set-config object request - %v", err) } @@ -1197,7 +881,7 @@ func TestAdminServerInfo(t *testing.T) { queryVal := url.Values{} queryVal.Set("info", "") - req, err := buildAdminRequest(queryVal, "", http.MethodGet, 0, nil) + req, err := buildAdminRequest(queryVal, http.MethodGet, "/info", 0, nil) if err != nil { t.Fatalf("Failed to construct get-config object request - %v", err) } @@ -1341,28 +1025,167 @@ func TestWriteSetConfigResponse(t *testing.T) { } } -// Test for newHealResult helper function. -func TestNewHealResult(t *testing.T) { - testCases := []struct { - healedDisks int - offlineDisks int - state healState - }{ - // 1. No disks healed, no disks offline. - {0, 0, healNone}, - // 2. No disks healed, non-zero disks offline. - {0, 1, healNone}, - // 3. Non-zero disks healed, no disks offline. - {1, 0, healOK}, - // 4. Non-zero disks healed, non-zero disks offline. - {1, 1, healPartial}, +func mkHealStartReq(t *testing.T, bucket, prefix string, + opts madmin.HealOpts) *http.Request { + + body, err := json.Marshal(opts) + if err != nil { + t.Fatalf("Unable marshal heal opts") } - for i, test := range testCases { - actual := newHealResult(test.healedDisks, test.offlineDisks) - if actual.State != test.state { - t.Errorf("Test %d: Expected %v but received %v", i+1, - test.state, actual.State) + path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket) + if bucket != "" && prefix != "" { + path += "/" + prefix + } + + req, err := newTestRequest("POST", path, + int64(len(body)), bytes.NewReader(body)) + if err != nil { + t.Fatalf("Failed to construct request - %v", err) + } + cred := globalServerConfig.GetCredential() + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + t.Fatalf("Failed to sign request - %v", err) + } + + return req +} + +func mkHealStatusReq(t *testing.T, bucket, prefix, + clientToken string) *http.Request { + + path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket) + if bucket != "" && prefix != "" { + path += "/" + prefix + } + path += fmt.Sprintf("?clientToken=%s", clientToken) + + req, err := newTestRequest("POST", path, 0, nil) + if err != nil { + t.Fatalf("Failed to construct request - %v", err) + } + cred := globalServerConfig.GetCredential() + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + t.Fatalf("Failed to sign request - %v", err) + } + + return req +} + +func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket, + prefix, clientToken string, timeLimitSecs int) madmin.HealTaskStatus { + + var res, cur madmin.HealTaskStatus + + // loop and fetch heal status. have a time-limit to loop over + // all statuses. + timeLimit := UTCNow().Add(time.Second * time.Duration(timeLimitSecs)) + for cur.Summary != healStoppedStatus && cur.Summary != healFinishedStatus { + if UTCNow().After(timeLimit) { + t.Fatalf("heal-status loop took too long - clientToken: %s", clientToken) + } + req := mkHealStatusReq(t, bucket, prefix, clientToken) + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if http.StatusOK != rec.Code { + t.Errorf("Unexpected status code - got %d but expected %d", + rec.Code, http.StatusOK) + break + } + err := json.NewDecoder(rec.Body).Decode(&cur) + if err != nil { + t.Errorf("unable to unmarshal resp: %v", err) + break + } + + // all results are accumulated into a slice + // and returned to caller in the end + allItems := append(res.Items, cur.Items...) + res = cur + res.Items = allItems + + time.Sleep(time.Millisecond * 200) + } + + return res +} + +func TestHealStartNStatusHandler(t *testing.T) { + adminTestBed, err := prepareAdminXLTestBed() + if err != nil { + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") + } + defer adminTestBed.TearDown() + + // gen. test data + adminTestBed.GenerateHealTestData(t) + defer adminTestBed.CleanupHealTestData(t) + + // Prepare heal-start request to send to the server. + healOpts := madmin.HealOpts{ + Recursive: true, + DryRun: false, + } + bucketName, objName := "mybucket", "myobject-0" + var hss madmin.HealStartSuccess + + { + req := mkHealStartReq(t, bucketName, objName, healOpts) + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if http.StatusOK != rec.Code { + t.Errorf("Unexpected status code - got %d but expected %d", + rec.Code, http.StatusOK) + } + + err = json.Unmarshal(rec.Body.Bytes(), &hss) + if err != nil { + t.Fatal("unable to unmarshal response") + } + + if hss.ClientToken == "" { + t.Errorf("unexpected result") + } + } + + { + // test with an invalid client token + req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken) + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("Unexpected status code") + } + } + + { + // fetch heal status + results := collectHealResults(t, adminTestBed, bucketName, + objName, hss.ClientToken, 5) + + // check if we got back an expected record + foundIt := false + for _, item := range results.Items { + if item.Type == madmin.HealItemObject && + item.Bucket == bucketName && item.Object == objName { + foundIt = true + } + } + if !foundIt { + t.Error("did not find expected heal record in heal results") + } + + // check that the heal settings in the results is the + // same as what we started the heal seq. with. + if results.HealSettings != healOpts { + t.Errorf("unexpected heal settings: %v", + results.HealSettings) + } + + if results.Summary == healStoppedStatus { + t.Errorf("heal sequence stopped unexpectedly") } } } diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go new file mode 100644 index 000000000..5d7dca68b --- /dev/null +++ b/cmd/admin-heal-ops.go @@ -0,0 +1,683 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "github.com/minio/minio/pkg/madmin" +) + +// healStatusSummary - overall short summary of a healing sequence +type healStatusSummary string + +// healStatusSummary constants +const ( + healNotStartedStatus healStatusSummary = "not started" + healRunningStatus = "running" + healStoppedStatus = "stopped" + healFinishedStatus = "finished" +) + +const ( + // a heal sequence with this many un-consumed heal result + // items blocks until heal-status consumption resumes or is + // aborted due to timeout. + maxUnconsumedHealResultItems = 1000 + + // if no heal-results are consumed (via the heal-status API) + // for this timeout duration, the heal sequence is aborted. + healUnconsumedTimeout = 24 * time.Hour + + // time-duration to keep heal sequence state after it + // completes. + keepHealSeqStateDuration = time.Minute * 10 +) + +var ( + errHealIdleTimeout = fmt.Errorf("healing results were not consumed for too long") + errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal") + errHealStopSignalled = fmt.Errorf("heal stop signalled") + + errFnHealFromAPIErr = func(err error) error { + errCode := toAPIErrorCode(err) + apiErr := getAPIError(errCode) + return fmt.Errorf("Heal internal error: %s: %s", + apiErr.Code, apiErr.Description) + } +) + +// healSequenceStatus - accumulated status of the heal sequence +type healSequenceStatus struct { + // lock to update this structure as it is concurrently + // accessed + updateLock *sync.RWMutex + + // summary and detail for failures + Summary healStatusSummary `json:"Summary"` + FailureDetail string `json:"Detail,omitempty"` + StartTime time.Time `json:"StartTime"` + + // disk information + NumDisks int `json:"NumDisks"` + + // settings for the heal sequence + HealSettings madmin.HealOpts `json:"Settings"` + + // slice of available heal result records + Items []madmin.HealResultItem `json:"Items"` +} + +// structure to hold state of all heal sequences in server memory +type allHealState struct { + sync.Mutex + + // map of heal path to heal sequence + healSeqMap map[string]*healSequence +} + +var ( + // global server heal state + globalAllHealState allHealState +) + +// initAllHealState - initialize healing apparatus +func initAllHealState(isErasureMode bool) { + if !isErasureMode { + return + } + + globalAllHealState = allHealState{ + healSeqMap: make(map[string]*healSequence), + } +} + +// getHealSequence - Retrieve a heal sequence by path. The second +// argument returns if a heal sequence actually exists. +func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) { + ahs.Lock() + defer ahs.Unlock() + h, exists = ahs.healSeqMap[path] + return h, exists +} + +// LaunchNewHealSequence - launches a background routine that performs +// healing according to the healSequence argument. For each heal +// sequence, state is stored in the `globalAllHealState`, which is a +// map of the heal path to `healSequence` which holds state about the +// heal sequence. +// +// Heal results are persisted in server memory for +// `keepHealSeqStateDuration`. This function also launches a +// background routine to clean up heal results after the +// aforementioned duration. +func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) ( + respBytes []byte, errCode APIErrorCode, errMsg string) { + + existsAndLive := false + he, exists := ahs.getHealSequence(h.path) + if exists { + if !he.hasEnded() || len(he.currentStatus.Items) > 0 { + existsAndLive = true + } + } + if existsAndLive { + // A heal sequence exists on the given path. + if h.forceStarted { + // stop the running heal sequence - wait for + // it to finish. + he.stop() + for !he.hasEnded() { + time.Sleep(10 * time.Second) + } + } else { + errMsg = "Heal is already running on the given path " + + "(use force-start option to stop and start afresh). " + + fmt.Sprintf("The heal was started by IP %s at %s", + h.clientAddress, h.startTime) + + return nil, ErrHealAlreadyRunning, errMsg + } + } + + ahs.Lock() + defer ahs.Unlock() + + // Check if new heal sequence to be started overlaps with any + // existing, running sequence + for k, hSeq := range ahs.healSeqMap { + if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) || + strings.HasPrefix(h.path, k)) { + + errMsg = "The provided heal sequence path overlaps with an existing " + + fmt.Sprintf("heal path: %s", k) + return nil, ErrHealOverlappingPaths, errMsg + } + } + + // Add heal state and start sequence + ahs.healSeqMap[h.path] = h + + // Launch top-level background heal go-routine + go h.healSequenceStart() + + // Launch clean-up routine to remove this heal sequence (after + // it ends) from the global state after timeout has elapsed. + go func() { + var keepStateTimeout <-chan time.Time + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + everyMinute := ticker.C + for { + select { + // Check every minute if heal sequence has ended. + case <-everyMinute: + if h.hasEnded() { + keepStateTimeout = time.After(keepHealSeqStateDuration) + everyMinute = nil + } + + // This case does not fire until the heal + // sequence completes. + case <-keepStateTimeout: + // Heal sequence has ended, keep + // results state duration has elapsed, + // so purge state. + ahs.Lock() + defer ahs.Unlock() + delete(ahs.healSeqMap, h.path) + return + + case <-globalServiceDoneCh: + // server could be restarting - need + // to exit immediately + return + } + } + }() + + b, err := json.Marshal(madmin.HealStartSuccess{ + h.clientToken, + h.clientAddress, + h.startTime, + }) + if err != nil { + errorIf(err, "Failed to marshal heal result into json.") + return nil, ErrInternalError, "" + } + return b, ErrNone, "" +} + +// PopHealStatusJSON - Called by heal-status API. It fetches the heal +// status results from global state and returns its JSON +// representation. The clientToken helps ensure there aren't +// conflicting clients fetching status. +func (ahs *allHealState) PopHealStatusJSON(path string, + clientToken string) ([]byte, APIErrorCode) { + + // fetch heal state for given path + h, exists := ahs.getHealSequence(path) + if !exists { + // If there is no such heal sequence, return error. + return nil, ErrHealNoSuchProcess + } + + // Check if client-token is valid + if clientToken != h.clientToken { + return nil, ErrHealInvalidClientToken + } + + // Take lock to access and update the heal-sequence + h.currentStatus.updateLock.Lock() + defer h.currentStatus.updateLock.Unlock() + + numItems := len(h.currentStatus.Items) + + // calculate index of most recently available heal result + // record. + lastResultIndex := h.lastSentResultIndex + if numItems > 0 { + lastResultIndex = h.currentStatus.Items[numItems-1].ResultIndex + } + + // After sending status to client, and before relinquishing + // the updateLock, reset Item to nil and record the result + // index sent to the client. + defer func(i int64) { + h.lastSentResultIndex = i + h.currentStatus.Items = nil + }(lastResultIndex) + + jbytes, err := json.Marshal(h.currentStatus) + if err != nil { + errorIf(err, "Failed to marshal heal result into json.") + return nil, ErrInternalError + } + + return jbytes, ErrNone +} + +// healSequence - state for each heal sequence initiated on the +// server. +type healSequence struct { + // bucket, and prefix on which heal seq. was initiated + bucket, objPrefix string + + // path is just bucket + "/" + objPrefix + path string + + // time at which heal sequence was started + startTime time.Time + + // Heal client info + clientToken, clientAddress string + + // was this heal sequence force started? + forceStarted bool + + // heal settings applied to this heal sequence + settings madmin.HealOpts + + // current accumulated status of the heal sequence + currentStatus healSequenceStatus + + // channel signalled by background routine when traversal has + // completed + traverseAndHealDoneCh chan error + + // channel to signal heal sequence to stop (e.g. from the + // heal-stop API) + stopSignalCh chan struct{} + + // the last result index sent to client + lastSentResultIndex int64 +} + +// NewHealSequence - creates healSettings, assumes bucket and +// objPrefix are already validated. +func newHealSequence(bucket, objPrefix, clientAddr string, + numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence { + + return &healSequence{ + bucket: bucket, + objPrefix: objPrefix, + path: bucket + "/" + objPrefix, + startTime: UTCNow(), + clientToken: mustGetUUID(), + clientAddress: clientAddr, + forceStarted: forceStart, + settings: hs, + currentStatus: healSequenceStatus{ + Summary: healNotStartedStatus, + HealSettings: hs, + NumDisks: numDisks, + updateLock: &sync.RWMutex{}, + }, + traverseAndHealDoneCh: make(chan error), + stopSignalCh: make(chan struct{}), + } +} + +// isQuitting - determines if the heal sequence is quitting (due to an +// external signal) +func (h *healSequence) isQuitting() bool { + select { + case <-h.stopSignalCh: + return true + default: + return false + } +} + +// check if the heal sequence has ended +func (h *healSequence) hasEnded() bool { + h.currentStatus.updateLock.RLock() + summary := h.currentStatus.Summary + h.currentStatus.updateLock.RUnlock() + return summary == healStoppedStatus || summary == healFinishedStatus +} + +// stops the heal sequence - safe to call multiple times. +func (h *healSequence) stop() { + select { + case <-h.stopSignalCh: + default: + close(h.stopSignalCh) + } +} + +// pushHealResultItem - pushes a heal result item for consumption in +// the heal-status API. It blocks if there are +// maxUnconsumedHealResultItems. When it blocks, the heal sequence +// routine is effectively paused - this happens when the server has +// accumulated the maximum number of heal records per heal +// sequence. When the client consumes further records, the heal +// sequence automatically resumes. The return value indicates if the +// operation succeeded. +func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error { + + // start a timer to keep an upper time limit to find an empty + // slot to add the given heal result - if no slot is found it + // means that the server is holding the maximum amount of + // heal-results in memory and the client has not consumed it + // for too long. + unconsumedTimer := time.NewTimer(healUnconsumedTimeout) + defer func() { + // stop the timeout timer so it is garbage collected. + if !unconsumedTimer.Stop() { + <-unconsumedTimer.C + } + }() + + var itemsLen int + for { + h.currentStatus.updateLock.Lock() + itemsLen = len(h.currentStatus.Items) + if itemsLen == maxUnconsumedHealResultItems { + // unlock and wait to check again if we can push + h.currentStatus.updateLock.Unlock() + + // wait for a second, or quit if an external + // stop signal is received or the + // unconsumedTimer fires. + select { + // Check after a second + case <-time.After(time.Second): + continue + + case <-h.stopSignalCh: + // discard result and return. + return errHealPushStopNDiscard + + // Timeout if no results consumed for too + // long. + case <-unconsumedTimer.C: + return errHealIdleTimeout + + } + } + break + } + + // Set the correct result index for the new result item + if itemsLen > 0 { + r.ResultIndex = 1 + h.currentStatus.Items[itemsLen-1].ResultIndex + } else { + r.ResultIndex = 1 + h.lastSentResultIndex + } + + // append to results + h.currentStatus.Items = append(h.currentStatus.Items, r) + + // release lock + h.currentStatus.updateLock.Unlock() + + // This is a "safe" point for the heal sequence to quit if + // signalled externally. + if h.isQuitting() { + return errHealStopSignalled + } + + return nil +} + +// healSequenceStart - this is the top-level background heal +// routine. It launches another go-routine that actually traverses +// on-disk data, checks and heals according to the selected +// settings. This go-routine itself, (1) monitors the traversal +// routine for completion, and (2) listens for external stop +// signals. When either event happens, it sets the finish status for +// the heal-sequence. +func (h *healSequence) healSequenceStart() { + // Set status as running + h.currentStatus.updateLock.Lock() + h.currentStatus.Summary = healRunningStatus + h.currentStatus.StartTime = UTCNow() + h.currentStatus.updateLock.Unlock() + + go h.traverseAndHeal() + + select { + case err, ok := <-h.traverseAndHealDoneCh: + h.currentStatus.updateLock.Lock() + defer h.currentStatus.updateLock.Unlock() + // Heal traversal is complete. + if ok { + // heal traversal had an error. + h.currentStatus.Summary = healStoppedStatus + h.currentStatus.FailureDetail = err.Error() + } else { + // heal traversal succeeded. + h.currentStatus.Summary = healFinishedStatus + } + + case <-h.stopSignalCh: + h.currentStatus.updateLock.Lock() + h.currentStatus.Summary = healStoppedStatus + h.currentStatus.FailureDetail = errHealStopSignalled.Error() + h.currentStatus.updateLock.Unlock() + + // drain traverse channel so the traversal + // go-routine does not leak. + go func() { + // Eventually the traversal go-routine closes + // the channel and returns, so this go-routine + // itself will not leak. + <-h.traverseAndHealDoneCh + }() + } +} + +// traverseAndHeal - traverses on-disk data and performs healing +// according to settings. At each "safe" point it also checks if an +// external quit signal has been received and quits if so. Since the +// healing traversal may be mutating on-disk data when an external +// quit signal is received, this routine cannot quit immediately and +// has to wait until a safe point is reached, such as between scanning +// two objects. +func (h *healSequence) traverseAndHeal() { + var err error + checkErr := func(f func() error) { + switch { + case err != nil: + return + case h.isQuitting(): + err = errHealStopSignalled + return + } + err = f() + } + + // Start with format healing + checkErr(h.healDiskFormat) + + // Heal buckets and objects + checkErr(h.healBuckets) + + if err != nil { + h.traverseAndHealDoneCh <- err + } + + close(h.traverseAndHealDoneCh) +} + +// healDiskFormat - heals format.json, return value indicates if a +// failure error occurred. +func (h *healSequence) healDiskFormat() error { + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return errServerNotInitialized + } + + // Create a new set of storage instances to heal format.json. + bootstrapDisks, err := initStorageDisks(globalEndpoints) + if err != nil { + return errFnHealFromAPIErr(err) + } + + // Wrap into retrying disks + retryingDisks := initRetryableStorageDisks(bootstrapDisks, + time.Millisecond, time.Millisecond*5, + globalStorageHealthCheckInterval, globalStorageRetryThreshold) + + // Heal format.json on available storage. + hres, err := healFormatXL(retryingDisks, h.settings.DryRun) + if err != nil { + return errFnHealFromAPIErr(err) + } + + // reload object layer global only if we healed some disk + onlineBefore, onlineAfter := hres.GetOnlineCounts() + numHealed := onlineAfter - onlineBefore + if numHealed > 0 { + // Instantiate new object layer with newly formatted + // storage. + newObjectAPI, err := newXLObjects(retryingDisks) + if err != nil { + return errFnHealFromAPIErr(err) + } + + // Set object layer with newly formatted storage to + // globalObjectAPI. + globalObjLayerMutex.Lock() + globalObjectAPI = newObjectAPI + globalObjLayerMutex.Unlock() + + // Shutdown storage belonging to old object layer + // instance. + objectAPI.Shutdown() + + // Inform peers to reinitialize storage with newly + // formatted storage. + reInitPeerDisks(globalAdminPeers) + } + + // Push format heal result + return h.pushHealResultItem(hres) +} + +// healBuckets - check for all buckets heal or just particular bucket. +func (h *healSequence) healBuckets() error { + // 1. If a bucket was specified, heal only the bucket. + if h.bucket != "" { + return h.healBucket(h.bucket) + } + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return errServerNotInitialized + } + + buckets, err := objectAPI.ListBucketsHeal() + if err != nil { + return errFnHealFromAPIErr(err) + } + + for _, bucket := range buckets { + err = h.healBucket(bucket.Name) + if err != nil { + return err + } + } + + return nil +} + +// healBucket - traverses and heals given bucket +func (h *healSequence) healBucket(bucket string) error { + if h.isQuitting() { + return errHealStopSignalled + } + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return errServerNotInitialized + } + + results, err := objectAPI.HealBucket(bucket, h.settings.DryRun) + // push any available results before checking for error + for _, result := range results { + if perr := h.pushHealResultItem(result); perr != nil { + return perr + } + } + // handle heal-bucket error + if err != nil { + return err + } + + if !h.settings.Recursive { + if h.objPrefix != "" { + // Check if an object named as the objPrefix exists, + // and if so heal it. + _, err = objectAPI.GetObjectInfo(bucket, h.objPrefix) + if err == nil { + err = h.healObject(bucket, h.objPrefix) + if err != nil { + return err + } + } + } + + return nil + } + + marker := "" + isTruncated := true + for isTruncated { + objectInfos, err := objectAPI.ListObjectsHeal(bucket, + h.objPrefix, marker, "", 1000) + if err != nil { + return errFnHealFromAPIErr(err) + } + + for _, o := range objectInfos.Objects { + if err := h.healObject(o.Bucket, o.Name); err != nil { + return err + } + } + + isTruncated = objectInfos.IsTruncated + marker = objectInfos.NextMarker + } + return nil +} + +// healObject - heal the given object and record result +func (h *healSequence) healObject(bucket, object string) error { + if h.isQuitting() { + return errHealStopSignalled + } + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return errServerNotInitialized + } + + hri, err := objectAPI.HealObject(bucket, object, h.settings.DryRun) + if err != nil { + hri.Detail = err.Error() + } + return h.pushHealResultItem(hri) +} diff --git a/cmd/admin-router.go b/cmd/admin-router.go index fa0b805b2..ad5a2c32c 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -16,7 +16,15 @@ package cmd -import router "github.com/gorilla/mux" +import ( + "net/http" + + router "github.com/gorilla/mux" +) + +const ( + adminAPIPathPrefix = "/minio/admin" +) // adminAPIHandlers provides HTTP handlers for Minio admin API. type adminAPIHandlers struct { @@ -27,46 +35,44 @@ func registerAdminRouter(mux *router.Router) { adminAPI := adminAPIHandlers{} // Admin router - adminRouter := mux.NewRoute().PathPrefix("/").Subrouter() + adminRouter := mux.NewRoute().PathPrefix(adminAPIPathPrefix).Subrouter() + + // Version handler + adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler) + + adminV1Router := adminRouter.PathPrefix("/v1").Subrouter() /// Service operations // Service status - adminRouter.Methods("GET").Queries("service", "").Headers(minioAdminOpHeader, "status").HandlerFunc(adminAPI.ServiceStatusHandler) + adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(adminAPI.ServiceStatusHandler) - // Service restart - adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "restart").HandlerFunc(adminAPI.ServiceRestartHandler) - // Service update credentials - adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "set-credentials").HandlerFunc(adminAPI.ServiceCredentialsHandler) + // Service restart and stop - TODO + adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(adminAPI.ServiceStopNRestartHandler) // Info operations - adminRouter.Methods("GET").Queries("info", "").HandlerFunc(adminAPI.ServerInfoHandler) + adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(adminAPI.ServerInfoHandler) /// Lock operations // List Locks - adminRouter.Methods("GET").Queries("lock", "").Headers(minioAdminOpHeader, "list").HandlerFunc(adminAPI.ListLocksHandler) + adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(adminAPI.ListLocksHandler) // Clear locks - adminRouter.Methods("POST").Queries("lock", "").Headers(minioAdminOpHeader, "clear").HandlerFunc(adminAPI.ClearLocksHandler) + adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(adminAPI.ClearLocksHandler) /// Heal operations - // List Objects needing heal. - adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-objects").HandlerFunc(adminAPI.ListObjectsHealHandler) - // List Buckets needing heal. - adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-buckets").HandlerFunc(adminAPI.ListBucketsHealHandler) - - // Heal Buckets. - adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler) - // Heal Objects. - adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler) - // Heal Format. - adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler) + // Heal processing endpoint. + adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(adminAPI.HealHandler) + adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(adminAPI.HealHandler) + adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(adminAPI.HealHandler) /// Config operations + // Update credentials + adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(adminAPI.UpdateCredentialsHandler) // Get config - adminRouter.Methods("GET").Queries("config", "").Headers(minioAdminOpHeader, "get").HandlerFunc(adminAPI.GetConfigHandler) - // Set Config - adminRouter.Methods("PUT").Queries("config", "").Headers(minioAdminOpHeader, "set").HandlerFunc(adminAPI.SetConfigHandler) + adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(adminAPI.GetConfigHandler) + // Set config + adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(adminAPI.SetConfigHandler) } diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go index 6d22b85a0..ed3be9911 100644 --- a/cmd/admin-rpc-client.go +++ b/cmd/admin-rpc-client.go @@ -34,7 +34,7 @@ import ( const ( // Admin service names - serviceRestartRPC = "Admin.Restart" + signalServiceRPC = "Admin.SignalService" listLocksRPC = "Admin.ListLocks" reInitDisksRPC = "Admin.ReInitDisks" serverInfoDataRPC = "Admin.ServerInfoData" @@ -56,7 +56,7 @@ type remoteAdminClient struct { // adminCmdRunner - abstracts local and remote execution of admin // commands like service stop and service restart. type adminCmdRunner interface { - Restart() error + SignalService(s serviceSignal) error ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) ReInitDisks() error ServerInfoData() (ServerInfoData, error) @@ -65,10 +65,16 @@ type adminCmdRunner interface { CommitConfig(tmpFileName string) error } -// Restart - Sends a message over channel to the go-routine -// responsible for restarting the process. -func (lc localAdminClient) Restart() error { - globalServiceSignalCh <- serviceRestart +var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported") + +// SignalService - sends a restart or stop signal to the local server +func (lc localAdminClient) SignalService(s serviceSignal) error { + switch s { + case serviceRestart, serviceStop: + globalServiceSignalCh <- s + default: + return errUnsupportedSignal + } return nil } @@ -77,25 +83,31 @@ func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Durati return listLocksInfo(bucket, prefix, duration), nil } -// Restart - Sends restart command to remote server via RPC. -func (rc remoteAdminClient) Restart() error { - args := AuthRPCArgs{} - reply := AuthRPCReply{} - return rc.Call(serviceRestartRPC, &args, &reply) +func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) { + switch s { + case serviceRestart, serviceStop: + reply := AuthRPCReply{} + err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s}, + &reply) + default: + err = errUnsupportedSignal + } + return err + } // ListLocks - Sends list locks command to remote server via RPC. func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { listArgs := ListLocksQuery{ - bucket: bucket, - prefix: prefix, - duration: duration, + Bucket: bucket, + Prefix: prefix, + Duration: duration, } var reply ListLocksReply if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil { return nil, err } - return reply.volLocks, nil + return reply.VolLocks, nil } // ReInitDisks - There is nothing to do here, heal format REST API @@ -225,7 +237,7 @@ func (rc remoteAdminClient) CommitConfig(tmpFileName string) error { return nil } -// adminPeer - represents an entity that implements Restart methods. +// adminPeer - represents an entity that implements admin API RPCs. type adminPeer struct { addr string cmdRunner adminCmdRunner @@ -274,11 +286,11 @@ func initGlobalAdminPeers(endpoints EndpointList) { globalAdminPeers = makeAdminPeers(endpoints) } -// invokeServiceCmd - Invoke Restart command. +// invokeServiceCmd - Invoke Restart/Stop command. func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) { switch cmd { - case serviceRestart: - err = cp.cmdRunner.Restart() + case serviceRestart, serviceStop: + err = cp.cmdRunner.SignalService(cmd) } return err } diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go index 9e3938d9b..cd8c2fe25 100644 --- a/cmd/admin-rpc-server.go +++ b/cmd/admin-rpc-server.go @@ -38,18 +38,24 @@ type adminCmd struct { AuthRPCServer } +// SignalServiceArgs - provides the signal argument to SignalService RPC +type SignalServiceArgs struct { + AuthRPCArgs + Sig serviceSignal +} + // ListLocksQuery - wraps ListLocks API's query values to send over RPC. type ListLocksQuery struct { AuthRPCArgs - bucket string - prefix string - duration time.Duration + Bucket string + Prefix string + Duration time.Duration } // ListLocksReply - wraps ListLocks response over RPC. type ListLocksReply struct { AuthRPCReply - volLocks []VolumeLockInfo + VolLocks []VolumeLockInfo } // ServerInfoDataReply - wraps the server info response over RPC. @@ -64,13 +70,13 @@ type ConfigReply struct { Config []byte // json-marshalled bytes of serverConfigV13 } -// Restart - Restart this instance of minio server. -func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error { +// SignalService - Send a restart or stop signal to the service +func (s *adminCmd) SignalService(args *SignalServiceArgs, reply *AuthRPCReply) error { if err := args.IsAuthenticated(); err != nil { return err } - globalServiceSignalCh <- serviceRestart + globalServiceSignalCh <- args.Sig return nil } @@ -79,8 +85,8 @@ func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error if err := query.IsAuthenticated(); err != nil { return err } - volLocks := listLocksInfo(query.bucket, query.prefix, query.duration) - *reply = ListLocksReply{volLocks: volLocks} + volLocks := listLocksInfo(query.Bucket, query.Prefix, query.Duration) + *reply = ListLocksReply{VolLocks: volLocks} return nil } diff --git a/cmd/admin-rpc-server_test.go b/cmd/admin-rpc-server_test.go index 16747dc98..0312bbc36 100644 --- a/cmd/admin-rpc-server_test.go +++ b/cmd/admin-rpc-server_test.go @@ -23,8 +23,8 @@ import ( ) func testAdminCmd(cmd cmdType, t *testing.T) { - // reset globals. - // this is to make sure that the tests are not affected by modified globals. + // reset globals. this is to make sure that the tests are not + // affected by modified globals. resetTestGlobals() rootPath, err := newTestConfig(globalMinioDefaultRegion) @@ -55,12 +55,22 @@ func testAdminCmd(cmd cmdType, t *testing.T) { <-globalServiceSignalCh }() - ga := AuthRPCArgs{AuthToken: token} + sa := SignalServiceArgs{ + AuthRPCArgs: AuthRPCArgs{AuthToken: token}, + Sig: cmd.toServiceSignal(), + } genReply := AuthRPCReply{} switch cmd { - case restartCmd: - if err = adminServer.Restart(&ga, &genReply); err != nil { - t.Errorf("restartCmd: Expected: , got: %v", err) + case restartCmd, stopCmd: + if err = adminServer.SignalService(&sa, &genReply); err != nil { + t.Errorf("restartCmd/stopCmd: Expected: , got: %v", + err) + } + default: + err = adminServer.SignalService(&sa, &genReply) + if err != nil && err.Error() != errUnsupportedSignal.Error() { + t.Errorf("invalidSignal %s: unexpected error got: %v", + cmd, err) } } } @@ -70,6 +80,16 @@ func TestAdminRestart(t *testing.T) { testAdminCmd(restartCmd, t) } +// TestAdminStop - test for Admin.Stop RPC service. +func TestAdminStop(t *testing.T) { + testAdminCmd(stopCmd, t) +} + +// TestAdminStatus - test for Admin.Status RPC service (error case) +func TestAdminStatus(t *testing.T) { + testAdminCmd(statusCmd, t) +} + // TestReInitDisks - test for Admin.ReInitDisks RPC service. func TestReInitDisks(t *testing.T) { // Reset global variables to start afresh. diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 3c205c6cd..fb0d9908f 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -41,8 +41,8 @@ type APIErrorResponse struct { Key string BucketName string Resource string - RequestID string `xml:"RequestId"` - HostID string `xml:"HostId"` + RequestID string `xml:"RequestId" json:"RequestId"` + HostID string `xml:"HostId" json:"HostId"` } // APIErrorCode type of error status. @@ -158,6 +158,7 @@ const ( ErrReadQuorum ErrWriteQuorum ErrStorageFull + ErrRequestBodyParse ErrObjectExistsAsDirectory ErrPolicyNesting ErrInvalidObjectName @@ -174,6 +175,7 @@ const ( // Please open a https://github.com/minio/minio/issues before adding // new error codes here. + ErrMalformedJSON ErrAdminInvalidAccessKey ErrAdminInvalidSecretKey ErrAdminConfigNoQuorum @@ -183,6 +185,11 @@ const ( ErrInsecureClientRequest ErrObjectTampered ErrHealNotImplemented + ErrHealNoSuchProcess + ErrHealInvalidClientToken + ErrHealMissingBucket + ErrHealAlreadyRunning + ErrHealOverlappingPaths ) // error code to APIError structure, these fields carry respective @@ -673,6 +680,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.", HTTPStatusCode: http.StatusInternalServerError, }, + ErrRequestBodyParse: { + Code: "XMinioRequestBodyParse", + Description: "The request body failed to parse.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrObjectExistsAsDirectory: { Code: "XMinioObjectExistsAsDirectory", Description: "Object name already exists as a directory.", @@ -708,6 +720,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Server not initialized, please try again.", HTTPStatusCode: http.StatusServiceUnavailable, }, + ErrMalformedJSON: { + Code: "XMinioMalformedJSON", + Description: "The JSON you provided was not well-formed or did not validate against our published format.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrAdminInvalidAccessKey: { Code: "XMinioAdminInvalidAccessKey", Description: "The access key is invalid.", @@ -764,11 +781,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: errObjectTampered.Error(), HTTPStatusCode: http.StatusPartialContent, }, - ErrHealNotImplemented: { - Code: "XMinioHealNotImplemented", - Description: "This server does not implement heal functionality.", - HTTPStatusCode: http.StatusBadRequest, - }, ErrMaximumExpires: { Code: "AuthorizationQueryParametersError", Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds", @@ -782,6 +794,36 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Invalid Request", HTTPStatusCode: http.StatusBadRequest, }, + ErrHealNotImplemented: { + Code: "XMinioHealNotImplemented", + Description: "This server does not implement heal functionality.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealNoSuchProcess: { + Code: "XMinioHealNoSuchProcess", + Description: "No such heal process is running on the server", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealInvalidClientToken: { + Code: "XMinioHealInvalidClientToken", + Description: "Client token mismatch", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealMissingBucket: { + Code: "XMinioHealMissingBucket", + Description: "A heal start request with a non-empty object-prefix parameter requires a bucket to be specified.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealAlreadyRunning: { + Code: "XMinioHealAlreadyRunning", + Description: "", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealOverlappingPaths: { + Code: "XMinioHealOverlappingPaths", + Description: "", + HTTPStatusCode: http.StatusBadRequest, + }, // Add your error structure here. } diff --git a/cmd/api-headers.go b/cmd/api-headers.go index 8d1889d9a..269296644 100644 --- a/cmd/api-headers.go +++ b/cmd/api-headers.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "encoding/json" "encoding/xml" "fmt" "net/http" @@ -53,6 +54,14 @@ func encodeResponse(response interface{}) []byte { return bytesBuffer.Bytes() } +// Encodes the response headers into JSON format. +func encodeResponseJSON(response interface{}) []byte { + var bytesBuffer bytes.Buffer + e := json.NewEncoder(&bytesBuffer) + e.Encode(response) + return bytesBuffer.Bytes() +} + // Write object header func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) { // set common headers diff --git a/cmd/api-response.go b/cmd/api-response.go index f0c2bfd86..eb0c7940c 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -166,13 +166,12 @@ type ListBucketsResponse struct { // Upload container for in progress multipart upload type Upload struct { - Key string - UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - StorageClass string - Initiated string - HealUploadInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"` + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass string + Initiated string } // CommonPrefix container for prefix response in ListObjectsResponse @@ -182,9 +181,8 @@ type CommonPrefix struct { // Bucket container for bucket metadata type Bucket struct { - Name string - CreationDate string // time string of format "2006-01-02T15:04:05.000Z" - HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"` + Name string + CreationDate string // time string of format "2006-01-02T15:04:05.000Z" } // Object container for object metadata @@ -198,8 +196,7 @@ type Object struct { Owner Owner // The class of storage used to store the object. - StorageClass string - HealObjectInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"` + StorageClass string } // CopyObjectResponse container returns ETag and LastModified of the successfully copied object @@ -308,7 +305,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { var listbucket = Bucket{} listbucket.Name = bucket.Name listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong) - listbucket.HealBucketInfo = bucket.HealBucketInfo listbuckets = append(listbuckets, listbucket) } @@ -339,8 +335,6 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max content.Size = object.Size content.StorageClass = globalMinioDefaultStorageClass content.Owner = owner - // object.HealObjectInfo is non-empty only when resp is constructed in ListObjectsHeal. - content.HealObjectInfo = object.HealObjectInfo contents = append(contents, content) } // TODO - support EncodingType in xml decoding @@ -498,7 +492,6 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult newUpload.UploadID = upload.UploadID newUpload.Key = upload.Object newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong) - newUpload.HealUploadInfo = upload.HealUploadInfo listMultipartUploadsResponse.Uploads[index] = newUpload } return listMultipartUploadsResponse @@ -584,3 +577,31 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode apiError := getAPIError(errorCode) writeResponse(w, apiError.HTTPStatusCode, nil, mimeNone) } + +// writeErrorResponseJSON - writes error response in JSON format; +// useful for admin APIs. +func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) { + apiError := getAPIError(errorCode) + // Generate error response. + errorResponse := getAPIErrorResponse(apiError, reqURL.Path) + encodedErrorResponse := encodeResponseJSON(errorResponse) + writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON) +} + +// writeCustomErrorResponseJSON - similar to writeErrorResponseJSON, +// but accepts the error message directly (this allows messages to be +// dynamically generated.) +func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, + errBody string, reqURL *url.URL) { + + apiError := getAPIError(errorCode) + errorResponse := APIErrorResponse{ + Code: apiError.Code, + Message: errBody, + Resource: reqURL.Path, + RequestID: "3L137", + HostID: "3L137", + } + encodedErrorResponse := encodeResponseJSON(errorResponse) + writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON) +} diff --git a/cmd/endpoint.go b/cmd/endpoint.go index f0a3d678d..c7c396cc3 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -186,6 +186,15 @@ func (endpoints EndpointList) IsHTTPS() bool { return endpoints[0].IsHTTPS() } +// GetString - returns endpoint string of i-th endpoint (0-based), +// and empty string for invalid indexes. +func (endpoints EndpointList) GetString(i int) string { + if i < 0 || i >= len(endpoints) { + return "" + } + return endpoints[i].String() +} + // NewEndpointList - returns new endpoint list based on input args. func NewEndpointList(args ...string) (endpoints EndpointList, err error) { // isValidDistribution - checks whether given count is a valid distribution for erasure coding. diff --git a/cmd/format-xl.go b/cmd/format-xl.go index 4cbc7e91b..96b1165f7 100644 --- a/cmd/format-xl.go +++ b/cmd/format-xl.go @@ -143,14 +143,14 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) { // Initialize format configs. var formats = make([]*formatXLV1, len(bootstrapDisks)) - // Make a volume entry on all underlying storage disks. + // Load format from each disk in parallel for index, disk := range bootstrapDisks { if disk == nil { sErrs[index] = errDiskNotFound continue } wg.Add(1) - // Make a volume inside a go-routine. + // Launch go-routine per disk. go func(index int, disk StorageAPI) { defer wg.Done() format, lErr := loadFormat(disk) @@ -162,15 +162,9 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) { }(index, disk) } - // Wait for all make vol to finish. + // Wait for all go-routines to finish. wg.Wait() - for _, err := range sErrs { - if err != nil { - // Return all formats and errors. - return formats, sErrs - } - } // Return all formats and nil return formats, sErrs } @@ -374,8 +368,10 @@ func loadFormat(disk StorageAPI) (format *formatXLV1, err error) { if err != nil { return nil, err } - if len(vols) > 1 { - // 'format.json' not found, but we found user data. + if len(vols) > 1 || (len(vols) == 1 && + vols[0].Name != minioMetaBucket) { + // 'format.json' not found, but we + // found user data. return nil, errCorruptedFormat } // No other data found, its a fresh disk. @@ -394,11 +390,10 @@ func loadFormat(disk StorageAPI) (format *formatXLV1, err error) { return format, nil } -// collectNSaveNewFormatConfigs - creates new format configs based on -// the reference config and saves it on all disks, this is to be -// called from healFormatXL* functions. +// collectNSaveNewFormatConfigs - generates new format configs based on +// the given ref. config and saves on each disk func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1, - orderedDisks []StorageAPI) error { + orderedDisks []StorageAPI, dryRun bool) error { // Collect new format configs that need to be written. var newFormatConfigs = make([]*formatXLV1, len(orderedDisks)) @@ -420,16 +415,19 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1, } // Save new `format.json` across all disks, in JBOD order. - return saveFormatXL(orderedDisks, newFormatConfigs) + if !dryRun { + return saveFormatXL(orderedDisks, newFormatConfigs) + } + return nil } // Heals any missing format.json on the drives. Returns error only for // unexpected errors as regular errors can be ignored since there // might be enough quorum to be operational. Heals only fresh disks. func healFormatXLFreshDisks(storageDisks []StorageAPI, - formats []*formatXLV1) error { + formats []*formatXLV1, dryRun bool) error { - // Reorder the disks based on the JBOD order. + // Reorder disks based on JBOD order, and get reference config. referenceConfig, orderedDisks, err := reorderDisks(storageDisks, formats, true) if err != nil { @@ -441,21 +439,24 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI, // and allowed fresh disks to be arranged anywhere. // Following block facilitates to put fresh disks. for index, format := range formats { + if format != nil { + continue + } + // Format is missing so we go through ordered disks. - if format == nil { - // At this point when disk is missing the fresh disk - // in the stack get it back from storageDisks. - for oIndex, disk := range orderedDisks { - if disk == nil { - orderedDisks[oIndex] = storageDisks[index] - break - } + // At this point when disk is missing the fresh disk + // in the stack get it back from storageDisks. + for oIndex, disk := range orderedDisks { + if disk == nil { + orderedDisks[oIndex] = storageDisks[index] + break } } } // apply new format config and save to all disks - return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks) + return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks, + dryRun) } // collectUnAssignedDisks - collect disks unassigned to orderedDisks @@ -540,9 +541,9 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, // Heals corrupted format json in all disks func healFormatXLCorruptedDisks(storageDisks []StorageAPI, - formats []*formatXLV1) error { + formats []*formatXLV1, dryRun bool) error { - // Reorder the disks based on the JBOD order. + // Reorder disks based on JBOD order, and update ref. config. referenceConfig, orderedDisks, err := reorderDisks(storageDisks, formats, true) if err != nil { @@ -570,8 +571,9 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI, } } - // apply new format config and save to all disks - return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks) + // generate and write new configs to all disks + return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks, + dryRun) } // loadFormatXL - loads XL `format.json` and returns back properly diff --git a/cmd/format-xl_test.go b/cmd/format-xl_test.go index 17271aa4d..8f6f5960d 100644 --- a/cmd/format-xl_test.go +++ b/cmd/format-xl_test.go @@ -278,7 +278,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) { formatConfigs, _ := loadAllFormats(storageDisks) // Start healing disks - err = healFormatXLFreshDisks(storageDisks, formatConfigs) + err = healFormatXLFreshDisks(storageDisks, formatConfigs, false) if err != nil { t.Fatal("healing corrupted disk failed: ", err) } @@ -352,7 +352,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) { formatConfigs, _ := loadAllFormats(permutedStorageDisks) // Start healing disks - err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs) + err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs, false) if err != nil { t.Fatal("healing corrupted disk failed: ", err) } @@ -761,7 +761,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { xl := obj.(*xlObjects) formatConfigs, _ := loadAllFormats(xl.storageDisks) - if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != nil { + if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err != nil { t.Fatal("Got an unexpected error: ", err) } @@ -784,7 +784,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk) formatConfigs, _ = loadAllFormats(xl.storageDisks) - if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk { + if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err != errFaultyDisk { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -806,7 +806,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } } formatConfigs, _ = loadAllFormats(xl.storageDisks) - if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err == nil { + if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err == nil { t.Fatal("Should get a json parsing error, ") } removeRoots(fsDirs) @@ -833,7 +833,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } xl := obj.(*xlObjects) formatConfigs, _ := loadAllFormats(xl.storageDisks) - if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil { + if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != nil { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -855,7 +855,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk) formatConfigs, _ = loadAllFormats(xl.storageDisks) - if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk { + if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != errFaultyDisk { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -873,7 +873,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { xl = obj.(*xlObjects) xl.storageDisks[0] = nil formatConfigs, _ = loadAllFormats(xl.storageDisks) - if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil { + if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != nil { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 892d31366..ee224f97c 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -31,6 +31,7 @@ import ( "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/lock" + "github.com/minio/minio/pkg/madmin" ) // fsObjects - Implements fs object layer. @@ -1039,13 +1040,17 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey } // HealObject - no-op for fs. Valid only for XL. -func (fs fsObjects) HealObject(bucket, object string) (int, int, error) { - return 0, 0, errors.Trace(NotImplemented{}) +func (fs fsObjects) HealObject(bucket, object string, dryRun bool) ( + res madmin.HealResultItem, err error) { + + return res, errors.Trace(NotImplemented{}) } // HealBucket - no-op for fs, Valid only for XL. -func (fs fsObjects) HealBucket(bucket string) error { - return errors.Trace(NotImplemented{}) +func (fs fsObjects) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, + error) { + + return nil, errors.Trace(NotImplemented{}) } // ListObjectsHeal - list all objects to be healed. Valid only for XL diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 51e3ac43a..2f6c7d47d 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -400,7 +400,7 @@ func TestFSHealObject(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - _, _, err := obj.HealObject("bucket", "object") + _, err := obj.HealObject("bucket", "object", false) if err == nil || !isSameType(errors.Cause(err), NotImplemented{}) { t.Fatalf("Heal Object should return NotImplemented error ") } diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index b5730db2d..9a3478c50 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -23,6 +23,7 @@ import ( "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" + "github.com/minio/minio/pkg/madmin" ) // GatewayUnsupported list of unsupported call stubs for gateway. @@ -38,8 +39,8 @@ func (a GatewayUnsupported) NewMultipartUpload(bucket string, object string, met return "", errors.Trace(NotImplemented{}) } -// CopyObjectPart copy part of object to other bucket and object -func (a GatewayUnsupported) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string, srcEtag string) (pi PartInfo, err error) { +// CopyObjectPart copy part of object to uploadID for another object +func (a GatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, metadata map[string]string, srcETag string) (pi PartInfo, err error) { return pi, errors.Trace(NotImplemented{}) } @@ -79,8 +80,8 @@ func (a GatewayUnsupported) DeleteBucketPolicies(bucket string) error { } // HealBucket - Not implemented stub -func (a GatewayUnsupported) HealBucket(bucket string) error { - return errors.Trace(NotImplemented{}) +func (a GatewayUnsupported) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, error) { + return nil, errors.Trace(NotImplemented{}) } // ListBucketsHeal - Not implemented stub @@ -89,8 +90,8 @@ func (a GatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) } // HealObject - Not implemented stub -func (a GatewayUnsupported) HealObject(bucket, object string) (int, int, error) { - return 0, 0, errors.Trace(NotImplemented{}) +func (a GatewayUnsupported) HealObject(bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) { + return h, errors.Trace(NotImplemented{}) } // ListObjectsV2 - Not implemented stub diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index b08df3646..f038ec001 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -246,6 +246,12 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.handler.ServeHTTP(w, r) } +// Check to allow access to the reserved "bucket" `/minio` for Admin +// API requests. +func isAdminReq(r *http.Request) bool { + return strings.HasPrefix(r.URL.Path, adminAPIPathPrefix+"/") +} + // Adds verification for incoming paths. type minioReservedBucketHandler struct { handler http.Handler @@ -256,8 +262,12 @@ func setReservedBucketHandler(h http.Handler) http.Handler { } func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if !guessIsRPCReq(r) && !guessIsBrowserReq(r) { - // For all non browser, non RPC requests, reject access to 'minioReservedBucketPath'. + switch { + case guessIsRPCReq(r), guessIsBrowserReq(r), isAdminReq(r): + // Allow access to reserved buckets + default: + // For all other requests reject access to reserved + // buckets bucketName, _ := urlPath2BucketObjectName(r.URL) if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) { writeErrorResponse(w, ErrAllAccessDisabled, r.URL) @@ -433,6 +443,11 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } + // A put method on path "/" doesn't make sense, ignore it. + if r.Method == http.MethodPut && r.URL.Path == "/" { + writeErrorResponse(w, ErrNotImplemented, r.URL) + return + } // Serve HTTP. h.handler.ServeHTTP(w, r) diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index ce64cb2f5..24d2f1660 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -50,21 +50,6 @@ type StorageInfo struct { } } -type healStatus int - -const ( - healthy healStatus = iota // Object is healthy - canHeal // Object can be healed - corrupted // Object can't be healed - quorumUnavailable // Object can't be healed until read quorum is available - canPartiallyHeal // Object can't be healed completely until outdated disk(s) are online. -) - -// HealBucketInfo - represents healing related information of a bucket. -type HealBucketInfo struct { - Status healStatus -} - // BucketInfo - represents bucket metadata. type BucketInfo struct { // Name of the bucket. @@ -72,16 +57,6 @@ type BucketInfo struct { // Date and time when the bucket was created. Created time.Time - - // Healing information - HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"` -} - -// HealObjectInfo - represents healing related information of an object. -type HealObjectInfo struct { - Status healStatus - MissingDataCount int - MissingParityCount int } // ObjectInfo - represents object metadata. @@ -113,8 +88,7 @@ type ObjectInfo struct { ContentEncoding string // User-Defined metadata - UserDefined map[string]string - HealObjectInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"` + UserDefined map[string]string } // ListPartsInfo - represents list of all parts. @@ -273,8 +247,6 @@ type MultipartInfo struct { Initiated time.Time StorageClass string // Not supported yet. - - HealUploadInfo *HealObjectInfo `xml:"HealUploadInfo,omitempty"` } // CompletePart - represents the part that was completed, this is sent by the client diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go index 7bb14955d..a23d065f0 100644 --- a/cmd/object-api-input-checks.go +++ b/cmd/object-api-input-checks.go @@ -153,9 +153,6 @@ func checkPutObjectArgs(bucket, object string, obj ObjectLayer) error { // Checks whether bucket exists and returns appropriate error if not. func checkBucketExist(bucket string, obj ObjectLayer) error { - if !IsValidBucketName(bucket) { - return BucketNameInvalid{Bucket: bucket} - } _, err := obj.GetBucketInfo(bucket) if err != nil { return errors.Cause(err) diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 839569c4c..c7a850737 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -21,6 +21,7 @@ import ( "time" "github.com/minio/minio/pkg/hash" + "github.com/minio/minio/pkg/madmin" ) // ObjectLayer implements primitives for object API layer. @@ -53,9 +54,9 @@ type ObjectLayer interface { CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) // Healing operations. - HealBucket(bucket string) error + HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, error) + HealObject(bucket, object string, dryRun bool) (madmin.HealResultItem, error) ListBucketsHeal() (buckets []BucketInfo, err error) - HealObject(bucket, object string) (int, int, error) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) // Locking operations diff --git a/cmd/routers.go b/cmd/routers.go index 76b27f73f..f575dd4a6 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -70,6 +70,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) { return nil, err } + // Add Admin router. + registerAdminRouter(mux) + // Register web router when its enabled. if globalIsBrowserEnabled { if err := registerWebRouter(mux); err != nil { @@ -77,9 +80,6 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) { } } - // Add Admin router. - registerAdminRouter(mux) - // Add API router. registerAPIRouter(mux) diff --git a/cmd/server-main.go b/cmd/server-main.go index 1a67dfe77..31dbaaaf6 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -193,6 +193,9 @@ func serverMain(ctx *cli.Context) { // Initialize name space lock. initNSLock(globalIsDistXL) + // Init global heal state + initAllHealState(globalIsXL) + // Configure server. var handler http.Handler handler, err = configureServerHandler(globalEndpoints) diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 9830324e9..3803aa62c 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -497,6 +497,17 @@ func resetGlobalStorageEnvs() { globalRRStorageClass = storageClass{} } +// reset global heal state +func resetGlobalHealState() { + globalAllHealState.Lock() + defer globalAllHealState.Unlock() + for _, v := range globalAllHealState.healSeqMap { + if !v.hasEnded() { + v.stop() + } + } +} + // Resets all the globals used modified in tests. // Resetting ensures that the changes made to globals by one test doesn't affect others. func resetTestGlobals() { @@ -518,6 +529,8 @@ func resetTestGlobals() { resetGlobalIsEnvs() // Reset global storage class flags resetGlobalStorageEnvs() + // Reset global heal state + resetGlobalHealState() } // Configure the server for the test run. diff --git a/cmd/xl-v1-healing-common.go b/cmd/xl-v1-healing-common.go index 5e0e964ef..414be858e 100644 --- a/cmd/xl-v1-healing-common.go +++ b/cmd/xl-v1-healing-common.go @@ -37,11 +37,7 @@ func commonTime(modTimes []time.Time) (modTime time.Time, count int) { // Find the common cardinality from previously collected // occurrences of elements. for time, count := range timeOccurenceMap { - if count == maxima && time.After(modTime) { - maxima = count - modTime = time - - } else if count > maxima { + if count > maxima || (count == maxima && time.After(modTime)) { maxima = count modTime = time } @@ -126,7 +122,7 @@ func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) { // List all the file commit ids from parts metadata. modTimes := listObjectModtimes(partsMetadata, errs) - // Count all lastest updated xlMeta values + // Count all latest updated xlMeta values var count int var latestXLMeta xlMetaV1 @@ -144,140 +140,13 @@ func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) { return latestXLMeta, count } -// outDatedDisks - return disks which don't have the latest object (i.e xl.json). -// disks that are offline are not 'marked' outdated. -func outDatedDisks(disks, latestDisks []StorageAPI, errs []error, partsMetadata []xlMetaV1, - bucket, object string) (outDatedDisks []StorageAPI) { - - outDatedDisks = make([]StorageAPI, len(disks)) - for index, latestDisk := range latestDisks { - if latestDisk != nil { - continue - } - // disk either has an older xl.json or doesn't have one. - switch errors.Cause(errs[index]) { - case nil, errFileNotFound: - outDatedDisks[index] = disks[index] - } - } - - return outDatedDisks -} - -// Returns if the object should be healed. -func xlShouldHeal(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket, object string) bool { - onlineDisks, _ := listOnlineDisks(disks, partsMetadata, - errs) - // Return true even if one of the disks have stale data. - for _, disk := range onlineDisks { - if disk == nil { - return true - } - } - - // Check if all parts of an object are available and their - // checksums are valid. - availableDisks, _, err := disksWithAllParts(onlineDisks, partsMetadata, - errs, bucket, object) - if err != nil { - // Note: This error is due to failure of blake2b - // checksum computation of a part. It doesn't clearly - // indicate if the object needs healing. At this - // juncture healing could fail with the same - // error. So, we choose to return that there is no - // need to heal. - return false - } - - // Return true even if one disk has xl.json or one or more - // parts missing. - for _, disk := range availableDisks { - if disk == nil { - return true - } - } - - return false -} - -// xlHealStat - returns a structure which describes how many data, -// parity erasure blocks are missing and if it is possible to heal -// with the blocks present. -func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObjectInfo { - // Less than quorum erasure coded blocks of the object have the same create time. - // This object can't be healed with the information we have. - modTime, count := commonTime(listObjectModtimes(partsMetadata, errs)) - - // get read quorum for this object - readQuorum, _, err := objectQuorumFromMeta(xl, partsMetadata, errs) - - if count < readQuorum || err != nil { - return HealObjectInfo{ - Status: quorumUnavailable, - MissingDataCount: 0, - MissingParityCount: 0, - } - } - - // If there isn't a valid xlMeta then we can't heal the object. - xlMeta, err := pickValidXLMeta(partsMetadata, modTime) - if err != nil { - return HealObjectInfo{ - Status: corrupted, - MissingDataCount: 0, - MissingParityCount: 0, - } - } - - // Compute heal statistics like bytes to be healed, missing - // data and missing parity count. - missingDataCount := 0 - missingParityCount := 0 - - disksMissing := false - for i, err := range errs { - // xl.json is not found, which implies the erasure - // coded blocks are unavailable in the corresponding disk. - // First half of the disks are data and the rest are parity. - switch realErr := errors.Cause(err); realErr { - case errDiskNotFound: - disksMissing = true - fallthrough - case errFileNotFound: - if xlMeta.Erasure.Distribution[i]-1 < xlMeta.Erasure.DataBlocks { - missingDataCount++ - } else { - missingParityCount++ - } - } - } - - // The object may not be healed completely, since some of the - // disks needing healing are unavailable. - if disksMissing { - return HealObjectInfo{ - Status: canPartiallyHeal, - MissingDataCount: missingDataCount, - MissingParityCount: missingParityCount, - } - } - - // This object can be healed. We have enough object metadata - // to reconstruct missing erasure coded blocks. - return HealObjectInfo{ - Status: canHeal, - MissingDataCount: missingDataCount, - MissingParityCount: missingParityCount, - } -} - // disksWithAllParts - This function needs to be called with // []StorageAPI returned by listOnlineDisks. Returns, // // - disks which have all parts specified in the latest xl.json. // -// - errs updated to have errFileNotFound in place of disks that had -// missing or corrupted parts. +// - slice of errors about the state of data files on disk - can have +// a not-found error or a hash-mismatch error. // // - non-nil error if any of the disks failed unexpectedly (i.e. error // other than file not found and not a checksum error). @@ -286,11 +155,13 @@ func disksWithAllParts(onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs availableDisks := make([]StorageAPI, len(onlineDisks)) buffer := []byte{} + dataErrs := make([]error, len(onlineDisks)) for i, onlineDisk := range onlineDisks { - if onlineDisk == OfflineDisk { + if onlineDisk == nil { continue } + // disk has a valid xl.json but may not have all the // parts. This is considered an outdated disk, since // it needs healing too. @@ -302,22 +173,25 @@ func disksWithAllParts(onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs // verification happens even if a 0-length // buffer is passed _, hErr := onlineDisk.ReadFile(bucket, partPath, 0, buffer, verifier) - if hErr != nil { - _, isCorrupted := hErr.(hashMismatchError) - if isCorrupted || hErr == errFileNotFound { - errs[i] = errFileNotFound - availableDisks[i] = OfflineDisk - break - } + + _, isCorrupt := hErr.(hashMismatchError) + switch { + case isCorrupt: + fallthrough + case hErr == errFileNotFound, hErr == errVolumeNotFound: + dataErrs[i] = hErr + break + case hErr != nil: + // abort on unhandled errors return nil, nil, errors.Trace(hErr) } } - if errs[i] == nil { + if dataErrs[i] == nil { // All parts verified, mark it as all data available. availableDisks[i] = onlineDisk } } - return availableDisks, errs, nil + return availableDisks, dataErrs, nil } diff --git a/cmd/xl-v1-healing-common_test.go b/cmd/xl-v1-healing-common_test.go index c93ef352d..a7df1060a 100644 --- a/cmd/xl-v1-healing-common_test.go +++ b/cmd/xl-v1-healing-common_test.go @@ -107,20 +107,6 @@ func partsMetaFromModTimes(modTimes []time.Time, algorithm BitrotAlgorithm, chec return partsMetadata } -// toPosix - fetches *posix object from StorageAPI. -func toPosix(disk StorageAPI) *posix { - retryDisk, ok := disk.(*retryStorage) - if !ok { - return nil - } - pDisk, ok := retryDisk.remoteStorage.(*posix) - if !ok { - return nil - } - return pDisk - -} - // TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks // are consistent with each other. func TestListOnlineDisks(t *testing.T) { @@ -273,62 +259,22 @@ func TestListOnlineDisks(t *testing.T) { partsMetadata := partsMetaFromModTimes(test.modTimes, DefaultBitrotAlgorithm, xlMeta.Erasure.Checksums) onlineDisks, modTime := listOnlineDisks(xlDisks, partsMetadata, test.errs) - availableDisks, newErrs, _ := disksWithAllParts(onlineDisks, partsMetadata, test.errs, bucket, object) - test.errs = newErrs - outdatedDisks := outDatedDisks(xlDisks, availableDisks, test.errs, partsMetadata, bucket, object) - if modTime.Equal(timeSentinel) { - t.Fatalf("Test %d: modTime should never be equal to timeSentinel, but found equal", - i+1) - } - - if test._tamperBackend != noTamper { - if tamperedIndex != -1 && outdatedDisks[tamperedIndex] == nil { - t.Fatalf("Test %d: disk (%v) with part.1 missing is an outdated disk, but wasn't listed by outDatedDisks", - i+1, xlDisks[tamperedIndex]) - } - - } - if !modTime.Equal(test.expectedTime) { t.Fatalf("Test %d: Expected modTime to be equal to %v but was found to be %v", i+1, test.expectedTime, modTime) } - // Check if a disk is considered both online and outdated, - // which is a contradiction, except if parts are missing. - overlappingDisks := make(map[string]*posix) - for _, availableDisk := range availableDisks { - if availableDisk == nil { - continue - } - pDisk := toPosix(availableDisk) - overlappingDisks[pDisk.diskPath] = pDisk - } - - for index, outdatedDisk := range outdatedDisks { - // ignore the intentionally tampered disk, - // this is expected to appear as outdated - // disk, since it doesn't have all the parts. - if index == tamperedIndex { - continue - } - - if outdatedDisk == nil { - continue - } + availableDisks, newErrs, _ := disksWithAllParts(onlineDisks, partsMetadata, test.errs, bucket, object) + test.errs = newErrs - pDisk := toPosix(outdatedDisk) - if _, ok := overlappingDisks[pDisk.diskPath]; ok { - t.Errorf("Test %d: Outdated disk %v was also detected as an online disk - %v %v", - i+1, pDisk, availableDisks, outdatedDisks) + if test._tamperBackend != noTamper { + if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil { + t.Fatalf("Test %d: disk (%v) with part.1 missing is not a disk with available data", + i+1, xlDisks[tamperedIndex]) } - // errors other than errFileNotFound doesn't imply that the disk is outdated. - if test.errs[index] != nil && test.errs[index] != errFileNotFound && outdatedDisk != nil { - t.Errorf("Test %d: error (%v) other than errFileNotFound doesn't imply that the disk (%v) could be outdated", - i+1, test.errs[index], pDisk) - } } + } } @@ -412,8 +358,8 @@ func TestDisksWithAllParts(t *testing.T) { } } - // Test that all disks are returned without any failures with unmodified - // meta data + // Test that all disks are returned without any failures with + // unmodified meta data partsMetadata, errs = readAllXLMetadata(xlDisks, bucket, object) if err != nil { t.Fatalf("Failed to read xl meta data %v", err) diff --git a/cmd/xl-v1-healing.go b/cmd/xl-v1-healing.go index 0b0fd8946..3ef1dbbc7 100644 --- a/cmd/xl-v1-healing.go +++ b/cmd/xl-v1-healing.go @@ -23,11 +23,14 @@ import ( "sync" "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/pkg/madmin" ) // healFormatXL - heals missing `format.json` on freshly or corrupted // disks (missing format.json but does have erasure coded data in it). -func healFormatXL(storageDisks []StorageAPI) (err error) { +func healFormatXL(storageDisks []StorageAPI, dryRun bool) (res madmin.HealResultItem, + err error) { + // Attempt to load all `format.json`. formatConfigs, sErrs := loadAllFormats(storageDisks) @@ -35,7 +38,34 @@ func healFormatXL(storageDisks []StorageAPI) (err error) { // - if (no quorum) return error // - if (disks not recognized) // Always error. if err = genericFormatCheckXL(formatConfigs, sErrs); err != nil { - return err + return res, err + } + + // Prepare heal-result + res = madmin.HealResultItem{ + Type: madmin.HealItemMetadata, + Detail: "disk-format", + DiskCount: len(storageDisks), + } + res.InitDrives() + // Existing formats are available (i.e. ok), so save it in + // result, also populate disks to be healed. + for i, format := range formatConfigs { + drive := globalEndpoints.GetString(i) + switch { + case format != nil: + res.DriveInfo.Before[drive] = madmin.DriveStateOk + case sErrs[i] == errCorruptedFormat: + res.DriveInfo.Before[drive] = madmin.DriveStateCorrupt + case sErrs[i] == errUnformattedDisk: + res.DriveInfo.Before[drive] = madmin.DriveStateMissing + default: + res.DriveInfo.Before[drive] = madmin.DriveStateOffline + } + } + // Copy "after" drive state too + for k, v := range res.DriveInfo.Before { + res.DriveInfo.After[k] = v } numDisks := len(storageDisks) @@ -45,61 +75,95 @@ func healFormatXL(storageDisks []StorageAPI) (err error) { switch { case unformattedDiskCount == numDisks: // all unformatted. - if err = initFormatXL(storageDisks); err != nil { - return err + if !dryRun { + err = initFormatXL(storageDisks) + if err != nil { + return res, err + } + for i := 0; i < len(storageDisks); i++ { + drive := globalEndpoints.GetString(i) + res.DriveInfo.After[drive] = madmin.DriveStateOk + } } + return res, nil case diskNotFoundCount > 0: - return fmt.Errorf("cannot proceed with heal as %s", + return res, fmt.Errorf("cannot proceed with heal as %s", errSomeDiskOffline) case otherErrCount > 0: - return fmt.Errorf("cannot proceed with heal as some disks had unhandled errors") + return res, fmt.Errorf("cannot proceed with heal as some disks had unhandled errors") case corruptedFormatCount > 0: - if err = healFormatXLCorruptedDisks(storageDisks, formatConfigs); err != nil { - return fmt.Errorf("Unable to repair corrupted format, %s", err) + // heal corrupted disks + err = healFormatXLCorruptedDisks(storageDisks, formatConfigs, + dryRun) + if err != nil { + return res, err } + // success + if !dryRun { + for i := 0; i < len(storageDisks); i++ { + drive := globalEndpoints.GetString(i) + res.DriveInfo.After[drive] = madmin.DriveStateOk + } + } + return res, nil case unformattedDiskCount > 0: - // All drives online but some report missing format.json. - if err = healFormatXLFreshDisks(storageDisks, formatConfigs); err != nil { - // There was an unexpected unrecoverable error - // during healing. - return fmt.Errorf("Unable to heal backend %s", err) + // heal unformatted disks + err = healFormatXLFreshDisks(storageDisks, formatConfigs, + dryRun) + if err != nil { + return res, err } - + // success + if !dryRun { + for i := 0; i < len(storageDisks); i++ { + drive := globalEndpoints.GetString(i) + res.DriveInfo.After[drive] = madmin.DriveStateOk + } + } + return res, nil } - return nil + + return res, nil } // Heals a bucket if it doesn't exist on one of the disks, additionally // also heals the missing entries for bucket metadata files // `policy.json, notification.xml, listeners.json`. -func (xl xlObjects) HealBucket(bucket string) error { - if err := checkBucketExist(bucket, xl); err != nil { - return err +func (xl xlObjects) HealBucket(bucket string, dryRun bool) ( + results []madmin.HealResultItem, err error) { + + if err = checkBucketExist(bucket, xl); err != nil { + return nil, err } // get write quorum for an object writeQuorum := len(xl.storageDisks)/2 + 1 bucketLock := xl.nsMutex.NewNSLock(bucket, "") - if err := bucketLock.GetLock(globalHealingTimeout); err != nil { - return err + if err = bucketLock.GetLock(globalHealingTimeout); err != nil { + return nil, err } defer bucketLock.Unlock() // Heal bucket. - if err := healBucket(xl.storageDisks, bucket, writeQuorum); err != nil { - return err + result, err := healBucket(xl.storageDisks, bucket, writeQuorum, dryRun) + if err != nil { + return results, err } + results = append(results, result) // Proceed to heal bucket metadata. - return healBucketMetadata(xl, bucket) + metaResults, err := healBucketMetadata(xl, bucket, dryRun) + results = append(results, metaResults...) + return results, err } // Heal bucket - create buckets on disks where it does not exist. -func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error { +func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int, + dryRun bool) (res madmin.HealResultItem, err error) { // Initialize sync waitgroup. var wg = &sync.WaitGroup{} @@ -107,24 +171,47 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error // Initialize list of errors. var dErrs = make([]error, len(storageDisks)) + // Disk states slices + beforeState := make([]string, len(storageDisks)) + afterState := make([]string, len(storageDisks)) + // Make a volume entry on all underlying storage disks. for index, disk := range storageDisks { if disk == nil { dErrs[index] = errors.Trace(errDiskNotFound) + beforeState[index] = madmin.DriveStateOffline + afterState[index] = madmin.DriveStateOffline continue } wg.Add(1) + // Make a volume inside a go-routine. go func(index int, disk StorageAPI) { defer wg.Done() if _, err := disk.StatVol(bucket); err != nil { - if err != errVolumeNotFound { + if errors.Cause(err) != errVolumeNotFound { + beforeState[index] = madmin.DriveStateCorrupt + afterState[index] = madmin.DriveStateCorrupt dErrs[index] = errors.Trace(err) return } - if err = disk.MakeVol(bucket); err != nil { - dErrs[index] = errors.Trace(err) + + beforeState[index] = madmin.DriveStateMissing + afterState[index] = madmin.DriveStateMissing + + // mutate only if not a dry-run + if dryRun { + return + } + + makeErr := disk.MakeVol(bucket) + dErrs[index] = errors.Trace(makeErr) + if makeErr == nil { + afterState[index] = madmin.DriveStateOk } + } else { + beforeState[index] = madmin.DriveStateOk + afterState[index] = madmin.DriveStateOk } }(index, disk) } @@ -132,44 +219,75 @@ func healBucket(storageDisks []StorageAPI, bucket string, writeQuorum int) error // Wait for all make vol to finish. wg.Wait() + // Initialize heal result info + res = madmin.HealResultItem{ + Type: madmin.HealItemBucket, + Bucket: bucket, + DiskCount: len(storageDisks), + } + res.InitDrives() + for i, before := range beforeState { + drive := globalEndpoints.GetString(i) + res.DriveInfo.Before[drive] = before + res.DriveInfo.After[drive] = afterState[i] + } + reducedErr := reduceWriteQuorumErrs(dErrs, bucketOpIgnoredErrs, writeQuorum) if errors.Cause(reducedErr) == errXLWriteQuorum { // Purge successfully created buckets if we don't have writeQuorum. undoMakeBucket(storageDisks, bucket) } - return reducedErr + return res, reducedErr } // Heals all the metadata associated for a given bucket, this function // heals `policy.json`, `notification.xml` and `listeners.json`. -func healBucketMetadata(xlObj xlObjects, bucket string) error { +func healBucketMetadata(xl xlObjects, bucket string, dryRun bool) ( + results []madmin.HealResultItem, err error) { + healBucketMetaFn := func(metaPath string) error { - if _, _, err := xlObj.HealObject(minioMetaBucket, metaPath); err != nil && !isErrObjectNotFound(err) { - return err + result, healErr := xl.HealObject(minioMetaBucket, metaPath, dryRun) + // If object is not found, no result to add. + if isErrObjectNotFound(healErr) { + return nil + } + if healErr != nil { + return healErr } + result.Type = madmin.HealItemBucketMetadata + results = append(results, result) return nil } - // Heal `policy.json` for missing entries, ignores if `policy.json` is not found. + // Heal `policy.json` for missing entries, ignores if + // `policy.json` is not found. policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig) - if err := healBucketMetaFn(policyPath); err != nil { - return err + err = healBucketMetaFn(policyPath) + if err != nil { + return results, err } - // Heal `notification.xml` for missing entries, ignores if `notification.xml` is not found. - nConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig) - if err := healBucketMetaFn(nConfigPath); err != nil { - return err + // Heal `notification.xml` for missing entries, ignores if + // `notification.xml` is not found. + nConfigPath := path.Join(bucketConfigPrefix, bucket, + bucketNotificationConfig) + err = healBucketMetaFn(nConfigPath) + if err != nil { + return results, err } - // Heal `listeners.json` for missing entries, ignores if `listeners.json` is not found. + // Heal `listeners.json` for missing entries, ignores if + // `listeners.json` is not found. lConfigPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig) - return healBucketMetaFn(lConfigPath) + err = healBucketMetaFn(lConfigPath) + return results, err } // listAllBuckets lists all buckets from all disks. It also // returns the occurrence of each buckets in all disks -func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, bucketsOcc map[string]int, err error) { +func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, + bucketsOcc map[string]int, err error) { + buckets = make(map[string]VolInfo) bucketsOcc = make(map[string]int) for _, disk := range storageDisks { @@ -178,122 +296,42 @@ func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, buck } var volsInfo []VolInfo volsInfo, err = disk.ListVols() - if err == nil { - for _, volInfo := range volsInfo { - // StorageAPI can send volume names which are - // incompatible with buckets, handle it and skip them. - if !IsValidBucketName(volInfo.Name) { - continue - } - // Skip special volume buckets. - if isMinioMetaBucketName(volInfo.Name) { - continue - } - // Increase counter per bucket name - bucketsOcc[volInfo.Name]++ - // Save volume info under bucket name - buckets[volInfo.Name] = volInfo + if err != nil { + if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { + continue } - continue - } - // Ignore any disks not found. - if errors.IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { - continue - } - break - } - return buckets, bucketsOcc, err -} - -// reduceHealStatus - fetches the worst heal status in a provided slice -func reduceHealStatus(status []healStatus) healStatus { - worstStatus := healthy - for _, st := range status { - if st > worstStatus { - worstStatus = st + break } - } - return worstStatus -} - -// bucketHealStatus - returns the heal status of the provided bucket. Internally, -// this function lists all object heal status of objects inside meta bucket config -// directory and returns the worst heal status that can be found -func (xl xlObjects) bucketHealStatus(bucketName string) (healStatus, error) { - // A list of all the bucket config files - configFiles := []string{bucketPolicyConfig, bucketNotificationConfig, bucketListenerConfig} - // The status of buckets config files - configsHealStatus := make([]healStatus, len(configFiles)) - // The list of errors found during checking heal status of each config file - configsErrs := make([]error, len(configFiles)) - // The path of meta bucket that contains all config files - configBucket := path.Join(minioMetaBucket, bucketConfigPrefix, bucketName) - - // Check of config files heal status in go-routines - var wg sync.WaitGroup - // Loop over config files - for idx, configFile := range configFiles { - wg.Add(1) - // Compute heal status of current config file - go func(bucket, object string, index int) { - defer wg.Done() - // Check - listObjectsHeal, err := xl.listObjectsHeal(bucket, object, "", "", 1) - // If any error, save and immediately quit - if err != nil { - configsErrs[index] = err - return - } - // Check if current bucket contains any not healthy config file and save heal status - if len(listObjectsHeal.Objects) > 0 { - configsHealStatus[index] = listObjectsHeal.Objects[0].HealObjectInfo.Status + for _, volInfo := range volsInfo { + // StorageAPI can send volume names which are + // incompatible with buckets - these are + // skipped, like the meta-bucket. + if !IsValidBucketName(volInfo.Name) || + isMinioMetaBucketName(volInfo.Name) { + continue } - }(configBucket, configFile, idx) - } - wg.Wait() - - // Return any found error - for _, err := range configsErrs { - if err != nil { - return healthy, err + // Increase counter per bucket name + bucketsOcc[volInfo.Name]++ + // Save volume info under bucket name + buckets[volInfo.Name] = volInfo } } - - // Reduce and return heal status - return reduceHealStatus(configsHealStatus), nil + return buckets, bucketsOcc, err } // ListBucketsHeal - Find all buckets that need to be healed func (xl xlObjects) ListBucketsHeal() ([]BucketInfo, error) { listBuckets := []BucketInfo{} // List all buckets that can be found in all disks - buckets, occ, err := listAllBuckets(xl.storageDisks) + buckets, _, err := listAllBuckets(xl.storageDisks) if err != nil { return listBuckets, err } // Iterate over all buckets for _, currBucket := range buckets { - // Check the status of bucket metadata - bucketHealStatus, err := xl.bucketHealStatus(currBucket.Name) - if err != nil { - return []BucketInfo{}, err - } - // If all metadata are sane, check if the bucket directory is present in all disks - if bucketHealStatus == healthy && occ[currBucket.Name] != len(xl.storageDisks) { - // Current bucket is missing in some of the storage disks - bucketHealStatus = canHeal - } - // Add current bucket to the returned result if not healthy - if bucketHealStatus != healthy { - listBuckets = append(listBuckets, - BucketInfo{ - Name: currBucket.Name, - Created: currBucket.Created, - HealBucketInfo: &HealBucketInfo{Status: bucketHealStatus}, - }) - } - + listBuckets = append(listBuckets, + BucketInfo{currBucket.Name, currBucket.Created}) } // Sort found buckets @@ -323,8 +361,8 @@ func quickHeal(xlObj xlObjects, writeQuorum int, readQuorum int) error { defer bucketLock.Unlock() // Heal bucket and then proceed to heal bucket metadata if any. - if err = healBucket(xlObj.storageDisks, bucketName, writeQuorum); err == nil { - if err = healBucketMetadata(xlObj, bucketName); err == nil { + if _, err = healBucket(xlObj.storageDisks, bucketName, writeQuorum, false); err == nil { + if _, err = healBucketMetadata(xlObj, bucketName, false); err == nil { continue } return err @@ -337,77 +375,108 @@ func quickHeal(xlObj xlObjects, writeQuorum int, readQuorum int) error { return nil } -// Heals an object only the corrupted/missing erasure blocks. -func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (int, int, error) { +// Heals an object by re-writing corrupt/missing erasure blocks. +func healObject(storageDisks []StorageAPI, bucket string, object string, + quorum int, dryRun bool) (result madmin.HealResultItem, err error) { partsMetadata, errs := readAllXLMetadata(storageDisks, bucket, object) + // readQuorum suffices for xl.json since we use monotonic // system time to break the tie when a split-brain situation // arises. - if rErr := reduceReadQuorumErrs(errs, nil, quorum); rErr != nil { - return 0, 0, toObjectErr(rErr, bucket, object) + if reducedErr := reduceReadQuorumErrs(errs, nil, quorum); reducedErr != nil { + return result, toObjectErr(reducedErr, bucket, object) } - // List of disks having latest version of the object. + // List of disks having latest version of the object xl.json + // (by modtime). latestDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs) - // List of disks having all parts as per latest xl.json - this - // does a full pass over the data and verifies all part files - // on disk - availableDisks, errs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket, - object) + // List of disks having all parts as per latest xl.json. + availableDisks, dataErrs, aErr := disksWithAllParts(latestDisks, partsMetadata, errs, bucket, object) if aErr != nil { - return 0, 0, toObjectErr(aErr, bucket, object) + return result, toObjectErr(aErr, bucket, object) } - // Number of disks which don't serve data. - numOfflineDisks := 0 - for index, disk := range storageDisks { - if disk == nil || errs[index] == errDiskNotFound { - numOfflineDisks++ - } + // Initialize heal result object + result = madmin.HealResultItem{ + Type: madmin.HealItemObject, + Bucket: bucket, + Object: object, + DiskCount: len(storageDisks), + + // Initialize object size to -1, so we can detect if we are + // unable to reliably find the object size. + ObjectSize: -1, } + result.InitDrives() - // Number of disks which have all parts of the given object. + // Loop to find number of disks with valid data, per-drive + // data state and a list of outdated disks on which data needs + // to be healed. + outDatedDisks := make([]StorageAPI, len(storageDisks)) numAvailableDisks := 0 - for _, disk := range availableDisks { - if disk != nil { + disksToHealCount := 0 + for i, v := range availableDisks { + driveState := "" + switch { + case v != nil: + driveState = madmin.DriveStateOk numAvailableDisks++ + // If data is sane on any one disk, we can + // extract the correct object size. + result.ObjectSize = partsMetadata[i].Stat.Size + result.ParityBlocks = partsMetadata[i].Erasure.ParityBlocks + result.DataBlocks = partsMetadata[i].Erasure.DataBlocks + case errors.Cause(errs[i]) == errDiskNotFound: + driveState = madmin.DriveStateOffline + case errors.Cause(errs[i]) == errFileNotFound, errors.Cause(errs[i]) == errVolumeNotFound: + fallthrough + case errors.Cause(dataErrs[i]) == errFileNotFound, errors.Cause(dataErrs[i]) == errVolumeNotFound: + driveState = madmin.DriveStateMissing + default: + // all remaining cases imply corrupt data/metadata + driveState = madmin.DriveStateCorrupt + } + drive := globalEndpoints.GetString(i) + result.DriveInfo.Before[drive] = driveState + // copy for 'after' state + result.DriveInfo.After[drive] = driveState + + // an online disk without valid data/metadata is + // outdated and can be healed. + if errs[i] != errDiskNotFound && v == nil { + outDatedDisks[i] = storageDisks[i] + disksToHealCount++ } - } - - if numAvailableDisks == len(storageDisks) { - // nothing to heal in this case - return 0, 0, nil } // If less than read quorum number of disks have all the parts // of the data, we can't reconstruct the erasure-coded data. if numAvailableDisks < quorum { - return 0, 0, toObjectErr(errXLReadQuorum, bucket, object) + return result, toObjectErr(errXLReadQuorum, bucket, object) } - // List of disks having outdated version of the object or missing object. - outDatedDisks := outDatedDisks(storageDisks, availableDisks, errs, partsMetadata, bucket, - object) + if disksToHealCount == 0 { + // Nothing to heal! + return result, nil + } - // Number of disks that had outdated content of the given - // object and are online to be healed. - numHealedDisks := 0 - for _, disk := range outDatedDisks { - if disk != nil { - numHealedDisks++ - } + // After this point, only have to repair data on disk - so + // return if it is a dry-run + if dryRun { + return result, nil } // Latest xlMetaV1 for reference. If a valid metadata is not // present, it is as good as object not found. latestMeta, pErr := pickValidXLMeta(partsMetadata, modTime) if pErr != nil { - return 0, 0, toObjectErr(pErr, bucket, object) + return result, toObjectErr(pErr, bucket, object) } - for index, disk := range outDatedDisks { + // Clear data files of the object on outdated disks + for _, disk := range outDatedDisks { // Before healing outdated disks, we need to remove // xl.json and part files from "bucket/object/" so // that rename(minioMetaBucket, "tmp/tmpuuid/", @@ -417,18 +486,10 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i continue } - // errFileNotFound implies that xl.json is missing. We - // may have object parts still present in the object - // directory. This needs to be deleted for object to - // healed successfully. - if errs[index] != nil && !errors.IsErr(errs[index], errFileNotFound) { - continue - } - // List and delete the object directory, ignoring // errors. - files, err := disk.ListDir(bucket, object) - if err == nil { + files, derr := disk.ListDir(bucket, object) + if derr == nil { for _, entry := range files { _ = disk.DeleteFile(bucket, pathJoin(object, entry)) @@ -452,10 +513,10 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i // Heal each part. erasureHealFile() will write the healed // part to .minio/tmp/uuid/ which needs to be renamed later to // the final location. - storage, err := NewErasureStorage(latestDisks, - latestMeta.Erasure.DataBlocks, latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize) + storage, err := NewErasureStorage(latestDisks, latestMeta.Erasure.DataBlocks, + latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize) if err != nil { - return 0, 0, toObjectErr(err, bucket, object) + return result, toObjectErr(err, bucket, object) } checksums := make([][]byte, len(latestDisks)) for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ { @@ -475,7 +536,7 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i erasure.BlockSize, minioMetaTmpBucket, pathJoin(tmpID, partName), partSize, algorithm, checksums) if hErr != nil { - return 0, 0, toObjectErr(hErr, bucket, object) + return result, toObjectErr(hErr, bucket, object) } // outDatedDisks that had write errors should not be // written to for remaining parts, so we nil it out. @@ -487,7 +548,7 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i // a healed part checksum had a write error. if file.Checksums[i] == nil { outDatedDisks[i] = nil - numHealedDisks-- + disksToHealCount-- continue } // append part checksums @@ -496,8 +557,8 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i } // If all disks are having errors, we give up. - if numHealedDisks == 0 { - return 0, 0, fmt.Errorf("all disks without up-to-date data had write errors") + if disksToHealCount == 0 { + return result, fmt.Errorf("all disks without up-to-date data had write errors") } } @@ -514,11 +575,11 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i outDatedDisks, aErr = writeUniqueXLMetadata(outDatedDisks, minioMetaTmpBucket, tmpID, partsMetadata, diskCount(outDatedDisks)) if aErr != nil { - return 0, 0, toObjectErr(aErr, bucket, object) + return result, toObjectErr(aErr, bucket, object) } // Rename from tmp location to the actual location. - for _, disk := range outDatedDisks { + for diskIndex, disk := range outDatedDisks { if disk == nil { continue } @@ -527,33 +588,47 @@ func healObject(storageDisks []StorageAPI, bucket, object string, quorum int) (i aErr = disk.RenameFile(minioMetaTmpBucket, retainSlash(tmpID), bucket, retainSlash(object)) if aErr != nil { - return 0, 0, toObjectErr(errors.Trace(aErr), bucket, object) + return result, toObjectErr(errors.Trace(aErr), bucket, object) } + + realDiskIdx := unshuffleIndex(diskIndex, + latestMeta.Erasure.Distribution) + drive := globalEndpoints.GetString(realDiskIdx) + result.DriveInfo.After[drive] = madmin.DriveStateOk } - return numOfflineDisks, numHealedDisks, nil + + // Set the size of the object in the heal result + result.ObjectSize = latestMeta.Stat.Size + + return result, nil } -// HealObject heals a given object for all its missing entries. +// HealObject - heal the given object. +// // FIXME: If an object object was deleted and one disk was down, // and later the disk comes back up again, heal on the object // should delete it. -func (xl xlObjects) HealObject(bucket, object string) (int, int, error) { +func (xl xlObjects) HealObject(bucket, object string, dryRun bool) ( + hr madmin.HealResultItem, err error) { + + // FIXME: Metadata is read again in the healObject() call below. // Read metadata files from all the disks partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, object) // get read quorum for this object - readQuorum, _, err := objectQuorumFromMeta(xl, partsMetadata, errs) + var readQuorum int + readQuorum, _, err = objectQuorumFromMeta(xl, partsMetadata, errs) if err != nil { - return 0, 0, err + return hr, err } // Lock the object before healing. objectLock := xl.nsMutex.NewNSLock(bucket, object) - if err := objectLock.GetRLock(globalHealingTimeout); err != nil { - return 0, 0, err + if lerr := objectLock.GetRLock(globalHealingTimeout); lerr != nil { + return hr, lerr } defer objectLock.RUnlock() // Heal the object. - return healObject(xl.storageDisks, bucket, object, readQuorum) + return healObject(xl.storageDisks, bucket, object, readQuorum, dryRun) } diff --git a/cmd/xl-v1-healing_test.go b/cmd/xl-v1-healing_test.go index 48ab81b18..54e14e9a2 100644 --- a/cmd/xl-v1-healing_test.go +++ b/cmd/xl-v1-healing_test.go @@ -23,6 +23,7 @@ import ( "path/filepath" "testing" + "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/pkg/errors" ) @@ -46,7 +47,7 @@ func TestHealFormatXL(t *testing.T) { t.Fatal(err) } xl := obj.(*xlObjects) - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { t.Fatal("Got an unexpected error: ", err) } @@ -67,7 +68,7 @@ func TestHealFormatXL(t *testing.T) { xl.storageDisks[i] = nil } - if err = healFormatXL(xl.storageDisks); err != errXLReadQuorum { + if _, err = healFormatXL(xl.storageDisks, false); err != errXLReadQuorum { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -90,7 +91,7 @@ func TestHealFormatXL(t *testing.T) { } xl.storageDisks[i] = newNaughtyDisk(posixDisk, nil, errDiskFull) } - if err = healFormatXL(xl.storageDisks); err != errXLReadQuorum { + if _, err = healFormatXL(xl.storageDisks, false); err != errXLReadQuorum { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -108,7 +109,7 @@ func TestHealFormatXL(t *testing.T) { } xl = obj.(*xlObjects) xl.storageDisks[0] = nil - if err = healFormatXL(xl.storageDisks); err != nil && err.Error() != "cannot proceed with heal as some disks are offline" { + if _, err = healFormatXL(xl.storageDisks, false); err != nil && err.Error() != "cannot proceed with heal as some disks are offline" { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -129,7 +130,7 @@ func TestHealFormatXL(t *testing.T) { t.Fatal(err) } } - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -150,7 +151,7 @@ func TestHealFormatXL(t *testing.T) { t.Fatal(err) } } - if err = healFormatXL(xl.storageDisks); err == nil { + if _, err = healFormatXL(xl.storageDisks, false); err == nil { t.Fatal("Should get a json parsing error, ") } removeRoots(fsDirs) @@ -171,7 +172,7 @@ func TestHealFormatXL(t *testing.T) { t.Fatal(err) } } - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -198,7 +199,7 @@ func TestHealFormatXL(t *testing.T) { } xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskNotFound) expectedErr := fmt.Errorf("cannot proceed with heal as %s", errSomeDiskOffline) - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { if err.Error() != expectedErr.Error() { t.Fatal("Got an unexpected error: ", err) } @@ -228,7 +229,7 @@ func TestHealFormatXL(t *testing.T) { } xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskAccessDenied) expectedErr = fmt.Errorf("cannot proceed with heal as some disks had unhandled errors") - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { if err.Error() != expectedErr.Error() { t.Fatal("Got an unexpected error: ", err) } @@ -254,7 +255,7 @@ func TestHealFormatXL(t *testing.T) { t.Fatal(err) } } - if err = healFormatXL(xl.storageDisks); err != nil { + if _, err = healFormatXL(xl.storageDisks, false); err != nil { t.Fatal("Got an unexpected error: ", err) } removeRoots(fsDirs) @@ -446,14 +447,18 @@ func TestListBucketsHeal(t *testing.T) { t.Fatal(err) } + bucketSet := set.CreateStringSet(saneBucket, corruptedBucketName) + // Check the number of buckets in list buckets heal result - if len(buckets) != 1 { - t.Fatalf("Length of missing buckets is incorrect, expected: 1, found: %d", len(buckets)) + if len(buckets) != len(bucketSet) { + t.Fatalf("Length of missing buckets is incorrect, expected: 2, found: %d", len(buckets)) } - // Check the name of bucket in list buckets heal result - if buckets[0].Name != corruptedBucketName { - t.Fatalf("Name of missing bucket is incorrect, expected: %s, found: %s", corruptedBucketName, buckets[0].Name) + // Check each bucket name is in `bucketSet`v + for _, b := range buckets { + if !bucketSet.Contains(b.Name) { + t.Errorf("Bucket %v is missing from bucket set", b.Name) + } } } @@ -520,7 +525,7 @@ func TestHealObjectXL(t *testing.T) { t.Fatalf("Failed to delete a file - %v", err) } - _, _, err = obj.HealObject(bucket, object) + _, err = obj.HealObject(bucket, object, false) if err != nil { t.Fatalf("Failed to heal object - %v", err) } @@ -536,7 +541,7 @@ func TestHealObjectXL(t *testing.T) { } // Try healing now, expect to receive errDiskNotFound. - _, _, err = obj.HealObject(bucket, object) + _, err = obj.HealObject(bucket, object, false) // since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum if errors.Cause(err) != errXLReadQuorum { t.Errorf("Expected %v but received %v", errDiskNotFound, err) diff --git a/cmd/xl-v1-list-objects-heal.go b/cmd/xl-v1-list-objects-heal.go index 36369c4c4..fde8c63fa 100644 --- a/cmd/xl-v1-list-objects-heal.go +++ b/cmd/xl-v1-list-objects-heal.go @@ -37,9 +37,6 @@ func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc { continue } - // Filter entries that have the prefix prefixEntry. - entries = filterMatchingPrefix(entries, prefixEntry) - // isLeaf() check has to happen here so that // trailing "/" for objects can be removed. for i, entry := range entries { @@ -63,6 +60,9 @@ func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc { mergedEntries = append(mergedEntries, newEntries...) sort.Strings(mergedEntries) } + + // Filter entries that have the prefix prefixEntry. + mergedEntries = filterMatchingPrefix(mergedEntries, prefixEntry) } return mergedEntries, false, nil } @@ -141,23 +141,15 @@ func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, ma continue } - // Check if the current object needs healing - objectLock := xl.nsMutex.NewNSLock(bucket, objInfo.Name) - if err := objectLock.GetRLock(globalHealingTimeout); err != nil { - return loi, err - } - partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, objInfo.Name) - if xlShouldHeal(xl.storageDisks, partsMetadata, errs, bucket, objInfo.Name) { - healStat := xlHealStat(xl, partsMetadata, errs) - result.Objects = append(result.Objects, ObjectInfo{ - Name: objInfo.Name, - ModTime: objInfo.ModTime, - Size: objInfo.Size, - IsDir: false, - HealObjectInfo: &healStat, - }) - } - objectLock.RUnlock() + // Add each object seen to the result - objects are + // checked for healing later. + result.Objects = append(result.Objects, ObjectInfo{ + Bucket: bucket, + Name: objInfo.Name, + ModTime: objInfo.ModTime, + Size: objInfo.Size, + IsDir: false, + }) } return result, nil } diff --git a/cmd/xl-v1-list-objects-heal_test.go b/cmd/xl-v1-list-objects-heal_test.go index 4cf03572a..c22d36310 100644 --- a/cmd/xl-v1-list-objects-heal_test.go +++ b/cmd/xl-v1-list-objects-heal_test.go @@ -54,14 +54,16 @@ func TestListObjectsHeal(t *testing.T) { // Put 5 objects under sane dir for i := 0; i < 5; i++ { - _, err = xl.PutObject(bucketName, "sane/"+objName+strconv.Itoa(i), mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = xl.PutObject(bucketName, "sane/"+objName+strconv.Itoa(i), + mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) if err != nil { t.Fatalf("XL Object upload failed: %s", err) } } - // Put 500 objects under unsane/subdir dir + // Put 5 objects under unsane/subdir dir for i := 0; i < 5; i++ { - _, err = xl.PutObject(bucketName, "unsane/subdir/"+objName+strconv.Itoa(i), mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = xl.PutObject(bucketName, "unsane/subdir/"+objName+strconv.Itoa(i), + mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) if err != nil { t.Fatalf("XL Object upload failed: %s", err) } @@ -101,7 +103,7 @@ func TestListObjectsHeal(t *testing.T) { // Inexistent object {bucketName, "inexistentObj", "", "", 1000, nil, 0}, // Test ListObjectsHeal when all objects are sane - {bucketName, "", "", "", 1000, nil, 0}, + {bucketName, "", "", "", 1000, nil, 10}, } for i, testCase := range testCases { testFunc(testCase, i+1) @@ -119,12 +121,12 @@ func TestListObjectsHeal(t *testing.T) { testCases = []testData{ // Test ListObjectsHeal when all objects under unsane/ need to be healed - {bucketName, "", "", "", 1000, nil, 5}, + {bucketName, "", "", "", 1000, nil, 10}, // List objects heal under unsane/, should return all elements {bucketName, "unsane/", "", "", 1000, nil, 5}, - // List healing objects under sane/, should return 0 - {bucketName, "sane/", "", "", 1000, nil, 0}, - // Max Keys == 200 + // List healing objects under sane/ + {bucketName, "sane/", "", "", 1000, nil, 5}, + // Max Keys == 2 {bucketName, "unsane/", "", "", 2, nil, 2}, // Max key > 1000 {bucketName, "unsane/", "", "", 5000, nil, 5}, diff --git a/cmd/xl-v1-object_test.go b/cmd/xl-v1-object_test.go index 28dd943c7..6dfd96ff4 100644 --- a/cmd/xl-v1-object_test.go +++ b/cmd/xl-v1-object_test.go @@ -316,7 +316,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - _, _, err = xl.HealObject(bucket, object) + _, err = xl.HealObject(bucket, object, false) if err != nil { t.Fatal(err) } @@ -340,7 +340,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - _, _, err = xl.HealObject(bucket, object) + _, err = xl.HealObject(bucket, object, false) if err != nil { t.Fatal(err) } @@ -362,7 +362,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } // This would create the bucket. - err = xl.HealBucket(bucket) + _, err = xl.HealBucket(bucket, false) if err != nil { t.Fatal(err) } diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 768cb1b6f..c521a100c 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -66,17 +66,18 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error) // Additionally a special error is provided to be returned in case // quorum is not satisfied. func reduceQuorumErrs(errs []error, ignoredErrs []error, quorum int, quorumErr error) (maxErr error) { - maxCount, maxErr := reduceErrs(errs, ignoredErrs) - if maxErr == nil && maxCount >= quorum { + var maxCount int + maxCount, maxErr = reduceErrs(errs, ignoredErrs) + switch { + case maxErr == nil && maxCount >= quorum: // Success in quorum. - return nil - } - if maxErr != nil && maxCount >= quorum { + case maxErr != nil && maxCount >= quorum: // Errors in quorum. - return errors2.Trace(maxErr, errs...) + maxErr = errors2.Trace(maxErr, errs...) + default: + // No quorum satisfied. + maxErr = errors2.Trace(quorumErr, errs...) } - // No quorum satisfied. - maxErr = errors2.Trace(quorumErr, errs...) return } @@ -365,6 +366,17 @@ func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []Stora return shuffledDisks } +// unshuffleIndex - performs reverse of the shuffleDisks operations +// for a single 0-based index. +func unshuffleIndex(n int, distribution []int) int { + for i, v := range distribution { + if v-1 == n { + return i + } + } + return -1 +} + // evalDisks - returns a new slice of disks where nil is set if // the corresponding error in errs slice is not nil func evalDisks(disks []StorageAPI, errs []error) []StorageAPI { diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md index 5b56c3a66..3b432e275 100644 --- a/pkg/madmin/API.md +++ b/pkg/madmin/API.md @@ -36,13 +36,11 @@ func main() { ``` -| Service operations|LockInfo operations|Healing operations|Config operations| Misc | -|:---|:---|:---|:---|:---| -|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)|[`GetConfig`](#GetConfig)| [`SetCredentials`](#SetCredentials)| -|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)|[`SetConfig`](#SetConfig)|| -| | |[`HealBucket`](#HealBucket) ||| -| | |[`HealObject`](#HealObject)||| -| | |[`HealFormat`](#HealFormat)||| +| Service operations | LockInfo operations | Healing operations | Config operations | Misc | +|:------------------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------| +| [`ServiceStatus`](#ServiceStatus) | [`ListLocks`](#ListLocks) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`SetCredentials`](#SetCredentials) | +| [`ServiceSendAction`](#ServiceSendAction) | [`ClearLocks`](#ClearLocks) | | [`SetConfig`](#SetConfig) | | + ## 1. Constructor @@ -60,8 +58,25 @@ __Parameters__ |`secretAccessKey` | _string_ |Secret key for the object storage endpoint. | |`ssl` | _bool_ | Set this value to 'true' to enable secure (HTTPS) access. | +## 2. Admin API Version + + +### VersionInfo() (AdminAPIVersionInfo, error) +Fetch server's supported Administrative API version. -## 2. Service operations + __Example__ + +``` go + + info, err := madmClnt.VersionInfo() + if err != nil { + log.Fatalln(err) + } + log.Printf("%s\n", info.Version) + +``` + +## 3. Service operations ### ServiceStatus() (ServiceStatusMetadata, error) @@ -102,17 +117,19 @@ Fetch service status, replies disk space used, backend type and total disks offl ``` - -### ServiceRestart() (error) -If successful restarts the running minio service, for distributed setup restarts all remote minio servers. + +### ServiceSendAction(act ServiceActionValue) (error) +Sends a service action command to service - possible actions are restarting and stopping the server. __Example__ ```go - - st, err := madmClnt.ServiceRestart() + // to restart + st, err := madmClnt.ServiceSendAction(ServiceActionValueRestart) + // or to stop + // st, err := madmClnt.ServiceSendAction(ServiceActionValueStop) if err != nil { log.Fatalln(err) } @@ -120,7 +137,7 @@ If successful restarts the running minio service, for distributed setup restarts ``` -## 3. Info operations +## 4. Info operations ### ServerInfo() ([]ServerInfo, error) @@ -143,7 +160,7 @@ Fetch all information for all cluster nodes, such as uptime, region, network sta ``` -## 4. Lock operations +## 5. Lock operations ### ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) @@ -175,146 +192,95 @@ __Example__ ``` -## 5. Heal operations +## 6. Heal operations - -### ListObjectsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan ObjectInfo, error) -If successful returns information on the list of objects that need healing in ``bucket`` matching ``prefix``. + +### Heal(bucket, prefix string, healOpts HealOpts, clientToken string, forceStart bool) (start HealStartSuccess, status HealTaskStatus, err error) -__Example__ +Start a heal sequence that scans data under given (possible empty) +`bucket` and `prefix`. The `recursive` bool turns on recursive +traversal under the given path. `dryRun` does not mutate on-disk data, +but performs data validation. `incomplete` enables healing of +multipart uploads that are in progress. `removeBadFiles` removes +unrecoverable files. `statisticsOnly` turns off detailed +heal-operations reporting in the status call. -``` go - // Create a done channel to control 'ListObjectsHeal' go routine. - doneCh := make(chan struct{}) +Two heal sequences on overlapping paths may not be initiated. - // Indicate to our routine to exit cleanly upon return. - defer close(doneCh) - - // Set true if recursive listing is needed. - isRecursive := true - // List objects that need healing for a given bucket and - // prefix. - healObjectCh, err := madmClnt.ListObjectsHeal("mybucket", "myprefix", isRecursive, doneCh) - if err != nil { - fmt.Println(err) - return - } - for object := range healObjectsCh { - if object.Err != nil { - log.Fatalln(err) - return - } - if object.HealObjectInfo != nil { - switch healInfo := *object.HealObjectInfo; healInfo.Status { - case madmin.CanHeal: - fmt.Println(object.Key, " can be healed.") - case madmin.QuorumUnavailable: - fmt.Println(object.Key, " can't be healed until quorum is available.") - case madmin.Corrupted: - fmt.Println(object.Key, " can't be healed, not enough information.") - } - } - fmt.Println("object: ", object) - } -``` - - -### ListBucketsHeal() error -If successful returns information on the list of buckets that need healing. +The progress of a heal should be followed using the `HealStatus` +API. The server accumulates results of the heal traversal and waits +for the client to receive and acknowledge them using the status +API. When the statistics-only option is set, the server only maintains +aggregates statistics - in this case, no acknowledgement of results is +required. __Example__ ``` go - // List buckets that need healing - healBucketsList, err := madmClnt.ListBucketsHeal() - if err != nil { - fmt.Println(err) - return - } - for bucket := range healBucketsList { - if bucket.HealBucketInfo != nil { - switch healInfo := *object.HealBucketInfo; healInfo.Status { - case madmin.CanHeal: - fmt.Println(bucket.Key, " can be healed.") - case madmin.QuorumUnavailable: - fmt.Println(bucket.Key, " can't be healed until quorum is available.") - case madmin.Corrupted: - fmt.Println(bucket.Key, " can't be healed, not enough information.") - } - } - fmt.Println("bucket: ", bucket) - } -``` - - -### HealBucket(bucket string, isDryRun bool) error -If bucket is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the bucket is not healed, but heal bucket request is validated by the server. e.g, if the bucket exists, if bucket name is valid etc. - -__Example__ -``` go - isDryRun := false - err := madmClnt.HealBucket("mybucket", isDryRun) + healPath, err := madmClnt.HealStart("", "", true, false, true, false, false) if err != nil { log.Fatalln(err) } - log.Println("successfully healed mybucket") + log.Printf("Heal sequence started at %s", healPath) ``` - -### HealObject(bucket, object string, isDryRun bool) (HealResult, error) -If object is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the object is not healed, but heal object request is validated by the server. e.g, if the object exists, if object name is valid etc. - -| Param | Type | Description | -|---|---|---| -|`h.State` | _HealState_ | Represents the result of heal operation. It could be one of `HealNone`, `HealPartial` or `HealOK`. | - - -| Value | Description | -|---|---| -|`HealNone` | Object wasn't healed on any of the disks | -|`HealPartial` | Object was healed on some of the disks needing heal | -| `HealOK` | Object was healed on all the disks needing heal | +#### HealTaskStatus structure +| Param | Type | Description | +|----|--------|--------| +| s.Summary | _string_ | Short status of heal sequence | +| s.FailureDetail | _string_ | Error message in case of heal sequence failure | +| s.HealSettings | _HealOpts_ | Contains the booleans set in the `HealStart` call | +| s.Items | _[]HealResultItem_ | Heal records for actions performed by server | +| s.Statistics | _HealStatistics_ | Aggregate of heal records from beginning | -__Example__ +#### HealResultItem structure -``` go - isDryRun = false - healResult, err := madmClnt.HealObject("mybucket", "myobject", isDryRun) - if err != nil { - log.Fatalln(err) - } +| Param | Type | Description | +|------|-------|---------| +| ResultIndex | _int64_ | Index of the heal-result record | +| Type | _HealItemType_ | Represents kind of heal operation in the heal record | +| Bucket | _string_ | Bucket name | +| Object | _string_ | Object name | +| Detail | _string_ | Details about heal operation | +| DiskInfo.AvailableOn | _[]int_ | List of disks on which the healed entity is present and healthy | +| DiskInfo.HealedOn | _[]int_ | List of disks on which the healed entity was restored | - log.Println("Heal-object result: ", healResult) +#### HealStatistics structure -``` +Most parameters represent the aggregation of heal operations since the +start of the heal sequence. - -### HealFormat(isDryRun bool) error -Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend. +| Param | Type | Description | +|-------|-----|----------| +| NumDisks | _int_ | Number of disks configured in the backend | +| NumBucketsScanned | _int64_ | Number of buckets scanned | +| BucketsMissingByDisk | _map[int]int64_ | Map of disk to number of buckets missing | +| BucketsAvailableByDisk | _map[int]int64_ | Map of disk to number of buckets available | +| BucketsHealedByDisk | _map[int]int64_ | Map of disk to number of buckets healed on | +| NumObjectsScanned | _int64_ | Number of objects scanned | +| NumUploadsScanned | _int64_ | Number of uploads scanned | +| ObjectsByAvailablePC | _map[int64]_ | Map of available part counts (after heal) to number of objects | +| ObjectsByHealedPC | _map[int64]_ | Map of healed part counts to number of objects | +| ObjectsMissingByDisk | _map[int64]_ | Map of disk number to number of objects with parts missing on that disk | +| ObjectsAvailableByDisk | _map[int64]_ | Map of disk number to number of objects available on that disk | +| ObjectsHealedByDisk | _map[int64]_ | Map of disk number to number of objects healed on that disk | __Example__ ``` go - isDryRun := true - err := madmClnt.HealFormat(isDryRun) - if err != nil { - log.Fatalln(err) - } - isDryRun = false - err = madmClnt.HealFormat(isDryRun) + res, err := madmClnt.HealStatus("", "") if err != nil { log.Fatalln(err) } - - log.Println("successfully healed storage format on available disks.") + log.Printf("Heal sequence status data %#v", res) ``` -## 6. Config operations +## 7. Config operations ### GetConfig() ([]byte, error) @@ -373,7 +339,7 @@ __Example__ log.Println("SetConfig: ", string(buf.Bytes())) ``` -## 7. Misc operations +## 8. Misc operations diff --git a/pkg/madmin/api-error-response.go b/pkg/madmin/api-error-response.go index d5a14f656..c85c0bfbb 100644 --- a/pkg/madmin/api-error-response.go +++ b/pkg/madmin/api-error-response.go @@ -54,7 +54,7 @@ func (e ErrorResponse) Error() string { } const ( - reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues." + reportIssue = "Please report this issue at https://github.com/minio/minio/issues." ) // httpRespToErrorResponse returns a new encoded ErrorResponse @@ -65,8 +65,8 @@ func httpRespToErrorResponse(resp *http.Response) error { return ErrInvalidArgument(msg) } var errResp ErrorResponse - // Decode the xml error - err := xmlDecoder(resp.Body, &errResp) + // Decode the json error + err := jsonDecoder(resp.Body, &errResp) if err != nil { return ErrorResponse{ Code: resp.Status, diff --git a/pkg/madmin/api.go b/pkg/madmin/api.go index 09cad8c22..d3d6c9f91 100644 --- a/pkg/madmin/api.go +++ b/pkg/madmin/api.go @@ -71,6 +71,8 @@ type AdminClient struct { const ( libraryName = "madmin-go" libraryVersion = "0.0.1" + + libraryAdminURLPrefix = "/minio/admin" ) // User Agent should always following the below style. @@ -176,6 +178,9 @@ type requestData struct { customHeaders http.Header queryValues url.Values + // Url path relative to admin API base endpoint + relPath string + contentBody io.Reader contentLength int64 contentSHA256Bytes []byte @@ -388,7 +393,7 @@ func (c AdminClient) newRequest(method string, reqData requestData) (req *http.R location := "us-east-1" // Construct a new target URL. - targetURL, err := c.makeTargetURL(reqData.queryValues) + targetURL, err := c.makeTargetURL(reqData) if err != nil { return nil, err } @@ -440,16 +445,16 @@ func (c AdminClient) newRequest(method string, reqData requestData) (req *http.R } // makeTargetURL make a new target url. -func (c AdminClient) makeTargetURL(queryValues url.Values) (*url.URL, error) { +func (c AdminClient) makeTargetURL(r requestData) (*url.URL, error) { host := c.endpointURL.Host scheme := c.endpointURL.Scheme - urlStr := scheme + "://" + host + "/" + urlStr := scheme + "://" + host + libraryAdminURLPrefix + r.relPath // If there are any query values, add them to the end. - if len(queryValues) > 0 { - urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) + if len(r.queryValues) > 0 { + urlStr = urlStr + "?" + s3utils.QueryEncode(r.queryValues) } u, err := url.Parse(urlStr) if err != nil { diff --git a/pkg/madmin/config-commands.go b/pkg/madmin/config-commands.go index e4f8afb53..4ced35314 100644 --- a/pkg/madmin/config-commands.go +++ b/pkg/madmin/config-commands.go @@ -20,14 +20,10 @@ package madmin import ( "bytes" "encoding/json" + "fmt" "io" "io/ioutil" "net/http" - "net/url" -) - -const ( - configQueryParam = "config" ) // NodeSummary - represents the result of an operation part of @@ -47,20 +43,14 @@ type SetConfigResult struct { // GetConfig - returns the config.json of a minio setup. func (adm *AdminClient) GetConfig() ([]byte, error) { - queryVal := make(url.Values) - queryVal.Set(configQueryParam, "") - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "get") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, + // No TLS? + if !adm.secure { + return nil, fmt.Errorf("credentials/configuration cannot be retrieved over an insecure connection") } - // Execute GET on /?config to get config of a setup. - resp, err := adm.executeMethod("GET", reqData) - + // Execute GET on /minio/admin/v1/config to get config of a setup. + resp, err := adm.executeMethod("GET", + requestData{relPath: "/v1/config"}) defer closeResponse(resp) if err != nil { return nil, err @@ -75,50 +65,42 @@ func (adm *AdminClient) GetConfig() ([]byte, error) { } // SetConfig - set config supplied as config.json for the setup. -func (adm *AdminClient) SetConfig(config io.Reader) (SetConfigResult, error) { - queryVal := url.Values{} - queryVal.Set(configQueryParam, "") - - // Set x-minio-operation to set. - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "set") +func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err error) { + // No TLS? + if !adm.secure { + return r, fmt.Errorf("credentials/configuration cannot be updated over an insecure connection") + } // Read config bytes to calculate MD5, SHA256 and content length. configBytes, err := ioutil.ReadAll(config) if err != nil { - return SetConfigResult{}, err + return r, err } reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, + relPath: "/v1/config", contentBody: bytes.NewReader(configBytes), contentMD5Bytes: sumMD5(configBytes), contentSHA256Bytes: sum256(configBytes), } - // Execute PUT on /?config to set config. + // Execute PUT on /minio/admin/v1/config to set config. resp, err := adm.executeMethod("PUT", reqData) defer closeResponse(resp) if err != nil { - return SetConfigResult{}, err + return r, err } if resp.StatusCode != http.StatusOK { - return SetConfigResult{}, httpRespToErrorResponse(resp) + return r, httpRespToErrorResponse(resp) } - var result SetConfigResult jsonBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return SetConfigResult{}, err - } - - err = json.Unmarshal(jsonBytes, &result) - if err != nil { - return SetConfigResult{}, err + return r, err } - return result, nil + err = json.Unmarshal(jsonBytes, &r) + return r, err } diff --git a/pkg/madmin/constants.go b/pkg/madmin/constants.go index 61fda4a61..dc8f9c1b3 100644 --- a/pkg/madmin/constants.go +++ b/pkg/madmin/constants.go @@ -19,7 +19,4 @@ package madmin const ( // Unsigned payload. unsignedPayload = "UNSIGNED-PAYLOAD" - - // Admin operation header. - minioAdminOpHeader = "X-Minio-Operation" ) diff --git a/pkg/madmin/generic-commands.go b/pkg/madmin/generic-commands.go index 384b74ce2..9699fdcf1 100644 --- a/pkg/madmin/generic-commands.go +++ b/pkg/madmin/generic-commands.go @@ -19,46 +19,50 @@ package madmin import ( "bytes" - "encoding/xml" + "encoding/json" + "fmt" "net/http" - "net/url" ) -// setCredsReq - xml to send to the server to set new credentials -type setCredsReq struct { - Username string `xml:"username"` - Password string `xml:"password"` +// SetCredsReq - xml to send to the server to set new credentials +type SetCredsReq struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` } -// SetCredentials - Call Set Credentials API to set new access and secret keys in the specified Minio server +// SetCredentials - Call Set Credentials API to set new access and +// secret keys in the specified Minio server func (adm *AdminClient) SetCredentials(access, secret string) error { - - // Setup new request - reqData := requestData{} - reqData.queryValues = make(url.Values) - reqData.queryValues.Set("service", "") - reqData.customHeaders = make(http.Header) - reqData.customHeaders.Set(minioAdminOpHeader, "set-credentials") - // Setup request's body - body, err := xml.Marshal(setCredsReq{Username: access, Password: secret}) + body, err := json.Marshal(SetCredsReq{access, secret}) if err != nil { return err } - reqData.contentBody = bytes.NewReader(body) - reqData.contentLength = int64(len(body)) - reqData.contentMD5Bytes = sumMD5(body) - reqData.contentSHA256Bytes = sum256(body) + + // No TLS? + if !adm.secure { + return fmt.Errorf("credentials cannot be updated over an insecure connection") + } + + // Setup new request + reqData := requestData{ + relPath: "/v1/config/credential", + contentBody: bytes.NewReader(body), + contentLength: int64(len(body)), + contentMD5Bytes: sumMD5(body), + contentSHA256Bytes: sum256(body), + } // Execute GET on bucket to list objects. - resp, err := adm.executeMethod("POST", reqData) + resp, err := adm.executeMethod("PUT", reqData) defer closeResponse(resp) if err != nil { return err } - // Return error to the caller if http response code is different from 200 + // Return error to the caller if http response code is + // different from 200 if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp) } diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go index 263bf4bff..c77182ad1 100644 --- a/pkg/madmin/heal-commands.go +++ b/pkg/madmin/heal-commands.go @@ -20,456 +20,157 @@ package madmin import ( + "bytes" "encoding/json" - "encoding/xml" "fmt" + "io" "io/ioutil" "net/http" "net/url" "time" ) -// listBucketHealResult container for listObjects response. -type listBucketHealResult struct { - // A response can contain CommonPrefixes only if you have - // specified a delimiter. - CommonPrefixes []commonPrefix - // Metadata about each object returned. - Contents []ObjectInfo - Delimiter string - - // Encoding type used to encode object keys in the response. - EncodingType string - - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - Marker string - MaxKeys int64 - Name string - - // When response is truncated (the IsTruncated element value in - // the response is true), you can use the key name in this field - // as marker in the subsequent request to get next set of objects. - // Object storage lists objects in alphabetical order Note: This - // element is returned only if you have delimiter request - // parameter specified. If response does not include the NextMaker - // and it is truncated, you can use the value of the last Key in - // the response as the marker in the subsequent request to get the - // next set of object keys. - NextMarker string - Prefix string -} - -// commonPrefix container for prefix response. -type commonPrefix struct { - Prefix string -} - -// Owner - bucket owner/principal -type Owner struct { - ID string - DisplayName string +// HealOpts - collection of options for a heal sequence +type HealOpts struct { + Recursive bool `json:"recursive"` + DryRun bool `json:"dryRun"` } -// Bucket container for bucket metadata -type Bucket struct { - Name string - CreationDate string // time string of format "2006-01-02T15:04:05.000Z" - - HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"` +// HealStartSuccess - holds information about a successfully started +// heal operation +type HealStartSuccess struct { + ClientToken string `json:"clientToken"` + ClientAddress string `json:"clientAddress"` + StartTime time.Time `json:"startTime"` } -// ListBucketsHealResponse - format for list buckets response -type ListBucketsHealResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` - - Owner Owner +// HealTaskStatus - status struct for a heal task +type HealTaskStatus struct { + Summary string `json:"summary"` + FailureDetail string `json:"detail"` + StartTime time.Time `json:"startTime"` + HealSettings HealOpts `json:"settings"` + NumDisks int `json:"numDisks"` - // Container for one or more buckets. - Buckets struct { - Buckets []Bucket `xml:"Bucket"` - } // Buckets are nested + Items []HealResultItem `json:"items,omitempty"` } -// HealStatus - represents different states of healing an object could be in. -type HealStatus int +// HealItemType - specify the type of heal operation in a healing +// result +type HealItemType string +// HealItemType constants const ( - // Healthy - Object that is already healthy - Healthy HealStatus = iota - // CanHeal - Object can be healed - CanHeal - // Corrupted - Object can't be healed - Corrupted - // QuorumUnavailable - Object can't be healed until read - // quorum is available - QuorumUnavailable - // CanPartiallyHeal - Object can't be healed completely until - // disks with missing parts come online - CanPartiallyHeal + HealItemMetadata HealItemType = "metadata" + HealItemBucket = "bucket" + HealItemBucketMetadata = "bucket-metadata" + HealItemObject = "object" ) -// HealBucketInfo - represents healing related information of a bucket. -type HealBucketInfo struct { - Status HealStatus -} - -// BucketInfo - represents bucket metadata. -type BucketInfo struct { - // Name of the bucket. - Name string - - // Date and time when the bucket was created. - Created time.Time - - // Healing information - HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"` -} - -// HealObjectInfo - represents healing related information of an object. -type HealObjectInfo struct { - Status HealStatus - MissingDataCount int - MissingParityCount int -} - -// ObjectInfo container for object metadata. -type ObjectInfo struct { - // An ETag is optionally set to md5sum of an object. In case of multipart objects, - // ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of - // each parts concatenated into one string. - ETag string `json:"etag"` - - Key string `json:"name"` // Name of the object - LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. - Size int64 `json:"size"` // Size in bytes of the object. - ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. - - // Collection of additional metadata on the object. - // eg: x-amz-meta-*, content-encoding etc. - Metadata http.Header `json:"metadata"` - - // Owner name. - Owner struct { - DisplayName string `json:"name"` - ID string `json:"id"` - } `json:"owner"` - - // The class of storage used to store the object. - StorageClass string `json:"storageClass"` - - // Error - Err error `json:"-"` - HealObjectInfo *HealObjectInfo `json:"healObjectInfo,omitempty"` -} - -type healQueryKey string - +// Drive state constants const ( - healBucket healQueryKey = "bucket" - healObject healQueryKey = "object" - healPrefix healQueryKey = "prefix" - healMarker healQueryKey = "marker" - healDelimiter healQueryKey = "delimiter" - healMaxKey healQueryKey = "max-key" - healDryRun healQueryKey = "dry-run" + DriveStateOk string = "ok" + DriveStateOffline = "offline" + DriveStateCorrupt = "corrupt" + DriveStateMissing = "missing" ) -// mkHealQueryVal - helper function to construct heal REST API query params. -func mkHealQueryVal(bucket, prefix, marker, delimiter, maxKeyStr string) url.Values { - queryVal := make(url.Values) - queryVal.Set("heal", "") - queryVal.Set(string(healBucket), bucket) - queryVal.Set(string(healPrefix), prefix) - queryVal.Set(string(healMarker), marker) - queryVal.Set(string(healDelimiter), delimiter) - queryVal.Set(string(healMaxKey), maxKeyStr) - return queryVal -} - -// listObjectsHeal - issues heal list API request for a batch of maxKeys objects to be healed. -func (adm *AdminClient) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (listBucketHealResult, error) { - // Construct query params. - maxKeyStr := fmt.Sprintf("%d", maxKeys) - queryVal := mkHealQueryVal(bucket, prefix, marker, delimiter, maxKeyStr) - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "list-objects") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - - // Empty 'list' of objects to be healed. - toBeHealedObjects := listBucketHealResult{} - - // Execute GET on /?heal to list objects needing heal. - resp, err := adm.executeMethod("GET", reqData) - - defer closeResponse(resp) - if err != nil { - return listBucketHealResult{}, err - } - - if resp.StatusCode != http.StatusOK { - return toBeHealedObjects, httpRespToErrorResponse(resp) - - } - - err = xml.NewDecoder(resp.Body).Decode(&toBeHealedObjects) - return toBeHealedObjects, err -} - -// ListObjectsHeal - Lists upto maxKeys objects that needing heal matching bucket, prefix, marker, delimiter. -func (adm *AdminClient) ListObjectsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan ObjectInfo, error) { - // Allocate new list objects channel. - objectStatCh := make(chan ObjectInfo, 1) - // Default listing is delimited at "/" - delimiter := "/" - if recursive { - // If recursive we do not delimit. - delimiter = "" - } - - // Initiate list objects goroutine here. - go func(objectStatCh chan<- ObjectInfo) { - defer close(objectStatCh) - // Save marker for next request. - var marker string - for { - // Get list of objects a maximum of 1000 per request. - result, err := adm.listObjectsHeal(bucket, prefix, marker, delimiter, 1000) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } - - // If contents are available loop through and send over channel. - for _, object := range result.Contents { - // Save the marker. - marker = object.Key - select { - // Send object content. - case objectStatCh <- object: - // If receives done from the caller, return here. - case <-doneCh: - return - } - } - - // Send all common prefixes if any. - // NOTE: prefixes are only present if the request is delimited. - for _, obj := range result.CommonPrefixes { - object := ObjectInfo{} - object.Key = obj.Prefix - object.Size = 0 - select { - // Send object prefixes. - case objectStatCh <- object: - // If receives done from the caller, return here. - case <-doneCh: - return - } - } - - // If next marker present, save it for next request. - if result.NextMarker != "" { - marker = result.NextMarker - } - - // Listing ends result is not truncated, return right here. - if !result.IsTruncated { - return - } +// HealResultItem - struct for an individual heal result item +type HealResultItem struct { + ResultIndex int64 `json:"resultId"` + Type HealItemType `json:"type"` + Bucket string `json:"bucket"` + Object string `json:"object"` + Detail string `json:"detail"` + ParityBlocks int `json:"parityBlocks,omitempty"` + DataBlocks int `json:"dataBlocks,omitempty"` + DiskCount int `json:"diskCount"` + DriveInfo struct { + // below maps are from drive endpoint to drive state + Before map[string]string `json:"before"` + After map[string]string `json:"after"` + } `json:"drives"` + ObjectSize int64 `json:"objectSize"` +} + +// InitDrives - initialize maps used to represent drive info +func (hri *HealResultItem) InitDrives() { + hri.DriveInfo.Before = make(map[string]string) + hri.DriveInfo.After = make(map[string]string) +} + +// GetOnlineCounts - returns the number of online disks before and +// after heal +func (hri *HealResultItem) GetOnlineCounts() (b, a int) { + if hri == nil { + return + } + for _, v := range hri.DriveInfo.Before { + if v == DriveStateOk { + b++ } - }(objectStatCh) - return objectStatCh, nil -} - -const timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. - -// ListBucketsHeal - issues heal bucket list API request -func (adm *AdminClient) ListBucketsHeal() ([]BucketInfo, error) { - queryVal := url.Values{} - queryVal.Set("heal", "") - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "list-buckets") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - - // Execute GET on /?heal to list objects needing heal. - resp, err := adm.executeMethod("GET", reqData) - - defer closeResponse(resp) - if err != nil { - return []BucketInfo{}, err - } - - if resp.StatusCode != http.StatusOK { - return []BucketInfo{}, httpRespToErrorResponse(resp) } - - var listBucketsHealResult ListBucketsHealResponse - - err = xml.NewDecoder(resp.Body).Decode(&listBucketsHealResult) - if err != nil { - return []BucketInfo{}, err - } - - var bucketsToBeHealed []BucketInfo - - for _, bucket := range listBucketsHealResult.Buckets.Buckets { - creationDate, err := time.Parse(timeFormatAMZLong, bucket.CreationDate) - if err != nil { - return []BucketInfo{}, err + for _, v := range hri.DriveInfo.After { + if v == DriveStateOk { + a++ } - bucketsToBeHealed = append(bucketsToBeHealed, - BucketInfo{ - Name: bucket.Name, - Created: creationDate, - HealBucketInfo: bucket.HealBucketInfo, - }) } - - return bucketsToBeHealed, nil + return } -// HealBucket - Heal the given bucket -func (adm *AdminClient) HealBucket(bucket string, dryrun bool) error { - // Construct query params. - queryVal := url.Values{} - queryVal.Set("heal", "") - queryVal.Set(string(healBucket), bucket) - if dryrun { - queryVal.Set(string(healDryRun), "") - } - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "bucket") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - - // Execute POST on /?heal&bucket=mybucket to heal a bucket. - resp, err := adm.executeMethod("POST", reqData) +// Heal - API endpoint to start heal and to fetch status +func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts, + clientToken string, forceStart bool) ( + healStart HealStartSuccess, healTaskStatus HealTaskStatus, err error) { - defer closeResponse(resp) + body, err := json.Marshal(healOpts) if err != nil { - return err + return healStart, healTaskStatus, err } - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp) + path := fmt.Sprintf("/v1/heal/%s", bucket) + if bucket != "" && prefix != "" { + path += "/" + prefix } - return nil -} - -// HealResult - represents result of heal-object admin API. -type HealResult struct { - State HealState `json:"state"` -} - -// HealState - different states of heal operation -type HealState int - -const ( - // HealNone - none of the disks healed - HealNone HealState = iota - // HealPartial - some disks were healed, others were offline - HealPartial - // HealOK - all disks were healed - HealOK -) - -// HealObject - Heal the given object. -func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) (HealResult, error) { - // Construct query params. - queryVal := url.Values{} - queryVal.Set("heal", "") - queryVal.Set(string(healBucket), bucket) - queryVal.Set(string(healObject), object) - if dryrun { - queryVal.Set(string(healDryRun), "") + // execute POST request to heal api + queryVals := make(url.Values) + var contentBody io.Reader + if clientToken != "" { + queryVals.Set("clientToken", clientToken) + } else { + // Set a body only if clientToken is not given + contentBody = bytes.NewReader(body) } - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "object") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, + if forceStart { + queryVals.Set("forceStart", "true") } - // Execute POST on /?heal&bucket=mybucket&object=myobject to heal an object. - resp, err := adm.executeMethod("POST", reqData) - + resp, err := adm.executeMethod("POST", requestData{ + relPath: path, + contentBody: contentBody, + contentSHA256Bytes: sum256(body), + queryValues: queryVals, + }) defer closeResponse(resp) if err != nil { - return HealResult{}, err + return healStart, healTaskStatus, err } if resp.StatusCode != http.StatusOK { - return HealResult{}, httpRespToErrorResponse(resp) - } - - // Healing is not performed so heal object result is empty. - if dryrun { - return HealResult{}, nil - } - - jsonBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return HealResult{}, err - } - - healResult := HealResult{} - err = json.Unmarshal(jsonBytes, &healResult) - if err != nil { - return HealResult{}, err + return healStart, healTaskStatus, httpRespToErrorResponse(resp) } - return healResult, nil -} - -// HealFormat - heal storage format on available disks. -func (adm *AdminClient) HealFormat(dryrun bool) error { - queryVal := url.Values{} - queryVal.Set("heal", "") - if dryrun { - queryVal.Set(string(healDryRun), "") - } - - // Set x-minio-operation to format. - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "format") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - - // Execute POST on /?heal to heal storage format. - resp, err := adm.executeMethod("POST", reqData) - - defer closeResponse(resp) + respBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return healStart, healTaskStatus, err } - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp) + // Was it a status request? + if clientToken == "" { + err = json.Unmarshal(respBytes, &healStart) + } else { + err = json.Unmarshal(respBytes, &healTaskStatus) } - - return nil + return healStart, healTaskStatus, err } diff --git a/pkg/madmin/info-commands.go b/pkg/madmin/info-commands.go index 60f46cfb2..dc57637a0 100644 --- a/pkg/madmin/info-commands.go +++ b/pkg/madmin/info-commands.go @@ -21,7 +21,6 @@ import ( "encoding/json" "io/ioutil" "net/http" - "net/url" "time" ) @@ -115,13 +114,7 @@ type ServerInfo struct { // ServerInfo - Connect to a minio server and call Server Info Management API // to fetch server's information represented by ServerInfo structure func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) { - // Prepare web service request - reqData := requestData{} - reqData.queryValues = make(url.Values) - reqData.queryValues.Set("info", "") - reqData.customHeaders = make(http.Header) - - resp, err := adm.executeMethod("GET", reqData) + resp, err := adm.executeMethod("GET", requestData{relPath: "/v1/info"}) defer closeResponse(resp) if err != nil { return nil, err diff --git a/pkg/madmin/lock-commands.go b/pkg/madmin/lock-commands.go index edfc548bb..7de125ddb 100644 --- a/pkg/madmin/lock-commands.go +++ b/pkg/madmin/lock-commands.go @@ -82,24 +82,19 @@ func getLockInfos(body io.Reader) ([]VolumeLockInfo, error) { // ListLocks - Calls List Locks Management API to fetch locks matching // bucket, prefix and held before the duration supplied. -func (adm *AdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { +func (adm *AdminClient) ListLocks(bucket, prefix string, + duration time.Duration) ([]VolumeLockInfo, error) { + queryVal := make(url.Values) - queryVal.Set("lock", "") queryVal.Set("bucket", bucket) queryVal.Set("prefix", prefix) - queryVal.Set("duration", duration.String()) - - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "list") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - - // Execute GET on /?lock to list locks. - resp, err := adm.executeMethod("GET", reqData) + queryVal.Set("older-than", duration.String()) + // Execute GET on /minio/admin/v1/locks to list locks. + resp, err := adm.executeMethod("GET", requestData{ + queryValues: queryVal, + relPath: "/v1/locks", + }) defer closeResponse(resp) if err != nil { return nil, err @@ -114,24 +109,19 @@ func (adm *AdminClient) ListLocks(bucket, prefix string, duration time.Duration) // ClearLocks - Calls Clear Locks Management API to clear locks held // on bucket, matching prefix older than duration supplied. -func (adm *AdminClient) ClearLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { +func (adm *AdminClient) ClearLocks(bucket, prefix string, + duration time.Duration) ([]VolumeLockInfo, error) { + queryVal := make(url.Values) - queryVal.Set("lock", "") queryVal.Set("bucket", bucket) queryVal.Set("prefix", prefix) queryVal.Set("duration", duration.String()) - hdrs := make(http.Header) - hdrs.Set(minioAdminOpHeader, "clear") - - reqData := requestData{ - queryValues: queryVal, - customHeaders: hdrs, - } - // Execute POST on /?lock to clear locks. - resp, err := adm.executeMethod("POST", reqData) - + resp, err := adm.executeMethod("DELETE", requestData{ + queryValues: queryVal, + relPath: "/v1/locks", + }) defer closeResponse(resp) if err != nil { return nil, err diff --git a/pkg/madmin/service-commands.go b/pkg/madmin/service-commands.go index 62bf8cb42..17a695109 100644 --- a/pkg/madmin/service-commands.go +++ b/pkg/madmin/service-commands.go @@ -18,69 +18,79 @@ package madmin import ( + "bytes" "encoding/json" "io/ioutil" "net/http" - "net/url" "time" ) -// ServiceStatusMetadata - contains the response of service status API -type ServiceStatusMetadata struct { - Uptime time.Duration `json:"uptime"` +// ServerVersion - server version +type ServerVersion struct { + Version string `json:"version"` + CommitID string `json:"commitID"` } -// ServiceStatus - Connect to a minio server and call Service Status Management API -// to fetch server's storage information represented by ServiceStatusMetadata structure -func (adm *AdminClient) ServiceStatus() (ServiceStatusMetadata, error) { - - // Prepare web service request - reqData := requestData{} - reqData.queryValues = make(url.Values) - reqData.queryValues.Set("service", "") - reqData.customHeaders = make(http.Header) - reqData.customHeaders.Set(minioAdminOpHeader, "status") +// ServiceStatus - contains the response of service status API +type ServiceStatus struct { + ServerVersion ServerVersion `json:"serverVersion"` + Uptime time.Duration `json:"uptime"` +} - // Execute GET on bucket to list objects. - resp, err := adm.executeMethod("GET", reqData) +// ServiceStatus - Connect to a minio server and call Service Status +// Management API to fetch server's storage information represented by +// ServiceStatusMetadata structure +func (adm *AdminClient) ServiceStatus() (ss ServiceStatus, err error) { + // Request API to GET service status + resp, err := adm.executeMethod("GET", requestData{relPath: "/v1/service"}) defer closeResponse(resp) if err != nil { - return ServiceStatusMetadata{}, err + return ss, err } // Check response http status code if resp.StatusCode != http.StatusOK { - return ServiceStatusMetadata{}, httpRespToErrorResponse(resp) + return ss, httpRespToErrorResponse(resp) } - // Unmarshal the server's json response - var serviceStatus ServiceStatusMetadata - respBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return ServiceStatusMetadata{}, err + return ss, err } - err = json.Unmarshal(respBytes, &serviceStatus) - if err != nil { - return ServiceStatusMetadata{}, err - } - - return serviceStatus, nil + err = json.Unmarshal(respBytes, &ss) + return ss, err } -// ServiceRestart - Call Service Restart API to restart a specified Minio server -func (adm *AdminClient) ServiceRestart() error { - // - reqData := requestData{} - reqData.queryValues = make(url.Values) - reqData.queryValues.Set("service", "") - reqData.customHeaders = make(http.Header) - reqData.customHeaders.Set(minioAdminOpHeader, "restart") +// ServiceActionValue - type to restrict service-action values +type ServiceActionValue string - // Execute GET on bucket to list objects. - resp, err := adm.executeMethod("POST", reqData) +const ( + // ServiceActionValueRestart represents restart action + ServiceActionValueRestart ServiceActionValue = "restart" + // ServiceActionValueStop represents stop action + ServiceActionValueStop = "stop" +) + +// ServiceAction - represents POST body for service action APIs +type ServiceAction struct { + Action ServiceActionValue `json:"action"` +} + +// ServiceSendAction - Call Service Restart/Stop API to restart/stop a +// Minio server +func (adm *AdminClient) ServiceSendAction(action ServiceActionValue) error { + body, err := json.Marshal(ServiceAction{action}) + if err != nil { + return err + } + // Request API to Restart server + resp, err := adm.executeMethod("POST", requestData{ + relPath: "/v1/service", + contentBody: bytes.NewReader(body), + contentSHA256Bytes: sum256(body), + }) defer closeResponse(resp) if err != nil { return err diff --git a/pkg/madmin/utils.go b/pkg/madmin/utils.go index 5e1163295..83b2c59e2 100644 --- a/pkg/madmin/utils.go +++ b/pkg/madmin/utils.go @@ -18,7 +18,7 @@ package madmin import ( "crypto/md5" - "encoding/xml" + "encoding/json" "io" "io/ioutil" "net" @@ -45,9 +45,9 @@ func sumMD5(data []byte) []byte { return hash.Sum(nil) } -// xmlDecoder provide decoded value in xml. -func xmlDecoder(body io.Reader, v interface{}) error { - d := xml.NewDecoder(body) +// jsonDecoder decode json to go type. +func jsonDecoder(body io.Reader, v interface{}) error { + d := json.NewDecoder(body) return d.Decode(v) } diff --git a/pkg/madmin/version-commands.go b/pkg/madmin/version-commands.go new file mode 100644 index 000000000..95cf1d547 --- /dev/null +++ b/pkg/madmin/version-commands.go @@ -0,0 +1,54 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package madmin + +import ( + "encoding/json" + "io/ioutil" + "net/http" +) + +// AdminAPIVersionInfo - contains admin API version information +type AdminAPIVersionInfo struct { + Version string `json:"version"` +} + +// VersionInfo - Connect to minio server and call the version API to +// retrieve the server API version +func (adm *AdminClient) VersionInfo() (verInfo AdminAPIVersionInfo, err error) { + var resp *http.Response + resp, err = adm.executeMethod("GET", requestData{relPath: "/version"}) + defer closeResponse(resp) + if err != nil { + return verInfo, err + } + + // Check response http status code + if resp.StatusCode != http.StatusOK { + return verInfo, httpRespToErrorResponse(resp) + } + + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return verInfo, err + } + + // Unmarshal the server's json response + err = json.Unmarshal(respBytes, &verInfo) + return verInfo, err +}