From d65a2c672518a8af5f37f07c3abf18222df1839a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 27 Aug 2019 11:37:47 -0700 Subject: [PATCH] Implement cluster-wide in-place updates (#8070) This PR is a breaking change and also deprecates `minio update` command, from this release onwards all users are advised to just use `mc admin update` --- cmd/admin-handlers.go | 204 ++++++++++------ cmd/admin-handlers_test.go | 124 ++-------- cmd/admin-router.go | 9 +- cmd/globals.go | 12 - cmd/main.go | 1 - cmd/peer-rest-client.go | 2 +- cmd/peer-rest-server.go | 25 +- cmd/service.go | 9 +- cmd/signals.go | 8 +- cmd/update-main.go | 222 ++++++++---------- pkg/madmin/README.md | 138 ++++++----- pkg/madmin/api-trace.go | 77 ------ .../examples/{trace.go => service-trace.go} | 2 +- pkg/madmin/service-commands.go | 142 ++++++++--- pkg/madmin/version-commands.go | 54 ----- 15 files changed, 456 insertions(+), 573 deletions(-) delete mode 100644 pkg/madmin/api-trace.go rename pkg/madmin/examples/{trace.go => service-trace.go} (95%) delete mode 100644 pkg/madmin/version-commands.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index a0e74887a..213f74f6d 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -21,7 +21,7 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -66,107 +66,122 @@ const ( mgmtForceStop = "forceStop" ) -var ( - // This struct literal represents the Admin API version that - // the server uses. - adminAPIVersionInfo = madmin.AdminAPIVersionInfo{ - Version: "1", +func updateServer() (us madmin.ServiceUpdateStatus, err error) { + minioMode := getMinioMode() + updateMsg, sha256Hex, _, latestReleaseTime, err := getUpdateInfo(updateTimeout, minioMode) + if err != nil { + return us, err } -) - -// VersionHandler - GET /minio/admin/version -// ----------- -// Returns Administration API version -func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "Version") - - objectAPI := validateAdminReq(ctx, w, r) - if objectAPI == nil { - return + if updateMsg == "" { + us.CurrentVersion = Version + us.UpdatedVersion = Version + return us, nil } - - jsonBytes, err := json.Marshal(adminAPIVersionInfo) - if err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return + if err = doUpdate(sha256Hex, minioMode, latestReleaseTime, true); err != nil { + return us, err } - - writeSuccessResponseJSON(w, jsonBytes) + us.CurrentVersion = Version + us.UpdatedVersion = latestReleaseTime.Format(minioReleaseTagTimeLayout) + return us, nil } -// ServiceStatusHandler - GET /minio/admin/v1/service +// ServiceActionHandler - POST /minio/admin/v1/service +// Body: {"action": } // ---------- -// Returns server version and uptime. -func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "ServiceStatus") +// restarts/updates/stops minio server gracefully. In a distributed setup, +// restarts/updates/stops all the servers in the cluster. Also asks for +// server version and uptime. +func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ServiceAction") + + vars := mux.Vars(r) + action := vars["action"] objectAPI := validateAdminReq(ctx, w, r) if objectAPI == nil { return } - // Fetch server version - serverVersion := madmin.ServerVersion{ - Version: Version, - CommitID: CommitID, + var serviceSig serviceSignal + switch madmin.ServiceAction(action) { + case madmin.ServiceActionRestart: + serviceSig = serviceRestart + case madmin.ServiceActionStop: + serviceSig = serviceStop + case madmin.ServiceActionUpdate: + if globalInplaceUpdateDisabled { + // if MINIO_UPDATE=off - inplace update is disabled, mostly + // in containers. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + // update, updates the server and restarts them. + serviceSig = serviceUpdate | serviceRestart + case madmin.ServiceActionStatus: + serviceSig = serviceStatus + default: + logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action)) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return } - // Fetch uptimes from all peers and pick the latest. - uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx)) + if serviceSig&serviceUpdate == serviceUpdate { + for _, nerr := range globalNotificationSys.SignalService(serviceSig) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + logger.LogIf(ctx, nerr.Err) + } + } - // Create API response - serverStatus := madmin.ServiceStatus{ - ServerVersion: serverVersion, - Uptime: uptime, - } + updateStatus, err := updateServer() + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } - // Marshal API response - jsonBytes, err := json.Marshal(serverStatus) - if err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + // Marshal API response + jsonBytes, err := json.Marshal(updateStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, jsonBytes) + + if updateStatus.CurrentVersion != updateStatus.UpdatedVersion { + // We did upgrade - restart all services. + globalServiceSignalCh <- serviceSig + } return } - // Reply with storage information (across nodes in a - // distributed setup) as json. - writeSuccessResponseJSON(w, jsonBytes) -} + if serviceSig == serviceStatus { + // Fetch server version + serverVersion := madmin.ServerVersion{ + Version: Version, + CommitID: CommitID, + } -// ServiceStopNRestartHandler - POST /minio/admin/v1/service -// Body: {"action": } -// ---------- -// Restarts/Stops minio server gracefully. In a distributed setup, -// restarts all the servers in the cluster. -func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "ServiceStopNRestart") + // Fetch uptimes from all peers and pick the latest. + uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx)) - objectAPI := validateAdminReq(ctx, w, r) - if objectAPI == nil { - return - } + // Create API response + serverStatus := madmin.ServiceStatus{ + ServerVersion: serverVersion, + Uptime: uptime, + } - var sa madmin.ServiceAction - err := json.NewDecoder(r.Body).Decode(&sa) - if err != nil { - writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrRequestBodyParse), r.URL) - return - } + // Marshal API response + jsonBytes, err := json.Marshal(serverStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } - var serviceSig serviceSignal - switch sa.Action { - case madmin.ServiceActionValueRestart: - serviceSig = serviceRestart - case madmin.ServiceActionValueStop: - serviceSig = serviceStop - default: - writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) - logger.LogIf(ctx, errors.New("Invalid service action received")) + writeSuccessResponseJSON(w, jsonBytes) return } - // Reply to the client before restarting minio server. - writeSuccessResponseHeadersOnly(w) - // Notify all other MinIO peers signal service. for _, nerr := range globalNotificationSys.SignalService(serviceSig) { if nerr.Err != nil { @@ -175,6 +190,9 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h } } + // Reply to the client before restarting, stopping MinIO server. + writeSuccessResponseHeadersOnly(w) + globalServiceSignalCh <- serviceSig } @@ -989,6 +1007,24 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re writeSuccessResponseJSON(w, []byte(econfigData)) } +// AdminError - is a generic error for all admin APIs. +type AdminError struct { + Code string + Message string + StatusCode int +} + +func (ae AdminError) Error() string { + return ae.Message +} + +// Admin API errors +const ( + AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure" + AdminUpdateURLNotReachable = "XMinioAdminUpdateURLNotReachable" + AdminUpdateApplyFailure = "XMinioAdminUpdateApplyFailure" +) + // toAdminAPIErrCode - converts errXLWriteQuorum error to admin API // specific error. func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode { @@ -1001,7 +1037,21 @@ func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode { } func toAdminAPIErr(ctx context.Context, err error) APIError { - return errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err)) + if err == nil { + return noError + } + apiErr := errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err)) + if apiErr.Code == "InternalError" { + switch e := err.(type) { + case AdminError: + apiErr = APIError{ + Code: e.Code, + Description: e.Message, + HTTPStatusCode: e.StatusCode, + } + } + } + return apiErr } // RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey= diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index e06dda200..c6690834b 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc. + * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -344,41 +344,6 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) { return objLayer, xlDirs, nil } -func TestAdminVersionHandler(t *testing.T) { - adminTestBed, err := prepareAdminXLTestBed() - if err != nil { - t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") - } - defer adminTestBed.TearDown() - - req, err := newTestRequest("GET", "/minio/admin/version", 0, nil) - if err != nil { - t.Fatalf("Failed to construct request - %v", err) - } - cred := globalServerConfig.GetCredential() - err = signRequestV4(req, cred.AccessKey, cred.SecretKey) - if err != nil { - t.Fatalf("Failed to sign request - %v", err) - } - - rec := httptest.NewRecorder() - adminTestBed.router.ServeHTTP(rec, req) - if http.StatusOK != rec.Code { - t.Errorf("Unexpected status code - got %d but expected %d", - rec.Code, http.StatusOK) - } - - var result madmin.AdminAPIVersionInfo - err = json.NewDecoder(rec.Body).Decode(&result) - if err != nil { - t.Errorf("json parse err: %v", err) - } - - if result != adminAPIVersionInfo { - t.Errorf("unexpected version: %v", result) - } -} - // cmdType - Represents different service subcomands like status, stop // and restart. type cmdType int @@ -387,52 +352,8 @@ const ( statusCmd cmdType = iota restartCmd stopCmd - setCreds ) -// String - String representation for cmdType -func (c cmdType) String() string { - switch c { - case statusCmd: - return "status" - case restartCmd: - return "restart" - case stopCmd: - return "stop" - case setCreds: - return "set-credentials" - } - return "" -} - -// apiMethod - Returns the HTTP method corresponding to the admin REST -// API for a given cmdType value. -func (c cmdType) apiMethod() string { - switch c { - case statusCmd: - return "GET" - case restartCmd: - return "POST" - case stopCmd: - return "POST" - case setCreds: - return "PUT" - } - return "GET" -} - -// apiEndpoint - Return endpoint for each admin REST API mapped to a -// command here. -func (c cmdType) apiEndpoint() string { - switch c { - case statusCmd, restartCmd, stopCmd: - return "/minio/admin/v1/service" - case setCreds: - return "/minio/admin/v1/config/credential" - } - return "" -} - // toServiceSignal - Helper function that translates a given cmdType // value to its corresponding serviceSignal value. func (c cmdType) toServiceSignal() serviceSignal { @@ -447,14 +368,16 @@ func (c cmdType) toServiceSignal() serviceSignal { return serviceStatus } -func (c cmdType) toServiceActionValue() madmin.ServiceActionValue { +func (c cmdType) toServiceAction() madmin.ServiceAction { switch c { case restartCmd: - return madmin.ServiceActionValueRestart + return madmin.ServiceActionRestart case stopCmd: - return madmin.ServiceActionValueStop + return madmin.ServiceActionStop + case statusCmd: + return madmin.ServiceActionStatus } - return madmin.ServiceActionValueStop + return madmin.ServiceActionStatus } // testServiceSignalReceiver - Helper function that simulates a @@ -469,19 +392,15 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) { // getServiceCmdRequest - Constructs a management REST API request for service // subcommands for a given cmdType value. -func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) { - req, err := newTestRequest(cmd.apiMethod(), cmd.apiEndpoint(), 0, nil) +func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) { + queryVal := url.Values{} + queryVal.Set("action", string(cmd.toServiceAction())) + resource := "/minio/admin/v1/service?" + queryVal.Encode() + req, err := newTestRequest(http.MethodPost, resource, 0, nil) if err != nil { return nil, err } - // Set body - req.Body = ioutil.NopCloser(bytes.NewReader(body)) - req.ContentLength = int64(len(body)) - - // Set sha-sum header - req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body)) - // management REST API uses signature V4 for authentication. err = signRequestV4(req, cred.AccessKey, cred.SecretKey) if err != nil { @@ -517,13 +436,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) { } credentials := globalServerConfig.GetCredential() - body, err := json.Marshal(madmin.ServiceAction{ - Action: cmd.toServiceActionValue()}) - if err != nil { - t.Fatalf("JSONify error: %v", err) - } - - req, err := getServiceCmdRequest(cmd, credentials, body) + req, err := getServiceCmdRequest(cmd, credentials) if err != nil { t.Fatalf("Failed to build service status request %v", err) } @@ -531,6 +444,11 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) { rec := httptest.NewRecorder() adminTestBed.router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + resp, _ := ioutil.ReadAll(rec.Body) + t.Errorf("Expected to receive %d status code but received %d. Body (%s)", + http.StatusOK, rec.Code, string(resp)) + } if cmd == statusCmd { expectedInfo := madmin.ServiceStatus{ ServerVersion: madmin.ServerVersion{Version: Version, CommitID: CommitID}, @@ -544,12 +462,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) { } } - if rec.Code != http.StatusOK { - resp, _ := ioutil.ReadAll(rec.Body) - t.Errorf("Expected to receive %d status code but received %d. Body (%s)", - http.StatusOK, rec.Code, string(resp)) - } - // Wait until testServiceSignalReceiver() called in a goroutine quits. wg.Wait() } diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 23c216726..174561246 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -38,17 +38,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter() // Version handler - adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(httpTraceAll(adminAPI.VersionHandler)) - adminV1Router := adminRouter.PathPrefix("/v1").Subrouter() /// Service operations - // Service status - adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStatusHandler)) - - // Service restart and stop - TODO - adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStopNRestartHandler)) + // Get status, update, restart and stop MinIO service. + adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}") // Info operations adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler)) diff --git a/cmd/globals.go b/cmd/globals.go index a4dbfb94a..8fe4f7b2c 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -339,18 +339,6 @@ var ( } return fmt.Sprintf }() - colorGreenBold = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgGreen, color.Bold).SprintfFunc() - } - return fmt.Sprintf - }() - colorRedBold = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgRed, color.Bold).SprintfFunc() - } - return fmt.Sprintf - }() ) // Returns minio global information, as a key value map. diff --git a/cmd/main.go b/cmd/main.go index fb92a7687..2048a640a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -116,7 +116,6 @@ func newApp(name string) *cli.App { // Register all commands. registerCommand(serverCmd) registerCommand(gatewayCmd) - registerCommand(updateCmd) registerCommand(versionCmd) // Set up app. diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index 0cb8f5a75..d303f1dea 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -506,7 +506,7 @@ func (client *peerRESTClient) LoadGroup(group string) error { // SignalService - sends signal to peer nodes. func (client *peerRESTClient) SignalService(sig serviceSignal) error { values := make(url.Values) - values.Set(peerRESTSignal, string(sig)) + values.Set(peerRESTSignal, strconv.Itoa(int(sig))) respBody, err := client.call(peerRESTMethodSignalService, values, nil, -1) if err != nil { return err diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 4b624e709..967965544 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -824,10 +824,29 @@ func (s *peerRESTServer) SignalServiceHandler(w http.ResponseWriter, r *http.Req s.writeErrorResponse(w, errors.New("signal name is missing")) return } - signal := serviceSignal(signalString) + si, err := strconv.Atoi(signalString) + if err != nil { + s.writeErrorResponse(w, err) + return + } + signal := serviceSignal(si) defer w.(http.Flusher).Flush() - switch signal { - case serviceRestart, serviceStop: + switch { + case signal&serviceUpdate == serviceUpdate: + us, err := updateServer() + if err != nil { + s.writeErrorResponse(w, err) + return + } + // We didn't upgrade, no need to restart + // the services. + if us.CurrentVersion == us.UpdatedVersion { + return + } + fallthrough + case signal&serviceRestart == serviceRestart: + fallthrough + case signal&serviceStop == serviceStop: globalServiceSignalCh <- signal default: s.writeErrorResponse(w, errUnsupportedSignal) diff --git a/cmd/service.go b/cmd/service.go index e770cfe1a..81514c2fc 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -23,12 +23,13 @@ import ( ) // Type of service signals currently supported. -type serviceSignal string +type serviceSignal int const ( - serviceStatus serviceSignal = "serviceStatus" // Gets status about the service. - serviceRestart = "serviceRestart" // Restarts the service. - serviceStop = "serviceStop" // Stops the server. + serviceStatus serviceSignal = iota // Gets status about the service. + serviceRestart // Restarts the service. + serviceStop // Stops the server. + serviceUpdate // Updates the server. // Add new service requests here. ) diff --git a/cmd/signals.go b/cmd/signals.go index 0c9202003..937876233 100644 --- a/cmd/signals.go +++ b/cmd/signals.go @@ -77,16 +77,14 @@ func handleSignals() { logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String())) exit(stopProcess()) case signal := <-globalServiceSignalCh: - switch signal { - case serviceStatus: - // Ignore this at the moment. - case serviceRestart: + switch { + case signal&serviceRestart == serviceRestart: logger.Info("Restarting on service signal") stop := stopProcess() rerr := restartProcess() logger.LogIf(context.Background(), rerr) exit(stop && rerr == nil) - case serviceStop: + case signal&serviceStop == serviceStop: logger.Info("Stopping on service signal") exit(stopProcess()) } diff --git a/cmd/update-main.go b/cmd/update-main.go index 2ba1b0fca..64313f7f3 100644 --- a/cmd/update-main.go +++ b/cmd/update-main.go @@ -23,6 +23,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "net" "net/http" "os" "path/filepath" @@ -30,45 +31,12 @@ import ( "strings" "time" - "github.com/fatih/color" "github.com/inconshreveable/go-update" - "github.com/minio/cli" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" _ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier. ) -// Check for new software updates. -var updateCmd = cli.Command{ - Name: "update", - Usage: "update minio to latest release", - Action: mainUpdate, - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "quiet", - Usage: "disable any update prompt message", - }, - }, - CustomHelpTemplate: `Name: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}}{{if .VisibleFlags}} [FLAGS]{{end}} -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -EXIT STATUS: - 0 - You are already running the most recent version. - 1 - New update was applied successfully. - -1 - Error in getting update information. - -EXAMPLES: - 1. Check and update minio: - {{.Prompt}} {{.HelpName}} -`, -} - const ( minioReleaseTagTimeLayout = "2006-01-02T15-04-05Z" minioOSARCH = runtime.GOOS + "-" + runtime.GOARCH @@ -298,38 +266,58 @@ func getUserAgent(mode string) string { } func downloadReleaseURL(releaseChecksumURL string, timeout time.Duration, mode string) (content string, err error) { - req, err := http.NewRequest("GET", releaseChecksumURL, nil) + req, err := http.NewRequest(http.MethodGet, releaseChecksumURL, nil) if err != nil { - return content, err + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } } req.Header.Set("User-Agent", getUserAgent(mode)) - client := &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - // need to close connection after usage. - DisableKeepAlives: true, - }, - } - + client := &http.Client{Transport: getUpdateTransport(timeout)} resp, err := client.Do(req) if err != nil { - return content, err + if isNetworkOrHostDown(err) { + return content, AdminError{ + Code: AdminUpdateURLNotReachable, + Message: err.Error(), + StatusCode: http.StatusServiceUnavailable, + } + } + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } } if resp == nil { - return content, fmt.Errorf("No response from server to download URL %s", releaseChecksumURL) + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("No response from server to download URL %s", releaseChecksumURL), + StatusCode: http.StatusInternalServerError, + } } defer xhttp.DrainBody(resp.Body) if resp.StatusCode != http.StatusOK { - return content, fmt.Errorf("Error downloading URL %s. Response: %v", releaseChecksumURL, resp.Status) + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("Error downloading URL %s. Response: %v", releaseChecksumURL, resp.Status), + StatusCode: resp.StatusCode, + } } contentBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return content, fmt.Errorf("Error reading response. %s", err) + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("Error reading response. %s", err), + StatusCode: http.StatusInternalServerError, + } } - return string(contentBytes), err + return string(contentBytes), nil } // DownloadReleaseData - downloads release data from minio official server. @@ -338,6 +326,7 @@ func DownloadReleaseData(timeout time.Duration, mode string) (data string, err e if runtime.GOOS == globalWindowsOSName { releaseURLs = minioReleaseWindowsInfoURLs } + return func() (data string, err error) { for _, url := range releaseURLs { data, err = downloadReleaseURL(url, timeout, mode) @@ -345,7 +334,7 @@ func DownloadReleaseData(timeout time.Duration, mode string) (data string, err e return data, nil } } - return data, fmt.Errorf("Failed to fetch release URL - last error: %s", err) + return data, err }() } @@ -385,6 +374,27 @@ func parseReleaseData(data string) (sha256Hex string, releaseTime time.Time, err return sha256Hex, releaseTime, err } +const updateTimeout = 10 * time.Second + +func getUpdateTransport(timeout time.Duration) http.RoundTripper { + var updateTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: timeout, + KeepAlive: timeout, + DualStack: true, + } + return dialer.DialContext(ctx, network, addr) + }, + IdleConnTimeout: timeout, + TLSHandshakeTimeout: timeout, + ExpectContinueTimeout: timeout, + DisableCompression: true, + } + return updateTransport +} + func getLatestReleaseTime(timeout time.Duration, mode string) (sha256Hex string, releaseTime time.Time, err error) { data, err := DownloadReleaseData(timeout, mode) if err != nil { @@ -450,21 +460,39 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256 return prepareUpdateMessage(downloadURL, older), sha256Hex, currentReleaseTime, latestReleaseTime, nil } -func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateStatusMsg string, err error) { - if !ok { - updateStatusMsg = colorRedBold("MinIO update to version RELEASE.%s canceled.", - latestReleaseTime.Format(minioReleaseTagTimeLayout)) - return updateStatusMsg, nil - } +func doUpdate(sha256Hex string, mode string, latestReleaseTime time.Time, ok bool) (err error) { var sha256Sum []byte sha256Sum, err = hex.DecodeString(sha256Hex) if err != nil { - return updateStatusMsg, err + return err + } + + clnt := &http.Client{Transport: getUpdateTransport(30 * time.Second)} + req, err := http.NewRequest(http.MethodGet, getDownloadURL(releaseTimeToReleaseTag(latestReleaseTime)), nil) + if err != nil { + return AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } } - resp, err := http.Get(getDownloadURL(releaseTimeToReleaseTag(latestReleaseTime))) + req.Header.Set("User-Agent", getUserAgent(mode)) + + resp, err := clnt.Do(req) if err != nil { - return updateStatusMsg, err + if isNetworkOrHostDown(err) { + return AdminError{ + Code: AdminUpdateURLNotReachable, + Message: err.Error(), + StatusCode: http.StatusServiceUnavailable, + } + } + return AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } } defer xhttp.DrainBody(resp.Body) @@ -475,73 +503,19 @@ func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateSta Checksum: sha256Sum, }, ); err != nil { - return updateStatusMsg, err - } - - return colorGreenBold("MinIO updated to version RELEASE.%s successfully.", - latestReleaseTime.Format(minioReleaseTagTimeLayout)), nil -} - -// Confirm continues prompting until the input is boolean-ish. -func confirm(prompt string, args ...interface{}) bool { - for { - var s string - fmt.Fprintf(color.Output, prompt+": ", args...) - fmt.Scanln(&s) - switch s { - case "Yes", "yes", "y", "Y": - return true - case "No", "no", "n", "N": - return false + if os.IsPermission(err) { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: err.Error(), + StatusCode: http.StatusForbidden, + } } - } -} - -func shouldUpdate(quiet bool, sha256Hex string, latestReleaseTime time.Time) (ok bool) { - ok = true - if !quiet { - ok = confirm(colorGreenBold("Update to RELEASE.%s ? [%s]", latestReleaseTime.Format(minioReleaseTagTimeLayout), "y/n")) - } - return ok -} - -func mainUpdate(ctx *cli.Context) { - if len(ctx.Args()) != 0 { - cli.ShowCommandHelpAndExit(ctx, "update", -1) - } - - handleCommonEnvVars() - - quiet := ctx.Bool("quiet") || ctx.GlobalBool("quiet") - if quiet { - logger.EnableQuiet() - } - - minioMode := "" - updateMsg, sha256Hex, _, latestReleaseTime, err := getUpdateInfo(10*time.Second, minioMode) - if err != nil { - logger.Info(err.Error()) - os.Exit(-1) - } - - // Nothing to update running the latest release. - if updateMsg == "" { - logger.Info(colorGreenBold("You are already running the most recent version of ‘minio’.")) - os.Exit(0) - } - - logger.Info(updateMsg) - // if the in-place update is disabled then we shouldn't ask the - // user to update the binaries. - if strings.Contains(updateMsg, minioReleaseURL) && !globalInplaceUpdateDisabled { - var updateStatusMsg string - updateStatusMsg, err = doUpdate(sha256Hex, latestReleaseTime, shouldUpdate(quiet, sha256Hex, latestReleaseTime)) - if err != nil { - logger.Info(colorRedBold("Unable to update ‘minio’.")) - logger.Info(err.Error()) - os.Exit(-1) + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, } - logger.Info(updateStatusMsg) - os.Exit(1) } + + return nil } diff --git a/pkg/madmin/README.md b/pkg/madmin/README.md index 62a15eebd..0cb5c9956 100644 --- a/pkg/madmin/README.md +++ b/pkg/madmin/README.md @@ -41,12 +41,13 @@ func main() { ``` -| Service operations | Info operations | Healing operations | Config operations | Top operations | IAM operations | Misc | -|:------------------------------------------|:--------------------------------------------|:-------------------|:----------------------------------|:------------------------|:--------------------------------------|:--------------------------------------------------| -| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser) | | -| [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | | [`SetConfig`](#SetConfig) | | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) | -| [`Trace`](#Trace) | [`ServerMemUsageInfo`](#ServerMemUsageInfo) | | [`GetConfigKeys`](#GetConfigKeys) | | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) | -| | | | [`SetConfigKeys`](#SetConfigKeys) | | [`AddCannedPolicy`](#AddCannedPolicy) | | +| Service operations | Info operations | Healing operations | Config operations | Top operations | IAM operations | Misc | +|:------------------------------------|:--------------------------------------------|:-------------------|:----------------------------------|:------------------------|:--------------------------------------|:--------------------------------------------------| +| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser) | | +| [`ServiceRestart`](#ServiceRestart) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | | [`SetConfig`](#SetConfig) | | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) | +| [`ServiceStop`](#ServiceStop) | [`ServerMemUsageInfo`](#ServerMemUsageInfo) | | [`GetConfigKeys`](#GetConfigKeys) | | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) | +| [`ServiceUpdate`](#ServiceUpdate) | | | [`SetConfigKeys`](#SetConfigKeys) | | [`AddCannedPolicy`](#AddCannedPolicy) | | +| [`ServiceTrace`](#ServiceTrace) | | | | | | | ## 1. Constructor @@ -64,25 +65,7 @@ __Parameters__ | `secretAccessKey` | _string_ | Secret key for the object storage endpoint. | | `ssl` | _bool_ | Set this value to 'true' to enable secure (HTTPS) access. | -## 2. Admin API Version - - -### VersionInfo() (AdminAPIVersionInfo, error) -Fetch server's supported Administrative API version. - - __Example__ - -``` go - - info, err := madmClnt.VersionInfo() - if err != nil { - log.Fatalln(err) - } - log.Printf("%s\n", info.Version) - -``` - -## 3. Service operations +## 2. Service operations ### ServiceStatus() (ServiceStatusMetadata, error) @@ -111,25 +94,73 @@ Fetch service status, replies disk space used, backend type and total disks offl ``` - -### ServiceSendAction(act ServiceActionValue) (error) -Sends a service action command to service - possible actions are restarting and stopping the server. + +### ServiceRestart() error +Sends a service action restart command to MinIO server. __Example__ +```go + // To restart the service, restarts all servers in the cluster. + err := madmClnt.ServiceRestart() + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +``` - ```go - // to restart - st, err := madmClnt.ServiceSendAction(ServiceActionValueRestart) - // or to stop - // st, err := madmClnt.ServiceSendAction(ServiceActionValueStop) - if err != nil { - log.Fatalln(err) - } - log.Printf("Success") - ``` + +### ServiceStop() error +Sends a service action stop command to MinIO server. + + __Example__ + +```go + // To stop the service, stops all servers in the cluster. + err := madmClnt.ServiceStop() + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +``` + + +### ServiceUpdate() (ServiceUpdateStatus, error) +Sends a service action update command to MinIO server, to update MinIO server to latest release. + + __Example__ + +```go + // To update the service, update and restarts all the servers in the cluster. + us, err := madmClnt.ServiceUpdate() + if err != nil { + log.Fatalln(err) + } + if us.CurrentVersion != us.UpdatedVersion { + log.Printf("Updated server version from %s to %s successfully", us.CurrentVersion, us.UpdatedVersion) + } +``` + + +### ServiceTrace(allTrace bool, doneCh <-chan struct{}) <-chan TraceInfo +Enable HTTP request tracing on all nodes in a MinIO cluster + +__Example__ + +``` go + doneCh := make(chan struct{}) + defer close(doneCh) + // listen to all trace including internal API calls + allTrace := true + // Start listening on all trace activity. + traceCh := madmClnt.ServiceTrace(allTrace, doneCh) + for traceInfo := range traceCh { + fmt.Println(traceInfo.String()) + } +``` -## 4. Info operations + +## 3. Info operations ### ServerInfo() ([]ServerInfo, error) @@ -262,7 +293,7 @@ Fetches Mem utilization for all cluster nodes. Returned value is in Bytes. | `mem.Usage.Mem` | _uint64_ | The total number of bytes obtained from the OS | | `mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info | -## 6. Heal operations +## 5. Heal operations ### Heal(bucket, prefix string, healOpts HealOpts, clientToken string, forceStart bool, forceStop bool) (start HealStartSuccess, status HealTaskStatus, err error) @@ -327,7 +358,7 @@ __Example__ | `DiskInfo.AvailableOn` | _[]int_ | List of disks on which the healed entity is present and healthy | | `DiskInfo.HealedOn` | _[]int_ | List of disks on which the healed entity was restored | -## 7. Config operations +## 6. Config operations ### GetConfig() ([]byte, error) @@ -405,7 +436,7 @@ __Example__ log.Println("New configuration successfully set") ``` -## 8. Top operations +## 7. Top operations ### TopLocks() (LockEntries, error) @@ -427,7 +458,7 @@ __Example__ log.Println("TopLocks received successfully: ", string(out)) ``` -## 9. IAM operations +## 8. IAM operations ### AddCannedPolicy(policyName string, policy string) error @@ -483,7 +514,7 @@ __Example__ } ``` -## 10. Misc operations +## 9. Misc operations ### StartProfiling(profiler string) error @@ -537,22 +568,3 @@ __Example__ log.Println("Profiling data successfully downloaded.") ``` - - -### Trace(allTrace bool,doneCh <-chan struct{}) <-chan TraceInfo -Enable HTTP request tracing on all nodes in a MinIO cluster - -__Example__ - -``` go - doneCh := make(chan struct{}) - defer close(doneCh) - // listen to all trace including internal API calls - allTrace := true - // Start listening on all trace activity. - traceCh := madmClnt.Trace(allTrace,doneCh) - for traceInfo := range traceCh { - fmt.Println(traceInfo.String()) - } - log.Println("Success") -``` \ No newline at end of file diff --git a/pkg/madmin/api-trace.go b/pkg/madmin/api-trace.go deleted file mode 100644 index 9b276bb10..000000000 --- a/pkg/madmin/api-trace.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package madmin - -import ( - "encoding/json" - "net/http" - "net/url" - "strconv" - - trace "github.com/minio/minio/pkg/trace" -) - -// TraceInfo holds http trace -type TraceInfo struct { - Trace trace.Info - Err error `json:"-"` -} - -// Trace - listen on http trace notifications. -func (adm AdminClient) Trace(allTrace, errTrace bool, doneCh <-chan struct{}) <-chan TraceInfo { - traceInfoCh := make(chan TraceInfo) - // Only success, start a routine to start reading line by line. - go func(traceInfoCh chan<- TraceInfo) { - defer close(traceInfoCh) - for { - urlValues := make(url.Values) - urlValues.Set("all", strconv.FormatBool(allTrace)) - urlValues.Set("err", strconv.FormatBool(errTrace)) - reqData := requestData{ - relPath: "/v1/trace", - queryValues: urlValues, - } - // Execute GET to call trace handler - resp, err := adm.executeMethod("GET", reqData) - if err != nil { - closeResponse(resp) - return - } - - if resp.StatusCode != http.StatusOK { - traceInfoCh <- TraceInfo{Err: httpRespToErrorResponse(resp)} - return - } - - dec := json.NewDecoder(resp.Body) - for { - var info trace.Info - if err = dec.Decode(&info); err != nil { - break - } - select { - case <-doneCh: - return - case traceInfoCh <- TraceInfo{Trace: info}: - } - } - } - }(traceInfoCh) - - // Returns the trace info channel, for caller to start reading from. - return traceInfoCh -} diff --git a/pkg/madmin/examples/trace.go b/pkg/madmin/examples/service-trace.go similarity index 95% rename from pkg/madmin/examples/trace.go rename to pkg/madmin/examples/service-trace.go index 7909b2447..645c33b4f 100644 --- a/pkg/madmin/examples/trace.go +++ b/pkg/madmin/examples/service-trace.go @@ -43,7 +43,7 @@ func main() { // in the minio cluster. allTrace := false errTrace := false - traceCh := madmClnt.Trace(allTrace, errTrace, doneCh) + traceCh := madmClnt.ServiceTrace(allTrace, errTrace, doneCh) for traceInfo := range traceCh { if traceInfo.Err != nil { fmt.Println(traceInfo.Err) diff --git a/pkg/madmin/service-commands.go b/pkg/madmin/service-commands.go index 833f0963d..ab20f8ddd 100644 --- a/pkg/madmin/service-commands.go +++ b/pkg/madmin/service-commands.go @@ -21,7 +21,11 @@ import ( "encoding/json" "io/ioutil" "net/http" + "net/url" + "strconv" "time" + + trace "github.com/minio/minio/pkg/trace" ) // ServerVersion - server version @@ -30,72 +34,134 @@ type ServerVersion struct { CommitID string `json:"commitID"` } +// ServiceUpdateStatus - contains the response of service update API +type ServiceUpdateStatus struct { + CurrentVersion string `json:"currentVersion"` + UpdatedVersion string `json:"updatedVersion"` +} + // ServiceStatus - contains the response of service status API type ServiceStatus struct { ServerVersion ServerVersion `json:"serverVersion"` Uptime time.Duration `json:"uptime"` } -// ServiceStatus - Connect to a minio server and call Service Status -// Management API to fetch server's storage information represented by -// ServiceStatusMetadata structure +// ServiceStatus - Returns current server uptime and current +// running version of MinIO server. func (adm *AdminClient) ServiceStatus() (ss ServiceStatus, err error) { - // Request API to GET service status - resp, err := adm.executeMethod("GET", requestData{relPath: "/v1/service"}) - defer closeResponse(resp) + respBytes, err := adm.serviceCallAction(ServiceActionStatus) if err != nil { return ss, err } + err = json.Unmarshal(respBytes, &ss) + return ss, err +} - // Check response http status code - if resp.StatusCode != http.StatusOK { - return ss, httpRespToErrorResponse(resp) - } +// ServiceRestart - restarts the MinIO cluster +func (adm *AdminClient) ServiceRestart() error { + _, err := adm.serviceCallAction(ServiceActionRestart) + return err +} - respBytes, err := ioutil.ReadAll(resp.Body) +// ServiceStop - stops the MinIO cluster +func (adm *AdminClient) ServiceStop() error { + _, err := adm.serviceCallAction(ServiceActionStop) + return err +} + +// ServiceUpdate - updates and restarts the MinIO cluster to latest version. +func (adm *AdminClient) ServiceUpdate() (us ServiceUpdateStatus, err error) { + respBytes, err := adm.serviceCallAction(ServiceActionUpdate) if err != nil { - return ss, err + return us, err } - - err = json.Unmarshal(respBytes, &ss) - return ss, err + err = json.Unmarshal(respBytes, &us) + return us, err } -// ServiceActionValue - type to restrict service-action values -type ServiceActionValue string +// ServiceAction - type to restrict service-action values +type ServiceAction string const ( - // ServiceActionValueRestart represents restart action - ServiceActionValueRestart ServiceActionValue = "restart" - // ServiceActionValueStop represents stop action - ServiceActionValueStop = "stop" + // ServiceActionStatus represents status action + ServiceActionStatus ServiceAction = "status" + // ServiceActionRestart represents restart action + ServiceActionRestart = "restart" + // ServiceActionStop represents stop action + ServiceActionStop = "stop" + // ServiceActionUpdate represents update action + ServiceActionUpdate = "update" ) -// ServiceAction - represents POST body for service action APIs -type ServiceAction struct { - Action ServiceActionValue `json:"action"` -} - -// ServiceSendAction - Call Service Restart/Stop API to restart/stop a -// MinIO server -func (adm *AdminClient) ServiceSendAction(action ServiceActionValue) error { - body, err := json.Marshal(ServiceAction{action}) - if err != nil { - return err - } +// serviceCallAction - call service restart/update/stop API. +func (adm *AdminClient) serviceCallAction(action ServiceAction) ([]byte, error) { + queryValues := url.Values{} + queryValues.Set("action", string(action)) // Request API to Restart server resp, err := adm.executeMethod("POST", requestData{ - relPath: "/v1/service", - content: body, + relPath: "/v1/service", + queryValues: queryValues, }) defer closeResponse(resp) if err != nil { - return err + return nil, err } if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp) + return nil, httpRespToErrorResponse(resp) } - return nil + + return ioutil.ReadAll(resp.Body) +} + +// ServiceTraceInfo holds http trace +type ServiceTraceInfo struct { + Trace trace.Info + Err error `json:"-"` +} + +// ServiceTrace - listen on http trace notifications. +func (adm AdminClient) ServiceTrace(allTrace, errTrace bool, doneCh <-chan struct{}) <-chan ServiceTraceInfo { + traceInfoCh := make(chan ServiceTraceInfo) + // Only success, start a routine to start reading line by line. + go func(traceInfoCh chan<- ServiceTraceInfo) { + defer close(traceInfoCh) + for { + urlValues := make(url.Values) + urlValues.Set("all", strconv.FormatBool(allTrace)) + urlValues.Set("err", strconv.FormatBool(errTrace)) + reqData := requestData{ + relPath: "/v1/trace", + queryValues: urlValues, + } + // Execute GET to call trace handler + resp, err := adm.executeMethod("GET", reqData) + if err != nil { + closeResponse(resp) + return + } + + if resp.StatusCode != http.StatusOK { + traceInfoCh <- ServiceTraceInfo{Err: httpRespToErrorResponse(resp)} + return + } + + dec := json.NewDecoder(resp.Body) + for { + var info trace.Info + if err = dec.Decode(&info); err != nil { + break + } + select { + case <-doneCh: + return + case traceInfoCh <- ServiceTraceInfo{Trace: info}: + } + } + } + }(traceInfoCh) + + // Returns the trace info channel, for caller to start reading from. + return traceInfoCh } diff --git a/pkg/madmin/version-commands.go b/pkg/madmin/version-commands.go deleted file mode 100644 index 9156d2762..000000000 --- a/pkg/madmin/version-commands.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package madmin - -import ( - "encoding/json" - "io/ioutil" - "net/http" -) - -// AdminAPIVersionInfo - contains admin API version information -type AdminAPIVersionInfo struct { - Version string `json:"version"` -} - -// VersionInfo - Connect to minio server and call the version API to -// retrieve the server API version -func (adm *AdminClient) VersionInfo() (verInfo AdminAPIVersionInfo, err error) { - var resp *http.Response - resp, err = adm.executeMethod("GET", requestData{relPath: "/version"}) - defer closeResponse(resp) - if err != nil { - return verInfo, err - } - - // Check response http status code - if resp.StatusCode != http.StatusOK { - return verInfo, httpRespToErrorResponse(resp) - } - - respBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return verInfo, err - } - - // Unmarshal the server's json response - err = json.Unmarshal(respBytes, &verInfo) - return verInfo, err -}