From 83abad0b3732da4bff851ef9c8a1bbabc9802ca2 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 21 Apr 2017 15:15:53 +0100 Subject: [PATCH] admin: ServerInfo() returns info for each node (#4150) ServerInfo() will gather information from all nodes before returning it back to the client. --- cmd/admin-handlers.go | 77 ++++++++++++++++--------------------- cmd/admin-handlers_test.go | 24 +++++++++--- cmd/admin-rpc-client.go | 51 +++++++++++++++++------- cmd/admin-rpc-server.go | 39 +++++++++++++------ cmd/http-stats.go | 8 ++++ cmd/server-mux.go | 8 +++- pkg/madmin/API.md | 65 +++++++++++++++++++++---------- pkg/madmin/info-commands.go | 29 ++++++++------ 8 files changed, 194 insertions(+), 107 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index b3773bdd3..8396e4e6b 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -26,6 +26,7 @@ import ( "net/url" "path" "strconv" + "sync" "time" ) @@ -229,14 +230,22 @@ type ServerHTTPStats struct { SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"` } -// ServerInfo holds the information that will be returned by ServerInfo API -type ServerInfo struct { +// ServerInfoData holds storage, connections and other +// information of a given server. +type ServerInfoData struct { StorageInfo StorageInfo `json:"storage"` ConnStats ServerConnStats `json:"network"` HTTPStats ServerHTTPStats `json:"http"` Properties ServerProperties `json:"server"` } +// ServerInfo holds server information result of one node +type ServerInfo struct { + Error error + Addr string + Data *ServerInfoData +} + // ServerInfoHandler - GET /?info // ---------- // Get server information @@ -248,55 +257,37 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt return } - // Build storage info - objLayer := newObjectLayerFn() - if objLayer == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) - return - } - storage := objLayer.StorageInfo() + // Web service response + reply := make([]ServerInfo, len(globalAdminPeers)) - // Build list of enabled ARNs queues - var arns []string - for queueArn := range globalEventNotifier.GetAllExternalTargets() { - arns = append(arns, queueArn) - } + var wg sync.WaitGroup - // Fetch uptimes from all peers. This may fail to due to lack - // of read-quorum availability. - uptime, err := getPeerUptimes(globalAdminPeers) - if err != nil { - writeErrorResponse(w, toAPIErrorCode(err), r.URL) - errorIf(err, "Unable to get uptime from majority of servers.") - return - } + // Gather server information for all nodes + for i, p := range globalAdminPeers { + wg.Add(1) - // Build server properties information - properties := ServerProperties{ - Version: Version, - CommitID: CommitID, - Region: serverConfig.GetRegion(), - SQSARN: arns, - Uptime: uptime, - } + // Gather information from a peer in a goroutine + go func(idx int, peer adminPeer) { + defer wg.Done() - // Build network info - connStats := ServerConnStats{ - TotalInputBytes: globalConnStats.getTotalInputBytes(), - TotalOutputBytes: globalConnStats.getTotalOutputBytes(), - } - httpStats := globalHTTPStats.toServerHTTPStats() + // Initialize server info at index + reply[idx] = ServerInfo{Addr: peer.addr} - // Build the whole returned information - info := ServerInfo{ - StorageInfo: storage, - ConnStats: connStats, - HTTPStats: httpStats, - Properties: properties, + serverInfoData, err := peer.cmdRunner.ServerInfoData() + if err != nil { + errorIf(err, "Unable to get server info from %s.", peer.addr) + reply[idx].Error = err + return + } + + reply[idx].Data = &serverInfoData + }(i, p) } + wg.Wait() + // Marshal API response - jsonBytes, err := json.Marshal(info) + jsonBytes, err := json.Marshal(reply) if err != nil { writeErrorResponse(w, ErrInternalError, r.URL) errorIf(err, "Failed to marshal storage info into json.") diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 4a6ad4d36..a8d984f9e 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -1300,17 +1300,29 @@ func TestAdminServerInfo(t *testing.T) { t.Errorf("Expected to succeed but failed with %d", rec.Code) } - result := ServerInfo{} - err = json.NewDecoder(rec.Body).Decode(&result) + results := []ServerInfo{} + err = json.NewDecoder(rec.Body).Decode(&results) if err != nil { t.Fatalf("Failed to decode set config result json %v", err) } - if result.StorageInfo.Free == 0 { - t.Error("Expected StorageInfo.Free to be non empty") + if len(results) == 0 { + t.Error("Expected at least one server info result") } - if result.Properties.Region != globalMinioDefaultRegion { - t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, result.Properties.Region) + + for _, serverInfo := range results { + if len(serverInfo.Addr) == 0 { + t.Error("Expected server address to be non empty") + } + if serverInfo.Error != nil { + t.Errorf("Unexpected error = %v\n", serverInfo.Error) + } + if serverInfo.Data.StorageInfo.Free == 0 { + t.Error("Expected StorageInfo.Free to be non empty") + } + if serverInfo.Data.Properties.Region != globalMinioDefaultRegion { + t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region) + } } } diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go index 52a27f4c3..13f65aa69 100644 --- a/cmd/admin-rpc-client.go +++ b/cmd/admin-rpc-client.go @@ -37,7 +37,7 @@ const ( serviceRestartRPC = "Admin.Restart" listLocksRPC = "Admin.ListLocks" reInitDisksRPC = "Admin.ReInitDisks" - uptimeRPC = "Admin.Uptime" + serverInfoDataRPC = "Admin.ServerInfoData" getConfigRPC = "Admin.GetConfig" writeTmpConfigRPC = "Admin.WriteTmpConfig" commitConfigRPC = "Admin.CommitConfig" @@ -59,7 +59,7 @@ type adminCmdRunner interface { Restart() error ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) ReInitDisks() error - Uptime() (time.Duration, error) + ServerInfoData() (ServerInfoData, error) GetConfig() ([]byte, error) WriteTmpConfig(tmpFileName string, configBytes []byte) error CommitConfig(tmpFileName string) error @@ -112,26 +112,48 @@ func (rc remoteAdminClient) ReInitDisks() error { return rc.Call(reInitDisksRPC, &args, &reply) } -// Uptime - Returns the uptime of this server. Timestamp is taken -// after object layer is initialized. -func (lc localAdminClient) Uptime() (time.Duration, error) { +// ServerInfoData - Returns the server info of this server. +func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) { if globalBootTime.IsZero() { - return time.Duration(0), errServerNotInitialized + return ServerInfoData{}, errServerNotInitialized } - return UTCNow().Sub(globalBootTime), nil + // Build storage info + objLayer := newObjectLayerFn() + if objLayer == nil { + return ServerInfoData{}, errServerNotInitialized + } + storage := objLayer.StorageInfo() + + var arns []string + for queueArn := range globalEventNotifier.GetAllExternalTargets() { + arns = append(arns, queueArn) + } + + return ServerInfoData{ + StorageInfo: storage, + ConnStats: globalConnStats.toServerConnStats(), + HTTPStats: globalHTTPStats.toServerHTTPStats(), + Properties: ServerProperties{ + Uptime: UTCNow().Sub(globalBootTime), + Version: Version, + CommitID: CommitID, + SQSARN: arns, + Region: serverConfig.GetRegion(), + }, + }, nil } -// Uptime - returns the uptime of the server to which the RPC call is made. -func (rc remoteAdminClient) Uptime() (time.Duration, error) { +// ServerInfo - returns the server info of the server to which the RPC call is made. +func (rc remoteAdminClient) ServerInfoData() (ServerInfoData, error) { args := AuthRPCArgs{} - reply := UptimeReply{} - err := rc.Call(uptimeRPC, &args, &reply) + reply := ServerInfoDataReply{} + err := rc.Call(serverInfoDataRPC, &args, &reply) if err != nil { - return time.Duration(0), err + return ServerInfoData{}, err } - return reply.Uptime, nil + return reply.ServerInfoData, nil } // GetConfig - returns config.json of the local server. @@ -384,7 +406,8 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) { wg.Add(1) go func(idx int, peer adminPeer) { defer wg.Done() - uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime() + serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData() + uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr }(i, peer) } wg.Wait() diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go index 5339cd7d0..25a03b310 100644 --- a/cmd/admin-rpc-server.go +++ b/cmd/admin-rpc-server.go @@ -53,10 +53,10 @@ type ListLocksReply struct { volLocks []VolumeLockInfo } -// UptimeReply - wraps the uptime response over RPC. -type UptimeReply struct { +// ServerInfoDataReply - wraps the server info response over RPC. +type ServerInfoDataReply struct { AuthRPCReply - Uptime time.Duration + ServerInfoData ServerInfoData } // ConfigReply - wraps the server config response over RPC. @@ -122,8 +122,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error { return nil } -// Uptime - returns the time when object layer was initialized on this server. -func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error { +// ServerInfo - returns the server info when object layer was initialized on this server. +func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error { if err := args.IsAuthenticated(); err != nil { return err } @@ -132,12 +132,29 @@ func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error { return errServerNotInitialized } - // N B The uptime is computed assuming that the system time is - // monotonic. This is not the case in time pkg in Go, see - // https://github.com/golang/go/issues/12914. This is expected - // to be fixed by go1.9. - *reply = UptimeReply{ - Uptime: UTCNow().Sub(globalBootTime), + // Build storage info + objLayer := newObjectLayerFn() + if objLayer == nil { + return errServerNotInitialized + } + storageInfo := objLayer.StorageInfo() + + var arns []string + for queueArn := range globalEventNotifier.GetAllExternalTargets() { + arns = append(arns, queueArn) + } + + reply.ServerInfoData = ServerInfoData{ + Properties: ServerProperties{ + Uptime: UTCNow().Sub(globalBootTime), + Version: Version, + CommitID: CommitID, + Region: serverConfig.GetRegion(), + SQSARN: arns, + }, + StorageInfo: storageInfo, + ConnStats: globalConnStats.toServerConnStats(), + HTTPStats: globalHTTPStats.toServerHTTPStats(), } return nil diff --git a/cmd/http-stats.go b/cmd/http-stats.go index ff917131d..9a8930e1c 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -52,6 +52,14 @@ func (s *ConnStats) getTotalOutputBytes() uint64 { return s.totalOutputBytes.Load() } +// Return connection stats (total input/output bytes) +func (s *ConnStats) toServerConnStats() ServerConnStats { + return ServerConnStats{ + TotalInputBytes: s.getTotalInputBytes(), + TotalOutputBytes: s.getTotalOutputBytes(), + } +} + // Prepare new ConnStats structure func newConnStats() *ConnStats { return &ConnStats{} diff --git a/cmd/server-mux.go b/cmd/server-mux.go index ed0cb2016..75839e26e 100644 --- a/cmd/server-mux.go +++ b/cmd/server-mux.go @@ -121,7 +121,9 @@ func (c *ConnMux) PeekProtocol() (string, error) { // bytes received from the client. func (c *ConnMux) Read(b []byte) (n int, err error) { // Update total incoming number of bytes. - defer globalConnStats.incInputBytes(n) + defer func() { + globalConnStats.incInputBytes(n) + }() n, err = c.peeker.Read(b) if err != nil { @@ -141,7 +143,9 @@ func (c *ConnMux) Read(b []byte) (n int, err error) { // keeps track of the total bytes written by the server. func (c *ConnMux) Write(b []byte) (n int, err error) { // Update total outgoing number of bytes. - defer globalConnStats.incOutputBytes(n) + defer func() { + globalConnStats.incOutputBytes(n) + }() // Call the conn write wrapper. return c.Conn.Write(b) diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md index dc09b9978..8d6c3d99e 100644 --- a/pkg/madmin/API.md +++ b/pkg/madmin/API.md @@ -122,7 +122,30 @@ If successful restarts the running minio service, for distributed setup restarts ``` -## 3. Lock operations +## 3. Info operations + + +### ServerInfo() ([]ServerInfo, error) +Fetch all information for all cluster nodes, such as uptime, region, network statistics, etc.. + + + __Example__ + + ```go + + serversInfo, err := madmClnt.ServerInfo() + if err != nil { + log.Fatalln(err) + } + + for _, peerInfo := range serversInfo { + log.Printf("Node: %s, Info: %v\n", peerInfo.Addr, peerInfo.Data) + } + + ``` + + +## 4. Lock operations ### ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) @@ -154,7 +177,7 @@ __Example__ ``` -## 4. Heal operations +## 5. Heal operations ### ListObjectsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan ObjectInfo, error) @@ -360,7 +383,7 @@ If upload is successfully healed returns nil, otherwise returns error indicating log.Println("Heal-upload result: ", healResult) ``` -## 5. Config operations +## 6. Config operations ### GetConfig() ([]byte, error) @@ -384,23 +407,6 @@ __Example__ log.Println("config received successfully: ", string(buf.Bytes())) ``` -## 6. Misc operations - - - -### SetCredentials() error -Set new credentials of a Minio setup. - -__Example__ - -``` go - err = madmClnt.SetCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY") - if err != nil { - log.Fatalln(err) - } - log.Println("New credentials successfully set.") - -``` ### SetConfig(config io.Reader) (SetConfigResult, error) @@ -435,3 +441,22 @@ __Example__ } log.Println("SetConfig: ", string(buf.Bytes())) ``` + +## 7. Misc operations + + + +### SetCredentials() error +Set new credentials of a Minio setup. + +__Example__ + +``` go + err = madmClnt.SetCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY") + if err != nil { + log.Fatalln(err) + } + log.Println("New credentials successfully set.") + +``` + diff --git a/pkg/madmin/info-commands.go b/pkg/madmin/info-commands.go index 1c41fdf80..01dcff234 100644 --- a/pkg/madmin/info-commands.go +++ b/pkg/madmin/info-commands.go @@ -74,17 +74,24 @@ type ServerConnStats struct { TotalOutputBytes uint64 `json:"received"` } -// ServerInfo holds the whole server information that will be -// returned by ServerInfo API. -type ServerInfo struct { +// ServerInfoData holds storage, connections and other +// information of a given server +type ServerInfoData struct { StorageInfo StorageInfo `json:"storage"` ConnStats ServerConnStats `json:"network"` Properties ServerProperties `json:"server"` } +// ServerInfo holds server information result of one node +type ServerInfo struct { + Error error `json:"error"` + Addr string `json:"addr"` + Data *ServerInfoData `json:"data"` +} + // ServerInfo - Connect to a minio server and call Server Info Management API // to fetch server's information represented by ServerInfo structure -func (adm *AdminClient) ServerInfo() (ServerInfo, error) { +func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) { // Prepare web service request reqData := requestData{} reqData.queryValues = make(url.Values) @@ -94,26 +101,26 @@ func (adm *AdminClient) ServerInfo() (ServerInfo, error) { resp, err := adm.executeMethod("GET", reqData) defer closeResponse(resp) if err != nil { - return ServerInfo{}, err + return nil, err } // Check response http status code if resp.StatusCode != http.StatusOK { - return ServerInfo{}, httpRespToErrorResponse(resp) + return nil, httpRespToErrorResponse(resp) } // Unmarshal the server's json response - var info ServerInfo + var serversInfo []ServerInfo respBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return ServerInfo{}, err + return nil, err } - err = json.Unmarshal(respBytes, &info) + err = json.Unmarshal(respBytes, &serversInfo) if err != nil { - return ServerInfo{}, err + return nil, err } - return info, nil + return serversInfo, nil }