admin: ServerInfo() returns info for each node (#4150)

ServerInfo() will gather information from all nodes before returning
it back to the client.

cmd/admin-handlers.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"path"
 	"strconv"
+	"sync"
 	"time"
 )
 
@@ -229,14 +230,22 @@ type ServerHTTPStats struct {
 	SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
 }
 
-// ServerInfo holds the information that will be returned by ServerInfo API
+// ServerInfoData holds storage, connections and other
-type ServerInfo struct {
+// information of a given server.
+type ServerInfoData struct {
 	StorageInfo StorageInfo      `json:"storage"`
 	ConnStats   ServerConnStats  `json:"network"`
 	HTTPStats   ServerHTTPStats  `json:"http"`
 	Properties  ServerProperties `json:"server"`
 }
 
+// ServerInfo holds server information result of one node
+type ServerInfo struct {
+	Error error
+	Addr  string
+	Data  *ServerInfoData
+}
+
 // ServerInfoHandler - GET /?info
 // ----------
 // Get server information
@@ -248,55 +257,37 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Build storage info
+	// Web service response
-	objLayer := newObjectLayerFn()
+	reply := make([]ServerInfo, len(globalAdminPeers))
-	if objLayer == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-	storage := objLayer.StorageInfo()
 
-	// Build list of enabled ARNs queues
+	var wg sync.WaitGroup
-	var arns []string
-	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
-		arns = append(arns, queueArn)
-	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
+	// Gather server information for all nodes
-	// of read-quorum availability.
+	for i, p := range globalAdminPeers {
-	uptime, err := getPeerUptimes(globalAdminPeers)
+		wg.Add(1)
+
+		// Gather information from a peer in a goroutine
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+
+			// Initialize server info at index
+			reply[idx] = ServerInfo{Addr: peer.addr}
+
+			serverInfoData, err := peer.cmdRunner.ServerInfoData()
 			if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				errorIf(err, "Unable to get server info from %s.", peer.addr)
-		errorIf(err, "Unable to get uptime from majority of servers.")
+				reply[idx].Error = err
 				return
 			}
 
-	// Build server properties information
+			reply[idx].Data = &serverInfoData
-	properties := ServerProperties{
+		}(i, p)
-		Version:  Version,
-		CommitID: CommitID,
-		Region:   serverConfig.GetRegion(),
-		SQSARN:   arns,
-		Uptime:   uptime,
 	}
 
-	// Build network info
+	wg.Wait()
-	connStats := ServerConnStats{
-		TotalInputBytes:  globalConnStats.getTotalInputBytes(),
-		TotalOutputBytes: globalConnStats.getTotalOutputBytes(),
-	}
-	httpStats := globalHTTPStats.toServerHTTPStats()
-
-	// Build the whole returned information
-	info := ServerInfo{
-		StorageInfo: storage,
-		ConnStats:   connStats,
-		HTTPStats:   httpStats,
-		Properties:  properties,
-	}
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(info)
+	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		errorIf(err, "Failed to marshal storage info into json.")

cmd/admin-handlers_test.go

@@ -1300,17 +1300,29 @@ func TestAdminServerInfo(t *testing.T) {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
 
-	result := ServerInfo{}
+	results := []ServerInfo{}
-	err = json.NewDecoder(rec.Body).Decode(&result)
+	err = json.NewDecoder(rec.Body).Decode(&results)
 	if err != nil {
 		t.Fatalf("Failed to decode set config result json %v", err)
 	}
 
-	if result.StorageInfo.Free == 0 {
+	if len(results) == 0 {
+		t.Error("Expected at least one server info result")
+	}
+
+	for _, serverInfo := range results {
+		if len(serverInfo.Addr) == 0 {
+			t.Error("Expected server address to be non empty")
+		}
+		if serverInfo.Error != nil {
+			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
+		}
+		if serverInfo.Data.StorageInfo.Free == 0 {
 			t.Error("Expected StorageInfo.Free to be non empty")
 		}
-	if result.Properties.Region != globalMinioDefaultRegion {
+		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
-		t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, result.Properties.Region)
+			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
+		}
 	}
 }
 

cmd/admin-rpc-client.go

@@ -37,7 +37,7 @@ const (
 	serviceRestartRPC = "Admin.Restart"
 	listLocksRPC      = "Admin.ListLocks"
 	reInitDisksRPC    = "Admin.ReInitDisks"
-	uptimeRPC         = "Admin.Uptime"
+	serverInfoDataRPC = "Admin.ServerInfoData"
 	getConfigRPC      = "Admin.GetConfig"
 	writeTmpConfigRPC = "Admin.WriteTmpConfig"
 	commitConfigRPC   = "Admin.CommitConfig"
@@ -59,7 +59,7 @@ type adminCmdRunner interface {
 	Restart() error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ReInitDisks() error
-	Uptime() (time.Duration, error)
+	ServerInfoData() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
 	CommitConfig(tmpFileName string) error
@@ -112,26 +112,48 @@ func (rc remoteAdminClient) ReInitDisks() error {
 	return rc.Call(reInitDisksRPC, &args, &reply)
 }
 
-// Uptime - Returns the uptime of this server. Timestamp is taken
+// ServerInfoData - Returns the server info of this server.
-// after object layer is initialized.
+func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) {
-func (lc localAdminClient) Uptime() (time.Duration, error) {
 	if globalBootTime.IsZero() {
-		return time.Duration(0), errServerNotInitialized
+		return ServerInfoData{}, errServerNotInitialized
 	}
 
-	return UTCNow().Sub(globalBootTime), nil
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return ServerInfoData{}, errServerNotInitialized
+	}
+	storage := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	return ServerInfoData{
+		StorageInfo: storage,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   arns,
+			Region:   serverConfig.GetRegion(),
+		},
+	}, nil
 }
 
-// Uptime - returns the uptime of the server to which the RPC call is made.
+// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) Uptime() (time.Duration, error) {
+func (rc remoteAdminClient) ServerInfoData() (ServerInfoData, error) {
 	args := AuthRPCArgs{}
-	reply := UptimeReply{}
+	reply := ServerInfoDataReply{}
-	err := rc.Call(uptimeRPC, &args, &reply)
+	err := rc.Call(serverInfoDataRPC, &args, &reply)
 	if err != nil {
-		return time.Duration(0), err
+		return ServerInfoData{}, err
 	}
 
-	return reply.Uptime, nil
+	return reply.ServerInfoData, nil
 }
 
 // GetConfig - returns config.json of the local server.
@@ -384,7 +406,8 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 		wg.Add(1)
 		go func(idx int, peer adminPeer) {
 			defer wg.Done()
-			uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
 		}(i, peer)
 	}
 	wg.Wait()

cmd/admin-rpc-server.go

@@ -53,10 +53,10 @@ type ListLocksReply struct {
 	volLocks []VolumeLockInfo
 }
 
-// UptimeReply - wraps the uptime response over RPC.
+// ServerInfoDataReply - wraps the server info response over RPC.
-type UptimeReply struct {
+type ServerInfoDataReply struct {
 	AuthRPCReply
-	Uptime time.Duration
+	ServerInfoData ServerInfoData
 }
 
 // ConfigReply - wraps the server config response over RPC.
@@ -122,8 +122,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	return nil
 }
 
-// Uptime - returns the time when object layer was initialized on this server.
+// ServerInfo - returns the server info when object layer was initialized on this server.
-func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
+func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
@@ -132,12 +132,29 @@ func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
 		return errServerNotInitialized
 	}
 
-	// N B The uptime is computed assuming that the system time is
+	// Build storage info
-	// monotonic. This is not the case in time pkg in Go, see
+	objLayer := newObjectLayerFn()
-	// https://github.com/golang/go/issues/12914. This is expected
+	if objLayer == nil {
-	// to be fixed by go1.9.
+		return errServerNotInitialized
-	*reply = UptimeReply{
+	}
+	storageInfo := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	reply.ServerInfoData = ServerInfoData{
+		Properties: ServerProperties{
 			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			Region:   serverConfig.GetRegion(),
+			SQSARN:   arns,
+		},
+		StorageInfo: storageInfo,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
 	}
 
 	return nil

cmd/http-stats.go

@@ -52,6 +52,14 @@ func (s *ConnStats) getTotalOutputBytes() uint64 {
 	return s.totalOutputBytes.Load()
 }
 
+// Return connection stats (total input/output bytes)
+func (s *ConnStats) toServerConnStats() ServerConnStats {
+	return ServerConnStats{
+		TotalInputBytes:  s.getTotalInputBytes(),
+		TotalOutputBytes: s.getTotalOutputBytes(),
+	}
+}
+
 // Prepare new ConnStats structure
 func newConnStats() *ConnStats {
 	return &ConnStats{}

cmd/server-mux.go

@@ -121,7 +121,9 @@ func (c *ConnMux) PeekProtocol() (string, error) {
 // bytes received from the client.
 func (c *ConnMux) Read(b []byte) (n int, err error) {
 	// Update total incoming number of bytes.
-	defer globalConnStats.incInputBytes(n)
+	defer func() {
+		globalConnStats.incInputBytes(n)
+	}()
 
 	n, err = c.peeker.Read(b)
 	if err != nil {
@@ -141,7 +143,9 @@ func (c *ConnMux) Read(b []byte) (n int, err error) {
 // keeps track of the total bytes written by the server.
 func (c *ConnMux) Write(b []byte) (n int, err error) {
 	// Update total outgoing number of bytes.
-	defer globalConnStats.incOutputBytes(n)
+	defer func() {
+		globalConnStats.incOutputBytes(n)
+	}()
 
 	// Call the conn write wrapper.
 	return c.Conn.Write(b)

pkg/madmin/API.md

@@ -122,7 +122,30 @@ If successful restarts the running minio service, for distributed setup restarts
 
  ```
 
-## 3. Lock operations
+## 3. Info operations
+
+<a name="ServerInfo"></a>
+### ServerInfo() ([]ServerInfo, error)
+Fetch all information for all cluster nodes, such as uptime, region, network statistics, etc..
+
+
+ __Example__
+
+ ```go
+
+	serversInfo, err := madmClnt.ServerInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	for _, peerInfo := range serversInfo {
+		log.Printf("Node: %s, Info: %v\n", peerInfo.Addr, peerInfo.Data)
+	}
+
+ ```
+
+
+## 4. Lock operations
 
 <a name="ListLocks"></a>
 ### ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
@@ -154,7 +177,7 @@ __Example__
 
 ```
 
-## 4. Heal operations
+## 5. Heal operations
 
 <a name="ListObjectsHeal"></a>
 ### ListObjectsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan ObjectInfo, error)
@@ -360,7 +383,7 @@ If upload is successfully healed returns nil, otherwise returns error indicating
     log.Println("Heal-upload result: ", healResult)
 ```
 
-## 5. Config operations
+## 6. Config operations
 
 <a name="GetConfig"></a>
 ### GetConfig() ([]byte, error)
@@ -384,23 +407,6 @@ __Example__
     log.Println("config received successfully: ", string(buf.Bytes()))
 ```
 
-## 6. Misc operations
-
-<a name="SetCredentials"></a>
-
-### SetCredentials() error
-Set new credentials of a Minio setup.
-
-__Example__
-
-``` go
-    err = madmClnt.SetCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY")
-    if err != nil {
-            log.Fatalln(err)
-    }
-    log.Println("New credentials successfully set.")
-
-```
 
 <a name="SetConfig"></a>
 ### SetConfig(config io.Reader) (SetConfigResult, error)
@@ -435,3 +441,22 @@ __Example__
     }
     log.Println("SetConfig: ", string(buf.Bytes()))
 ```
+
+## 7. Misc operations
+
+<a name="SetCredentials"></a>
+
+### SetCredentials() error
+Set new credentials of a Minio setup.
+
+__Example__
+
+``` go
+    err = madmClnt.SetCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY")
+    if err != nil {
+            log.Fatalln(err)
+    }
+    log.Println("New credentials successfully set.")
+
+```
+

pkg/madmin/info-commands.go

@@ -74,17 +74,24 @@ type ServerConnStats struct {
 	TotalOutputBytes uint64 `json:"received"`
 }
 
-// ServerInfo holds the whole server information that will be
+// ServerInfoData holds storage, connections and other
-// returned by ServerInfo API.
+// information of a given server
-type ServerInfo struct {
+type ServerInfoData struct {
 	StorageInfo StorageInfo      `json:"storage"`
 	ConnStats   ServerConnStats  `json:"network"`
 	Properties  ServerProperties `json:"server"`
 }
 
+// ServerInfo holds server information result of one node
+type ServerInfo struct {
+	Error error           `json:"error"`
+	Addr  string          `json:"addr"`
+	Data  *ServerInfoData `json:"data"`
+}
+
 // ServerInfo - Connect to a minio server and call Server Info Management API
 // to fetch server's information represented by ServerInfo structure
-func (adm *AdminClient) ServerInfo() (ServerInfo, error) {
+func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) {
 	// Prepare web service request
 	reqData := requestData{}
 	reqData.queryValues = make(url.Values)
@@ -94,26 +101,26 @@ func (adm *AdminClient) ServerInfo() (ServerInfo, error) {
 	resp, err := adm.executeMethod("GET", reqData)
 	defer closeResponse(resp)
 	if err != nil {
-		return ServerInfo{}, err
+		return nil, err
 	}
 
 	// Check response http status code
 	if resp.StatusCode != http.StatusOK {
-		return ServerInfo{}, httpRespToErrorResponse(resp)
+		return nil, httpRespToErrorResponse(resp)
 	}
 
 	// Unmarshal the server's json response
-	var info ServerInfo
+	var serversInfo []ServerInfo
 
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return ServerInfo{}, err
+		return nil, err
 	}
 
-	err = json.Unmarshal(respBytes, &info)
+	err = json.Unmarshal(respBytes, &serversInfo)
 	if err != nil {
-		return ServerInfo{}, err
+		return nil, err
 	}
 
-	return info, nil
+	return serversInfo, nil
 }