From 2ecaab55a6543a5d9c00e1cc33f1c61053ad28d8 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Mon, 21 Dec 2020 18:35:19 +0100 Subject: [PATCH] admin: ServerInfo returns info without object layer initialized (#11142) --- cmd/admin-handlers.go | 97 ++++++++++++++++------------------ cmd/admin-server-info.go | 8 +++ cmd/endpoint.go | 8 +++ cmd/erasure-server-sets.go | 31 ++++++----- cmd/erasure-sets.go | 4 -- cmd/erasure.go | 89 ++++++++++++++++--------------- cmd/fs-v1.go | 5 ++ cmd/gateway-unsupported.go | 5 ++ cmd/metrics.go | 3 +- cmd/object-api-datatypes.go | 35 ++++++------ cmd/object-api-interface.go | 2 + cmd/server-startup-msg.go | 5 +- cmd/server-startup-msg_test.go | 12 ++++- pkg/madmin/info-commands.go | 12 ++++- 14 files changed, 179 insertions(+), 137 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index da9e235dd..d452b50ec 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1497,22 +1497,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r)) - objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction) - if objectAPI == nil { + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) return } - buckets := madmin.Buckets{} - objects := madmin.Objects{} - usage := madmin.Usage{} - - dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) - if err == nil { - buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} - objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} - usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} - } - vault := fetchVaultStatus() ldap := madmin.LDAP{} @@ -1534,31 +1525,54 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque // Get the notification target info notifyTarget := fetchLambdaInfo() - // Fetching the Storage information, ignore any errors. - storageInfo, _ := objectAPI.StorageInfo(ctx, false) + server := getLocalServerProperty(globalEndpoints, r) + servers := globalNotificationSys.ServerInfo() + servers = append(servers, server) var backend interface{} - if storageInfo.Backend.Type == BackendType(madmin.Erasure) { - backend = madmin.ErasureBackend{ - Type: madmin.ErasureType, - OnlineDisks: storageInfo.Backend.OnlineDisks.Sum(), - OfflineDisks: storageInfo.Backend.OfflineDisks.Sum(), - StandardSCData: storageInfo.Backend.StandardSCData, - StandardSCParity: storageInfo.Backend.StandardSCParity, - RRSCData: storageInfo.Backend.RRSCData, - RRSCParity: storageInfo.Backend.RRSCParity, + mode := madmin.ObjectLayerInitializing + + buckets := madmin.Buckets{} + objects := madmin.Objects{} + usage := madmin.Usage{} + + objectAPI := newObjectLayerFn() + if objectAPI != nil { + mode = madmin.ObjectLayerOnline + // Load data usage + dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) + if err == nil { + buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} + objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} + usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} } - } else { - backend = madmin.FSBackend{ - Type: madmin.FsType, + + // Fetching the backend information + backendInfo := objectAPI.BackendInfo() + if backendInfo.Type == BackendType(madmin.Erasure) { + // Calculate the number of online/offline disks of all nodes + var allDisks []madmin.Disk + for _, s := range servers { + allDisks = append(allDisks, s.Disks...) + } + onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks) + + backend = madmin.ErasureBackend{ + Type: madmin.ErasureType, + OnlineDisks: onlineDisks.Sum(), + OfflineDisks: offlineDisks.Sum(), + StandardSCData: backendInfo.StandardSCData, + StandardSCParity: backendInfo.StandardSCParity, + RRSCData: backendInfo.RRSCData, + RRSCParity: backendInfo.RRSCParity, + } + } else { + backend = madmin.FSBackend{ + Type: madmin.FsType, + } } } - mode := "online" - server := getLocalServerProperty(globalEndpoints, r) - servers := globalNotificationSys.ServerInfo() - servers = append(servers, server) - domain := globalDomainNames services := madmin.Services{ Vault: vault, @@ -1568,25 +1582,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque Notifications: notifyTarget, } - // find all disks which belong to each respective endpoints - for i := range servers { - for _, disk := range storageInfo.Disks { - if strings.Contains(disk.Endpoint, servers[i].Endpoint) { - servers[i].Disks = append(servers[i].Disks, disk) - } - } - } - - // add all the disks local to this server. - for _, disk := range storageInfo.Disks { - if disk.DrivePath == "" && disk.Endpoint == "" { - continue - } - if disk.Endpoint == disk.DrivePath { - servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk) - } - } - infoMsg := madmin.InfoMessage{ Mode: mode, Domain: domain, diff --git a/cmd/admin-server-info.go b/cmd/admin-server-info.go index d8e67271c..aef2b178b 100644 --- a/cmd/admin-server-info.go +++ b/cmd/admin-server-info.go @@ -25,6 +25,7 @@ import ( // getLocalServerProperty - returns madmin.ServerProperties for only the // local endpoints from given list of endpoints func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties { + var localEndpoints Endpoints addr := r.Host if globalIsDistErasure { addr = GetLocalPeer(endpointServerPools) @@ -39,6 +40,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req if endpoint.IsLocal { // Only proceed for local endpoints network[nodeName] = "online" + localEndpoints = append(localEndpoints, endpoint) continue } _, present := network[nodeName] @@ -52,6 +54,11 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req } } + localDisks, _ := initStorageDisksWithErrors(localEndpoints) + defer closeStorageDisks(localDisks) + + storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings()) + return madmin.ServerProperties{ State: "ok", Endpoint: addr, @@ -59,5 +66,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req Version: Version, CommitID: CommitID, Network: network, + Disks: storageInfo.Disks, } } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index bfc96d944..62ea1d0eb 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -353,6 +353,14 @@ func (endpoints Endpoints) GetString(i int) string { return endpoints[i].String() } +// GetAllStrings - returns allstring of all endpoints +func (endpoints Endpoints) GetAllStrings() (all []string) { + for _, e := range endpoints { + all = append(all, e.String()) + } + return +} + func hostResolveToLocalhost(endpoint Endpoint) bool { hostIPs, err := getHostIP(endpoint.Hostname()) if err != nil { diff --git a/cmd/erasure-server-sets.go b/cmd/erasure-server-sets.go index 6c9181937..1fe124e0d 100644 --- a/cmd/erasure-server-sets.go +++ b/cmd/erasure-server-sets.go @@ -273,9 +273,24 @@ func (z *erasureServerPools) Shutdown(ctx context.Context) error { return nil } +func (z *erasureServerPools) BackendInfo() (b BackendInfo) { + b.Type = BackendErasure + + scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD) + if scParity == 0 { + scParity = z.SetDriveCount() / 2 + } + b.StandardSCData = z.SetDriveCount() - scParity + b.StandardSCParity = scParity + + rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS) + b.RRSCData = z.SetDriveCount() - rrSCParity + b.RRSCParity = rrSCParity + return +} + func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) { var storageInfo StorageInfo - storageInfo.Backend.Type = BackendErasure storageInfos := make([]StorageInfo, len(z.serverPools)) storageInfosErrs := make([][]error, len(z.serverPools)) @@ -291,23 +306,11 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (Stora // Wait for the go routines. g.Wait() + storageInfo.Backend = z.BackendInfo() for _, lstorageInfo := range storageInfos { storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) - storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks) - storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks) - } - - scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD) - if scParity == 0 { - scParity = z.SetDriveCount() / 2 } - storageInfo.Backend.StandardSCData = z.SetDriveCount() - scParity - storageInfo.Backend.StandardSCParity = scParity - rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS) - storageInfo.Backend.RRSCData = z.SetDriveCount() - rrSCParity - storageInfo.Backend.RRSCParity = rrSCParity - var errs []error for i := range z.serverPools { errs = append(errs, storageInfosErrs[i]...) diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 7c4ebff00..472362cab 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -491,8 +491,6 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo { for _, lstorageInfo := range storageInfos { storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) - storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks) - storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks) } return storageInfo @@ -530,8 +528,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo, for _, lstorageInfo := range storageInfos { storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) - storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks) - storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks) } if local { diff --git a/cmd/erasure.go b/cmd/erasure.go index 2fc168044..e2ba9bbd8 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -109,13 +109,12 @@ func diskErrToDriveState(err error) (state string) { return } -// getDisksInfo - fetch disks info across all other storage API. -func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error, onlineDisks, offlineDisks madmin.BackendDisks) { - disksInfo = make([]madmin.Disk, len(disks)) +func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDisks madmin.BackendDisks) { onlineDisks = make(madmin.BackendDisks) offlineDisks = make(madmin.BackendDisks) - for _, ep := range endpoints { + for _, disk := range disksInfo { + ep := disk.Endpoint if _, ok := offlineDisks[ep]; !ok { offlineDisks[ep] = 0 } @@ -124,6 +123,47 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di } } + // Wait for the routines. + for _, disk := range disksInfo { + ep := disk.Endpoint + state := disk.State + if state != madmin.DriveStateOk && state != madmin.DriveStateUnformatted { + offlineDisks[ep]++ + continue + } + onlineDisks[ep]++ + } + + rootDiskCount := 0 + for _, di := range disksInfo { + if di.RootDisk { + rootDiskCount++ + } + } + + // Count offline disks as well to ensure consistent + // reportability of offline drives on local setups. + if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) { + // Success. + return onlineDisks, offlineDisks + } + + // Root disk should be considered offline + for i := range disksInfo { + ep := disksInfo[i].Endpoint + if disksInfo[i].RootDisk { + offlineDisks[ep]++ + onlineDisks[ep]-- + } + } + + return onlineDisks, offlineDisks +} + +// getDisksInfo - fetch disks info across all other storage API. +func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) { + disksInfo = make([]madmin.Disk, len(disks)) + g := errgroup.WithNErrs(len(disks)) for index := range disks { index := index @@ -157,46 +197,12 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di }, index) } - errs = g.Wait() - // Wait for the routines. - for i, diskInfoErr := range errs { - ep := disksInfo[i].Endpoint - if diskInfoErr != nil && !errors.Is(diskInfoErr, errUnformattedDisk) { - offlineDisks[ep]++ - continue - } - onlineDisks[ep]++ - } - - rootDiskCount := 0 - for _, di := range disksInfo { - if di.RootDisk { - rootDiskCount++ - } - } - - // Count offline disks as well to ensure consistent - // reportability of offline drives on local setups. - if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) { - // Success. - return disksInfo, errs, onlineDisks, offlineDisks - } - - // Root disk should be considered offline - for i := range disksInfo { - ep := disksInfo[i].Endpoint - if disksInfo[i].RootDisk { - offlineDisks[ep]++ - onlineDisks[ep]-- - } - } - - return disksInfo, errs, onlineDisks, offlineDisks + return disksInfo, g.Wait() } // Get an aggregated storage info across all disks. func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) { - disksInfo, errs, onlineDisks, offlineDisks := getDisksInfo(disks, endpoints) + disksInfo, errs := getDisksInfo(disks, endpoints) // Sort so that the first element is the smallest. sort.Sort(byDiskTotal(disksInfo)) @@ -206,9 +212,6 @@ func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []erro } storageInfo.Backend.Type = BackendErasure - storageInfo.Backend.OnlineDisks = onlineDisks - storageInfo.Backend.OfflineDisks = offlineDisks - return storageInfo, errs } diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 39c3c5a47..28a99a297 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -200,6 +200,11 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error { return fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)) } +// BackendInfo - returns backend information +func (fs *FSObjects) BackendInfo() BackendInfo { + return BackendInfo{Type: BackendFS} +} + // StorageInfo - returns underlying storage statistics. func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []error) { diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index 801e4450b..af755caaf 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -34,6 +34,11 @@ import ( // GatewayUnsupported list of unsupported call stubs for gateway. type GatewayUnsupported struct{} +// BackendInfo returns the underlying backend information +func (a GatewayUnsupported) BackendInfo() BackendInfo { + return BackendInfo{Type: BackendGateway} +} + // CrawlAndGetDataUsage - crawl is not implemented for gateway func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error { logger.CriticalIf(ctx, errors.New("not implemented")) diff --git a/cmd/metrics.go b/cmd/metrics.go index 21d82fef5..d680e6494 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -500,8 +500,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) { // Fetch disk space info, ignore errors storageInfo, _ := objLayer.StorageInfo(GlobalContext, true) - offlineDisks := storageInfo.Backend.OfflineDisks - onlineDisks := storageInfo.Backend.OnlineDisks + onlineDisks, offlineDisks := getOnlineOfflineDisksStats(storageInfo.Disks) totalDisks := offlineDisks.Merge(onlineDisks) // MinIO Offline Disks per node diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index c9d585118..b129c8bb7 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -42,26 +42,25 @@ const ( // Add your own backend. ) +// BackendInfo - contains info of the underlying backend +type BackendInfo struct { + // Represents various backend types, currently on FS, Erasure and Gateway + Type BackendType + + // Following fields are only meaningful if BackendType is Gateway. + GatewayOnline bool + + // Following fields are only meaningful if BackendType is Erasure. + StandardSCData int // Data disks for currently configured Standard storage class. + StandardSCParity int // Parity disks for currently configured Standard storage class. + RRSCData int // Data disks for currently configured Reduced Redundancy storage class. + RRSCParity int // Parity disks for currently configured Reduced Redundancy storage class. +} + // StorageInfo - represents total capacity of underlying storage. type StorageInfo struct { - Disks []madmin.Disk - - // Backend type. - Backend struct { - // Represents various backend types, currently on FS, Erasure and Gateway - Type BackendType - - // Following fields are only meaningful if BackendType is Gateway. - GatewayOnline bool - - // Following fields are only meaningful if BackendType is Erasure. - OnlineDisks madmin.BackendDisks // Online disks during server startup. - OfflineDisks madmin.BackendDisks // Offline disks during server startup. - StandardSCData int // Data disks for currently configured Standard storage class. - StandardSCParity int // Parity disks for currently configured Standard storage class. - RRSCData int // Data disks for currently configured Reduced Redundancy storage class. - RRSCParity int // Parity disks for currently configured Reduced Redundancy storage class. - } + Disks []madmin.Disk + Backend BackendInfo } // objectHistogramInterval is an interval that will be diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 9a3c04370..6ad574f9e 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -79,6 +79,8 @@ type ObjectLayer interface { // Storage operations. Shutdown(context.Context) error CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error + + BackendInfo() BackendInfo StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) // local queries only local disks // Bucket operations. diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 3f0adb9de..d08ae0899 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -205,12 +205,13 @@ func printObjectAPIMsg() { func getStorageInfoMsg(storageInfo StorageInfo) string { var msg string var mcMessage string + onlineDisks, offlineDisks := getOnlineOfflineDisksStats(storageInfo.Disks) if storageInfo.Backend.Type == BackendErasure { - if storageInfo.Backend.OfflineDisks.Sum() > 0 { + if offlineDisks.Sum() > 0 { mcMessage = "Use `mc admin info` to look for latest server/disk info\n" } - diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum()) + diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", onlineDisks.Sum(), offlineDisks.Sum()) msg += color.Blue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo) if len(mcMessage) > 0 { msg = fmt.Sprintf("%s %s", mcMessage, msg) diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go index e085b39cc..a9e7b2516 100644 --- a/cmd/server-startup-msg_test.go +++ b/cmd/server-startup-msg_test.go @@ -28,9 +28,17 @@ import ( // Tests if we generate storage info. func TestStorageInfoMsg(t *testing.T) { infoStorage := StorageInfo{} + infoStorage.Disks = []madmin.Disk{ + {Endpoint: "http://127.0.0.1:9000/data/1/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9000/data/2/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9000/data/3/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9000/data/4/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9001/data/1/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9001/data/2/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9001/data/3/", State: madmin.DriveStateOk}, + {Endpoint: "http://127.0.0.1:9001/data/4/", State: madmin.DriveStateOffline}, + } infoStorage.Backend.Type = BackendErasure - infoStorage.Backend.OnlineDisks = madmin.BackendDisks{"127.0.0.1:9000": 4, "127.0.0.1:9001": 3} - infoStorage.Backend.OfflineDisks = madmin.BackendDisks{"127.0.0.1:9000": 0, "127.0.0.1:9001": 1} if msg := getStorageInfoMsg(infoStorage); !strings.Contains(msg, "7 Online, 1 Offline") { t.Fatal("Unexpected storage info message, found:", msg) diff --git a/pkg/madmin/info-commands.go b/pkg/madmin/info-commands.go index 66087c560..5bcb13401 100644 --- a/pkg/madmin/info-commands.go +++ b/pkg/madmin/info-commands.go @@ -39,6 +39,16 @@ const ( // Add your own backend. ) +// ObjectLayerState - represents the status of the object layer +type ObjectLayerState string + +const ( + // ObjectLayerInitializing indicates that the object layer is still in initialization phase + ObjectLayerInitializing = ObjectLayerState("initializing") + // ObjectLayerOnline indicates that the object layer is ready + ObjectLayerOnline = ObjectLayerState("online") +) + // StorageInfo - represents total capacity of underlying storage. type StorageInfo struct { Disks []Disk @@ -163,7 +173,7 @@ func (adm *AdminClient) DataUsageInfo(ctx context.Context) (DataUsageInfo, error // InfoMessage container to hold server admin related information. type InfoMessage struct { - Mode string `json:"mode,omitempty"` + Mode ObjectLayerState `json:"mode,omitempty"` Domain []string `json:"domain,omitempty"` Region string `json:"region,omitempty"` SQSARN []string `json:"sqsARN,omitempty"`