From b2db8123ec75df1a08b7d1f42fdae4fc2170944d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 28 May 2020 13:03:04 -0700 Subject: [PATCH] Preserve errors returned by diskInfo to detect disk errors (#9727) This PR basically reverts #9720 and re-implements it differently --- cmd/admin-handlers.go | 11 +-- cmd/background-heal-ops.go | 5 +- cmd/fs-v1.go | 6 +- cmd/gateway/azure/gateway-azure.go | 4 +- cmd/gateway/gcs/gateway-gcs.go | 4 +- cmd/gateway/hdfs/gateway-hdfs.go | 18 ++--- cmd/gateway/nas/gateway-nas.go | 14 ++-- cmd/gateway/s3/gateway-s3.go | 4 +- cmd/metrics.go | 4 +- cmd/object-api-interface.go | 2 +- cmd/posix.go | 1 + cmd/server-startup-msg.go | 9 ++- cmd/storage-interface.go | 2 +- cmd/storage-rest-client.go | 18 ++--- cmd/storage-rest-common.go | 1 - cmd/storage-rest-server.go | 17 ---- cmd/web-handlers.go | 2 +- cmd/xl-sets.go | 125 ++++++++++++++++++----------- cmd/xl-v1.go | 26 +++--- cmd/xl-zones.go | 13 ++- 20 files changed, 152 insertions(+), 134 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 0d1d78a1b..0fba12d09 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -284,7 +284,8 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ return } - storageInfo := objectAPI.StorageInfo(ctx, false) + // ignores any errors here. + storageInfo, _ := objectAPI.StorageInfo(ctx, false) // Marshal API response jsonBytes, err := json.Marshal(storageInfo) @@ -707,8 +708,8 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) { } } - // find number of disks in the setup - info := objectAPI.StorageInfo(ctx, false) + // find number of disks in the setup, ignore any errors here. + info, _ := objectAPI.StorageInfo(ctx, false) numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum() healPath := pathJoin(hip.bucket, hip.objPrefix) @@ -1354,8 +1355,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque // Get the notification target info notifyTarget := fetchLambdaInfo(cfg) - // Fetching the Storage information - storageInfo := objectAPI.StorageInfo(ctx, false) + // Fetching the Storage information, ignore any errors. + storageInfo, _ := objectAPI.StorageInfo(ctx, false) var OnDisks int var OffDisks int diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go index eaea99e83..20271977b 100644 --- a/cmd/background-heal-ops.go +++ b/cmd/background-heal-ops.go @@ -114,8 +114,9 @@ func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) { go globalBackgroundHealRoutine.run(ctx, objAPI) // Launch the background healer sequence to track - // background healing operations - info := objAPI.StorageInfo(ctx, false) + // background healing operations, ignore errors + // errors are handled into offline disks already. + info, _ := objAPI.StorageInfo(ctx, false) numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum() nh := newBgHealSequence(numDisks) globalBackgroundHealState.LaunchNewHealSequence(nh) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 8dd8b3165..6cb95d4be 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -203,7 +203,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error { } // StorageInfo - returns underlying storage statistics. -func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo { +func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []error) { atomic.AddInt64(&fs.activeIOCount, 1) defer func() { @@ -212,7 +212,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo { di, err := getDiskInfo(fs.fsPath) if err != nil { - return StorageInfo{} + return StorageInfo{}, []error{err} } used := di.Total - di.Free if !fs.diskMount { @@ -226,7 +226,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo { MountPaths: []string{fs.fsPath}, } storageInfo.Backend.Type = BackendFS - return storageInfo + return storageInfo, nil } func (fs *FSObjects) waitForLowActiveIO() { diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index 819126435..828b7cdf4 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -542,10 +542,10 @@ func (a *azureObjects) Shutdown(ctx context.Context) error { } // StorageInfo - Not relevant to Azure backend. -func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) { +func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) { si.Backend.Type = minio.BackendGateway si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint) - return si + return si, nil } // MakeBucketWithLocation - Create a new container on azure backend. diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 66d51117a..00df5c1df 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -414,10 +414,10 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error { } // StorageInfo - Not relevant to GCS backend. -func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) { +func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) { si.Backend.Type = minio.BackendGateway si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com") - return si + return si, nil } // MakeBucketWithLocation - Create a new container on GCS backend. diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index 65e76b8e7..dec961545 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -205,16 +205,15 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error { return n.clnt.Close() } -func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo { +func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, errs []error) { fsInfo, err := n.clnt.StatFs() if err != nil { - return minio.StorageInfo{} + return minio.StorageInfo{}, []error{err} } - sinfo := minio.StorageInfo{} - sinfo.Used = []uint64{fsInfo.Used} - sinfo.Backend.Type = minio.BackendGateway - sinfo.Backend.GatewayOnline = true - return sinfo + si.Used = []uint64{fsInfo.Used} + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = true + return si, nil } // hdfsObjects implements gateway for Minio and S3 compatible object storage servers. @@ -758,6 +757,7 @@ func (n *hdfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, } // IsReady returns whether the layer is ready to take requests. -func (n *hdfsObjects) IsReady(_ context.Context) bool { - return true +func (n *hdfsObjects) IsReady(ctx context.Context) bool { + si, _ := n.StorageInfo(ctx, false) + return si.Backend.GatewayOnline } diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go index d2f9b6a1d..6cfcbf20d 100644 --- a/cmd/gateway/nas/gateway-nas.go +++ b/cmd/gateway/nas/gateway-nas.go @@ -110,11 +110,11 @@ func (n *nasObjects) IsListenBucketSupported() bool { return false } -func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo { - sinfo := n.ObjectLayer.StorageInfo(ctx, false) - sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS - sinfo.Backend.Type = minio.BackendGateway - return sinfo +func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) { + si, errs := n.ObjectLayer.StorageInfo(ctx, false) + si.Backend.GatewayOnline = si.Backend.Type == minio.BackendFS + si.Backend.Type = minio.BackendGateway + return si, errs } // nasObjects implements gateway for MinIO and S3 compatible object storage servers. @@ -134,8 +134,8 @@ func (n *nasObjects) SetBucketObjectLockConfig(ctx context.Context, bucket strin // IsReady returns whether the layer is ready to take requests. func (n *nasObjects) IsReady(ctx context.Context) bool { - sinfo := n.ObjectLayer.StorageInfo(ctx, false) - return sinfo.Backend.Type == minio.BackendFS + si, _ := n.StorageInfo(ctx, false) + return si.Backend.GatewayOnline } func (n *nasObjects) IsTaggingSupported() bool { diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 29e75fd08..cf3edb8e9 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -280,10 +280,10 @@ func (l *s3Objects) Shutdown(ctx context.Context) error { } // StorageInfo is not relevant to S3 backend. -func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) { +func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) { si.Backend.Type = minio.BackendGateway si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String()) - return si + return si, nil } // MakeBucket creates a new container on S3 backend. diff --git a/cmd/metrics.go b/cmd/metrics.go index 8284cf953..b534cf8af 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -416,8 +416,8 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) { return } - // Fetch disk space info - storageInfo := objLayer.StorageInfo(GlobalContext, true) + // Fetch disk space info, ignore errors + storageInfo, _ := objLayer.StorageInfo(GlobalContext, true) offlineDisks := storageInfo.Backend.OfflineDisks onlineDisks := storageInfo.Backend.OnlineDisks diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 65ee01253..13254d6ad 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -59,7 +59,7 @@ type ObjectLayer interface { // Storage operations. Shutdown(context.Context) error CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error - StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks + StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) // local queries only local disks // Bucket operations. MakeBucketWithLocation(ctx context.Context, bucket string, location string, lockEnabled bool) error diff --git a/cmd/posix.go b/cmd/posix.go index 8c017b748..a16fcf0d1 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -413,6 +413,7 @@ type DiskInfo struct { Used uint64 RootDisk bool MountPath string + Error string // reports any error returned by underlying disk } // DiskInfo provides current information about disk space usage, diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 13454a255..d004dad45 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -47,6 +47,11 @@ func getFormatStr(strLen int, padding int) string { return "%" + formatStr } +func mustGetStorageInfo(objAPI ObjectLayer) StorageInfo { + storageInfo, _ := objAPI.StorageInfo(GlobalContext, false) + return storageInfo +} + func printStartupSafeModeMessage(apiEndpoints []string, err error) { logStartupMessage(color.RedBold("Server startup failed with '%v'", err)) logStartupMessage(color.RedBold("Server switching to safe mode")) @@ -55,7 +60,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) { // Object layer is initialized then print StorageInfo in safe mode. objAPI := newObjectLayerWithoutSafeModeFn() if objAPI != nil { - if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(GlobalContext, false)); msg != "" { + if msg := getStorageInfoMsgSafeMode(mustGetStorageInfo(objAPI)); msg != "" { logStartupMessage(msg) } } @@ -117,7 +122,7 @@ func printStartupMessage(apiEndpoints []string) { // Object layer is initialized then print StorageInfo. objAPI := newObjectLayerFn() if objAPI != nil { - printStorageInfo(objAPI.StorageInfo(GlobalContext, false)) + printStorageInfo(mustGetStorageInfo(objAPI)) } // Prints credential, region and browser access. diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index e3a8d4a94..7cfe37f8d 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -31,7 +31,7 @@ type StorageAPI interface { IsLocal() bool Hostname() string // Returns host name if remote host. Close() error - GetDiskID() (string, error) // Could be expensive + GetDiskID() (string, error) SetDiskID(id string) DiskInfo() (info DiskInfo, err error) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 5a81eb047..3e9b48093 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -177,19 +177,11 @@ func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache } func (client *storageRESTClient) GetDiskID() (string, error) { - respBody, err := client.call(storageRESTMethodGetDiskID, nil, nil, -1) - if err != nil { - // Ignore when other nodes does not support GetDiskID call, this check - // can be removed when the storage API version is bumped. - if strings.Contains(err.Error(), "404 page not found") { - return client.diskID, nil - } - return "", err - } - defer http.DrainBody(respBody) - var s string - err = gob.NewDecoder(respBody).Decode(&s) - return s, err + // This call should never be over the network, this is always + // a cached value - caller should make sure to use this + // function on a fresh disk or make sure to look at the error + // from a different networked call to validate the GetDiskID() + return client.diskID, nil } func (client *storageRESTClient) SetDiskID(id string) { diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go index f4ab9c144..e9d8ed5c7 100644 --- a/cmd/storage-rest-common.go +++ b/cmd/storage-rest-common.go @@ -24,7 +24,6 @@ const ( const ( storageRESTMethodDiskInfo = "/diskinfo" - storageRESTMethodGetDiskID = "/getdiskid" storageRESTMethodCrawlAndGetDataUsage = "/crawlandgetdatausage" storageRESTMethodMakeVol = "/makevol" storageRESTMethodMakeVolBulk = "/makevolbulk" diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index d170839a2..62abd6287 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -132,22 +132,6 @@ func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Reque gob.NewEncoder(w).Encode(info) } -// GetDiskIDHandler - returns disk id. -func (s *storageRESTServer) GetDiskIDHandler(w http.ResponseWriter, r *http.Request) { - if err := storageServerRequestValidate(r); err != nil { - s.writeErrorResponse(w, err) - return - } - - info, err := s.storage.GetDiskID() - if err != nil { - s.writeErrorResponse(w, err) - return - } - defer w.(http.Flusher).Flush() - gob.NewEncoder(w).Encode(info) -} - func (s *storageRESTServer) CrawlAndGetDataUsageHandler(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { return @@ -800,7 +784,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter() subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler)) - subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodGetDiskID).HandlerFunc(httpTraceHdrs(server.GetDiskIDHandler)) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCrawlAndGetDataUsage).HandlerFunc(httpTraceHdrs(server.CrawlAndGetDataUsageHandler)) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(httpTraceHdrs(server.MakeVolBulkHandler)).Queries(restQueries(storageRESTVolumes)...) diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 87192c915..b702f923e 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -129,7 +129,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re if authErr != nil { return toJSONError(ctx, authErr) } - reply.StorageInfo = objectAPI.StorageInfo(ctx, false) + reply.StorageInfo, _ = objectAPI.StorageInfo(ctx, false) reply.UIVersion = browser.UIVersion return nil } diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index b0cd84d8d..104982ace 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -37,20 +37,9 @@ import ( "github.com/minio/minio/pkg/sync/errgroup" ) -// setsStorageAPI is encapsulated type for Close() -type setsStorageAPI [][]StorageAPI - // setsDsyncLockers is encapsulated type for Close() type setsDsyncLockers [][]dsync.NetLocker -func (s setsStorageAPI) Copy() [][]StorageAPI { - copyS := make(setsStorageAPI, len(s)) - for i, disks := range s { - copyS[i] = append(copyS[i], disks...) - } - return copyS -} - // Information of a new disk connection type diskConnectInfo struct { setIndex int @@ -71,7 +60,7 @@ type xlSets struct { xlDisksMu sync.RWMutex // Re-ordered list of disks per set. - xlDisks setsStorageAPI + xlDisks [][]StorageAPI // Distributed locker clients. xlLockers setsDsyncLockers @@ -369,32 +358,65 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, objects ...string return s.getHashedSet("").NewNSLock(ctx, bucket, objects...) } -// StorageInfo - combines output of StorageInfo across all erasure coded object sets. -// Caches values for 1 second. -func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo { +// StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets. +// This only returns disk usage info for Zones to perform placement decision, this call +// is not implemented in Object interface and is not meant to be used by other object +// layer implementations. +func (s *xlSets) StorageUsageInfo(ctx context.Context) StorageInfo { + storageUsageInfo := func() StorageInfo { + var storageInfo StorageInfo + storageInfos := make([]StorageInfo, len(s.sets)) + storageInfo.Backend.Type = BackendErasure + + g := errgroup.WithNErrs(len(s.sets)) + for index := range s.sets { + index := index + g.Go(func() error { + // ignoring errors on purpose + storageInfos[index], _ = s.sets[index].StorageInfo(ctx, false) + return nil + }, index) + } + + // Wait for the go routines. + g.Wait() + + for _, lstorageInfo := range storageInfos { + storageInfo.Used = append(storageInfo.Used, lstorageInfo.Used...) + storageInfo.Total = append(storageInfo.Total, lstorageInfo.Total...) + storageInfo.Available = append(storageInfo.Available, lstorageInfo.Available...) + storageInfo.MountPaths = append(storageInfo.MountPaths, lstorageInfo.MountPaths...) + storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks) + storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks) + } + + return storageInfo + } + s.disksStorageInfoCache.Once.Do(func() { s.disksStorageInfoCache.TTL = time.Second s.disksStorageInfoCache.Update = func() (interface{}, error) { - return s.storageInfo(ctx, local), nil + return storageUsageInfo(), nil } }) + v, _ := s.disksStorageInfoCache.Get() return v.(StorageInfo) } -// storageInfo - combines output of StorageInfo across all erasure coded object sets. -// Use StorageInfo for a cached version. -func (s *xlSets) storageInfo(ctx context.Context, local bool) StorageInfo { +// StorageInfo - combines output of StorageInfo across all erasure coded object sets. +func (s *xlSets) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) { var storageInfo StorageInfo storageInfos := make([]StorageInfo, len(s.sets)) + storageInfoErrs := make([][]error, len(s.sets)) storageInfo.Backend.Type = BackendErasure g := errgroup.WithNErrs(len(s.sets)) for index := range s.sets { index := index g.Go(func() error { - storageInfos[index] = s.sets[index].StorageInfo(ctx, local) + storageInfos[index], storageInfoErrs[index] = s.sets[index].StorageInfo(ctx, local) return nil }, index) } @@ -428,49 +450,56 @@ func (s *xlSets) storageInfo(ctx context.Context, local bool) StorageInfo { } if local { - // if local is true, we don't need to read format.json - return storageInfo + // if local is true, we are not interested in the drive UUID info. + // this is called primarily by prometheus + return storageInfo, nil } - s.xlDisksMu.RLock() - storageDisks := s.xlDisks.Copy() - s.xlDisksMu.RUnlock() - - for i := 0; i < s.setCount; i++ { - for j := 0; j < s.drivesPerSet; j++ { - if storageDisks[i][j] == nil { + for i, set := range s.sets { + storageDisks := set.getDisks() + for j, storageErr := range storageInfoErrs[i] { + if storageDisks[j] == OfflineDisk { storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ State: madmin.DriveStateOffline, Endpoint: s.endpointStrings[i*s.drivesPerSet+j], } continue } - diskID, err := storageDisks[i][j].GetDiskID() - if err != nil { - if err == errUnformattedDisk { - storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ - State: madmin.DriveStateUnformatted, - Endpoint: storageDisks[i][j].String(), - UUID: "", - } - } else { - storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ - State: madmin.DriveStateCorrupt, - Endpoint: storageDisks[i][j].String(), - UUID: "", - } + var diskID string + if storageErr == nil { + // No errors returned by storage, look for its DiskID() + diskID, storageErr = storageDisks[j].GetDiskID() + } + if storageErr == nil { + storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ + State: madmin.DriveStateOk, + Endpoint: storageDisks[j].String(), + UUID: diskID, } continue } - storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ - State: madmin.DriveStateOk, - Endpoint: storageDisks[i][j].String(), - UUID: diskID, + if storageErr == errUnformattedDisk { + storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ + State: madmin.DriveStateUnformatted, + Endpoint: storageDisks[j].String(), + UUID: "", + } + } else { + storageInfo.Backend.Sets[i][j] = madmin.DriveInfo{ + State: madmin.DriveStateCorrupt, + Endpoint: storageDisks[j].String(), + UUID: "", + } } } } - return storageInfo + var errs []error + for i := range s.sets { + errs = append(errs, storageInfoErrs[i]...) + } + + return storageInfo, errs } func (s *xlSets) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error { diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index 343261676..f831c82d2 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -93,8 +93,9 @@ func (d byDiskTotal) Less(i, j int) bool { } // getDisksInfo - fetch disks info across all other storage API. -func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlineDisks madmin.BackendDisks) { +func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, errs []error, onlineDisks, offlineDisks madmin.BackendDisks) { disksInfo = make([]DiskInfo, len(disks)) + errs = make([]error, len(disks)) g := errgroup.WithNErrs(len(disks)) for index := range disks { @@ -106,12 +107,12 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin } info, err := disks[index].DiskInfo() if err != nil { - if IsErr(err, baseErrs...) { - return err + if !IsErr(err, baseErrs...) { + reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String()) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + logger.LogIf(ctx, err) } - reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String()) - ctx := logger.SetReqInfo(GlobalContext, reqInfo) - logger.LogIf(ctx, err) + return err } disksInfo[index] = info return nil @@ -121,8 +122,9 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin onlineDisks = make(madmin.BackendDisks) offlineDisks = make(madmin.BackendDisks) + errs = g.Wait() // Wait for the routines. - for i, diskInfoErr := range g.Wait() { + for i, diskInfoErr := range errs { if disks[i] == nil { continue } @@ -157,12 +159,12 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin } // Success. - return disksInfo, onlineDisks, offlineDisks + return disksInfo, errs, onlineDisks, offlineDisks } // Get an aggregated storage info across all disks. -func getStorageInfo(disks []StorageAPI) StorageInfo { - disksInfo, onlineDisks, offlineDisks := getDisksInfo(disks) +func getStorageInfo(disks []StorageAPI) (StorageInfo, []error) { + disksInfo, errs, onlineDisks, offlineDisks := getDisksInfo(disks) // Sort so that the first element is the smallest. sort.Sort(byDiskTotal(disksInfo)) @@ -191,11 +193,11 @@ func getStorageInfo(disks []StorageAPI) StorageInfo { storageInfo.Backend.OnlineDisks = onlineDisks storageInfo.Backend.OfflineDisks = offlineDisks - return storageInfo + return storageInfo, errs } // StorageInfo - returns underlying storage statistics. -func (xl xlObjects) StorageInfo(ctx context.Context, local bool) StorageInfo { +func (xl xlObjects) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) { disks := xl.getDisks() if local { diff --git a/cmd/xl-zones.go b/cmd/xl-zones.go index 3d8d7f2e0..2a822b24f 100644 --- a/cmd/xl-zones.go +++ b/cmd/xl-zones.go @@ -130,7 +130,7 @@ func (z *xlZones) getZonesAvailableSpace(ctx context.Context) zonesAvailableSpac for index := range z.zones { index := index g.Go(func() error { - storageInfos[index] = z.zones[index].StorageInfo(ctx, false) + storageInfos[index] = z.zones[index].StorageUsageInfo(ctx) return nil }, index) } @@ -175,7 +175,7 @@ func (z *xlZones) Shutdown(ctx context.Context) error { return nil } -func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo { +func (z *xlZones) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) { if z.SingleZone() { return z.zones[0].StorageInfo(ctx, local) } @@ -183,11 +183,12 @@ func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo { var storageInfo StorageInfo storageInfos := make([]StorageInfo, len(z.zones)) + storageInfosErrs := make([][]error, len(z.zones)) g := errgroup.WithNErrs(len(z.zones)) for index := range z.zones { index := index g.Go(func() error { - storageInfos[index] = z.zones[index].StorageInfo(ctx, local) + storageInfos[index], storageInfosErrs[index] = z.zones[index].StorageInfo(ctx, local) return nil }, index) } @@ -211,7 +212,11 @@ func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo { storageInfo.Backend.RRSCData = storageInfos[0].Backend.RRSCData storageInfo.Backend.RRSCParity = storageInfos[0].Backend.RRSCParity - return storageInfo + var errs []error + for i := range z.zones { + errs = append(errs, storageInfosErrs[i]...) + } + return storageInfo, errs } func (z *xlZones) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {