From 2b4eb87d77b8478376148f494930fa8a2ed9320d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 29 Sep 2020 22:54:02 -0700 Subject: [PATCH] pick disks which are common maximally used (#10600) further optimization to ensure that good disks are always used for listing, other than healing we only use disks that are maximally used. --- cmd/disk-cache-backend.go | 6 +++--- cmd/erasure-common.go | 17 ++++++++++++++--- cmd/erasure-zones.go | 8 ++++---- cmd/fs-v1.go | 3 +-- cmd/server-main.go | 16 ++++++++++++++++ cmd/xl-storage.go | 2 +- pkg/disk/disk.go | 4 +--- pkg/disk/stat_bsd.go | 5 +++++ pkg/disk/stat_linux.go | 1 + pkg/disk/stat_linux_32bit.go | 1 + pkg/disk/stat_linux_s390x.go | 1 + pkg/disk/stat_netbsd.go | 6 ++++++ pkg/disk/stat_openbsd.go | 5 +++++ pkg/disk/stat_solaris.go | 6 ++++++ pkg/disk/stat_windows.go | 5 +++++ 15 files changed, 70 insertions(+), 16 deletions(-) diff --git a/cmd/disk-cache-backend.go b/cmd/disk-cache-backend.go index b48e8443b..4b2c3e389 100644 --- a/cmd/disk-cache-backend.go +++ b/cmd/disk-cache-backend.go @@ -194,7 +194,7 @@ func (c *diskCache) diskUsageLow() bool { logger.LogIf(ctx, err) return false } - usedPercent := (di.Total - di.Free) * 100 / di.Total + usedPercent := (di.Used / di.Total) * 100 low := int(usedPercent) < gcStopPct atomic.StoreUint64(&c.stats.UsagePercent, usedPercent) if low { @@ -218,7 +218,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool { logger.Info("diskCache: Received 0 total disk size") return false } - usedPercent := float64(di.Total-di.Free) * 100 / float64(di.Total) + usedPercent := float64(di.Used) * 100 / float64(di.Total) if usedPercent >= float64(gcTriggerPct) { atomic.StoreInt32(&c.stats.UsageState, 1) c.queueGC() @@ -226,7 +226,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool { atomic.StoreUint64(&c.stats.UsagePercent, uint64(usedPercent)) // Recalculate percentage with provided size added. - usedPercent = float64(di.Total-di.Free+uint64(size)) * 100 / float64(di.Total) + usedPercent = float64(di.Used+uint64(size)) * 100 / float64(di.Total) return usedPercent < float64(c.quotaPct) } diff --git a/cmd/erasure-common.go b/cmd/erasure-common.go index d6a64ad08..df1b38d7b 100644 --- a/cmd/erasure-common.go +++ b/cmd/erasure-common.go @@ -59,7 +59,7 @@ func (er erasureObjects) getLoadBalancedDisks() []StorageAPI { var wg sync.WaitGroup var mu sync.Mutex - var newDisks []StorageAPI + var newDisks = map[uint64][]StorageAPI{} // Based on the random shuffling return back randomized disks. for _, i := range hashOrder(UTCNow().String(), len(disks)) { i := i @@ -79,13 +79,24 @@ func (er erasureObjects) getLoadBalancedDisks() []StorageAPI { // - Future: skip busy disks return } + mu.Lock() - newDisks = append(newDisks, disks[i-1]) + // Capture disks usage wise + newDisks[di.Used] = append(newDisks[di.Used], disks[i-1]) mu.Unlock() }() } wg.Wait() - return newDisks + + var max uint64 + for k := range newDisks { + if k > max { + max = k + } + } + + // Return disks which have maximum disk usage common. + return newDisks[max] } // This function does the following check, suppose diff --git a/cmd/erasure-zones.go b/cmd/erasure-zones.go index 0733852e2..fceee6bcc 100644 --- a/cmd/erasure-zones.go +++ b/cmd/erasure-zones.go @@ -662,7 +662,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix, for _, zone := range z.zones { zonesEntryChs = append(zonesEntryChs, zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet, false)) - zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1) + zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2) } var objInfos []ObjectInfo @@ -784,7 +784,7 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) - zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1) + zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2) } entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet) @@ -876,7 +876,7 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker, } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) - zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1) + zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2) } entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet) @@ -1278,7 +1278,7 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) - zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-1) + zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2) } entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index d03ce9dca..169a5e7da 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -216,12 +216,11 @@ func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []er if err != nil { return StorageInfo{}, []error{err} } - used := di.Total - di.Free storageInfo := StorageInfo{ Disks: []madmin.Disk{ { TotalSpace: di.Total, - UsedSpace: used, + UsedSpace: di.Used, AvailableSpace: di.Free, DrivePath: fs.fsPath, }, diff --git a/cmd/server-main.go b/cmd/server-main.go index 9f41d579d..ca03716bf 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -319,6 +319,22 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) { if err != nil { return fmt.Errorf("Unable to list buckets to heal: %w", err) } + rquorum := InsufficientReadQuorum{} + wquorum := InsufficientWriteQuorum{} + for _, bucket := range buckets { + if err = newObject.MakeBucketWithLocation(ctx, bucket.Name, BucketOptions{}); err != nil { + if errors.As(err, &wquorum) || errors.As(err, &rquorum) { + // Return the error upwards for the caller to retry. + return fmt.Errorf("Unable to heal bucket: %w", err) + } + if _, ok := err.(BucketExists); !ok { + // ignore any other error and log for investigation. + logger.LogIf(ctx, err) + continue + } + // Bucket already exists, nothing that needs to be done. + } + } } else { buckets, err = newObject.ListBuckets(ctx) if err != nil { diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index df2db5c70..ce2653a02 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -479,7 +479,7 @@ func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) { } dcinfo.Total = di.Total dcinfo.Free = di.Free - dcinfo.Used = di.Total - di.Free + dcinfo.Used = di.Used dcinfo.FSType = di.FSType diskID, err := s.GetDiskID() diff --git a/pkg/disk/disk.go b/pkg/disk/disk.go index 504b5fd64..5d326d5d7 100644 --- a/pkg/disk/disk.go +++ b/pkg/disk/disk.go @@ -25,10 +25,8 @@ package disk type Info struct { Total uint64 Free uint64 + Used uint64 Files uint64 Ffree uint64 FSType string - - // Usage is calculated per tenant. - Usage uint64 } diff --git a/pkg/disk/stat_bsd.go b/pkg/disk/stat_bsd.go index d286a271b..9d0305ec0 100644 --- a/pkg/disk/stat_bsd.go +++ b/pkg/disk/stat_bsd.go @@ -19,6 +19,7 @@ package disk import ( + "fmt" "syscall" ) @@ -37,5 +38,9 @@ func GetInfo(path string) (info Info, err error) { Ffree: uint64(s.Ffree), FSType: getFSType(s.Fstypename[:]), } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_linux.go b/pkg/disk/stat_linux.go index 8cd11cd09..d9b5ab86c 100644 --- a/pkg/disk/stat_linux.go +++ b/pkg/disk/stat_linux.go @@ -45,5 +45,6 @@ func GetInfo(path string) (info Info, err error) { if info.Free > info.Total { return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_linux_32bit.go b/pkg/disk/stat_linux_32bit.go index c83a6e7cd..019e465b8 100644 --- a/pkg/disk/stat_linux_32bit.go +++ b/pkg/disk/stat_linux_32bit.go @@ -76,5 +76,6 @@ func GetInfo(path string) (info Info, err error) { if info.Free > info.Total { return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_linux_s390x.go b/pkg/disk/stat_linux_s390x.go index d4810ecbc..efb3145db 100644 --- a/pkg/disk/stat_linux_s390x.go +++ b/pkg/disk/stat_linux_s390x.go @@ -76,5 +76,6 @@ func GetInfo(path string) (info Info, err error) { if info.Free > info.Total { return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_netbsd.go b/pkg/disk/stat_netbsd.go index 5591cc5a8..abd34d849 100644 --- a/pkg/disk/stat_netbsd.go +++ b/pkg/disk/stat_netbsd.go @@ -19,6 +19,8 @@ package disk import ( + "fmt" + "golang.org/x/sys/unix" ) @@ -36,5 +38,9 @@ func GetInfo(path string) (info Info, err error) { Ffree: uint64(s.Ffree), FSType: string(s.Fstypename[:]), } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_openbsd.go b/pkg/disk/stat_openbsd.go index 7f43a10e8..df6e4f58e 100644 --- a/pkg/disk/stat_openbsd.go +++ b/pkg/disk/stat_openbsd.go @@ -19,6 +19,7 @@ package disk import ( + "fmt" "syscall" ) @@ -37,5 +38,9 @@ func GetInfo(path string) (info Info, err error) { Ffree: uint64(s.F_ffree), FSType: getFSType(s.F_fstypename[:]), } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_solaris.go b/pkg/disk/stat_solaris.go index ea4802d1b..28ab0cdcf 100644 --- a/pkg/disk/stat_solaris.go +++ b/pkg/disk/stat_solaris.go @@ -19,6 +19,8 @@ package disk import ( + "fmt" + "golang.org/x/sys/unix" ) @@ -36,5 +38,9 @@ func GetInfo(path string) (info Info, err error) { Ffree: uint64(s.Ffree), FSType: getFSType(s.Fstr[:]), } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/disk/stat_windows.go b/pkg/disk/stat_windows.go index 62ed08c51..d95f1dcc7 100644 --- a/pkg/disk/stat_windows.go +++ b/pkg/disk/stat_windows.go @@ -19,6 +19,7 @@ package disk import ( + "fmt" "os" "syscall" "unsafe" @@ -90,5 +91,9 @@ func GetInfo(path string) (info Info, err error) { info.Files = uint64(lpTotalNumberOfClusters) info.Ffree = uint64(lpNumberOfFreeClusters) + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free return info, nil }