From 2165d45d3f60cfbe280d87fcdc0c6df0550e8635 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Thu, 6 Feb 2020 19:35:55 -0800 Subject: [PATCH] Time getSize and use to estimate latency (#8959) Remove the random sleep. This is running in 4 goroutines, so mostly doing nothing. We use the getSize latency to estimate system load, meaning when there is little load on the system and we get the result fast we sleep a little. If it took a long time we have high load and release ourselves longer. We are sleeping inside the mutex so this affects all goroutines doing IO. --- cmd/data-usage.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/data-usage.go b/cmd/data-usage.go index a77463c02..3232adbbd 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "encoding/json" - "math/rand" "os" "path/filepath" "sync" @@ -160,19 +159,12 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun } numWorkers := 4 - walkInterval := 1 * time.Millisecond - var mutex sync.Mutex // Mutex to update dataUsageInfo - r := rand.New(rand.NewSource(UTCNow().UnixNano())) - fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error { // Wait for I/O to go down. waitForLowActiveIO() - // Randomize sleep intervals, to stagger the walk. - defer time.Sleep(time.Duration(r.Float64() * float64(walkInterval))) - bucket, entry := path2BucketObjectWithBasePath(basePath, path) if bucket == "" { return nil @@ -197,7 +189,13 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun return nil } + t := time.Now() size, err := getSize(Item{path, typ}) + // Use the response time of the getSize call to guess system load. + // Sleep equivalent time. + if d := time.Since(t); d > 100*time.Microsecond { + time.Sleep(d) + } if err != nil { return errSkipFile }