Remove crawler concurrency (#9023)

Only have one crawler per disk. Removes locking, but keep
fastwalk itself able to run concurrently.
master
Klaus Post 5 years ago committed by GitHub
parent ab7d3cd508
commit b2db1e96e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      cmd/data-usage.go

@ -22,7 +22,6 @@ import (
"encoding/json" "encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"time" "time"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
@ -160,10 +159,7 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun
ObjectsSizesHistogram: make(map[string]uint64), ObjectsSizesHistogram: make(map[string]uint64),
} }
numWorkers := 4 fastWalk(basePath, 1, doneCh, func(path string, typ os.FileMode) error {
var mutex sync.Mutex // Mutex to update dataUsageInfo
fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error {
// Wait for I/O to go down. // Wait for I/O to go down.
waitForLowActiveIO() waitForLowActiveIO()
@ -177,16 +173,11 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun
} }
if entry == "" && typ&os.ModeDir != 0 { if entry == "" && typ&os.ModeDir != 0 {
mutex.Lock()
dataUsageInfo.BucketsCount++ dataUsageInfo.BucketsCount++
dataUsageInfo.BucketsSizes[bucket] = 0 dataUsageInfo.BucketsSizes[bucket] = 0
mutex.Unlock()
return nil return nil
} }
mutex.Lock()
defer mutex.Unlock()
if typ&os.ModeDir != 0 { if typ&os.ModeDir != 0 {
return nil return nil
} }

Loading…
Cancel
Save