fix: inherit heal opts globally, including bitrot settings (#11166)

Bonus re-use ReadFileStream internal io.Copy buffers, fixes
lots of chatty allocations when reading metacache readers
with many sustained concurrent listing operations

```
   17.30GB  1.27% 84.80%    35.26GB  2.58%  io.copyBuffer
```
master
Harshavardhana 4 years ago committed by GitHub
parent 027e17468a
commit 59d3639396
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      cmd/admin-heal-ops.go
  2. 6
      cmd/data-crawler.go
  3. 2
      cmd/metacache-set.go
  4. 5
      cmd/storage-rest-server.go

@ -677,11 +677,10 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
} }
if source.opts != nil { if source.opts != nil {
task.opts = *source.opts task.opts = *source.opts
} else { }
if opts.Bitrot { if opts.Bitrot {
task.opts.ScanMode = madmin.HealDeepScan task.opts.ScanMode = madmin.HealDeepScan
} }
}
// Wait and proceed if there are active requests // Wait and proceed if there are active requests
waitForLowHTTPReq(opts.IOCount, opts.Sleep) waitForLowHTTPReq(opts.IOCount, opts.Sleep)

@ -570,9 +570,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
bucket: bucket, bucket: bucket,
object: entry.name, object: entry.name,
versionID: "", versionID: "",
opts: &madmin.HealOpts{
Remove: true,
},
}, madmin.HealItemObject) }, madmin.HealItemObject)
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
@ -588,9 +585,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
bucket: bucket, bucket: bucket,
object: fiv.Name, object: fiv.Name,
versionID: ver.VersionID, versionID: ver.VersionID,
opts: &madmin.HealOpts{
Remove: true,
},
}, madmin.HealItemObject) }, madmin.HealItemObject)
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)

@ -823,7 +823,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
defer cancel() defer cancel()
askDisks := len(disks) askDisks := len(disks)
var readers = make([]*metacacheReader, askDisks) readers := make([]*metacacheReader, askDisks)
for i := range disks { for i := range disks {
r, w := io.Pipe() r, w := io.Pipe()
d := disks[i] d := disks[i]

@ -522,7 +522,10 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
w.Header().Set(xhttp.ContentLength, strconv.Itoa(length)) w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
io.Copy(w, rc) bufp := s.storage.pool.Get().(*[]byte)
defer s.storage.pool.Put(bufp)
io.CopyBuffer(w, rc, *bufp)
w.(http.Flusher).Flush() w.(http.Flusher).Flush()
} }

Loading…
Cancel
Save