From 59d36393960b8659d962defdb6becf569eafe17e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 24 Dec 2020 23:04:03 -0800 Subject: [PATCH] fix: inherit heal opts globally, including bitrot settings (#11166) Bonus re-use ReadFileStream internal io.Copy buffers, fixes lots of chatty allocations when reading metacache readers with many sustained concurrent listing operations ``` 17.30GB 1.27% 84.80% 35.26GB 2.58% io.copyBuffer ``` --- cmd/admin-heal-ops.go | 7 +++---- cmd/data-crawler.go | 6 ------ cmd/metacache-set.go | 2 +- cmd/storage-rest-server.go | 5 ++++- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index a7c18cb1a..fc0d50cfd 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -677,10 +677,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem } if source.opts != nil { task.opts = *source.opts - } else { - if opts.Bitrot { - task.opts.ScanMode = madmin.HealDeepScan - } + } + if opts.Bitrot { + task.opts.ScanMode = madmin.HealDeepScan } // Wait and proceed if there are active requests diff --git a/cmd/data-crawler.go b/cmd/data-crawler.go index efb65de47..d199046d4 100644 --- a/cmd/data-crawler.go +++ b/cmd/data-crawler.go @@ -570,9 +570,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo bucket: bucket, object: entry.name, versionID: "", - opts: &madmin.HealOpts{ - Remove: true, - }, }, madmin.HealItemObject) if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { logger.LogIf(ctx, err) @@ -588,9 +585,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo bucket: bucket, object: fiv.Name, versionID: ver.VersionID, - opts: &madmin.HealOpts{ - Remove: true, - }, }, madmin.HealItemObject) if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { logger.LogIf(ctx, err) diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go index 2c093c9c8..522478b97 100644 --- a/cmd/metacache-set.go +++ b/cmd/metacache-set.go @@ -823,7 +823,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) { defer cancel() askDisks := len(disks) - var readers = make([]*metacacheReader, askDisks) + readers := make([]*metacacheReader, askDisks) for i := range disks { r, w := io.Pipe() d := disks[i] diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 8c8333b92..d199e9587 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -522,7 +522,10 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http w.Header().Set(xhttp.ContentLength, strconv.Itoa(length)) - io.Copy(w, rc) + bufp := s.storage.pool.Get().(*[]byte) + defer s.storage.pool.Put(bufp) + + io.CopyBuffer(w, rc, *bufp) w.(http.Flusher).Flush() }