fix: background heal to call HealFormat only if needed (#9491)

In large setups this avoids unnecessary data transfer
across nodes and potential locks.

This PR also optimizes heal result channel, which should
be avoided for each queueHealTask as its expensive
to create/close channels for large number of objects.

cmd/admin-heal-ops.go

@@ -309,7 +309,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 // healSource denotes single entity and heal option.
 type healSource struct {
 	path string          // entity path (format, buckets, objects) to heal
-	opts *madmin.HealOpts // optional heal option overrides default setting
+	opts madmin.HealOpts // optional heal option overrides default setting
 }
 
 // healSequence - state for each heal sequence initiated on the
@@ -321,9 +321,12 @@ type healSequence struct {
 	// path is just pathJoin(bucket, objPrefix)
 	path string
 
-	// List of entities (format, buckets, objects) to heal
+	// A channel of entities (format, buckets, objects) to heal
 	sourceCh chan healSource
 
+	// A channel of entities with heal result
+	respCh chan healResult
+
 	// Report healing progress
 	reportProgress bool
 
@@ -385,6 +388,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	return &healSequence{
+		respCh:         make(chan healResult),
 		bucket:         bucket,
 		objPrefix:      objPrefix,
 		path:           pathJoin(bucket, objPrefix),
@@ -636,20 +640,19 @@ func (h *healSequence) healSequenceStart() {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
-	var respCh = make(chan healResult)
-	defer close(respCh)
 	// Send heal request
 	task := healTask{
 		path:       source.path,
-		responseCh: respCh,
 		opts:       h.settings,
+		responseCh: h.respCh,
 	}
-	if source.opts != nil {
-		task.opts = *source.opts
+	if !source.opts.Equal(h.settings) {
+		task.opts = source.opts
 	}
 	globalBackgroundHealRoutine.queueHealTask(task)
-	// Wait for answer and push result to the client
-	res := <-respCh
+
+	select {
+	case res := <-h.respCh:
 		if !h.reportProgress {
 			h.mutex.Lock()
 			defer h.mutex.Unlock()
@@ -683,6 +686,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 			res.result.Detail = res.err.Error()
 		}
 		return h.pushHealResultItem(res.result)
+	case <-h.ctx.Done():
+		return nil
+	case <-h.traverseAndHealDoneCh:
+		return nil
+	}
+
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {

cmd/background-heal-ops.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"path"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -89,10 +88,10 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case bucket != "" && object != "":
 				res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
 			}
-			ObjectPathUpdated(path.Join(bucket, object))
-			if task.responseCh != nil {
-				task.responseCh <- healResult{result: res, err: err}
+			if task.path != slashSeparator && task.path != nopHeal {
+				ObjectPathUpdated(task.path)
 			}
+			task.responseCh <- healResult{result: res, err: err}
 		case <-h.doneCh:
 			return
 		case <-ctx.Done():

cmd/background-newdisks-heal-ops.go

@@ -57,6 +57,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		case <-time.After(defaultMonitorNewDiskInterval):
 			// Attempt a heal as the server starts-up first.
 			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+			var healNewDisks bool
 			for i, ep := range globalEndpoints {
 				localDisksToHeal := Endpoints{}
 				for _, endpoint := range ep.Endpoints {
@@ -74,6 +75,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 					continue
 				}
 				localDisksInZoneHeal[i] = localDisksToHeal
+				healNewDisks = true
+			}
+
+			// Reformat disks only if needed.
+			if !healNewDisks {
+				continue
 			}
 
 			// Reformat disks

cmd/global-heal.go

@@ -48,6 +48,7 @@ func newBgHealSequence(numDisks int) *healSequence {
 
 	return &healSequence{
 		sourceCh:    make(chan healSource),
+		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
 		settings:    hs,
@@ -125,7 +126,7 @@ func deepHealObject(objectPath string) {
 
 	bgSeq.sourceCh <- healSource{
 		path: objectPath,
-		opts: &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+		opts: madmin.HealOpts{ScanMode: madmin.HealDeepScan},
 	}
 }
 

pkg/madmin/heal-commands.go

@@ -45,6 +45,20 @@ type HealOpts struct {
 	ScanMode  HealScanMode `json:"scanMode"`
 }
 
+// Equal returns true if no is same as o.
+func (o HealOpts) Equal(no HealOpts) bool {
+	if o.Recursive != no.Recursive {
+		return false
+	}
+	if o.DryRun != no.DryRun {
+		return false
+	}
+	if o.Remove != no.Remove {
+		return false
+	}
+	return o.ScanMode == no.ScanMode
+}
+
 // HealStartSuccess - holds information about a successfully started
 // heal operation
 type HealStartSuccess struct {