xl/zones: return errNoHealRequired when no heal is required (#8821)

Zone abstraction of object layer was returning `nil`
incorrectly under situations where disk healing is
not required. Returning `nil` is considered as healing
successful, which leads to unexpected ReloadFormat()
peer notification calls during startup.

This PR fixes this behavior properly for zones.

buildscripts/verify-healing.sh

@@ -75,7 +75,7 @@ function __init__()
 }
 
 function perform_test_1() {
-    minio_pids=( $(start_minio_3_node 30) )
+    minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
         kill "$pid"
     done
@@ -111,7 +111,7 @@ function perform_test_1() {
 }
 
 function perform_test_2() {
-    minio_pids=( $(start_minio_3_node 30) )
+    minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
         kill "$pid"
     done
@@ -148,7 +148,7 @@ function perform_test_2() {
 }
 
 function perform_test_3() {
-    minio_pids=( $(start_minio_3_node 30) )
+    minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
         kill "$pid"
     done

cmd/format-xl.go

@@ -762,7 +762,6 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
 	formats := make([]*formatXLV3, len(storageDisks))
 	wantAtMost := ecDrivesNoConfig(drivesPerSet)
 
-	logger.Info("Formatting zone, %v set(s), %v drives per set.", setCount, drivesPerSet)
 	for i := 0; i < setCount; i++ {
 		hostCount := make(map[string]int, drivesPerSet)
 		for j := 0; j < drivesPerSet; j++ {

cmd/xl-zones.go

@@ -60,8 +60,11 @@ func newXLZones(endpointZones EndpointZones) (ObjectLayer, error) {
 		formats = make([]*formatXLV3, len(endpointZones))
 		z       = &xlZones{zones: make([]*xlSets, len(endpointZones))}
 	)
+	local := endpointZones.FirstLocal()
 	for i, ep := range endpointZones {
-		formats[i], err = waitForFormatXL(endpointZones.FirstLocal(), ep.Endpoints,
+		logger.Info("Formatting %v zone, %v set(s), %v drives per set.",
+			i+1, ep.SetCount, ep.DrivesPerSet)
+		formats[i], err = waitForFormatXL(local, ep.Endpoints,
 			ep.SetCount, ep.DrivesPerSet, deploymentID)
 		if err != nil {
 			return nil, err
@@ -69,8 +72,6 @@ func newXLZones(endpointZones EndpointZones) (ObjectLayer, error) {
 		if deploymentID == "" {
 			deploymentID = formats[i].ID
 		}
-	}
-	for i, ep := range endpointZones {
 		z.zones[i], err = newXLSets(ep.Endpoints, formats[i], ep.SetCount, ep.DrivesPerSet)
 		if err != nil {
 			return nil, err
@@ -1261,17 +1262,28 @@ func (z *xlZones) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResul
 		Type:   madmin.HealItemMetadata,
 		Detail: "disk-format",
 	}
+
+	var countNoHeal int
 	for _, zone := range z.zones {
 		result, err := zone.HealFormat(ctx, dryRun)
 		if err != nil && err != errNoHealRequired {
 			logger.LogIf(ctx, err)
 			continue
 		}
+		// Count errNoHealRequired across all zones,
+		// to return appropriate error to the caller
+		if err == errNoHealRequired {
+			countNoHeal++
+		}
 		r.DiskCount += result.DiskCount
 		r.SetCount += result.SetCount
 		r.Before.Drives = append(r.Before.Drives, result.Before.Drives...)
 		r.After.Drives = append(r.After.Drives, result.After.Drives...)
 	}
+	// No heal returned by all zones, return errNoHealRequired
+	if countNoHeal == len(z.zones) {
+		return r, errNoHealRequired
+	}
 	return r, nil
 }