fix: save healing tracker right before healing (#10915)

this change avoids a situation where accidentally
if the user deleted the healing tracker or drives
were replaced again within the 10sec window.

cmd/background-newdisks-heal-ops.go

@@ -118,6 +118,7 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 //  2. Only the node hosting the disk is responsible to perform the heal
 func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
+wait:
 	for {
 		select {
 		case <-ctx.Done():
@@ -176,6 +177,26 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *
 					for _, disk := range disks {
 						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
 
+						// So someone changed the drives underneath, healing tracker missing.
+						if !disk.Healing() {
+							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1))
+							diskID, err := disk.GetDiskID()
+							if err != nil {
+								logger.LogIf(ctx, err)
+								// reading format.json failed or not found, proceed to look
+								// for new disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+
+							if err := saveHealingTracker(disk, diskID); err != nil {
+								logger.LogIf(ctx, err)
+								// Unable to write healing tracker, permission denied or some
+								// other unexpected error occurred. Proceed to look for new
+								// disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+						}
+
 						lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks()
 						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
 							logger.LogIf(ctx, err)

cmd/format-erasure.go

@@ -339,6 +339,19 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 	return formats, g.Wait()
 }
 
+func saveHealingTracker(disk StorageAPI, diskID string) error {
+	htracker := healingTracker{
+		ID: diskID,
+	}
+	htrackerBytes, err := htracker.MarshalMsg(nil)
+	if err != nil {
+		return err
+	}
+	return disk.WriteAll(context.TODO(), minioMetaBucket,
+		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		htrackerBytes)
+}
+
 func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error {
 	if disk == nil || format == nil {
 		return errDiskNotFound
@@ -373,16 +386,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 
 	disk.SetDiskID(diskID)
 	if heal {
-		htracker := healingTracker{
-			ID: diskID,
-		}
-		htrackerBytes, err := htracker.MarshalMsg(nil)
-		if err != nil {
-			return err
-		}
-		return disk.WriteAll(context.TODO(), minioMetaBucket,
-			pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-			htrackerBytes)
+		return saveHealingTracker(disk, diskID)
 	}
 	return nil
 }