fix: heal object shards with ec.index and ec.distribution mismatches (#10773)

Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/erasure-healing-common.go

@@ -160,6 +160,25 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
+	inconsistent := 0
+	for i, meta := range partsMetadata {
+		if !meta.IsValid() {
+			// Since for majority of the cases erasure.Index matches with erasure.Distribution we can
+			// consider the offline disks as consistent.
+			continue
+		}
+		if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+			inconsistent++
+		}
+	}
+
+	erasureDistributionReliable := true
+	if inconsistent > len(partsMetadata)/2 {
+		// If there are too many inconsistent files, then we can't trust erasure.Distribution (most likely
+		// because of bugs found in CopyObject/PutObjectTags) https://github.com/minio/minio/pull/10772
+		erasureDistributionReliable = false
+	}
+
 	for i, onlineDisk := range onlineDisks {
 		if errs[i] != nil {
 			dataErrs[i] = errs[i]
@@ -169,6 +188,18 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			dataErrs[i] = errDiskNotFound
 			continue
 		}
+		if erasureDistributionReliable {
+			meta := partsMetadata[i]
+			if !meta.IsValid() {
+				continue
+			}
+			// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
+			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
 
 		switch scanMode {
 		case madmin.HealDeepScan:

cmd/erasure-healing.go

@@ -221,17 +221,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	partsMetadata []FileInfo, errs []error, latestFileInfo FileInfo,
 	dryRun bool, remove bool, scanMode madmin.HealScanMode) (result madmin.HealResultItem, err error) {
 
-	for i, metadata := range shufflePartsMetadata(partsMetadata, latestFileInfo.Erasure.Distribution) {
-		if !metadata.IsValid() {
-			continue
-		}
-		if i != metadata.Erasure.Index-1 {
-			// FIXME: fix in the next release for objects which erasure.Index does not match
-			// with expected distribution.
-			return result, fmt.Errorf("Unable to heal object %s, disk ordering issue detected", pathJoin(bucket, object))
-		}
-	}
-
 	dataBlocks := latestFileInfo.Erasure.DataBlocks
 
 	storageDisks := er.getDisks()