xl: All nodes create meta volumes in its local disks (#8786)

Meta volumes directories, tmp/, background-ops/, etc..
undr .minio.sys are created when disks are formatted
but also when the cluster is started.

However using MakeVolBulk() is not appropriate in the
case of a user migrating from a version which does not
have .minio.sys/background-ops/. The reason is that
MakeVolBulk() exits early when an error is occured:
errVolumeExists in this case, which is expected since
some directories such as tmp/ already exist.

This commit will avoid use MakeVolBulk and use MakeVol
instead.

Also the PR will make each node creates meta volumes
in its local disks and stop relying on the first disk
since the first node could be offline.

cmd/format-xl.go

@@ -621,23 +621,28 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
 }
 
-// Initializes meta volume on all input storage disks.
-func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
-	// This happens for the first time, but keep this here since this
-	// is the only place where it can be made expensive optimizing all
-	// other calls. Create minio meta volume, if it doesn't exist yet.
+// Initializes meta volume only on local storage disks.
+func initXLMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatXLV3) error {
+
+	// Compute the local disks eligible for meta volumes (re)initialization
+	var disksToInit []StorageAPI
+	for index := range storageDisks {
+		if formats[index] == nil || storageDisks[index] == nil || storageDisks[index].Hostname() != "" {
+			// Ignore create meta volume on disks which are not found or not local.
+			continue
+		}
+		disksToInit = append(disksToInit, storageDisks[index])
+	}
 
 	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(storageDisks))
+	g := errgroup.WithNErrs(len(disksToInit))
 
 	// Initialize all disks in parallel.
-	for index := range storageDisks {
+	for index := range disksToInit {
+		// Initialize a new index variable in each loop so each
+		// goroutine will return its own instance of index variable.
 		index := index
 		g.Go(func() error {
-			if formats[index] == nil || storageDisks[index] == nil {
-				// Ignore create meta volume on disks which are not found.
-				return nil
-			}
 			return makeFormatXLMetaVolumes(storageDisks[index])
 		}, index)
 	}
@@ -821,13 +826,7 @@ func ecDrivesNoConfig(drivesPerSet int) int {
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
-	if err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	return nil
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/format-xl_test.go

@@ -101,7 +101,7 @@ func TestFixFormatV3(t *testing.T) {
 		formats[j] = newFormat
 	}
 
-	if err = initFormatXLMetaVolume(storageDisks, formats); err != nil {
+	if err = initXLMetaVolumesInLocalDisks(storageDisks, formats); err != nil {
 		t.Fatal(err)
 	}
 

cmd/prepare-storage.go

@@ -267,12 +267,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return format, nil
 	}
 
-	// The first will always recreate some directories inside .minio.sys
-	// such as, tmp, multipart and background-ops
-	if firstDisk {
-		initFormatXLMetaVolume(storageDisks, formatConfigs)
-	}
-
 	// Return error when quorum unformatted disks - indicating we are
 	// waiting for first server to be online.
 	if quorumUnformattedDisks(sErrs) && !firstDisk {
@@ -331,6 +325,10 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return nil, err
 	}
 
+	// The will always recreate some directories inside .minio.sys of
+	// the local disk such as tmp, multipart and background-ops
+	initXLMetaVolumesInLocalDisks(storageDisks, formatConfigs)
+
 	return format, nil
 }