Show offline nodes after a fixed number of init retry (#3107)

cmd/prepare-storage.go

@@ -196,9 +196,16 @@ func retryFormattingDisks(firstDisk bool, firstEndpoint string, storageDisks []S
 	defer close(doneCh)
 
 	// Wait on the jitter retry loop.
-	for range newRetryTimer(time.Second, time.Second*30, MaxJitter, doneCh) {
+	for retryCounter := range newRetryTimer(time.Second, time.Second*30, MaxJitter, doneCh) {
 		// Attempt to load all `format.json`.
 		formatConfigs, sErrs := loadAllFormats(storageDisks)
+		if retryCounter > 5 {
+			for i, e := range sErrs {
+				if e == errDiskNotFound {
+					console.Printf("%s still unreachable.\n", storageDisks[i])
+				}
+			}
+		}
 		// Check if this is a XL or distributed XL, anything > 1 is considered XL backend.
 		if len(formatConfigs) > 1 {
 			switch prepForInitXL(firstDisk, sErrs, len(storageDisks)) {

cmd/retry.go

@@ -58,8 +58,8 @@ var globalRandomSource = rand.New(&lockedRandSource{
 
 // newRetryTimer creates a timer with exponentially increasing delays
 // until the maximum retry attempts are reached.
-func newRetryTimer(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan struct{} {
-	attemptCh := make(chan struct{})
+func newRetryTimer(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
+	attemptCh := make(chan int)
 
 	// computes the exponential backoff duration according to
 	// https://www.awsarchitectureblog.com/2015/03/backoff.html
@@ -89,7 +89,7 @@ func newRetryTimer(unit time.Duration, cap time.Duration, jitter float64, doneCh
 		for {
 			select {
 			// Attempts starts.
-			case attemptCh <- struct{}{}:
+			case attemptCh <- nextBackoff:
 				nextBackoff++
 			case <-globalWakeupCh:
 				// Reset nextBackoff to reduce the subsequent wait and re-read