Reduce logging when peer is offline (#11184)

cmd/erasure-metadata-utils.go

@@ -140,9 +140,8 @@ func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, objec
 			}
 			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, checkDataDir)
 			if err != nil {
-				if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
-					logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())
-					logger.LogIf(ctx, err)
+				if !IsErr(err, errFileNotFound, errVolumeNotFound, errFileVersionNotFound, errDiskNotFound) {
+					logger.LogOnceIf(ctx, err, disks[index].String())
 				}
 			}
 			return err

cmd/rest/client.go

@@ -119,8 +119,9 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
+			if c.MarkOffline() {
 				logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err))
-			c.MarkOffline()
+			}
 		}
 		return nil, &NetworkError{err}
 	}
@@ -150,8 +151,9 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
 		if err != nil {
 			if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
+				if c.MarkOffline() {
 					logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err))
-				c.MarkOffline()
+				}
 			}
 			return nil, err
 		}
@@ -190,7 +192,8 @@ func (c *Client) IsOnline() bool {
 
 // MarkOffline - will mark a client as being offline and spawns
 // a goroutine that will attempt to reconnect if HealthCheckFn is set.
-func (c *Client) MarkOffline() {
+// returns true if the node changed state from online to offline
+func (c *Client) MarkOffline() bool {
 	// Start goroutine that will attempt to reconnect.
 	// If server is already trying to reconnect this will have no effect.
 	if c.HealthCheckFn != nil && atomic.CompareAndSwapInt32(&c.connected, online, offline) {
@@ -201,12 +204,15 @@ func (c *Client) MarkOffline() {
 					return
 				}
 				if c.HealthCheckFn() {
-					atomic.CompareAndSwapInt32(&c.connected, offline, online)
+					if atomic.CompareAndSwapInt32(&c.connected, offline, online) {
 						logger.Info("Client %s online", c.url.String())
+					}
 					return
 				}
 				time.Sleep(time.Duration(r.Float64() * float64(c.HealthCheckInterval)))
 			}
 		}()
+		return true
 	}
+	return false
 }