storage: Increase the timeout of storage REST requests (#7218)

This commit increases storage REST requests to 5 minutes, this includes
the opening TCP connection, and sending/receiving data. This will reduce
clients receiving errors when the server is under high load.

cmd/rpc.go

@@ -31,6 +31,12 @@ import (
 // DefaultSkewTime - skew time is 15 minutes between minio peers.
 const DefaultSkewTime = 15 * time.Minute
 
+// defaultRPCTimeout - default RPC timeout is one minute.
+const defaultRPCTimeout = 5 * time.Minute
+
+// defaultRPCRetryTime - default RPC time to wait before retry after a network error
+const defaultRPCRetryTime = 1 * time.Minute
+
 var errRPCRetry = fmt.Errorf("rpc: retry error")
 
 func isNetError(err error) bool {
@@ -217,7 +223,7 @@ func (client *RPCClient) Call(serviceMethod string, args interface {
 		}
 
 		if isNetError(err) {
-			client.setRetryTicker(time.NewTicker(xrpc.DefaultRPCTimeout))
+			client.setRetryTicker(time.NewTicker(defaultRPCRetryTime))
 		} else {
 			client.setRetryTicker(nil)
 		}
@@ -265,6 +271,6 @@ func NewRPCClient(args RPCClientArgs) (*RPCClient, error) {
 	return &RPCClient{
 		args:      args,
 		authToken: args.NewAuthTokenFunc(),
-		rpcClient: xrpc.NewClient(args.ServiceURL, args.TLSConfig, xrpc.DefaultRPCTimeout),
+		rpcClient: xrpc.NewClient(args.ServiceURL, args.TLSConfig, defaultRPCTimeout),
 	}, nil
 }

cmd/storage-rest-client.go

@@ -25,6 +25,7 @@ import (
 	"net/url"
 	"path"
 	"strconv"
+	"time"
 
 	"encoding/gob"
 	"encoding/hex"
@@ -38,6 +39,10 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+// The timeout of TCP connect and sending/receiving
+// data for all internode storage REST requests.
+const storageRESTTimeout = 5 * time.Minute
+
 func isNetworkDisconnectError(err error) bool {
 	if err == nil {
 		return false
@@ -376,6 +381,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 		}
 	}
 
-	restClient := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
+	restClient := rest.NewClient(serverURL, tlsConfig, storageRESTTimeout, newAuthToken)
 	return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: true}
 }