diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 4e0ca2105..3548f586e 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -986,7 +986,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { // Use buffered channel to take care of burst sends or slow w.Write() traceCh := make(chan interface{}, 4000) - peers := getRestClients(globalEndpoints) + peers := newPeerRestClients(globalEndpoints) globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool { return mustTrace(entry, trcAll, trcErr) @@ -1051,7 +1051,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque logCh := make(chan interface{}, 4000) - peers := getRestClients(globalEndpoints) + peers := newPeerRestClients(globalEndpoints) globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil) @@ -1482,7 +1482,9 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus { // Fetch the configured targets - targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), NewGatewayHTTPTransport(), true, false) + tr := NewGatewayHTTPTransport() + defer tr.CloseIdleConnections() + targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), tr, true, false) if err != nil && err != notify.ErrTargetsOffline { logger.LogIf(GlobalContext, err) return nil @@ -1605,12 +1607,8 @@ func checkConnection(endpointStr string, timeout time.Duration) error { return pErr } - tr := newCustomHTTPTransport( - &tls.Config{RootCAs: globalRootCAs}, - timeout, - 0, /* Default value */ - )() - + tr := newCustomHTTPTransport(&tls.Config{RootCAs: globalRootCAs}, timeout)() + defer tr.CloseIdleConnections() if dErr := u.DialHTTP(tr); dErr != nil { if urlErr, ok := dErr.(*url.Error); ok { // To treat "connection refused" errors as un reachable endpoint. diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go index c9d4150fd..4692d15ad 100644 --- a/cmd/bootstrap-peer-server.go +++ b/cmd/bootstrap-peer-server.go @@ -247,7 +247,7 @@ func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) { } } - trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout) + trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout) restClient, err := rest.NewClient(serverURL, trFn, newAuthToken) if err != nil { return nil, err diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 1fc665832..a3ff048f2 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -286,7 +286,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit // Use buffered channel to take care of burst sends or slow w.Write() listenCh := make(chan interface{}, 4000) - peers := getRestClients(globalEndpoints) + peers := newPeerRestClients(globalEndpoints) globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool { ev, ok := evI.(event.Event) diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go index 644375ab9..4af32cba6 100644 --- a/cmd/lock-rest-client.go +++ b/cmd/lock-rest-client.go @@ -170,7 +170,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient { } } - trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout) + trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout) restClient, err := rest.NewClient(serverURL, trFn, newAuthToken) if err != nil { logger.LogIf(GlobalContext, err) diff --git a/cmd/notification.go b/cmd/notification.go index caaf6b24d..1194dccf8 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -1171,7 +1171,7 @@ func NewNotificationSys(endpoints EndpointZones) *NotificationSys { targetList: event.NewTargetList(), bucketRulesMap: make(map[string]event.RulesMap), bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap), - peerClients: getRestClients(endpoints), + peerClients: newPeerRestClients(endpoints), } } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 451502923..ed043bab8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -27,6 +27,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" @@ -663,6 +664,10 @@ func getCpObjTagsFromHeader(ctx context.Context, r *http.Request, tags string) ( return tags, nil } +// getRemoteInstanceTransport contains a singleton roundtripper. +var getRemoteInstanceTransport http.RoundTripper +var getRemoteInstanceTransportOnce sync.Once + // Returns a minio-go Client configured to access remote host described by destDNSRecord // Applicable only in a federated deployment var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { @@ -673,7 +678,10 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, if err != nil { return nil, err } - core.SetCustomTransport(NewGatewayHTTPTransport()) + getRemoteInstanceTransportOnce.Do(func() { + getRemoteInstanceTransport = NewGatewayHTTPTransport() + }) + core.SetCustomTransport(getRemoteInstanceTransport) return core, nil } diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index b0a04075b..7a87c4c86 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -1002,7 +1002,8 @@ func getRemoteHosts(endpointZones EndpointZones) []*xnet.Host { return remoteHosts } -func getRestClients(endpoints EndpointZones) []*peerRESTClient { +// newPeerRestClients creates new peer clients. +func newPeerRestClients(endpoints EndpointZones) []*peerRESTClient { peerHosts := getRemoteHosts(endpoints) restClients := make([]*peerRESTClient, len(peerHosts)) for i, host := range peerHosts { @@ -1019,7 +1020,6 @@ func getRestClients(endpoints EndpointZones) []*peerRESTClient { // Returns a peer rest client. func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) { - scheme := "http" if globalIsSSL { scheme = "https" @@ -1039,7 +1039,7 @@ func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) { } } - trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout) + trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout) restClient, err := rest.NewClient(serverURL, trFn, newAuthToken) if err != nil { return nil, err diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 682820a5c..b708a92a5 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -201,12 +201,12 @@ func IsServerResolvable(endpoint Endpoint) error { } httpClient := &http.Client{ - Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)(), + Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)(), } + defer httpClient.CloseIdleConnections() resp, err := httpClient.Do(req) if err != nil { - httpClient.CloseIdleConnections() return err } defer xhttp.DrainBody(resp.Body) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index fd3c071c2..42009502a 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -572,7 +572,7 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient { } } - trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout) + trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout) restClient, err := rest.NewClient(serverURL, trFn, newAuthToken) if err != nil { logger.LogIf(GlobalContext, err) diff --git a/cmd/utils.go b/cmd/utils.go index da8aaf9fe..ee83ed391 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -172,8 +172,7 @@ const ( globalMaxPartID = 10000 // Default values used while communicating for internode communication. - defaultDialTimeout = 5 * time.Second - defaultDialKeepAlive = 15 * time.Second + defaultDialTimeout = 5 * time.Second ) // isMaxObjectSize - verify if max object size @@ -464,14 +463,16 @@ func newCustomDialContext(dialTimeout, dialKeepAlive time.Duration) dialContext } } -func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive time.Duration) func() *http.Transport { +func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport { // For more details about various values used here refer // https://golang.org/pkg/net/http/#Transport documentation tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, - DialContext: newCustomDialContext(dialTimeout, dialKeepAlive), - MaxIdleConnsPerHost: 256, - IdleConnTimeout: time.Minute, + DialContext: newCustomDialContext(dialTimeout, 15*time.Second), + MaxIdleConnsPerHost: 16, + MaxIdleConns: 16, + MaxConnsPerHost: 64, // This is used per drive/rpc host. More requests will block until free. + IdleConnTimeout: 1 * time.Minute, ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode. TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 10 * time.Second, @@ -493,9 +494,14 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive ti func NewGatewayHTTPTransport() *http.Transport { tr := newCustomHTTPTransport(&tls.Config{ RootCAs: globalRootCAs, - }, defaultDialTimeout, defaultDialKeepAlive)() + }, defaultDialTimeout)() // Set aggressive timeouts for gateway tr.ResponseHeaderTimeout = 30 * time.Second + + // Allow more requests to be in flight. + tr.MaxConnsPerHost = 256 + tr.MaxIdleConnsPerHost = 16 + tr.MaxIdleConns = 256 return tr } diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 1e81d638e..2a68cf75e 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -2109,10 +2109,10 @@ func (web *webAPIHandlers) LoginSTS(r *http.Request, args *LoginSTSArgs, reply * clnt := &http.Client{ Transport: NewGatewayHTTPTransport(), } + defer clnt.CloseIdleConnections() resp, err := clnt.Do(req) if err != nil { - clnt.CloseIdleConnections() return toJSONError(ctx, err) } defer xhttp.DrainBody(resp.Body)