From f355374962387f4c0ec1a388eb986c85a2ef0a79 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 11 Sep 2020 23:03:08 -0700 Subject: [PATCH] add support for configurable remote transport deadline (#10447) configurable remote transport timeouts for some special cases where this value needs to be bumped to a higher value when transferring large data between federated instances. --- cmd/config-current.go | 5 +++++ cmd/config/api/api.go | 48 +++++++++++++++++++++++++++--------------- cmd/config/api/help.go | 6 ++++++ cmd/handler-api.go | 15 +++++++------ cmd/object-handlers.go | 41 ++++++++---------------------------- cmd/web-handlers.go | 4 ++-- 6 files changed, 62 insertions(+), 57 deletions(-) diff --git a/cmd/config-current.go b/cmd/config-current.go index 2416f8801..accef0a2b 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -398,6 +398,11 @@ func lookupConfigs(s config.Config, setDriveCount int) { globalAPIConfig.init(apiConfig, setDriveCount) + // Initialize remote instance transport once. + getRemoteInstanceTransportOnce.Do(func() { + getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline) + }) + if globalIsErasure { globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount) if err != nil { diff --git a/cmd/config/api/api.go b/cmd/config/api/api.go index 3ae52dbe5..f485d77ca 100644 --- a/cmd/config/api/api.go +++ b/cmd/config/api/api.go @@ -29,15 +29,17 @@ import ( // API sub-system constants const ( - apiRequestsMax = "requests_max" - apiRequestsDeadline = "requests_deadline" - apiReadyDeadline = "ready_deadline" - apiCorsAllowOrigin = "cors_allow_origin" - - EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX" - EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE" - EnvAPIReadyDeadline = "MINIO_API_READY_DEADLINE" - EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN" + apiRequestsMax = "requests_max" + apiRequestsDeadline = "requests_deadline" + apiReadyDeadline = "ready_deadline" + apiCorsAllowOrigin = "cors_allow_origin" + apiRemoteTransportDeadline = "remote_transport_deadline" + + EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX" + EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE" + EnvAPIReadyDeadline = "MINIO_API_READY_DEADLINE" + EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN" + EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE" ) // DefaultKVS - default storage class config @@ -59,15 +61,20 @@ var ( Key: apiCorsAllowOrigin, Value: "*", }, + config.KV{ + Key: apiRemoteTransportDeadline, + Value: "2h", + }, } ) // Config storage class configuration type Config struct { - APIRequestsMax int `json:"requests_max"` - APIRequestsDeadline time.Duration `json:"requests_deadline"` - APIReadyDeadline time.Duration `json:"ready_deadline"` - APICorsAllowOrigin []string `json:"cors_allow_origin"` + RequestsMax int `json:"requests_max"` + RequestsDeadline time.Duration `json:"requests_deadline"` + ReadyDeadline time.Duration `json:"ready_deadline"` + CorsAllowOrigin []string `json:"cors_allow_origin"` + RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"` } // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON. @@ -108,10 +115,17 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { } corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",") + + remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.Get(apiRemoteTransportDeadline))) + if err != nil { + return cfg, err + } + return Config{ - APIRequestsMax: requestsMax, - APIRequestsDeadline: requestsDeadline, - APIReadyDeadline: readyDeadline, - APICorsAllowOrigin: corsAllowOrigin, + RequestsMax: requestsMax, + RequestsDeadline: requestsDeadline, + ReadyDeadline: readyDeadline, + CorsAllowOrigin: corsAllowOrigin, + RemoteTransportDeadline: remoteTransportDeadline, }, nil } diff --git a/cmd/config/api/help.go b/cmd/config/api/help.go index f9072a6df..c176b60d1 100644 --- a/cmd/config/api/help.go +++ b/cmd/config/api/help.go @@ -39,5 +39,11 @@ var ( Optional: true, Type: "csv", }, + config.HelpKV{ + Key: apiRemoteTransportDeadline, + Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"`, + Optional: true, + Type: "duration", + }, } ) diff --git a/cmd/handler-api.go b/cmd/handler-api.go index 076c1e1df..be86d4960 100644 --- a/cmd/handler-api.go +++ b/cmd/handler-api.go @@ -38,10 +38,11 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) { t.mu.Lock() defer t.mu.Unlock() - t.readyDeadline = cfg.APIReadyDeadline - t.corsAllowOrigins = cfg.APICorsAllowOrigin + t.readyDeadline = cfg.ReadyDeadline + t.corsAllowOrigins = cfg.CorsAllowOrigin + var apiRequestsMaxPerNode int - if cfg.APIRequestsMax <= 0 { + if cfg.RequestsMax <= 0 { stats, err := sys.GetStats() if err != nil { return @@ -51,21 +52,23 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) { // ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size) apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*readBlockSize+blockSizeV1*2)) } else { - apiRequestsMaxPerNode = cfg.APIRequestsMax + apiRequestsMaxPerNode = cfg.RequestsMax if len(globalEndpoints.Hostnames()) > 0 { apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames()) } } t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode) - t.requestsDeadline = cfg.APIRequestsDeadline + t.requestsDeadline = cfg.RequestsDeadline } func (t *apiConfig) getCorsAllowOrigins() []string { t.mu.RLock() defer t.mu.RUnlock() - return t.corsAllowOrigins + corsAllowOrigins := make([]string, len(t.corsAllowOrigins)) + copy(corsAllowOrigins, t.corsAllowOrigins) + return corsAllowOrigins } func (t *apiConfig) getReadyDeadline() time.Duration { diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 4c378f3e6..0c19d3baa 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -711,40 +711,17 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m } // getRemoteInstanceTransport contains a singleton roundtripper. -var getRemoteInstanceTransport *http.Transport -var getRemoteInstanceTransportLongTO *http.Transport -var getRemoteInstanceTransportOnce sync.Once +var ( + getRemoteInstanceTransport *http.Transport + getRemoteInstanceTransportOnce sync.Once +) // Returns a minio-go Client configured to access remote host described by destDNSRecord // Applicable only in a federated deployment var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { - getRemoteInstanceTransportOnce.Do(func() { - getRemoteInstanceTransport = NewGatewayHTTPTransport() - getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour) - }) - - cred := getReqAccessCred(r, globalServerRegion) - // In a federated deployment, all the instances share config files - // and hence expected to have same credentials. - core, err := miniogo.NewCore(host, &miniogo.Options{ - Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""), - Secure: globalIsSSL, - Transport: getRemoteInstanceTransport, - }) - if err != nil { - return nil, err + if newObjectLayerFn() == nil { + return nil, errServerNotInitialized } - return core, nil -} - -// Returns a minio-go Client configured to access remote host described by destDNSRecord -// Applicable only in a federated deployment. -// The transport does not contain any timeout except for dialing. -func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo.Core, error) { - getRemoteInstanceTransportOnce.Do(func() { - getRemoteInstanceTransport = NewGatewayHTTPTransport() - getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour) - }) cred := getReqAccessCred(r, globalServerRegion) // In a federated deployment, all the instances share config files @@ -752,7 +729,7 @@ func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo. core, err := miniogo.NewCore(host, &miniogo.Options{ Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""), Secure: globalIsSSL, - Transport: getRemoteInstanceTransportLongTO, + Transport: getRemoteInstanceTransport, }) if err != nil { return nil, err @@ -1222,7 +1199,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // Send PutObject request to appropriate instance (in federated deployment) - core, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords)) + core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords)) if rerr != nil { writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r)) return @@ -1932,7 +1909,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt } // Send PutObject request to appropriate instance (in federated deployment) - core, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords)) + core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords)) if rerr != nil { writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r)) return diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 1e6176d6d..6a0cee208 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -427,7 +427,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r } return toJSONError(ctx, err, args.BucketName) } - core, err := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(sr)) + core, err := getRemoteInstanceClient(r, getHostFromSrv(sr)) if err != nil { return toJSONError(ctx, err, args.BucketName) } @@ -653,7 +653,7 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, } return toJSONError(ctx, err, args.BucketName) } - core, err := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(sr)) + core, err := getRemoteInstanceClient(r, getHostFromSrv(sr)) if err != nil { return toJSONError(ctx, err, args.BucketName) }