diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 932d106d5..a3ea868c6 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -334,7 +334,8 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio r.Body = ioutil.NopCloser(bytes.NewReader(payload)) } - if cred.AccessKey == "" { + if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" { + // Anonymous checks are not meant for ListBuckets action if globalPolicySys.IsAllowed(policy.Args{ AccountName: cred.AccessKey, Action: action, @@ -378,6 +379,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio // Request is allowed return the appropriate access key. return cred.AccessKey, owner, ErrNone } + if action == policy.ListBucketVersionsAction { // In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission // verify as a fallback. diff --git a/cmd/bucket-targets.go b/cmd/bucket-targets.go index 7ea65669f..32aad0ebd 100644 --- a/cmd/bucket-targets.go +++ b/cmd/bucket-targets.go @@ -20,6 +20,7 @@ import ( "context" "net/http" "sync" + "time" minio "github.com/minio/minio-go/v7" miniogo "github.com/minio/minio-go/v7" @@ -281,8 +282,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*m creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "") getRemoteTargetInstanceTransportOnce.Do(func() { - getRemoteTargetInstanceTransport = NewGatewayHTTPTransport() + getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour) }) + core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{ Creds: creds, Secure: tcfg.Secure, diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 10ca105c7..471979548 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -771,11 +771,7 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) { } // allow transport to be HTTP/1.1 for proxying. - tr := newCustomHTTP11Transport(tlsConfig, rest.DefaultTimeout)() - - // Allow more requests to be in flight with higher response header timeout. - tr.ResponseHeaderTimeout = 30 * time.Minute - tr.MaxIdleConnsPerHost = 64 + tr := newCustomHTTPProxyTransport(tlsConfig, rest.DefaultTimeout)() proxyEps = append(proxyEps, ProxyEndpoint{ Endpoint: endpoint, diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 768ea96f9..f2c1f7189 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -449,7 +449,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s } } - defer er.deleteObject(ctx, minioMetaTmpBucket, tmpID, len(storageDisks)/2+1) + defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpID, len(storageDisks)/2+1) // Generate and write `xl.meta` generated from other disks. outDatedDisks, err = writeUniqueFileInfo(ctx, outDatedDisks, minioMetaTmpBucket, tmpID, diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index ee6b975fe..01f7676b5 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -277,7 +277,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, // Delete the tmp path later in case we fail to commit (ignore // returned errors) - this will be a no-op in case of a commit // success. - defer er.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum) + defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempUploadIDPath, writeQuorum) var partsMetadata = make([]FileInfo, len(onlineDisks)) for i := range onlineDisks { @@ -396,7 +396,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo tmpPartPath := pathJoin(tmpPart, partSuffix) // Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete. - defer er.deleteObject(ctx, minioMetaTmpBucket, tmpPart, writeQuorum) + defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum) erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) if err != nil { diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 800287c9c..7a60bb1cf 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -271,6 +271,17 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt } } +// GetAllLockers return a list of all lockers for all sets. +func (s *erasureSets) GetAllLockers() []dsync.NetLocker { + allLockers := make([]dsync.NetLocker, s.setDriveCount*s.setCount) + for i, lockers := range s.erasureLockers { + for j, locker := range lockers { + allLockers[i*s.setDriveCount+j] = locker + } + } + return allLockers +} + func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) { return func() ([]dsync.NetLocker, string) { lockers := make([]dsync.NetLocker, s.setDriveCount) diff --git a/cmd/erasure-zones.go b/cmd/erasure-zones.go index f3d80d4c0..932b9e444 100644 --- a/cmd/erasure-zones.go +++ b/cmd/erasure-zones.go @@ -33,6 +33,7 @@ import ( "github.com/minio/minio/cmd/config/storageclass" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/dsync" "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/sync/errgroup" ) @@ -92,6 +93,10 @@ func (z *erasureZones) NewNSLock(ctx context.Context, bucket string, objects ... return z.zones[0].NewNSLock(ctx, bucket, objects...) } +func (z *erasureZones) GetAllLockers() []dsync.NetLocker { + return z.zones[0].GetAllLockers() +} + func (z *erasureZones) SetDriveCount() int { return z.zones[0].SetDriveCount() } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 9c911e804..0db4cf369 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -289,7 +289,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { globalHTTPServer = httpServer globalObjLayerMutex.Unlock() - signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM) + signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) newObject, err := gw.NewGatewayLayer(globalActiveCred) if err != nil { @@ -323,8 +323,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) { } if enableIAMOps { - // Initialize IAM sys. - startBackgroundIAMLoad(GlobalContext, newObject) + // Initialize users credentials and policies in background. + go globalIAMSys.Init(GlobalContext, newObject) } if globalCacheConfig.Enabled { diff --git a/cmd/http/listen_nix.go b/cmd/http/listen_nix.go index cbda9002c..dd9cb199b 100644 --- a/cmd/http/listen_nix.go +++ b/cmd/http/listen_nix.go @@ -27,9 +27,9 @@ import ( var cfg = &tcplisten.Config{ DeferAccept: true, FastOpen: true, - // Bump up the soMaxConn value from 128 to 2048 to + // Bump up the soMaxConn value from 128 to 4096 to // handle large incoming concurrent requests. - Backlog: 2048, + Backlog: 4096, } // Unix listener with special TCP options. diff --git a/cmd/iam.go b/cmd/iam.go index 38c26e5b6..fa6bb8d69 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -408,13 +408,6 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error { return sys.store.migrateBackendFormat(ctx) } -// Loads IAM users and policies in background, any un-handled -// error means this code can potentially crash the server -// in such a situation manual intervention is necessary. -func startBackgroundIAMLoad(ctx context.Context, objAPI ObjectLayer) { - go globalIAMSys.Init(ctx, objAPI) -} - // Init - initializes config system by reading entries from config/iam func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) { if objAPI == nil { diff --git a/cmd/lock-rest-server.go b/cmd/lock-rest-server.go index d130af540..68ec14a94 100644 --- a/cmd/lock-rest-server.go +++ b/cmd/lock-rest-server.go @@ -247,6 +247,11 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error { return nil } + z, ok := objAPI.(*erasureZones) + if !ok { + return nil + } + type nlock struct { locks int writer bool @@ -265,6 +270,8 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error { } } + allLockersFn := z.GetAllLockers + // Validate if long lived locks are indeed clean. // Get list of long lived locks to check for staleness. for lendpoint, nlrips := range getLongLivedLocks(interval) { @@ -273,8 +280,7 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error { // Locks are only held on first zone, make sure that // we only look for ownership of locks from endpoints // on first zone. - for _, endpoint := range globalEndpoints[0].Endpoints { - c := newLockAPI(endpoint) + for _, c := range allLockersFn() { if !c.IsOnline() { updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer) continue @@ -292,16 +298,12 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error { cancel() if err != nil { updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer) - c.Close() continue } if !expired { updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer) } - - // Close the connection regardless of the call response. - c.Close() } // Read locks we assume quorum for be N/2 success diff --git a/cmd/logger/logger.go b/cmd/logger/logger.go index 6509165ef..3c8b3b8f8 100644 --- a/cmd/logger/logger.go +++ b/cmd/logger/logger.go @@ -19,6 +19,7 @@ package logger import ( "context" "encoding/hex" + "errors" "fmt" "go/build" "hash" @@ -60,11 +61,6 @@ var globalDeploymentID string // TimeFormat - logging time format. const TimeFormat string = "15:04:05 MST 01/02/2006" -// List of error strings to be ignored by LogIf -const ( - diskNotFoundError = "disk not found" -) - var matchingFuncNames = [...]string{ "http.HandlerFunc.ServeHTTP", "cmd.serverMain", @@ -303,7 +299,7 @@ func LogIf(ctx context.Context, err error, errKind ...interface{}) { return } - if err.Error() != diskNotFoundError { + if !errors.Is(err, context.Canceled) { logIf(ctx, err, errKind...) } } diff --git a/cmd/rest/client.go b/cmd/rest/client.go index 1c1c9dac8..ef1aeeab7 100644 --- a/cmd/rest/client.go +++ b/cmd/rest/client.go @@ -74,11 +74,10 @@ type Client struct { // Should only be modified before any calls are made. MaxErrResponseSize int64 - httpClient *http.Client - httpIdleConnsCloser func() - url *url.URL - newAuthToken func(audience string) string - connected int32 + httpClient *http.Client + url *url.URL + newAuthToken func(audience string) string + connected int32 } // URL query separator constants @@ -157,9 +156,6 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod // Close closes all idle connections of the underlying http client func (c *Client) Close() { atomic.StoreInt32(&c.connected, closed) - if c.httpIdleConnsCloser != nil { - c.httpIdleConnsCloser() - } } // NewClient - returns new REST client. @@ -169,7 +165,6 @@ func NewClient(url *url.URL, newCustomTransport func() *http.Transport, newAuthT tr := newCustomTransport() return &Client{ httpClient: &http.Client{Transport: tr}, - httpIdleConnsCloser: tr.CloseIdleConnections, url: url, newAuthToken: newAuthToken, connected: online, diff --git a/cmd/server-main.go b/cmd/server-main.go index 78cff470e..51650e36b 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -190,7 +190,7 @@ func newAllSubsystems() { globalBucketTargetSys = NewBucketTargetSys() } -func initServer(ctx context.Context, newObject ObjectLayer) (err error) { +func initServer(ctx context.Context, newObject ObjectLayer) error { // Create cancel context to control 'newRetryTimer' go routine. retryCtx, cancel := context.WithCancel(ctx) @@ -203,39 +203,6 @@ func initServer(ctx context.Context, newObject ObjectLayer) (err error) { // appropriately. This is also true for rotation of encrypted // content. txnLk := newObject.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/transaction.lock") - defer func() { - if err != nil { - var cerr config.Err - // For any config error, we don't need to drop into safe-mode - // instead its a user error and should be fixed by user. - if errors.As(err, &cerr) { - logger.FatalIf(err, "Unable to initialize the server") - return - } - - // If context was canceled - if errors.Is(err, context.Canceled) { - logger.FatalIf(err, "Server startup canceled upon user request") - return - } - } - - // Prints the formatted startup message, if err is not nil then it prints additional information as well. - printStartupMessage(getAPIEndpoints(), err) - - if globalActiveCred.Equal(auth.DefaultCredentials) { - msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred) - logger.StartupMessage(color.RedBold(msg)) - } - - <-globalOSSignalCh - }() - - // Enable background operations for erasure coding - if globalIsErasure { - initAutoHeal(ctx, newObject) - initBackgroundReplication(ctx, newObject) - } // allocate dynamic timeout once before the loop configLockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second) @@ -252,7 +219,9 @@ func initServer(ctx context.Context, newObject ObjectLayer) (err error) { // version is needed, migration is needed etc. rquorum := InsufficientReadQuorum{} wquorum := InsufficientWriteQuorum{} - for range retry.NewTimerWithJitter(retryCtx, 250*time.Millisecond, 500*time.Millisecond, retry.MaxJitter) { + + var err error + for range retry.NewTimerWithJitter(retryCtx, 500*time.Millisecond, time.Second, retry.MaxJitter) { // let one of the server acquire the lock, if not let them timeout. // which shall be retried again by this loop. if err = txnLk.GetLock(configLockTimeout); err != nil { @@ -389,7 +358,7 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) { // serverMain handler called for 'minio server' command. func serverMain(ctx *cli.Context) { - signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM) + signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) go handleSignals() @@ -509,10 +478,38 @@ func serverMain(ctx *cli.Context) { go initDataCrawler(GlobalContext, newObject) - // Initialize users credentials and policies in background. - go startBackgroundIAMLoad(GlobalContext, newObject) + // Enable background operations for erasure coding + if globalIsErasure { + initAutoHeal(GlobalContext, newObject) + initBackgroundReplication(GlobalContext, newObject) + } + + if err = initServer(GlobalContext, newObject); err != nil { + var cerr config.Err + // For any config error, we don't need to drop into safe-mode + // instead its a user error and should be fixed by user. + if errors.As(err, &cerr) { + logger.FatalIf(err, "Unable to initialize the server") + } + + // If context was canceled + if errors.Is(err, context.Canceled) { + logger.FatalIf(err, "Server startup canceled upon user request") + } + } + + // Initialize users credentials and policies in background right after config has initialized. + go globalIAMSys.Init(GlobalContext, newObject) + + // Prints the formatted startup message, if err is not nil then it prints additional information as well. + printStartupMessage(getAPIEndpoints(), err) + + if globalActiveCred.Equal(auth.DefaultCredentials) { + msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred) + logger.StartupMessage(color.RedBold(msg)) + } - initServer(GlobalContext, newObject) + <-globalOSSignalCh } // Initialize object layer with the supplied disks, objectLayer is nil upon any error. diff --git a/cmd/utils.go b/cmd/utils.go index d713b7781..19d1f7f58 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -469,8 +469,8 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: xhttp.NewInternodeDialContext(dialTimeout), - MaxIdleConnsPerHost: 16, - IdleConnTimeout: 30 * time.Second, + MaxIdleConnsPerHost: 1024, + IdleConnTimeout: 15 * time.Second, ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode. TLSHandshakeTimeout: 15 * time.Second, ExpectContinueTimeout: 15 * time.Second, @@ -490,15 +490,16 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) } } -func newCustomHTTP11Transport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport { +// Used by only proxied requests, specifically only supports HTTP/1.1 +func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport { // For more details about various values used here refer // https://golang.org/pkg/net/http/#Transport documentation tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: xhttp.NewCustomDialContext(dialTimeout), - MaxIdleConnsPerHost: 16, - IdleConnTimeout: 1 * time.Minute, - ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode. + MaxIdleConnsPerHost: 1024, + IdleConnTimeout: 15 * time.Second, + ResponseHeaderTimeout: 30 * time.Minute, // Set larger timeouts for proxied requests. TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 10 * time.Second, TLSClientConfig: tlsConfig, @@ -519,8 +520,8 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: xhttp.NewCustomDialContext(dialTimeout), - MaxIdleConnsPerHost: 16, - IdleConnTimeout: 1 * time.Minute, + MaxIdleConnsPerHost: 1024, + IdleConnTimeout: 15 * time.Second, ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode. TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 10 * time.Second, @@ -553,9 +554,8 @@ func newGatewayHTTPTransport(timeout time.Duration) *http.Transport { RootCAs: globalRootCAs, }, defaultDialTimeout)() - // Allow more requests to be in flight. + // Customize response header timeout for gateway transport. tr.ResponseHeaderTimeout = timeout - tr.MaxIdleConnsPerHost = 16 return tr } diff --git a/pkg/net/url.go b/pkg/net/url.go index 7d7e36d0c..283bf8acf 100644 --- a/pkg/net/url.go +++ b/pkg/net/url.go @@ -148,21 +148,24 @@ func IsNetworkOrHostDown(err error) bool { if errors.Is(err, context.Canceled) { return false } + // We need to figure if the error either a timeout // or a non-temporary error. e, ok := err.(net.Error) if ok { - urlErr, ok := e.(*url.Error) - if ok { + if urlErr, ok := e.(*url.Error); ok { switch urlErr.Err.(type) { case *net.DNSError, *net.OpError, net.UnknownNetworkError: return true } } + if e.Timeout() { return true } + } + ok = false // Fallback to other mechanisms. if strings.Contains(err.Error(), "Connection closed by foreign host") {