diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go index 249e0ae1f..ab9921ed0 100644 --- a/cmd/admin-handlers-config-kv.go +++ b/cmd/admin-handlers-config-kv.go @@ -19,6 +19,7 @@ package cmd import ( "bufio" "bytes" + "context" "encoding/json" "io" "net/http" @@ -29,11 +30,29 @@ import ( "github.com/minio/minio/pkg/madmin" ) +func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer { + // Get current object layer instance. + objectAPI := globalObjectAPI + if objectAPI == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return nil + } + + // Validate request signature. + adminAPIErr := checkAdminRequestAuthType(ctx, r, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return nil + } + + return objectAPI +} + // DelConfigKVHandler - DELETE /minio/admin/v2/del-config-kv func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DelConfigKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -66,6 +85,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ oldCfg := cfg.Clone() scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) for scanner.Scan() { + // Skip any empty lines if scanner.Text() == "" { continue } @@ -89,7 +109,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "SetConfigKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -123,6 +143,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ oldCfg := cfg.Clone() scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) for scanner.Scan() { + // Skip any empty lines if scanner.Text() == "" { continue } @@ -158,16 +179,22 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetConfigKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } + cfg, err := getValidConfig(objectAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + vars := mux.Vars(r) var buf = &bytes.Buffer{} key := vars["key"] if key != "" { - kvs, err := globalServerConfig.GetKVS(key) + kvs, err := cfg.GetKVS(key) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -181,7 +208,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ } } } else { - buf.WriteString(globalServerConfig.String()) + buf.WriteString(cfg.String()) } password := globalActiveCred.SecretKey @@ -197,7 +224,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ClearConfigHistoryKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -232,7 +259,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r * func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "RestoreConfigHistoryKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -260,6 +287,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r oldCfg := cfg.Clone() scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) for scanner.Scan() { + // Skip any empty lines if scanner.Text() == "" { continue } @@ -290,7 +318,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListConfigHistoryKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -314,7 +342,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "HelpConfigKVHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -340,7 +368,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "SetConfigHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } @@ -391,7 +419,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetConfigHandler") - objectAPI := validateAdminReq(ctx, w, r) + objectAPI := validateAdminReqConfigKV(ctx, w, r) if objectAPI == nil { return } diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 62f2abe6c..a067e6b1e 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -933,7 +933,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r * func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil { writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) return nil diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 9ac39cf27..a4937c429 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -90,7 +90,11 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) { globalPolicySys = NewPolicySys() globalPolicySys.Init(buckets, objLayer) - globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints) + globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints) + if err != nil { + return nil, err + } + globalNotificationSys.Init(buckets, objLayer) // Setup admin mgmt REST API handlers. diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index 72beb6227..3f5b15471 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -660,7 +660,7 @@ func (h *healSequence) traverseAndHeal() { func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error { return func() error { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -692,7 +692,7 @@ func (h *healSequence) healDiskFormat() error { } // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -712,7 +712,7 @@ func (h *healSequence) healBuckets() error { } // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -734,7 +734,7 @@ func (h *healSequence) healBuckets() error { // healBucket - traverses and heals given bucket func (h *healSequence) healBucket(bucket string) error { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -771,7 +771,7 @@ func (h *healSequence) healObject(bucket, object string) error { } // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } diff --git a/cmd/api-router.go b/cmd/api-router.go index de539691f..c1b0f38ce 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -37,8 +37,18 @@ type objectAPIHandlers struct { func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) { // Initialize API. api := objectAPIHandlers{ - ObjectAPI: newObjectLayerFn, - CacheAPI: newCacheObjectsFn, + ObjectAPI: func() ObjectLayer { + if !globalSafeMode { + return globalObjectAPI + } + return nil + }, + CacheAPI: func() CacheObjectLayer { + if !globalSafeMode { + return globalCacheObjectAPI + } + return nil + }, EncryptionEnabled: func() bool { return encryptionEnabled }, diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go index cbac2f891..b0c335174 100644 --- a/cmd/background-heal-ops.go +++ b/cmd/background-heal-ops.go @@ -107,7 +107,7 @@ func startBackgroundHealing() { var objAPI ObjectLayer for { - objAPI = newObjectLayerFn() + objAPI = globalObjectAPI if objAPI == nil { time.Sleep(time.Second) continue @@ -135,7 +135,7 @@ func initBackgroundHealing() { // failure error occurred. func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return madmin.HealResultItem{}, errServerNotInitialized } @@ -165,7 +165,7 @@ func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealRes // bghealBucket - traverses and heals given bucket func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return madmin.HealResultItem{}, errServerNotInitialized } @@ -176,7 +176,7 @@ func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (mad // bgHealObject - heal the given object and record result func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) { // Get current object layer instance. - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return madmin.HealResultItem{}, errServerNotInitialized } diff --git a/cmd/background-newdisks-heal-ops.go b/cmd/background-newdisks-heal-ops.go index 4ce8d736f..cca358c74 100644 --- a/cmd/background-newdisks-heal-ops.go +++ b/cmd/background-newdisks-heal-ops.go @@ -36,7 +36,7 @@ func monitorLocalDisksAndHeal() { // Wait until the object layer is ready var objAPI ObjectLayer for { - objAPI = newObjectLayerFn() + objAPI = globalObjectAPI if objAPI == nil { time.Sleep(time.Second) continue diff --git a/cmd/common-main.go b/cmd/common-main.go index db3bc26d4..3fd2d6f2d 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -162,8 +162,9 @@ func handleCommonEnvVars() { logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable") } - for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), config.ValueSeparator) { - if domainName != "" { + domains := env.Get(config.EnvDomain, "") + if len(domains) != 0 { + for _, domainName := range strings.Split(domains, config.ValueSeparator) { if _, ok := dns2.IsDomainName(domainName); !ok { logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName), "Invalid MINIO_DOMAIN value in environment variable") @@ -172,9 +173,9 @@ func handleCommonEnvVars() { } } - minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs) - if ok { - minioEndpoints := strings.Split(minioEndpointsEnv, config.ValueSeparator) + publicIPs := env.Get(config.EnvPublicIPs, "") + if len(publicIPs) != 0 { + minioEndpoints := strings.Split(publicIPs, config.ValueSeparator) var domainIPs = set.NewStringSet() for _, endpoint := range minioEndpoints { if net.ParseIP(endpoint) == nil { diff --git a/cmd/config-current.go b/cmd/config-current.go index 6a4bcbb82..54b40d357 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -18,7 +18,6 @@ package cmd import ( "context" - "fmt" "strings" "sync" @@ -46,6 +45,12 @@ var ( ) func validateConfig(s config.Config) error { + // Disable merging env values with config for validation. + env.SetEnvOff() + + // Enable env values upon return. + defer env.SetEnvOn() + if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil { return err } @@ -91,25 +96,23 @@ func validateConfig(s config.Config) error { return notify.TestNotificationTargets(s, GlobalServiceDoneCh, globalRootCAs) } -func lookupConfigs(s config.Config) { - var err error - +func lookupConfigs(s config.Config) (err error) { if !globalActiveCred.IsValid() { // Env doesn't seem to be set, we fallback to lookup creds from the config. globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default]) if err != nil { - logger.Fatal(err, "Invalid credentials configuration") + return config.Errorf("Invalid credentials configuration: %s", err) } } etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs) if err != nil { - logger.Fatal(err, "Unable to initialize etcd config") + return config.Errorf("Unable to initialize etcd config: %s", err) } globalEtcdClient, err = etcd.New(etcdCfg) if err != nil { - logger.Fatal(err, "Unable to initialize etcd config") + return config.Errorf("Unable to initialize etcd config: %s", err) } if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil { @@ -120,52 +123,51 @@ func lookupConfigs(s config.Config) { dns.CoreDNSPath(etcdCfg.CoreDNSPath), ) if err != nil { - logger.Fatal(err, "Unable to initialize DNS config for %s.", globalDomainNames) + return config.Errorf("Unable to initialize DNS config for %s: %s", globalDomainNames, err) } } globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default]) if err != nil { - logger.Fatal(err, "Invalid region configuration") + return config.Errorf("Invalid region configuration: %s", err) } globalWORMEnabled, err = config.LookupWorm(s[config.WormSubSys][config.Default]) if err != nil { - logger.Fatal(config.ErrInvalidWormValue(err), - "Invalid worm configuration") + return config.Errorf("Invalid worm configuration: %s", err) + } if globalIsXL { globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], globalXLSetDriveCount) if err != nil { - logger.FatalIf(err, "Unable to initialize storage class config") + return config.Errorf("Unable to initialize storage class config: %s", err) } } globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default]) if err != nil { - logger.FatalIf(err, "Unable to setup cache") + return config.Errorf("Unable to setup cache: %s", err) } if globalCacheConfig.Enabled { if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" { globalCacheKMS, err = crypto.ParseMasterKey(cacheEncKey) if err != nil { - logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err), - "Unable to setup encryption cache") + return config.Errorf("Unable to setup encryption cache: %s", err) } } } kmsCfg, err := crypto.LookupConfig(s[config.KmsVaultSubSys][config.Default]) if err != nil { - logger.FatalIf(err, "Unable to setup KMS config") + return config.Errorf("Unable to setup KMS config: %s", err) } GlobalKMS, err = crypto.NewKMS(kmsCfg) if err != nil { - logger.FatalIf(err, "Unable to setup KMS with current KMS config") + return config.Errorf("Unable to setup KMS with current KMS config: %s", err) } // Enable auto-encryption if enabled @@ -173,19 +175,19 @@ func lookupConfigs(s config.Config) { globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default]) if err != nil { - logger.FatalIf(err, "Unable to setup Compression") + return config.Errorf("Unable to setup Compression: %s", err) } globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default], NewCustomHTTPTransport(), xhttp.DrainBody) if err != nil { - logger.FatalIf(err, "Unable to initialize OpenID") + return config.Errorf("Unable to initialize OpenID: %s", err) } opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default], NewCustomHTTPTransport(), xhttp.DrainBody) if err != nil { - logger.FatalIf(err, "Unable to initialize OPA") + return config.Errorf("Unable to initialize OPA: %s", err) } globalOpenIDValidators = getOpenIDValidators(globalOpenIDConfig) @@ -194,7 +196,7 @@ func lookupConfigs(s config.Config) { globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs) if err != nil { - logger.FatalIf(err, "Unable to parse LDAP configuration") + return config.Errorf("Unable to parse LDAP configuration: %s", err) } // Load logger targets based on user's configuration @@ -202,7 +204,7 @@ func lookupConfigs(s config.Config) { loggerCfg, err := logger.LookupConfig(s) if err != nil { - logger.FatalIf(err, "Unable to initialize logger") + return config.Errorf("Unable to initialize logger: %s", err) } for _, l := range loggerCfg.HTTP { @@ -221,6 +223,8 @@ func lookupConfigs(s config.Config) { // Enable console logging logger.AddTarget(globalConsoleSys.Console()) + + return nil } var helpMap = map[string]config.HelpKV{ @@ -255,18 +259,18 @@ func GetHelp(subSys, key string, envOnly bool) (config.HelpKV, error) { } subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2) if len(subSystemValue) == 0 { - return nil, config.Error(fmt.Sprintf("invalid number of arguments %s", subSys)) + return nil, config.Errorf("invalid number of arguments %s", subSys) } if !config.SubSystems.Contains(subSystemValue[0]) { - return nil, config.Error(fmt.Sprintf("unknown sub-system %s", subSys)) + return nil, config.Errorf("unknown sub-system %s", subSys) } help := helpMap[subSystemValue[0]] if key != "" { value, ok := help[key] if !ok { - return nil, config.Error(fmt.Sprintf("unknown key %s for sub-system %s", key, subSys)) + return nil, config.Errorf("unknown key %s for sub-system %s", key, subSys) } help = config.HelpKV{ key: value, @@ -341,7 +345,9 @@ func newSrvConfig(objAPI ObjectLayer) error { srvCfg := newServerConfig() // Override any values from ENVs. - lookupConfigs(srvCfg) + if err := lookupConfigs(srvCfg); err != nil { + return err + } // hold the mutex lock before a new config is assigned. globalServerConfigMu.Lock() @@ -378,7 +384,9 @@ func loadConfig(objAPI ObjectLayer) error { } // Override any values from ENVs. - lookupConfigs(srvCfg) + if err = lookupConfigs(srvCfg); err != nil { + return err + } // hold the mutex lock before a new config is assigned. globalServerConfigMu.Lock() diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 763e51f6f..b681a7b09 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -445,7 +445,7 @@ func migrateV5ToV6() error { if cv5.Logger.ElasticSearch.URL != "" { var url *xnet.URL - url, err = xnet.ParseURL(cv5.Logger.ElasticSearch.URL) + url, err = xnet.ParseHTTPURL(cv5.Logger.ElasticSearch.URL) if err != nil { return err } diff --git a/cmd/config/cache/config.go b/cmd/config/cache/config.go index a5e1bc245..0f1d56464 100644 --- a/cmd/config/cache/config.go +++ b/cmd/config/cache/config.go @@ -89,6 +89,9 @@ func parseCacheDrives(drives []string) ([]string, error) { } for _, d := range endpoints { + if len(d) == 0 { + return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir cannot be an empty path") + } if !filepath.IsAbs(d) { return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d) } diff --git a/cmd/config/cache/config_test.go b/cmd/config/cache/config_test.go index 430de0f26..a21801f0d 100644 --- a/cmd/config/cache/config_test.go +++ b/cmd/config/cache/config_test.go @@ -30,11 +30,15 @@ func TestParseCacheDrives(t *testing.T) { expectedPatterns []string success bool }{ - // valid input + // Invalid input {"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false}, {"bucket1", []string{}, false}, + {";;;", []string{}, false}, + {",;,;,;", []string{}, false}, } + + // Valid inputs if runtime.GOOS == "windows" { testCases = append(testCases, struct { driveStr string @@ -91,8 +95,12 @@ func TestParseCacheExclude(t *testing.T) { expectedPatterns []string success bool }{ - // valid input + // Invalid input {"/home/drive1;/home/drive2;/home/drive3", []string{}, false}, + {"/", []string{}, false}, + {";;;", []string{}, false}, + + // valid input {"bucket1/*;*.png;images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true}, {"bucket1", []string{"bucket1"}, true}, } diff --git a/cmd/config/certs.go b/cmd/config/certs.go index b34e13365..340bc2701 100644 --- a/cmd/config/certs.go +++ b/cmd/config/certs.go @@ -119,8 +119,8 @@ func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) { return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data") } if x509.IsEncryptedPEMBlock(key) { - password, ok := env.Lookup(EnvCertPassword) - if !ok { + password := env.Get(EnvCertPassword, "") + if len(password) == 0 { return tls.Certificate{}, ErrSSLNoPassword(nil) } decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password)) diff --git a/cmd/config/compress/compress.go b/cmd/config/compress/compress.go index c294818f7..bc5926903 100644 --- a/cmd/config/compress/compress.go +++ b/cmd/config/compress/compress.go @@ -56,11 +56,15 @@ var ( ) // Parses the given compression exclude list `extensions` or `content-types`. -func parseCompressIncludes(includes []string) ([]string, error) { +func parseCompressIncludes(include string) ([]string, error) { + includes := strings.Split(include, config.ValueSeparator) for _, e := range includes { if len(e) == 0 { return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty") } + if e == "/" { + return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'") + } } return includes, nil } @@ -90,22 +94,21 @@ func LookupConfig(kvs config.KVS) (Config, error) { compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes)) if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" { if compressExtensions != "" { - extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator)) + extensions, err := parseCompressIncludes(compressExtensions) if err != nil { return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions) } cfg.Extensions = extensions } if compressMimeTypes != "" { - mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator)) + mimeTypes, err := parseCompressIncludes(compressMimeTypes) if err != nil { return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) } cfg.MimeTypes = mimeTypes } if compressMimeTypesLegacy != "" { - mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypesLegacy, - config.ValueSeparator)) + mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy) if err != nil { return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) } diff --git a/cmd/config/compress/compress_test.go b/cmd/config/compress/compress_test.go new file mode 100644 index 000000000..a620755d7 --- /dev/null +++ b/cmd/config/compress/compress_test.go @@ -0,0 +1,57 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package compress + +import ( + "reflect" + "testing" +) + +func TestParseCompressIncludes(t *testing.T) { + testCases := []struct { + str string + expectedPatterns []string + success bool + }{ + // invalid input + {",,,", []string{}, false}, + {"", []string{}, false}, + {",", []string{}, false}, + {"/", []string{}, false}, + {"text/*,/", []string{}, false}, + + // valid input + {".txt,.log", []string{".txt", ".log"}, true}, + {"text/*,application/json", []string{"text/*", "application/json"}, true}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.str, func(t *testing.T) { + gotPatterns, err := parseCompressIncludes(testCase.str) + if !testCase.success && err == nil { + t.Error("expected failure but success instead") + } + if testCase.success && err != nil { + t.Errorf("expected success but failed instead %s", err) + } + if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) { + t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns) + } + }) + } +} diff --git a/cmd/config/config.go b/cmd/config/config.go index 7df08996c..c1dba84fb 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -29,6 +29,12 @@ import ( // Error config error type type Error string +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type config.Error +func Errorf(format string, a ...interface{}) error { + return Error(fmt.Sprintf(format, a...)) +} + func (e Error) Error() string { return string(e) } @@ -403,10 +409,6 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error { kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator) continue } - if len(kv[1]) == 0 { - err := fmt.Sprintf("value for key '%s' cannot be empty", kv[0]) - return Error(err) - } prevK = kv[0] kvs[kv[0]] = sanitizeValue(kv[1]) } diff --git a/cmd/config/etcd/etcd.go b/cmd/config/etcd/etcd.go index 765841507..4f8d036ed 100644 --- a/cmd/config/etcd/etcd.go +++ b/cmd/config/etcd/etcd.go @@ -19,6 +19,7 @@ package etcd import ( "crypto/tls" "crypto/x509" + "fmt" "strings" "time" @@ -75,6 +76,25 @@ func New(cfg Config) (*clientv3.Client, error) { return clientv3.New(cfg.Config) } +func parseEndpoints(endpoints string) ([]string, bool, error) { + etcdEndpoints := strings.Split(endpoints, config.ValueSeparator) + + var etcdSecure bool + for _, endpoint := range etcdEndpoints { + u, err := xnet.ParseHTTPURL(endpoint) + if err != nil { + return nil, false, err + } + if etcdSecure && u.Scheme == "http" { + return nil, false, fmt.Errorf("all endpoints should be https or http: %s", endpoint) + } + // If one of the endpoint is https, we will use https directly. + etcdSecure = etcdSecure || u.Scheme == "https" + } + + return etcdEndpoints, etcdSecure, nil +} + // LookupConfig - Initialize new etcd config. func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) { cfg := Config{} @@ -96,22 +116,12 @@ func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) { return cfg, nil } - cfg.Enabled = true - etcdEndpoints := strings.Split(endpoints, config.ValueSeparator) - - var etcdSecure bool - for _, endpoint := range etcdEndpoints { - if endpoint == "" { - continue - } - u, err := xnet.ParseURL(endpoint) - if err != nil { - return cfg, err - } - // If one of the endpoint is https, we will use https directly. - etcdSecure = etcdSecure || u.Scheme == "https" + etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints) + if err != nil { + return cfg, err } + cfg.Enabled = true cfg.DialTimeout = defaultDialTimeout cfg.DialKeepAliveTime = defaultDialKeepAlive cfg.Endpoints = etcdEndpoints diff --git a/cmd/config/etcd/etcd_test.go b/cmd/config/etcd/etcd_test.go new file mode 100644 index 000000000..9b6805246 --- /dev/null +++ b/cmd/config/etcd/etcd_test.go @@ -0,0 +1,66 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "reflect" + "testing" +) + +// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs. +func TestParseEndpoints(t *testing.T) { + testCases := []struct { + s string + endpoints []string + secure bool + success bool + }{ + // Invalid inputs + {"https://localhost:2379,http://localhost:2380", nil, false, false}, + {",,,", nil, false, false}, + {"", nil, false, false}, + {"ftp://localhost:2379", nil, false, false}, + {"http://localhost:2379000", nil, false, false}, + + // Valid inputs + {"https://localhost:2379,https://localhost:2380", []string{ + "https://localhost:2379", "https://localhost:2380"}, + true, true}, + {"http://localhost:2379", []string{"http://localhost:2379"}, false, true}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.s, func(t *testing.T) { + endpoints, secure, err := parseEndpoints(testCase.s) + if err != nil && testCase.success { + t.Errorf("expected to succeed but failed with %s", err) + } + if !testCase.success && err == nil { + t.Error("expected failure but succeeded instead") + } + if testCase.success { + if !reflect.DeepEqual(endpoints, testCase.endpoints) { + t.Errorf("expected %s, got %s", testCase.endpoints, endpoints) + } + if secure != testCase.secure { + t.Errorf("expected %t, got %t", testCase.secure, secure) + } + } + }) + } +} diff --git a/cmd/config/identity/openid/jwt.go b/cmd/config/identity/openid/jwt.go index 17cb5a3c6..198e6ac4c 100644 --- a/cmd/config/identity/openid/jwt.go +++ b/cmd/config/identity/openid/jwt.go @@ -284,7 +284,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io. configURL := env.Get(EnvIdentityOpenIDURL, kv.Get(ConfigURL)) if configURL != "" { - c.URL, err = xnet.ParseURL(configURL) + c.URL, err = xnet.ParseHTTPURL(configURL) if err != nil { return c, err } @@ -315,7 +315,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io. closeRespFn: closeRespFn, } - c.JWKS.URL, err = xnet.ParseURL(jwksURL) + c.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL) if err != nil { return c, err } diff --git a/cmd/config/identity/openid/jwt_test.go b/cmd/config/identity/openid/jwt_test.go index 22978ba53..67a825363 100644 --- a/cmd/config/identity/openid/jwt_test.go +++ b/cmd/config/identity/openid/jwt_test.go @@ -53,7 +53,7 @@ func TestJWT(t *testing.T) { } } - u1, err := xnet.ParseURL("http://localhost:8443") + u1, err := xnet.ParseHTTPURL("http://localhost:8443") if err != nil { t.Fatal(err) } diff --git a/cmd/config/identity/openid/validators_test.go b/cmd/config/identity/openid/validators_test.go index d8925e3f4..27a65d0b1 100644 --- a/cmd/config/identity/openid/validators_test.go +++ b/cmd/config/identity/openid/validators_test.go @@ -87,7 +87,7 @@ func TestValidators(t *testing.T) { t.Fatalf("Unexpected number of vids %v", vids) } - u, err := xnet.ParseURL(ts.URL) + u, err := xnet.ParseHTTPURL(ts.URL) if err != nil { t.Fatal(err) } diff --git a/cmd/config/notify/parse.go b/cmd/config/notify/parse.go index ef9a75eb2..cda60a988 100644 --- a/cmd/config/notify/parse.go +++ b/cmd/config/notify/parse.go @@ -17,7 +17,6 @@ package notify import ( - "context" "crypto/tls" "crypto/x509" "strconv" @@ -61,7 +60,57 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root return nil, err } - for id, args := range GetNotifyAMQP(cfg) { + amqpTargets, err := GetNotifyAMQP(cfg[config.NotifyAMQPSubSys]) + if err != nil { + return nil, err + } + + esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys]) + if err != nil { + return nil, err + } + + kafkaTargets, err := GetNotifyKafka(cfg[config.NotifyKafkaSubSys]) + if err != nil { + return nil, err + } + + mqttTargets, err := GetNotifyMQTT(cfg[config.NotifyMQTTSubSys], rootCAs) + if err != nil { + return nil, err + } + + mysqlTargets, err := GetNotifyMySQL(cfg[config.NotifyMySQLSubSys]) + if err != nil { + return nil, err + } + + natsTargets, err := GetNotifyNATS(cfg[config.NotifyNATSSubSys]) + if err != nil { + return nil, err + } + + nsqTargets, err := GetNotifyNSQ(cfg[config.NotifyNSQSubSys]) + if err != nil { + return nil, err + } + + postgresTargets, err := GetNotifyPostgres(cfg[config.NotifyPostgresSubSys]) + if err != nil { + return nil, err + } + + redisTargets, err := GetNotifyRedis(cfg[config.NotifyRedisSubSys]) + if err != nil { + return nil, err + } + + webhookTargets, err := GetNotifyWebhook(cfg[config.NotifyWebhookSubSys], rootCAs) + if err != nil { + return nil, err + } + + for id, args := range amqpTargets { if !args.Enable { continue } @@ -78,7 +127,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyES(cfg) { + for id, args := range esTargets { if !args.Enable { continue } @@ -96,7 +145,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyKafka(cfg) { + for id, args := range kafkaTargets { if !args.Enable { continue } @@ -114,7 +163,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyMQTT(cfg, rootCAs) { + for id, args := range mqttTargets { if !args.Enable { continue } @@ -132,7 +181,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyMySQL(cfg) { + for id, args := range mysqlTargets { if !args.Enable { continue } @@ -149,7 +198,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyNATS(cfg) { + for id, args := range natsTargets { if !args.Enable { continue } @@ -166,7 +215,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyNSQ(cfg) { + for id, args := range nsqTargets { if !args.Enable { continue } @@ -183,7 +232,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyPostgres(cfg) { + for id, args := range postgresTargets { if !args.Enable { continue } @@ -200,7 +249,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyRedis(cfg) { + for id, args := range redisTargets { if !args.Enable { continue } @@ -217,7 +266,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root } } - for id, args := range GetNotifyWebhook(cfg, rootCAs) { + for id, args := range webhookTargets { if !args.Enable { continue } @@ -307,17 +356,16 @@ var ( ) // GetNotifyKafka - returns a map of registered notification 'kafka' targets -func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { +func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs, error) { kafkaTargets := make(map[string]target.KafkaArgs) - for k, kv := range mergeTargets(s[config.NotifyKafkaSubSys], target.EnvKafkaState, DefaultKafkaKVS) { + for k, kv := range mergeTargets(kafkaKVS, target.EnvKafkaState, DefaultKafkaKVS) { stateEnv := target.EnvKafkaState if k != config.Default { stateEnv = stateEnv + config.Default + k } enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -328,6 +376,9 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { brokersEnv = brokersEnv + config.Default + k } kafkaBrokers := env.Get(brokersEnv, kv.Get(target.KafkaBrokers)) + if len(kafkaBrokers) == 0 { + return nil, config.Error("kafka 'brokers' cannot be empty") + } for _, s := range strings.Split(kafkaBrokers, config.ValueSeparator) { var host *xnet.Host host, err = xnet.ParseHost(s) @@ -337,8 +388,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { brokers = append(brokers, *host) } if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvKafkaQueueLimit @@ -347,8 +397,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { } queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.KafkaQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } clientAuthEnv := target.EnvKafkaTLSClientAuth @@ -357,8 +406,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { } clientAuth, err := strconv.Atoi(env.Get(clientAuthEnv, kv.Get(target.KafkaTLSClientAuth))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } topicEnv := target.EnvKafkaTopic @@ -408,14 +456,13 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs { kafkaArgs.SASL.Password = env.Get(saslPasswordEnv, kv.Get(target.KafkaSASLPassword)) if err = kafkaArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } kafkaTargets[k] = kafkaArgs } - return kafkaTargets + return kafkaTargets, nil } // DefaultMQTTKVS - default MQTT config @@ -436,9 +483,9 @@ var ( ) // GetNotifyMQTT - returns a map of registered notification 'mqtt' targets -func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQTTArgs { +func GetNotifyMQTT(mqttKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.MQTTArgs, error) { mqttTargets := make(map[string]target.MQTTArgs) - for k, kv := range mergeTargets(s[config.NotifyMQTTSubSys], target.EnvMQTTState, DefaultMQTTKVS) { + for k, kv := range mergeTargets(mqttKVS, target.EnvMQTTState, DefaultMQTTKVS) { stateEnv := target.EnvMQTTState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -446,8 +493,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -459,8 +505,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ } brokerURL, err := xnet.ParseURL(env.Get(brokerEnv, kv.Get(target.MqttBroker))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } reconnectIntervalEnv := target.EnvMQTTReconnectInterval @@ -470,8 +515,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ reconnectInterval, err := time.ParseDuration(env.Get(reconnectIntervalEnv, kv.Get(target.MqttReconnectInterval))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } keepAliveIntervalEnv := target.EnvMQTTKeepAliveInterval @@ -481,8 +525,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ keepAliveInterval, err := time.ParseDuration(env.Get(keepAliveIntervalEnv, kv.Get(target.MqttKeepAliveInterval))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvMQTTQueueLimit @@ -491,8 +534,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ } queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MqttQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } qosEnv := target.EnvMQTTQoS @@ -503,8 +545,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ // Parse uint8 value qos, err := strconv.ParseUint(env.Get(qosEnv, kv.Get(target.MqttQoS)), 10, 8) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } topicEnv := target.EnvMQTTTopic @@ -542,12 +583,11 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ } if err = mqttArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } mqttTargets[k] = mqttArgs } - return mqttTargets + return mqttTargets, nil } // DefaultMySQLKVS - default KV for MySQL @@ -569,9 +609,9 @@ var ( ) // GetNotifyMySQL - returns a map of registered notification 'mysql' targets -func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs { +func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs, error) { mysqlTargets := make(map[string]target.MySQLArgs) - for k, kv := range mergeTargets(s[config.NotifyMySQLSubSys], target.EnvMySQLState, DefaultMySQLKVS) { + for k, kv := range mergeTargets(mysqlKVS, target.EnvMySQLState, DefaultMySQLKVS) { stateEnv := target.EnvMySQLState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -579,8 +619,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs { enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -593,8 +632,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs { host, err := xnet.ParseURL(env.Get(hostEnv, kv.Get(target.MySQLHost))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvMySQLQueueLimit @@ -603,8 +641,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs { } queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MySQLQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } formatEnv := target.EnvMySQLFormat @@ -653,12 +690,11 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs { QueueLimit: queueLimit, } if err = mysqlArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } mysqlTargets[k] = mysqlArgs } - return mysqlTargets + return mysqlTargets, nil } // DefaultNATSKVS - NATS KV for nats config. @@ -683,9 +719,9 @@ var ( ) // GetNotifyNATS - returns a map of registered notification 'nats' targets -func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { +func GetNotifyNATS(natsKVS map[string]config.KVS) (map[string]target.NATSArgs, error) { natsTargets := make(map[string]target.NATSArgs) - for k, kv := range mergeTargets(s[config.NotifyNATSSubSys], target.EnvNATSState, DefaultNATSKVS) { + for k, kv := range mergeTargets(natsKVS, target.EnvNATSState, DefaultNATSKVS) { stateEnv := target.EnvNATSState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -693,8 +729,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -707,8 +742,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { address, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NATSAddress))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } pingIntervalEnv := target.EnvNATSPingInterval @@ -718,8 +752,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { pingInterval, err := strconv.ParseInt(env.Get(pingIntervalEnv, kv.Get(target.NATSPingInterval)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvNATSQueueLimit @@ -729,8 +762,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NATSQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } secureEnv := target.EnvNATSSecure @@ -794,8 +826,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { maxPubAcksInflight, err := strconv.Atoi(env.Get(maxPubAcksInflightEnv, kv.Get(target.NATSStreamingMaxPubAcksInFlight))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } clusterIDEnv := target.EnvNATSStreamingClusterID if k != config.Default { @@ -808,13 +839,12 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs { } if err = natsArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } natsTargets[k] = natsArgs } - return natsTargets + return natsTargets, nil } // DefaultNSQKVS - NSQ KV for config @@ -832,9 +862,9 @@ var ( ) // GetNotifyNSQ - returns a map of registered notification 'nsq' targets -func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs { +func GetNotifyNSQ(nsqKVS map[string]config.KVS) (map[string]target.NSQArgs, error) { nsqTargets := make(map[string]target.NSQArgs) - for k, kv := range mergeTargets(s[config.NotifyNSQSubSys], target.EnvNSQState, DefaultNSQKVS) { + for k, kv := range mergeTargets(nsqKVS, target.EnvNSQState, DefaultNSQKVS) { stateEnv := target.EnvNSQState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -842,8 +872,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs { enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -855,8 +884,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs { } nsqdAddress, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NSQAddress))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } tlsEnableEnv := target.EnvNSQTLSEnable if k != config.Default { @@ -873,8 +901,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs { } queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NSQQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } topicEnv := target.EnvNSQTopic @@ -897,13 +924,12 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs { nsqArgs.TLS.SkipVerify = env.Get(tlsSkipVerifyEnv, kv.Get(target.NSQTLSSkipVerify)) == config.StateOn if err = nsqArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } nsqTargets[k] = nsqArgs } - return nsqTargets + return nsqTargets, nil } // DefaultPostgresKVS - default Postgres KV for server config. @@ -925,9 +951,9 @@ var ( ) // GetNotifyPostgres - returns a map of registered notification 'postgres' targets -func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs { +func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.PostgreSQLArgs, error) { psqlTargets := make(map[string]target.PostgreSQLArgs) - for k, kv := range mergeTargets(s[config.NotifyPostgresSubSys], target.EnvPostgresState, DefaultPostgresKVS) { + for k, kv := range mergeTargets(postgresKVS, target.EnvPostgresState, DefaultPostgresKVS) { stateEnv := target.EnvPostgresState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -935,8 +961,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs { enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -949,8 +974,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs { host, err := xnet.ParseHost(env.Get(hostEnv, kv.Get(target.PostgresHost))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvPostgresQueueLimit @@ -960,8 +984,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs { queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.PostgresQueueLimit))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } formatEnv := target.EnvPostgresFormat @@ -1018,12 +1041,12 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs { QueueLimit: uint64(queueLimit), } if err = psqlArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } psqlTargets[k] = psqlArgs } - return psqlTargets + + return psqlTargets, nil } // DefaultRedisKVS - default KV for redis config @@ -1041,9 +1064,9 @@ var ( ) // GetNotifyRedis - returns a map of registered notification 'redis' targets -func GetNotifyRedis(s config.Config) map[string]target.RedisArgs { +func GetNotifyRedis(redisKVS map[string]config.KVS) (map[string]target.RedisArgs, error) { redisTargets := make(map[string]target.RedisArgs) - for k, kv := range mergeTargets(s[config.NotifyRedisSubSys], target.EnvRedisState, DefaultRedisKVS) { + for k, kv := range mergeTargets(redisKVS, target.EnvRedisState, DefaultRedisKVS) { stateEnv := target.EnvRedisState if k != config.Default { stateEnv = stateEnv + config.Default + k @@ -1051,8 +1074,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs { enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -1064,8 +1086,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs { } addr, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.RedisAddress))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvRedisQueueLimit if k != config.Default { @@ -1073,8 +1094,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs { } queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.RedisQueueLimit))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } formatEnv := target.EnvRedisFormat if k != config.Default { @@ -1102,12 +1122,11 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs { QueueLimit: uint64(queueLimit), } if err = redisArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } redisTargets[k] = redisArgs } - return redisTargets + return redisTargets, nil } // DefaultWebhookKVS - default KV for webhook config @@ -1123,17 +1142,16 @@ var ( ) // GetNotifyWebhook - returns a map of registered notification 'webhook' targets -func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target.WebhookArgs { +func GetNotifyWebhook(webhookKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.WebhookArgs, error) { webhookTargets := make(map[string]target.WebhookArgs) - for k, kv := range mergeTargets(s[config.NotifyWebhookSubSys], target.EnvWebhookState, DefaultWebhookKVS) { + for k, kv := range mergeTargets(webhookKVS, target.EnvWebhookState, DefaultWebhookKVS) { stateEnv := target.EnvWebhookState if k != config.Default { stateEnv = stateEnv + config.Default + k } enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -1142,10 +1160,9 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target if k != config.Default { urlEnv = urlEnv + config.Default + k } - url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint))) + url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvWebhookQueueLimit if k != config.Default { @@ -1153,8 +1170,7 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target } queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.WebhookQueueLimit))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueDirEnv := target.EnvWebhookQueueDir if k != config.Default { @@ -1174,12 +1190,11 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target QueueLimit: uint64(queueLimit), } if err = webhookArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } webhookTargets[k] = webhookArgs } - return webhookTargets + return webhookTargets, nil } // DefaultESKVS - default KV config for Elasticsearch target @@ -1196,17 +1211,16 @@ var ( ) // GetNotifyES - returns a map of registered notification 'elasticsearch' targets -func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs { +func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchArgs, error) { esTargets := make(map[string]target.ElasticsearchArgs) - for k, kv := range mergeTargets(s[config.NotifyESSubSys], target.EnvElasticState, DefaultESKVS) { + for k, kv := range mergeTargets(esKVS, target.EnvElasticState, DefaultESKVS) { stateEnv := target.EnvElasticState if k != config.Default { stateEnv = stateEnv + config.Default + k } enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue @@ -1217,10 +1231,9 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs { urlEnv = urlEnv + config.Default + k } - url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.ElasticURL))) + url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.ElasticURL))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } queueLimitEnv := target.EnvElasticQueueLimit @@ -1230,8 +1243,7 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs { queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.ElasticQueueLimit))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } formatEnv := target.EnvElasticFormat @@ -1258,12 +1270,11 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs { QueueLimit: uint64(queueLimit), } if err = esArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } esTargets[k] = esArgs } - return esTargets + return esTargets, nil } // DefaultAMQPKVS - default KV for AMQP config @@ -1287,30 +1298,27 @@ var ( ) // GetNotifyAMQP - returns a map of registered notification 'amqp' targets -func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs { +func GetNotifyAMQP(amqpKVS map[string]config.KVS) (map[string]target.AMQPArgs, error) { amqpTargets := make(map[string]target.AMQPArgs) - for k, kv := range mergeTargets(s[config.NotifyAMQPSubSys], target.EnvAMQPState, DefaultAMQPKVS) { + for k, kv := range mergeTargets(amqpKVS, target.EnvAMQPState, DefaultAMQPKVS) { stateEnv := target.EnvAMQPState if k != config.Default { stateEnv = stateEnv + config.Default + k } enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } if !enabled { continue } - urlEnv := target.EnvAMQPURL if k != config.Default { urlEnv = urlEnv + config.Default + k } url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.AmqpURL))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } deliveryModeEnv := target.EnvAMQPDeliveryMode if k != config.Default { @@ -1318,8 +1326,7 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs { } deliveryMode, err := strconv.Atoi(env.Get(deliveryModeEnv, kv.Get(target.AmqpDeliveryMode))) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } exchangeEnv := target.EnvAMQPExchange if k != config.Default { @@ -1367,8 +1374,7 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs { } queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.AmqpQueueLimit)), 10, 64) if err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } amqpArgs := target.AMQPArgs{ Enable: enabled, @@ -1387,10 +1393,9 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs { QueueLimit: queueLimit, } if err = amqpArgs.Validate(); err != nil { - logger.LogIf(context.Background(), err) - continue + return nil, err } amqpTargets[k] = amqpArgs } - return amqpTargets + return amqpTargets, nil } diff --git a/cmd/config/policy/opa/config.go b/cmd/config/policy/opa/config.go index 5fe6eb13f..0562c89ff 100644 --- a/cmd/config/policy/opa/config.go +++ b/cmd/config/policy/opa/config.go @@ -124,7 +124,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io. authToken = env.Get(EnvPolicyOpaAuthToken, kv.Get(AuthToken)) } - u, err := xnet.ParseURL(opaURL) + u, err := xnet.ParseHTTPURL(opaURL) if err != nil { return args, err } diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go index 9590282c3..393a3617e 100644 --- a/cmd/crypto/config.go +++ b/cmd/crypto/config.go @@ -21,6 +21,7 @@ import ( "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/env" + xnet "github.com/minio/minio/pkg/net" ) // KMSConfig has the KMS config for hashicorp vault @@ -148,9 +149,21 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) { if !stateBool { return kmsCfg, nil } - vcfg := VaultConfig{} - // Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present - vcfg.Endpoint = env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint)) + vcfg := VaultConfig{ + Auth: VaultAuth{ + Type: "approle", + }, + } + endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint)) + if endpointStr != "" { + // Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present + endpoint, err := xnet.ParseHTTPURL(endpointStr) + if err != nil { + return kmsCfg, err + } + endpointStr = endpoint.String() + } + vcfg.Endpoint = endpointStr vcfg.CAPath = env.Get(EnvKMSVaultCAPath, kvs.Get(KMSVaultCAPath)) vcfg.Auth.Type = env.Get(EnvKMSVaultAuthType, kvs.Get(KMSVaultAuthType)) vcfg.Auth.AppRole.ID = env.Get(EnvKMSVaultAppRoleID, kvs.Get(KMSVaultAppRoleID)) @@ -177,7 +190,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) { // NewKMS - initialize a new KMS. func NewKMS(cfg KMSConfig) (kms KMS, err error) { // Lookup KMS master keys - only available through ENV. - if masterKeyLegacy, ok := env.Lookup(EnvKMSMasterKeyLegacy); ok { + if masterKeyLegacy := env.Get(EnvKMSMasterKeyLegacy, ""); len(masterKeyLegacy) != 0 { if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time") } @@ -185,7 +198,7 @@ func NewKMS(cfg KMSConfig) (kms KMS, err error) { if err != nil { return kms, err } - } else if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok { + } else if masterKey := env.Get(EnvKMSMasterKey, ""); len(masterKey) != 0 { if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time") } diff --git a/cmd/crypto/legacy.go b/cmd/crypto/legacy.go index 599555872..9963b691a 100644 --- a/cmd/crypto/legacy.go +++ b/cmd/crypto/legacy.go @@ -22,6 +22,7 @@ import ( "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/env" + xnet "github.com/minio/minio/pkg/net" ) const ( @@ -119,9 +120,16 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) { if err != nil { return KMSConfig{}, err } + cfg := KMSConfig{ AutoEncryption: autoBool, + Vault: VaultConfig{ + Auth: VaultAuth{ + Type: "approle", + }, + }, } + // Assume default as "on" for legacy config since we didn't have a _STATE // flag to turn it off, but we should honor it nonetheless to turn it off // if the vault endpoint is down and there is no way to start the server. @@ -132,29 +140,37 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) { if !stateBool { return cfg, nil } - vcfg := VaultConfig{} - // Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present - vcfg.Endpoint = env.Get(EnvVaultEndpoint, kvs.Get(KMSVaultEndpoint)) - vcfg.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath)) - vcfg.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType)) - vcfg.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID)) - vcfg.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret)) - vcfg.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName)) - vcfg.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace)) + + endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint)) + if endpointStr != "" { + // Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present + endpoint, err := xnet.ParseHTTPURL(endpointStr) + if err != nil { + return cfg, err + } + endpointStr = endpoint.String() + } + + cfg.Vault.Endpoint = endpointStr + cfg.Vault.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath)) + cfg.Vault.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType)) + cfg.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID)) + cfg.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret)) + cfg.Vault.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName)) + cfg.Vault.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace)) keyVersion := env.Get(EnvVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)) if keyVersion != "" { - vcfg.Key.Version, err = strconv.Atoi(keyVersion) + cfg.Vault.Key.Version, err = strconv.Atoi(keyVersion) if err != nil { return cfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion) } } - if err = vcfg.Verify(); err != nil { + if err = cfg.Vault.Verify(); err != nil { return cfg, err } - cfg.Vault = vcfg return cfg, nil } diff --git a/cmd/daily-lifecycle-ops.go b/cmd/daily-lifecycle-ops.go index ce8463809..0d99fa638 100644 --- a/cmd/daily-lifecycle-ops.go +++ b/cmd/daily-lifecycle-ops.go @@ -55,7 +55,7 @@ func startDailyLifecycle() { // Wait until the object API is ready for { - objAPI = newObjectLayerFn() + objAPI = globalObjectAPI if objAPI == nil { time.Sleep(time.Second) continue diff --git a/cmd/disk-cache.go b/cmd/disk-cache.go index 659e0224c..75531658e 100644 --- a/cmd/disk-cache.go +++ b/cmd/disk-cache.go @@ -553,23 +553,23 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec migrating: migrateSw, migMutex: sync.Mutex{}, GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { - return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts) + return globalObjectAPI.GetObjectInfo(ctx, bucket, object, opts) }, GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) { - return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts) + return globalObjectAPI.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts) }, DeleteObjectFn: func(ctx context.Context, bucket, object string) error { - return newObjectLayerFn().DeleteObject(ctx, bucket, object) + return globalObjectAPI.DeleteObject(ctx, bucket, object) }, DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) { errs := make([]error, len(objects)) for idx, object := range objects { - errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object) + errs[idx] = globalObjectAPI.DeleteObject(ctx, bucket, object) } return errs, nil }, PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { - return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts) + return globalObjectAPI.PutObject(ctx, bucket, object, data, opts) }, } if migrateSw { diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 80e5537d7..9437451fc 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -32,7 +32,6 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/v6/pkg/set" "github.com/minio/minio/cmd/config" - "github.com/minio/minio/cmd/config/etcd" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/mountinfo" @@ -590,10 +589,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, return serverAddr, endpoints, setupType, err } - _, dok := env.Lookup(config.EnvDomain) - _, eok := env.Lookup(etcd.EnvEtcdEndpoints) - _, iok := env.Lookup(config.EnvPublicIPs) - if dok && eok && !iok { + publicIPs := env.Get(config.EnvPublicIPs, "") + if len(publicIPs) == 0 { updateDomainIPs(uniqueArgs) } diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index ef7123f6d..90085c160 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -374,15 +374,6 @@ func parseGatewaySSE(s string) (gatewaySSE, error) { // handle gateway env vars func handleGatewayEnvVars() { - gwsseVal, ok := env.Lookup("MINIO_GATEWAY_SSE") - if ok { - var err error - GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal) - if err != nil { - logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal) - } - } - accessKey := env.Get(config.EnvAccessKey, "") secretKey := env.Get(config.EnvSecretKey, "") cred, err := auth.CreateCredentials(accessKey, secretKey) @@ -391,4 +382,13 @@ func handleGatewayEnvVars() { "Unable to validate credentials inherited from the shell environment") } globalActiveCred = cred + + gwsseVal := env.Get("MINIO_GATEWAY_SSE", "") + if len(gwsseVal) != 0 { + var err error + GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal) + if err != nil { + logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal) + } + } } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 8864f54ac..040cda669 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -206,7 +206,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) { srvCfg := newServerConfig() // Override any values from ENVs. - lookupConfigs(srvCfg) + if err := lookupConfigs(srvCfg); err != nil { + logger.FatalIf(err, "Unable to initialize server config") + } // hold the mutex lock before a new config is assigned. globalServerConfigMu.Lock() @@ -232,7 +234,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Create a new config system. globalConfigSys = NewConfigSys() - // Load globalServerConfig from etcd + // Load globalServerConfig from disk logger.LogIf(context.Background(), globalConfigSys.Init(newObject)) // Start watching disk for reloading config, this @@ -267,7 +269,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) { globalLifecycleSys = NewLifecycleSys() // Create new notification system. - globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints) + globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints) + if err != nil { + logger.FatalIf(err, "Unable to initialize notification system") + } // Verify if object layer supports // - encryption diff --git a/cmd/gateway-startup-msg.go b/cmd/gateway-startup-msg.go index 86442d44d..3e12ed68e 100644 --- a/cmd/gateway-startup-msg.go +++ b/cmd/gateway-startup-msg.go @@ -28,9 +28,8 @@ import ( func printGatewayStartupMessage(apiEndPoints []string, backendType string) { strippedAPIEndpoints := stripStandardPorts(apiEndPoints) // If cache layer is enabled, print cache capacity. - cacheObjectAPI := newCacheObjectsFn() - if cacheObjectAPI != nil { - printCacheStorageInfo(cacheObjectAPI.StorageInfo(context.Background())) + if globalCacheObjectAPI != nil { + printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background())) } // Prints credential. printGatewayCommonMsg(strippedAPIEndpoints) diff --git a/cmd/global-heal.go b/cmd/global-heal.go index 5cbab051a..e52922300 100644 --- a/cmd/global-heal.go +++ b/cmd/global-heal.go @@ -156,7 +156,7 @@ func execLeaderTasks(sets *xlSets) { func startGlobalHeal() { var objAPI ObjectLayer for { - objAPI = newObjectLayerFn() + objAPI = globalObjectAPI if objAPI == nil { time.Sleep(time.Second) continue diff --git a/cmd/globals.go b/cmd/globals.go index 0dec7d2eb..4ffbd161d 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -251,6 +251,11 @@ var ( globalBackgroundHealRoutine *healRoutine globalBackgroundHealState *allHealState + // Only enabled when one of the sub-systems fail + // to initialize, this allows for administrators to + // fix the system. + globalSafeMode bool + // Add new variable global values here. ) diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index b89d71769..f77b420d7 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -52,9 +52,9 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "LivenessCheckHandler") - objLayer := newObjectLayerFn() + objLayer := globalObjectAPI // Service not initialized yet - if objLayer == nil { + if objLayer == nil || globalSafeMode { // Respond with 200 OK while server initializes to ensure a distributed cluster // is able to start on orchestration platforms like Docker Swarm. // Refer https://github.com/minio/minio/issues/8140 for more details. diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go index 5b3c5272f..1a34b968b 100644 --- a/cmd/iam-object-store.go +++ b/cmd/iam-object-store.go @@ -47,7 +47,7 @@ func (iamOS *IAMObjectStore) getObjectAPI() ObjectLayer { if iamOS.objAPI != nil { return iamOS.objAPI } - return newObjectLayerFn() + return globalObjectAPI } func (iamOS *IAMObjectStore) setObjectAPI(objAPI ObjectLayer) { diff --git a/cmd/iam.go b/cmd/iam.go index 1bd71eff6..1e15fdd1d 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -406,7 +406,7 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error { // DeletePolicy - deletes a canned policy from backend or etcd. func (sys *IAMSys) DeletePolicy(policyName string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -431,7 +431,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error { // InfoPolicy - expands the canned policy into its JSON structure. func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return nil, errServerNotInitialized } @@ -448,7 +448,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) { // ListPolicies - lists all canned policies. func (sys *IAMSys) ListPolicies() (map[string][]byte, error) { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return nil, errServerNotInitialized } @@ -471,7 +471,7 @@ func (sys *IAMSys) ListPolicies() (map[string][]byte, error) { // SetPolicy - sets a new name policy. func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -492,7 +492,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error { // DeleteUser - delete user (only for long-term users not STS users). func (sys *IAMSys) DeleteUser(accessKey string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -521,7 +521,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error { // SetTempUser - set temporary user credentials, these credentials have an expiry. func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -561,7 +561,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa // ListUsers - list all users. func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return nil, errServerNotInitialized } @@ -587,7 +587,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) { // GetUserInfo - get info on a user. func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return u, errServerNotInitialized } @@ -617,7 +617,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) { // SetUserStatus - sets current user status, supports disabled or enabled. func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -653,7 +653,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) // SetUser - set user credentials and policy. func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -685,7 +685,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error { // SetUserSecretKey - sets user secret key func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -724,7 +724,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) { // AddUsersToGroup - adds users to a group, creating the group if // needed. No error if user(s) already are in the group. func (sys *IAMSys) AddUsersToGroup(group string, members []string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -782,7 +782,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error { // RemoveUsersFromGroup - remove users from group. If no users are // given, and the group is empty, deletes the group as well. func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -863,7 +863,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error { // SetGroupStatus - enable/disabled a group func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -952,7 +952,7 @@ func (sys *IAMSys) ListGroups() (r []string, err error) { // PolicyDB. This function applies only long-term users. For STS // users, policy is set directly by called sys.policyDBSet(). func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error { - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return errServerNotInitialized } @@ -1007,7 +1007,7 @@ func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) { return nil, errInvalidArgument } - objectAPI := newObjectLayerFn() + objectAPI := globalObjectAPI if objectAPI == nil { return nil, errServerNotInitialized } diff --git a/cmd/lifecycle.go b/cmd/lifecycle.go index 938520325..aa39ac217 100644 --- a/cmd/lifecycle.go +++ b/cmd/lifecycle.go @@ -59,7 +59,7 @@ func (sys *LifecycleSys) Get(bucketName string) (lifecycle lifecycle.Lifecycle, if globalIsGateway { // When gateway is enabled, no cached value // is used to validate life cycle policies. - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { return } diff --git a/cmd/metrics.go b/cmd/metrics.go index 4be5b05cb..2e27747d8 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -83,9 +83,9 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) { minioVersionInfo.WithLabelValues(Version, CommitID).Set(float64(1.0)) // Fetch disk space info - objLayer := newObjectLayerFn() + objLayer := globalObjectAPI // Service not initialized yet - if objLayer == nil { + if objLayer == nil || globalSafeMode { return } diff --git a/cmd/notification.go b/cmd/notification.go index 12ea4459e..2bb0621ef 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -1098,10 +1098,10 @@ func (sys *NotificationSys) NetworkInfo() []madmin.ServerNetworkHardwareInfo { } // NewNotificationSys - creates new notification system object. -func NewNotificationSys(cfg config.Config, endpoints EndpointList) *NotificationSys { +func NewNotificationSys(cfg config.Config, endpoints EndpointList) (*NotificationSys, error) { targetList, err := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, globalRootCAs) if err != nil { - logger.FatalIf(err, "Unable to start notification sub system") + return nil, config.Errorf("Unable to start notification sub system: %s", err) } remoteHosts := getRemoteHosts(endpoints) @@ -1113,7 +1113,7 @@ func NewNotificationSys(cfg config.Config, endpoints EndpointList) *Notification bucketRulesMap: make(map[string]event.RulesMap), bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap), peerClients: remoteClients, - } + }, nil } type eventArgs struct { diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index e9aea37b7..d27426a22 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -44,7 +44,7 @@ const ( // Global object layer mutex, used for safely updating object layer. var globalObjLayerMutex *sync.RWMutex -// Global object layer, only accessed by newObjectLayerFn(). +// Global object layer, only accessed by globalObjectAPI. var globalObjectAPI ObjectLayer //Global cacheObjects, only accessed by newCacheObjectsFn(). diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 7c3f1c8f6..285330154 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -46,8 +46,8 @@ func getServerInfo() (*ServerInfoData, error) { return nil, errServerNotInitialized } - objLayer := newObjectLayerFn() - if objLayer == nil { + objLayer := globalObjectAPI + if objLayer == nil || globalSafeMode { return nil, errServerNotInitialized } @@ -166,7 +166,7 @@ func (s *peerRESTServer) DeletePolicyHandler(w http.ResponseWriter, r *http.Requ return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -194,7 +194,7 @@ func (s *peerRESTServer) LoadPolicyHandler(w http.ResponseWriter, r *http.Reques return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -222,7 +222,7 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -251,7 +251,7 @@ func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Reques return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -279,7 +279,7 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request) return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -329,7 +329,7 @@ func (s *peerRESTServer) LoadGroupHandler(w http.ResponseWriter, r *http.Request return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return @@ -539,7 +539,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ return } - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI == nil { s.writeErrorResponse(w, errServerNotInitialized) return diff --git a/cmd/policy.go b/cmd/policy.go index 7e1e5ba7c..89c987d1c 100644 --- a/cmd/policy.go +++ b/cmd/policy.go @@ -69,7 +69,7 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool { if globalIsGateway { // When gateway is enabled, no cached value // is used to validate bucket policies. - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI != nil { config, err := objAPI.GetBucketPolicy(context.Background(), args.BucketName) if err == nil { diff --git a/cmd/routers.go b/cmd/routers.go index 2350244c5..16e8cc74a 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -22,17 +22,6 @@ import ( "github.com/gorilla/mux" ) -func newObjectLayerFn() (layer ObjectLayer) { - globalObjLayerMutex.RLock() - layer = globalObjectAPI - globalObjLayerMutex.RUnlock() - return -} - -func newCacheObjectsFn() CacheObjectLayer { - return globalCacheObjectAPI -} - // Composed function registering routers for only distributed XL setup. func registerDistXLRouters(router *mux.Router, endpoints EndpointList) { // Register storage rpc router only if its a distributed setup. diff --git a/cmd/server-main.go b/cmd/server-main.go index d04f76657..1468cd003 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -131,11 +131,8 @@ EXAMPLES: // Checks if endpoints are either available through environment // or command line, returns false if both fails. func endpointsPresent(ctx *cli.Context) bool { - _, ok := env.Lookup(config.EnvEndpoints) - if !ok { - ok = ctx.Args().Present() - } - return ok + endpoints := env.Get(config.EnvEndpoints, strings.Join(ctx.Args(), config.ValueSeparator)) + return len(endpoints) != 0 } func serverHandleCmdArgs(ctx *cli.Context) { @@ -195,6 +192,55 @@ func serverHandleEnvVars() { } +func initAllSubsystems(newObject ObjectLayer) { + // Create a new config system. + globalConfigSys = NewConfigSys() + + // Initialize config system. + if err := globalConfigSys.Init(newObject); err != nil { + logger.Fatal(err, "Unable to initialize config system") + } + + // Create new IAM system. + globalIAMSys = NewIAMSys() + + if err := globalIAMSys.Init(newObject); err != nil { + logger.Fatal(err, "Unable to initialize IAM system") + } + + buckets, err := newObject.ListBuckets(context.Background()) + if err != nil { + logger.Fatal(err, "Unable to list buckets") + } + + // Create new notification system and initialize notification targets + globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints) + if err != nil { + logger.Fatal(err, "Unable to initialize notification targets") + } + + // Initialize notification system. + if err = globalNotificationSys.Init(buckets, newObject); err != nil { + logger.Fatal(err, "Unable to initialize notification system") + } + + // Create new policy system. + globalPolicySys = NewPolicySys() + + // Initialize policy system. + if err = globalPolicySys.Init(buckets, newObject); err != nil { + logger.Fatal(err, "Unable to initialize policy system") + } + + // Create new lifecycle system. + globalLifecycleSys = NewLifecycleSys() + + // Initialize lifecycle system. + if err = globalLifecycleSys.Init(buckets, newObject); err != nil { + logger.Fatal(err, "Unable to initialize lifecycle system") + } +} + // serverMain handler called for 'minio server' command. func serverMain(ctx *cli.Context) { if ctx.Args().First() == "help" || !endpointsPresent(ctx) { @@ -304,13 +350,8 @@ func serverMain(ctx *cli.Context) { // Re-enable logging logger.Disable = false - // Create a new config system. - globalConfigSys = NewConfigSys() - - // Initialize config system. - if err = globalConfigSys.Init(newObject); err != nil { - logger.Fatal(err, "Unable to initialize config system") - } + // Validate and initialize all subsystems. + initAllSubsystems(newObject) if globalCacheConfig.Enabled { logger.StartupMessage(color.Red(color.Bold("Disk caching is recommended only for gateway deployments"))) @@ -320,46 +361,6 @@ func serverMain(ctx *cli.Context) { logger.FatalIf(err, "Unable to initialize disk caching") } - // Create new IAM system. - globalIAMSys = NewIAMSys() - if err = globalIAMSys.Init(newObject); err != nil { - logger.Fatal(err, "Unable to initialize IAM system") - } - - buckets, err := newObject.ListBuckets(context.Background()) - if err != nil { - logger.Fatal(err, "Unable to list buckets on your backend") - } - - // Create new policy system. - globalPolicySys = NewPolicySys() - - // Initialize policy system. - if err = globalPolicySys.Init(buckets, newObject); err != nil { - logger.Fatal(err, "Unable to initialize policy system") - } - - // Create new lifecycle system. - globalLifecycleSys = NewLifecycleSys() - - // Initialize lifecycle system. - if err = globalLifecycleSys.Init(buckets, newObject); err != nil { - logger.Fatal(err, "Unable to initialize lifecycle system") - } - - // Create new notification system. - globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints) - - // Initialize notification system. - if err = globalNotificationSys.Init(buckets, newObject); err != nil { - logger.Fatal(err, "Unable to initialize notification system") - } - - // Verify if object layer supports - // - encryption - // - compression - verifyObjectLayerFeatures("server", newObject) - initDailyLifecycle() if globalIsXL { @@ -368,9 +369,7 @@ func serverMain(ctx *cli.Context) { initGlobalHeal() } - globalObjLayerMutex.Lock() globalObjectAPI = newObject - globalObjLayerMutex.Unlock() // Prints the formatted startup message once object layer is initialized. printStartupMessage(getAPIEndpoints()) diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 4a8d733d7..b2f813cfe 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -50,12 +50,11 @@ func printStartupMessage(apiEndPoints []string) { strippedAPIEndpoints := stripStandardPorts(apiEndPoints) // If cache layer is enabled, print cache capacity. - cacheObjectAPI := newCacheObjectsFn() - if cacheObjectAPI != nil { - printCacheStorageInfo(cacheObjectAPI.StorageInfo(context.Background())) + if globalCacheObjectAPI != nil { + printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background())) } // Object layer is initialized then print StorageInfo. - objAPI := newObjectLayerFn() + objAPI := globalObjectAPI if objAPI != nil { printStorageInfo(objAPI.StorageInfo(context.Background())) } @@ -97,7 +96,7 @@ func stripStandardPorts(apiEndpoints []string) (newAPIEndpoints []string) { newAPIEndpoints = make([]string, len(apiEndpoints)) // Check all API endpoints for standard ports and strip them. for i, apiEndpoint := range apiEndpoints { - u, err := xnet.ParseURL(apiEndpoint) + u, err := xnet.ParseHTTPURL(apiEndpoint) if err != nil { newAPIEndpoints[i] = apiEndpoint continue diff --git a/cmd/signals.go b/cmd/signals.go index d0f73e344..d5eb72600 100644 --- a/cmd/signals.go +++ b/cmd/signals.go @@ -55,7 +55,7 @@ func handleSignals() { // send signal to various go-routines that they need to quit. close(GlobalServiceDoneCh) - if objAPI := newObjectLayerFn(); objAPI != nil { + if objAPI := globalObjectAPI; objAPI != nil { oerr = objAPI.Shutdown(context.Background()) logger.LogIf(context.Background(), oerr) } @@ -66,7 +66,7 @@ func handleSignals() { for { select { case err := <-globalHTTPServerErrorCh: - if objAPI := newObjectLayerFn(); objAPI != nil { + if objAPI := globalObjectAPI; objAPI != nil { objAPI.Shutdown(context.Background()) } if err != nil { diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go index ccdb558e7..1a542fc49 100644 --- a/cmd/storage-rest_test.go +++ b/cmd/storage-rest_test.go @@ -498,7 +498,7 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES router := mux.NewRouter() httpServer := httptest.NewServer(router) - url, err := xnet.ParseURL(httpServer.URL) + url, err := xnet.ParseHTTPURL(httpServer.URL) if err != nil { t.Fatalf("unexpected error %v", err) } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 96ae4d84c..f73d5bfa7 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -370,7 +370,11 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer { globalPolicySys = NewPolicySys() globalPolicySys.Init(buckets, objLayer) - globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks) + globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks) + if err != nil { + t.Fatalf("Unable to initialize notification system %s", err) + } + globalNotificationSys.Init(buckets, objLayer) globalLifecycleSys = NewLifecycleSys() @@ -1636,7 +1640,10 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro globalIAMSys.Init(xl) globalPolicySys = NewPolicySys() - globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints) + globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints) + if err != nil { + return xl, err + } return xl, nil } @@ -2168,8 +2175,18 @@ func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFuncti // to underlying cache layer to manage object layer operation and disk caching // operation api := objectAPIHandlers{ - ObjectAPI: newObjectLayerFn, - CacheAPI: newCacheObjectsFn, + ObjectAPI: func() ObjectLayer { + if !globalSafeMode { + return globalObjectAPI + } + return nil + }, + CacheAPI: func() CacheObjectLayer { + if !globalSafeMode { + return globalCacheObjectAPI + } + return nil + }, EncryptionEnabled: func() bool { return true }, } diff --git a/cmd/web-router.go b/cmd/web-router.go index 1cf9b74b6..f8362ce16 100644 --- a/cmd/web-router.go +++ b/cmd/web-router.go @@ -62,8 +62,18 @@ const specialAssets = "index_bundle.*.js|loader.css|logo.svg|firefox.png|safari. func registerWebRouter(router *mux.Router) error { // Initialize Web. web := &webAPIHandlers{ - ObjectAPI: newObjectLayerFn, - CacheAPI: newCacheObjectsFn, + ObjectAPI: func() ObjectLayer { + if !globalSafeMode { + return globalObjectAPI + } + return nil + }, + CacheAPI: func() CacheObjectLayer { + if !globalSafeMode { + return globalCacheObjectAPI + } + return nil + }, } // Initialize a new json2 codec. diff --git a/pkg/env/env.go b/pkg/env/env.go index 1b3c7a56c..7e1f18c25 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -3,26 +3,47 @@ package env import ( "os" "strings" + "sync" ) +var ( + privateMutex sync.RWMutex + envOff bool +) + +// SetEnvOff - turns off env lookup +func SetEnvOff() { + privateMutex.Lock() + defer privateMutex.Unlock() + + envOff = true +} + +// SetEnvOn - turns on env lookup +func SetEnvOn() { + privateMutex.Lock() + defer privateMutex.Unlock() + + envOff = false +} + // Get retrieves the value of the environment variable named // by the key. If the variable is present in the environment the // value (which may be empty) is returned. Otherwise it returns // the specified default value. func Get(key, defaultValue string) string { + privateMutex.RLock() + ok := envOff + privateMutex.RUnlock() + if ok { + return defaultValue + } if v, ok := os.LookupEnv(key); ok { return v } return defaultValue } -// Lookup retrieves the value of the environment variable named -// by the key. If the variable is present in the environment the -// value (which may be empty) is returned and the boolean is true. -// Otherwise the returned value will be empty and the boolean will -// be false. -func Lookup(key string) (string, bool) { return os.LookupEnv(key) } - // List all envs with a given prefix. func List(prefix string) (envs []string) { for _, env := range os.Environ() { diff --git a/pkg/event/target/webhook.go b/pkg/event/target/webhook.go index d33cda852..aae8aa2ee 100644 --- a/pkg/event/target/webhook.go +++ b/pkg/event/target/webhook.go @@ -98,7 +98,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error { if target.store != nil { return target.store.Put(eventData) } - u, pErr := xnet.ParseURL(target.args.Endpoint.String()) + u, pErr := xnet.ParseHTTPURL(target.args.Endpoint.String()) if pErr != nil { return pErr } @@ -153,7 +153,7 @@ func (target *WebhookTarget) send(eventData event.Event) error { // Send - reads an event from store and sends it to webhook. func (target *WebhookTarget) Send(eventKey string) error { - u, pErr := xnet.ParseURL(target.args.Endpoint.String()) + u, pErr := xnet.ParseHTTPURL(target.args.Endpoint.String()) if pErr != nil { return pErr } diff --git a/pkg/net/url.go b/pkg/net/url.go index bf0b6a7de..7e9dc4909 100644 --- a/pkg/net/url.go +++ b/pkg/net/url.go @@ -19,6 +19,7 @@ package net import ( "encoding/json" "errors" + "fmt" "net" "net/http" "net/url" @@ -103,6 +104,21 @@ func (u URL) DialHTTP() error { return nil } +// ParseHTTPURL - parses a string into HTTP URL, string is +// expected to be of form http:// or https:// +func ParseHTTPURL(s string) (u *URL, err error) { + u, err = ParseURL(s) + if err != nil { + return nil, err + } + switch u.Scheme { + default: + return nil, fmt.Errorf("unexpected scheme found %s", u.Scheme) + case "http", "https": + return u, nil + } +} + // ParseURL - parses string into URL. func ParseURL(s string) (u *URL, err error) { var uu *url.URL diff --git a/pkg/net/url_test.go b/pkg/net/url_test.go index 49dcd4339..5d9973bf4 100644 --- a/pkg/net/url_test.go +++ b/pkg/net/url_test.go @@ -133,6 +133,42 @@ func TestURLUnmarshalJSON(t *testing.T) { } } +func TestParseHTTPURL(t *testing.T) { + testCases := []struct { + s string + expectedURL *URL + expectErr bool + }{ + {"http://play", &URL{Scheme: "http", Host: "play"}, false}, + {"https://play.min.io:0", &URL{Scheme: "https", Host: "play.min.io:0"}, false}, + {"https://147.75.201.93:9000/", &URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, false}, + {"https://s3.amazonaws.com/?location", &URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, false}, + {"http://myminio:10000/mybucket//myobject/", &URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject/"}, false}, + {"ftp://myftp.server:10000/myuser", nil, true}, + {"https://my.server:10000000/myuser", nil, true}, + {"myserver:1000", nil, true}, + {"http://:1000/mybucket", nil, true}, + {"https://147.75.201.93:90000/", nil, true}, + {"http:/play", nil, true}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.s, func(t *testing.T) { + url, err := ParseHTTPURL(testCase.s) + expectErr := (err != nil) + if expectErr != testCase.expectErr { + t.Fatalf("error: expected: %v, got: %v", testCase.expectErr, expectErr) + } + if !testCase.expectErr { + if !reflect.DeepEqual(url, testCase.expectedURL) { + t.Fatalf("host: expected: %#v, got: %#v", testCase.expectedURL, url) + } + } + }) + } +} + func TestParseURL(t *testing.T) { testCases := []struct { s string