diff --git a/cmd/certs.go b/cmd/certs.go index c886d72b7..c41818e8a 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -24,9 +24,10 @@ import ( "crypto/x509" "encoding/pem" "io/ioutil" - "os" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/certs" + "github.com/minio/minio/pkg/env" ) // TLSPrivateKeyPassword is the environment variable which contains the password used @@ -49,19 +50,19 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er for len(current) > 0 { var pemBlock *pem.Block if pemBlock, current = pem.Decode(current); pemBlock == nil { - return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile) + return nil, config.ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile) } var x509Cert *x509.Certificate if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil { - return nil, uiErrSSLUnexpectedData(err) + return nil, config.ErrSSLUnexpectedData(err) } x509Certs = append(x509Certs, x509Cert) } if len(x509Certs) == 0 { - return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile) + return nil, config.ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile) } return x509Certs, nil @@ -105,37 +106,38 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) { func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) { certPEMBlock, err := ioutil.ReadFile(certFile) if err != nil { - return tls.Certificate{}, uiErrSSLUnexpectedError(err) + return tls.Certificate{}, config.ErrSSLUnexpectedError(err) } keyPEMBlock, err := ioutil.ReadFile(keyFile) if err != nil { - return tls.Certificate{}, uiErrSSLUnexpectedError(err) + return tls.Certificate{}, config.ErrSSLUnexpectedError(err) } key, rest := pem.Decode(keyPEMBlock) if len(rest) > 0 { - return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data") + return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg("The private key contains additional data") } if x509.IsEncryptedPEMBlock(key) { - password, ok := os.LookupEnv(TLSPrivateKeyPassword) + password, ok := env.Lookup(TLSPrivateKeyPassword) if !ok { - return tls.Certificate{}, uiErrSSLNoPassword(nil) + return tls.Certificate{}, config.ErrSSLNoPassword(nil) } decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password)) if decErr != nil { - return tls.Certificate{}, uiErrSSLWrongPassword(decErr) + return tls.Certificate{}, config.ErrSSLWrongPassword(decErr) } keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey}) } cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) if err != nil { - return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error()) + return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg(err.Error()) } // Ensure that the private key is not a P-384 or P-521 EC key. // The Go TLS stack does not provide constant-time implementations of P-384 and P-521. if priv, ok := cert.PrivateKey.(crypto.Signer); ok { if pub, ok := priv.Public().(*ecdsa.PublicKey); ok { - if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check - return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name) + if name := pub.Params().Name; name == "P-384" || name == "P-521" { + // unfortunately there is no cleaner way to check + return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name) } } } diff --git a/cmd/common-main.go b/cmd/common-main.go index aa746d8f1..a405c44fe 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -22,7 +22,6 @@ import ( "net" "os" "path/filepath" - "strconv" "strings" "time" @@ -30,10 +29,12 @@ import ( dns2 "github.com/miekg/dns" "github.com/minio/cli" "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger/target/http" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/dns" + "github.com/minio/minio/pkg/env" xnet "github.com/minio/minio/pkg/net" ) @@ -45,7 +46,7 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) { if strings.HasPrefix(name, "gateway") { if GlobalGatewaySSE.IsSet() && GlobalKMS == nil { - uiErr := uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured") + uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured") logger.Fatal(uiErr, "Unable to start gateway with SSE") } } @@ -75,13 +76,13 @@ func checkUpdate(mode string) { func loadLoggers() { loggerUserAgent := getUserAgent(getMinioMode()) - auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT") + auditEndpoint, ok := env.Lookup("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT") if ok { // Enable audit HTTP logging through ENV. logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport())) } - loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT") + loggerEndpoint, ok := env.Lookup("MINIO_LOGGER_HTTP_ENDPOINT") if ok { // Enable HTTP logging through ENV. logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport())) @@ -189,31 +190,20 @@ func handleCommonCmdArgs(ctx *cli.Context) { globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat") } -// Parses the given compression exclude list `extensions` or `content-types`. -func parseCompressIncludes(includes []string) ([]string, error) { - for _, e := range includes { - if len(e) == 0 { - return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e) - } - } - return includes, nil -} - func handleCommonEnvVars() { - compressEnvDelimiter := "," // Start profiler if env is set. - if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { + if profiler := env.Get("_MINIO_PROFILER", ""); profiler != "" { var err error globalProfiler, err = startProfiler(profiler, "") logger.FatalIf(err, "Unable to setup a profiler") } - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") + accessKey := env.Get("MINIO_ACCESS_KEY", "") + secretKey := env.Get("MINIO_SECRET_KEY", "") if accessKey != "" && secretKey != "" { cred, err := auth.CreateCredentials(accessKey, secretKey) if err != nil { - logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment") + logger.Fatal(config.ErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment") } cred.Expiration = timeSentinel @@ -222,10 +212,10 @@ func handleCommonEnvVars() { globalActiveCred = cred } - if browser := os.Getenv("MINIO_BROWSER"); browser != "" { + if browser := env.Get("MINIO_BROWSER", "on"); browser != "" { browserFlag, err := ParseBoolFlag(browser) if err != nil { - logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable") + logger.Fatal(config.ErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable") } // browser Envs are set globally, this does not represent @@ -234,7 +224,7 @@ func handleCommonEnvVars() { globalIsBrowserEnabled = bool(browserFlag) } - etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS") + etcdEndpointsEnv, ok := env.Lookup("MINIO_ETCD_ENDPOINTS") if ok { etcdEndpoints := strings.Split(etcdEndpointsEnv, ",") @@ -252,8 +242,8 @@ func handleCommonEnvVars() { if etcdSecure { // This is only to support client side certificate authentication // https://coreos.com/etcd/docs/latest/op-guide/security.html - etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT") - etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY") + etcdClientCertFile, ok1 := env.Lookup("MINIO_ETCD_CLIENT_CERT") + etcdClientCertKey, ok2 := env.Lookup("MINIO_ETCD_CLIENT_CERT_KEY") var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) if ok1 && ok2 { getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) { @@ -281,18 +271,18 @@ func handleCommonEnvVars() { logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints) } - v, ok := os.LookupEnv("MINIO_DOMAIN") + v, ok := env.Lookup("MINIO_DOMAIN") if ok { for _, domainName := range strings.Split(v, ",") { if _, ok = dns2.IsDomainName(domainName); !ok { - logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName), + logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName), "Invalid MINIO_DOMAIN value in environment variable") } globalDomainNames = append(globalDomainNames, domainName) } } - minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS") + minioEndpointsEnv, ok := env.Lookup("MINIO_PUBLIC_IPS") if ok { minioEndpoints := strings.Split(minioEndpointsEnv, ",") var domainIPs = set.NewStringSet() @@ -323,53 +313,10 @@ func handleCommonEnvVars() { logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames) } - if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" { - driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter)) - if err != nil { - logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives) - } - globalCacheDrives = driveList - globalIsDiskCacheEnabled = true - } - - if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" { - excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter)) - if err != nil { - logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes) - } - globalCacheExcludes = excludeList - } - - if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" { - expiry, err := strconv.Atoi(expiryStr) - if err != nil { - logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr) - } - globalCacheExpiry = expiry - } - - if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" { - maxUse, err := strconv.Atoi(maxUseStr) - if err != nil { - logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr) - } - // maxUse should be a valid percentage. - if maxUse > 0 && maxUse <= 100 { - globalCacheMaxUse = maxUse - } - } - - var err error - if cacheEncKey := os.Getenv("MINIO_CACHE_ENCRYPTION_MASTER_KEY"); cacheEncKey != "" { - globalCacheKMSKeyID, globalCacheKMS, err = parseKMSMasterKey(cacheEncKey) - if err != nil { - logger.Fatal(uiErrInvalidCacheEncryptionKey(err), "Invalid cache encryption master key") - } - } // In place update is true by default if the MINIO_UPDATE is not set // or is not set to 'off', if MINIO_UPDATE is set to 'off' then // in-place update is off. - globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off") + globalInplaceUpdateDisabled = strings.EqualFold(env.Get("MINIO_UPDATE", "off"), "off") // Validate and store the storage class env variables only for XL/Dist XL setups if globalIsXL { @@ -402,10 +349,10 @@ func handleCommonEnvVars() { } // Get WORM environment variable. - if worm := os.Getenv("MINIO_WORM"); worm != "" { + if worm := env.Get("MINIO_WORM", "off"); worm != "" { wormFlag, err := ParseBoolFlag(worm) if err != nil { - logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable") + logger.Fatal(config.ErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable") } // worm Envs are set globally, this does not represent @@ -414,29 +361,6 @@ func handleCommonEnvVars() { globalWORMEnabled = bool(wormFlag) } - if compress := os.Getenv("MINIO_COMPRESS"); compress != "" { - globalIsCompressionEnabled = strings.EqualFold(compress, "true") - } - - compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS") - compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES") - if compressExtensions != "" || compressMimeTypes != "" { - globalIsEnvCompression = true - if compressExtensions != "" { - extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter)) - if err != nil { - logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions) - } - globalCompressExtensions = extensions - } - if compressMimeTypes != "" { - contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter)) - if err != nil { - logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes) - } - globalCompressMimeTypes = contenttypes - } - } } func logStartupMessage(msg string, data ...interface{}) { diff --git a/cmd/config-current.go b/cmd/config-current.go index a989d7594..b09be2d4f 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -20,14 +20,18 @@ import ( "context" "errors" "fmt" - "os" "reflect" "sync" + "github.com/minio/minio/cmd/config" + "github.com/minio/minio/cmd/config/cache" + "github.com/minio/minio/cmd/config/compress" + xldap "github.com/minio/minio/cmd/config/ldap" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/event/target" "github.com/minio/minio/pkg/iam/openid" @@ -134,18 +138,10 @@ func (s *serverConfig) GetWorm() bool { return bool(s.Worm) } -// SetCacheConfig sets the current cache config -func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) { - s.Cache.Drives = drives - s.Cache.Exclude = exclude - s.Cache.Expiry = expiry - s.Cache.MaxUse = maxuse -} - // GetCacheConfig gets the current cache config -func (s *serverConfig) GetCacheConfig() CacheConfig { +func (s *serverConfig) GetCacheConfig() cache.Config { if globalIsDiskCacheEnabled { - return CacheConfig{ + return cache.Config{ Drives: globalCacheDrives, Exclude: globalCacheExcludes, Expiry: globalCacheExpiry, @@ -153,7 +149,7 @@ func (s *serverConfig) GetCacheConfig() CacheConfig { } } if s == nil { - return CacheConfig{} + return cache.Config{} } return s.Cache } @@ -246,7 +242,7 @@ func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []str } // GetCompressionConfig gets the current compression config -func (s *serverConfig) GetCompressionConfig() compressionConfig { +func (s *serverConfig) GetCompressionConfig() compress.Config { return s.Compression } @@ -268,19 +264,39 @@ func (s *serverConfig) loadFromEnvs() { s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass) } - if globalIsDiskCacheEnabled { - s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse) + var err error + s.Cache, err = cache.LookupConfig(s.Cache) + if err != nil { + logger.FatalIf(err, "Unable to setup cache") } - if err := Environment.LookupKMSConfig(s.KMS); err != nil { - logger.FatalIf(err, "Unable to setup the KMS") + if len(s.Cache.Drives) > 0 { + globalIsDiskCacheEnabled = true + globalCacheDrives = s.Cache.Drives + globalCacheExcludes = s.Cache.Exclude + globalCacheExpiry = s.Cache.Expiry + globalCacheMaxUse = s.Cache.MaxUse + + var err error + if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" { + globalCacheKMSKeyID, globalCacheKMS, err = parseKMSMasterKey(cacheEncKey) + if err != nil { + logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err), + "Unable to setup encryption cache") + } + } + } + + if err = LookupKMSConfig(s.KMS); err != nil { + logger.FatalIf(err, "Unable to setup KMS") } - if globalIsEnvCompression { - s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes) + s.Compression, err = compress.LookupConfig(s.Compression) + if err != nil { + logger.FatalIf(err, "Unable to setup Compression") } - if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok { + if jwksURL, ok := env.Lookup("MINIO_IAM_JWKS_URL"); ok { u, err := xnet.ParseURL(jwksURL) if err != nil { logger.FatalIf(err, "Unable to parse MINIO_IAM_JWKS_URL %s", jwksURL) @@ -288,14 +304,14 @@ func (s *serverConfig) loadFromEnvs() { s.OpenID.JWKS.URL = u } - if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok { + if opaURL, ok := env.Lookup("MINIO_IAM_OPA_URL"); ok { u, err := xnet.ParseURL(opaURL) if err != nil { logger.FatalIf(err, "Unable to parse MINIO_IAM_OPA_URL %s", opaURL) } opaArgs := iampolicy.OpaArgs{ URL: u, - AuthToken: os.Getenv("MINIO_IAM_OPA_AUTHTOKEN"), + AuthToken: env.Get("MINIO_IAM_OPA_AUTHTOKEN", ""), Transport: NewCustomHTTPTransport(), CloseRespFn: xhttp.DrainBody, } @@ -304,8 +320,7 @@ func (s *serverConfig) loadFromEnvs() { s.Policy.OPA.AuthToken = opaArgs.AuthToken } - var err error - s.LDAPServerConfig, err = newLDAPConfigFromEnv(globalRootCAs) + s.LDAPServerConfig, err = xldap.Lookup(s.LDAPServerConfig, globalRootCAs) if err != nil { logger.FatalIf(err, "Unable to parse LDAP configuration from env") } @@ -484,7 +499,7 @@ func newServerConfig() *serverConfig { Standard: storageClass{}, RRS: storageClass{}, }, - Cache: CacheConfig{ + Cache: cache.Config{ Drives: []string{}, Exclude: []string{}, Expiry: globalCacheExpiry, @@ -492,7 +507,7 @@ func newServerConfig() *serverConfig { }, KMS: crypto.KMSConfig{}, Notify: notifier{}, - Compression: compressionConfig{ + Compression: compress.Config{ Enabled: false, Extensions: globalCompressExtensions, MimeTypes: globalCompressMimeTypes, @@ -555,7 +570,7 @@ func (s *serverConfig) loadToCachedConfigs() { globalCacheExpiry = cacheConf.Expiry globalCacheMaxUse = cacheConf.MaxUse } - if err := Environment.LookupKMSConfig(s.KMS); err != nil { + if err := LookupKMSConfig(s.KMS); err != nil { logger.FatalIf(err, "Unable to setup the KMS %s", s.KMS.Vault.Endpoint) } @@ -571,7 +586,7 @@ func (s *serverConfig) loadToCachedConfigs() { "Unable to populate public key from JWKS URL %s", s.OpenID.JWKS.URL) } - globalIAMValidators = getOpenIDValidators(s) + globalOpenIDValidators = getOpenIDValidators(s) if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" { opaArgs := iampolicy.OpaArgs{ @@ -621,7 +636,7 @@ func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) { func loadConfig(objAPI ObjectLayer) error { srvCfg, err := getValidConfig(objAPI) if err != nil { - return uiErrInvalidConfig(nil).Msg(err.Error()) + return config.ErrInvalidConfig(nil).Msg(err.Error()) } // Override any values from ENVs. diff --git a/cmd/config-versions.go b/cmd/config-versions.go index 24ffec3c4..d4d6ace7e 100644 --- a/cmd/config-versions.go +++ b/cmd/config-versions.go @@ -19,6 +19,9 @@ package cmd import ( "sync" + "github.com/minio/minio/cmd/config/cache" + "github.com/minio/minio/cmd/config/compress" + xldap "github.com/minio/minio/cmd/config/ldap" "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/event/target" @@ -592,7 +595,7 @@ type serverConfigV23 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // Notification queue configuration. Notify notifierV3 `json:"notify"` @@ -616,7 +619,7 @@ type serverConfigV24 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // Notification queue configuration. Notify notifierV3 `json:"notify"` @@ -643,14 +646,14 @@ type serverConfigV25 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // Notification queue configuration. Notify notifierV3 `json:"notify"` } // serverConfigV26 is just like version '25', stores additionally -// cache max use value in 'CacheConfig'. +// cache max use value in 'cache.Config'. type serverConfigV26 struct { quick.Config `json:"-"` // ignore interfaces @@ -667,7 +670,7 @@ type serverConfigV26 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // Notification queue configuration. Notify notifierV3 `json:"notify"` @@ -708,7 +711,7 @@ type serverConfigV27 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // Notification queue configuration. Notify notifierV3 `json:"notify"` @@ -736,7 +739,7 @@ type serverConfigV28 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // KMS configuration KMS crypto.KMSConfig `json:"kms"` @@ -751,13 +754,6 @@ type serverConfigV28 struct { // serverConfigV29 is just like version '28'. type serverConfigV29 serverConfigV28 -// compressionConfig represents the compression settings. -type compressionConfig struct { - Enabled bool `json:"enabled"` - Extensions []string `json:"extensions"` - MimeTypes []string `json:"mime-types"` -} - // serverConfigV30 is just like version '29', stores additionally // extensions and mimetypes fields for compression. type serverConfigV30 struct { @@ -772,7 +768,7 @@ type serverConfigV30 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // KMS configuration KMS crypto.KMSConfig `json:"kms"` @@ -784,7 +780,7 @@ type serverConfigV30 struct { Logger loggerConfig `json:"logger"` // Compression configuration - Compression compressionConfig `json:"compress"` + Compression compress.Config `json:"compress"` } // serverConfigV31 is just like version '30', with OPA and OpenID configuration. @@ -800,7 +796,7 @@ type serverConfigV31 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // KMS configuration KMS crypto.KMSConfig `json:"kms"` @@ -812,7 +808,7 @@ type serverConfigV31 struct { Logger loggerConfig `json:"logger"` // Compression configuration - Compression compressionConfig `json:"compress"` + Compression compress.Config `json:"compress"` // OpenID configuration OpenID struct { @@ -855,7 +851,7 @@ type serverConfigV32 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // KMS configuration KMS crypto.KMSConfig `json:"kms"` @@ -867,7 +863,7 @@ type serverConfigV32 struct { Logger loggerConfig `json:"logger"` // Compression configuration - Compression compressionConfig `json:"compress"` + Compression compress.Config `json:"compress"` // OpenID configuration OpenID struct { @@ -899,7 +895,7 @@ type serverConfigV33 struct { StorageClass storageClassConfig `json:"storageclass"` // Cache configuration - Cache CacheConfig `json:"cache"` + Cache cache.Config `json:"cache"` // KMS configuration KMS crypto.KMSConfig `json:"kms"` @@ -911,7 +907,7 @@ type serverConfigV33 struct { Logger loggerConfig `json:"logger"` // Compression configuration - Compression compressionConfig `json:"compress"` + Compression compress.Config `json:"compress"` // OpenID configuration OpenID struct { @@ -927,5 +923,5 @@ type serverConfigV33 struct { // Add new external policy enforcements here. } `json:"policy"` - LDAPServerConfig ldapServerConfig `json:"ldapserverconfig"` + LDAPServerConfig xldap.Config `json:"ldapserverconfig"` } diff --git a/cmd/disk-cache-config.go b/cmd/config/cache/config.go similarity index 78% rename from cmd/disk-cache-config.go rename to cmd/config/cache/config.go index 429752603..b3f603a8f 100644 --- a/cmd/disk-cache-config.go +++ b/cmd/config/cache/config.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2018 MinIO, Inc. + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package cmd +package cache import ( "encoding/json" @@ -22,11 +22,12 @@ import ( "path/filepath" "strings" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/ellipses" ) -// CacheConfig represents cache config settings -type CacheConfig struct { +// Config represents cache config settings +type Config struct { Drives []string `json:"drives"` Expiry int `json:"expiry"` MaxUse int `json:"maxuse"` @@ -35,8 +36,8 @@ type CacheConfig struct { // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling // json entries for CacheConfig. -func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) { - type Alias CacheConfig +func (cfg *Config) UnmarshalJSON(data []byte) (err error) { + type Alias Config var _cfg = &struct { *Alias }{ @@ -83,7 +84,7 @@ func parseCacheDrives(drives []string) ([]string, error) { for _, d := range endpoints { if !filepath.IsAbs(d) { - return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d) + return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d) } } return endpoints, nil @@ -93,7 +94,7 @@ func parseCacheDrives(drives []string) ([]string, error) { func parseCacheDrivePaths(arg string) (ep []string, err error) { patterns, perr := ellipses.FindEllipsesPatterns(arg) if perr != nil { - return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error()) + return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error()) } for _, lbls := range patterns.Expand() { @@ -107,10 +108,10 @@ func parseCacheDrivePaths(arg string) (ep []string, err error) { func parseCacheExcludes(excludes []string) ([]string, error) { for _, e := range excludes { if len(e) == 0 { - return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e) + return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e) } - if hasPrefix(e, SlashSeparator) { - return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e) + if strings.HasPrefix(e, "/") { + return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e) } } return excludes, nil diff --git a/cmd/disk-cache-config_test.go b/cmd/config/cache/config_test.go similarity index 97% rename from cmd/disk-cache-config_test.go rename to cmd/config/cache/config_test.go index f4cc02529..56fa69e62 100644 --- a/cmd/disk-cache-config_test.go +++ b/cmd/config/cache/config_test.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2018 MinIO, Inc. + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package cmd +package cache import ( "reflect" @@ -35,7 +35,7 @@ func TestParseCacheDrives(t *testing.T) { {"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false}, {"bucket1", []string{}, false}, } - if runtime.GOOS == globalWindowsOSName { + if runtime.GOOS == "windows" { testCases = append(testCases, struct { driveStr string expectedPatterns []string diff --git a/cmd/config/cache/lookup.go b/cmd/config/cache/lookup.go new file mode 100644 index 000000000..a3213de6d --- /dev/null +++ b/cmd/config/cache/lookup.go @@ -0,0 +1,79 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cache + +import ( + "strconv" + "strings" + + "github.com/minio/minio/cmd/config" + "github.com/minio/minio/pkg/env" +) + +// Cache ENVs +const ( + EnvCacheDrives = "MINIO_CACHE_DRIVES" + EnvCacheExclude = "MINIO_CACHE_EXCLUDE" + EnvCacheExpiry = "MINIO_CACHE_EXPIRY" + EnvCacheMaxUse = "MINIO_CACHE_MAXUSE" + EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY" +) + +const ( + cacheEnvDelimiter = ";" +) + +// LookupConfig - extracts cache configuration provided by environment +// variables and merge them with provided CacheConfiguration. +func LookupConfig(cfg Config) (Config, error) { + if drives := env.Get(EnvCacheDrives, strings.Join(cfg.Drives, ",")); drives != "" { + driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter)) + if err != nil { + return cfg, err + } + cfg.Drives = driveList + } + + if excludes := env.Get(EnvCacheExclude, strings.Join(cfg.Exclude, ",")); excludes != "" { + excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter)) + if err != nil { + return cfg, err + } + cfg.Exclude = excludeList + } + + if expiryStr := env.Get(EnvCacheExpiry, strconv.Itoa(cfg.Expiry)); expiryStr != "" { + expiry, err := strconv.Atoi(expiryStr) + if err != nil { + return cfg, config.ErrInvalidCacheExpiryValue(err) + } + cfg.Expiry = expiry + } + + if maxUseStr := env.Get(EnvCacheMaxUse, strconv.Itoa(cfg.MaxUse)); maxUseStr != "" { + maxUse, err := strconv.Atoi(maxUseStr) + if err != nil { + return cfg, config.ErrInvalidCacheMaxUse(err) + } + // maxUse should be a valid percentage. + if maxUse > 0 && maxUse <= 100 { + cfg.MaxUse = maxUse + } + } + + return cfg, nil +} diff --git a/cmd/config/compress/compress.go b/cmd/config/compress/compress.go new file mode 100644 index 000000000..0281820c3 --- /dev/null +++ b/cmd/config/compress/compress.go @@ -0,0 +1,79 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package compress + +import ( + "fmt" + "strconv" + "strings" + + "github.com/minio/minio/cmd/config" + "github.com/minio/minio/pkg/env" +) + +// Config represents the compression settings. +type Config struct { + Enabled bool `json:"enabled"` + Extensions []string `json:"extensions"` + MimeTypes []string `json:"mime-types"` +} + +// Compression environment variables +const ( + EnvMinioCompress = "MINIO_COMPRESS" + EnvMinioCompressExtensions = "MINIO_COMPRESS_EXTENSIONS" + EnvMinioCompressMimeTypes = "MINIO_COMPRESS_MIMETYPES" +) + +// Parses the given compression exclude list `extensions` or `content-types`. +func parseCompressIncludes(includes []string) ([]string, error) { + for _, e := range includes { + if len(e) == 0 { + return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e) + } + } + return includes, nil +} + +// LookupConfig - lookup compression config. +func LookupConfig(cfg Config) (Config, error) { + const compressEnvDelimiter = "," + if compress := env.Get(EnvMinioCompress, strconv.FormatBool(cfg.Enabled)); compress != "" { + cfg.Enabled = strings.EqualFold(compress, "true") + } + + compressExtensions := env.Get(EnvMinioCompressExtensions, strings.Join(cfg.Extensions, ",")) + compressMimeTypes := env.Get(EnvMinioCompressMimeTypes, strings.Join(cfg.MimeTypes, ",")) + if compressExtensions != "" || compressMimeTypes != "" { + if compressExtensions != "" { + extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter)) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions) + } + cfg.Extensions = extensions + } + if compressMimeTypes != "" { + contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter)) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes) + } + cfg.MimeTypes = contenttypes + } + } + + return cfg, nil +} diff --git a/cmd/ui-errors-utils.go b/cmd/config/errors-utls.go similarity index 60% rename from cmd/ui-errors-utils.go rename to cmd/config/errors-utls.go index a5eeab94b..2bec6fd65 100644 --- a/cmd/ui-errors-utils.go +++ b/cmd/config/errors-utls.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package cmd +package config import ( "errors" @@ -22,12 +22,14 @@ import ( "io" "os" "syscall" + + "github.com/minio/minio/pkg/color" ) -// uiErr is a structure which contains all information +// Err is a structure which contains all information // to print a fatal error message in json or pretty mode -// uiErr implements error so we can use it anywhere -type uiErr struct { +// Err implements error so we can use it anywhere +type Err struct { msg string detail string action string @@ -35,16 +37,16 @@ type uiErr struct { } // Return the error message -func (u uiErr) Error() string { +func (u Err) Error() string { if u.detail == "" { return u.msg } return u.detail } -// Replace the current error's message -func (u uiErr) Msg(m string, args ...interface{}) uiErr { - return uiErr{ +// Msg - Replace the current error's message +func (u Err) Msg(m string, args ...interface{}) Err { + return Err{ msg: fmt.Sprintf(m, args...), detail: u.detail, action: u.action, @@ -52,14 +54,15 @@ func (u uiErr) Msg(m string, args ...interface{}) uiErr { } } -type uiErrFn func(err error) uiErr +// ErrFn function wrapper +type ErrFn func(err error) Err // Create a UI error generator, this is needed to simplify // the update of the detailed error message in several places // in MinIO code -func newUIErrFn(msg, action, hint string) uiErrFn { - return func(err error) uiErr { - u := uiErr{ +func newErrFn(msg, action, hint string) ErrFn { + return func(err error) Err { + u := Err{ msg: msg, action: action, hint: hint, @@ -71,35 +74,35 @@ func newUIErrFn(msg, action, hint string) uiErrFn { } } -// errorToUIError inspects the passed error and transforms it +// ErrorToErr inspects the passed error and transforms it // to the appropriate UI error. -func errorToUIErr(err error) uiErr { - // If this is already a uiErr, do nothing - if e, ok := err.(uiErr); ok { +func ErrorToErr(err error) Err { + // If this is already a Err, do nothing + if e, ok := err.(Err); ok { return e } // Show a generic message for known golang errors if errors.Is(err, syscall.EADDRINUSE) { - return uiErrPortAlreadyInUse(err).Msg("Specified port is already in use") + return ErrPortAlreadyInUse(err).Msg("Specified port is already in use") } else if errors.Is(err, syscall.EACCES) { - return uiErrPortAccess(err).Msg("Insufficient permissions to use specified port") + return ErrPortAccess(err).Msg("Insufficient permissions to use specified port") } else if os.IsPermission(err) { - return uiErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path") + return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path") } else if errors.Is(err, io.ErrUnexpectedEOF) { - return uiErrUnexpectedDataContent(err) + return ErrUnexpectedDataContent(err) } else { // Failed to identify what type of error this, return a simple UI error - return uiErr{msg: err.Error()} + return Err{msg: err.Error()} } } -// fmtError() converts a fatal error message to a more clear error +// FmtError converts a fatal error message to a more clear error // using some colors -func fmtError(introMsg string, err error, jsonFlag bool) string { +func FmtError(introMsg string, err error, jsonFlag bool) string { renderedTxt := "" - uiErr := errorToUIErr(err) + uiErr := ErrorToErr(err) // JSON print if jsonFlag { // Message text in json should be simple @@ -111,18 +114,18 @@ func fmtError(introMsg string, err error, jsonFlag bool) string { // Pretty print error message introMsg += ": " if uiErr.msg != "" { - introMsg += colorBold(uiErr.msg) + introMsg += color.Bold(uiErr.msg) } else { - introMsg += colorBold(err.Error()) + introMsg += color.Bold(err.Error()) } - renderedTxt += colorRed(introMsg) + "\n" + renderedTxt += color.Red(introMsg) + "\n" // Add action message if uiErr.action != "" { - renderedTxt += "> " + colorBgYellow(colorBlack(uiErr.action)) + "\n" + renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n" } // Add hint if uiErr.hint != "" { - renderedTxt += colorBold("HINT:") + "\n" + renderedTxt += color.Bold("HINT:") + "\n" renderedTxt += " " + uiErr.hint } return renderedTxt diff --git a/cmd/ui-errors.go b/cmd/config/errors.go similarity index 80% rename from cmd/ui-errors.go rename to cmd/config/errors.go index 87296d2dd..0f0397a82 100644 --- a/cmd/ui-errors.go +++ b/cmd/config/errors.go @@ -14,101 +14,102 @@ * limitations under the License. */ -package cmd +package config +// UI errors var ( - uiErrInvalidConfig = newUIErrFn( + ErrInvalidConfig = newErrFn( "Invalid value found in the configuration file", "Please ensure a valid value in the configuration file", "For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide", ) - uiErrInvalidBrowserValue = newUIErrFn( + ErrInvalidBrowserValue = newErrFn( "Invalid browser value", "Please check the passed value", "Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`", ) - uiErrInvalidDomainValue = newUIErrFn( + ErrInvalidDomainValue = newErrFn( "Invalid domain value", "Please check the passed value", "Domain can only accept DNS compatible values", ) - uiErrInvalidErasureSetSize = newUIErrFn( + ErrInvalidErasureSetSize = newErrFn( "Invalid erasure set size", "Please check the passed value", "Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values", ) - uiErrInvalidWormValue = newUIErrFn( + ErrInvalidWormValue = newErrFn( "Invalid WORM value", "Please check the passed value", "WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`", ) - uiErrInvalidCacheDrivesValue = newUIErrFn( + ErrInvalidCacheDrivesValue = newErrFn( "Invalid cache drive value", "Please check the value in this ENV variable", "MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`", ) - uiErrInvalidCacheExcludesValue = newUIErrFn( + ErrInvalidCacheExcludesValue = newErrFn( "Invalid cache excludes value", "Please check the passed value", "MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`", ) - uiErrInvalidCacheExpiryValue = newUIErrFn( + ErrInvalidCacheExpiryValue = newErrFn( "Invalid cache expiry value", "Please check the passed value", "MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days", ) - uiErrInvalidCacheMaxUse = newUIErrFn( + ErrInvalidCacheMaxUse = newErrFn( "Invalid cache max-use value", "Please check the passed value", "MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100", ) - uiErrInvalidCacheEncryptionKey = newUIErrFn( + ErrInvalidCacheEncryptionKey = newErrFn( "Invalid cache encryption master key value", "Please check the passed value", "MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide", ) - uiErrInvalidCredentials = newUIErrFn( + ErrInvalidCredentials = newErrFn( "Invalid credentials", "Please provide correct credentials", `Access key length should be between minimum 3 characters in length. Secret key should be in between 8 and 40 characters`, ) - uiErrEnvCredentialsMissingGateway = newUIErrFn( + ErrEnvCredentialsMissingGateway = newErrFn( "Credentials missing", "Please set your credentials in the environment", `In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`, ) - uiErrEnvCredentialsMissingDistributed = newUIErrFn( + ErrEnvCredentialsMissingDistributed = newErrFn( "Credentials missing", "Please set your credentials in the environment", `In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`, ) - uiErrInvalidErasureEndpoints = newUIErrFn( + ErrInvalidErasureEndpoints = newErrFn( "Invalid endpoint(s) in erasure mode", "Please provide correct combination of local/remote paths", "For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide", ) - uiErrInvalidNumberOfErasureEndpoints = newUIErrFn( + ErrInvalidNumberOfErasureEndpoints = newErrFn( "Invalid total number of endpoints for erasure mode", "Please provide an even number of endpoints greater or equal to 4", "For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide", ) - uiErrStorageClassValue = newUIErrFn( + ErrStorageClassValue = newErrFn( "Invalid storage class value", "Please check the value", `MINIO_STORAGE_CLASS_STANDARD: Format "EC:" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request @@ -116,13 +117,13 @@ MINIO_STORAGE_CLASS_RRS: Format "EC:" ( Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`, ) - uiErrUnexpectedBackendVersion = newUIErrFn( + ErrUnexpectedBackendVersion = newErrFn( "Backend version seems to be too recent", "Please update to the latest MinIO version", "", ) - uiErrInvalidAddressFlag = newUIErrFn( + ErrInvalidAddressFlag = newErrFn( "--address input is invalid", "Please check --address parameter", `--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000') @@ -131,7 +132,7 @@ Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storag --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`, ) - uiErrInvalidFSEndpoint = newUIErrFn( + ErrInvalidFSEndpoint = newErrFn( "Invalid endpoint for standalone FS mode", "Please check the FS endpoint", `FS mode requires only one writable disk path @@ -139,91 +140,91 @@ Example 1: $ minio server /data/minio/`, ) - uiErrUnableToWriteInBackend = newUIErrFn( + ErrUnableToWriteInBackend = newErrFn( "Unable to write to the backend", "Please ensure MinIO binary has write permissions for the backend", `Verify if MinIO binary is running as the same user who has write permissions for the backend`, ) - uiErrPortAlreadyInUse = newUIErrFn( + ErrPortAlreadyInUse = newErrFn( "Port is already in use", "Please ensure no other program uses the same address/port", "", ) - uiErrPortAccess = newUIErrFn( + ErrPortAccess = newErrFn( "Unable to use specified port", "Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions", `Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`, ) - uiErrNoPermissionsToAccessDirFiles = newUIErrFn( + ErrNoPermissionsToAccessDirFiles = newErrFn( "Missing permissions to access the specified path", "Please ensure the specified path can be accessed", "", ) - uiErrSSLUnexpectedError = newUIErrFn( + ErrSSLUnexpectedError = newErrFn( "Invalid TLS certificate", "Please check the content of your certificate data", `Only PEM (x.509) format is accepted as valid public & private certificates`, ) - uiErrSSLUnexpectedData = newUIErrFn( + ErrSSLUnexpectedData = newErrFn( "Invalid TLS certificate", "Please check your certificate", "", ) - uiErrSSLNoPassword = newUIErrFn( + ErrSSLNoPassword = newErrFn( "Missing TLS password", - "Please set the password to environment variable `"+TLSPrivateKeyPassword+"` so that the private key can be decrypted", + "Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted", "", ) - uiErrNoCertsAndHTTPSEndpoints = newUIErrFn( + ErrNoCertsAndHTTPSEndpoints = newErrFn( "HTTPS specified in endpoints, but no TLS certificate is found on the local machine", "Please add TLS certificate or use HTTP endpoints only", "Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server", ) - uiErrCertsAndHTTPEndpoints = newUIErrFn( + ErrCertsAndHTTPEndpoints = newErrFn( "HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate", "Please remove the certificate in the configuration directory or switch to HTTPS", "", ) - uiErrSSLWrongPassword = newUIErrFn( + ErrSSLWrongPassword = newErrFn( "Unable to decrypt the private key using the provided password", - "Please set the correct password in environment variable "+TLSPrivateKeyPassword, + "Please set the correct password in environment variable `MINIO_CERT_PASSWD`", "", ) - uiErrUnexpectedDataContent = newUIErrFn( + ErrUnexpectedDataContent = newErrFn( "Unexpected data content", "Please contact MinIO at https://slack.min.io", "", ) - uiErrUnexpectedError = newUIErrFn( + ErrUnexpectedError = newErrFn( "Unexpected error", "Please contact MinIO at https://slack.min.io", "", ) - uiErrInvalidCompressionIncludesValue = newUIErrFn( + ErrInvalidCompressionIncludesValue = newErrFn( "Invalid compression include value", "Please check the passed value", "Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"", ) - uiErrInvalidGWSSEValue = newUIErrFn( + ErrInvalidGWSSEValue = newErrFn( "Invalid gateway SSE value", "Please check the passed value", "MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value", ) - uiErrInvalidGWSSEEnvValue = newUIErrFn( + ErrInvalidGWSSEEnvValue = newErrFn( "Invalid gateway SSE configuration", "", "Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE", diff --git a/cmd/config/ldap/config.go b/cmd/config/ldap/config.go new file mode 100644 index 000000000..e059902c4 --- /dev/null +++ b/cmd/config/ldap/config.go @@ -0,0 +1,201 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ldap + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "regexp" + "time" + + "github.com/minio/minio/pkg/env" + ldap "gopkg.in/ldap.v3" +) + +const ( + defaultLDAPExpiry = time.Hour * 1 +) + +// Config contains AD/LDAP server connectivity information. +type Config struct { + IsEnabled bool `json:"enabled"` + + // E.g. "ldap.minio.io:636" + ServerAddr string `json:"serverAddr"` + + // STS credentials expiry duration + STSExpiryDuration string `json:"stsExpiryDuration"` + stsExpiryDuration time.Duration // contains converted value + + RootCAs *x509.CertPool `json:"-"` + + // Format string for usernames + UsernameFormat string `json:"usernameFormat"` + + GroupSearchBaseDN string `json:"groupSearchBaseDN"` + GroupSearchFilter string `json:"groupSearchFilter"` + GroupNameAttribute string `json:"groupNameAttribute"` +} + +// LDAP keys and envs. +const ( + ServerAddr = "server_addr" + STSExpiry = "sts_expiry" + UsernameFormat = "username_format" + GroupSearchFilter = "group_search_filter" + GroupNameAttribute = "group_name_attribute" + GroupSearchBaseDN = "group_search_base_dn" + + EnvServerAddr = "MINIO_IDENTITY_LDAP_SERVER_ADDR" + EnvSTSExpiry = "MINIO_IDENTITY_LDAP_STS_EXPIRY" + EnvUsernameFormat = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT" + EnvGroupSearchFilter = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER" + EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE" + EnvGroupSearchBaseDN = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN" +) + +// Connect connect to ldap server. +func (l *Config) Connect() (ldapConn *ldap.Conn, err error) { + if l == nil { + // Happens when LDAP is not configured. + return + } + return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.RootCAs}) +} + +// GetExpiryDuration - return parsed expiry duration. +func (l Config) GetExpiryDuration() time.Duration { + return l.stsExpiryDuration +} + +// Lookup - initializes LDAP config, overrides config, if any ENV values are set. +func Lookup(cfg Config, rootCAs *x509.CertPool) (l Config, err error) { + if cfg.ServerAddr == "" && cfg.IsEnabled { + return l, errors.New("ldap server cannot initialize with empty LDAP server") + } + l.RootCAs = rootCAs + ldapServer := env.Get(EnvServerAddr, cfg.ServerAddr) + if ldapServer == "" { + return l, nil + } + l.IsEnabled = true + l.ServerAddr = ldapServer + l.stsExpiryDuration = defaultLDAPExpiry + if v := env.Get(EnvSTSExpiry, cfg.STSExpiryDuration); v != "" { + expDur, err := time.ParseDuration(v) + if err != nil { + return l, errors.New("LDAP expiry time err:" + err.Error()) + } + if expDur <= 0 { + return l, errors.New("LDAP expiry time has to be positive") + } + l.STSExpiryDuration = v + l.stsExpiryDuration = expDur + } + + if v := env.Get(EnvUsernameFormat, cfg.UsernameFormat); v != "" { + subs, err := NewSubstituter("username", "test") + if err != nil { + return l, err + } + if _, err := subs.Substitute(v); err != nil { + return l, fmt.Errorf("Only username may be substituted in the username format: %s", err) + } + l.UsernameFormat = v + } + + grpSearchFilter := env.Get(EnvGroupSearchFilter, cfg.GroupSearchFilter) + grpSearchNameAttr := env.Get(EnvGroupNameAttribute, cfg.GroupNameAttribute) + grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, cfg.GroupSearchBaseDN) + + // Either all group params must be set or none must be set. + allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == "" + allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != "" + if !allNotSet && !allSet { + return l, errors.New("All group related parameters must be set") + } + + if allSet { + subs, err := NewSubstituter("username", "test", "usernamedn", "test2") + if err != nil { + return l, err + } + if _, err := subs.Substitute(grpSearchFilter); err != nil { + return l, fmt.Errorf("Only username and usernamedn may be substituted in the group search filter string: %s", err) + } + l.GroupSearchFilter = grpSearchFilter + l.GroupNameAttribute = grpSearchNameAttr + subs, err = NewSubstituter("username", "test", "usernamedn", "test2") + if err != nil { + return l, err + } + if _, err := subs.Substitute(grpSearchBaseDN); err != nil { + return l, fmt.Errorf("Only username and usernamedn may be substituted in the base DN string: %s", err) + } + l.GroupSearchBaseDN = grpSearchBaseDN + } + return +} + +// Substituter - This type is to allow restricted runtime +// substitutions of variables in LDAP configuration items during +// runtime. +type Substituter struct { + vals map[string]string +} + +// NewSubstituter - sets up the substituter for usage, for e.g.: +// +// subber := NewSubstituter("username", "john") +func NewSubstituter(v ...string) (Substituter, error) { + if len(v)%2 != 0 { + return Substituter{}, errors.New("Need an even number of arguments") + } + vals := make(map[string]string) + for i := 0; i < len(v); i += 2 { + vals[v[i]] = v[i+1] + } + return Substituter{vals: vals}, nil +} + +// Substitute - performs substitution on the given string `t`. Returns +// an error if there are any variables in the input that do not have +// values in the substituter. E.g.: +// +// subber.Substitute("uid=${username},cn=users,dc=example,dc=com") +// +// returns "uid=john,cn=users,dc=example,dc=com" +// +// whereas: +// +// subber.Substitute("uid=${usernamedn}") +// +// returns an error. +func (s *Substituter) Substitute(t string) (string, error) { + for k, v := range s.vals { + re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k)) + t = re.ReplaceAllLiteralString(t, v) + } + // Check if all requested substitutions have been made. + re := regexp.MustCompile(`\$\{.*\}`) + if re.MatchString(t) { + return "", errors.New("unsupported substitution requested") + } + return t, nil +} diff --git a/cmd/config/ldap/config_test.go b/cmd/config/ldap/config_test.go new file mode 100644 index 000000000..526929e04 --- /dev/null +++ b/cmd/config/ldap/config_test.go @@ -0,0 +1,65 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ldap + +import ( + "fmt" + "testing" +) + +func TestSubstituter(t *testing.T) { + tests := []struct { + KV []string + SubstitutableStr string + SubstitutedStr string + ErrExpected bool + }{ + { + KV: []string{"username", "john"}, + SubstitutableStr: "uid=${username},cn=users,dc=example,dc=com", + SubstitutedStr: "uid=john,cn=users,dc=example,dc=com", + ErrExpected: false, + }, + { + KV: []string{"username", "john"}, + SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com", + ErrExpected: true, + }, + { + KV: []string{"username"}, + SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com", + ErrExpected: true, + }, + } + + for i, test := range tests { + test := test + t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) { + subber, err := NewSubstituter(test.KV...) + if err != nil && !test.ErrExpected { + t.Errorf("Unexpected failure %s", err) + } + gotStr, err := subber.Substitute(test.SubstitutableStr) + if err != nil && !test.ErrExpected { + t.Errorf("Unexpected failure %s", err) + } + if gotStr != test.SubstitutedStr { + t.Errorf("Expected %s, got %s", test.SubstitutedStr, gotStr) + } + }) + } +} diff --git a/cmd/disk-cache-backend.go b/cmd/disk-cache-backend.go index 7949025f8..736ed18ff 100644 --- a/cmd/disk-cache-backend.go +++ b/cmd/disk-cache-backend.go @@ -47,8 +47,6 @@ const ( cacheDataFile = "part.1" cacheMetaVersion = "1.0.0" - cacheEnvDelimiter = ";" - // SSECacheEncrypted is the metadata key indicating that the object // is a cache entry encrypted with cache KMS master key in globalCacheKMS. SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache" diff --git a/cmd/disk-cache.go b/cmd/disk-cache.go index b63782e45..4a59169c5 100644 --- a/cmd/disk-cache.go +++ b/cmd/disk-cache.go @@ -13,7 +13,9 @@ import ( "time" "github.com/djherbis/atime" + "github.com/minio/minio/cmd/config/cache" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/color" "github.com/minio/minio/pkg/wildcard" ) @@ -389,7 +391,7 @@ func (c *cacheObjects) hashIndex(bucket, object string) int { // newCache initializes the cacheFSObjects for the "drives" specified in config.json // or the global env overrides. -func newCache(config CacheConfig) ([]*diskCache, bool, error) { +func newCache(config cache.Config) ([]*diskCache, bool, error) { var caches []*diskCache ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{}) formats, migrating, err := loadAndValidateCacheFormat(ctx, config.Drives) @@ -446,7 +448,7 @@ func checkAtimeSupport(dir string) (err error) { return } func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) { - logStartupMessage(colorBlue("Cache migration initiated ....")) + logStartupMessage(color.Blue("Cache migration initiated ....")) var wg sync.WaitGroup errs := make([]error, len(c.cache)) @@ -482,7 +484,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) { c.migMutex.Lock() defer c.migMutex.Unlock() c.migrating = false - logStartupMessage(colorBlue("Cache migration completed successfully.")) + logStartupMessage(color.Blue("Cache migration completed successfully.")) } // PutObject - caches the uploaded object for single Put operations @@ -535,7 +537,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r * } // Returns cacheObjects for use by Server. -func newServerCacheObjects(ctx context.Context, config CacheConfig) (CacheObjectLayer, error) { +func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjectLayer, error) { // list of disk caches for cache "drives" specified in config.json or MINIO_CACHE_DRIVES env var. cache, migrateSw, err := newCache(config) if err != nil { diff --git a/cmd/endpoint-ellipses.go b/cmd/endpoint-ellipses.go index d456f98ba..0dfdbff16 100644 --- a/cmd/endpoint-ellipses.go +++ b/cmd/endpoint-ellipses.go @@ -18,12 +18,13 @@ package cmd import ( "fmt" - "os" "strconv" "strings" "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/ellipses" + "github.com/minio/minio/pkg/env" ) // This file implements and supports ellipses pattern for @@ -77,13 +78,13 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e } var customSetDriveCount uint64 - if v := os.Getenv("MINIO_ERASURE_SET_DRIVE_COUNT"); v != "" { + if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" { customSetDriveCount, err = strconv.ParseUint(v, 10, 64) if err != nil { - return nil, uiErrInvalidErasureSetSize(err) + return nil, config.ErrInvalidErasureSetSize(err) } if !isValidSetSize(customSetDriveCount) { - return nil, uiErrInvalidErasureSetSize(nil) + return nil, config.ErrInvalidErasureSetSize(nil) } } @@ -91,7 +92,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e for _, totalSize := range totalSizes { // Check if totalSize has minimum range upto setSize if totalSize < setSizes[0] || totalSize < customSetDriveCount { - return nil, uiErrInvalidNumberOfErasureEndpoints(nil) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil) } } @@ -125,17 +126,17 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e if customSetDriveCount > 0 { msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize)) if customSetDriveCount > setSize { - return nil, uiErrInvalidErasureSetSize(nil).Msg(msg) + return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg) } if setSize%customSetDriveCount != 0 { - return nil, uiErrInvalidErasureSetSize(nil).Msg(msg) + return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg) } setSize = customSetDriveCount } // Check whether setSize is with the supported range. if !isValidSetSize(setSize) { - return nil, uiErrInvalidNumberOfErasureEndpoints(nil) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil) } for i := range totalSizes { @@ -198,14 +199,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) { for i, arg := range args { patterns, perr := ellipses.FindEllipsesPatterns(arg) if perr != nil { - return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error()) + return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(perr.Error()) } argPatterns[i] = patterns } ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns)) if err != nil { - return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error()) + return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } ep.argPatterns = argPatterns @@ -254,7 +255,7 @@ func GetAllSets(args ...string) ([][]string, error) { for _, sargs := range setArgs { for _, arg := range sargs { if uniqueArgs.Contains(arg) { - return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args)) + return nil, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args)) } uniqueArgs.Add(arg) } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 80bbe13fa..31db5f766 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -21,7 +21,6 @@ import ( "fmt" "net" "net/url" - "os" "path" "path/filepath" "runtime" @@ -32,7 +31,9 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/minio/cli" "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/mountinfo" ) @@ -377,14 +378,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, return serverAddr, endpoints, setupType, err } if endpoint.Type() != PathEndpointType { - return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup") + return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup") } endpoints = append(endpoints, endpoint) setupType = FSSetupType // Check for cross device mounts if any. if err = checkCrossDeviceMounts(endpoints); err != nil { - return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error()) + return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error()) } return serverAddr, endpoints, setupType, nil } @@ -395,12 +396,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, var eps EndpointList eps, err = NewEndpointList(iargs...) if err != nil { - return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error()) + return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } // Check for cross device mounts if any. if err = checkCrossDeviceMounts(eps); err != nil { - return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error()) + return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } for _, ep := range eps { @@ -417,7 +418,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, } if err := endpoints.UpdateIsLocal(); err != nil { - return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error()) + return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } // Here all endpoints are URL style. @@ -445,7 +446,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, // No local endpoint found. if localEndpointCount == 0 { - return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found") + return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found") } // Check whether same path is not used in endpoints of a host on different port. @@ -461,7 +462,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, if IPSet, ok := pathIPMap[endpoint.Path]; ok { if !IPSet.Intersection(hostIPSet).IsEmpty() { return serverAddr, endpoints, setupType, - uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path)) + config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path)) } pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet) } else { @@ -479,7 +480,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, } if localPathSet.Contains(endpoint.Path) { return serverAddr, endpoints, setupType, - uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path)) + config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path)) } localPathSet.Add(endpoint.Path) } @@ -490,10 +491,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, if !localPortSet.Contains(serverAddrPort) { if len(localPortSet) > 1 { return serverAddr, endpoints, setupType, - uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints") + config.ErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints") } return serverAddr, endpoints, setupType, - uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports") + config.ErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports") } } @@ -571,9 +572,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, return serverAddr, endpoints, setupType, err } - _, dok := os.LookupEnv("MINIO_DOMAIN") - _, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS") - _, iok := os.LookupEnv("MINIO_PUBLIC_IPS") + _, dok := env.Lookup("MINIO_DOMAIN") + _, eok := env.Lookup("MINIO_ETCD_ENDPOINTS") + _, iok := env.Lookup("MINIO_PUBLIC_IPS") if dok && eok && !iok { updateDomainIPs(uniqueArgs) } diff --git a/cmd/environment.go b/cmd/environment.go index 52d585aad..ca95b07b4 100644 --- a/cmd/environment.go +++ b/cmd/environment.go @@ -18,11 +18,11 @@ import ( "encoding/hex" "errors" "fmt" - "os" "strconv" "strings" "github.com/minio/minio/cmd/crypto" + "github.com/minio/minio/pkg/env" ) const ( @@ -76,30 +76,6 @@ const ( EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE" ) -// Environment provides functions for accessing environment -// variables. -var Environment = environment{} - -type environment struct{} - -// Get retrieves the value of the environment variable named -// by the key. If the variable is present in the environment the -// value (which may be empty) is returned. Otherwise it returns -// the specified default value. -func (environment) Get(key, defaultValue string) string { - if v, ok := os.LookupEnv(key); ok { - return v - } - return defaultValue -} - -// Lookup retrieves the value of the environment variable named -// by the key. If the variable is present in the environment the -// value (which may be empty) is returned and the boolean is true. -// Otherwise the returned value will be empty and the boolean will -// be false. -func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) } - // LookupKMSConfig extracts the KMS configuration provided by environment // variables and merge them with the provided KMS configuration. The // merging follows the following rules: @@ -113,7 +89,7 @@ func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) // // It sets the global KMS configuration according to the merged configuration // on success. -func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) { +func LookupKMSConfig(config crypto.KMSConfig) (err error) { // Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint) config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath) diff --git a/cmd/format-fs.go b/cmd/format-fs.go index 588237079..3f0201caa 100644 --- a/cmd/format-fs.go +++ b/cmd/format-fs.go @@ -24,6 +24,7 @@ import ( "path" "time" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" ) @@ -153,7 +154,7 @@ func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) e return err } if version != formatFSVersionV2 { - return uiErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version)) + return config.ErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version)) } return nil } diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index d7150c925..7d5152d8c 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -32,6 +32,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lifecycle" "github.com/minio/minio/pkg/lock" @@ -116,7 +117,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { if err == errMinDiskSize { return nil, err } - return nil, uiErrUnableToWriteInBackend(err) + return nil, config.ErrUnableToWriteInBackend(err) } // Assign a new UUID for FS minio mode. Each server instance diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index eace4360e..7a34bd22c 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2017 MinIO, Inc. + * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,12 @@ package cmd import ( "net/http" - "os" "strings" + "github.com/minio/minio/cmd/config" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/hash" xnet "github.com/minio/minio/pkg/net" @@ -365,14 +366,14 @@ func parseGatewaySSE(s string) (gatewaySSE, error) { gwSlice = append(gwSlice, v) continue } - return nil, uiErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v) + return nil, config.ErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v) } return gatewaySSE(gwSlice), nil } // handle gateway env vars func handleGatewayEnvVars() { - gwsseVal, ok := os.LookupEnv("MINIO_GATEWAY_SSE") + gwsseVal, ok := env.Lookup("MINIO_GATEWAY_SSE") if ok { var err error GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal) diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 260495f61..71f518307 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -27,14 +27,17 @@ import ( "github.com/gorilla/mux" "github.com/minio/cli" + "github.com/minio/minio/cmd/config" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/certs" + "github.com/minio/minio/pkg/color" + "github.com/minio/minio/pkg/env" ) func init() { logger.Init(GOPATH, GOROOT) - logger.RegisterUIError(fmtError) + logger.RegisterError(config.FmtError) } var ( @@ -141,7 +144,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Validate if we have access, secret set through environment. if !globalIsEnvCreds { - logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway") + logger.Fatal(config.ErrEnvCredentialsMissingGateway(nil), "Unable to start gateway") } // Set system resources to maximum. @@ -250,7 +253,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { loadLoggers() // This is only to uniquely identify each gateway deployments. - globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID") + globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID()) logger.SetDeploymentID(globalDeploymentID) var cacheConfig = globalServerConfig.GetCacheConfig() @@ -298,7 +301,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { // Print a warning message if gateway is not ready for production before the startup banner. if !gw.Production() { - logStartupMessage(colorYellow(" *** Warning: Not Ready for Production ***")) + logStartupMessage(color.Yellow(" *** Warning: Not Ready for Production ***")) } // Print gateway startup message. diff --git a/cmd/gateway-startup-msg.go b/cmd/gateway-startup-msg.go index 324da7942..12d363ed1 100644 --- a/cmd/gateway-startup-msg.go +++ b/cmd/gateway-startup-msg.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "strings" + + "github.com/minio/minio/pkg/color" ) // Prints the formatted startup message. @@ -55,15 +57,15 @@ func printGatewayCommonMsg(apiEndpoints []string) { apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. - logStartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) - if isTerminal() && !globalCLIContext.Anonymous { - logStartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) - logStartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) + logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) + if color.IsTerminal() && !globalCLIContext.Anonymous { + logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey))) + logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey))) } printEventNotifiers() if globalIsBrowserEnabled { - logStartupMessage(colorBlue("\nBrowser Access:")) + logStartupMessage(color.Blue("\nBrowser Access:")) logStartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } } diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 17aa815d4..baa6ba7f3 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -30,7 +30,6 @@ import ( "path" "strconv" - "os" "regexp" "strings" "time" @@ -41,6 +40,7 @@ import ( miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/policy" "github.com/minio/minio/pkg/policy/condition" @@ -158,7 +158,7 @@ EXAMPLES: // Handler for 'minio gateway gcs' command line. func gcsGatewayMain(ctx *cli.Context) { projectID := ctx.Args().First() - if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + if projectID == "" && env.Get("GOOGLE_APPLICATION_CREDENTIALS", "") == "" { logger.LogIf(context.Background(), errGCSProjectIDNotFound) cli.ShowCommandHelpAndExit(ctx, "gcs", 1) } @@ -190,7 +190,7 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) if g.projectID == "" { // If project ID is not provided on command line, we figure it out // from the credentials.json file. - g.projectID, err = gcsParseProjectID(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) + g.projectID, err = gcsParseProjectID(env.Get("GOOGLE_APPLICATION_CREDENTIALS", "")) if err != nil { return nil, err } diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index d54a23cf1..fa52dfb5b 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -41,6 +41,7 @@ import ( minio "github.com/minio/minio/cmd" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/env" xnet "github.com/minio/minio/pkg/net" ) @@ -126,32 +127,24 @@ func (g *HDFS) Name() string { } func getKerberosClient() (*krb.Client, error) { - configPath := os.Getenv("KRB5_CONFIG") - if configPath == "" { - configPath = "/etc/krb5.conf" + cfg, err := config.Load(env.Get("KRB5_CONFIG", "/etc/krb5.conf")) + if err != nil { + return nil, err } - cfg, err := config.Load(configPath) + u, err := user.Current() if err != nil { return nil, err } - // Determine the ccache location from the environment, - // falling back to the default location. - ccachePath := os.Getenv("KRB5CCNAME") + // Determine the ccache location from the environment, falling back to the default location. + ccachePath := env.Get("KRB5CCNAME", fmt.Sprintf("/tmp/krb5cc_%s", u.Uid)) if strings.Contains(ccachePath, ":") { if strings.HasPrefix(ccachePath, "FILE:") { ccachePath = strings.TrimPrefix(ccachePath, "FILE:") } else { return nil, fmt.Errorf("unable to use kerberos ccache: %s", ccachePath) } - } else if ccachePath == "" { - u, err := user.Current() - if err != nil { - return nil, err - } - - ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) } ccache, err := credentials.LoadCCache(ccachePath) @@ -192,20 +185,18 @@ func (g *HDFS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error opts.Addresses = addresses } + u, err := user.Current() + if err != nil { + return nil, fmt.Errorf("Unable to lookup local user: %s", err) + } + if opts.KerberosClient != nil { opts.KerberosClient, err = getKerberosClient() if err != nil { return nil, fmt.Errorf("Unable to initialize kerberos client: %s", err) } } else { - opts.User = os.Getenv("HADOOP_USER_NAME") - if opts.User == "" { - u, err := user.Current() - if err != nil { - return nil, fmt.Errorf("Unable to lookup local user: %s", err) - } - opts.User = u.Username - } + opts.User = env.Get("HADOOP_USER_NAME", u.Username) } clnt, err := hdfs.NewClient(opts) diff --git a/cmd/globals.go b/cmd/globals.go index d5efb7de7..915051c32 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -18,16 +18,13 @@ package cmd import ( "crypto/x509" - "fmt" "os" "time" - isatty "github.com/mattn/go-isatty" "github.com/minio/minio-go/v6/pkg/set" etcd "github.com/coreos/etcd/clientv3" humanize "github.com/dustin/go-humanize" - "github.com/fatih/color" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/pkg/auth" @@ -269,7 +266,7 @@ var ( standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"} // Authorization validators list. - globalIAMValidators *openid.Validators + globalOpenIDValidators *openid.Validators // OPA policy system. globalPolicyOPA *iampolicy.Opa @@ -288,64 +285,6 @@ var ( // Add new variable global values here. ) -// global colors. -var ( - // Check if we stderr, stdout are dumb terminals, we do not apply - // ansi coloring on dumb terminals. - isTerminal = func() bool { - return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd()) - } - - colorBold = func() func(a ...interface{}) string { - if isTerminal() { - return color.New(color.Bold).SprintFunc() - } - return fmt.Sprint - }() - colorRed = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgRed).SprintfFunc() - } - return fmt.Sprintf - }() - colorBlue = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgBlue).SprintfFunc() - } - return fmt.Sprintf - }() - colorYellow = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgYellow).SprintfFunc() - } - return fmt.Sprintf - }() - colorCyanBold = func() func(a ...interface{}) string { - if isTerminal() { - color.New(color.FgCyan, color.Bold).SprintFunc() - } - return fmt.Sprint - }() - colorYellowBold = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgYellow, color.Bold).SprintfFunc() - } - return fmt.Sprintf - }() - colorBgYellow = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.BgYellow).SprintfFunc() - } - return fmt.Sprintf - }() - colorBlack = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgBlack).SprintfFunc() - } - return fmt.Sprintf - }() -) - // Returns minio global information, as a key value map. // returned list of global values is not an exhaustive // list. Feel free to add new relevant fields. diff --git a/cmd/ldap-ops.go b/cmd/ldap-ops.go deleted file mode 100644 index 0fe345f71..000000000 --- a/cmd/ldap-ops.go +++ /dev/null @@ -1,180 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "log" - "os" - "regexp" - "time" - - ldap "gopkg.in/ldap.v3" -) - -const ( - defaultLDAPExpiry = time.Hour * 1 -) - -// ldapServerConfig contains server connectivity information. -type ldapServerConfig struct { - IsEnabled bool `json:"enabled"` - - // E.g. "ldap.minio.io:636" - ServerAddr string `json:"serverAddr"` - - // STS credentials expiry duration - STSExpiryDuration string `json:"stsExpiryDuration"` - stsExpiryDuration time.Duration // contains converted value - rootCAs *x509.CertPool // contains custom CAs for ldaps server. - - // Skips TLS verification (for testing, not - // recommended in production). - SkipTLSVerify bool `json:"skipTLSverify"` - - // Format string for usernames - UsernameFormat string `json:"usernameFormat"` - - GroupSearchBaseDN string `json:"groupSearchBaseDN"` - GroupSearchFilter string `json:"groupSearchFilter"` - GroupNameAttribute string `json:"groupNameAttribute"` -} - -func (l *ldapServerConfig) Connect() (ldapConn *ldap.Conn, err error) { - if l == nil { - // Happens when LDAP is not configured. - return - } - if l.SkipTLSVerify { - ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.rootCAs, InsecureSkipVerify: true}) - } else { - ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.rootCAs}) - } - return -} - -// newLDAPConfigFromEnv loads configuration from the environment -func newLDAPConfigFromEnv(rootCAs *x509.CertPool) (l ldapServerConfig, err error) { - if ldapServer, ok := os.LookupEnv("MINIO_IDENTITY_LDAP_SERVER_ADDR"); ok { - l.IsEnabled = ok - l.ServerAddr = ldapServer - - // Save root CAs - l.rootCAs = rootCAs - l.SkipTLSVerify = os.Getenv("MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY") == "true" - - if v := os.Getenv("MINIO_IDENTITY_LDAP_STS_EXPIRY"); v != "" { - expDur, err := time.ParseDuration(v) - if err != nil { - return l, errors.New("LDAP expiry time err:" + err.Error()) - } - if expDur <= 0 { - return l, errors.New("LDAP expiry time has to be positive") - } - l.STSExpiryDuration = v - l.stsExpiryDuration = expDur - } else { - l.stsExpiryDuration = defaultLDAPExpiry - } - - if v := os.Getenv("MINIO_IDENTITY_LDAP_USERNAME_FORMAT"); v != "" { - subs := newSubstituter("username", "test") - if _, err := subs.substitute(v); err != nil { - return l, errors.New("Only username may be substituted in the username format") - } - l.UsernameFormat = v - } - - grpSearchFilter := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER") - grpSearchNameAttr := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE") - grpSearchBaseDN := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN") - - // Either all group params must be set or none must be set. - allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == "" - allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != "" - if !allNotSet && !allSet { - return l, errors.New("All group related parameters must be set") - } - - if allSet { - subs := newSubstituter("username", "test", "usernamedn", "test2") - if _, err := subs.substitute(grpSearchFilter); err != nil { - return l, errors.New("Only username and usernamedn may be substituted in the group search filter string") - } - l.GroupSearchFilter = grpSearchFilter - - l.GroupNameAttribute = grpSearchNameAttr - - subs = newSubstituter("username", "test", "usernamedn", "test2") - if _, err := subs.substitute(grpSearchBaseDN); err != nil { - return l, errors.New("Only username and usernamedn may be substituted in the base DN string") - } - l.GroupSearchBaseDN = grpSearchBaseDN - } - } - return -} - -// substituter - This type is to allow restricted runtime -// substitutions of variables in LDAP configuration items during -// runtime. -type substituter struct { - vals map[string]string -} - -// newSubstituter - sets up the substituter for usage, for e.g.: -// -// subber := newSubstituter("username", "john") -func newSubstituter(v ...string) substituter { - if len(v)%2 != 0 { - log.Fatal("Need an even number of arguments") - } - vals := make(map[string]string) - for i := 0; i < len(v); i += 2 { - vals[v[i]] = v[i+1] - } - return substituter{vals: vals} -} - -// substitute - performs substitution on the given string `t`. Returns -// an error if there are any variables in the input that do not have -// values in the substituter. E.g.: -// -// subber.substitute("uid=${username},cn=users,dc=example,dc=com") -// -// returns "uid=john,cn=users,dc=example,dc=com" -// -// whereas: -// -// subber.substitute("uid=${usernamedn}") -// -// returns an error. -func (s *substituter) substitute(t string) (string, error) { - for k, v := range s.vals { - re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k)) - t = re.ReplaceAllLiteralString(t, v) - } - // Check if all requested substitutions have been made. - re := regexp.MustCompile(`\$\{.*\}`) - if re.MatchString(t) { - return "", errors.New("unsupported substitution requested") - } - return t, nil -} diff --git a/cmd/logger/console.go b/cmd/logger/console.go index ec9c69093..b23b232b0 100644 --- a/cmd/logger/console.go +++ b/cmd/logger/console.go @@ -25,6 +25,7 @@ import ( c "github.com/minio/mc/pkg/console" "github.com/minio/minio/cmd/logger/message/log" + "github.com/minio/minio/pkg/color" ) // Console interface describes the methods that need to be implemented to satisfy the interface requirements. @@ -89,8 +90,8 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) { var ( logTag = "ERROR" - logBanner = ColorBgRed(ColorFgWhite(ColorBold(logTag))) + " " - emptyBanner = ColorBgRed(strings.Repeat(" ", len(logTag))) + " " + logBanner = color.BgRed(color.FgWhite(color.Bold(logTag))) + " " + emptyBanner = color.BgRed(strings.Repeat(" ", len(logTag))) + " " bannerWidth = len(logTag) + 1 ) diff --git a/cmd/logger/logger.go b/cmd/logger/logger.go index cbd36007e..9db9ce2d0 100644 --- a/cmd/logger/logger.go +++ b/cmd/logger/logger.go @@ -139,9 +139,9 @@ func IsQuiet() bool { return quietFlag } -// RegisterUIError registers the specified rendering function. This latter +// RegisterError registers the specified rendering function. This latter // will be called for a pretty rendering of fatal errors. -func RegisterUIError(f func(string, error, bool) string) { +func RegisterError(f func(string, error, bool) string) { errorFmtFunc = f } diff --git a/cmd/logger/target/console/console.go b/cmd/logger/target/console/console.go index bfd1f751a..3cd7be559 100644 --- a/cmd/logger/target/console/console.go +++ b/cmd/logger/target/console/console.go @@ -24,6 +24,7 @@ import ( "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger/message/log" + "github.com/minio/minio/pkg/color" ) // Target implements loggerTarget to send log @@ -103,7 +104,7 @@ func (c *Target) Send(e interface{}) error { tagString = "\n " + tagString } - var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message)) + var msg = color.FgRed(color.Bold(entry.Trace.Message)) var output = fmt.Sprintf("\n%s\n%s%s%s%s%s%s\nError: %s%s\n%s", apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent, msg, tagString, strings.Join(trace, "\n")) diff --git a/cmd/logger/utils.go b/cmd/logger/utils.go index 9a1837d05..22649c2c1 100644 --- a/cmd/logger/utils.go +++ b/cmd/logger/utils.go @@ -18,45 +18,9 @@ package logger import ( "fmt" - "os" "regexp" - "github.com/fatih/color" - isatty "github.com/mattn/go-isatty" -) - -// Global colors. -var ( - // Check if we stderr, stdout are dumb terminals, we do not apply - // ansi coloring on dumb terminals. - isTerminal = func() bool { - return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd()) - } - - ColorBold = func() func(a ...interface{}) string { - if isTerminal() { - return color.New(color.Bold).SprintFunc() - } - return fmt.Sprint - }() - ColorFgRed = func() func(a ...interface{}) string { - if isTerminal() { - return color.New(color.FgRed).SprintFunc() - } - return fmt.Sprint - }() - ColorBgRed = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.BgRed).SprintfFunc() - } - return fmt.Sprintf - }() - ColorFgWhite = func() func(format string, a ...interface{}) string { - if isTerminal() { - return color.New(color.FgWhite).SprintfFunc() - } - return fmt.Sprintf - }() + "github.com/minio/minio/pkg/color" ) var ansiRE = regexp.MustCompile("(\x1b[^m]*m)") @@ -68,19 +32,19 @@ func ansiEscape(format string, args ...interface{}) { } func ansiMoveRight(n int) { - if isTerminal() { + if color.IsTerminal() { ansiEscape("[%dC", n) } } func ansiSaveAttributes() { - if isTerminal() { + if color.IsTerminal() { ansiEscape("7") } } func ansiRestoreAttributes() { - if isTerminal() { + if color.IsTerminal() { ansiEscape("8") } diff --git a/cmd/net.go b/cmd/net.go index dadccc334..52154622a 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -27,6 +27,7 @@ import ( "syscall" "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" ) @@ -334,7 +335,7 @@ func sameLocalAddrs(addr1, addr2 string) (bool, error) { func CheckLocalServerAddr(serverAddr string) error { host, port, err := net.SplitHostPort(serverAddr) if err != nil { - return uiErrInvalidAddressFlag(err) + return config.ErrInvalidAddressFlag(err) } // Strip off IPv6 zone information. @@ -345,9 +346,9 @@ func CheckLocalServerAddr(serverAddr string) error { // Check whether port is a valid port number. p, err := strconv.Atoi(port) if err != nil { - return uiErrInvalidAddressFlag(err).Msg("invalid port number") + return config.ErrInvalidAddressFlag(err).Msg("invalid port number") } else if p < 1 || p > 65535 { - return uiErrInvalidAddressFlag(nil).Msg("port number must be between 1 to 65535") + return config.ErrInvalidAddressFlag(nil).Msg("port number must be between 1 to 65535") } // 0.0.0.0 is a wildcard address and refers to local network @@ -359,7 +360,7 @@ func CheckLocalServerAddr(serverAddr string) error { return err } if !isLocalHost { - return uiErrInvalidAddressFlag(nil).Msg("host in server address should be this server") + return config.ErrInvalidAddressFlag(nil).Msg("host in server address should be this server") } } diff --git a/cmd/server-main.go b/cmd/server-main.go index b6facdf78..32eda2ef9 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -28,14 +28,17 @@ import ( "github.com/minio/cli" "github.com/minio/dsync/v2" + "github.com/minio/minio/cmd/config" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/certs" + "github.com/minio/minio/pkg/color" + "github.com/minio/minio/pkg/env" ) func init() { logger.Init(GOPATH, GOROOT) - logger.RegisterUIError(fmtError) + logger.RegisterError(config.FmtError) gob.Register(VerifyFileError("")) gob.Register(DeleteFileError("")) } @@ -133,7 +136,7 @@ EXAMPLES: // Checks if endpoints are either available through environment // or command line, returns false if both fails. func endpointsPresent(ctx *cli.Context) bool { - _, ok := os.LookupEnv("MINIO_ENDPOINTS") + _, ok := env.Lookup("MINIO_ENDPOINTS") if !ok { ok = ctx.Args().Present() } @@ -150,12 +153,12 @@ func serverHandleCmdArgs(ctx *cli.Context) { var err error if len(ctx.Args()) > serverCommandLineArgsMax { - uErr := uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments", + uErr := config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments", len(ctx.Args()))) logger.FatalIf(uErr, "Unable to validate passed endpoints") } - endpoints := strings.Fields(os.Getenv("MINIO_ENDPOINTS")) + endpoints := strings.Fields(env.Get("MINIO_ENDPOINTS", "")) if len(endpoints) > 0 { globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...) } else { @@ -184,7 +187,7 @@ func serverHandleEnvVars() { // Handle common environment variables. handleCommonEnvVars() - if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" { + if serverRegion := env.Get("MINIO_REGION", ""); serverRegion != "" { // region Envs are set globally. globalIsEnvRegion = true globalServerRegion = serverRegion @@ -222,10 +225,10 @@ func serverMain(ctx *cli.Context) { // Is distributed setup, error out if no certificates are found for HTTPS endpoints. if globalIsDistXL { if globalEndpoints.IsHTTPS() && !globalIsSSL { - logger.Fatal(uiErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server") + logger.Fatal(config.ErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server") } if !globalEndpoints.IsHTTPS() && globalIsSSL { - logger.Fatal(uiErrCertsAndHTTPEndpoints(nil), "Unable to start the server") + logger.Fatal(config.ErrCertsAndHTTPEndpoints(nil), "Unable to start the server") } } @@ -235,7 +238,7 @@ func serverMain(ctx *cli.Context) { } if globalIsDiskCacheEnabled { - logger.StartupMessage(colorRed(colorBold("Disk caching is allowed only for gateway deployments"))) + logger.StartupMessage(color.Red(color.Bold("Disk caching is allowed only for gateway deployments"))) } // FIXME: This code should be removed in future releases and we should have mandatory @@ -245,14 +248,14 @@ func serverMain(ctx *cli.Context) { // Check for backward compatibility and newer style. if !globalIsEnvCreds && globalIsDistXL { // Try to load old config file if any, for backward compatibility. - var config = &serverConfig{} - if _, err = Load(getConfigFile(), config); err == nil { - globalActiveCred = config.Credential + var cfg = &serverConfig{} + if _, err = Load(getConfigFile(), cfg); err == nil { + globalActiveCred = cfg.Credential } if os.IsNotExist(err) { - if _, err = Load(getConfigFile()+".deprecated", config); err == nil { - globalActiveCred = config.Credential + if _, err = Load(getConfigFile()+".deprecated", cfg); err == nil { + globalActiveCred = cfg.Credential } } @@ -262,7 +265,8 @@ func serverMain(ctx *cli.Context) { logger.Info(`Supplying credentials from your 'config.json' is **DEPRECATED**, Access key and Secret key in distributed server mode is expected to be specified with environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY. This approach will become mandatory in future releases, please migrate to this approach soon.`) } else { // Credential is not available anywhere by both means, we cannot start distributed setup anymore, fail eagerly. - logger.Fatal(uiErrEnvCredentialsMissingDistributed(nil), "Unable to initialize the server in distributed mode") + logger.Fatal(config.ErrEnvCredentialsMissingDistributed(nil), + "Unable to initialize the server in distributed mode") } } } @@ -293,7 +297,7 @@ func serverMain(ctx *cli.Context) { var handler http.Handler handler, err = configureServerHandler(globalEndpoints) if err != nil { - logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services") + logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services") } var getCert certs.GetCertificateFunc diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 02b0e7856..06683dae0 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -25,6 +25,7 @@ import ( "strings" humanize "github.com/dustin/go-humanize" + color "github.com/minio/minio/pkg/color" xnet "github.com/minio/minio/pkg/net" ) @@ -121,18 +122,18 @@ func printServerCommonMsg(apiEndpoints []string) { apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. - logStartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) - if isTerminal() && !globalCLIContext.Anonymous { - logStartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) - logStartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) + logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) + if color.IsTerminal() && !globalCLIContext.Anonymous { + logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey))) + logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey))) if region != "" { - logStartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) + logStartupMessage(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 3), region))) } } printEventNotifiers() if globalIsBrowserEnabled { - logStartupMessage(colorBlue("\nBrowser Access:")) + logStartupMessage(color.Blue("\nBrowser Access:")) logStartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } } @@ -144,9 +145,9 @@ func printEventNotifiers() { return } - arnMsg := colorBlue("SQS ARNs: ") + arnMsg := color.Blue("SQS ARNs: ") for _, arn := range arns { - arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(arn), 1), arn)) + arnMsg += color.Bold(fmt.Sprintf(getFormatStr(len(arn), 1), arn)) } logStartupMessage(arnMsg) @@ -159,13 +160,15 @@ func printCLIAccessMsg(endPoint string, alias string) { cred := globalServerConfig.GetCredential() // Configure 'mc', following block prints platform specific information for minio client. - if isTerminal() { - logStartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) + if color.IsTerminal() { + logStartupMessage(color.Blue("\nCommand-line Access: ") + mcQuickStartGuide) if runtime.GOOS == globalWindowsOSName { - mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) + mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, + endPoint, cred.AccessKey, cred.SecretKey) logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } else { - mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) + mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, + endPoint, cred.AccessKey, cred.SecretKey) logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } } @@ -173,12 +176,12 @@ func printCLIAccessMsg(endPoint string, alias string) { // Prints startup message for Object API acces, prints link to our SDK documentation. func printObjectAPIMsg() { - logStartupMessage(colorBlue("\nObject API (Amazon S3 compatible):")) - logStartupMessage(colorBlue(" Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide)) - logStartupMessage(colorBlue(" Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide)) - logStartupMessage(colorBlue(" Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide)) - logStartupMessage(colorBlue(" JavaScript: ") + jsQuickStartGuide) - logStartupMessage(colorBlue(" .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide)) + logStartupMessage(color.Blue("\nObject API (Amazon S3 compatible):")) + logStartupMessage(color.Blue(" Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide)) + logStartupMessage(color.Blue(" Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide)) + logStartupMessage(color.Blue(" Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide)) + logStartupMessage(color.Blue(" JavaScript: ") + jsQuickStartGuide) + logStartupMessage(color.Blue(" .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide)) } // Get formatted disk/storage info message. @@ -186,7 +189,7 @@ func getStorageInfoMsg(storageInfo StorageInfo) string { var msg string if storageInfo.Backend.Type == BackendErasure { diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks, storageInfo.Backend.OfflineDisks) - msg += colorBlue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo) + msg += color.Blue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo) } return msg } @@ -199,7 +202,7 @@ func printStorageInfo(storageInfo StorageInfo) { } func printCacheStorageInfo(storageInfo CacheStorageInfo) { - msg := fmt.Sprintf("%s %s Free, %s Total", colorBlue("Cache Capacity:"), + msg := fmt.Sprintf("%s %s Free, %s Total", color.Blue("Cache Capacity:"), humanize.IBytes(uint64(storageInfo.Free)), humanize.IBytes(uint64(storageInfo.Total))) logStartupMessage(msg) @@ -207,14 +210,14 @@ func printCacheStorageInfo(storageInfo CacheStorageInfo) { // Prints certificate expiry date warning func getCertificateChainMsg(certs []*x509.Certificate) string { - msg := colorBlue("\nCertificate expiry info:\n") + msg := color.Blue("\nCertificate expiry info:\n") totalCerts := len(certs) var expiringCerts int for i := totalCerts - 1; i >= 0; i-- { cert := certs[i] if cert.NotAfter.Before(UTCNow().Add(globalMinioCertExpireWarnDays)) { expiringCerts++ - msg += fmt.Sprintf(colorBold("#%d %s will expire on %s\n"), expiringCerts, cert.Subject.CommonName, cert.NotAfter) + msg += fmt.Sprintf(color.Bold("#%d %s will expire on %s\n"), expiringCerts, cert.Subject.CommonName, cert.NotAfter) } } if expiringCerts > 0 { diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go index b23a02256..87839d0e6 100644 --- a/cmd/server-startup-msg_test.go +++ b/cmd/server-startup-msg_test.go @@ -25,6 +25,8 @@ import ( "strings" "testing" "time" + + "github.com/minio/minio/pkg/color" ) // Tests if we generate storage info. @@ -53,8 +55,8 @@ func TestCertificateExpiryInfo(t *testing.T) { }, } - expectedMsg := colorBlue("\nCertificate expiry info:\n") + - colorBold(fmt.Sprintf("#1 Test cert will expire on %s\n", expiredDate)) + expectedMsg := color.Blue("\nCertificate expiry info:\n") + + color.Bold(fmt.Sprintf("#1 Test cert will expire on %s\n", expiredDate)) // When msg := getCertificateChainMsg(fakeCerts) diff --git a/cmd/storage-class.go b/cmd/storage-class.go index 370ce71ee..ba610ec3a 100644 --- a/cmd/storage-class.go +++ b/cmd/storage-class.go @@ -22,6 +22,8 @@ import ( "fmt" "strconv" "strings" + + "github.com/minio/minio/cmd/config" ) const ( @@ -104,20 +106,20 @@ func parseStorageClass(storageClassEnv string) (sc storageClass, err error) { // only two elements allowed in the string - "scheme" and "number of parity disks" if len(s) > 2 { - return storageClass{}, uiErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv) + return storageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv) } else if len(s) < 2 { - return storageClass{}, uiErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv) + return storageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv) } // only allowed scheme is "EC" if s[0] != supportedStorageClassScheme { - return storageClass{}, uiErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC") + return storageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC") } // Number of parity disks should be integer parityDisks, err := strconv.Atoi(s[1]) if err != nil { - return storageClass{}, uiErrStorageClassValue(err) + return storageClass{}, config.ErrStorageClassValue(err) } sc = storageClass{ diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index ef7b95d73..ff05b115a 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -29,6 +29,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/minio/minio/cmd/config" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" ) @@ -573,7 +574,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) { } storage, err := newPosix(endpoint.Path) if err != nil { - logger.Fatal(uiErrUnableToWriteInBackend(err), "Unable to initialize posix backend") + logger.Fatal(config.ErrUnableToWriteInBackend(err), + "Unable to initialize posix backend") } server := &storageRESTServer{storage, mustGetUUID()} diff --git a/cmd/sts-handlers.go b/cmd/sts-handlers.go index bc5008430..f2707fdc3 100644 --- a/cmd/sts-handlers.go +++ b/cmd/sts-handlers.go @@ -24,6 +24,7 @@ import ( "net/http" "github.com/gorilla/mux" + xldap "github.com/minio/minio/cmd/config/ldap" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" @@ -263,12 +264,12 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ ctx = newContext(r, w, action) defer logger.AuditLog(w, r, action, nil) - if globalIAMValidators == nil { + if globalOpenIDValidators == nil { writeSTSErrorResponse(ctx, w, ErrSTSNotInitialized, errServerNotInitialized) return } - v, err := globalIAMValidators.Get("jwt") + v, err := globalOpenIDValidators.Get("jwt") if err != nil { writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) return @@ -471,10 +472,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r * return } - usernameSubs := newSubstituter("username", ldapUsername) + usernameSubs, _ := xldap.NewSubstituter("username", ldapUsername) // We ignore error below as we already validated the username // format string at startup. - usernameDN, _ := usernameSubs.substitute(globalServerConfig.LDAPServerConfig.UsernameFormat) + usernameDN, _ := usernameSubs.Substitute(globalServerConfig.LDAPServerConfig.UsernameFormat) // Bind with user credentials to validate the password err = ldapConn.Bind(usernameDN, ldapPassword) if err != nil { @@ -486,14 +487,14 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r * if globalServerConfig.LDAPServerConfig.GroupSearchFilter != "" { // Verified user credentials. Now we find the groups they are // a member of. - searchSubs := newSubstituter( + searchSubs, _ := xldap.NewSubstituter( "username", ldapUsername, "usernamedn", usernameDN, ) // We ignore error below as we already validated the search string // at startup. - groupSearchFilter, _ := searchSubs.substitute(globalServerConfig.LDAPServerConfig.GroupSearchFilter) - baseDN, _ := searchSubs.substitute(globalServerConfig.LDAPServerConfig.GroupSearchBaseDN) + groupSearchFilter, _ := searchSubs.Substitute(globalServerConfig.LDAPServerConfig.GroupSearchFilter) + baseDN, _ := searchSubs.Substitute(globalServerConfig.LDAPServerConfig.GroupSearchBaseDN) searchRequest := ldap.NewSearchRequest( baseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, @@ -513,7 +514,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r * groups = append(groups, entry.Attributes[0].Values...) } } - expiryDur := globalServerConfig.LDAPServerConfig.stsExpiryDuration + expiryDur := globalServerConfig.LDAPServerConfig.GetExpiryDuration() m := map[string]interface{}{ "exp": UTCNow().Add(expiryDur).Unix(), "ldapUser": ldapUsername, diff --git a/cmd/update-notifier.go b/cmd/update-notifier.go index cf9f1bd77..f80c7d532 100644 --- a/cmd/update-notifier.go +++ b/cmd/update-notifier.go @@ -25,6 +25,7 @@ import ( "github.com/cheggaaa/pb" humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/pkg/color" ) // prepareUpdateMessage - prepares the update message, only if a @@ -54,8 +55,8 @@ func colorizeUpdateMessage(updateString string, newerThan string) string { line2Length := len(fmt.Sprintf(msgLine2Fmt, updateString)) // Populate lines with color coding. - line1InColor := fmt.Sprintf(msgLine1Fmt, colorYellowBold(newerThan)) - line2InColor := fmt.Sprintf(msgLine2Fmt, colorCyanBold(updateString)) + line1InColor := fmt.Sprintf(msgLine1Fmt, color.YellowBold(newerThan)) + line2InColor := fmt.Sprintf(msgLine2Fmt, color.CyanBold(updateString)) // calculate the rectangular box size. maxContentWidth := int(math.Max(float64(line1Length), float64(line2Length))) @@ -89,10 +90,10 @@ func colorizeUpdateMessage(updateString string, newerThan string) string { } lines := []string{ - colorYellowBold(topLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + topRightChar), + color.YellowBold(topLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + topRightChar), vertBarChar + line1InColor + strings.Repeat(" ", maxContentWidth-line1Length) + vertBarChar, vertBarChar + line2InColor + strings.Repeat(" ", maxContentWidth-line2Length) + vertBarChar, - colorYellowBold(bottomLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + bottomRightChar), + color.YellowBold(bottomLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + bottomRightChar), } return "\n" + strings.Join(lines, "\n") + "\n" } diff --git a/cmd/update-notifier_test.go b/cmd/update-notifier_test.go index d8f32badf..b175ede39 100644 --- a/cmd/update-notifier_test.go +++ b/cmd/update-notifier_test.go @@ -21,6 +21,8 @@ import ( "strings" "testing" "time" + + "github.com/minio/minio/pkg/color" ) // Tests update notifier string builder. @@ -68,8 +70,8 @@ func TestPrepareUpdateMessage(t *testing.T) { for i, testCase := range testCases { output := prepareUpdateMessage(testCase.dlURL, testCase.older) - line1 := fmt.Sprintf("%s %s", plainMsg, colorYellowBold(testCase.expectedSubStr)) - line2 := fmt.Sprintf("Update: %s", colorCyanBold(testCase.dlURL)) + line1 := fmt.Sprintf("%s %s", plainMsg, color.YellowBold(testCase.expectedSubStr)) + line2 := fmt.Sprintf("Update: %s", color.CyanBold(testCase.dlURL)) // Uncomment below to see message appearance: // fmt.Println(output) switch { diff --git a/cmd/update.go b/cmd/update.go index ce01b2544..88fda517d 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -35,6 +35,7 @@ import ( "github.com/inconshreveable/go-update" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/env" xnet "github.com/minio/minio/pkg/net" _ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier. ) @@ -144,7 +145,7 @@ func IsDocker() bool { func IsDCOS() bool { // http://mesos.apache.org/documentation/latest/docker-containerizer/ // Mesos docker containerizer sets this value - return os.Getenv("MESOS_CONTAINER_NAME") != "" + return env.Get("MESOS_CONTAINER_NAME", "") != "" } // IsKubernetes returns true if minio is running in kubernetes. @@ -153,7 +154,7 @@ func IsKubernetes() bool { // indeed running inside a kubernetes pod // is KUBERNETES_SERVICE_HOST but in future // we might need to enhance this. - return os.Getenv("KUBERNETES_SERVICE_HOST") != "" + return env.Get("KUBERNETES_SERVICE_HOST", "") != "" } // IsBOSH returns true if minio is deployed from a bosh package @@ -247,7 +248,7 @@ func getUserAgent(mode string) string { uaAppend(" MinIO/", ReleaseTag) uaAppend(" MinIO/", CommitID) if IsDCOS() { - universePkgVersion := os.Getenv("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION") + universePkgVersion := env.Get("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION", "") // On DC/OS environment try to the get universe package version. if universePkgVersion != "" { uaAppend(" MinIO/universe-", universePkgVersion) @@ -262,7 +263,7 @@ func getUserAgent(mode string) string { } } - pcfTileVersion := os.Getenv("MINIO_PCF_TILE_VERSION") + pcfTileVersion := env.Get("MINIO_PCF_TILE_VERSION", "") if pcfTileVersion != "" { uaAppend(" MinIO/pcf-tile-", pcfTileVersion) } diff --git a/pkg/color/color.go b/pkg/color/color.go new file mode 100644 index 000000000..ecd405f84 --- /dev/null +++ b/pkg/color/color.go @@ -0,0 +1,85 @@ +package color + +import ( + "fmt" + "os" + + "github.com/fatih/color" + "github.com/mattn/go-isatty" +) + +// global colors. +var ( + // Check if we stderr, stdout are dumb terminals, we do not apply + // ansi coloring on dumb terminals. + IsTerminal = func() bool { + return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd()) + } + + Bold = func() func(a ...interface{}) string { + if IsTerminal() { + return color.New(color.Bold).SprintFunc() + } + return fmt.Sprint + }() + Red = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgRed).SprintfFunc() + } + return fmt.Sprintf + }() + Blue = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgBlue).SprintfFunc() + } + return fmt.Sprintf + }() + Yellow = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgYellow).SprintfFunc() + } + return fmt.Sprintf + }() + CyanBold = func() func(a ...interface{}) string { + if IsTerminal() { + color.New(color.FgCyan, color.Bold).SprintFunc() + } + return fmt.Sprint + }() + YellowBold = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgYellow, color.Bold).SprintfFunc() + } + return fmt.Sprintf + }() + BgYellow = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.BgYellow).SprintfFunc() + } + return fmt.Sprintf + }() + Black = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgBlack).SprintfFunc() + } + return fmt.Sprintf + }() + FgRed = func() func(a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgRed).SprintFunc() + } + return fmt.Sprint + }() + BgRed = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.BgRed).SprintfFunc() + } + return fmt.Sprintf + }() + FgWhite = func() func(format string, a ...interface{}) string { + if IsTerminal() { + return color.New(color.FgWhite).SprintfFunc() + } + return fmt.Sprintf + }() +) diff --git a/pkg/env/env.go b/pkg/env/env.go new file mode 100644 index 000000000..b2305672b --- /dev/null +++ b/pkg/env/env.go @@ -0,0 +1,21 @@ +package env + +import "os" + +// Get retrieves the value of the environment variable named +// by the key. If the variable is present in the environment the +// value (which may be empty) is returned. Otherwise it returns +// the specified default value. +func Get(key, defaultValue string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return defaultValue +} + +// Lookup retrieves the value of the environment variable named +// by the key. If the variable is present in the environment the +// value (which may be empty) is returned and the boolean is true. +// Otherwise the returned value will be empty and the boolean will +// be false. +func Lookup(key string) (string, bool) { return os.LookupEnv(key) }