oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Extend further validation of config values (#8469)

Browse Source 
- This PR allows config KVS to be validated properly
  without being affected by ENV overrides, rejects
  invalid values during set operation

- Expands unit tests and refactors the error handling
  for notification targets, returns error instead of
  ignoring targets for invalid KVS

- Does all the prep-work for implementing safe-mode
  style operation for MinIO server, introduces a new
  global variable to toggle safe mode based operations
  NOTE: this PR itself doesn't provide safe mode operations
master
Harshavardhana 5 years ago committed by kannappanr
parent 599aae5ba6
commit 9e7a3e6adc
53 changed files with 723 additions and 396 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 50
      cmd/admin-handlers-config-kv.go
  2. 2
      cmd/admin-handlers.go
  3. 6
      cmd/admin-handlers_test.go
  4. 10
      cmd/admin-heal-ops.go
  5. 14
      cmd/api-router.go
  6. 8
      cmd/background-heal-ops.go
  7. 2
      cmd/background-newdisks-heal-ops.go
  8. 11
      cmd/common-main.go
  9. 62
      cmd/config-current.go
  10. 2
      cmd/config-migrate.go
  11. 3
      cmd/config/cache/config.go
  12. 12
      cmd/config/cache/config_test.go
  13. 4
      cmd/config/certs.go
  14. 13
      cmd/config/compress/compress.go
  15. 57
      cmd/config/compress/compress_test.go
  16. 10
      cmd/config/config.go
  17. 38
      cmd/config/etcd/etcd.go
  18. 66
      cmd/config/etcd/etcd_test.go
  19. 4
      cmd/config/identity/openid/jwt.go
  20. 2
      cmd/config/identity/openid/jwt_test.go
  21. 2
      cmd/config/identity/openid/validators_test.go
  22. 281
      cmd/config/notify/parse.go
  23. 2
      cmd/config/policy/opa/config.go
  24. 23
      cmd/crypto/config.go
  25. 40
      cmd/crypto/legacy.go
  26. 2
      cmd/daily-lifecycle-ops.go
  27. 10
      cmd/disk-cache.go
  28. 7
      cmd/endpoint.go
  29. 18
      cmd/gateway-common.go
  30. 11
      cmd/gateway-main.go
  31. 5
      cmd/gateway-startup-msg.go
  32. 2
      cmd/global-heal.go
  33. 5
      cmd/globals.go
  34. 4
      cmd/healthcheck-handler.go
  35. 2
      cmd/iam-object-store.go
  36. 32
      cmd/iam.go
  37. 2
      cmd/lifecycle.go
  38. 4
      cmd/metrics.go
  39. 6
      cmd/notification.go
  40. 2
      cmd/object-api-common.go
  41. 18
      cmd/peer-rest-server.go
  42. 2
      cmd/policy.go
  43. 11
      cmd/routers.go
  44. 107
      cmd/server-main.go
  45. 9
      cmd/server-startup-msg.go
  46. 4
      cmd/signals.go
  47. 2
      cmd/storage-rest_test.go
  48. 25
      cmd/test-utils_test.go
  49. 14
      cmd/web-router.go
  50. 35
      pkg/env/env.go
  51. 4
      pkg/event/target/webhook.go
  52. 16
      pkg/net/url.go
  53. 36
      pkg/net/url_test.go

50
cmd/admin-handlers-config-kv.go
Unescape View File

@ -19,6 +19,7 @@ package cmd
import (
"bufio"
"bytes"
"context"
"encoding/json"
"io"
"net/http"
@ -29,11 +30,29 @@ import (
"github.com/minio/minio/pkg/madmin"
)
func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
// Get current object layer instance.
objectAPI := globalObjectAPI
if objectAPI == nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return nil
}
// Validate request signature.
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
if adminAPIErr != ErrNone {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
return nil
}
return objectAPI
}
// DelConfigKVHandler - DELETE /minio/admin/v2/del-config-kv
func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "DelConfigKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -66,6 +85,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines
if scanner.Text() == "" {
continue
}
@ -89,7 +109,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetConfigKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -123,6 +143,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines
if scanner.Text() == "" {
continue
}
@ -158,16 +179,22 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetConfigKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
cfg, err := getValidConfig(objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
vars := mux.Vars(r)
var buf = &bytes.Buffer{}
key := vars["key"]
if key != "" {
kvs, err := globalServerConfig.GetKVS(key)
kvs, err := cfg.GetKVS(key)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@ -181,7 +208,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
}
}
} else {
buf.WriteString(globalServerConfig.String())
buf.WriteString(cfg.String())
}
password := globalActiveCred.SecretKey
@ -197,7 +224,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ClearConfigHistoryKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -232,7 +259,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RestoreConfigHistoryKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -260,6 +287,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines
if scanner.Text() == "" {
continue
}
@ -290,7 +318,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListConfigHistoryKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -314,7 +342,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "HelpConfigKVHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -340,7 +368,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetConfigHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}
@ -391,7 +419,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetConfigHandler")
objectAPI := validateAdminReq(ctx, w, r)
objectAPI := validateAdminReqConfigKV(ctx, w, r)
if objectAPI == nil {
return
}

2
cmd/admin-handlers.go
Unescape View File

@ -933,7 +933,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return nil

6
cmd/admin-handlers_test.go
Unescape View File

@ -90,7 +90,11 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
globalPolicySys = NewPolicySys()
globalPolicySys.Init(buckets, objLayer)
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
if err != nil {
return nil, err
}
globalNotificationSys.Init(buckets, objLayer)
// Setup admin mgmt REST API handlers.

10
cmd/admin-heal-ops.go
Unescape View File

@ -660,7 +660,7 @@ func (h *healSequence) traverseAndHeal() {
func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
return func() error {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -692,7 +692,7 @@ func (h *healSequence) healDiskFormat() error {
}
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -712,7 +712,7 @@ func (h *healSequence) healBuckets() error {
}
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -734,7 +734,7 @@ func (h *healSequence) healBuckets() error {
// healBucket - traverses and heals given bucket
func (h *healSequence) healBucket(bucket string) error {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -771,7 +771,7 @@ func (h *healSequence) healObject(bucket, object string) error {
}
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}

14
cmd/api-router.go
Unescape View File

@ -37,8 +37,18 @@ type objectAPIHandlers struct {
func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
// Initialize API.
api := objectAPIHandlers{
ObjectAPI: newObjectLayerFn,
CacheAPI: newCacheObjectsFn,
ObjectAPI: func() ObjectLayer {
if !globalSafeMode {
return globalObjectAPI
}
return nil
},
CacheAPI: func() CacheObjectLayer {
if !globalSafeMode {
return globalCacheObjectAPI
}
return nil
},
EncryptionEnabled: func() bool {
return encryptionEnabled
},

8
cmd/background-heal-ops.go
Unescape View File

@ -107,7 +107,7 @@ func startBackgroundHealing() {
var objAPI ObjectLayer
for {
objAPI = newObjectLayerFn()
objAPI = globalObjectAPI
if objAPI == nil {
time.Sleep(time.Second)
continue
@ -135,7 +135,7 @@ func initBackgroundHealing() {
// failure error occurred.
func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return madmin.HealResultItem{}, errServerNotInitialized
}
@ -165,7 +165,7 @@ func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealRes
// bghealBucket - traverses and heals given bucket
func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return madmin.HealResultItem{}, errServerNotInitialized
}
@ -176,7 +176,7 @@ func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (mad
// bgHealObject - heal the given object and record result
func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return madmin.HealResultItem{}, errServerNotInitialized
}

2
cmd/background-newdisks-heal-ops.go
Unescape View File

@ -36,7 +36,7 @@ func monitorLocalDisksAndHeal() {
// Wait until the object layer is ready
var objAPI ObjectLayer
for {
objAPI = newObjectLayerFn()
objAPI = globalObjectAPI
if objAPI == nil {
time.Sleep(time.Second)
continue

11
cmd/common-main.go
Unescape View File

@ -162,8 +162,9 @@ func handleCommonEnvVars() {
logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
}
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), config.ValueSeparator) {
if domainName != "" {
domains := env.Get(config.EnvDomain, "")
if len(domains) != 0 {
for _, domainName := range strings.Split(domains, config.ValueSeparator) {
if _, ok := dns2.IsDomainName(domainName); !ok {
logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
"Invalid MINIO_DOMAIN value in environment variable")
@ -172,9 +173,9 @@ func handleCommonEnvVars() {
}
}
minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
if ok {
minioEndpoints := strings.Split(minioEndpointsEnv, config.ValueSeparator)
publicIPs := env.Get(config.EnvPublicIPs, "")
if len(publicIPs) != 0 {
minioEndpoints := strings.Split(publicIPs, config.ValueSeparator)
var domainIPs = set.NewStringSet()
for _, endpoint := range minioEndpoints {
if net.ParseIP(endpoint) == nil {

62
cmd/config-current.go
Unescape View File

@ -18,7 +18,6 @@ package cmd
import (
"context"
"fmt"
"strings"
"sync"
@ -46,6 +45,12 @@ var (
)
func validateConfig(s config.Config) error {
// Disable merging env values with config for validation.
env.SetEnvOff()
// Enable env values upon return.
defer env.SetEnvOn()
if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
return err
}
@ -91,25 +96,23 @@ func validateConfig(s config.Config) error {
return notify.TestNotificationTargets(s, GlobalServiceDoneCh, globalRootCAs)
}
func lookupConfigs(s config.Config) {
var err error
func lookupConfigs(s config.Config) (err error) {
if !globalActiveCred.IsValid() {
// Env doesn't seem to be set, we fallback to lookup creds from the config.
globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid credentials configuration")
return config.Errorf("Invalid credentials configuration: %s", err)
}
}
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
return config.Errorf("Unable to initialize etcd config: %s", err)
}
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
return config.Errorf("Unable to initialize etcd config: %s", err)
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
@ -120,52 +123,51 @@ func lookupConfigs(s config.Config) {
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
)
if err != nil {
logger.Fatal(err, "Unable to initialize DNS config for %s.", globalDomainNames)
return config.Errorf("Unable to initialize DNS config for %s: %s", globalDomainNames, err)
}
}
globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid region configuration")
return config.Errorf("Invalid region configuration: %s", err)
}
globalWORMEnabled, err = config.LookupWorm(s[config.WormSubSys][config.Default])
if err != nil {
logger.Fatal(config.ErrInvalidWormValue(err),
"Invalid worm configuration")
return config.Errorf("Invalid worm configuration: %s", err)
}
if globalIsXL {
globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
globalXLSetDriveCount)
if err != nil {
logger.FatalIf(err, "Unable to initialize storage class config")
return config.Errorf("Unable to initialize storage class config: %s", err)
}
}
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
if err != nil {
logger.FatalIf(err, "Unable to setup cache")
return config.Errorf("Unable to setup cache: %s", err)
}
if globalCacheConfig.Enabled {
if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" {
globalCacheKMS, err = crypto.ParseMasterKey(cacheEncKey)
if err != nil {
logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err),
"Unable to setup encryption cache")
return config.Errorf("Unable to setup encryption cache: %s", err)
}
}
}
kmsCfg, err := crypto.LookupConfig(s[config.KmsVaultSubSys][config.Default])
if err != nil {
logger.FatalIf(err, "Unable to setup KMS config")
return config.Errorf("Unable to setup KMS config: %s", err)
}
GlobalKMS, err = crypto.NewKMS(kmsCfg)
if err != nil {
logger.FatalIf(err, "Unable to setup KMS with current KMS config")
return config.Errorf("Unable to setup KMS with current KMS config: %s", err)
}
// Enable auto-encryption if enabled
@ -173,19 +175,19 @@ func lookupConfigs(s config.Config) {
globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
if err != nil {
logger.FatalIf(err, "Unable to setup Compression")
return config.Errorf("Unable to setup Compression: %s", err)
}
globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
NewCustomHTTPTransport(), xhttp.DrainBody)
if err != nil {
logger.FatalIf(err, "Unable to initialize OpenID")
return config.Errorf("Unable to initialize OpenID: %s", err)
}
opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
NewCustomHTTPTransport(), xhttp.DrainBody)
if err != nil {
logger.FatalIf(err, "Unable to initialize OPA")
return config.Errorf("Unable to initialize OPA: %s", err)
}
globalOpenIDValidators = getOpenIDValidators(globalOpenIDConfig)
@ -194,7 +196,7 @@ func lookupConfigs(s config.Config) {
globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
globalRootCAs)
if err != nil {
logger.FatalIf(err, "Unable to parse LDAP configuration")
return config.Errorf("Unable to parse LDAP configuration: %s", err)
}
// Load logger targets based on user's configuration
@ -202,7 +204,7 @@ func lookupConfigs(s config.Config) {
loggerCfg, err := logger.LookupConfig(s)
if err != nil {
logger.FatalIf(err, "Unable to initialize logger")
return config.Errorf("Unable to initialize logger: %s", err)
}
for _, l := range loggerCfg.HTTP {
@ -221,6 +223,8 @@ func lookupConfigs(s config.Config) {
// Enable console logging
logger.AddTarget(globalConsoleSys.Console())
return nil
}
var helpMap = map[string]config.HelpKV{
@ -255,18 +259,18 @@ func GetHelp(subSys, key string, envOnly bool) (config.HelpKV, error) {
}
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return nil, config.Error(fmt.Sprintf("invalid number of arguments %s", subSys))
return nil, config.Errorf("invalid number of arguments %s", subSys)
}
if !config.SubSystems.Contains(subSystemValue[0]) {
return nil, config.Error(fmt.Sprintf("unknown sub-system %s", subSys))
return nil, config.Errorf("unknown sub-system %s", subSys)
}
help := helpMap[subSystemValue[0]]
if key != "" {
value, ok := help[key]
if !ok {
return nil, config.Error(fmt.Sprintf("unknown key %s for sub-system %s", key, subSys))
return nil, config.Errorf("unknown key %s for sub-system %s", key, subSys)
}
help = config.HelpKV{
key: value,
@ -341,7 +345,9 @@ func newSrvConfig(objAPI ObjectLayer) error {
srvCfg := newServerConfig()
// Override any values from ENVs.
lookupConfigs(srvCfg)
if err := lookupConfigs(srvCfg); err != nil {
return err
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
@ -378,7 +384,9 @@ func loadConfig(objAPI ObjectLayer) error {
}
// Override any values from ENVs.
lookupConfigs(srvCfg)
if err = lookupConfigs(srvCfg); err != nil {
return err
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()

2
cmd/config-migrate.go
Unescape View File

@ -445,7 +445,7 @@ func migrateV5ToV6() error {
if cv5.Logger.ElasticSearch.URL != "" {
var url *xnet.URL
url, err = xnet.ParseURL(cv5.Logger.ElasticSearch.URL)
url, err = xnet.ParseHTTPURL(cv5.Logger.ElasticSearch.URL)
if err != nil {
return err
}

3
cmd/config/cache/config.go vendored
Unescape View File

@ -89,6 +89,9 @@ func parseCacheDrives(drives []string) ([]string, error) {
}
for _, d := range endpoints {
if len(d) == 0 {
return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir cannot be an empty path")
}
if !filepath.IsAbs(d) {
return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
}

12
cmd/config/cache/config_test.go vendored
Unescape View File

@ -30,11 +30,15 @@ func TestParseCacheDrives(t *testing.T) {
expectedPatterns []string
success bool
}{
// valid input
// Invalid input
{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
{"bucket1", []string{}, false},
{";;;", []string{}, false},
{",;,;,;", []string{}, false},
}
// Valid inputs
if runtime.GOOS == "windows" {
testCases = append(testCases, struct {
driveStr string
@ -91,8 +95,12 @@ func TestParseCacheExclude(t *testing.T) {
expectedPatterns []string
success bool
}{
// valid input
// Invalid input
{"/home/drive1;/home/drive2;/home/drive3", []string{}, false},
{"/", []string{}, false},
{";;;", []string{}, false},
// valid input
{"bucket1/*;*.png;images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
{"bucket1", []string{"bucket1"}, true},
}

4
cmd/config/certs.go
Unescape View File

@ -119,8 +119,8 @@ func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
}
if x509.IsEncryptedPEMBlock(key) {
password, ok := env.Lookup(EnvCertPassword)
if !ok {
password := env.Get(EnvCertPassword, "")
if len(password) == 0 {
return tls.Certificate{}, ErrSSLNoPassword(nil)
}
decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))

13
cmd/config/compress/compress.go
Unescape View File

@ -56,11 +56,15 @@ var (
)
// Parses the given compression exclude list `extensions` or `content-types`.
func parseCompressIncludes(includes []string) ([]string, error) {
func parseCompressIncludes(include string) ([]string, error) {
includes := strings.Split(include, config.ValueSeparator)
for _, e := range includes {
if len(e) == 0 {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
}
if e == "/" {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'")
}
}
return includes, nil
}
@ -90,22 +94,21 @@ func LookupConfig(kvs config.KVS) (Config, error) {
compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
if compressExtensions != "" {
extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator))
extensions, err := parseCompressIncludes(compressExtensions)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
}
cfg.Extensions = extensions
}
if compressMimeTypes != "" {
mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
mimeTypes, err := parseCompressIncludes(compressMimeTypes)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
if compressMimeTypesLegacy != "" {
mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypesLegacy,
config.ValueSeparator))
mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
}

57
cmd/config/compress/compress_test.go
Unescape View File

@ -0,0 +1,57 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package compress
import (
"reflect"
"testing"
)
func TestParseCompressIncludes(t *testing.T) {
testCases := []struct {
str string
expectedPatterns []string
success bool
}{
// invalid input
{",,,", []string{}, false},
{"", []string{}, false},
{",", []string{}, false},
{"/", []string{}, false},
{"text/*,/", []string{}, false},
// valid input
{".txt,.log", []string{".txt", ".log"}, true},
{"text/*,application/json", []string{"text/*", "application/json"}, true},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.str, func(t *testing.T) {
gotPatterns, err := parseCompressIncludes(testCase.str)
if !testCase.success && err == nil {
t.Error("expected failure but success instead")
}
if testCase.success && err != nil {
t.Errorf("expected success but failed instead %s", err)
}
if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) {
t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns)
}
})
}
}

10
cmd/config/config.go
Unescape View File

@ -29,6 +29,12 @@ import (
// Error config error type
type Error string
// Errorf - formats according to a format specifier and returns
// the string as a value that satisfies error of type config.Error
func Errorf(format string, a ...interface{}) error {
return Error(fmt.Sprintf(format, a...))
}
func (e Error) Error() string {
return string(e)
}
@ -403,10 +409,6 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator)
continue
}
if len(kv[1]) == 0 {
err := fmt.Sprintf("value for key '%s' cannot be empty", kv[0])
return Error(err)
}
prevK = kv[0]
kvs[kv[0]] = sanitizeValue(kv[1])
}

38
cmd/config/etcd/etcd.go
Unescape View File

@ -19,6 +19,7 @@ package etcd
import (
"crypto/tls"
"crypto/x509"
"fmt"
"strings"
"time"
@ -75,6 +76,25 @@ func New(cfg Config) (*clientv3.Client, error) {
return clientv3.New(cfg.Config)
}
func parseEndpoints(endpoints string) ([]string, bool, error) {
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
u, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
return nil, false, err
}
if etcdSecure && u.Scheme == "http" {
return nil, false, fmt.Errorf("all endpoints should be https or http: %s", endpoint)
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
return etcdEndpoints, etcdSecure, nil
}
// LookupConfig - Initialize new etcd config.
func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg := Config{}
@ -96,22 +116,12 @@ func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
return cfg, nil
}
cfg.Enabled = true
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
if endpoint == "" {
continue
}
u, err := xnet.ParseURL(endpoint)
if err != nil {
return cfg, err
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints)
if err != nil {
return cfg, err
}
cfg.Enabled = true
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
cfg.Endpoints = etcdEndpoints

66
cmd/config/etcd/etcd_test.go
Unescape View File

@ -0,0 +1,66 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package etcd
import (
"reflect"
"testing"
)
// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs.
func TestParseEndpoints(t *testing.T) {
testCases := []struct {
s string
endpoints []string
secure bool
success bool
}{
// Invalid inputs
{"https://localhost:2379,http://localhost:2380", nil, false, false},
{",,,", nil, false, false},
{"", nil, false, false},
{"ftp://localhost:2379", nil, false, false},
{"http://localhost:2379000", nil, false, false},
// Valid inputs
{"https://localhost:2379,https://localhost:2380", []string{
"https://localhost:2379", "https://localhost:2380"},
true, true},
{"http://localhost:2379", []string{"http://localhost:2379"}, false, true},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.s, func(t *testing.T) {
endpoints, secure, err := parseEndpoints(testCase.s)
if err != nil && testCase.success {
t.Errorf("expected to succeed but failed with %s", err)
}
if !testCase.success && err == nil {
t.Error("expected failure but succeeded instead")
}
if testCase.success {
if !reflect.DeepEqual(endpoints, testCase.endpoints) {
t.Errorf("expected %s, got %s", testCase.endpoints, endpoints)
}
if secure != testCase.secure {
t.Errorf("expected %t, got %t", testCase.secure, secure)
}
}
})
}
}

4
cmd/config/identity/openid/jwt.go
Unescape View File

@ -284,7 +284,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
configURL := env.Get(EnvIdentityOpenIDURL, kv.Get(ConfigURL))
if configURL != "" {
c.URL, err = xnet.ParseURL(configURL)
c.URL, err = xnet.ParseHTTPURL(configURL)
if err != nil {
return c, err
}
@ -315,7 +315,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
closeRespFn: closeRespFn,
}
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
c.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL)
if err != nil {
return c, err
}

2
cmd/config/identity/openid/jwt_test.go
Unescape View File

@ -53,7 +53,7 @@ func TestJWT(t *testing.T) {
}
}
u1, err := xnet.ParseURL("http://localhost:8443")
u1, err := xnet.ParseHTTPURL("http://localhost:8443")
if err != nil {
t.Fatal(err)
}

2
cmd/config/identity/openid/validators_test.go
Unescape View File

@ -87,7 +87,7 @@ func TestValidators(t *testing.T) {
t.Fatalf("Unexpected number of vids %v", vids)
}
u, err := xnet.ParseURL(ts.URL)
u, err := xnet.ParseHTTPURL(ts.URL)
if err != nil {
t.Fatal(err)
}

281
cmd/config/notify/parse.go
Unescape View File

@ -17,7 +17,6 @@
package notify
import (
"context"
"crypto/tls"
"crypto/x509"
"strconv"
@ -61,7 +60,57 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
return nil, err
}
for id, args := range GetNotifyAMQP(cfg) {
amqpTargets, err := GetNotifyAMQP(cfg[config.NotifyAMQPSubSys])
if err != nil {
return nil, err
}
esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys])
if err != nil {
return nil, err
}
kafkaTargets, err := GetNotifyKafka(cfg[config.NotifyKafkaSubSys])
if err != nil {
return nil, err
}
mqttTargets, err := GetNotifyMQTT(cfg[config.NotifyMQTTSubSys], rootCAs)
if err != nil {
return nil, err
}
mysqlTargets, err := GetNotifyMySQL(cfg[config.NotifyMySQLSubSys])
if err != nil {
return nil, err
}
natsTargets, err := GetNotifyNATS(cfg[config.NotifyNATSSubSys])
if err != nil {
return nil, err
}
nsqTargets, err := GetNotifyNSQ(cfg[config.NotifyNSQSubSys])
if err != nil {
return nil, err
}
postgresTargets, err := GetNotifyPostgres(cfg[config.NotifyPostgresSubSys])
if err != nil {
return nil, err
}
redisTargets, err := GetNotifyRedis(cfg[config.NotifyRedisSubSys])
if err != nil {
return nil, err
}
webhookTargets, err := GetNotifyWebhook(cfg[config.NotifyWebhookSubSys], rootCAs)
if err != nil {
return nil, err
}
for id, args := range amqpTargets {
if !args.Enable {
continue
}
@ -78,7 +127,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyES(cfg) {
for id, args := range esTargets {
if !args.Enable {
continue
}
@ -96,7 +145,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyKafka(cfg) {
for id, args := range kafkaTargets {
if !args.Enable {
continue
}
@ -114,7 +163,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyMQTT(cfg, rootCAs) {
for id, args := range mqttTargets {
if !args.Enable {
continue
}
@ -132,7 +181,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyMySQL(cfg) {
for id, args := range mysqlTargets {
if !args.Enable {
continue
}
@ -149,7 +198,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyNATS(cfg) {
for id, args := range natsTargets {
if !args.Enable {
continue
}
@ -166,7 +215,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyNSQ(cfg) {
for id, args := range nsqTargets {
if !args.Enable {
continue
}
@ -183,7 +232,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyPostgres(cfg) {
for id, args := range postgresTargets {
if !args.Enable {
continue
}
@ -200,7 +249,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyRedis(cfg) {
for id, args := range redisTargets {
if !args.Enable {
continue
}
@ -217,7 +266,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, root
}
}
for id, args := range GetNotifyWebhook(cfg, rootCAs) {
for id, args := range webhookTargets {
if !args.Enable {
continue
}
@ -307,17 +356,16 @@ var (
)
// GetNotifyKafka - returns a map of registered notification 'kafka' targets
func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs, error) {
kafkaTargets := make(map[string]target.KafkaArgs)
for k, kv := range mergeTargets(s[config.NotifyKafkaSubSys], target.EnvKafkaState, DefaultKafkaKVS) {
for k, kv := range mergeTargets(kafkaKVS, target.EnvKafkaState, DefaultKafkaKVS) {
stateEnv := target.EnvKafkaState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
}
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -328,6 +376,9 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
brokersEnv = brokersEnv + config.Default + k
}
kafkaBrokers := env.Get(brokersEnv, kv.Get(target.KafkaBrokers))
if len(kafkaBrokers) == 0 {
return nil, config.Error("kafka 'brokers' cannot be empty")
}
for _, s := range strings.Split(kafkaBrokers, config.ValueSeparator) {
var host *xnet.Host
host, err = xnet.ParseHost(s)
@ -337,8 +388,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
brokers = append(brokers, *host)
}
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvKafkaQueueLimit
@ -347,8 +397,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
}
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.KafkaQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
clientAuthEnv := target.EnvKafkaTLSClientAuth
@ -357,8 +406,7 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
}
clientAuth, err := strconv.Atoi(env.Get(clientAuthEnv, kv.Get(target.KafkaTLSClientAuth)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
topicEnv := target.EnvKafkaTopic
@ -408,14 +456,13 @@ func GetNotifyKafka(s config.Config) map[string]target.KafkaArgs {
kafkaArgs.SASL.Password = env.Get(saslPasswordEnv, kv.Get(target.KafkaSASLPassword))
if err = kafkaArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
kafkaTargets[k] = kafkaArgs
}
return kafkaTargets
return kafkaTargets, nil
}
// DefaultMQTTKVS - default MQTT config
@ -436,9 +483,9 @@ var (
)
// GetNotifyMQTT - returns a map of registered notification 'mqtt' targets
func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQTTArgs {
func GetNotifyMQTT(mqttKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.MQTTArgs, error) {
mqttTargets := make(map[string]target.MQTTArgs)
for k, kv := range mergeTargets(s[config.NotifyMQTTSubSys], target.EnvMQTTState, DefaultMQTTKVS) {
for k, kv := range mergeTargets(mqttKVS, target.EnvMQTTState, DefaultMQTTKVS) {
stateEnv := target.EnvMQTTState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -446,8 +493,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -459,8 +505,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
}
brokerURL, err := xnet.ParseURL(env.Get(brokerEnv, kv.Get(target.MqttBroker)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
reconnectIntervalEnv := target.EnvMQTTReconnectInterval
@ -470,8 +515,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
reconnectInterval, err := time.ParseDuration(env.Get(reconnectIntervalEnv,
kv.Get(target.MqttReconnectInterval)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
keepAliveIntervalEnv := target.EnvMQTTKeepAliveInterval
@ -481,8 +525,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
keepAliveInterval, err := time.ParseDuration(env.Get(keepAliveIntervalEnv,
kv.Get(target.MqttKeepAliveInterval)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvMQTTQueueLimit
@ -491,8 +534,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
}
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MqttQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
qosEnv := target.EnvMQTTQoS
@ -503,8 +545,7 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
// Parse uint8 value
qos, err := strconv.ParseUint(env.Get(qosEnv, kv.Get(target.MqttQoS)), 10, 8)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
topicEnv := target.EnvMQTTTopic
@ -542,12 +583,11 @@ func GetNotifyMQTT(s config.Config, rootCAs *x509.CertPool) map[string]target.MQ
}
if err = mqttArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
mqttTargets[k] = mqttArgs
}
return mqttTargets
return mqttTargets, nil
}
// DefaultMySQLKVS - default KV for MySQL
@ -569,9 +609,9 @@ var (
)
// GetNotifyMySQL - returns a map of registered notification 'mysql' targets
func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs {
func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs, error) {
mysqlTargets := make(map[string]target.MySQLArgs)
for k, kv := range mergeTargets(s[config.NotifyMySQLSubSys], target.EnvMySQLState, DefaultMySQLKVS) {
for k, kv := range mergeTargets(mysqlKVS, target.EnvMySQLState, DefaultMySQLKVS) {
stateEnv := target.EnvMySQLState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -579,8 +619,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs {
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -593,8 +632,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs {
host, err := xnet.ParseURL(env.Get(hostEnv, kv.Get(target.MySQLHost)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvMySQLQueueLimit
@ -603,8 +641,7 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs {
}
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MySQLQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
formatEnv := target.EnvMySQLFormat
@ -653,12 +690,11 @@ func GetNotifyMySQL(s config.Config) map[string]target.MySQLArgs {
QueueLimit: queueLimit,
}
if err = mysqlArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
mysqlTargets[k] = mysqlArgs
}
return mysqlTargets
return mysqlTargets, nil
}
// DefaultNATSKVS - NATS KV for nats config.
@ -683,9 +719,9 @@ var (
)
// GetNotifyNATS - returns a map of registered notification 'nats' targets
func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
func GetNotifyNATS(natsKVS map[string]config.KVS) (map[string]target.NATSArgs, error) {
natsTargets := make(map[string]target.NATSArgs)
for k, kv := range mergeTargets(s[config.NotifyNATSSubSys], target.EnvNATSState, DefaultNATSKVS) {
for k, kv := range mergeTargets(natsKVS, target.EnvNATSState, DefaultNATSKVS) {
stateEnv := target.EnvNATSState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -693,8 +729,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -707,8 +742,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
address, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NATSAddress)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
pingIntervalEnv := target.EnvNATSPingInterval
@ -718,8 +752,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
pingInterval, err := strconv.ParseInt(env.Get(pingIntervalEnv, kv.Get(target.NATSPingInterval)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvNATSQueueLimit
@ -729,8 +762,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NATSQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
secureEnv := target.EnvNATSSecure
@ -794,8 +826,7 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
maxPubAcksInflight, err := strconv.Atoi(env.Get(maxPubAcksInflightEnv,
kv.Get(target.NATSStreamingMaxPubAcksInFlight)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
clusterIDEnv := target.EnvNATSStreamingClusterID
if k != config.Default {
@ -808,13 +839,12 @@ func GetNotifyNATS(s config.Config) map[string]target.NATSArgs {
}
if err = natsArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
natsTargets[k] = natsArgs
}
return natsTargets
return natsTargets, nil
}
// DefaultNSQKVS - NSQ KV for config
@ -832,9 +862,9 @@ var (
)
// GetNotifyNSQ - returns a map of registered notification 'nsq' targets
func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs {
func GetNotifyNSQ(nsqKVS map[string]config.KVS) (map[string]target.NSQArgs, error) {
nsqTargets := make(map[string]target.NSQArgs)
for k, kv := range mergeTargets(s[config.NotifyNSQSubSys], target.EnvNSQState, DefaultNSQKVS) {
for k, kv := range mergeTargets(nsqKVS, target.EnvNSQState, DefaultNSQKVS) {
stateEnv := target.EnvNSQState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -842,8 +872,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs {
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -855,8 +884,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs {
}
nsqdAddress, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NSQAddress)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
tlsEnableEnv := target.EnvNSQTLSEnable
if k != config.Default {
@ -873,8 +901,7 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs {
}
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NSQQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
topicEnv := target.EnvNSQTopic
@ -897,13 +924,12 @@ func GetNotifyNSQ(s config.Config) map[string]target.NSQArgs {
nsqArgs.TLS.SkipVerify = env.Get(tlsSkipVerifyEnv, kv.Get(target.NSQTLSSkipVerify)) == config.StateOn
if err = nsqArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
nsqTargets[k] = nsqArgs
}
return nsqTargets
return nsqTargets, nil
}
// DefaultPostgresKVS - default Postgres KV for server config.
@ -925,9 +951,9 @@ var (
)
// GetNotifyPostgres - returns a map of registered notification 'postgres' targets
func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs {
func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.PostgreSQLArgs, error) {
psqlTargets := make(map[string]target.PostgreSQLArgs)
for k, kv := range mergeTargets(s[config.NotifyPostgresSubSys], target.EnvPostgresState, DefaultPostgresKVS) {
for k, kv := range mergeTargets(postgresKVS, target.EnvPostgresState, DefaultPostgresKVS) {
stateEnv := target.EnvPostgresState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -935,8 +961,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs {
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -949,8 +974,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs {
host, err := xnet.ParseHost(env.Get(hostEnv, kv.Get(target.PostgresHost)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvPostgresQueueLimit
@ -960,8 +984,7 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs {
queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.PostgresQueueLimit)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
formatEnv := target.EnvPostgresFormat
@ -1018,12 +1041,12 @@ func GetNotifyPostgres(s config.Config) map[string]target.PostgreSQLArgs {
QueueLimit: uint64(queueLimit),
}
if err = psqlArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
psqlTargets[k] = psqlArgs
}
return psqlTargets
return psqlTargets, nil
}
// DefaultRedisKVS - default KV for redis config
@ -1041,9 +1064,9 @@ var (
)
// GetNotifyRedis - returns a map of registered notification 'redis' targets
func GetNotifyRedis(s config.Config) map[string]target.RedisArgs {
func GetNotifyRedis(redisKVS map[string]config.KVS) (map[string]target.RedisArgs, error) {
redisTargets := make(map[string]target.RedisArgs)
for k, kv := range mergeTargets(s[config.NotifyRedisSubSys], target.EnvRedisState, DefaultRedisKVS) {
for k, kv := range mergeTargets(redisKVS, target.EnvRedisState, DefaultRedisKVS) {
stateEnv := target.EnvRedisState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
@ -1051,8 +1074,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs {
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -1064,8 +1086,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs {
}
addr, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.RedisAddress)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvRedisQueueLimit
if k != config.Default {
@ -1073,8 +1094,7 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs {
}
queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.RedisQueueLimit)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
formatEnv := target.EnvRedisFormat
if k != config.Default {
@ -1102,12 +1122,11 @@ func GetNotifyRedis(s config.Config) map[string]target.RedisArgs {
QueueLimit: uint64(queueLimit),
}
if err = redisArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
redisTargets[k] = redisArgs
}
return redisTargets
return redisTargets, nil
}
// DefaultWebhookKVS - default KV for webhook config
@ -1123,17 +1142,16 @@ var (
)
// GetNotifyWebhook - returns a map of registered notification 'webhook' targets
func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target.WebhookArgs {
func GetNotifyWebhook(webhookKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.WebhookArgs, error) {
webhookTargets := make(map[string]target.WebhookArgs)
for k, kv := range mergeTargets(s[config.NotifyWebhookSubSys], target.EnvWebhookState, DefaultWebhookKVS) {
for k, kv := range mergeTargets(webhookKVS, target.EnvWebhookState, DefaultWebhookKVS) {
stateEnv := target.EnvWebhookState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
}
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -1142,10 +1160,9 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target
if k != config.Default {
urlEnv = urlEnv + config.Default + k
}
url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint)))
url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvWebhookQueueLimit
if k != config.Default {
@ -1153,8 +1170,7 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target
}
queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.WebhookQueueLimit)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueDirEnv := target.EnvWebhookQueueDir
if k != config.Default {
@ -1174,12 +1190,11 @@ func GetNotifyWebhook(s config.Config, rootCAs *x509.CertPool) map[string]target
QueueLimit: uint64(queueLimit),
}
if err = webhookArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
webhookTargets[k] = webhookArgs
}
return webhookTargets
return webhookTargets, nil
}
// DefaultESKVS - default KV config for Elasticsearch target
@ -1196,17 +1211,16 @@ var (
)
// GetNotifyES - returns a map of registered notification 'elasticsearch' targets
func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs {
func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchArgs, error) {
esTargets := make(map[string]target.ElasticsearchArgs)
for k, kv := range mergeTargets(s[config.NotifyESSubSys], target.EnvElasticState, DefaultESKVS) {
for k, kv := range mergeTargets(esKVS, target.EnvElasticState, DefaultESKVS) {
stateEnv := target.EnvElasticState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
}
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
@ -1217,10 +1231,9 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs {
urlEnv = urlEnv + config.Default + k
}
url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.ElasticURL)))
url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.ElasticURL)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
queueLimitEnv := target.EnvElasticQueueLimit
@ -1230,8 +1243,7 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs {
queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.ElasticQueueLimit)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
formatEnv := target.EnvElasticFormat
@ -1258,12 +1270,11 @@ func GetNotifyES(s config.Config) map[string]target.ElasticsearchArgs {
QueueLimit: uint64(queueLimit),
}
if err = esArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
esTargets[k] = esArgs
}
return esTargets
return esTargets, nil
}
// DefaultAMQPKVS - default KV for AMQP config
@ -1287,30 +1298,27 @@ var (
)
// GetNotifyAMQP - returns a map of registered notification 'amqp' targets
func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs {
func GetNotifyAMQP(amqpKVS map[string]config.KVS) (map[string]target.AMQPArgs, error) {
amqpTargets := make(map[string]target.AMQPArgs)
for k, kv := range mergeTargets(s[config.NotifyAMQPSubSys], target.EnvAMQPState, DefaultAMQPKVS) {
for k, kv := range mergeTargets(amqpKVS, target.EnvAMQPState, DefaultAMQPKVS) {
stateEnv := target.EnvAMQPState
if k != config.Default {
stateEnv = stateEnv + config.Default + k
}
enabled, err := config.ParseBool(env.Get(stateEnv, kv.Get(config.State)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
if !enabled {
continue
}
urlEnv := target.EnvAMQPURL
if k != config.Default {
urlEnv = urlEnv + config.Default + k
}
url, err := xnet.ParseURL(env.Get(urlEnv, kv.Get(target.AmqpURL)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
deliveryModeEnv := target.EnvAMQPDeliveryMode
if k != config.Default {
@ -1318,8 +1326,7 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs {
}
deliveryMode, err := strconv.Atoi(env.Get(deliveryModeEnv, kv.Get(target.AmqpDeliveryMode)))
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
exchangeEnv := target.EnvAMQPExchange
if k != config.Default {
@ -1367,8 +1374,7 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs {
}
queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.AmqpQueueLimit)), 10, 64)
if err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
amqpArgs := target.AMQPArgs{
Enable: enabled,
@ -1387,10 +1393,9 @@ func GetNotifyAMQP(s config.Config) map[string]target.AMQPArgs {
QueueLimit: queueLimit,
}
if err = amqpArgs.Validate(); err != nil {
logger.LogIf(context.Background(), err)
continue
return nil, err
}
amqpTargets[k] = amqpArgs
}
return amqpTargets
return amqpTargets, nil
}

2
cmd/config/policy/opa/config.go
Unescape View File

@ -124,7 +124,7 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
authToken = env.Get(EnvPolicyOpaAuthToken, kv.Get(AuthToken))
}
u, err := xnet.ParseURL(opaURL)
u, err := xnet.ParseHTTPURL(opaURL)
if err != nil {
return args, err
}

23
cmd/crypto/config.go
Unescape View File

@ -21,6 +21,7 @@ import (
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/env"
xnet "github.com/minio/minio/pkg/net"
)
// KMSConfig has the KMS config for hashicorp vault
@ -148,9 +149,21 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
if !stateBool {
return kmsCfg, nil
}
vcfg := VaultConfig{}
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
vcfg.Endpoint = env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint))
vcfg := VaultConfig{
Auth: VaultAuth{
Type: "approle",
},
}
endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint))
if endpointStr != "" {
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
endpoint, err := xnet.ParseHTTPURL(endpointStr)
if err != nil {
return kmsCfg, err
}
endpointStr = endpoint.String()
}
vcfg.Endpoint = endpointStr
vcfg.CAPath = env.Get(EnvKMSVaultCAPath, kvs.Get(KMSVaultCAPath))
vcfg.Auth.Type = env.Get(EnvKMSVaultAuthType, kvs.Get(KMSVaultAuthType))
vcfg.Auth.AppRole.ID = env.Get(EnvKMSVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
@ -177,7 +190,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
// NewKMS - initialize a new KMS.
func NewKMS(cfg KMSConfig) (kms KMS, err error) {
// Lookup KMS master keys - only available through ENV.
if masterKeyLegacy, ok := env.Lookup(EnvKMSMasterKeyLegacy); ok {
if masterKeyLegacy := env.Get(EnvKMSMasterKeyLegacy, ""); len(masterKeyLegacy) != 0 {
if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
}
@ -185,7 +198,7 @@ func NewKMS(cfg KMSConfig) (kms KMS, err error) {
if err != nil {
return kms, err
}
} else if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
} else if masterKey := env.Get(EnvKMSMasterKey, ""); len(masterKey) != 0 {
if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
}

40
cmd/crypto/legacy.go
Unescape View File

@ -22,6 +22,7 @@ import (
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/env"
xnet "github.com/minio/minio/pkg/net"
)
const (
@ -119,9 +120,16 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
if err != nil {
return KMSConfig{}, err
}
cfg := KMSConfig{
AutoEncryption: autoBool,
Vault: VaultConfig{
Auth: VaultAuth{
Type: "approle",
},
},
}
// Assume default as "on" for legacy config since we didn't have a _STATE
// flag to turn it off, but we should honor it nonetheless to turn it off
// if the vault endpoint is down and there is no way to start the server.
@ -132,29 +140,37 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
if !stateBool {
return cfg, nil
}
vcfg := VaultConfig{}
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
vcfg.Endpoint = env.Get(EnvVaultEndpoint, kvs.Get(KMSVaultEndpoint))
vcfg.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath))
vcfg.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType))
vcfg.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
vcfg.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
vcfg.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName))
vcfg.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace))
endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint))
if endpointStr != "" {
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
endpoint, err := xnet.ParseHTTPURL(endpointStr)
if err != nil {
return cfg, err
}
endpointStr = endpoint.String()
}
cfg.Vault.Endpoint = endpointStr
cfg.Vault.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath))
cfg.Vault.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType))
cfg.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
cfg.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
cfg.Vault.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName))
cfg.Vault.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace))
keyVersion := env.Get(EnvVaultKeyVersion, kvs.Get(KMSVaultKeyVersion))
if keyVersion != "" {
vcfg.Key.Version, err = strconv.Atoi(keyVersion)
cfg.Vault.Key.Version, err = strconv.Atoi(keyVersion)
if err != nil {
return cfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
EnvVaultKeyVersion, keyVersion)
}
}
if err = vcfg.Verify(); err != nil {
if err = cfg.Vault.Verify(); err != nil {
return cfg, err
}
cfg.Vault = vcfg
return cfg, nil
}

2
cmd/daily-lifecycle-ops.go
Unescape View File

@ -55,7 +55,7 @@ func startDailyLifecycle() {
// Wait until the object API is ready
for {
objAPI = newObjectLayerFn()
objAPI = globalObjectAPI
if objAPI == nil {
time.Sleep(time.Second)
continue

10
cmd/disk-cache.go
Unescape View File

@ -553,23 +553,23 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
migrating: migrateSw,
migMutex: sync.Mutex{},
GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
return globalObjectAPI.GetObjectInfo(ctx, bucket, object, opts)
},
GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
return globalObjectAPI.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
},
DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
return newObjectLayerFn().DeleteObject(ctx, bucket, object)
return globalObjectAPI.DeleteObject(ctx, bucket, object)
},
DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
errs := make([]error, len(objects))
for idx, object := range objects {
errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
errs[idx] = globalObjectAPI.DeleteObject(ctx, bucket, object)
}
return errs, nil
},
PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
return globalObjectAPI.PutObject(ctx, bucket, object, data, opts)
},
}
if migrateSw {

7
cmd/endpoint.go
Unescape View File

@ -32,7 +32,6 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/v6/pkg/set"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/env"
"github.com/minio/minio/pkg/mountinfo"
@ -590,10 +589,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
return serverAddr, endpoints, setupType, err
}
_, dok := env.Lookup(config.EnvDomain)
_, eok := env.Lookup(etcd.EnvEtcdEndpoints)
_, iok := env.Lookup(config.EnvPublicIPs)
if dok && eok && !iok {
publicIPs := env.Get(config.EnvPublicIPs, "")
if len(publicIPs) == 0 {
updateDomainIPs(uniqueArgs)
}

18
cmd/gateway-common.go
Unescape View File

@ -374,15 +374,6 @@ func parseGatewaySSE(s string) (gatewaySSE, error) {
// handle gateway env vars
func handleGatewayEnvVars() {
gwsseVal, ok := env.Lookup("MINIO_GATEWAY_SSE")
if ok {
var err error
GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)
if err != nil {
logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal)
}
}
accessKey := env.Get(config.EnvAccessKey, "")
secretKey := env.Get(config.EnvSecretKey, "")
cred, err := auth.CreateCredentials(accessKey, secretKey)
@ -391,4 +382,13 @@ func handleGatewayEnvVars() {
"Unable to validate credentials inherited from the shell environment")
}
globalActiveCred = cred
gwsseVal := env.Get("MINIO_GATEWAY_SSE", "")
if len(gwsseVal) != 0 {
var err error
GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)
if err != nil {
logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal)
}
}
}

11
cmd/gateway-main.go
Unescape View File

@ -206,7 +206,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
srvCfg := newServerConfig()
// Override any values from ENVs.
lookupConfigs(srvCfg)
if err := lookupConfigs(srvCfg); err != nil {
logger.FatalIf(err, "Unable to initialize server config")
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
@ -232,7 +234,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
// Create a new config system.
globalConfigSys = NewConfigSys()
// Load globalServerConfig from etcd
// Load globalServerConfig from disk
logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
// Start watching disk for reloading config, this
@ -267,7 +269,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
globalLifecycleSys = NewLifecycleSys()
// Create new notification system.
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
if err != nil {
logger.FatalIf(err, "Unable to initialize notification system")
}
// Verify if object layer supports
// - encryption

5
cmd/gateway-startup-msg.go
Unescape View File

@ -28,9 +28,8 @@ import (
func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
// If cache layer is enabled, print cache capacity.
cacheObjectAPI := newCacheObjectsFn()
if cacheObjectAPI != nil {
printCacheStorageInfo(cacheObjectAPI.StorageInfo(context.Background()))
if globalCacheObjectAPI != nil {
printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background()))
}
// Prints credential.
printGatewayCommonMsg(strippedAPIEndpoints)

2
cmd/global-heal.go
Unescape View File

@ -156,7 +156,7 @@ func execLeaderTasks(sets *xlSets) {
func startGlobalHeal() {
var objAPI ObjectLayer
for {
objAPI = newObjectLayerFn()
objAPI = globalObjectAPI
if objAPI == nil {
time.Sleep(time.Second)
continue

5
cmd/globals.go
Unescape View File

@ -251,6 +251,11 @@ var (
globalBackgroundHealRoutine *healRoutine
globalBackgroundHealState *allHealState
// Only enabled when one of the sub-systems fail
// to initialize, this allows for administrators to
// fix the system.
globalSafeMode bool
// Add new variable global values here.
)

4
cmd/healthcheck-handler.go
Unescape View File

@ -52,9 +52,9 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "LivenessCheckHandler")
objLayer := newObjectLayerFn()
objLayer := globalObjectAPI
// Service not initialized yet
if objLayer == nil {
if objLayer == nil || globalSafeMode {
// Respond with 200 OK while server initializes to ensure a distributed cluster
// is able to start on orchestration platforms like Docker Swarm.
// Refer https://github.com/minio/minio/issues/8140 for more details.

2
cmd/iam-object-store.go
Unescape View File

@ -47,7 +47,7 @@ func (iamOS *IAMObjectStore) getObjectAPI() ObjectLayer {
if iamOS.objAPI != nil {
return iamOS.objAPI
}
return newObjectLayerFn()
return globalObjectAPI
}
func (iamOS *IAMObjectStore) setObjectAPI(objAPI ObjectLayer) {

32
cmd/iam.go
Unescape View File

@ -406,7 +406,7 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
// DeletePolicy - deletes a canned policy from backend or etcd.
func (sys *IAMSys) DeletePolicy(policyName string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -431,7 +431,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
// InfoPolicy - expands the canned policy into its JSON structure.
func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return nil, errServerNotInitialized
}
@ -448,7 +448,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
// ListPolicies - lists all canned policies.
func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return nil, errServerNotInitialized
}
@ -471,7 +471,7 @@ func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
// SetPolicy - sets a new name policy.
func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -492,7 +492,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
// DeleteUser - delete user (only for long-term users not STS users).
func (sys *IAMSys) DeleteUser(accessKey string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -521,7 +521,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
// SetTempUser - set temporary user credentials, these credentials have an expiry.
func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -561,7 +561,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
// ListUsers - list all users.
func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return nil, errServerNotInitialized
}
@ -587,7 +587,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
// GetUserInfo - get info on a user.
func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return u, errServerNotInitialized
}
@ -617,7 +617,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
// SetUserStatus - sets current user status, supports disabled or enabled.
func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -653,7 +653,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
// SetUser - set user credentials and policy.
func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -685,7 +685,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
// SetUserSecretKey - sets user secret key
func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -724,7 +724,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
// AddUsersToGroup - adds users to a group, creating the group if
// needed. No error if user(s) already are in the group.
func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -782,7 +782,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
// RemoveUsersFromGroup - remove users from group. If no users are
// given, and the group is empty, deletes the group as well.
func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -863,7 +863,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
// SetGroupStatus - enable/disabled a group
func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -952,7 +952,7 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
// PolicyDB. This function applies only long-term users. For STS
// users, policy is set directly by called sys.policyDBSet().
func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return errServerNotInitialized
}
@ -1007,7 +1007,7 @@ func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
return nil, errInvalidArgument
}
objectAPI := newObjectLayerFn()
objectAPI := globalObjectAPI
if objectAPI == nil {
return nil, errServerNotInitialized
}

2
cmd/lifecycle.go
Unescape View File

@ -59,7 +59,7 @@ func (sys *LifecycleSys) Get(bucketName string) (lifecycle lifecycle.Lifecycle,
if globalIsGateway {
// When gateway is enabled, no cached value
// is used to validate life cycle policies.
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
return
}

4
cmd/metrics.go
Unescape View File

@ -83,9 +83,9 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
minioVersionInfo.WithLabelValues(Version, CommitID).Set(float64(1.0))
// Fetch disk space info
objLayer := newObjectLayerFn()
objLayer := globalObjectAPI
// Service not initialized yet
if objLayer == nil {
if objLayer == nil || globalSafeMode {
return
}

6
cmd/notification.go
Unescape View File

@ -1098,10 +1098,10 @@ func (sys *NotificationSys) NetworkInfo() []madmin.ServerNetworkHardwareInfo {
}
// NewNotificationSys - creates new notification system object.
func NewNotificationSys(cfg config.Config, endpoints EndpointList) *NotificationSys {
func NewNotificationSys(cfg config.Config, endpoints EndpointList) (*NotificationSys, error) {
targetList, err := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, globalRootCAs)
if err != nil {
logger.FatalIf(err, "Unable to start notification sub system")
return nil, config.Errorf("Unable to start notification sub system: %s", err)
}
remoteHosts := getRemoteHosts(endpoints)
@ -1113,7 +1113,7 @@ func NewNotificationSys(cfg config.Config, endpoints EndpointList) *Notification
bucketRulesMap: make(map[string]event.RulesMap),
bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
peerClients: remoteClients,
}
}, nil
}
type eventArgs struct {

2
cmd/object-api-common.go
Unescape View File

@ -44,7 +44,7 @@ const (
// Global object layer mutex, used for safely updating object layer.
var globalObjLayerMutex *sync.RWMutex
// Global object layer, only accessed by newObjectLayerFn().
// Global object layer, only accessed by globalObjectAPI.
var globalObjectAPI ObjectLayer
//Global cacheObjects, only accessed by newCacheObjectsFn().

18
cmd/peer-rest-server.go
Unescape View File

@ -46,8 +46,8 @@ func getServerInfo() (*ServerInfoData, error) {
return nil, errServerNotInitialized
}
objLayer := newObjectLayerFn()
if objLayer == nil {
objLayer := globalObjectAPI
if objLayer == nil || globalSafeMode {
return nil, errServerNotInitialized
}
@ -166,7 +166,7 @@ func (s *peerRESTServer) DeletePolicyHandler(w http.ResponseWriter, r *http.Requ
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -194,7 +194,7 @@ func (s *peerRESTServer) LoadPolicyHandler(w http.ResponseWriter, r *http.Reques
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -222,7 +222,7 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -251,7 +251,7 @@ func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Reques
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -279,7 +279,7 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -329,7 +329,7 @@ func (s *peerRESTServer) LoadGroupHandler(w http.ResponseWriter, r *http.Request
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return
@ -539,7 +539,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ
return
}
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI == nil {
s.writeErrorResponse(w, errServerNotInitialized)
return

2
cmd/policy.go
Unescape View File

@ -69,7 +69,7 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
if globalIsGateway {
// When gateway is enabled, no cached value
// is used to validate bucket policies.
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI != nil {
config, err := objAPI.GetBucketPolicy(context.Background(), args.BucketName)
if err == nil {

11
cmd/routers.go
Unescape View File

@ -22,17 +22,6 @@ import (
"github.com/gorilla/mux"
)
func newObjectLayerFn() (layer ObjectLayer) {
globalObjLayerMutex.RLock()
layer = globalObjectAPI
globalObjLayerMutex.RUnlock()
return
}
func newCacheObjectsFn() CacheObjectLayer {
return globalCacheObjectAPI
}
// Composed function registering routers for only distributed XL setup.
func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
// Register storage rpc router only if its a distributed setup.

107
cmd/server-main.go
Unescape View File

@ -131,11 +131,8 @@ EXAMPLES:
// Checks if endpoints are either available through environment
// or command line, returns false if both fails.
func endpointsPresent(ctx *cli.Context) bool {
_, ok := env.Lookup(config.EnvEndpoints)
if !ok {
ok = ctx.Args().Present()
}
return ok
endpoints := env.Get(config.EnvEndpoints, strings.Join(ctx.Args(), config.ValueSeparator))
return len(endpoints) != 0
}
func serverHandleCmdArgs(ctx *cli.Context) {
@ -195,6 +192,55 @@ func serverHandleEnvVars() {
}
func initAllSubsystems(newObject ObjectLayer) {
// Create a new config system.
globalConfigSys = NewConfigSys()
// Initialize config system.
if err := globalConfigSys.Init(newObject); err != nil {
logger.Fatal(err, "Unable to initialize config system")
}
// Create new IAM system.
globalIAMSys = NewIAMSys()
if err := globalIAMSys.Init(newObject); err != nil {
logger.Fatal(err, "Unable to initialize IAM system")
}
buckets, err := newObject.ListBuckets(context.Background())
if err != nil {
logger.Fatal(err, "Unable to list buckets")
}
// Create new notification system and initialize notification targets
globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
if err != nil {
logger.Fatal(err, "Unable to initialize notification targets")
}
// Initialize notification system.
if err = globalNotificationSys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize notification system")
}
// Create new policy system.
globalPolicySys = NewPolicySys()
// Initialize policy system.
if err = globalPolicySys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize policy system")
}
// Create new lifecycle system.
globalLifecycleSys = NewLifecycleSys()
// Initialize lifecycle system.
if err = globalLifecycleSys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize lifecycle system")
}
}
// serverMain handler called for 'minio server' command.
func serverMain(ctx *cli.Context) {
if ctx.Args().First() == "help" || !endpointsPresent(ctx) {
@ -304,13 +350,8 @@ func serverMain(ctx *cli.Context) {
// Re-enable logging
logger.Disable = false
// Create a new config system.
globalConfigSys = NewConfigSys()
// Initialize config system.
if err = globalConfigSys.Init(newObject); err != nil {
logger.Fatal(err, "Unable to initialize config system")
}
// Validate and initialize all subsystems.
initAllSubsystems(newObject)
if globalCacheConfig.Enabled {
logger.StartupMessage(color.Red(color.Bold("Disk caching is recommended only for gateway deployments")))
@ -320,46 +361,6 @@ func serverMain(ctx *cli.Context) {
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Create new IAM system.
globalIAMSys = NewIAMSys()
if err = globalIAMSys.Init(newObject); err != nil {
logger.Fatal(err, "Unable to initialize IAM system")
}
buckets, err := newObject.ListBuckets(context.Background())
if err != nil {
logger.Fatal(err, "Unable to list buckets on your backend")
}
// Create new policy system.
globalPolicySys = NewPolicySys()
// Initialize policy system.
if err = globalPolicySys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize policy system")
}
// Create new lifecycle system.
globalLifecycleSys = NewLifecycleSys()
// Initialize lifecycle system.
if err = globalLifecycleSys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize lifecycle system")
}
// Create new notification system.
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
// Initialize notification system.
if err = globalNotificationSys.Init(buckets, newObject); err != nil {
logger.Fatal(err, "Unable to initialize notification system")
}
// Verify if object layer supports
// - encryption
// - compression
verifyObjectLayerFeatures("server", newObject)
initDailyLifecycle()
if globalIsXL {
@ -368,9 +369,7 @@ func serverMain(ctx *cli.Context) {
initGlobalHeal()
}
globalObjLayerMutex.Lock()
globalObjectAPI = newObject
globalObjLayerMutex.Unlock()
// Prints the formatted startup message once object layer is initialized.
printStartupMessage(getAPIEndpoints())

9
cmd/server-startup-msg.go
Unescape View File

@ -50,12 +50,11 @@ func printStartupMessage(apiEndPoints []string) {
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
// If cache layer is enabled, print cache capacity.
cacheObjectAPI := newCacheObjectsFn()
if cacheObjectAPI != nil {
printCacheStorageInfo(cacheObjectAPI.StorageInfo(context.Background()))
if globalCacheObjectAPI != nil {
printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background()))
}
// Object layer is initialized then print StorageInfo.
objAPI := newObjectLayerFn()
objAPI := globalObjectAPI
if objAPI != nil {
printStorageInfo(objAPI.StorageInfo(context.Background()))
}
@ -97,7 +96,7 @@ func stripStandardPorts(apiEndpoints []string) (newAPIEndpoints []string) {
newAPIEndpoints = make([]string, len(apiEndpoints))
// Check all API endpoints for standard ports and strip them.
for i, apiEndpoint := range apiEndpoints {
u, err := xnet.ParseURL(apiEndpoint)
u, err := xnet.ParseHTTPURL(apiEndpoint)
if err != nil {
newAPIEndpoints[i] = apiEndpoint
continue

4
cmd/signals.go
Unescape View File

@ -55,7 +55,7 @@ func handleSignals() {
// send signal to various go-routines that they need to quit.
close(GlobalServiceDoneCh)
if objAPI := newObjectLayerFn(); objAPI != nil {
if objAPI := globalObjectAPI; objAPI != nil {
oerr = objAPI.Shutdown(context.Background())
logger.LogIf(context.Background(), oerr)
}
@ -66,7 +66,7 @@ func handleSignals() {
for {
select {
case err := <-globalHTTPServerErrorCh:
if objAPI := newObjectLayerFn(); objAPI != nil {
if objAPI := globalObjectAPI; objAPI != nil {
objAPI.Shutdown(context.Background())
}
if err != nil {

2
cmd/storage-rest_test.go
Unescape View File

@ -498,7 +498,7 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
router := mux.NewRouter()
httpServer := httptest.NewServer(router)
url, err := xnet.ParseURL(httpServer.URL)
url, err := xnet.ParseHTTPURL(httpServer.URL)
if err != nil {
t.Fatalf("unexpected error %v", err)
}

25
cmd/test-utils_test.go
Unescape View File

@ -370,7 +370,11 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
globalPolicySys = NewPolicySys()
globalPolicySys.Init(buckets, objLayer)
globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks)
if err != nil {
t.Fatalf("Unable to initialize notification system %s", err)
}
globalNotificationSys.Init(buckets, objLayer)
globalLifecycleSys = NewLifecycleSys()
@ -1636,7 +1640,10 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
globalIAMSys.Init(xl)
globalPolicySys = NewPolicySys()
globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints)
if err != nil {
return xl, err
}
return xl, nil
}
@ -2168,8 +2175,18 @@ func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFuncti
// to underlying cache layer to manage object layer operation and disk caching
// operation
api := objectAPIHandlers{
ObjectAPI: newObjectLayerFn,
CacheAPI: newCacheObjectsFn,
ObjectAPI: func() ObjectLayer {
if !globalSafeMode {
return globalObjectAPI
}
return nil
},
CacheAPI: func() CacheObjectLayer {
if !globalSafeMode {
return globalCacheObjectAPI
}
return nil
},
EncryptionEnabled: func() bool { return true },
}

14
cmd/web-router.go
Unescape View File

@ -62,8 +62,18 @@ const specialAssets = "index_bundle.*.js|loader.css|logo.svg|firefox.png|safari.
func registerWebRouter(router *mux.Router) error {
// Initialize Web.
web := &webAPIHandlers{
ObjectAPI: newObjectLayerFn,
CacheAPI: newCacheObjectsFn,
ObjectAPI: func() ObjectLayer {
if !globalSafeMode {
return globalObjectAPI
}
return nil
},
CacheAPI: func() CacheObjectLayer {
if !globalSafeMode {
return globalCacheObjectAPI
}
return nil
},
}
// Initialize a new json2 codec.

35
pkg/env/env.go vendored
Unescape View File

@ -3,26 +3,47 @@ package env
import (
"os"
"strings"
"sync"
)
var (
privateMutex sync.RWMutex
envOff bool
)
// SetEnvOff - turns off env lookup
func SetEnvOff() {
privateMutex.Lock()
defer privateMutex.Unlock()
envOff = true
}
// SetEnvOn - turns on env lookup
func SetEnvOn() {
privateMutex.Lock()
defer privateMutex.Unlock()
envOff = false
}
// Get retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
// value (which may be empty) is returned. Otherwise it returns
// the specified default value.
func Get(key, defaultValue string) string {
privateMutex.RLock()
ok := envOff
privateMutex.RUnlock()
if ok {
return defaultValue
}
if v, ok := os.LookupEnv(key); ok {
return v
}
return defaultValue
}
// Lookup retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
// value (which may be empty) is returned and the boolean is true.
// Otherwise the returned value will be empty and the boolean will
// be false.
func Lookup(key string) (string, bool) { return os.LookupEnv(key) }
// List all envs with a given prefix.
func List(prefix string) (envs []string) {
for _, env := range os.Environ() {

4
pkg/event/target/webhook.go
Unescape View File

@ -98,7 +98,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
if target.store != nil {
return target.store.Put(eventData)
}
u, pErr := xnet.ParseURL(target.args.Endpoint.String())
u, pErr := xnet.ParseHTTPURL(target.args.Endpoint.String())
if pErr != nil {
return pErr
}
@ -153,7 +153,7 @@ func (target *WebhookTarget) send(eventData event.Event) error {
// Send - reads an event from store and sends it to webhook.
func (target *WebhookTarget) Send(eventKey string) error {
u, pErr := xnet.ParseURL(target.args.Endpoint.String())
u, pErr := xnet.ParseHTTPURL(target.args.Endpoint.String())
if pErr != nil {
return pErr
}

16
pkg/net/url.go
Unescape View File

@ -19,6 +19,7 @@ package net
import (
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"net/url"
@ -103,6 +104,21 @@ func (u URL) DialHTTP() error {
return nil
}
// ParseHTTPURL - parses a string into HTTP URL, string is
// expected to be of form http:// or https://
func ParseHTTPURL(s string) (u *URL, err error) {
u, err = ParseURL(s)
if err != nil {
return nil, err
}
switch u.Scheme {
default:
return nil, fmt.Errorf("unexpected scheme found %s", u.Scheme)
case "http", "https":
return u, nil
}
}
// ParseURL - parses string into URL.
func ParseURL(s string) (u *URL, err error) {
var uu *url.URL

36
pkg/net/url_test.go
Unescape View File

@ -133,6 +133,42 @@ func TestURLUnmarshalJSON(t *testing.T) {
}
}
func TestParseHTTPURL(t *testing.T) {
testCases := []struct {
s string
expectedURL *URL
expectErr bool
}{
{"http://play", &URL{Scheme: "http", Host: "play"}, false},
{"https://play.min.io:0", &URL{Scheme: "https", Host: "play.min.io:0"}, false},
{"https://147.75.201.93:9000/", &URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, false},
{"https://s3.amazonaws.com/?location", &URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, false},
{"http://myminio:10000/mybucket//myobject/", &URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject/"}, false},
{"ftp://myftp.server:10000/myuser", nil, true},
{"https://my.server:10000000/myuser", nil, true},
{"myserver:1000", nil, true},
{"http://:1000/mybucket", nil, true},
{"https://147.75.201.93:90000/", nil, true},
{"http:/play", nil, true},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.s, func(t *testing.T) {
url, err := ParseHTTPURL(testCase.s)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("error: expected: %v, got: %v", testCase.expectErr, expectErr)
}
if !testCase.expectErr {
if !reflect.DeepEqual(url, testCase.expectedURL) {
t.Fatalf("host: expected: %#v, got: %#v", testCase.expectedURL, url)
}
}
})
}
}
func TestParseURL(t *testing.T) {
testCases := []struct {
s string

Write Preview
Loading…
Cancel
Save