From 5ac4b517c9053297064592b0cbbfb3e1cde04ed8 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 20 Nov 2019 15:10:24 -0800 Subject: [PATCH] Order all keys in config (#8541) New changes - return default values when sub-sys is not configured. - state is hidden parameter now - remove worm mode to be saved in config --- cmd/admin-handlers-config-kv.go | 11 +- cmd/config-current.go | 32 +- cmd/config-migrate.go | 1 - cmd/config-migrate_test.go | 4 +- cmd/config/cache/config_test.go | 4 +- cmd/config/cache/help.go | 13 +- cmd/config/cache/legacy.go | 32 +- cmd/config/cache/lookup.go | 38 +- cmd/config/compress/compress.go | 16 +- cmd/config/compress/help.go | 9 +- cmd/config/compress/legacy.go | 15 +- cmd/config/config.go | 205 +++++---- cmd/config/constants.go | 3 +- cmd/config/errors.go | 4 +- cmd/config/etcd/etcd.go | 26 +- cmd/config/etcd/help.go | 7 +- cmd/config/help.go | 19 - cmd/config/identity/ldap/config.go | 41 +- cmd/config/identity/ldap/help.go | 5 - cmd/config/identity/ldap/legacy.go | 35 +- cmd/config/identity/openid/help.go | 5 - cmd/config/identity/openid/jwt.go | 21 +- cmd/config/identity/openid/legacy.go | 20 +- cmd/config/legacy.go | 37 +- cmd/config/notify/help.go | 54 +-- cmd/config/notify/legacy.go | 512 ++++++++++++++++++----- cmd/config/notify/parse.go | 500 +++++++++++++++++----- cmd/config/policy/opa/config.go | 16 +- cmd/config/policy/opa/help.go | 5 - cmd/config/policy/opa/legacy.go | 15 +- cmd/config/storageclass/help.go | 5 - cmd/config/storageclass/legacy.go | 15 +- cmd/config/storageclass/storage-class.go | 16 +- cmd/crypto/config.go | 46 +- cmd/crypto/help.go | 5 - cmd/crypto/legacy.go | 55 ++- cmd/gateway/azure/gateway-azure.go | 8 +- cmd/gateway/b2/gateway-b2.go | 8 +- cmd/gateway/gcs/gateway-gcs.go | 6 +- cmd/gateway/hdfs/gateway-hdfs.go | 8 +- cmd/gateway/nas/gateway-nas.go | 8 +- cmd/gateway/oss/gateway-oss.go | 8 +- cmd/gateway/s3/gateway-s3.go | 8 +- cmd/logger/config.go | 32 +- cmd/logger/help.go | 10 - cmd/logger/legacy.go | 30 +- docs/config/README.md | 27 +- docs/disk-caching/DESIGN.md | 8 +- docs/disk-caching/README.md | 4 +- docs/zh_CN/disk-caching/README.md | 4 +- pkg/madmin/parse-kv.go | 62 ++- 51 files changed, 1436 insertions(+), 642 deletions(-) diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go index a14124020..8717449a4 100644 --- a/cmd/admin-handlers-config-kv.go +++ b/cmd/admin-handlers-config-kv.go @@ -156,7 +156,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) { continue } - if err = cfg.SetKVS(scanner.Text()); err != nil { + if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } @@ -212,7 +212,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ var buf = &bytes.Buffer{} key := vars["key"] if key != "" { - kvs, err := cfg.GetKVS(key) + kvs, err := cfg.GetKVS(key, defaultKVS()) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -311,11 +311,12 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r oldCfg := cfg.Clone() scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) for scanner.Scan() { - // Skip any empty lines - if scanner.Text() == "" { + // Skip any empty lines, or comment like characters + if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) { continue } - if err = cfg.SetKVS(scanner.Text()); err != nil { + + if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } diff --git a/cmd/config-current.go b/cmd/config-current.go index 054416eef..923c059d4 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -59,10 +59,6 @@ func validateConfig(s config.Config) error { return err } - if _, err := config.LookupWorm(s[config.WormSubSys][config.Default]); err != nil { - return err - } - if globalIsXL { if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], globalXLSetDriveCount); err != nil { @@ -165,7 +161,7 @@ func lookupConfigs(s config.Config) (err error) { return config.Errorf("Invalid region configuration: %s", err) } - globalWORMEnabled, err = config.LookupWorm(s[config.WormSubSys][config.Default]) + globalWORMEnabled, err = config.LookupWorm() if err != nil { return config.Errorf("Invalid worm configuration: %s", err) @@ -260,16 +256,33 @@ func lookupConfigs(s config.Config) (err error) { return nil } +func defaultKVS() map[string]config.KVS { + var kvs = map[string]config.KVS{ + config.EtcdSubSys: etcd.DefaultKVS, + config.CacheSubSys: cache.DefaultKVS, + config.CompressionSubSys: compress.DefaultKVS, + config.StorageClassSubSys: storageclass.DefaultKVS, + config.IdentityLDAPSubSys: xldap.DefaultKVS, + config.IdentityOpenIDSubSys: openid.DefaultKVS, + config.PolicyOPASubSys: opa.DefaultKVS, + config.RegionSubSys: config.DefaultRegionKVS, + config.CredentialsSubSys: config.DefaultCredentialKVS, + config.KmsVaultSubSys: crypto.DefaultKVS, + config.LoggerWebhookSubSys: logger.DefaultKVS, + config.AuditWebhookSubSys: logger.DefaultAuditKVS, + } + for k, v := range notify.DefaultNotificationKVS { + kvs[k] = v + } + return kvs +} + // Captures help for each sub-system var helpSubSys = config.HelpKVS{ config.HelpKV{ Key: config.RegionSubSys, Description: "Configure to describe the physical location of the server", }, - config.HelpKV{ - Key: config.WormSubSys, - Description: "Configure to enable WORM mode, turns-off any overwrites and deletes of content", - }, config.HelpKV{ Key: config.StorageClassSubSys, Description: "Configure to control data and parity per object", @@ -367,7 +380,6 @@ var helpSubSys = config.HelpKVS{ var helpMap = map[string]config.HelpKVS{ config.RegionSubSys: config.RegionHelp, - config.WormSubSys: config.WormHelp, config.EtcdSubSys: etcd.Help, config.CacheSubSys: cache.Help, config.CompressionSubSys: compress.Help, diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 173d354ff..8919fb7cb 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -2729,7 +2729,6 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error { config.SetCredentials(newCfg, cfg.Credential) config.SetRegion(newCfg, cfg.Region) - config.SetWorm(newCfg, bool(cfg.Worm)) storageclass.SetStorageClass(newCfg, cfg.StorageClass) diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go index 0fc82695b..5cd087b30 100644 --- a/cmd/config-migrate_test.go +++ b/cmd/config-migrate_test.go @@ -220,12 +220,12 @@ func TestServerConfigMigrateV2toV33(t *testing.T) { } // Check if accessKey and secretKey are not altered during migration - caccessKey := globalServerConfig[config.CredentialsSubSys][config.Default][config.AccessKey] + caccessKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.AccessKey) if caccessKey != accessKey { t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, caccessKey) } - csecretKey := globalServerConfig[config.CredentialsSubSys][config.Default][config.SecretKey] + csecretKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.SecretKey) if csecretKey != secretKey { t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, csecretKey) } diff --git a/cmd/config/cache/config_test.go b/cmd/config/cache/config_test.go index a21801f0d..73e5152b8 100644 --- a/cmd/config/cache/config_test.go +++ b/cmd/config/cache/config_test.go @@ -73,7 +73,7 @@ func TestParseCacheDrives(t *testing.T) { }{"/home/drive{1..3}", []string{}, false}) } for i, testCase := range testCases { - drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheDelimiter)) + drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheDelimiterLegacy)) if err != nil && testCase.success { t.Errorf("Test %d: Expected success but failed instead %s", i+1, err) } @@ -106,7 +106,7 @@ func TestParseCacheExclude(t *testing.T) { } for i, testCase := range testCases { - excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheDelimiter)) + excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheDelimiterLegacy)) if err != nil && testCase.success { t.Errorf("Test %d: Expected success but failed instead %s", i+1, err) } diff --git a/cmd/config/cache/help.go b/cmd/config/cache/help.go index 17a1e6c8e..6eefa45b6 100644 --- a/cmd/config/cache/help.go +++ b/cmd/config/cache/help.go @@ -21,21 +21,16 @@ import "github.com/minio/minio/cmd/config" // Help template for caching feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if caching is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: Drives, - Description: `List of mounted drives or directories delimited by ";"`, - Type: "delimited-string", + Description: `List of mounted drives or directories delimited by ","`, + Type: "csv", }, config.HelpKV{ Key: Exclude, - Description: `List of wildcard based cache exclusion patterns delimited by ";"`, + Description: `List of wildcard based cache exclusion patterns delimited by ","`, Optional: true, - Type: "delimited-string", + Type: "csv", }, config.HelpKV{ Key: Expiry, diff --git a/cmd/config/cache/legacy.go b/cmd/config/cache/legacy.go index 3f51a9590..50afb1037 100644 --- a/cmd/config/cache/legacy.go +++ b/cmd/config/cache/legacy.go @@ -23,16 +23,36 @@ import ( "github.com/minio/minio/cmd/config" ) +const ( + cacheDelimiterLegacy = ";" +) + // SetCacheConfig - One time migration code needed, for migrating from older config to new for Cache. func SetCacheConfig(s config.Config, cfg Config) { if len(cfg.Drives) == 0 { // Do not save cache if no settings available. return } - s[config.CacheSubSys][config.Default] = DefaultKVS - s[config.CacheSubSys][config.Default][Drives] = strings.Join(cfg.Drives, cacheDelimiter) - s[config.CacheSubSys][config.Default][Exclude] = strings.Join(cfg.Exclude, cacheDelimiter) - s[config.CacheSubSys][config.Default][Expiry] = fmt.Sprintf("%d", cfg.Expiry) - s[config.CacheSubSys][config.Default][Quota] = fmt.Sprintf("%d", cfg.MaxUse) - s[config.CacheSubSys][config.Default][config.State] = config.StateOn + s[config.CacheSubSys][config.Default] = config.KVS{ + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: Drives, + Value: strings.Join(cfg.Drives, cacheDelimiter), + }, + config.KV{ + Key: Exclude, + Value: strings.Join(cfg.Exclude, cacheDelimiter), + }, + config.KV{ + Key: Expiry, + Value: fmt.Sprintf("%d", cfg.Expiry), + }, + config.KV{ + Key: Quota, + Value: fmt.Sprintf("%d", cfg.MaxUse), + }, + } } diff --git a/cmd/config/cache/lookup.go b/cmd/config/cache/lookup.go index 3a0152f39..8baac04b1 100644 --- a/cmd/config/cache/lookup.go +++ b/cmd/config/cache/lookup.go @@ -48,17 +48,31 @@ const ( // DefaultKVS - default KV settings for caching. var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default cache configuration, only applicable in gateway setups", - Drives: "", - Exclude: "", - Expiry: DefaultExpiry, - Quota: DefaultQuota, + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: Drives, + Value: "", + }, + config.KV{ + Key: Exclude, + Value: "", + }, + config.KV{ + Key: Expiry, + Value: DefaultExpiry, + }, + config.KV{ + Key: Quota, + Value: DefaultQuota, + }, } ) const ( - cacheDelimiter = ";" + cacheDelimiter = "," ) // LookupConfig - extracts cache configuration provided by environment @@ -92,14 +106,20 @@ func LookupConfig(kvs config.KVS) (Config, error) { cfg.Drives, err = parseCacheDrives(strings.Split(drives, cacheDelimiter)) if err != nil { - return cfg, err + cfg.Drives, err = parseCacheDrives(strings.Split(drives, cacheDelimiterLegacy)) + if err != nil { + return cfg, err + } } cfg.Enabled = true if excludes := env.Get(EnvCacheExclude, kvs.Get(Exclude)); excludes != "" { cfg.Exclude, err = parseCacheExcludes(strings.Split(excludes, cacheDelimiter)) if err != nil { - return cfg, err + cfg.Exclude, err = parseCacheExcludes(strings.Split(excludes, cacheDelimiterLegacy)) + if err != nil { + return cfg, err + } } } diff --git a/cmd/config/compress/compress.go b/cmd/config/compress/compress.go index 463da3dfa..5b536bc4c 100644 --- a/cmd/config/compress/compress.go +++ b/cmd/config/compress/compress.go @@ -48,10 +48,18 @@ const ( // DefaultKVS - default KV config for compression settings var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default compression configuration", - Extensions: DefaultExtensions, - MimeTypes: DefaultMimeTypes, + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: Extensions, + Value: DefaultExtensions, + }, + config.KV{ + Key: MimeTypes, + Value: DefaultMimeTypes, + }, } ) diff --git a/cmd/config/compress/help.go b/cmd/config/compress/help.go index 70b3f6480..11dfc14f0 100644 --- a/cmd/config/compress/help.go +++ b/cmd/config/compress/help.go @@ -21,22 +21,17 @@ import "github.com/minio/minio/cmd/config" // Help template for compress feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if compression is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: Extensions, Description: `Comma separated file extensions to compress eg: ".txt,.log,.csv"`, Optional: true, - Type: "delimited-string", + Type: "csv", }, config.HelpKV{ Key: MimeTypes, Description: `Comma separate wildcard mime-types to compress eg: "text/*,application/json,application/xml"`, Optional: true, - Type: "delimited-string", + Type: "csv", }, config.HelpKV{ Key: config.Comment, diff --git a/cmd/config/compress/legacy.go b/cmd/config/compress/legacy.go index ebac5f24a..a591a08a8 100644 --- a/cmd/config/compress/legacy.go +++ b/cmd/config/compress/legacy.go @@ -35,8 +35,17 @@ func SetCompressionConfig(s config.Config, cfg Config) { return } s[config.CompressionSubSys][config.Default] = config.KVS{ - config.State: config.StateOn, - Extensions: strings.Join(cfg.Extensions, config.ValueSeparator), - MimeTypes: strings.Join(cfg.MimeTypes, config.ValueSeparator), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: Extensions, + Value: strings.Join(cfg.Extensions, config.ValueSeparator), + }, + config.KV{ + Key: MimeTypes, + Value: strings.Join(cfg.MimeTypes, config.ValueSeparator), + }, } } diff --git a/cmd/config/config.go b/cmd/config/config.go index 5eafd868e..7483ce911 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -62,7 +62,6 @@ const ( PolicyOPASubSys = "policy_opa" IdentityOpenIDSubSys = "identity_openid" IdentityLDAPSubSys = "identity_ldap" - WormSubSys = "worm" CacheSubSys = "cache" RegionSubSys = "region" EtcdSubSys = "etcd" @@ -94,7 +93,6 @@ const ( // SubSystems - all supported sub-systems var SubSystems = set.CreateStringSet([]string{ CredentialsSubSys, - WormSubSys, RegionSubSys, EtcdSubSys, CacheSubSys, @@ -121,7 +119,6 @@ var SubSystems = set.CreateStringSet([]string{ // SubSystemsSingleTargets - subsystems which only support single target. var SubSystemsSingleTargets = set.CreateStringSet([]string{ CredentialsSubSys, - WormSubSys, RegionSubSys, EtcdSubSys, CacheSubSys, @@ -148,9 +145,15 @@ const ( EnvWordDelimiter = `_` ) +// KV - is a shorthand of each key value. +type KV struct { + Key string `json:"key"` + Value string `json:"value"` +} + // KVS - is a shorthand for some wrapper functions // to operate on list of key values. -type KVS map[string]string +type KVS []KV // Empty - return if kv is empty func (kvs KVS) Empty() bool { @@ -159,15 +162,15 @@ func (kvs KVS) Empty() bool { func (kvs KVS) String() string { var s strings.Builder - for k, v := range kvs { + for _, kv := range kvs { // Do not need to print if state is on - if k == State && v == StateOn { + if kv.Key == State && kv.Value == StateOn { continue } - s.WriteString(k) + s.WriteString(kv.Key) s.WriteString(KvSeparator) s.WriteString(KvDoubleQuote) - s.WriteString(v) + s.WriteString(kv.Value) s.WriteString(KvDoubleQuote) s.WriteString(KvSpaceSeparator) } @@ -176,7 +179,21 @@ func (kvs KVS) String() string { // Get - returns the value of a key, if not found returns empty. func (kvs KVS) Get(key string) string { - return kvs[key] + v, ok := kvs.Lookup(key) + if ok { + return v + } + return "" +} + +// Lookup - lookup a key in a list of KVS +func (kvs KVS) Lookup(key string) (string, bool) { + for _, kv := range kvs { + if kv.Key == key { + return kv.Value, true + } + } + return "", false } // Config - MinIO server config structure. @@ -202,21 +219,29 @@ func (c Config) String() string { // Default KV configs for worm and region var ( DefaultCredentialKVS = KVS{ - State: StateOff, - Comment: "This is a default credential configuration", - AccessKey: auth.DefaultAccessKey, - SecretKey: auth.DefaultSecretKey, - } - - DefaultWormKVS = KVS{ - State: StateOff, - Comment: "This is a default WORM configuration", + KV{ + Key: State, + Value: StateOff, + }, + KV{ + Key: AccessKey, + Value: auth.DefaultAccessKey, + }, + KV{ + Key: SecretKey, + Value: auth.DefaultSecretKey, + }, } DefaultRegionKVS = KVS{ - State: StateOff, - Comment: "This is a default Region configuration", - RegionName: "", + KV{ + Key: State, + Value: StateOff, + }, + KV{ + Key: RegionName, + Value: "", + }, } ) @@ -259,30 +284,20 @@ func LookupRegion(kv KVS) (string, error) { // returns error if it find extra or superflous keys. func CheckValidKeys(subSys string, kv KVS, validKVS KVS) error { nkv := KVS{} - for k, v := range kv { - if _, ok := validKVS[k]; !ok { - nkv[k] = v + for _, kv := range kv { + if _, ok := validKVS.Lookup(kv.Key); !ok { + nkv = append(nkv, kv) } } if len(nkv) > 0 { - return Error(fmt.Sprintf("found invalid keys (%s) for '%s' sub-system", nkv.String(), subSys)) + return Errorf("found invalid keys (%s) for '%s' sub-system", nkv.String(), subSys) } return nil } // LookupWorm - check if worm is enabled -func LookupWorm(kv KVS) (bool, error) { - if err := CheckValidKeys(WormSubSys, kv, DefaultWormKVS); err != nil { - return false, err - } - worm := env.Get(EnvWorm, "") - if worm == "" { - worm = env.Get(EnvWormState, kv.Get(State)) - if worm == "" { - return false, nil - } - } - return ParseBool(worm) +func LookupWorm() (bool, error) { + return ParseBool(env.Get(EnvWorm, StateOff)) } // New - initialize a new server config. @@ -295,17 +310,17 @@ func New() Config { } // GetKVS - get kvs from specific subsystem. -func (c Config) GetKVS(s string) (map[string]KVS, error) { +func (c Config) GetKVS(s string, defaultKVS map[string]KVS) (map[string]KVS, error) { if len(s) == 0 { return nil, Error("input cannot be empty") } inputs := strings.Fields(s) if len(inputs) > 1 { - return nil, Error(fmt.Sprintf("invalid number of arguments %s", s)) + return nil, Errorf("invalid number of arguments %s", s) } subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2) if len(subSystemValue) == 0 { - return nil, Error(fmt.Sprintf("invalid number of arguments %s", s)) + return nil, Errorf("invalid number of arguments %s", s) } found := SubSystems.Contains(subSystemValue[0]) if !found { @@ -314,36 +329,38 @@ func (c Config) GetKVS(s string) (map[string]KVS, error) { found = !SubSystems.FuncMatch(strings.HasPrefix, subSystemValue[0]).IsEmpty() && len(subSystemValue) == 1 } if !found { - return nil, Error(fmt.Sprintf("unknown sub-system %s", s)) + return nil, Errorf("unknown sub-system %s", s) } kvs := make(map[string]KVS) var ok bool + subSysPrefix := subSystemValue[0] if len(subSystemValue) == 2 { if len(subSystemValue[1]) == 0 { - err := fmt.Sprintf("sub-system target '%s' cannot be empty", s) - return nil, Error(err) + return nil, Errorf("sub-system target '%s' cannot be empty", s) } - kvs[inputs[0]], ok = c[subSystemValue[0]][subSystemValue[1]] + kvs[inputs[0]], ok = c[subSysPrefix][subSystemValue[1]] if !ok { - err := fmt.Sprintf("sub-system target '%s' doesn't exist", s) - return nil, Error(err) - } - return kvs, nil - } - - for subSys, subSysTgts := range c { - if !strings.HasPrefix(subSys, subSystemValue[0]) { - continue + return nil, Errorf("sub-system target '%s' doesn't exist", s) } - for k, kv := range subSysTgts { - if k != Default { - kvs[subSys+SubSystemSeparator+k] = kv - } else { - kvs[subSys] = kv + } else { + for subSys, subSysTgts := range c { + if !strings.HasPrefix(subSys, subSysPrefix) { + continue + } + for k, kv := range subSysTgts { + if k != Default { + kvs[subSys+SubSystemSeparator+k] = kv + } else { + kvs[subSys] = kv + } } } } + if len(kvs) == 0 { + kvs[subSysPrefix] = defaultKVS[subSysPrefix] + return kvs, nil + } return kvs, nil } @@ -354,24 +371,28 @@ func (c Config) DelKVS(s string) error { } inputs := strings.Fields(s) if len(inputs) > 1 { - return Error(fmt.Sprintf("invalid number of arguments %s", s)) + return Errorf("invalid number of arguments %s", s) } subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2) if len(subSystemValue) == 0 { - return Error(fmt.Sprintf("invalid number of arguments %s", s)) + return Errorf("invalid number of arguments %s", s) } if !SubSystems.Contains(subSystemValue[0]) { - return Error(fmt.Sprintf("unknown sub-system %s", s)) + return Errorf("unknown sub-system %s", s) } + tgt := Default + subSys := subSystemValue[0] if len(subSystemValue) == 2 { if len(subSystemValue[1]) == 0 { - err := fmt.Sprintf("sub-system target '%s' cannot be empty", s) - return Error(err) + return Errorf("sub-system target '%s' cannot be empty", s) } - delete(c[subSystemValue[0]], subSystemValue[1]) - return nil + tgt = subSystemValue[1] + } + _, ok := c[subSys][tgt] + if !ok { + return Errorf("sub-system %s already deleted", s) } - delete(c[subSystemValue[0]], Default) + delete(c[subSys], tgt) return nil } @@ -387,35 +408,32 @@ func (c Config) Clone() Config { for subSys, tgtKV := range c { cp[subSys] = make(map[string]KVS) for tgt, kv := range tgtKV { - cp[subSys][tgt] = KVS{} - for k, v := range kv { - cp[subSys][tgt][k] = v - } + cp[subSys][tgt] = append(cp[subSys][tgt], kv...) } } return cp } // SetKVS - set specific key values per sub-system. -func (c Config) SetKVS(s string) error { +func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error { if len(s) == 0 { return Error("input arguments cannot be empty") } inputs := strings.SplitN(s, KvSpaceSeparator, 2) if len(inputs) <= 1 { - return Error(fmt.Sprintf("invalid number of arguments '%s'", s)) + return Errorf("invalid number of arguments '%s'", s) } subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2) if len(subSystemValue) == 0 { - return Error(fmt.Sprintf("invalid number of arguments %s", s)) + return Errorf("invalid number of arguments %s", s) } if !SubSystems.Contains(subSystemValue[0]) { - return Error(fmt.Sprintf("unknown sub-system %s", s)) + return Errorf("unknown sub-system %s", s) } if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 { - return Error(fmt.Sprintf("sub-system '%s' only supports single target", subSystemValue[0])) + return Errorf("sub-system '%s' only supports single target", subSystemValue[0]) } var kvs = KVS{} @@ -426,35 +444,48 @@ func (c Config) SetKVS(s string) error { continue } if len(kv) == 1 && prevK != "" { - kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator) + kvs = append(kvs, KV{ + Key: prevK, + Value: strings.Join([]string{kvs.Get(prevK), sanitizeValue(kv[0])}, KvSpaceSeparator), + }) continue } if len(kv) == 1 { - return Error(fmt.Sprintf("key '%s', cannot have empty value", kv[0])) + return Errorf("key '%s', cannot have empty value", kv[0]) } prevK = kv[0] - kvs[kv[0]] = sanitizeValue(kv[1]) + kvs = append(kvs, KV{ + Key: kv[0], + Value: sanitizeValue(kv[1]), + }) } tgt := Default + subSys := subSystemValue[0] if len(subSystemValue) == 2 { tgt = subSystemValue[1] } - _, ok := c[subSystemValue[0]][tgt] + _, ok := c[subSys][tgt] if !ok { - c[subSystemValue[0]][tgt] = KVS{} - } - - for k, v := range kvs { - c[subSystemValue[0]][tgt][k] = v + c[subSys][tgt] = KVS{} + } else { + c[subSys][tgt] = kvs } - _, ok = c[subSystemValue[0]][tgt][State] + _, ok = c[subSys][tgt].Lookup(State) if !ok { // implicit state "on" if not specified. - c[subSystemValue[0]][tgt][State] = StateOn + c[subSys][tgt] = append(c[subSys][tgt], KV{ + Key: State, + Value: StateOn, + }) + } + for _, kv := range defaultKVS[subSys] { + _, ok := c[subSys][tgt].Lookup(kv.Key) + if !ok { + c[subSys][tgt] = append(c[subSys][tgt], kv) + } } - return nil } diff --git a/cmd/config/constants.go b/cmd/config/constants.go index a81349ef0..160edd98f 100644 --- a/cmd/config/constants.go +++ b/cmd/config/constants.go @@ -33,8 +33,7 @@ const ( EnvPublicIPs = "MINIO_PUBLIC_IPS" EnvEndpoints = "MINIO_ENDPOINTS" - EnvUpdate = "MINIO_UPDATE" - EnvWormState = "MINIO_WORM_STATE" + EnvUpdate = "MINIO_UPDATE" EnvWorm = "MINIO_WORM" // legacy EnvRegion = "MINIO_REGION" // legacy diff --git a/cmd/config/errors.go b/cmd/config/errors.go index 9244b6056..12d6af0bf 100644 --- a/cmd/config/errors.go +++ b/cmd/config/errors.go @@ -45,13 +45,13 @@ var ( ErrInvalidCacheDrivesValue = newErrFn( "Invalid cache drive value", "Please check the value in this ENV variable", - "MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`", + "MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `,`", ) ErrInvalidCacheExcludesValue = newErrFn( "Invalid cache excludes value", "Please check the passed value", - "MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`", + "MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `,`", ) ErrInvalidCacheExpiryValue = newErrFn( diff --git a/cmd/config/etcd/etcd.go b/cmd/config/etcd/etcd.go index ebb6d3d5a..859af3503 100644 --- a/cmd/config/etcd/etcd.go +++ b/cmd/config/etcd/etcd.go @@ -52,12 +52,26 @@ const ( // DefaultKVS - default KV settings for etcd. var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default etcd configuration", - Endpoints: "", - CoreDNSPath: "/skydns", - ClientCert: "", - ClientCertKey: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: Endpoints, + Value: "", + }, + config.KV{ + Key: CoreDNSPath, + Value: "/skydns", + }, + config.KV{ + Key: ClientCert, + Value: "", + }, + config.KV{ + Key: ClientCertKey, + Value: "", + }, } ) diff --git a/cmd/config/etcd/help.go b/cmd/config/etcd/help.go index 2bb2e2639..a0a4ba1ed 100644 --- a/cmd/config/etcd/help.go +++ b/cmd/config/etcd/help.go @@ -21,15 +21,10 @@ import "github.com/minio/minio/cmd/config" // etcd config documented in default config var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if etcd config is on or off", - Type: "on|off", - }, config.HelpKV{ Key: Endpoints, Description: `Comma separated list of etcd endpoints eg: "http://localhost:2379"`, - Type: "delimited-string", + Type: "csv", }, config.HelpKV{ Key: CoreDNSPath, diff --git a/cmd/config/help.go b/cmd/config/help.go index 30677ba2f..7619ebfff 100644 --- a/cmd/config/help.go +++ b/cmd/config/help.go @@ -44,11 +44,6 @@ func (hkvs HelpKVS) Lookup(key string) (HelpKV, bool) { // Region and Worm help is documented in default config var ( RegionHelp = HelpKVS{ - HelpKV{ - Key: State, - Type: "on|off", - Description: "Indicates if config region is honored or ignored", - }, HelpKV{ Key: RegionName, Type: "string", @@ -62,18 +57,4 @@ var ( Optional: true, }, } - - WormHelp = HelpKVS{ - HelpKV{ - Key: State, - Type: "on|off", - Description: `Indicates if worm is "on" or "off"`, - }, - HelpKV{ - Key: Comment, - Type: "sentence", - Description: "A comment to describe the worm state", - Optional: true, - }, - } ) diff --git a/cmd/config/identity/ldap/config.go b/cmd/config/identity/ldap/config.go index 368dcc3b2..4843a1ff7 100644 --- a/cmd/config/identity/ldap/config.go +++ b/cmd/config/identity/ldap/config.go @@ -78,15 +78,38 @@ const ( // DefaultKVS - default config for LDAP config var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default LDAP configuration", - ServerAddr: "", - STSExpiry: "1h", - UsernameFormat: "", - GroupSearchFilter: "", - GroupNameAttribute: "", - GroupSearchBaseDN: "", - TLSSkipVerify: config.StateOff, + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: ServerAddr, + Value: "", + }, + config.KV{ + Key: STSExpiry, + Value: "1h", + }, + config.KV{ + Key: UsernameFormat, + Value: "", + }, + config.KV{ + Key: GroupSearchFilter, + Value: "", + }, + config.KV{ + Key: GroupNameAttribute, + Value: "", + }, + config.KV{ + Key: GroupSearchBaseDN, + Value: "", + }, + config.KV{ + Key: TLSSkipVerify, + Value: config.StateOff, + }, } ) diff --git a/cmd/config/identity/ldap/help.go b/cmd/config/identity/ldap/help.go index f0df4bea6..cb9f464ad 100644 --- a/cmd/config/identity/ldap/help.go +++ b/cmd/config/identity/ldap/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for LDAP identity feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Enable or disable LDAP/AD identity", - Type: "on|off", - }, config.HelpKV{ Key: ServerAddr, Description: `AD/LDAP server address eg: "myldapserver.com:636"`, diff --git a/cmd/config/identity/ldap/legacy.go b/cmd/config/identity/ldap/legacy.go index a51a76771..5837c2e47 100644 --- a/cmd/config/identity/ldap/legacy.go +++ b/cmd/config/identity/ldap/legacy.go @@ -25,12 +25,33 @@ func SetIdentityLDAP(s config.Config, ldapArgs Config) { return } s[config.IdentityLDAPSubSys][config.Default] = config.KVS{ - config.State: config.StateOn, - ServerAddr: ldapArgs.ServerAddr, - STSExpiry: ldapArgs.STSExpiryDuration, - UsernameFormat: ldapArgs.UsernameFormat, - GroupSearchFilter: ldapArgs.GroupSearchFilter, - GroupNameAttribute: ldapArgs.GroupNameAttribute, - GroupSearchBaseDN: ldapArgs.GroupSearchBaseDN, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: ServerAddr, + Value: ldapArgs.ServerAddr, + }, + config.KV{ + Key: STSExpiry, + Value: ldapArgs.STSExpiryDuration, + }, + config.KV{ + Key: UsernameFormat, + Value: ldapArgs.UsernameFormat, + }, + config.KV{ + Key: GroupSearchFilter, + Value: ldapArgs.GroupSearchFilter, + }, + config.KV{ + Key: GroupNameAttribute, + Value: ldapArgs.GroupNameAttribute, + }, + config.KV{ + Key: GroupSearchBaseDN, + Value: ldapArgs.GroupSearchBaseDN, + }, } } diff --git a/cmd/config/identity/openid/help.go b/cmd/config/identity/openid/help.go index d060ec49b..ea40950e0 100644 --- a/cmd/config/identity/openid/help.go +++ b/cmd/config/identity/openid/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for OpenID identity feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if OpenID identity is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: ConfigURL, Description: `OpenID discovery documented endpoint. eg: "https://accounts.google.com/.well-known/openid-configuration"`, diff --git a/cmd/config/identity/openid/jwt.go b/cmd/config/identity/openid/jwt.go index 5b0f73f8a..b924a6eef 100644 --- a/cmd/config/identity/openid/jwt.go +++ b/cmd/config/identity/openid/jwt.go @@ -258,11 +258,22 @@ func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func( // DefaultKVS - default config for OpenID config var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default OpenID configuration", - JwksURL: "", - ConfigURL: "", - ClaimPrefix: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: JwksURL, + Value: "", + }, + config.KV{ + Key: ConfigURL, + Value: "", + }, + config.KV{ + Key: ClaimPrefix, + Value: "", + }, } ) diff --git a/cmd/config/identity/openid/legacy.go b/cmd/config/identity/openid/legacy.go index 71f2ca0b5..6a0f9069a 100644 --- a/cmd/config/identity/openid/legacy.go +++ b/cmd/config/identity/openid/legacy.go @@ -30,9 +30,21 @@ func SetIdentityOpenID(s config.Config, cfg Config) { return } s[config.IdentityOpenIDSubSys][config.Default] = config.KVS{ - config.State: config.StateOn, - JwksURL: cfg.JWKS.URL.String(), - ConfigURL: "", - ClaimPrefix: "", + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: JwksURL, + Value: cfg.JWKS.URL.String(), + }, + config.KV{ + Key: ConfigURL, + Value: "", + }, + config.KV{ + Key: ClaimPrefix, + Value: "", + }, } } diff --git a/cmd/config/legacy.go b/cmd/config/legacy.go index 2c0e15142..e675d9774 100644 --- a/cmd/config/legacy.go +++ b/cmd/config/legacy.go @@ -31,9 +31,18 @@ func SetCredentials(c Config, cred auth.Credentials) { return } c[CredentialsSubSys][Default] = KVS{ - State: StateOn, - AccessKey: cred.AccessKey, - SecretKey: cred.SecretKey, + KV{ + Key: State, + Value: StateOn, + }, + KV{ + Key: AccessKey, + Value: cred.AccessKey, + }, + KV{ + Key: SecretKey, + Value: cred.SecretKey, + }, } } @@ -43,19 +52,13 @@ func SetRegion(c Config, name string) { return } c[RegionSubSys][Default] = KVS{ - RegionName: name, - State: StateOn, - } -} - -// SetWorm - One time migration code needed, for migrating from older config to new for Worm mode. -func SetWorm(c Config, b bool) { - if !b { - // We don't save disabled configs - return - } - // Set the new value. - c[WormSubSys][Default] = KVS{ - State: StateOn, + KV{ + Key: RegionName, + Value: name, + }, + KV{ + Key: State, + Value: StateOn, + }, } } diff --git a/cmd/config/notify/help.go b/cmd/config/notify/help.go index bc087f257..c882c2eb4 100644 --- a/cmd/config/notify/help.go +++ b/cmd/config/notify/help.go @@ -24,11 +24,6 @@ import ( // Help template inputs for all notification targets var ( HelpAMQP = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.AmqpURL, Description: "AMQP server endpoint, e.g. `amqp://myuser:mypassword@localhost:5672`", @@ -109,15 +104,10 @@ var ( } HelpKafka = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.KafkaBrokers, - Description: "Command separated list of Kafka broker addresses", - Type: "delimited-string", + Description: "Comma separated list of Kafka broker addresses", + Type: "csv", }, config.HelpKV{ Key: target.KafkaTopic, @@ -182,11 +172,6 @@ var ( } HelpMQTT = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.MqttBroker, Description: "MQTT server endpoint, e.g. `tcp://localhost:1883`", @@ -248,11 +233,6 @@ var ( } HelpES = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.ElasticURL, Description: "The Elasticsearch server's address, with optional authentication info", @@ -289,11 +269,6 @@ var ( } HelpWebhook = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.WebhookEndpoint, Description: "Webhook server endpoint eg: http://localhost:8080/minio/events", @@ -326,11 +301,6 @@ var ( } HelpRedis = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.RedisAddress, Description: "The Redis server's address. For example: `localhost:6379`", @@ -373,11 +343,6 @@ var ( } HelpPostgres = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.PostgresConnectionString, Description: "Connection string parameters for the PostgreSQL server", @@ -444,11 +409,6 @@ var ( } HelpMySQL = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.MySQLDSNString, Description: "Data-Source-Name connection string for the MySQL server", @@ -515,11 +475,6 @@ var ( } HelpNATS = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.NATSAddress, Description: "NATS server address eg: '0.0.0.0:4222'", @@ -623,11 +578,6 @@ var ( } HelpNSQ = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Is this server endpoint configuration active/enabled", - Type: "on|off", - }, config.HelpKV{ Key: target.NSQAddress, Description: "NSQ server address eg: '127.0.0.1:4150'", diff --git a/cmd/config/notify/legacy.go b/cmd/config/notify/legacy.go index e6c39beff..23489a609 100644 --- a/cmd/config/notify/legacy.go +++ b/cmd/config/notify/legacy.go @@ -20,23 +20,56 @@ func SetNotifyKafka(s config.Config, kName string, cfg target.KafkaArgs) error { } s[config.NotifyKafkaSubSys][kName] = config.KVS{ - config.State: config.StateOn, - target.KafkaBrokers: func() string { - var brokers []string - for _, broker := range cfg.Brokers { - brokers = append(brokers, broker.String()) - } - return strings.Join(brokers, config.ValueSeparator) - }(), - target.KafkaTopic: cfg.Topic, - target.KafkaQueueDir: cfg.QueueDir, - target.KafkaQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), - target.KafkaTLS: config.FormatBool(cfg.TLS.Enable), - target.KafkaTLSSkipVerify: config.FormatBool(cfg.TLS.SkipVerify), - target.KafkaTLSClientAuth: strconv.Itoa(int(cfg.TLS.ClientAuth)), - target.KafkaSASL: config.FormatBool(cfg.SASL.Enable), - target.KafkaSASLUsername: cfg.SASL.User, - target.KafkaSASLPassword: cfg.SASL.Password, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.KafkaBrokers, + Value: func() string { + var brokers []string + for _, broker := range cfg.Brokers { + brokers = append(brokers, broker.String()) + } + return strings.Join(brokers, config.ValueSeparator) + }(), + }, + config.KV{ + Key: target.KafkaTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.KafkaQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.KafkaQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.KafkaTLS, + Value: config.FormatBool(cfg.TLS.Enable), + }, + config.KV{ + Key: target.KafkaTLSSkipVerify, + Value: config.FormatBool(cfg.TLS.SkipVerify), + }, + config.KV{ + Key: target.KafkaTLSClientAuth, + Value: strconv.Itoa(int(cfg.TLS.ClientAuth)), + }, + config.KV{ + Key: target.KafkaSASL, + Value: config.FormatBool(cfg.SASL.Enable), + }, + config.KV{ + Key: target.KafkaSASLUsername, + Value: cfg.SASL.User, + }, + config.KV{ + Key: target.KafkaSASLPassword, + Value: cfg.SASL.Password, + }, } return nil } @@ -52,19 +85,58 @@ func SetNotifyAMQP(s config.Config, amqpName string, cfg target.AMQPArgs) error } s[config.NotifyAMQPSubSys][amqpName] = config.KVS{ - config.State: config.StateOn, - target.AmqpURL: cfg.URL.String(), - target.AmqpExchange: cfg.Exchange, - target.AmqpRoutingKey: cfg.RoutingKey, - target.AmqpExchangeType: cfg.ExchangeType, - target.AmqpDeliveryMode: strconv.Itoa(int(cfg.DeliveryMode)), - target.AmqpMandatory: config.FormatBool(cfg.Mandatory), - target.AmqpInternal: config.FormatBool(cfg.Immediate), - target.AmqpDurable: config.FormatBool(cfg.Durable), - target.AmqpNoWait: config.FormatBool(cfg.NoWait), - target.AmqpAutoDeleted: config.FormatBool(cfg.AutoDeleted), - target.AmqpQueueDir: cfg.QueueDir, - target.AmqpQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.AmqpURL, + Value: cfg.URL.String(), + }, + config.KV{ + Key: target.AmqpExchange, + Value: cfg.Exchange, + }, + config.KV{ + Key: target.AmqpRoutingKey, + Value: cfg.RoutingKey, + }, + config.KV{ + Key: target.AmqpExchangeType, + Value: cfg.ExchangeType, + }, + config.KV{ + Key: target.AmqpDeliveryMode, + Value: strconv.Itoa(int(cfg.DeliveryMode)), + }, + config.KV{ + Key: target.AmqpMandatory, + Value: config.FormatBool(cfg.Mandatory), + }, + config.KV{ + Key: target.AmqpInternal, + Value: config.FormatBool(cfg.Immediate), + }, + config.KV{ + Key: target.AmqpDurable, + Value: config.FormatBool(cfg.Durable), + }, + config.KV{ + Key: target.AmqpNoWait, + Value: config.FormatBool(cfg.NoWait), + }, + config.KV{ + Key: target.AmqpAutoDeleted, + Value: config.FormatBool(cfg.AutoDeleted), + }, + config.KV{ + Key: target.AmqpQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.AmqpQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -81,12 +153,30 @@ func SetNotifyES(s config.Config, esName string, cfg target.ElasticsearchArgs) e } s[config.NotifyESSubSys][esName] = config.KVS{ - config.State: config.StateOn, - target.ElasticFormat: cfg.Format, - target.ElasticURL: cfg.URL.String(), - target.ElasticIndex: cfg.Index, - target.ElasticQueueDir: cfg.QueueDir, - target.ElasticQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.ElasticFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.ElasticURL, + Value: cfg.URL.String(), + }, + config.KV{ + Key: target.ElasticIndex, + Value: cfg.Index, + }, + config.KV{ + Key: target.ElasticQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.ElasticQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -103,13 +193,34 @@ func SetNotifyRedis(s config.Config, redisName string, cfg target.RedisArgs) err } s[config.NotifyRedisSubSys][redisName] = config.KVS{ - config.State: config.StateOn, - target.RedisFormat: cfg.Format, - target.RedisAddress: cfg.Addr.String(), - target.RedisPassword: cfg.Password, - target.RedisKey: cfg.Key, - target.RedisQueueDir: cfg.QueueDir, - target.RedisQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.RedisFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.RedisAddress, + Value: cfg.Addr.String(), + }, + config.KV{ + Key: target.RedisPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.RedisKey, + Value: cfg.Key, + }, + config.KV{ + Key: target.RedisQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.RedisQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -126,11 +237,26 @@ func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) er } s[config.NotifyWebhookSubSys][whName] = config.KVS{ - config.State: config.StateOn, - target.WebhookEndpoint: cfg.Endpoint.String(), - target.WebhookAuthToken: cfg.AuthToken, - target.WebhookQueueDir: cfg.QueueDir, - target.WebhookQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.WebhookEndpoint, + Value: cfg.Endpoint.String(), + }, + config.KV{ + Key: target.WebhookAuthToken, + Value: cfg.AuthToken, + }, + config.KV{ + Key: target.WebhookQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.WebhookQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -147,17 +273,50 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg } s[config.NotifyPostgresSubSys][psqName] = config.KVS{ - config.State: config.StateOn, - target.PostgresFormat: cfg.Format, - target.PostgresConnectionString: cfg.ConnectionString, - target.PostgresTable: cfg.Table, - target.PostgresHost: cfg.Host.String(), - target.PostgresPort: cfg.Port, - target.PostgresUsername: cfg.User, - target.PostgresPassword: cfg.Password, - target.PostgresDatabase: cfg.Database, - target.PostgresQueueDir: cfg.QueueDir, - target.PostgresQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.PostgresFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.PostgresConnectionString, + Value: cfg.ConnectionString, + }, + config.KV{ + Key: target.PostgresTable, + Value: cfg.Table, + }, + config.KV{ + Key: target.PostgresHost, + Value: cfg.Host.String(), + }, + config.KV{ + Key: target.PostgresPort, + Value: cfg.Port, + }, + config.KV{ + Key: target.PostgresUsername, + Value: cfg.User, + }, + config.KV{ + Key: target.PostgresPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.PostgresDatabase, + Value: cfg.Database, + }, + config.KV{ + Key: target.PostgresQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.PostgresQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -174,13 +333,34 @@ func SetNotifyNSQ(s config.Config, nsqName string, cfg target.NSQArgs) error { } s[config.NotifyNSQSubSys][nsqName] = config.KVS{ - config.State: config.StateOn, - target.NSQAddress: cfg.NSQDAddress.String(), - target.NSQTopic: cfg.Topic, - target.NSQTLS: config.FormatBool(cfg.TLS.Enable), - target.NSQTLSSkipVerify: config.FormatBool(cfg.TLS.SkipVerify), - target.NSQQueueDir: cfg.QueueDir, - target.NSQQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.NSQAddress, + Value: cfg.NSQDAddress.String(), + }, + config.KV{ + Key: target.NSQTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.NSQTLS, + Value: config.FormatBool(cfg.TLS.Enable), + }, + config.KV{ + Key: target.NSQTLSSkipVerify, + Value: config.FormatBool(cfg.TLS.SkipVerify), + }, + config.KV{ + Key: target.NSQQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.NSQQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -197,28 +377,79 @@ func SetNotifyNATS(s config.Config, natsName string, cfg target.NATSArgs) error } s[config.NotifyNATSSubSys][natsName] = config.KVS{ - config.State: config.StateOn, - target.NATSAddress: cfg.Address.String(), - target.NATSSubject: cfg.Subject, - target.NATSUsername: cfg.Username, - target.NATSPassword: cfg.Password, - target.NATSToken: cfg.Token, - target.NATSCertAuthority: cfg.CertAuthority, - target.NATSClientCert: cfg.ClientCert, - target.NATSClientKey: cfg.ClientKey, - target.NATSSecure: config.FormatBool(cfg.Secure), - target.NATSPingInterval: strconv.FormatInt(cfg.PingInterval, 10), - target.NATSQueueDir: cfg.QueueDir, - target.NATSQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), - target.NATSStreaming: func() string { - if cfg.Streaming.Enable { - return config.StateOn - } - return config.StateOff - }(), - target.NATSStreamingClusterID: cfg.Streaming.ClusterID, - target.NATSStreamingAsync: config.FormatBool(cfg.Streaming.Async), - target.NATSStreamingMaxPubAcksInFlight: strconv.Itoa(cfg.Streaming.MaxPubAcksInflight), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.NATSAddress, + Value: cfg.Address.String(), + }, + config.KV{ + Key: target.NATSSubject, + Value: cfg.Subject, + }, + config.KV{ + Key: target.NATSUsername, + Value: cfg.Username, + }, + config.KV{ + Key: target.NATSPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.NATSToken, + Value: cfg.Token, + }, + config.KV{ + Key: target.NATSCertAuthority, + Value: cfg.CertAuthority, + }, + config.KV{ + Key: target.NATSClientCert, + Value: cfg.ClientCert, + }, + config.KV{ + Key: target.NATSClientKey, + Value: cfg.ClientKey, + }, + config.KV{ + Key: target.NATSSecure, + Value: config.FormatBool(cfg.Secure), + }, + config.KV{ + Key: target.NATSPingInterval, + Value: strconv.FormatInt(cfg.PingInterval, 10), + }, + config.KV{ + Key: target.NATSQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.NATSQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.NATSStreaming, + Value: func() string { + if cfg.Streaming.Enable { + return config.StateOn + } + return config.StateOff + }(), + }, + config.KV{ + Key: target.NATSStreamingClusterID, + Value: cfg.Streaming.ClusterID, + }, + config.KV{ + Key: target.NATSStreamingAsync, + Value: config.FormatBool(cfg.Streaming.Async), + }, + config.KV{ + Key: target.NATSStreamingMaxPubAcksInFlight, + Value: strconv.Itoa(cfg.Streaming.MaxPubAcksInflight), + }, } return nil @@ -235,17 +466,50 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error } s[config.NotifyMySQLSubSys][sqlName] = config.KVS{ - config.State: config.StateOn, - target.MySQLFormat: cfg.Format, - target.MySQLDSNString: cfg.DSN, - target.MySQLTable: cfg.Table, - target.MySQLHost: cfg.Host.String(), - target.MySQLPort: cfg.Port, - target.MySQLUsername: cfg.User, - target.MySQLPassword: cfg.Password, - target.MySQLDatabase: cfg.Database, - target.MySQLQueueDir: cfg.QueueDir, - target.MySQLQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.MySQLFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.MySQLDSNString, + Value: cfg.DSN, + }, + config.KV{ + Key: target.MySQLTable, + Value: cfg.Table, + }, + config.KV{ + Key: target.MySQLHost, + Value: cfg.Host.String(), + }, + config.KV{ + Key: target.MySQLPort, + Value: cfg.Port, + }, + config.KV{ + Key: target.MySQLUsername, + Value: cfg.User, + }, + config.KV{ + Key: target.MySQLPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.MySQLDatabase, + Value: cfg.Database, + }, + config.KV{ + Key: target.MySQLQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.MySQLQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil @@ -262,16 +526,46 @@ func SetNotifyMQTT(s config.Config, mqttName string, cfg target.MQTTArgs) error } s[config.NotifyMQTTSubSys][mqttName] = config.KVS{ - config.State: config.StateOn, - target.MqttBroker: cfg.Broker.String(), - target.MqttTopic: cfg.Topic, - target.MqttQoS: fmt.Sprintf("%d", cfg.QoS), - target.MqttUsername: cfg.User, - target.MqttPassword: cfg.Password, - target.MqttReconnectInterval: cfg.MaxReconnectInterval.String(), - target.MqttKeepAliveInterval: cfg.KeepAlive.String(), - target.MqttQueueDir: cfg.QueueDir, - target.MqttQueueLimit: strconv.Itoa(int(cfg.QueueLimit)), + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: target.MqttBroker, + Value: cfg.Broker.String(), + }, + config.KV{ + Key: target.MqttTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.MqttQoS, + Value: fmt.Sprintf("%d", cfg.QoS), + }, + config.KV{ + Key: target.MqttUsername, + Value: cfg.User, + }, + config.KV{ + Key: target.MqttPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.MqttReconnectInterval, + Value: cfg.MaxReconnectInterval.String(), + }, + config.KV{ + Key: target.MqttKeepAliveInterval, + Value: cfg.KeepAlive.String(), + }, + config.KV{ + Key: target.MqttQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.MqttQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, } return nil diff --git a/cmd/config/notify/parse.go b/cmd/config/notify/parse.go index fa7e9b266..f83f9447e 100644 --- a/cmd/config/notify/parse.go +++ b/cmd/config/notify/parse.go @@ -340,18 +340,50 @@ func mergeTargets(cfgTargets map[string]config.KVS, envname string, defaultKVS c // DefaultKakfaKVS - default KV for kafka target var ( DefaultKafkaKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for Kafka notification", - target.KafkaTopic: "", - target.KafkaBrokers: "", - target.KafkaSASLUsername: "", - target.KafkaSASLPassword: "", - target.KafkaTLSClientAuth: "0", - target.KafkaSASL: config.StateOff, - target.KafkaTLS: config.StateOff, - target.KafkaTLSSkipVerify: config.StateOff, - target.KafkaQueueLimit: "0", - target.KafkaQueueDir: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.KafkaTopic, + Value: "", + }, + config.KV{ + Key: target.KafkaBrokers, + Value: "", + }, + config.KV{ + Key: target.KafkaSASLUsername, + Value: "", + }, + config.KV{ + Key: target.KafkaSASLPassword, + Value: "", + }, + config.KV{ + Key: target.KafkaTLSClientAuth, + Value: "0", + }, + config.KV{ + Key: target.KafkaSASL, + Value: config.StateOff, + }, + config.KV{ + Key: target.KafkaTLS, + Value: config.StateOff, + }, + config.KV{ + Key: target.KafkaTLSSkipVerify, + Value: config.StateOff, + }, + config.KV{ + Key: target.KafkaQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.KafkaQueueDir, + Value: "", + }, } ) @@ -468,17 +500,46 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs // DefaultMQTTKVS - default MQTT config var ( DefaultMQTTKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for MQTT notification", - target.MqttBroker: "", - target.MqttTopic: "", - target.MqttPassword: "", - target.MqttUsername: "", - target.MqttQoS: "0", - target.MqttKeepAliveInterval: "0s", - target.MqttReconnectInterval: "0s", - target.MqttQueueDir: "", - target.MqttQueueLimit: "0", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.MqttBroker, + Value: "", + }, + config.KV{ + Key: target.MqttTopic, + Value: "", + }, + config.KV{ + Key: target.MqttPassword, + Value: "", + }, + config.KV{ + Key: target.MqttUsername, + Value: "", + }, + config.KV{ + Key: target.MqttQoS, + Value: "0", + }, + config.KV{ + Key: target.MqttKeepAliveInterval, + Value: "0s", + }, + config.KV{ + Key: target.MqttReconnectInterval, + Value: "0s", + }, + config.KV{ + Key: target.MqttQueueDir, + Value: "", + }, + config.KV{ + Key: target.MqttQueueLimit, + Value: "0", + }, } ) @@ -593,18 +654,50 @@ func GetNotifyMQTT(mqttKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[s // DefaultMySQLKVS - default KV for MySQL var ( DefaultMySQLKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for MySQL notification", - target.MySQLFormat: formatNamespace, - target.MySQLHost: "", - target.MySQLPort: "", - target.MySQLUsername: "", - target.MySQLPassword: "", - target.MySQLDatabase: "", - target.MySQLDSNString: "", - target.MySQLTable: "", - target.MySQLQueueLimit: "0", - target.MySQLQueueDir: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.MySQLFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.MySQLHost, + Value: "", + }, + config.KV{ + Key: target.MySQLPort, + Value: "", + }, + config.KV{ + Key: target.MySQLUsername, + Value: "", + }, + config.KV{ + Key: target.MySQLPassword, + Value: "", + }, + config.KV{ + Key: target.MySQLDatabase, + Value: "", + }, + config.KV{ + Key: target.MySQLDSNString, + Value: "", + }, + config.KV{ + Key: target.MySQLTable, + Value: "", + }, + config.KV{ + Key: target.MySQLQueueDir, + Value: "", + }, + config.KV{ + Key: target.MySQLQueueLimit, + Value: "0", + }, } ) @@ -700,24 +793,74 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs // DefaultNATSKVS - NATS KV for nats config. var ( DefaultNATSKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for NATS notification", - target.NATSAddress: "", - target.NATSSubject: "", - target.NATSUsername: "", - target.NATSPassword: "", - target.NATSToken: "", - target.NATSCertAuthority: "", - target.NATSClientCert: "", - target.NATSClientKey: "", - target.NATSSecure: config.StateOff, - target.NATSPingInterval: "0", - target.NATSQueueLimit: "0", - target.NATSQueueDir: "", - target.NATSStreaming: config.StateOff, - target.NATSStreamingAsync: config.StateOff, - target.NATSStreamingMaxPubAcksInFlight: "0", - target.NATSStreamingClusterID: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.NATSAddress, + Value: "", + }, + config.KV{ + Key: target.NATSSubject, + Value: "", + }, + config.KV{ + Key: target.NATSUsername, + Value: "", + }, + config.KV{ + Key: target.NATSPassword, + Value: "", + }, + config.KV{ + Key: target.NATSToken, + Value: "", + }, + config.KV{ + Key: target.NATSCertAuthority, + Value: "", + }, + config.KV{ + Key: target.NATSClientCert, + Value: "", + }, + config.KV{ + Key: target.NATSClientKey, + Value: "", + }, + config.KV{ + Key: target.NATSSecure, + Value: config.StateOff, + }, + config.KV{ + Key: target.NATSPingInterval, + Value: "0", + }, + config.KV{ + Key: target.NATSStreaming, + Value: config.StateOff, + }, + config.KV{ + Key: target.NATSStreamingAsync, + Value: config.StateOff, + }, + config.KV{ + Key: target.NATSStreamingMaxPubAcksInFlight, + Value: "0", + }, + config.KV{ + Key: target.NATSStreamingClusterID, + Value: "", + }, + config.KV{ + Key: target.NATSQueueDir, + Value: "", + }, + config.KV{ + Key: target.NATSQueueLimit, + Value: "0", + }, } ) @@ -871,14 +1014,34 @@ func GetNotifyNATS(natsKVS map[string]config.KVS) (map[string]target.NATSArgs, e // DefaultNSQKVS - NSQ KV for config var ( DefaultNSQKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for NSQ notification", - target.NSQAddress: "", - target.NSQTopic: "", - target.NSQTLS: config.StateOff, - target.NSQTLSSkipVerify: config.StateOff, - target.NSQQueueLimit: "0", - target.NSQQueueDir: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.NSQAddress, + Value: "", + }, + config.KV{ + Key: target.NSQTopic, + Value: "", + }, + config.KV{ + Key: target.NSQTLS, + Value: config.StateOff, + }, + config.KV{ + Key: target.NSQTLSSkipVerify, + Value: config.StateOff, + }, + config.KV{ + Key: target.NSQQueueDir, + Value: "", + }, + config.KV{ + Key: target.NSQQueueLimit, + Value: "0", + }, } ) @@ -956,18 +1119,50 @@ func GetNotifyNSQ(nsqKVS map[string]config.KVS) (map[string]target.NSQArgs, erro // DefaultPostgresKVS - default Postgres KV for server config. var ( DefaultPostgresKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for Postgres notification", - target.PostgresFormat: formatNamespace, - target.PostgresConnectionString: "", - target.PostgresTable: "", - target.PostgresHost: "", - target.PostgresPort: "", - target.PostgresUsername: "", - target.PostgresPassword: "", - target.PostgresDatabase: "", - target.PostgresQueueDir: "", - target.PostgresQueueLimit: "0", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.PostgresFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.PostgresConnectionString, + Value: "", + }, + config.KV{ + Key: target.PostgresTable, + Value: "", + }, + config.KV{ + Key: target.PostgresHost, + Value: "", + }, + config.KV{ + Key: target.PostgresPort, + Value: "", + }, + config.KV{ + Key: target.PostgresUsername, + Value: "", + }, + config.KV{ + Key: target.PostgresPassword, + Value: "", + }, + config.KV{ + Key: target.PostgresDatabase, + Value: "", + }, + config.KV{ + Key: target.PostgresQueueDir, + Value: "", + }, + config.KV{ + Key: target.PostgresQueueLimit, + Value: "0", + }, } ) @@ -1073,14 +1268,34 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos // DefaultRedisKVS - default KV for redis config var ( DefaultRedisKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for Redis notification", - target.RedisFormat: formatNamespace, - target.RedisAddress: "", - target.RedisKey: "", - target.RedisPassword: "", - target.RedisQueueDir: "", - target.RedisQueueLimit: "0", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.RedisFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.RedisAddress, + Value: "", + }, + config.KV{ + Key: target.RedisKey, + Value: "", + }, + config.KV{ + Key: target.RedisPassword, + Value: "", + }, + config.KV{ + Key: target.RedisQueueDir, + Value: "", + }, + config.KV{ + Key: target.RedisQueueLimit, + Value: "0", + }, } ) @@ -1153,12 +1368,26 @@ func GetNotifyRedis(redisKVS map[string]config.KVS) (map[string]target.RedisArgs // DefaultWebhookKVS - default KV for webhook config var ( DefaultWebhookKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for Webhook notification", - target.WebhookEndpoint: "", - target.WebhookAuthToken: "", - target.WebhookQueueLimit: "0", - target.WebhookQueueDir: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.WebhookEndpoint, + Value: "", + }, + config.KV{ + Key: target.WebhookAuthToken, + Value: "", + }, + config.KV{ + Key: target.WebhookQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.WebhookQueueDir, + Value: "", + }, } ) @@ -1221,13 +1450,30 @@ func GetNotifyWebhook(webhookKVS map[string]config.KVS, rootCAs *x509.CertPool) // DefaultESKVS - default KV config for Elasticsearch target var ( DefaultESKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for Elasticsearch notification", - target.ElasticURL: "", - target.ElasticFormat: formatNamespace, - target.ElasticIndex: "", - target.ElasticQueueDir: "", - target.ElasticQueueLimit: "0", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.ElasticURL, + Value: "", + }, + config.KV{ + Key: target.ElasticFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.ElasticIndex, + Value: "", + }, + config.KV{ + Key: target.ElasticQueueDir, + Value: "", + }, + config.KV{ + Key: target.ElasticQueueLimit, + Value: "0", + }, } ) @@ -1301,20 +1547,58 @@ func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchAr // DefaultAMQPKVS - default KV for AMQP config var ( DefaultAMQPKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "Default settings for AMQP notification", - target.AmqpURL: "", - target.AmqpExchange: "", - target.AmqpExchangeType: "", - target.AmqpRoutingKey: "", - target.AmqpMandatory: config.StateOff, - target.AmqpDurable: config.StateOff, - target.AmqpNoWait: config.StateOff, - target.AmqpInternal: config.StateOff, - target.AmqpAutoDeleted: config.StateOff, - target.AmqpDeliveryMode: "0", - target.AmqpQueueLimit: "0", - target.AmqpQueueDir: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpURL, + Value: "", + }, + config.KV{ + Key: target.AmqpExchange, + Value: "", + }, + config.KV{ + Key: target.AmqpExchangeType, + Value: "", + }, + config.KV{ + Key: target.AmqpRoutingKey, + Value: "", + }, + config.KV{ + Key: target.AmqpMandatory, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpDurable, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpNoWait, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpInternal, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpAutoDeleted, + Value: config.StateOff, + }, + config.KV{ + Key: target.AmqpDeliveryMode, + Value: "0", + }, + config.KV{ + Key: target.AmqpQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.AmqpQueueDir, + Value: "", + }, } ) diff --git a/cmd/config/policy/opa/config.go b/cmd/config/policy/opa/config.go index 0562c89ff..8e211642b 100644 --- a/cmd/config/policy/opa/config.go +++ b/cmd/config/policy/opa/config.go @@ -41,10 +41,18 @@ const ( // DefaultKVS - default config for OPA config var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default OPA configuration", - URL: "", - AuthToken: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: URL, + Value: "", + }, + config.KV{ + Key: AuthToken, + Value: "", + }, } ) diff --git a/cmd/config/policy/opa/help.go b/cmd/config/policy/opa/help.go index 92f19f41f..10b7df2b4 100644 --- a/cmd/config/policy/opa/help.go +++ b/cmd/config/policy/opa/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for OPA policy feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if OPA policy is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: URL, Description: `Points to URL for OPA HTTP API endpoint. eg: "http://localhost:8181/v1/data/httpapi/authz/allow"`, diff --git a/cmd/config/policy/opa/legacy.go b/cmd/config/policy/opa/legacy.go index 01e2f0ffd..634b97203 100644 --- a/cmd/config/policy/opa/legacy.go +++ b/cmd/config/policy/opa/legacy.go @@ -33,8 +33,17 @@ func SetPolicyOPAConfig(s config.Config, opaArgs Args) { return } s[config.PolicyOPASubSys][config.Default] = config.KVS{ - config.State: config.StateOn, - URL: opaArgs.URL.String(), - AuthToken: opaArgs.AuthToken, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: URL, + Value: opaArgs.URL.String(), + }, + config.KV{ + Key: AuthToken, + Value: opaArgs.AuthToken, + }, } } diff --git a/cmd/config/storageclass/help.go b/cmd/config/storageclass/help.go index 13f85d05d..d12d71c0e 100644 --- a/cmd/config/storageclass/help.go +++ b/cmd/config/storageclass/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for storageclass feature. var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if storageclass is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: ClassRRS, Description: `Set reduced redundancy storage class parity ratio. eg: "EC:2"`, diff --git a/cmd/config/storageclass/legacy.go b/cmd/config/storageclass/legacy.go index 23ed334a0..af3f83938 100644 --- a/cmd/config/storageclass/legacy.go +++ b/cmd/config/storageclass/legacy.go @@ -27,8 +27,17 @@ func SetStorageClass(s config.Config, cfg Config) { return } s[config.StorageClassSubSys][config.Default] = config.KVS{ - ClassStandard: cfg.Standard.String(), - ClassRRS: cfg.RRS.String(), - config.State: config.StateOn, + config.KV{ + Key: ClassStandard, + Value: cfg.Standard.String(), + }, + config.KV{ + Key: ClassRRS, + Value: cfg.RRS.String(), + }, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, } } diff --git a/cmd/config/storageclass/storage-class.go b/cmd/config/storageclass/storage-class.go index 9a5dd76ab..20c176c01 100644 --- a/cmd/config/storageclass/storage-class.go +++ b/cmd/config/storageclass/storage-class.go @@ -59,10 +59,18 @@ const ( // DefaultKVS - default storage class config var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default StorageClass configuration, only applicable in erasure coded setups", - ClassStandard: "", - ClassRRS: "EC:2", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: ClassStandard, + Value: "", + }, + config.KV{ + Key: ClassRRS, + Value: "EC:2", + }, } ) diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go index 4ddd2ae30..ff68c92ab 100644 --- a/cmd/crypto/config.go +++ b/cmd/crypto/config.go @@ -46,16 +46,42 @@ const ( // DefaultKVS - default KV crypto config var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default Vault configuration", - KMSVaultEndpoint: "", - KMSVaultCAPath: "", - KMSVaultKeyName: "", - KMSVaultKeyVersion: "", - KMSVaultNamespace: "", - KMSVaultAuthType: "approle", - KMSVaultAppRoleID: "", - KMSVaultAppRoleSecret: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: KMSVaultEndpoint, + Value: "", + }, + config.KV{ + Key: KMSVaultKeyName, + Value: "", + }, + config.KV{ + Key: KMSVaultAuthType, + Value: "approle", + }, + config.KV{ + Key: KMSVaultAppRoleID, + Value: "", + }, + config.KV{ + Key: KMSVaultAppRoleSecret, + Value: "", + }, + config.KV{ + Key: KMSVaultCAPath, + Value: "", + }, + config.KV{ + Key: KMSVaultKeyVersion, + Value: "", + }, + config.KV{ + Key: KMSVaultNamespace, + Value: "", + }, } ) diff --git a/cmd/crypto/help.go b/cmd/crypto/help.go index 0ff4b7f84..eead8b047 100644 --- a/cmd/crypto/help.go +++ b/cmd/crypto/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for KMS vault var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if KMS Vault is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: KMSVaultEndpoint, Description: `Points to Vault API endpoint eg: "http://vault-endpoint-ip:8200"`, diff --git a/cmd/crypto/legacy.go b/cmd/crypto/legacy.go index 393c20738..07181381f 100644 --- a/cmd/crypto/legacy.go +++ b/cmd/crypto/legacy.go @@ -83,20 +83,47 @@ func SetKMSConfig(s config.Config, cfg KMSConfig) { return } s[config.KmsVaultSubSys][config.Default] = config.KVS{ - KMSVaultEndpoint: cfg.Vault.Endpoint, - KMSVaultCAPath: cfg.Vault.CAPath, - KMSVaultAuthType: func() string { - if cfg.Vault.Auth.Type != "" { - return cfg.Vault.Auth.Type - } - return "approle" - }(), - KMSVaultAppRoleID: cfg.Vault.Auth.AppRole.ID, - KMSVaultAppRoleSecret: cfg.Vault.Auth.AppRole.Secret, - KMSVaultKeyName: cfg.Vault.Key.Name, - KMSVaultKeyVersion: strconv.Itoa(cfg.Vault.Key.Version), - KMSVaultNamespace: cfg.Vault.Namespace, - config.State: config.StateOn, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: KMSVaultEndpoint, + Value: cfg.Vault.Endpoint, + }, + config.KV{ + Key: KMSVaultCAPath, + Value: cfg.Vault.CAPath, + }, + config.KV{ + Key: KMSVaultAuthType, + Value: func() string { + if cfg.Vault.Auth.Type != "" { + return cfg.Vault.Auth.Type + } + return "approle" + }(), + }, + config.KV{ + Key: KMSVaultAppRoleID, + Value: cfg.Vault.Auth.AppRole.ID, + }, + config.KV{ + Key: KMSVaultAppRoleSecret, + Value: cfg.Vault.Auth.AppRole.Secret, + }, + config.KV{ + Key: KMSVaultKeyName, + Value: cfg.Vault.Key.Name, + }, + config.KV{ + Key: KMSVaultKeyVersion, + Value: strconv.Itoa(cfg.Vault.Key.Version), + }, + config.KV{ + Key: KMSVaultNamespace, + Value: cfg.Vault.Namespace, + }, } } diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index 82a33dc55..63249843c 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -83,8 +83,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -102,8 +102,8 @@ EXAMPLES: 3. Start minio gateway server for Azure Blob Storage backend with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 18c6bc4d8..129b66a12 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -69,8 +69,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -83,8 +83,8 @@ EXAMPLES: 2. Start minio gateway server for B2 backend with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index c6f5da989..6d54bed23 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -120,8 +120,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -140,7 +140,7 @@ EXAMPLES: {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index 8a01aac4b..ed82013c4 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -76,8 +76,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -90,8 +90,8 @@ EXAMPLES: 2. Start minio gateway server for HDFS with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} hdfs://namenode:8200 diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go index 28e32d787..ec2d9633a 100644 --- a/cmd/gateway/nas/gateway-nas.go +++ b/cmd/gateway/nas/gateway-nas.go @@ -53,8 +53,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -67,8 +67,8 @@ EXAMPLES: 2. Start minio gateway server for NAS with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} /shared/nasvol diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index 350b4ad89..f129f587b 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -72,8 +72,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -91,8 +91,8 @@ EXAMPLES: 3. Start minio gateway server for Aliyun OSS backend with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index a4b856989..ad925996c 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -67,8 +67,8 @@ ENVIRONMENT VARIABLES: MINIO_DOMAIN: To enable virtual-host-style requests, set this value to MinIO host domain name. CACHE: - MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";". - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";". + MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ",". + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ",". MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). @@ -97,8 +97,8 @@ EXAMPLES: 4. Start minio gateway server for AWS S3 backend with edge caching enabled. {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" + {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80 {{.Prompt}} {{.HelpName}} diff --git a/cmd/logger/config.go b/cmd/logger/config.go index bbfd4c856..e5e095da1 100644 --- a/cmd/logger/config.go +++ b/cmd/logger/config.go @@ -57,16 +57,32 @@ const ( // Default KVS for loggerHTTP and loggerAuditHTTP var ( DefaultKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default HTTP logging configuration", - Endpoint: "", - AuthToken: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: Endpoint, + Value: "", + }, + config.KV{ + Key: AuthToken, + Value: "", + }, } DefaultAuditKVS = config.KVS{ - config.State: config.StateOff, - config.Comment: "This is a default HTTP Audit logging configuration", - Endpoint: "", - AuthToken: "", + config.KV{ + Key: config.State, + Value: config.StateOff, + }, + config.KV{ + Key: Endpoint, + Value: "", + }, + config.KV{ + Key: AuthToken, + Value: "", + }, } ) diff --git a/cmd/logger/help.go b/cmd/logger/help.go index 536944e42..9929b4014 100644 --- a/cmd/logger/help.go +++ b/cmd/logger/help.go @@ -21,11 +21,6 @@ import "github.com/minio/minio/cmd/config" // Help template for logger http and audit var ( Help = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if HTTP logger is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: Endpoint, Description: `HTTP logger endpoint eg: "http://localhost:8080/minio/logs/server"`, @@ -46,11 +41,6 @@ var ( } HelpAudit = config.HelpKVS{ - config.HelpKV{ - Key: config.State, - Description: "Indicates if HTTP Audit logger is enabled or not", - Type: "on|off", - }, config.HelpKV{ Key: Endpoint, Description: `HTTP Audit logger endpoint eg: "http://localhost:8080/minio/logs/audit"`, diff --git a/cmd/logger/legacy.go b/cmd/logger/legacy.go index d78d8fd47..58e6d4494 100644 --- a/cmd/logger/legacy.go +++ b/cmd/logger/legacy.go @@ -30,9 +30,18 @@ func SetLoggerHTTPAudit(scfg config.Config, k string, args HTTP) { return } scfg[config.AuditWebhookSubSys][k] = config.KVS{ - config.State: config.StateOn, - Endpoint: args.Endpoint, - AuthToken: args.AuthToken, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: Endpoint, + Value: args.Endpoint, + }, + config.KV{ + Key: AuthToken, + Value: args.AuthToken, + }, } } @@ -44,8 +53,17 @@ func SetLoggerHTTP(scfg config.Config, k string, args HTTP) { } scfg[config.LoggerWebhookSubSys][k] = config.KVS{ - config.State: config.StateOn, - Endpoint: args.Endpoint, - AuthToken: args.AuthToken, + config.KV{ + Key: config.State, + Value: config.StateOn, + }, + config.KV{ + Key: Endpoint, + Value: args.Endpoint, + }, + config.KV{ + Key: AuthToken, + Value: args.AuthToken, + }, } } diff --git a/docs/config/README.md b/docs/config/README.md index fb7d18c9c..400b56cf4 100644 --- a/docs/config/README.md +++ b/docs/config/README.md @@ -70,19 +70,6 @@ export MINIO_REGION_NAME="my_region" minio server /data ``` -#### Worm - -| Field | Type | Description | -|:-----------------------|:---------|:--------------------------------------------------------------------------------------------------------------------------------------------------| -| ``worm state=on`` | _string_ | Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. You may override this field with ``MINIO_WORM`` environment variable. | - -Example: - -```sh -export MINIO_WORM_STATE=on -minio server /data -``` - ### Storage Class | Field | Type | Description | @@ -96,8 +83,8 @@ By default, parity for objects with standard storage class is set to `N/2`, and ### Cache | Field | Type | Description | |:-------------------------------------------------------------|:-----------|:------------------------------------------------------------------------------------------------------------| -| ``cache drives="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"`` | _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled | -| ``cache exclude="*.pdf;mybucket/*"`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache | +| ``cache drives="/mnt/drive1,/mnt/drive2,/mnt/cache{1...3}"`` | _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled | +| ``cache exclude="*.pdf,mybucket/*"`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache | | ``cache expiry=90`` | _int_ | Days to cache expiry | | ``cache quota=70`` | _int_ | Percentage of disk available to cache | | | | | @@ -121,6 +108,16 @@ All configuration changes can be made using [`mc admin config` get/set commands] ## Environment only settings +#### Worm +Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. Set ``MINIO_WORM=on`` environment variable to enable WORM mode. + +Example: + +```sh +export MINIO_WORM=on +minio server /data +``` + ### Browser Enable or disable access to web UI. By default it is set to `on`. You may override this field with `MINIO_BROWSER` environment variable. diff --git a/docs/disk-caching/DESIGN.md b/docs/disk-caching/DESIGN.md index 6d2c36e86..d241d3969 100644 --- a/docs/disk-caching/DESIGN.md +++ b/docs/disk-caching/DESIGN.md @@ -9,8 +9,8 @@ minio gateway -h ... ... CACHE: - MINIO_CACHE_DRIVES: List of mounted cache drives or directories delimited by ";" - MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";" + MINIO_CACHE_DRIVES: List of mounted cache drives or directories delimited by "," + MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by "," MINIO_CACHE_EXPIRY: Cache expiry duration in days MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100). ... @@ -19,8 +19,8 @@ minio gateway -h 7. Start MinIO gateway to s3 with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24', exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension with expiry up to 40 days. - $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}" - $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf" + $ export MINIO_CACHE_DRIVES="/mnt/drive1,/mnt/drive2,/mnt/export{1..24}" + $ export MINIO_CACHE_EXCLUDE="mybucket/*,*.pdf" $ export MINIO_CACHE_EXPIRY=40 $ export MINIO_CACHE_QUOTA=80 $ minio gateway s3 diff --git a/docs/disk-caching/README.md b/docs/disk-caching/README.md index 4346aa67c..6ca136e60 100644 --- a/docs/disk-caching/README.md +++ b/docs/disk-caching/README.md @@ -19,9 +19,9 @@ Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cac ```bash export MINIO_CACHE="on" -export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}" +export MINIO_CACHE_DRIVES="/mnt/drive1,/mnt/drive2,/mnt/cache{1...3}" export MINIO_CACHE_EXPIRY=90 -export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*" +export MINIO_CACHE_EXCLUDE="*.pdf,mybucket/*" export MINIO_CACHE_QUOTA=80 minio gateway s3 ``` diff --git a/docs/zh_CN/disk-caching/README.md b/docs/zh_CN/disk-caching/README.md index 7335eb864..dbb947bc3 100644 --- a/docs/zh_CN/disk-caching/README.md +++ b/docs/zh_CN/disk-caching/README.md @@ -23,9 +23,9 @@ 缓存设置也可以通过环境变量设置。设置后，环境变量会覆盖任何`cache`配置中的值。下面示例使用`/mnt/drive1`, `/mnt/drive2` 和 `/mnt/drive3`来做缓存，90天失效，并且`mybucket`下的所有对象以及后缀名为`.pdf`的对象不做缓存。 ```bash -export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3" +export MINIO_CACHE_DRIVES="/mnt/drive1,/mnt/drive2,/mnt/drive3" export MINIO_CACHE_EXPIRY=90 -export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*" +export MINIO_CACHE_EXCLUDE="*.pdf,mybucket/*" minio server /export{1...24} ``` diff --git a/pkg/madmin/parse-kv.go b/pkg/madmin/parse-kv.go index 0fdf373b1..679e29225 100644 --- a/pkg/madmin/parse-kv.go +++ b/pkg/madmin/parse-kv.go @@ -25,8 +25,39 @@ import ( "unicode" ) -// KVS each sub-system key, value -type KVS map[string]string +// KV - is a shorthand of each key value. +type KV struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// KVS - is a shorthand for some wrapper functions +// to operate on list of key values. +type KVS []KV + +// Empty - return if kv is empty +func (kvs KVS) Empty() bool { + return len(kvs) == 0 +} + +// Get - returns the value of a key, if not found returns empty. +func (kvs KVS) Get(key string) string { + v, ok := kvs.Lookup(key) + if ok { + return v + } + return "" +} + +// Lookup - lookup a key in a list of KVS +func (kvs KVS) Lookup(key string) (string, bool) { + for _, kv := range kvs { + if kv.Key == key { + return kv.Value, true + } + } + return "", false +} // Targets sub-system targets type Targets map[string]map[string]KVS @@ -34,28 +65,25 @@ type Targets map[string]map[string]KVS const ( stateKey = "state" commentKey = "comment" - - stateOn = "on" - stateOff = "off" ) func (kvs KVS) String() string { var s strings.Builder - for k, v := range kvs { - // Do not need to print if state is on - if k == stateKey && v == stateOn { + for _, kv := range kvs { + // Do not need to print state + if kv.Key == stateKey { continue } - if k == commentKey && v == "" { + if kv.Key == commentKey && kv.Value == "" { continue } - s.WriteString(k) + s.WriteString(kv.Key) s.WriteString(KvSeparator) - spc := hasSpace(v) + spc := hasSpace(kv.Value) if spc { s.WriteString(KvDoubleQuote) } - s.WriteString(v) + s.WriteString(kv.Value) if spc { s.WriteString(KvDoubleQuote) } @@ -140,14 +168,20 @@ func convertTargets(s string, targets Targets) error { continue } if len(kv) == 1 && prevK != "" { - kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator) + kvs = append(kvs, KV{ + Key: prevK, + Value: strings.Join([]string{kvs.Get(prevK), sanitizeValue(kv[0])}, KvSpaceSeparator), + }) continue } if len(kv) == 1 { return fmt.Errorf("value for key '%s' cannot be empty", kv[0]) } prevK = kv[0] - kvs[kv[0]] = sanitizeValue(kv[1]) + kvs = append(kvs, KV{ + Key: kv[0], + Value: sanitizeValue(kv[1]), + }) } _, ok := targets[subSystemValue[0]]