From 30e80d0a862b340a518c601100c1abe6ee37d4be Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 27 Nov 2019 09:36:08 -0800 Subject: [PATCH] Add ReadFrom,WriteTo helpers for server config (#8580) --- cmd/admin-handlers-config-kv.go | 108 ++--------- cmd/config-current.go | 313 ++++++++++++++++---------------- cmd/config.go | 5 + cmd/config/config.go | 106 +++++++++++ 4 files changed, 285 insertions(+), 247 deletions(-) diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go index 2fa67e2d5..3f6cf86ee 100644 --- a/cmd/admin-handlers-config-kv.go +++ b/cmd/admin-handlers-config-kv.go @@ -17,14 +17,12 @@ package cmd import ( - "bufio" "bytes" "context" "encoding/json" "io" "net/http" "strconv" - "strings" "github.com/gorilla/mux" "github.com/minio/minio/cmd/config" @@ -79,24 +77,14 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) return } + cfg, err := readServerConfig(ctx, objectAPI) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } - scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) - for scanner.Scan() { - // Skip any empty lines - if scanner.Text() == "" { - continue - } - if err = cfg.DelKVS(scanner.Text()); err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - } - if err = scanner.Err(); err != nil { + if err = cfg.DelFrom(bytes.NewReader(kvBytes)); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } @@ -135,31 +123,9 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) return } + cfg, err := readServerConfig(ctx, objectAPI) if err != nil { - // Config not found for some reason, allow things to continue - // by initializing a new fresh config in safe mode. - if err == errConfigNotFound && globalSafeMode { - cfg = newServerConfig() - err = nil - } else { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - } - - scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) - for scanner.Scan() { - // Skip any empty lines, or comment like characters - if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) { - continue - } - if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - } - if err = scanner.Err(); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } @@ -210,23 +176,10 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ vars := mux.Vars(r) var buf = &bytes.Buffer{} - key := vars["key"] - if key != "" { - kvs, err := cfg.GetKVS(key, defaultKVS()) - if err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - for k, kv := range kvs { - buf.WriteString(k) - buf.WriteString(config.KvSpaceSeparator) - buf.WriteString(kv.String()) - if len(kvs) > 1 { - buf.WriteString(config.KvNewline) - } - } - } else { - buf.WriteString(cfg.String()) + cw := config.NewConfigWriteTo(cfg, vars["key"]) + if _, err := cw.WriteTo(buf); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return } password := globalActiveCred.SecretKey @@ -297,30 +250,11 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r cfg, err := readServerConfig(ctx, objectAPI) if err != nil { - // Config not found for some reason, allow things to continue - // by initializing a new fresh config in safe mode. - if err == errConfigNotFound && globalSafeMode { - cfg = newServerConfig() - err = nil - } else { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return } - scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) - for scanner.Scan() { - // Skip any empty lines, or comment like characters - if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) { - continue - } - - if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - } - if err = scanner.Err(); err != nil { + if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } @@ -432,19 +366,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques } cfg := newServerConfig() - scanner := bufio.NewScanner(bytes.NewReader(kvBytes)) - for scanner.Scan() { - // Skip any empty lines, or comment like characters - if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) { - continue - } - if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil { - writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) - return - } - } - - if err = scanner.Err(); err != nil { + if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } @@ -484,14 +406,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques return } - config, err := readServerConfig(ctx, objectAPI) + cfg, err := readServerConfig(ctx, objectAPI) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } var buf = &bytes.Buffer{} - buf.WriteString(config.String()) + cw := config.NewConfigWriteTo(cfg, "") + if _, err = cw.WriteTo(buf); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } password := globalActiveCred.SecretKey econfigData, err := madmin.EncryptData(password, buf.Bytes()) diff --git a/cmd/config-current.go b/cmd/config-current.go index 37f54b682..fcdaebee2 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -39,6 +39,154 @@ import ( "github.com/minio/minio/pkg/env" ) +func init() { + var kvs = map[string]config.KVS{ + config.EtcdSubSys: etcd.DefaultKVS, + config.CacheSubSys: cache.DefaultKVS, + config.CompressionSubSys: compress.DefaultKVS, + config.StorageClassSubSys: storageclass.DefaultKVS, + config.IdentityLDAPSubSys: xldap.DefaultKVS, + config.IdentityOpenIDSubSys: openid.DefaultKVS, + config.PolicyOPASubSys: opa.DefaultKVS, + config.RegionSubSys: config.DefaultRegionKVS, + config.CredentialsSubSys: config.DefaultCredentialKVS, + config.KmsVaultSubSys: crypto.DefaultKVS, + config.LoggerWebhookSubSys: logger.DefaultKVS, + config.AuditWebhookSubSys: logger.DefaultAuditKVS, + } + for k, v := range notify.DefaultNotificationKVS { + kvs[k] = v + } + config.RegisterDefaultKVS(kvs) + + // Captures help for each sub-system + var helpSubSys = config.HelpKVS{ + config.HelpKV{ + Key: config.RegionSubSys, + Description: "Configure to describe the physical location of the server", + }, + config.HelpKV{ + Key: config.StorageClassSubSys, + Description: "Configure to control data and parity per object", + }, + config.HelpKV{ + Key: config.CacheSubSys, + Description: "Configure to enable edge caching", + }, + config.HelpKV{ + Key: config.CompressionSubSys, + Description: "Configure to enable streaming on disk compression", + }, + config.HelpKV{ + Key: config.EtcdSubSys, + Description: "Configure to enable 'etcd' configuration", + }, + config.HelpKV{ + Key: config.IdentityOpenIDSubSys, + Description: "Configure to enable OpenID SSO support", + }, + config.HelpKV{ + Key: config.IdentityLDAPSubSys, + Description: "Configure to enable LDAP SSO support", + }, + config.HelpKV{ + Key: config.PolicyOPASubSys, + Description: "Configure to enable external OPA policy support", + }, + config.HelpKV{ + Key: config.KmsVaultSubSys, + Description: "Configure to enable Vault based external KMS", + }, + config.HelpKV{ + Key: config.LoggerWebhookSubSys, + Description: "Configure to enable Webhook based logger", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.AuditWebhookSubSys, + Description: "Configure to enable Webhook based audit logger", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyWebhookSubSys, + Description: "Configure to publish events to Webhook target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyAMQPSubSys, + Description: "Configure to publish events to AMQP target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyKafkaSubSys, + Description: "Configure to publish events to Kafka target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyMQTTSubSys, + Description: "Configure to publish events to MQTT target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyNATSSubSys, + Description: "Configure to publish events to NATS target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyNSQSubSys, + Description: "Configure to publish events to NSQ target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyMySQLSubSys, + Description: "Configure to publish events to MySQL target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyPostgresSubSys, + Description: "Configure to publish events to Postgres target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyRedisSubSys, + Description: "Configure to publish events to Redis target", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyESSubSys, + Description: "Configure to publish events to Elasticsearch target", + MultipleTargets: true, + }, + } + + var helpMap = map[string]config.HelpKVS{ + "": helpSubSys, // Help for all sub-systems. + config.RegionSubSys: config.RegionHelp, + config.EtcdSubSys: etcd.Help, + config.CacheSubSys: cache.Help, + config.CompressionSubSys: compress.Help, + config.StorageClassSubSys: storageclass.Help, + config.IdentityOpenIDSubSys: openid.Help, + config.IdentityLDAPSubSys: xldap.Help, + config.PolicyOPASubSys: opa.Help, + config.KmsVaultSubSys: crypto.Help, + config.LoggerWebhookSubSys: logger.Help, + config.AuditWebhookSubSys: logger.HelpAudit, + config.NotifyAMQPSubSys: notify.HelpAMQP, + config.NotifyKafkaSubSys: notify.HelpKafka, + config.NotifyMQTTSubSys: notify.HelpMQTT, + config.NotifyNATSSubSys: notify.HelpNATS, + config.NotifyNSQSubSys: notify.HelpNSQ, + config.NotifyMySQLSubSys: notify.HelpMySQL, + config.NotifyPostgresSubSys: notify.HelpPostgres, + config.NotifyRedisSubSys: notify.HelpRedis, + config.NotifyWebhookSubSys: notify.HelpWebhook, + config.NotifyESSubSys: notify.HelpES, + } + + config.RegisterHelpSubSys(helpMap) +} + var ( // globalServerConfig server config. globalServerConfig config.Config @@ -257,152 +405,6 @@ func lookupConfigs(s config.Config) (err error) { return nil } -func defaultKVS() map[string]config.KVS { - var kvs = map[string]config.KVS{ - config.EtcdSubSys: etcd.DefaultKVS, - config.CacheSubSys: cache.DefaultKVS, - config.CompressionSubSys: compress.DefaultKVS, - config.StorageClassSubSys: storageclass.DefaultKVS, - config.IdentityLDAPSubSys: xldap.DefaultKVS, - config.IdentityOpenIDSubSys: openid.DefaultKVS, - config.PolicyOPASubSys: opa.DefaultKVS, - config.RegionSubSys: config.DefaultRegionKVS, - config.CredentialsSubSys: config.DefaultCredentialKVS, - config.KmsVaultSubSys: crypto.DefaultKVS, - config.LoggerWebhookSubSys: logger.DefaultKVS, - config.AuditWebhookSubSys: logger.DefaultAuditKVS, - } - for k, v := range notify.DefaultNotificationKVS { - kvs[k] = v - } - return kvs -} - -// Captures help for each sub-system -var helpSubSys = config.HelpKVS{ - config.HelpKV{ - Key: config.RegionSubSys, - Description: "Configure to describe the physical location of the server", - }, - config.HelpKV{ - Key: config.StorageClassSubSys, - Description: "Configure to control data and parity per object", - }, - config.HelpKV{ - Key: config.CacheSubSys, - Description: "Configure to enable edge caching", - }, - config.HelpKV{ - Key: config.CompressionSubSys, - Description: "Configure to enable streaming on disk compression", - }, - config.HelpKV{ - Key: config.EtcdSubSys, - Description: "Configure to enable 'etcd' configuration", - }, - config.HelpKV{ - Key: config.IdentityOpenIDSubSys, - Description: "Configure to enable OpenID SSO support", - }, - config.HelpKV{ - Key: config.IdentityLDAPSubSys, - Description: "Configure to enable LDAP SSO support", - }, - config.HelpKV{ - Key: config.PolicyOPASubSys, - Description: "Configure to enable external OPA policy support", - }, - config.HelpKV{ - Key: config.KmsVaultSubSys, - Description: "Configure to enable Vault based external KMS", - }, - config.HelpKV{ - Key: config.LoggerWebhookSubSys, - Description: "Configure to enable Webhook based logger", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.AuditWebhookSubSys, - Description: "Configure to enable Webhook based audit logger", - MultipleTargets: true, - }, - - config.HelpKV{ - Key: config.NotifyWebhookSubSys, - Description: "Configure to publish events to Webhook target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyAMQPSubSys, - Description: "Configure to publish events to AMQP target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyKafkaSubSys, - Description: "Configure to publish events to Kafka target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyMQTTSubSys, - Description: "Configure to publish events to MQTT target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyNATSSubSys, - Description: "Configure to publish events to NATS target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyNSQSubSys, - Description: "Configure to publish events to NSQ target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyMySQLSubSys, - Description: "Configure to publish events to MySQL target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyPostgresSubSys, - Description: "Configure to publish events to Postgres target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyRedisSubSys, - Description: "Configure to publish events to Redis target", - MultipleTargets: true, - }, - config.HelpKV{ - Key: config.NotifyESSubSys, - Description: "Configure to publish events to Elasticsearch target", - MultipleTargets: true, - }, -} - -var helpMap = map[string]config.HelpKVS{ - config.RegionSubSys: config.RegionHelp, - config.EtcdSubSys: etcd.Help, - config.CacheSubSys: cache.Help, - config.CompressionSubSys: compress.Help, - config.StorageClassSubSys: storageclass.Help, - config.IdentityOpenIDSubSys: openid.Help, - config.IdentityLDAPSubSys: xldap.Help, - config.PolicyOPASubSys: opa.Help, - config.KmsVaultSubSys: crypto.Help, - config.LoggerWebhookSubSys: logger.Help, - config.AuditWebhookSubSys: logger.HelpAudit, - config.NotifyAMQPSubSys: notify.HelpAMQP, - config.NotifyKafkaSubSys: notify.HelpKafka, - config.NotifyMQTTSubSys: notify.HelpMQTT, - config.NotifyNATSSubSys: notify.HelpNATS, - config.NotifyNSQSubSys: notify.HelpNSQ, - config.NotifyMySQLSubSys: notify.HelpMySQL, - config.NotifyPostgresSubSys: notify.HelpPostgres, - config.NotifyRedisSubSys: notify.HelpRedis, - config.NotifyWebhookSubSys: notify.HelpWebhook, - config.NotifyESSubSys: notify.HelpES, -} - // Help - return sub-system level help type Help struct { SubSys string `json:"subSys"` @@ -414,18 +416,24 @@ type Help struct { // GetHelp - returns help for sub-sys, a key for a sub-system or all the help. func GetHelp(subSys, key string, envOnly bool) (Help, error) { if len(subSys) == 0 { - return Help{KeysHelp: helpSubSys}, nil + return Help{KeysHelp: config.HelpSubSysMap[subSys]}, nil } subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2) if len(subSystemValue) == 0 { return Help{}, config.Errorf("invalid number of arguments %s", subSys) } - if !config.SubSystems.Contains(subSystemValue[0]) { + subSys = subSystemValue[0] + + subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys) + if !ok { return Help{}, config.Errorf("unknown sub-system %s", subSys) } - h := helpMap[subSystemValue[0]] + h, ok := config.HelpSubSysMap[subSys] + if !ok { + return Help{}, config.Errorf("unknown sub-system %s", subSys) + } if key != "" { value, ok := h.Lookup(key) if !ok { @@ -434,8 +442,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) { h = config.HelpKVS{value} } - subSys = subSystemValue[0] - envHelp := config.HelpKVS{} if envOnly { for _, hkv := range h { @@ -452,11 +458,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) { h = envHelp } - subSysHelp, ok := helpSubSys.Lookup(subSys) - if !ok { - return Help{}, config.Errorf("unknown sub-system %s", subSys) - } - return Help{ SubSys: subSys, Description: subSysHelp.Description, diff --git a/cmd/config.go b/cmd/config.go index 6077df105..3d5c456e1 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -151,6 +151,11 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e configFile := path.Join(minioConfigPrefix, minioConfigFile) configData, err := readConfig(ctx, objAPI, configFile) if err != nil { + // Config not found for some reason, allow things to continue + // by initializing a new fresh config in safe mode. + if err == errConfigNotFound && globalSafeMode { + return newServerConfig(), nil + } return nil, err } diff --git a/cmd/config/config.go b/cmd/config/config.go index fc8b7d32a..90f407a0e 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -18,7 +18,9 @@ package config import ( + "bufio" "fmt" + "io" "regexp" "strings" @@ -145,6 +147,35 @@ const ( EnvWordDelimiter = `_` ) +// DefaultKVS - default kvs for all sub-systems +var DefaultKVS map[string]KVS + +// RegisterDefaultKVS - this function saves input kvsMap +// globally, this should be called only once preferably +// during `init()`. +func RegisterDefaultKVS(kvsMap map[string]KVS) { + DefaultKVS = map[string]KVS{} + for subSys, kvs := range kvsMap { + DefaultKVS[subSys] = kvs + } +} + +// HelpSubSysMap - help for all individual KVS for each sub-systems +// also carries a special empty sub-system which dumps +// help for each sub-system key. +var HelpSubSysMap map[string]HelpKVS + +// RegisterHelpSubSys - this function saves +// input help KVS for each sub-system globally, +// this function should be called only once +// preferably in during `init()`. +func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) { + HelpSubSysMap = map[string]HelpKVS{} + for subSys, hkvs := range helpKVSMap { + HelpSubSysMap[subSys] = hkvs + } +} + // KV - is a shorthand of each key value. type KV struct { Key string `json:"key"` @@ -216,6 +247,81 @@ func (c Config) String() string { return s.String() } +// DelFrom - deletes all keys in the input reader. +func (c Config) DelFrom(r io.Reader) error { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + // Skip any empty lines, or comment like characters + if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) { + continue + } + if err := c.DelKVS(scanner.Text()); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + return nil +} + +// ReadFrom - implements io.ReaderFrom interface +func (c Config) ReadFrom(r io.Reader) (int64, error) { + var n int + scanner := bufio.NewScanner(r) + for scanner.Scan() { + // Skip any empty lines, or comment like characters + if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) { + continue + } + if err := c.SetKVS(scanner.Text(), DefaultKVS); err != nil { + return 0, err + } + n += len(scanner.Text()) + } + if err := scanner.Err(); err != nil { + return 0, err + } + return int64(n), nil +} + +type configWriteTo struct { + Config + filterByKey string +} + +// NewConfigWriteTo - returns a struct which +// allows for serializing the config/kv struct +// to a io.WriterTo +func NewConfigWriteTo(cfg Config, key string) io.WriterTo { + return &configWriteTo{Config: cfg, filterByKey: key} +} + +// WriteTo - implements io.WriterTo interface implementation for config. +func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) { + if c.filterByKey == "" { + n, err := w.Write([]byte(c.String())) + return int64(n), err + } + kvs, err := c.GetKVS(c.filterByKey, DefaultKVS) + if err != nil { + return 0, err + } + var n int + for k, kv := range kvs { + m1, _ := w.Write([]byte(k)) + m2, _ := w.Write([]byte(KvSpaceSeparator)) + m3, _ := w.Write([]byte(kv.String())) + if len(kvs) > 1 { + m4, _ := w.Write([]byte(KvNewline)) + n += m1 + m2 + m3 + m4 + } else { + n += m1 + m2 + m3 + } + } + return int64(n), nil +} + // Default KV configs for worm and region var ( DefaultCredentialKVS = KVS{