oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add ReadFrom,WriteTo helpers for server config (#8580)

Browse Source
master
Harshavardhana 5 years ago committed by GitHub
parent 87443af49e
commit 30e80d0a86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 285 additions and 247 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 108
      cmd/admin-handlers-config-kv.go
  2. 313
      cmd/config-current.go
  3. 5
      cmd/config.go
  4. 106
      cmd/config/config.go

108
cmd/admin-handlers-config-kv.go
Unescape View File

@ -17,14 +17,12 @@
package cmd
import (
"bufio"
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
@ -79,24 +77,14 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
return
}
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines
if scanner.Text() == "" {
continue
}
if err = cfg.DelKVS(scanner.Text()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if err = cfg.DelFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -135,31 +123,9 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
return
}
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
cfg = newServerConfig()
err = nil
} else {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -210,23 +176,10 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
vars := mux.Vars(r)
var buf = &bytes.Buffer{}
key := vars["key"]
if key != "" {
kvs, err := cfg.GetKVS(key, defaultKVS())
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for k, kv := range kvs {
buf.WriteString(k)
buf.WriteString(config.KvSpaceSeparator)
buf.WriteString(kv.String())
if len(kvs) > 1 {
buf.WriteString(config.KvNewline)
}
}
} else {
buf.WriteString(cfg.String())
cw := config.NewConfigWriteTo(cfg, vars["key"])
if _, err := cw.WriteTo(buf); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
password := globalActiveCred.SecretKey
@ -297,30 +250,11 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
cfg = newServerConfig()
err = nil
} else {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -432,19 +366,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
}
cfg := newServerConfig()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), config.KvComment) {
continue
}
if err = cfg.SetKVS(scanner.Text(), defaultKVS()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
if err = scanner.Err(); err != nil {
if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@ -484,14 +406,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
return
}
config, err := readServerConfig(ctx, objectAPI)
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
var buf = &bytes.Buffer{}
buf.WriteString(config.String())
cw := config.NewConfigWriteTo(cfg, "")
if _, err = cw.WriteTo(buf); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
password := globalActiveCred.SecretKey
econfigData, err := madmin.EncryptData(password, buf.Bytes())

313
cmd/config-current.go
Unescape View File

@ -39,6 +39,154 @@ import (
"github.com/minio/minio/pkg/env"
)
func init() {
var kvs = map[string]config.KVS{
config.EtcdSubSys: etcd.DefaultKVS,
config.CacheSubSys: cache.DefaultKVS,
config.CompressionSubSys: compress.DefaultKVS,
config.StorageClassSubSys: storageclass.DefaultKVS,
config.IdentityLDAPSubSys: xldap.DefaultKVS,
config.IdentityOpenIDSubSys: openid.DefaultKVS,
config.PolicyOPASubSys: opa.DefaultKVS,
config.RegionSubSys: config.DefaultRegionKVS,
config.CredentialsSubSys: config.DefaultCredentialKVS,
config.KmsVaultSubSys: crypto.DefaultKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS,
}
for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v
}
config.RegisterDefaultKVS(kvs)
// Captures help for each sub-system
var helpSubSys = config.HelpKVS{
config.HelpKV{
Key: config.RegionSubSys,
Description: "Configure to describe the physical location of the server",
},
config.HelpKV{
Key: config.StorageClassSubSys,
Description: "Configure to control data and parity per object",
},
config.HelpKV{
Key: config.CacheSubSys,
Description: "Configure to enable edge caching",
},
config.HelpKV{
Key: config.CompressionSubSys,
Description: "Configure to enable streaming on disk compression",
},
config.HelpKV{
Key: config.EtcdSubSys,
Description: "Configure to enable 'etcd' configuration",
},
config.HelpKV{
Key: config.IdentityOpenIDSubSys,
Description: "Configure to enable OpenID SSO support",
},
config.HelpKV{
Key: config.IdentityLDAPSubSys,
Description: "Configure to enable LDAP SSO support",
},
config.HelpKV{
Key: config.PolicyOPASubSys,
Description: "Configure to enable external OPA policy support",
},
config.HelpKV{
Key: config.KmsVaultSubSys,
Description: "Configure to enable Vault based external KMS",
},
config.HelpKV{
Key: config.LoggerWebhookSubSys,
Description: "Configure to enable Webhook based logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.AuditWebhookSubSys,
Description: "Configure to enable Webhook based audit logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyWebhookSubSys,
Description: "Configure to publish events to Webhook target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyAMQPSubSys,
Description: "Configure to publish events to AMQP target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyKafkaSubSys,
Description: "Configure to publish events to Kafka target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMQTTSubSys,
Description: "Configure to publish events to MQTT target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNATSSubSys,
Description: "Configure to publish events to NATS target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNSQSubSys,
Description: "Configure to publish events to NSQ target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMySQLSubSys,
Description: "Configure to publish events to MySQL target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyPostgresSubSys,
Description: "Configure to publish events to Postgres target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyRedisSubSys,
Description: "Configure to publish events to Redis target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyESSubSys,
Description: "Configure to publish events to Elasticsearch target",
MultipleTargets: true,
},
}
var helpMap = map[string]config.HelpKVS{
"": helpSubSys, // Help for all sub-systems.
config.RegionSubSys: config.RegionHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
config.IdentityOpenIDSubSys: openid.Help,
config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help,
config.KmsVaultSubSys: crypto.Help,
config.LoggerWebhookSubSys: logger.Help,
config.AuditWebhookSubSys: logger.HelpAudit,
config.NotifyAMQPSubSys: notify.HelpAMQP,
config.NotifyKafkaSubSys: notify.HelpKafka,
config.NotifyMQTTSubSys: notify.HelpMQTT,
config.NotifyNATSSubSys: notify.HelpNATS,
config.NotifyNSQSubSys: notify.HelpNSQ,
config.NotifyMySQLSubSys: notify.HelpMySQL,
config.NotifyPostgresSubSys: notify.HelpPostgres,
config.NotifyRedisSubSys: notify.HelpRedis,
config.NotifyWebhookSubSys: notify.HelpWebhook,
config.NotifyESSubSys: notify.HelpES,
}
config.RegisterHelpSubSys(helpMap)
}
var (
// globalServerConfig server config.
globalServerConfig config.Config
@ -257,152 +405,6 @@ func lookupConfigs(s config.Config) (err error) {
return nil
}
func defaultKVS() map[string]config.KVS {
var kvs = map[string]config.KVS{
config.EtcdSubSys: etcd.DefaultKVS,
config.CacheSubSys: cache.DefaultKVS,
config.CompressionSubSys: compress.DefaultKVS,
config.StorageClassSubSys: storageclass.DefaultKVS,
config.IdentityLDAPSubSys: xldap.DefaultKVS,
config.IdentityOpenIDSubSys: openid.DefaultKVS,
config.PolicyOPASubSys: opa.DefaultKVS,
config.RegionSubSys: config.DefaultRegionKVS,
config.CredentialsSubSys: config.DefaultCredentialKVS,
config.KmsVaultSubSys: crypto.DefaultKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS,
}
for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v
}
return kvs
}
// Captures help for each sub-system
var helpSubSys = config.HelpKVS{
config.HelpKV{
Key: config.RegionSubSys,
Description: "Configure to describe the physical location of the server",
},
config.HelpKV{
Key: config.StorageClassSubSys,
Description: "Configure to control data and parity per object",
},
config.HelpKV{
Key: config.CacheSubSys,
Description: "Configure to enable edge caching",
},
config.HelpKV{
Key: config.CompressionSubSys,
Description: "Configure to enable streaming on disk compression",
},
config.HelpKV{
Key: config.EtcdSubSys,
Description: "Configure to enable 'etcd' configuration",
},
config.HelpKV{
Key: config.IdentityOpenIDSubSys,
Description: "Configure to enable OpenID SSO support",
},
config.HelpKV{
Key: config.IdentityLDAPSubSys,
Description: "Configure to enable LDAP SSO support",
},
config.HelpKV{
Key: config.PolicyOPASubSys,
Description: "Configure to enable external OPA policy support",
},
config.HelpKV{
Key: config.KmsVaultSubSys,
Description: "Configure to enable Vault based external KMS",
},
config.HelpKV{
Key: config.LoggerWebhookSubSys,
Description: "Configure to enable Webhook based logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.AuditWebhookSubSys,
Description: "Configure to enable Webhook based audit logger",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyWebhookSubSys,
Description: "Configure to publish events to Webhook target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyAMQPSubSys,
Description: "Configure to publish events to AMQP target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyKafkaSubSys,
Description: "Configure to publish events to Kafka target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMQTTSubSys,
Description: "Configure to publish events to MQTT target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNATSSubSys,
Description: "Configure to publish events to NATS target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyNSQSubSys,
Description: "Configure to publish events to NSQ target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyMySQLSubSys,
Description: "Configure to publish events to MySQL target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyPostgresSubSys,
Description: "Configure to publish events to Postgres target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyRedisSubSys,
Description: "Configure to publish events to Redis target",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyESSubSys,
Description: "Configure to publish events to Elasticsearch target",
MultipleTargets: true,
},
}
var helpMap = map[string]config.HelpKVS{
config.RegionSubSys: config.RegionHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
config.IdentityOpenIDSubSys: openid.Help,
config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help,
config.KmsVaultSubSys: crypto.Help,
config.LoggerWebhookSubSys: logger.Help,
config.AuditWebhookSubSys: logger.HelpAudit,
config.NotifyAMQPSubSys: notify.HelpAMQP,
config.NotifyKafkaSubSys: notify.HelpKafka,
config.NotifyMQTTSubSys: notify.HelpMQTT,
config.NotifyNATSSubSys: notify.HelpNATS,
config.NotifyNSQSubSys: notify.HelpNSQ,
config.NotifyMySQLSubSys: notify.HelpMySQL,
config.NotifyPostgresSubSys: notify.HelpPostgres,
config.NotifyRedisSubSys: notify.HelpRedis,
config.NotifyWebhookSubSys: notify.HelpWebhook,
config.NotifyESSubSys: notify.HelpES,
}
// Help - return sub-system level help
type Help struct {
SubSys string `json:"subSys"`
@ -414,18 +416,24 @@ type Help struct {
// GetHelp - returns help for sub-sys, a key for a sub-system or all the help.
func GetHelp(subSys, key string, envOnly bool) (Help, error) {
if len(subSys) == 0 {
return Help{KeysHelp: helpSubSys}, nil
return Help{KeysHelp: config.HelpSubSysMap[subSys]}, nil
}
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return Help{}, config.Errorf("invalid number of arguments %s", subSys)
}
if !config.SubSystems.Contains(subSystemValue[0]) {
subSys = subSystemValue[0]
subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys)
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
h := helpMap[subSystemValue[0]]
h, ok := config.HelpSubSysMap[subSys]
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
if key != "" {
value, ok := h.Lookup(key)
if !ok {
@ -434,8 +442,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
h = config.HelpKVS{value}
}
subSys = subSystemValue[0]
envHelp := config.HelpKVS{}
if envOnly {
for _, hkv := range h {
@ -452,11 +458,6 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
h = envHelp
}
subSysHelp, ok := helpSubSys.Lookup(subSys)
if !ok {
return Help{}, config.Errorf("unknown sub-system %s", subSys)
}
return Help{
SubSys: subSys,
Description: subSysHelp.Description,

5
cmd/config.go
Unescape View File

@ -151,6 +151,11 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
configFile := path.Join(minioConfigPrefix, minioConfigFile)
configData, err := readConfig(ctx, objAPI, configFile)
if err != nil {
// Config not found for some reason, allow things to continue
// by initializing a new fresh config in safe mode.
if err == errConfigNotFound && globalSafeMode {
return newServerConfig(), nil
}
return nil, err
}

106
cmd/config/config.go
Unescape View File

@ -18,7 +18,9 @@
package config
import (
"bufio"
"fmt"
"io"
"regexp"
"strings"
@ -145,6 +147,35 @@ const (
EnvWordDelimiter = `_`
)
// DefaultKVS - default kvs for all sub-systems
var DefaultKVS map[string]KVS
// RegisterDefaultKVS - this function saves input kvsMap
// globally, this should be called only once preferably
// during `init()`.
func RegisterDefaultKVS(kvsMap map[string]KVS) {
DefaultKVS = map[string]KVS{}
for subSys, kvs := range kvsMap {
DefaultKVS[subSys] = kvs
}
}
// HelpSubSysMap - help for all individual KVS for each sub-systems
// also carries a special empty sub-system which dumps
// help for each sub-system key.
var HelpSubSysMap map[string]HelpKVS
// RegisterHelpSubSys - this function saves
// input help KVS for each sub-system globally,
// this function should be called only once
// preferably in during `init()`.
func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) {
HelpSubSysMap = map[string]HelpKVS{}
for subSys, hkvs := range helpKVSMap {
HelpSubSysMap[subSys] = hkvs
}
}
// KV - is a shorthand of each key value.
type KV struct {
Key string `json:"key"`
@ -216,6 +247,81 @@ func (c Config) String() string {
return s.String()
}
// DelFrom - deletes all keys in the input reader.
func (c Config) DelFrom(r io.Reader) error {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) {
continue
}
if err := c.DelKVS(scanner.Text()); err != nil {
return err
}
}
if err := scanner.Err(); err != nil {
return err
}
return nil
}
// ReadFrom - implements io.ReaderFrom interface
func (c Config) ReadFrom(r io.Reader) (int64, error) {
var n int
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Skip any empty lines, or comment like characters
if scanner.Text() == "" || strings.HasPrefix(scanner.Text(), KvComment) {
continue
}
if err := c.SetKVS(scanner.Text(), DefaultKVS); err != nil {
return 0, err
}
n += len(scanner.Text())
}
if err := scanner.Err(); err != nil {
return 0, err
}
return int64(n), nil
}
type configWriteTo struct {
Config
filterByKey string
}
// NewConfigWriteTo - returns a struct which
// allows for serializing the config/kv struct
// to a io.WriterTo
func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
return &configWriteTo{Config: cfg, filterByKey: key}
}
// WriteTo - implements io.WriterTo interface implementation for config.
func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
if c.filterByKey == "" {
n, err := w.Write([]byte(c.String()))
return int64(n), err
}
kvs, err := c.GetKVS(c.filterByKey, DefaultKVS)
if err != nil {
return 0, err
}
var n int
for k, kv := range kvs {
m1, _ := w.Write([]byte(k))
m2, _ := w.Write([]byte(KvSpaceSeparator))
m3, _ := w.Write([]byte(kv.String()))
if len(kvs) > 1 {
m4, _ := w.Write([]byte(KvNewline))
n += m1 + m2 + m3 + m4
} else {
n += m1 + m2 + m3
}
}
return int64(n), nil
}
// Default KV configs for worm and region
var (
DefaultCredentialKVS = KVS{

Write Preview
Loading…
Cancel
Save