From 11e15f9b4c96a14493c8132abde66b0c741bf3ed Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Thu, 23 Mar 2017 16:27:22 +0100 Subject: [PATCH] config: Remove level in console/file loggers (#3938) Also rename fileName field in file logger to filename --- cmd/config-migrate.go | 109 +++++++++++++++++- cmd/config-migrate_test.go | 12 +- cmd/config-old.go | 51 ++++++-- cmd/{config-v15.go => config-v16.go} | 49 ++++---- ...{config-v15_test.go => config-v16_test.go} | 27 ++--- cmd/gateway-main.go | 4 +- cmd/logger-console-hook.go | 21 +--- cmd/logger-file-hook.go | 18 ++- 8 files changed, 205 insertions(+), 86 deletions(-) rename cmd/{config-v15.go => config-v16.go} (88%) rename cmd/{config-v15_test.go => config-v16_test.go} (94%) diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 063d19ac6..7b58f9d75 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -82,6 +82,10 @@ func migrateConfig() error { if err := migrateV14ToV15(); err != nil { return err } + // Migration version '15' to '16'. + if err := migrateV15ToV16(); err != nil { + return err + } return nil } @@ -136,11 +140,11 @@ func migrateV2ToV3() error { // Region needs to be set for AWS Signature V4. srvConfig.Region = globalMinioDefaultRegion } - srvConfig.Logger.Console = consoleLogger{ + srvConfig.Logger.Console = consoleLoggerV1{ Enable: true, Level: "fatal", } - flogger := fileLogger{} + flogger := fileLoggerV1{} flogger.Level = "error" if cv2.FileLogger.Filename != "" { flogger.Enable = true @@ -745,7 +749,7 @@ func migrateV12ToV13() error { // Copy over fields from V12 into V13 config struct srvConfig := &serverConfigV13{ - Logger: &logger{}, + Logger: &loggerV7{}, Notify: ¬ifier{}, } srvConfig.Version = "13" @@ -825,7 +829,7 @@ func migrateV13ToV14() error { // Copy over fields from V13 into V14 config struct srvConfig := &serverConfigV14{ - Logger: &logger{}, + Logger: &loggerV7{}, Notify: ¬ifier{}, } srvConfig.Version = "14" @@ -910,7 +914,7 @@ func migrateV14ToV15() error { // Copy over fields from V14 into V15 config struct srvConfig := &serverConfigV15{ - Logger: &logger{}, + Logger: &loggerV7{}, Notify: ¬ifier{}, } srvConfig.Version = "15" @@ -981,3 +985,98 @@ func migrateV14ToV15() error { console.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv14.Version, srvConfig.Version) return nil } + +// Version '15' to '16' migration. Remove log level in loggers +// and rename 'fileName' filed in File logger to 'filename' +func migrateV15ToV16() error { + configFile := getConfigFile() + + cv15 := &serverConfigV15{} + _, err := quick.Load(configFile, cv15) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return fmt.Errorf("Unable to load config version ‘15’. %v", err) + } + if cv15.Version != "15" { + return nil + } + + // Copy over fields from V15 into V16 config struct + srvConfig := &serverConfigV16{ + Logger: &logger{}, + Notify: ¬ifier{}, + } + srvConfig.Version = "16" + srvConfig.Credential = cv15.Credential + srvConfig.Region = cv15.Region + if srvConfig.Region == "" { + // Region needs to be set for AWS Signature Version 4. + srvConfig.Region = globalMinioDefaultRegion + } + + // check and set notifiers config + if len(cv15.Notify.AMQP) == 0 { + srvConfig.Notify.AMQP = make(map[string]amqpNotify) + srvConfig.Notify.AMQP["1"] = amqpNotify{} + } else { + srvConfig.Notify.AMQP = cv15.Notify.AMQP + } + if len(cv15.Notify.ElasticSearch) == 0 { + srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify) + srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{} + } else { + srvConfig.Notify.ElasticSearch = cv15.Notify.ElasticSearch + } + if len(cv15.Notify.Redis) == 0 { + srvConfig.Notify.Redis = make(map[string]redisNotify) + srvConfig.Notify.Redis["1"] = redisNotify{} + } else { + srvConfig.Notify.Redis = cv15.Notify.Redis + } + if len(cv15.Notify.PostgreSQL) == 0 { + srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify) + srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{} + } else { + srvConfig.Notify.PostgreSQL = cv15.Notify.PostgreSQL + } + if len(cv15.Notify.Kafka) == 0 { + srvConfig.Notify.Kafka = make(map[string]kafkaNotify) + srvConfig.Notify.Kafka["1"] = kafkaNotify{} + } else { + srvConfig.Notify.Kafka = cv15.Notify.Kafka + } + if len(cv15.Notify.NATS) == 0 { + srvConfig.Notify.NATS = make(map[string]natsNotify) + srvConfig.Notify.NATS["1"] = natsNotify{} + } else { + srvConfig.Notify.NATS = cv15.Notify.NATS + } + if len(cv15.Notify.Webhook) == 0 { + srvConfig.Notify.Webhook = make(map[string]webhookNotify) + srvConfig.Notify.Webhook["1"] = webhookNotify{} + } else { + srvConfig.Notify.Webhook = cv15.Notify.Webhook + } + if len(cv15.Notify.MySQL) == 0 { + srvConfig.Notify.MySQL = make(map[string]mySQLNotify) + srvConfig.Notify.MySQL["1"] = mySQLNotify{} + } else { + srvConfig.Notify.MySQL = cv15.Notify.MySQL + } + + // Load browser config from existing config in the file. + srvConfig.Browser = cv15.Browser + + // Migrate console and file fields + srvConfig.Logger.Console = consoleLogger{Enable: cv15.Logger.Console.Enable} + srvConfig.Logger.File = fileLogger{Enable: cv15.Logger.File.Enable, Filename: cv15.Logger.File.Filename} + + if err = quick.Save(configFile, srvConfig); err != nil { + return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err) + } + + console.Printf("Migration from version ‘%s’ to ‘%s’ completed successfully.\n", cv15.Version, srvConfig.Version) + + return nil +} diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go index 201d087af..9774bc412 100644 --- a/cmd/config-migrate_test.go +++ b/cmd/config-migrate_test.go @@ -112,10 +112,13 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) { if err := migrateV14ToV15(); err != nil { t.Fatal("migrate v14 to v15 should succeed when no config file is found") } + if err := migrateV15ToV16(); err != nil { + t.Fatal("migrate v15 to v16 should succeed when no config file is found") + } } -// Test if a config migration from v2 to v15 is successfully done -func TestServerConfigMigrateV2toV15(t *testing.T) { +// Test if a config migration from v2 to v16 is successfully done +func TestServerConfigMigrateV2toV16(t *testing.T) { rootPath, err := newTestConfig(globalMinioDefaultRegion) if err != nil { t.Fatalf("Init Test config failed") @@ -154,7 +157,7 @@ func TestServerConfigMigrateV2toV15(t *testing.T) { } // Check the version number in the upgraded config file - expectedVersion := v15 + expectedVersion := v16 if serverConfig.Version != expectedVersion { t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version) } @@ -225,4 +228,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) { if err := migrateV14ToV15(); err == nil { t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json") } + if err := migrateV15ToV16(); err == nil { + t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json") + } } diff --git a/cmd/config-old.go b/cmd/config-old.go index 720f6f9ea..5eed563bf 100644 --- a/cmd/config-old.go +++ b/cmd/config-old.go @@ -186,10 +186,22 @@ type configV5 struct { Logger loggerV5 `json:"logger"` } +// consoleLogger - default logger if not other logging is enabled. +type consoleLoggerV1 struct { + Enable bool `json:"enable"` + Level string `json:"level"` +} + +type fileLoggerV1 struct { + Enable bool `json:"enable"` + Filename string `json:"fileName"` + Level string `json:"level"` +} + type loggerV6 struct { - Console consoleLogger `json:"console"` - File fileLogger `json:"file"` - Syslog syslogLoggerV3 `json:"syslog"` + Console consoleLoggerV1 `json:"console"` + File fileLoggerV1 `json:"file"` + Syslog syslogLoggerV3 `json:"syslog"` } // configV6 server configuration version '6'. @@ -285,6 +297,12 @@ type serverConfigV9 struct { rwMutex *sync.RWMutex } +type loggerV7 struct { + sync.RWMutex + Console consoleLoggerV1 `json:"console"` + File fileLoggerV1 `json:"file"` +} + // serverConfigV10 server configuration version '10' which is like // version '9' except it drops support of syslog config, and makes the // RWMutex global (so it does not exist in this struct). @@ -296,7 +314,7 @@ type serverConfigV10 struct { Region string `json:"region"` // Additional error logging configuration. - Logger logger `json:"logger"` + Logger loggerV7 `json:"logger"` // Notification queue configuration. Notify notifierV1 `json:"notify"` @@ -324,7 +342,7 @@ type serverConfigV11 struct { Region string `json:"region"` // Additional error logging configuration. - Logger logger `json:"logger"` + Logger loggerV7 `json:"logger"` // Notification queue configuration. Notify notifierV1 `json:"notify"` @@ -340,7 +358,7 @@ type serverConfigV12 struct { Region string `json:"region"` // Additional error logging configuration. - Logger logger `json:"logger"` + Logger loggerV7 `json:"logger"` // Notification queue configuration. Notify notifierV2 `json:"notify"` @@ -356,7 +374,7 @@ type serverConfigV13 struct { Region string `json:"region"` // Additional error logging configuration. - Logger *logger `json:"logger"` + Logger *loggerV7 `json:"logger"` // Notification queue configuration. Notify *notifier `json:"notify"` @@ -373,7 +391,24 @@ type serverConfigV14 struct { Browser string `json:"browser"` // Additional error logging configuration. - Logger *logger `json:"logger"` + Logger *loggerV7 `json:"logger"` + + // Notification queue configuration. + Notify *notifier `json:"notify"` +} + +// serverConfigV15 server configuration version '15' which is like +// version '14' except it adds mysql support +type serverConfigV15 struct { + Version string `json:"version"` + + // S3 API configuration. + Credential credential `json:"credential"` + Region string `json:"region"` + Browser string `json:"browser"` + + // Additional error logging configuration. + Logger *loggerV7 `json:"logger"` // Notification queue configuration. Notify *notifier `json:"notify"` diff --git a/cmd/config-v15.go b/cmd/config-v16.go similarity index 88% rename from cmd/config-v15.go rename to cmd/config-v16.go index fd9c09032..1577caeb6 100644 --- a/cmd/config-v15.go +++ b/cmd/config-v16.go @@ -30,12 +30,12 @@ import ( // Read Write mutex for safe access to ServerConfig. var serverConfigMu sync.RWMutex -// Config version -const v15 = "15" +var v16 = "16" -// serverConfigV15 server configuration version '15' which is like -// version '14' except it adds support of MySQL notifications. -type serverConfigV15 struct { +// serverConfigV16 server configuration version '16' which is like +// version '15' except it removes log level field and renames `fileName` +// field of File logger to `filename` +type serverConfigV16 struct { Version string `json:"version"` // S3 API configuration. @@ -50,9 +50,9 @@ type serverConfigV15 struct { Notify *notifier `json:"notify"` } -func newServerConfigV15() *serverConfigV15 { - srvCfg := &serverConfigV15{ - Version: v15, +func newServerConfigV16() *serverConfigV16 { + srvCfg := &serverConfigV16{ + Version: v16, Region: globalMinioDefaultRegion, Logger: &logger{}, Notify: ¬ifier{}, @@ -62,7 +62,6 @@ func newServerConfigV15() *serverConfigV15 { // Enable console logger by default on a fresh run. srvCfg.Logger.Console = consoleLogger{ Enable: true, - Level: "error", } // Make sure to initialize notification configs. @@ -90,7 +89,7 @@ func newServerConfigV15() *serverConfigV15 { // found, otherwise use default parameters func newConfig(envParams envParams) error { // Initialize server config. - srvCfg := newServerConfigV15() + srvCfg := newServerConfigV16() // If env is set for a fresh start, save them to config file. if globalIsEnvCreds { @@ -122,12 +121,12 @@ func newConfig(envParams envParams) error { func loadConfig(envParams envParams) error { configFile := getConfigFile() - srvCfg := &serverConfigV15{} + srvCfg := &serverConfigV16{} if _, err := quick.Load(configFile, srvCfg); err != nil { return err } - if srvCfg.Version != v15 { - return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", srvCfg.Version, v15) + if srvCfg.Version != v16 { + return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", srvCfg.Version, v16) } // If env is set override the credentials from config file. @@ -206,14 +205,14 @@ func checkDupJSONKeys(json string) error { func validateConfig() error { configFile := getConfigFile() - srvCfg := &serverConfigV15{} + srvCfg := &serverConfigV16{} if _, err := quick.Load(configFile, srvCfg); err != nil { return err } // Check if config version is valid - if srvCfg.Version != v15 { - return errors.New("bad config version, expected: " + v15) + if srvCfg.Version != v16 { + return errors.New("bad config version, expected: " + v16) } // Load config file json and check for duplication json keys @@ -257,10 +256,10 @@ func validateConfig() error { } // serverConfig server config. -var serverConfig *serverConfigV15 +var serverConfig *serverConfigV16 // GetVersion get current config version. -func (s serverConfigV15) GetVersion() string { +func (s serverConfigV16) GetVersion() string { serverConfigMu.RLock() defer serverConfigMu.RUnlock() @@ -268,7 +267,7 @@ func (s serverConfigV15) GetVersion() string { } // SetRegion set new region. -func (s *serverConfigV15) SetRegion(region string) { +func (s *serverConfigV16) SetRegion(region string) { serverConfigMu.Lock() defer serverConfigMu.Unlock() @@ -280,7 +279,7 @@ func (s *serverConfigV15) SetRegion(region string) { } // GetRegion get current region. -func (s serverConfigV15) GetRegion() string { +func (s serverConfigV16) GetRegion() string { serverConfigMu.RLock() defer serverConfigMu.RUnlock() @@ -293,7 +292,7 @@ func (s serverConfigV15) GetRegion() string { } // SetCredentials set new credentials. -func (s *serverConfigV15) SetCredential(creds credential) { +func (s *serverConfigV16) SetCredential(creds credential) { serverConfigMu.Lock() defer serverConfigMu.Unlock() @@ -302,7 +301,7 @@ func (s *serverConfigV15) SetCredential(creds credential) { } // GetCredentials get current credentials. -func (s serverConfigV15) GetCredential() credential { +func (s serverConfigV16) GetCredential() credential { serverConfigMu.RLock() defer serverConfigMu.RUnlock() @@ -310,7 +309,7 @@ func (s serverConfigV15) GetCredential() credential { } // SetBrowser set if browser is enabled. -func (s *serverConfigV15) SetBrowser(v string) { +func (s *serverConfigV16) SetBrowser(v string) { serverConfigMu.Lock() defer serverConfigMu.Unlock() @@ -324,7 +323,7 @@ func (s *serverConfigV15) SetBrowser(v string) { } // GetCredentials get current credentials. -func (s serverConfigV15) GetBrowser() string { +func (s serverConfigV16) GetBrowser() string { serverConfigMu.RLock() defer serverConfigMu.RUnlock() @@ -337,7 +336,7 @@ func (s serverConfigV15) GetBrowser() string { } // Save config. -func (s serverConfigV15) Save() error { +func (s serverConfigV16) Save() error { serverConfigMu.RLock() defer serverConfigMu.RUnlock() diff --git a/cmd/config-v15_test.go b/cmd/config-v16_test.go similarity index 94% rename from cmd/config-v15_test.go rename to cmd/config-v16_test.go index 8da013256..fe6e2a5aa 100644 --- a/cmd/config-v15_test.go +++ b/cmd/config-v16_test.go @@ -113,8 +113,8 @@ func TestServerConfig(t *testing.T) { }) // Match version. - if serverConfig.GetVersion() != v15 { - t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v15) + if serverConfig.GetVersion() != v16 { + t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v16) } // Attempt to save. @@ -219,7 +219,7 @@ func TestValidateConfig(t *testing.T) { configPath := filepath.Join(rootPath, minioConfigFile) - v := v15 + v := v16 testCases := []struct { configData string @@ -255,31 +255,28 @@ func TestValidateConfig(t *testing.T) { // Test 10 - duplicated json keys {`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false}, - // Test 11 - Wrong Console logger level - {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "console": { "enable": true, "level": "foo" } }}`, false}, + // Test 11 - empty filename field in File + {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false}, - // Test 12 - Wrong File logger level - {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "level": "foo" } }}`, false}, - - // Test 13 - Test AMQP + // Test 12 - Test AMQP {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false}, - // Test 14 - Test NATS + // Test 13 - Test NATS {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false}, - // Test 15 - Test ElasticSearch + // Test 14 - Test ElasticSearch {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false}, - // Test 16 - Test Redis + // Test 15 - Test Redis {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false}, - // Test 17 - Test PostgreSQL + // Test 16 - Test PostgreSQL {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false}, - // Test 18 - Test Kafka + // Test 17 - Test Kafka {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false}, - // Test 19 - Test Webhook + // Test 18 - Test Webhook {`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false}, // Test 19 - Test MySQL diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 5a6716b3f..4a5882934 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -96,7 +96,7 @@ func newGatewayLayer(backendType, accessKey, secretKey string) (GatewayLayer, er // only used in memory. func newGatewayConfig(accessKey, secretKey, region string) error { // Initialize server config. - srvCfg := newServerConfigV15() + srvCfg := newServerConfigV16() // If env is set for a fresh start, save them to config file. srvCfg.SetCredential(credential{ @@ -105,7 +105,7 @@ func newGatewayConfig(accessKey, secretKey, region string) error { }) // Set default printing to console. - srvCfg.Logger.SetConsole(consoleLogger{true, "error"}) + srvCfg.Logger.SetConsole(consoleLogger{true}) // Set custom region. srvCfg.SetRegion(region) diff --git a/cmd/logger-console-hook.go b/cmd/logger-console-hook.go index a437ec1ad..e932938fa 100644 --- a/cmd/logger-console-hook.go +++ b/cmd/logger-console-hook.go @@ -16,24 +16,14 @@ package cmd -import ( - "fmt" - "strings" - - "github.com/Sirupsen/logrus" -) +import "github.com/Sirupsen/logrus" // consoleLogger - default logger if not other logging is enabled. type consoleLogger struct { - Enable bool `json:"enable"` - Level string `json:"level"` + Enable bool `json:"enable"` } func (c *consoleLogger) Validate() error { - level := strings.ToLower(c.Level) - if level != "error" && level != "fatal" && level != "" { - return fmt.Errorf("`%s` level value not recognized", c.Level) - } return nil } @@ -46,12 +36,7 @@ func enableConsoleLogger() { consoleLogger := logrus.New() - // log.Out and log.Formatter use the default versions. - // Only set specific log level. - lvl, err := logrus.ParseLevel(clogger.Level) - fatalIf(err, "Unknown log level found in the config file.") - - consoleLogger.Level = lvl + consoleLogger.Level = logrus.DebugLevel consoleLogger.Formatter = new(logrus.TextFormatter) log.mu.Lock() log.loggers = append(log.loggers, consoleLogger) diff --git a/cmd/logger-file-hook.go b/cmd/logger-file-hook.go index 9fbd20ea2..fa3399697 100644 --- a/cmd/logger-file-hook.go +++ b/cmd/logger-file-hook.go @@ -17,24 +17,25 @@ package cmd import ( + "errors" "fmt" "io/ioutil" "os" - "strings" "github.com/Sirupsen/logrus" ) type fileLogger struct { Enable bool `json:"enable"` - Filename string `json:"fileName"` - Level string `json:"level"` + Filename string `json:"filename"` } func (f *fileLogger) Validate() error { - level := strings.ToLower(f.Level) - if level != "error" && level != "fatal" && level != "" { - return fmt.Errorf("`%s` level value not recognized", f.Level) + if !f.Enable { + return nil + } + if f.Filename == "" { + return errors.New("Filename field empty") } return nil } @@ -58,13 +59,10 @@ func enableFileLogger() { // Add a local file hook. fileLogger.Hooks.Add(&localFile{file}) - lvl, err := logrus.ParseLevel(flogger.Level) - fatalIf(err, "Unknown log level found in the config file.") - // Set default JSON formatter. fileLogger.Out = ioutil.Discard fileLogger.Formatter = new(logrus.JSONFormatter) - fileLogger.Level = lvl // Minimum log level. + fileLogger.Level = logrus.DebugLevel // Minimum log level. log.mu.Lock() log.loggers = append(log.loggers, fileLogger)