From 24d9d7e5fa5c93328e1aa7e84c4fb587c8303366 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Wed, 17 Jan 2018 09:24:46 -0600 Subject: [PATCH] Removes logrus package and refactors logging messages (#5293) This fix removes logrus package dependency and refactors the console logging as the only logging mechanism by removing file logging support. It rearranges the log message format and adds stack trace information whenever trace information is not available in the error structure. It also adds `--json` flag support for server logging. When minio server is started with `--json` flag, all log messages are displayed in json format, with no start-up and informational log messages. Fixes #5265 #5220 #5197 --- cmd/config-migrate.go | 8 -- cmd/config-versions.go | 15 +++ cmd/console-logger.go | 74 ------------ cmd/file-logger.go | 86 -------------- cmd/gateway-main.go | 7 ++ cmd/globals.go | 1 + cmd/logger.go | 223 +++++++++++++++++++++---------------- cmd/logger_test.go | 31 ------ cmd/main.go | 4 + cmd/namespace-lock.go | 17 +++ cmd/namespace-lock_test.go | 16 +++ cmd/prepare-storage-msg.go | 7 +- cmd/prepare-storage.go | 9 +- cmd/server-main.go | 7 ++ cmd/test-utils_test.go | 3 - 15 files changed, 201 insertions(+), 307 deletions(-) delete mode 100644 cmd/console-logger.go delete mode 100644 cmd/file-logger.go delete mode 100644 cmd/logger_test.go diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 36bc17ba9..583db2838 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -1147,14 +1147,6 @@ func migrateV15ToV16() error { // Load browser config from existing config in the file. srvConfig.Browser = cv15.Browser - // Migrate console and file fields - if cv15.Logger.Console.Enable { - srvConfig.Logger.Console = NewConsoleLogger() - } - if cv15.Logger.File.Enable { - srvConfig.Logger.File = NewFileLogger(cv15.Logger.File.Filename) - } - if err = quick.Save(configFile, srvConfig); err != nil { return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err) } diff --git a/cmd/config-versions.go b/cmd/config-versions.go index 6fa551dcb..58c9da2ac 100644 --- a/cmd/config-versions.go +++ b/cmd/config-versions.go @@ -418,6 +418,21 @@ type serverConfigV15 struct { Notify *notifier `json:"notify"` } +// FileLogger is introduced to workaround the dependency about logrus +type FileLogger struct { + Enable bool `json:"enable"` + Filename string `json:"filename"` +} + +// ConsoleLogger is introduced to workaround the dependency about logrus +type ConsoleLogger struct { + Enable bool `json:"enable"` +} + +// Loggers struct is defined with FileLogger and ConsoleLogger +// although they are removed from logging logic. They are +// kept here just to workaround the dependency migration +// code/logic has on them. type loggers struct { sync.RWMutex Console ConsoleLogger `json:"console"` diff --git a/cmd/console-logger.go b/cmd/console-logger.go deleted file mode 100644 index bb17b96da..000000000 --- a/cmd/console-logger.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "fmt" - "os" - - "github.com/Sirupsen/logrus" -) - -// ConsoleLogger - console logger which logs into stderr. -type ConsoleLogger struct { - BaseLogTarget -} - -// Fire - log entry handler. -func (logger ConsoleLogger) Fire(entry *logrus.Entry) error { - if !logger.Enable { - return nil - } - - msgBytes, err := logger.formatter.Format(entry) - if err == nil { - fmt.Fprintf(os.Stderr, string(msgBytes)) - } - - return err -} - -// String - represents ConsoleLogger as string. -func (logger ConsoleLogger) String() string { - enableStr := "disabled" - if logger.Enable { - enableStr = "enabled" - } - - return fmt.Sprintf("console:%s", enableStr) -} - -// NewConsoleLogger - return new console logger object. -func NewConsoleLogger() (logger ConsoleLogger) { - logger.Enable = true - logger.formatter = new(logrus.TextFormatter) - - return logger -} - -// InitConsoleLogger - initializes console logger. -func InitConsoleLogger(logger *ConsoleLogger) { - if !logger.Enable { - return - } - - if logger.formatter == nil { - logger.formatter = new(logrus.TextFormatter) - } - - return -} diff --git a/cmd/file-logger.go b/cmd/file-logger.go deleted file mode 100644 index fcf0e5b26..000000000 --- a/cmd/file-logger.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "fmt" - "os" - - "github.com/Sirupsen/logrus" -) - -// FileLogger - file logger which logs to a file. -type FileLogger struct { - BaseLogTarget - Filename string `json:"filename"` - file *os.File -} - -// Fire - log entry handler. -func (logger FileLogger) Fire(entry *logrus.Entry) (err error) { - if !logger.Enable { - return nil - } - - msgBytes, err := logger.formatter.Format(entry) - if err != nil { - return err - } - - if _, err = logger.file.Write(msgBytes); err != nil { - return err - } - - err = logger.file.Sync() - return err -} - -// String - represents ConsoleLogger as string. -func (logger FileLogger) String() string { - enableStr := "disabled" - if logger.Enable { - enableStr = "enabled" - } - - return fmt.Sprintf("file:%s:%s", enableStr, logger.Filename) -} - -// NewFileLogger - creates new file logger object. -func NewFileLogger(filename string) (logger FileLogger) { - logger.Enable = true - logger.formatter = new(logrus.JSONFormatter) - logger.Filename = filename - - return logger -} - -// InitFileLogger - initializes file logger. -func InitFileLogger(logger *FileLogger) (err error) { - if !logger.Enable { - return err - } - - if logger.formatter == nil { - logger.formatter = new(logrus.JSONFormatter) - } - - if logger.file == nil { - logger.file, err = os.OpenFile(logger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664) - } - - return err -} diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 42a8e0d4c..619754b8e 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -112,6 +112,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) { cli.ShowCommandHelpAndExit(ctx, gatewayName, 1) } + // Get "json" flag from command line argument and + // enable json and quite modes if jason flag is turned on. + jsonFlag := ctx.Bool("json") || ctx.GlobalBool("json") + if jsonFlag { + log.EnableJSON() + } + // Get quiet flag from command line argument. quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet") if quietFlag { diff --git a/cmd/globals.go b/cmd/globals.go index 4e0b842e7..353bf743e 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -174,6 +174,7 @@ var ( colorBold = color.New(color.Bold).SprintFunc() colorBlue = color.New(color.FgBlue).SprintfFunc() colorYellow = color.New(color.FgYellow).SprintfFunc() + colorRed = color.New(color.FgRed).SprintfFunc() ) // Returns minio global information, as a key value map. diff --git a/cmd/logger.go b/cmd/logger.go index 04d3f0162..94ca487f1 100644 --- a/cmd/logger.go +++ b/cmd/logger.go @@ -17,78 +17,68 @@ package cmd import ( + "encoding/json" "fmt" - "io/ioutil" - "path" + "os" + "path/filepath" "runtime" "strings" + "time" - "github.com/Sirupsen/logrus" "github.com/minio/mc/pkg/console" "github.com/minio/minio/pkg/errors" ) var log = NewLogger() +var trimStrings []string -// LogTarget - interface for log target. -type LogTarget interface { - Fire(entry *logrus.Entry) error - String() string -} +// Level type +type Level int8 -// BaseLogTarget - base log target. -type BaseLogTarget struct { - Enable bool `json:"enable"` - formatter logrus.Formatter -} +// Enumerated level types +const ( + Error Level = iota + 1 + Fatal +) -// Logger - higher level logger. -type Logger struct { - logger *logrus.Logger - consoleTarget ConsoleLogger - targets []LogTarget - quiet bool +func (level Level) String() string { + var lvlStr string + switch level { + case Error: + lvlStr = "ERROR" + case Fatal: + lvlStr = "FATAL" + } + return lvlStr } -// AddTarget - add logger to this hook. -func (log *Logger) AddTarget(logTarget LogTarget) { - log.targets = append(log.targets, logTarget) +type logEntry struct { + Level string `json:"level"` + Message string `json:"message"` + Time string `json:"time"` + Cause string `json:"cause"` + Trace []string `json:"trace"` } -// SetConsoleTarget - sets console target to this hook. -func (log *Logger) SetConsoleTarget(consoleTarget ConsoleLogger) { - log.consoleTarget = consoleTarget +// Logger - for console messages +type Logger struct { + quiet bool + json bool } -// Fire - log entry handler to save logs. -func (log *Logger) Fire(entry *logrus.Entry) (err error) { - if err = log.consoleTarget.Fire(entry); err != nil { - log.Printf("Unable to log to console target. %s\n", err) - } - - for _, logTarget := range log.targets { - if err = logTarget.Fire(entry); err != nil { - log.Printf("Unable to log to target %s. %s\n", logTarget, err) - } - } - - return err +// NewLogger - to create a new Logger object +func NewLogger() *Logger { + return &Logger{} } -// Levels - returns list of log levels support. -func (log *Logger) Levels() []logrus.Level { - return []logrus.Level{ - logrus.PanicLevel, - logrus.FatalLevel, - logrus.ErrorLevel, - logrus.WarnLevel, - logrus.InfoLevel, - logrus.DebugLevel, - } +// EnableQuiet - turns quiet option on. +func (log *Logger) EnableQuiet() { + log.quiet = true } -// EnableQuiet - sets quiet option. -func (log *Logger) EnableQuiet() { +// EnableJSON - outputs logs in json format. +func (log *Logger) EnableJSON() { + log.json = true log.quiet = true } @@ -106,38 +96,66 @@ func (log *Logger) Printf(format string, args ...interface{}) { } } -// NewLogger - returns a new initialized logger. -func NewLogger() *Logger { - logger := logrus.New() - logger.Out = ioutil.Discard - logger.Level = logrus.DebugLevel - - l := &Logger{ - logger: logger, - consoleTarget: NewConsoleLogger(), +func init() { + var goPathList []string + // Add all possible GOPATH paths into trimStrings + // Split GOPATH depending on the OS type + if runtime.GOOS == "windows" { + goPathList = strings.Split(GOPATH, ";") + } else { + // All other types of OSs + goPathList = strings.Split(GOPATH, ":") } - // Adds a console logger. - logger.Hooks.Add(l) + // Add trim string "{GOROOT}/src/" into trimStrings + trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)} - return l + // Add all possible path from GOPATH=path1:path2...:pathN + // as "{path#}/src/" into trimStrings + for _, goPathString := range goPathList { + trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator)) + } + // Add "github.com/minio/minio" as the last to cover + // paths like "{GOROOT}/src/github.com/minio/minio" + // and "{GOPATH}/src/github.com/minio/minio" + trimStrings = append(trimStrings, filepath.Join("github.com", "minio", "minio")+string(filepath.Separator)) } -func getSource() string { - var funcName string - pc, filename, lineNum, ok := runtime.Caller(2) - if ok { - filename = path.Base(filename) - funcName = strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "github.com/minio/minio/cmd.") - } else { - filename = "" - lineNum = 0 +func trimTrace(f string) string { + for _, trimString := range trimStrings { + f = strings.TrimPrefix(filepath.ToSlash(f), filepath.ToSlash(trimString)) } + return filepath.FromSlash(f) +} - return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName) +// getTrace method - creates and returns stack trace +func getTrace(traceLevel int) []string { + var trace []string + pc, file, lineNumber, ok := runtime.Caller(traceLevel) + + for ok { + // Clean up the common prefixes + file = trimTrace(file) + // Get the function name + _, funcName := filepath.Split(runtime.FuncForPC(pc).Name()) + // Skip duplicate traces that start with file name, "" + // and also skip traces with function name that starts with "runtime." + if !strings.HasPrefix(file, "") && + !strings.HasPrefix(funcName, "runtime.") { + // Form and append a line of stack trace into a + // collection, 'trace', to build full stack trace + trace = append(trace, fmt.Sprintf("%v:%v:%v()", file, lineNumber, funcName)) + } + traceLevel++ + // Read stack trace information from PC + pc, file, lineNumber, ok = runtime.Caller(traceLevel) + } + return trace } -func logIf(level logrus.Level, source string, err error, msg string, data ...interface{}) { +func logIf(level Level, err error, msg string, + data ...interface{}) { + isErrIgnored := func(err error) (ok bool) { err = errors.Cause(err) switch err.(type) { @@ -154,36 +172,51 @@ func logIf(level logrus.Level, source string, err error, msg string, data ...int if err == nil || isErrIgnored(err) { return } + cause := strings.Title(err.Error()) + // Get full stack trace + trace := getTrace(3) + // Get time + timeOfError := UTCNow().Format(time.RFC3339Nano) + // Output the formatted log message at console + var output string + message := fmt.Sprintf(msg, data...) + if log.json { + logJSON, err := json.Marshal(&logEntry{ + Level: level.String(), + Message: message, + Time: timeOfError, + Cause: cause, + Trace: trace, + }) + if err != nil { + panic("json marshal of logEntry failed: " + err.Error()) + } + output = string(logJSON) + } else { + // Add a sequence number and formatting for each stack trace + // No formatting is required for the first entry + trace[0] = "1: " + trace[0] + for i, element := range trace[1:] { + trace[i+1] = fmt.Sprintf("%8v: %s", i+2, element) + } + errMsg := fmt.Sprintf("[%s] [%s] %s (%s)", + timeOfError, level.String(), message, cause) - fields := logrus.Fields{ - "source": source, - "cause": err.Error(), - } - - if terr, ok := err.(*errors.Error); ok { - fields["stack"] = strings.Join(terr.Stack(), " ") + output = fmt.Sprintf("\nTrace: %s\n%s", + strings.Join(trace, "\n"), + colorRed(colorBold(errMsg))) } + fmt.Println(output) - switch level { - case logrus.PanicLevel: - log.logger.WithFields(fields).Panicf(msg, data...) - case logrus.FatalLevel: - log.logger.WithFields(fields).Fatalf(msg, data...) - case logrus.ErrorLevel: - log.logger.WithFields(fields).Errorf(msg, data...) - case logrus.WarnLevel: - log.logger.WithFields(fields).Warnf(msg, data...) - case logrus.InfoLevel: - log.logger.WithFields(fields).Infof(msg, data...) - default: - log.logger.WithFields(fields).Debugf(msg, data...) + if level == Fatal { + os.Exit(1) } } func errorIf(err error, msg string, data ...interface{}) { - logIf(logrus.ErrorLevel, getSource(), err, msg, data...) + logIf(Error, err, msg, data...) } func fatalIf(err error, msg string, data ...interface{}) { - logIf(logrus.FatalLevel, getSource(), err, msg, data...) + logIf(Fatal, err, msg, data...) } diff --git a/cmd/logger_test.go b/cmd/logger_test.go deleted file mode 100644 index 12e431678..000000000 --- a/cmd/logger_test.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Minio Cloud Storage (C) 2015, 2016, 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "testing" -) - -// Tests getSource(). -func TestGetSource(t *testing.T) { - currentSource := func() string { return getSource() } - gotSource := currentSource() - expectedSource := "[logger_test.go:26:TestGetSource()]" - if gotSource != expectedSource { - t.Errorf("expected : %s, got : %s", expectedSource, gotSource) - } -} diff --git a/cmd/main.go b/cmd/main.go index 66d9223d2..5863f4f41 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,6 +38,10 @@ var globalFlags = []cli.Flag{ Name: "quiet", Usage: "Disable startup information.", }, + cli.BoolFlag{ + Name: "json", + Usage: "Output server logs and startup information in json format.", + }, } // Help template for minio. diff --git a/cmd/namespace-lock.go b/cmd/namespace-lock.go index 4c9a5dcb4..9d7f58369 100644 --- a/cmd/namespace-lock.go +++ b/cmd/namespace-lock.go @@ -19,6 +19,8 @@ package cmd import ( "errors" pathutil "path" + "runtime" + "strings" "sync" "fmt" @@ -366,3 +368,18 @@ func (li *lockInstance) RUnlock() { readLock := true li.ns.unlock(li.volume, li.path, li.opsID, readLock) } + +func getSource() string { + var funcName string + pc, filename, lineNum, ok := runtime.Caller(2) + if ok { + filename = pathutil.Base(filename) + funcName = strings.TrimPrefix(runtime.FuncForPC(pc).Name(), + "github.com/minio/minio/cmd.") + } else { + filename = "" + lineNum = 0 + } + + return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName) +} diff --git a/cmd/namespace-lock_test.go b/cmd/namespace-lock_test.go index 463a310ea..2e9a95865 100644 --- a/cmd/namespace-lock_test.go +++ b/cmd/namespace-lock_test.go @@ -21,6 +21,22 @@ import ( "time" ) +// WARNING: +// +// Expected source line number is hard coded, 32, in the +// following test. Adding new code before this test or changing its +// position will cause the line number to change and the test to FAIL +// Tests getSource(). +func TestGetSource(t *testing.T) { + currentSource := func() string { return getSource() } + gotSource := currentSource() + // Hard coded line number, 32, in the "expectedSource" value + expectedSource := "[namespace-lock_test.go:32:TestGetSource()]" + if gotSource != expectedSource { + t.Errorf("expected : %s, got : %s", expectedSource, gotSource) + } +} + // Tests functionality provided by namespace lock. func TestNamespaceLockTest(t *testing.T) { // List of test cases. diff --git a/cmd/prepare-storage-msg.go b/cmd/prepare-storage-msg.go index faa3b1ee1..7acf28f22 100644 --- a/cmd/prepare-storage-msg.go +++ b/cmd/prepare-storage-msg.go @@ -21,7 +21,6 @@ import ( "sync" humanize "github.com/dustin/go-humanize" - "github.com/minio/mc/pkg/console" ) // Helper to generate integer sequences into a friendlier user consumable format. @@ -44,7 +43,7 @@ func printOnceFn() printOnceFunc { var once sync.Once return func(msg string) { once.Do(func() { - console.Println(msg) + log.Println(msg) }) } } @@ -91,7 +90,7 @@ func getHealMsg(endpoints EndpointList, storageDisks []StorageAPI) string { // Prints regular message when we have sufficient disks to start the cluster. func printRegularMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) { - msg := getStorageInitMsg("\nInitializing data volume.", endpoints, storageDisks) + msg := getStorageInitMsg("Initializing data volume.", endpoints, storageDisks) fn(msg) } @@ -121,7 +120,7 @@ func getStorageInitMsg(titleMsg string, endpoints EndpointList, storageDisks []S // Prints initialization message when cluster is being initialized for the first time. func printFormatMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) { - msg := getStorageInitMsg("\nInitializing data volume for the first time.", endpoints, storageDisks) + msg := getStorageInitMsg("Initializing data volume for the first time.", endpoints, storageDisks) fn(msg) } diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 3745e763b..02d4f3780 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/minio/mc/pkg/console" "github.com/minio/minio/pkg/errors" ) @@ -289,11 +288,9 @@ func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks case Abort: return reduceInitXLErrs(storageDisks, sErrs) case FormatDisks: - console.Eraseline() printFormatMsg(endpoints, storageDisks, printOnceFn()) return initFormatXL(storageDisks) case InitObjectLayer: - console.Eraseline() // Validate formats loaded before proceeding forward. err := genericFormatCheckXL(formatConfigs, sErrs) if err == nil { @@ -308,7 +305,7 @@ func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks } return err case WaitForQuorum: - console.Printf( + log.Printf( "Initializing data volume. Waiting for minimum %d servers to come online. (elapsed %s)\n", len(storageDisks)/2+1, getElapsedTime(), ) @@ -316,9 +313,9 @@ func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks // Print configuration errors. return reduceInitXLErrs(storageDisks, sErrs) case WaitForAll: - console.Printf("Initializing data volume for first time. Waiting for other servers to come online (elapsed %s)\n", getElapsedTime()) + log.Printf("Initializing data volume for first time. Waiting for other servers to come online (elapsed %s)\n", getElapsedTime()) case WaitForFormatting: - console.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime()) + log.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime()) } case <-globalServiceDoneCh: return fmt.Errorf("Initializing data volumes gracefully stopped") diff --git a/cmd/server-main.go b/cmd/server-main.go index a69414c58..d59a1b2aa 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -132,6 +132,13 @@ func serverMain(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "server", 1) } + // Get "json" flag from command line argument and + // enable json and quite modes if jason flag is turned on. + jsonFlag := ctx.Bool("json") || ctx.GlobalBool("json") + if jsonFlag { + log.EnableJSON() + } + // Get quiet flag from command line argument. quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet") if quietFlag { diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 4a97f2cb1..9830324e9 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -71,9 +71,6 @@ func init() { // Set system resources to maximum. setMaxResources() - - // Quiet logging. - log.logger.Hooks = nil } // concurreny level for certain parallel tests.