Bring in safe mode support (#8478)

This PR refactors object layer handling such
that upon failure in sub-system initialization
server reaches a stage of safe-mode operation
wherein only certain API operations are enabled
and available.

This allows for fixing many scenarios such as

 - incorrect configuration in vault, etcd,
   notification targets
 - missing files, incomplete config migrations
   unable to read encrypted content etc
 - any other issues related to notification,
   policies, lifecycle etc

cmd/admin-handlers-config-kv.go

@@ -33,7 +33,7 @@ import (
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil
@@ -136,9 +136,16 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
+		// Config not found for some reason, allow things to continue
+		// by initializing a new fresh config in safe mode.
+		if err == errConfigNotFound && globalSafeMode {
+			cfg = newServerConfig()
+			err = nil
+		} else {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+	}
 
 	defaultKVS := configDefaultKVS()
 	oldCfg := cfg.Clone()
@@ -174,6 +181,9 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	// Make sure to write backend is encrypted
+	saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 }
 
 // GetConfigKVHandler - GET /minio/admin/v2/get-config-kv?key={key}
@@ -280,9 +290,16 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 
 	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
+		// Config not found for some reason, allow things to continue
+		// by initializing a new fresh config in safe mode.
+		if err == errConfigNotFound && globalSafeMode {
+			cfg = newServerConfig()
+			err = nil
+		} else {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+	}
 
 	defaultKVS := configDefaultKVS()
 	oldCfg := cfg.Clone()
@@ -425,6 +442,9 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Make sure to write backend is encrypted
+	saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+
 	// Reply to the client before restarting minio server.
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/admin-handlers-users.go

@@ -0,0 +1,529 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return nil
+	}
+
+	return objectAPI
+}
+
+// RemoveUser - DELETE /minio/admin/v2/remove-user?accessKey=<access_key>
+func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveUser")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to delete user.
+	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// ListUsers - GET /minio/admin/v2/list-users
+func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListUsers")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	allCredentials, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(allCredentials)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetUserInfo - GET /minio/admin/v2/user-info
+func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetUserInfo")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	name := vars["accessKey"]
+
+	userInfo, err := globalIAMSys.GetUserInfo(name)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(userInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, data)
+}
+
+// UpdateGroupMembers - PUT /minio/admin/v2/update-group-members
+func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "UpdateGroupMembers")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	defer r.Body.Close()
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	var updReq madmin.GroupAddRemove
+	err = json.Unmarshal(data, &updReq)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if updReq.IsRemove {
+		err = globalIAMSys.RemoveUsersFromGroup(updReq.Group, updReq.Members)
+	} else {
+		err = globalIAMSys.AddUsersToGroup(updReq.Group, updReq.Members)
+	}
+
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to load group.
+	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// GetGroup - /minio/admin/v2/group?group=mygroup1
+func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetGroup")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+
+	gdesc, err := globalIAMSys.GetGroupDescription(group)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(gdesc)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// ListGroups - GET /minio/admin/v2/groups
+func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListGroups")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	groups, err := globalIAMSys.ListGroups()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(groups)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SetGroupStatus - PUT /minio/admin/v2/set-group-status?group=mygroup1&status=enabled
+func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetGroupStatus")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+	status := vars["status"]
+
+	var err error
+	if status == statusEnabled {
+		err = globalIAMSys.SetGroupStatus(group, true)
+	} else if status == statusDisabled {
+		err = globalIAMSys.SetGroupStatus(group, false)
+	} else {
+		err = errInvalidArgument
+	}
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadGroup(group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// SetUserStatus - PUT /minio/admin/v2/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserStatus")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	status := vars["status"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// AddUser - PUT /minio/admin/v2/add-user?accessKey=<access_key>
+func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddUser")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var uinfo madmin.UserInfo
+	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(data)
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPolicies - GET /minio/admin/v2/list-canned-policies
+func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPolicies")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	policies, err := globalIAMSys.ListPolicies()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(policies); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// RemoveCannedPolicy - DELETE /minio/admin/v2/remove-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveCannedPolicy")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to delete policy
+	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// AddCannedPolicy - PUT /minio/admin/v2/add-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddCannedPolicy")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	// Error out if Content-Length is missing.
+	if r.ContentLength <= 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
+		return
+	}
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if iamPolicy.Version == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetPolicy(policyName, *iamPolicy); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// SetPolicyForUserOrGroup - PUT /minio/admin/v2/set-policy?policy=xxx&user-or-group=?[&is-group]
+func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
+
+	objectAPI := validateAdminUsersReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["policyName"]
+	entityName := vars["userOrGroup"]
+	isGroup := vars["isGroup"] == "true"
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.PolicyDBSet(entityName, policyName, isGroup); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}

cmd/admin-handlers.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -42,7 +41,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/handlers"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mem"
 	xnet "github.com/minio/minio/pkg/net"
@@ -94,7 +92,7 @@ func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us
 	return us, nil
 }
 
-// ServerUpdateHandler - POST /minio/admin/v1/update?updateURL={updateURL}
+// ServerUpdateHandler - POST /minio/admin/v2/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
@@ -173,7 +171,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// ServiceActionHandler - POST /minio/admin/v1/service?action={action}
+// ServiceActionHandler - POST /minio/admin/v2/service?action={action}
 // ----------
 // restarts/stops minio server gracefully. In a distributed setup,
 func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
@@ -262,7 +260,7 @@ type ServerInfo struct {
 	Data  *ServerInfoData `json:"data"`
 }
 
-// ServerInfoHandler - GET /minio/admin/v1/info
+// ServerInfoHandler - GET /minio/admin/v2/info
 // ----------
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -303,7 +301,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// ServerInfoHandler - GET /minio/admin/v1/storageinfo
+// ServerInfoHandler - GET /minio/admin/v2/storageinfo
 // ----------
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -355,7 +353,7 @@ type ServerNetReadPerfInfo struct {
 	Error          string `json:"error,omitempty"`
 }
 
-// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
+// PerfInfoHandler - GET /minio/admin/v2/performance?perfType={perfType}
 // ----------
 // Get all performance information based on input type
 // Supported types = drive
@@ -561,7 +559,7 @@ type StartProfilingResult struct {
 	Error    string `json:"error"`
 }
 
-// StartProfilingHandler - POST /minio/admin/v1/profiling/start?profilerType={profilerType}
+// StartProfilingHandler - POST /minio/admin/v2/profiling/start?profilerType={profilerType}
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
@@ -643,7 +641,7 @@ func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
 func (f dummyFileInfo) IsDir() bool        { return f.isDir }
 func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 
-// DownloadProfilingHandler - POST /minio/admin/v1/profiling/download
+// DownloadProfilingHandler - POST /minio/admin/v2/profiling/download
 // ----------
 // Download profiling information of all nodes in a zip format
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
@@ -725,7 +723,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	return
 }
 
-// HealHandler - POST /minio/admin/v1/heal/
+// HealHandler - POST /minio/admin/v2/heal/
 // -----------
 // Start heal processing and return heal status items.
 //
@@ -927,8 +925,8 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil
 	}
@@ -992,490 +990,17 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			HTTPStatusCode: e.StatusCode,
 		}
 	default:
-		apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
+		if err == errConfigNotFound {
-	}
+			apiErr = APIError{
-	return apiErr
+				Code:           "XMinioConfigError",
-}
+				Description:    err.Error(),
-
+				HTTPStatusCode: http.StatusNotFound,
-// RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey=<access_key>
-func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveUser")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
-
-	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to delete user.
-	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// ListUsers - GET /minio/admin/v1/list-users
-func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListUsers")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	allCredentials, err := globalIAMSys.ListUsers()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	data, err := json.Marshal(allCredentials)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	password := globalActiveCred.SecretKey
-	econfigData, err := madmin.EncryptData(password, data)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, econfigData)
-}
-
-// GetUserInfo - GET /minio/admin/v1/user-info
-func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetUserInfo")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	name := vars["accessKey"]
-
-	userInfo, err := globalIAMSys.GetUserInfo(name)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	data, err := json.Marshal(userInfo)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, data)
-}
-
-// UpdateGroupMembers - PUT /minio/admin/v1/update-group-members
-func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "UpdateGroupMembers")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	defer r.Body.Close()
-	data, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-		return
-	}
-
-	var updReq madmin.GroupAddRemove
-	err = json.Unmarshal(data, &updReq)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-		return
-	}
-
-	if updReq.IsRemove {
-		err = globalIAMSys.RemoveUsersFromGroup(updReq.Group, updReq.Members)
-	} else {
-		err = globalIAMSys.AddUsersToGroup(updReq.Group, updReq.Members)
-	}
-
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to load group.
-	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// GetGroup - /minio/admin/v1/group?group=mygroup1
-func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetGroup")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	group := vars["group"]
-
-	gdesc, err := globalIAMSys.GetGroupDescription(group)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	body, err := json.Marshal(gdesc)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, body)
-}
-
-// ListGroups - GET /minio/admin/v1/groups
-func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListGroups")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	groups, err := globalIAMSys.ListGroups()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	body, err := json.Marshal(groups)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, body)
-}
-
-// SetGroupStatus - PUT /minio/admin/v1/set-group-status?group=mygroup1&status=enabled
-func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetGroupStatus")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
 			}
-
-	vars := mux.Vars(r)
-	group := vars["group"]
-	status := vars["status"]
-
-	var err error
-	if status == statusEnabled {
-		err = globalIAMSys.SetGroupStatus(group, true)
-	} else if status == statusDisabled {
-		err = globalIAMSys.SetGroupStatus(group, false)
 		} else {
-		err = errInvalidArgument
+			apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
-	}
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadGroup(group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// SetUserStatus - PUT /minio/admin/v1/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
-func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetUserStatus")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
-	status := vars["status"]
-
-	// Custom IAM policies not allowed for admin user.
-	if accessKey == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-		return
-	}
-
-	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
-func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddUser")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
-
-	// Custom IAM policies not allowed for admin user.
-	if accessKey == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
-		return
-	}
-
-	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
-		// More than maxConfigSize bytes were available
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
-		return
-	}
-
-	password := globalActiveCred.SecretKey
-	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
-		return
-	}
-
-	var uinfo madmin.UserInfo
-	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
-		return
-	}
-
-	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// InfoCannedPolicy - GET /minio/admin/v1/info-canned-policy?name={policyName}
-func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoCannedPolicy")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	w.Write(data)
-	w.(http.Flusher).Flush()
-}
-
-// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
-func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListCannedPolicies")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	policies, err := globalIAMSys.ListPolicies()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if err = json.NewEncoder(w).Encode(policies); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	w.(http.Flusher).Flush()
-}
-
-// RemoveCannedPolicy - DELETE /minio/admin/v1/remove-canned-policy?name=<policy_name>
-func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveCannedPolicy")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	policyName := vars["name"]
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to delete policy
-	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
-func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddCannedPolicy")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	policyName := vars["name"]
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	// Error out if Content-Length is missing.
-	if r.ContentLength <= 0 {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
-		return
-	}
-
-	// Error out if Content-Length is beyond allowed size.
-	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
-		return
-	}
-
-	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
-		return
-	}
-
-	// Version in policy must not be empty
-	if iamPolicy.Version == "" {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
-		return
-	}
-
-	if err = globalIAMSys.SetPolicy(policyName, *iamPolicy); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to reload policy
-	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-}
-
-// SetPolicyForUserOrGroup - PUT /minio/admin/v1/set-policy?policy=xxx&user-or-group=?[&is-group]
-func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	policyName := vars["policyName"]
-	entityName := vars["userOrGroup"]
-	isGroup := vars["isGroup"] == "true"
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	if err := globalIAMSys.PolicyDBSet(entityName, policyName, isGroup); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Notify all other MinIO peers to reload policy
-	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
 		}
 	}
+	return apiErr
 }
 
 // Returns true if the trace.Info should be traced,
@@ -1495,7 +1020,7 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 	return trace
 }
 
-// TraceHandler - POST /minio/admin/v1/trace
+// TraceHandler - POST /minio/admin/v2/trace
 // ----------
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
@@ -1519,13 +1044,16 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(getRemoteHosts(globalEndpoints))
+	peers := getRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, doneCh, func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
 	})
 
 	for _, peer := range peers {
+		if peer == nil {
+			continue
+		}
 		peer.Trace(traceCh, doneCh, trcAll, trcErr)
 	}
 
@@ -1583,12 +1111,14 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	defer close(doneCh)
 	logCh := make(chan interface{}, 4000)
 
-	remoteHosts := getRemoteHosts(globalEndpoints)
+	peers := getRestClients(globalEndpoints)
-	peers := getRestClients(remoteHosts)
 
 	globalConsoleSys.Subscribe(logCh, doneCh, node, limitLines, logKind, nil)
 
 	for _, peer := range peers {
+		if peer == nil {
+			continue
+		}
 		if node == "" || strings.EqualFold(peer.host.Name, node) {
 			peer.ConsoleLog(logCh, doneCh)
 		}
@@ -1620,7 +1150,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	}
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v1/kms/key/status?key-id=<master-key-id>
+// KMSKeyStatusHandler - GET /minio/admin/v2/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatusHandler")
 
@@ -1702,7 +1232,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-// ServerHardwareInfoHandler - GET /minio/admin/v1/hardwareinfo?Type={hwType}
+// ServerHardwareInfoHandler - GET /minio/admin/v2/hardwareinfo?Type={hwType}
 // ----------
 // Get all hardware information based on input type
 // Supported types = cpu

cmd/admin-handlers_test.go

@@ -90,11 +90,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalPolicySys = NewPolicySys()
 	globalPolicySys.Init(buckets, objLayer)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
+	globalNotificationSys = NewNotificationSys(globalEndpoints)
-	if err != nil {
-		return nil, err
-	}
-
 	globalNotificationSys.Init(buckets, objLayer)
 
 	// Setup admin mgmt REST API handlers.

cmd/admin-heal-ops.go

@@ -660,7 +660,7 @@ func (h *healSequence) traverseAndHeal() {
 func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 	return func() error {
 		// Get current object layer instance.
-		objectAPI := globalObjectAPI
+		objectAPI := newObjectLayerWithoutSafeModeFn()
 		if objectAPI == nil {
 			return errServerNotInitialized
 		}
@@ -692,7 +692,7 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -712,7 +712,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -734,7 +734,7 @@ func (h *healSequence) healBuckets() error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -771,7 +771,7 @@ func (h *healSequence) healObject(bucket, object string) error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}

cmd/api-router.go

@@ -23,6 +23,31 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 )
 
+func newObjectLayerWithoutSafeModeFn() ObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+	return globalObjectAPI
+}
+
+func newObjectLayerFn() ObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+	if globalSafeMode {
+		return nil
+	}
+	return globalObjectAPI
+}
+
+func newCachedObjectLayerFn() CacheObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+
+	if globalSafeMode {
+		return nil
+	}
+	return globalCacheObjectAPI
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -37,18 +62,8 @@ type objectAPIHandlers struct {
 func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
 	// Initialize API.
 	api := objectAPIHandlers{
-		ObjectAPI: func() ObjectLayer {
+		ObjectAPI: newObjectLayerFn,
-			if !globalSafeMode {
+		CacheAPI:  newCachedObjectLayerFn,
-				return globalObjectAPI
-			}
-			return nil
-		},
-		CacheAPI: func() CacheObjectLayer {
-			if !globalSafeMode {
-				return globalCacheObjectAPI
-			}
-			return nil
-		},
 		EncryptionEnabled: func() bool {
 			return encryptionEnabled
 		},

cmd/background-heal-ops.go

@@ -107,7 +107,7 @@ func startBackgroundHealing() {
 
 	var objAPI ObjectLayer
 	for {
-		objAPI = globalObjectAPI
+		objAPI = newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue
@@ -135,7 +135,7 @@ func initBackgroundHealing() {
 // failure error occurred.
 func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
@@ -165,7 +165,7 @@ func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealRes
 // bghealBucket - traverses and heals given bucket
 func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
@@ -176,7 +176,7 @@ func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (mad
 // bgHealObject - heal the given object and record result
 func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}

cmd/background-newdisks-heal-ops.go

@@ -36,7 +36,7 @@ func monitorLocalDisksAndHeal() {
 	// Wait until the object layer is ready
 	var objAPI ObjectLayer
 	for {
-		objAPI = globalObjectAPI
+		objAPI = newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue

cmd/bucket-handlers.go

@@ -51,11 +51,8 @@ import (
 // -- If no, make an entry
 // -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
 // -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
-func initFederatorBackend(objLayer ObjectLayer) {
+func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
-	// Get buckets in the backend
+	if len(buckets) == 0 {
-	b, err := objLayer.ListBuckets(context.Background())
-	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -69,21 +66,21 @@ func initFederatorBackend(objLayer ObjectLayer) {
 	bucketSet := set.NewStringSet()
 
 	// Add buckets that are not registered with the DNS
-	g := errgroup.WithNErrs(len(b))
+	g := errgroup.WithNErrs(len(buckets))
-	for index := range b {
+	for index := range buckets {
-		bucketSet.Add(b[index].Name)
+		bucketSet.Add(buckets[index].Name)
 		index := index
 		g.Go(func() error {
-			r, gerr := globalDNSConfig.Get(b[index].Name)
+			r, gerr := globalDNSConfig.Get(buckets[index].Name)
 			if gerr != nil {
 				if gerr == dns.ErrNoEntriesFound {
-					return globalDNSConfig.Put(b[index].Name)
+					return globalDNSConfig.Put(buckets[index].Name)
 				}
 				return gerr
 			}
 			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
 				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
-				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice())
 			}
 			return nil
 		}, index)

cmd/config-current.go

@@ -48,51 +48,84 @@ func validateConfig(s config.Config) error {
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
-	// Enable env values upon return.
+	// Enable env values to validate KMS.
 	defer env.SetEnvOn()
 
 	if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
 		return err
 	}
+
 	if _, err := config.LookupRegion(s[config.RegionSubSys][config.Default]); err != nil {
 		return err
 	}
+
 	if _, err := config.LookupWorm(s[config.WormSubSys][config.Default]); err != nil {
 		return err
 	}
+
 	if globalIsXL {
 		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
 			globalXLSetDriveCount); err != nil {
 			return err
 		}
 	}
-	if _, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs); err != nil {
+
+	if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
 		return err
 	}
-	if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
+
+	if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
 		return err
 	}
-	if _, err := crypto.LookupConfig(s[config.KmsVaultSubSys][config.Default]); err != nil {
+
+	{
+		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+		if err != nil {
 			return err
 		}
-	if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
+		if etcdCfg.Enabled {
+			etcdClnt, err := etcd.New(etcdCfg)
+			if err != nil {
 				return err
 			}
+			etcdClnt.Close()
+		}
+	}
+	{
+		kmsCfg, err := crypto.LookupConfig(s[config.KmsVaultSubSys][config.Default])
+		if err != nil {
+			return err
+		}
+		if kmsCfg.Vault.Enabled {
+			// Set env to enable master key validation.
+			// this is needed only for KMS.
+			env.SetEnvOn()
+
+			if _, err = crypto.NewKMS(kmsCfg); err != nil {
+				return err
+			}
+		}
+	}
+
 	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
 		NewCustomHTTPTransport(), xhttp.DrainBody); err != nil {
 		return err
 	}
+
 	if _, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
 		globalRootCAs); err != nil {
 		return err
 	}
+
 	if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
 		NewCustomHTTPTransport(), xhttp.DrainBody); err != nil {
 		return err
 	}
+
 	if _, err := logger.LookupConfig(s); err != nil {
 		return err
 	}
+
 	return notify.TestNotificationTargets(s, GlobalServiceDoneCh, globalRootCAs)
 }
 

cmd/config-encrypted.go

@@ -19,9 +19,11 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"os"
 	"strings"
 
 	etcd "github.com/coreos/etcd/clientv3"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -97,6 +99,8 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		if err != nil {
 			return err
 		}
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
 	}
 
 	// Migrating Config backend needs a retry mechanism for
@@ -124,7 +128,8 @@ const (
 )
 
 var (
-	backendEncryptedFileValue = []byte("encrypted")
+	backendEncryptedMigrationIncomplete = []byte("incomplete")
+	backendEncryptedMigrationComplete   = []byte("encrypted")
 )
 
 func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
@@ -132,7 +137,7 @@ func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool,
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
-	return err == nil && bytes.Equal(data, backendEncryptedFileValue), nil
+	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
 }
 
 func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
@@ -140,7 +145,7 @@ func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
-	return err == nil && bytes.Equal(data, backendEncryptedFileValue), nil
+	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
 }
 
 func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
@@ -178,6 +183,9 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		if err != nil {
 			return err
 		}
+		// Once we have obtained the rotating creds
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
 	}
 
 	if encrypted {
@@ -191,18 +199,21 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		if activeCredOld.Equal(globalActiveCred) {
 			return nil
 		}
-	}
 
-	if !activeCredOld.IsValid() {
+		logger.Info("Attempting rotation of encrypted IAM users and policies on etcd with newly supplied credentials")
-		logger.Info("Attempting a one time encrypt of all IAM users and policies on etcd")
 	} else {
-		logger.Info("Attempting a rotation of encrypted IAM users and policies on etcd with newly supplied credentials")
+		logger.Info("Attempting encryption of all IAM users and policies on etcd")
 	}
 
 	r, err := client.Get(ctx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
 	}
+
+	if err = saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationIncomplete); err != nil {
+		return err
+	}
+
 	for _, kv := range r.Kvs {
 		var (
 			cdata    []byte
@@ -217,18 +228,32 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 			}
 			return err
 		}
+
+		var data []byte
 		// Is rotating of creds requested?
 		if activeCredOld.IsValid() {
-			cdata, err = madmin.DecryptData(activeCredOld.String(), bytes.NewReader(cdata))
+			data, err = madmin.DecryptData(activeCredOld.String(), bytes.NewReader(cdata))
 			if err != nil {
 				if err == madmin.ErrMaliciousData {
+					data, err = madmin.DecryptData(globalActiveCred.String(),
+						bytes.NewReader(cdata))
+					if err != nil {
 						return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
 					}
+				} else {
 					return err
 				}
 			}
+		}
 
-		cencdata, err = madmin.EncryptData(globalActiveCred.String(), cdata)
+		// Attempt to unmarshal JSON content
+		var dummy map[string]interface{}
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err = json.Unmarshal(data, &dummy); err != nil {
+			return err
+		}
+
+		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
 		if err != nil {
 			return err
 		}
@@ -237,7 +262,10 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 			return err
 		}
 	}
-	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedFileValue)
+	if encrypted && globalActiveCred.IsValid() {
+		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+	}
+	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
 }
 
 func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Credentials, encrypted bool) error {
@@ -252,12 +280,14 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		if activeCredOld.Equal(globalActiveCred) {
 			return nil
 		}
+		logger.Info("Attempting rotation of encrypted config, IAM users and policies on MinIO with newly supplied credentials")
+	} else {
+		logger.Info("Attempting encryption of all config, IAM users and policies on MinIO backend")
 	}
 
-	if !activeCredOld.IsValid() {
+	err := saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
-		logger.Info("Attempting a one time encrypt of all config, IAM users and policies on MinIO backend")
+	if err != nil {
-	} else {
+		return err
-		logger.Info("Attempting a rotation of encrypted config, IAM users and policies on MinIO with newly supplied credentials")
 	}
 
 	var marker string
@@ -277,18 +307,31 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 				return err
 			}
 
+			var data []byte
 			// Is rotating of creds requested?
 			if activeCredOld.IsValid() {
-				cdata, err = madmin.DecryptData(activeCredOld.String(), bytes.NewReader(cdata))
+				data, err = madmin.DecryptData(activeCredOld.String(), bytes.NewReader(cdata))
 				if err != nil {
 					if err == madmin.ErrMaliciousData {
+						data, err = madmin.DecryptData(globalActiveCred.String(),
+							bytes.NewReader(cdata))
+						if err != nil {
 							return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
 						}
+					} else {
 						return err
 					}
 				}
+			}
+
+			// Attempt to unmarshal JSON content
+			var dummy map[string]interface{}
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
+			if err = json.Unmarshal(data, &dummy); err != nil {
+				return err
+			}
 
-			cencdata, err = madmin.EncryptData(globalActiveCred.String(), cdata)
+			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
 			if err != nil {
 				return err
 			}
@@ -305,5 +348,9 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		marker = res.NextMarker
 	}
 
-	return saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedFileValue)
+	if encrypted && globalActiveCred.IsValid() {
+		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+	}
+
+	return saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 }

cmd/config/config.go

@@ -386,11 +386,6 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 		return Error(fmt.Sprintf("invalid number of arguments %s", s))
 	}
 
-	if subSystemValue[0] == CredentialsSubSys {
-		return Error(fmt.Sprintf("changing '%s' sub-system values is not allowed, use ENVs instead",
-			subSystemValue[0]))
-	}
-
 	if !SubSystems.Contains(subSystemValue[0]) {
 		return Error(fmt.Sprintf("unknown sub-system %s", s))
 	}

cmd/config/etcd/etcd.go

@@ -31,7 +31,7 @@ import (
 
 const (
 	// Default values used while communicating with etcd.
-	defaultDialTimeout   = 30 * time.Second
+	defaultDialTimeout   = 5 * time.Second
 	defaultDialKeepAlive = 30 * time.Second
 )
 
@@ -112,7 +112,7 @@ func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
 		return cfg, config.Error("'endpoints' key cannot be empty if you wish to enable etcd")
 	}
 
-	if len(endpoints) == 0 {
+	if len(endpoints) == 0 && !stateBool {
 		return cfg, nil
 	}
 

cmd/crypto/config.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"errors"
 	"fmt"
+	"reflect"
 	"strconv"
 
 	"github.com/minio/minio/cmd/config"
@@ -112,6 +113,12 @@ const (
 	EnvKMSVaultNamespace = "MINIO_KMS_VAULT_NAMESPACE"
 )
 
+var defaultCfg = VaultConfig{
+	Auth: VaultAuth{
+		Type: "approle",
+	},
+}
+
 // LookupConfig extracts the KMS configuration provided by environment
 // variables and merge them with the provided KMS configuration. The
 // merging follows the following rules:
@@ -139,7 +146,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 			return kmsCfg, err
 		}
 	}
-	if !kmsCfg.Vault.IsEmpty() {
+	if kmsCfg.Vault.Enabled {
 		return kmsCfg, nil
 	}
 	stateBool, err := config.ParseBool(env.Get(EnvKMSVaultState, kvs.Get(config.State)))
@@ -166,23 +173,30 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 	vcfg.Endpoint = endpointStr
 	vcfg.CAPath = env.Get(EnvKMSVaultCAPath, kvs.Get(KMSVaultCAPath))
 	vcfg.Auth.Type = env.Get(EnvKMSVaultAuthType, kvs.Get(KMSVaultAuthType))
+	if vcfg.Auth.Type == "" {
+		vcfg.Auth.Type = "approle"
+	}
 	vcfg.Auth.AppRole.ID = env.Get(EnvKMSVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
 	vcfg.Auth.AppRole.Secret = env.Get(EnvKMSVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
 	vcfg.Key.Name = env.Get(EnvKMSVaultKeyName, kvs.Get(KMSVaultKeyName))
 	vcfg.Namespace = env.Get(EnvKMSVaultNamespace, kvs.Get(KMSVaultNamespace))
-	keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion))
+	if keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)); keyVersion != "" {
-
-	if keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
 			return kmsCfg, fmt.Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
 		}
 	}
 
+	if reflect.DeepEqual(vcfg, defaultCfg) {
+		return kmsCfg, nil
+	}
+
+	// Verify all the proper settings.
 	if err = vcfg.Verify(); err != nil {
 		return kmsCfg, err
 	}
 
+	vcfg.Enabled = true
 	kmsCfg.Vault = vcfg
 	return kmsCfg, nil
 }
@@ -191,7 +205,7 @@ func LookupConfig(kvs config.KVS) (KMSConfig, error) {
 func NewKMS(cfg KMSConfig) (kms KMS, err error) {
 	// Lookup KMS master keys - only available through ENV.
 	if masterKeyLegacy := env.Get(EnvKMSMasterKeyLegacy, ""); len(masterKeyLegacy) != 0 {
-		if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
+		if cfg.Vault.Enabled { // Vault and KMS master key provided
 			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
 		}
 		kms, err = ParseMasterKey(masterKeyLegacy)
@@ -199,15 +213,14 @@ func NewKMS(cfg KMSConfig) (kms KMS, err error) {
 			return kms, err
 		}
 	} else if masterKey := env.Get(EnvKMSMasterKey, ""); len(masterKey) != 0 {
-		if !cfg.Vault.IsEmpty() { // Vault and KMS master key provided
+		if cfg.Vault.Enabled { // Vault and KMS master key provided
 			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
 		}
 		kms, err = ParseMasterKey(masterKey)
 		if err != nil {
 			return kms, err
 		}
-	}
+	} else if cfg.Vault.Enabled {
-	if !cfg.Vault.IsEmpty() {
 		kms, err = NewVault(cfg.Vault)
 		if err != nil {
 			return kms, err

cmd/crypto/legacy.go

@@ -18,6 +18,7 @@ package crypto
 
 import (
 	"fmt"
+	"reflect"
 	"strconv"
 
 	"github.com/minio/minio/cmd/config"
@@ -40,40 +41,40 @@ const (
 )
 
 const (
-	// EnvVaultEndpoint is the environment variable used to specify
+	// EnvLegacyVaultEndpoint is the environment variable used to specify
 	// the vault HTTPS endpoint.
-	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
+	EnvLegacyVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
 
-	// EnvVaultAuthType is the environment variable used to specify
+	// EnvLegacyVaultAuthType is the environment variable used to specify
 	// the authentication type for vault.
-	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
+	EnvLegacyVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
 
-	// EnvVaultAppRoleID is the environment variable used to specify
+	// EnvLegacyVaultAppRoleID is the environment variable used to specify
 	// the vault AppRole ID.
-	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
+	EnvLegacyVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
 
-	// EnvVaultAppSecretID is the environment variable used to specify
+	// EnvLegacyVaultAppSecretID is the environment variable used to specify
 	// the vault AppRole secret corresponding to the AppRole ID.
-	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
+	EnvLegacyVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
 
-	// EnvVaultKeyVersion is the environment variable used to specify
+	// EnvLegacyVaultKeyVersion is the environment variable used to specify
 	// the vault key version.
-	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
+	EnvLegacyVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
 
-	// EnvVaultKeyName is the environment variable used to specify
+	// EnvLegacyVaultKeyName is the environment variable used to specify
 	// the vault named key-ring. In the S3 context it's referred as
 	// customer master key ID (CMK-ID).
-	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
+	EnvLegacyVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
 
-	// EnvVaultCAPath is the environment variable used to specify the
+	// EnvLegacyVaultCAPath is the environment variable used to specify the
 	// path to a directory of PEM-encoded CA cert files. These CA cert
 	// files are used to authenticate MinIO to Vault over mTLS.
-	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+	EnvLegacyVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
 
-	// EnvVaultNamespace is the environment variable used to specify
+	// EnvLegacyVaultNamespace is the environment variable used to specify
 	// vault namespace. The vault namespace is used if the enterprise
 	// version of Hashicorp Vault is used.
-	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
+	EnvLegacyVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
 )
 
 // SetKMSConfig helper to migrate from older KMSConfig to new KV.
@@ -93,7 +94,7 @@ func SetKMSConfig(s config.Config, cfg KMSConfig) {
 		KMSVaultKeyVersion:    strconv.Itoa(cfg.Vault.Key.Version),
 		KMSVaultNamespace:     cfg.Vault.Namespace,
 		config.State: func() string {
-			if !cfg.Vault.IsEmpty() {
+			if cfg.Vault.Endpoint != "" {
 				return config.StateOn
 			}
 			return config.StateOff
@@ -141,7 +142,7 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
 		return cfg, nil
 	}
 
-	endpointStr := env.Get(EnvKMSVaultEndpoint, kvs.Get(KMSVaultEndpoint))
+	endpointStr := env.Get(EnvLegacyVaultEndpoint, "")
 	if endpointStr != "" {
 		// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
 		endpoint, err := xnet.ParseHTTPURL(endpointStr)
@@ -152,25 +153,31 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
 	}
 
 	cfg.Vault.Endpoint = endpointStr
-	cfg.Vault.CAPath = env.Get(EnvVaultCAPath, kvs.Get(KMSVaultCAPath))
+	cfg.Vault.CAPath = env.Get(EnvLegacyVaultCAPath, "")
-	cfg.Vault.Auth.Type = env.Get(EnvVaultAuthType, kvs.Get(KMSVaultAuthType))
+	cfg.Vault.Auth.Type = env.Get(EnvLegacyVaultAuthType, "")
-	cfg.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, kvs.Get(KMSVaultAppRoleID))
+	if cfg.Vault.Auth.Type == "" {
-	cfg.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, kvs.Get(KMSVaultAppRoleSecret))
+		cfg.Vault.Auth.Type = "approle"
-	cfg.Vault.Key.Name = env.Get(EnvVaultKeyName, kvs.Get(KMSVaultKeyName))
+	}
-	cfg.Vault.Namespace = env.Get(EnvVaultNamespace, kvs.Get(KMSVaultNamespace))
+	cfg.Vault.Auth.AppRole.ID = env.Get(EnvLegacyVaultAppRoleID, "")
-	keyVersion := env.Get(EnvVaultKeyVersion, kvs.Get(KMSVaultKeyVersion))
+	cfg.Vault.Auth.AppRole.Secret = env.Get(EnvLegacyVaultAppSecretID, "")
-
+	cfg.Vault.Key.Name = env.Get(EnvLegacyVaultKeyName, "")
-	if keyVersion != "" {
+	cfg.Vault.Namespace = env.Get(EnvLegacyVaultNamespace, "")
+	if keyVersion := env.Get(EnvLegacyVaultKeyVersion, ""); keyVersion != "" {
 		cfg.Vault.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
 			return cfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
-				EnvVaultKeyVersion, keyVersion)
+				EnvLegacyVaultKeyVersion, keyVersion)
+		}
 	}
+
+	if reflect.DeepEqual(cfg.Vault, defaultCfg) {
+		return cfg, nil
 	}
 
 	if err = cfg.Vault.Verify(); err != nil {
 		return cfg, err
 	}
 
+	cfg.Vault.Enabled = true
 	return cfg, nil
 }

cmd/crypto/vault.go

@@ -51,6 +51,7 @@ type VaultAppRole struct {
 
 // VaultConfig represents vault configuration.
 type VaultConfig struct {
+	Enabled   bool      `json:"-"`
 	Endpoint  string    `json:"endpoint"` // The vault API endpoint as URL
 	CAPath    string    `json:"-"`        // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
 	Auth      VaultAuth `json:"auth"`     // The vault authentication configuration
@@ -68,24 +69,10 @@ type vaultService struct {
 
 var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
 
-// empty/default vault configuration used to check whether a particular is empty.
-var emptyVaultConfig = VaultConfig{
-	Auth: VaultAuth{
-		Type: "approle",
-	},
-}
-
-// IsEmpty returns true if the vault config struct is an
-// empty configuration.
-func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
-
 // Verify returns a nil error if the vault configuration
 // is valid. A valid configuration is either empty or
 // contains valid non-default values.
 func (v *VaultConfig) Verify() (err error) {
-	if v.IsEmpty() {
-		return // an empty configuration is valid
-	}
 	switch {
 	case v.Endpoint == "":
 		err = errors.New("crypto: missing hashicorp vault endpoint")
@@ -107,8 +94,8 @@ func (v *VaultConfig) Verify() (err error) {
 // to Vault with the credentials in config and gets a client
 // token for future api calls.
 func NewVault(config VaultConfig) (KMS, error) {
-	if config.IsEmpty() {
+	if !config.Enabled {
-		return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
+		return nil, nil
 	}
 	if err := config.Verify(); err != nil {
 		return nil, err

cmd/crypto/vault_test.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -23,17 +22,17 @@ var verifyVaultConfigTests = []struct {
 	Config     VaultConfig
 	ShouldFail bool
 }{
-	{
-		ShouldFail: false, // 0
-		Config:     emptyVaultConfig,
-	},
 	{
 		ShouldFail: true,
-		Config:     VaultConfig{Endpoint: "https://127.0.0.1:8080"},
+		Config: VaultConfig{
+			Endpoint: "https://127.0.0.1:8080",
+			Enabled:  true,
+		},
 	},
 	{
 		ShouldFail: true, // 1
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth:     VaultAuth{Type: "unsupported"},
 		},
@@ -41,6 +40,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 2
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -51,6 +51,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 3
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -61,6 +62,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 4
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -71,6 +73,7 @@ var verifyVaultConfigTests = []struct {
 	{
 		ShouldFail: true, // 5
 		Config: VaultConfig{
+			Enabled:  true,
 			Endpoint: "https://127.0.0.1:8080",
 			Auth: VaultAuth{
 				Type:    "approle",
@@ -82,9 +85,9 @@ var verifyVaultConfigTests = []struct {
 }
 
 func TestVerifyVaultConfig(t *testing.T) {
-	for i, test := range verifyVaultConfigTests {
+	for _, test := range verifyVaultConfigTests {
 		test := test
-		t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) {
+		t.Run(test.Config.Endpoint, func(t *testing.T) {
 			err := test.Config.Verify()
 			if test.ShouldFail && err == nil {
 				t.Errorf("Verify should fail but returned 'err == nil'")

cmd/daily-lifecycle-ops.go

@@ -56,7 +56,7 @@ func startDailyLifecycle() {
 
 	// Wait until the object API is ready
 	for {
-		objAPI = globalObjectAPI
+		objAPI = newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue

cmd/disk-cache.go

@@ -553,23 +553,23 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		migrating: migrateSw,
 		migMutex:  sync.Mutex{},
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-			return globalObjectAPI.GetObjectInfo(ctx, bucket, object, opts)
+			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
 		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-			return globalObjectAPI.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
-			return globalObjectAPI.DeleteObject(ctx, bucket, object)
+			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
 		},
 		DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
 			errs := make([]error, len(objects))
 			for idx, object := range objects {
-				errs[idx] = globalObjectAPI.DeleteObject(ctx, bucket, object)
+				errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
 			}
 			return errs, nil
 		},
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-			return globalObjectAPI.PutObject(ctx, bucket, object, data, opts)
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
 	}
 	if migrateSw {

cmd/gateway-main.go

@@ -222,9 +222,22 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	// Re-enable logging
+	logger.Disable = false
+
+	// Once endpoints are finalized, initialize the new object api in safe mode.
+	globalObjLayerMutex.Lock()
+	globalSafeMode = true
+	globalObjectAPI = newObject
+	globalObjLayerMutex.Unlock()
+
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
-		initFederatorBackend(newObject)
+		buckets, err := newObject.ListBuckets(context.Background())
+		if err != nil {
+			logger.Fatal(err, "Unable to list buckets")
+		}
+		initFederatorBackend(buckets, newObject)
 	}
 
 	// Migrate all backend configs to encrypted backend, also handles rotation as well.
@@ -233,16 +246,15 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	logger.FatalIf(handleEncryptedConfigBackend(newObject, enableConfigOps),
 		"Unable to handle encrypted backend for config, iam and policies")
 
+	// Calls all New() for all sub-systems.
+	newAllSubsystems()
+
 	// ****  WARNING ****
 	// Migrating to encrypted backend should happen before initialization of any
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
 
 	if enableConfigOps {
-		// Create a new config system.
+		logger.FatalIf(globalConfigSys.Init(newObject), "Unable to initialize config system")
-		globalConfigSys = NewConfigSys()
-
-		// Load globalServerConfig from disk
-		logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
 
 		// Start watching disk for reloading config, this
 		// is only enabled for "NAS" gateway.
@@ -261,32 +273,20 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 			"Unable to handle encrypted backend for iam and policies")
 	}
 
-	if globalCacheConfig.Enabled {
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
-	// Re-enable logging
-	logger.Disable = false
-
-	// Create new IAM system.
-	globalIAMSys = NewIAMSys()
 	if enableIAMOps {
 		// Initialize IAM sys.
-		logger.LogIf(context.Background(), globalIAMSys.Init(newObject))
+		logger.FatalIf(globalIAMSys.Init(newObject), "Unable to initialize IAM system")
 	}
 
-	// Create new policy system.
+	if globalCacheConfig.Enabled {
-	globalPolicySys = NewPolicySys()
+		// initialize the new disk cache objects.
-
+		var cacheAPI CacheObjectLayer
-	// Create new lifecycle system.
+		cacheAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
-	globalLifecycleSys = NewLifecycleSys()
+		logger.FatalIf(err, "Unable to initialize disk caching")
 
-	// Create new notification system.
+		globalObjLayerMutex.Lock()
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
+		globalCacheObjectAPI = cacheAPI
-	if err != nil {
+		globalObjLayerMutex.Unlock()
-		logger.FatalIf(err, "Unable to initialize notification system")
 	}
 
 	// Verify if object layer supports
@@ -294,11 +294,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// - compression
 	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
-	// Once endpoints are finalized, initialize the new object api.
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = newObject
-	globalObjLayerMutex.Unlock()
-
 	// Prints the formatted startup message once object layer is initialized.
 	if !globalCLIContext.Quiet {
 		mode := globalMinioModeGatewayPrefix + gatewayName
@@ -314,6 +309,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
 	}
 
+	// Disable safe mode operation, after all initialization is over.
+	globalObjLayerMutex.Lock()
+	globalSafeMode = false
+	globalObjLayerMutex.Unlock()
+
 	// Set uptime time after object layer has initialized.
 	globalBootTime = UTCNow()
 

cmd/gateway-startup-msg.go

@@ -28,8 +28,9 @@ import (
 func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
 	// If cache layer is enabled, print cache capacity.
-	if globalCacheObjectAPI != nil {
+	cacheAPI := newCachedObjectLayerFn()
-		printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background()))
+	if cacheAPI != nil {
+		printCacheStorageInfo(cacheAPI.StorageInfo(context.Background()))
 	}
 	// Prints credential.
 	printGatewayCommonMsg(strippedAPIEndpoints)

cmd/global-heal.go

@@ -156,7 +156,7 @@ func execLeaderTasks(sets *xlSets) {
 func startGlobalHeal() {
 	var objAPI ObjectLayer
 	for {
-		objAPI = globalObjectAPI
+		objAPI = newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue

cmd/healthcheck-handler.go

@@ -52,9 +52,9 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "LivenessCheckHandler")
 
-	objLayer := globalObjectAPI
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
-	if objLayer == nil || globalSafeMode {
+	if objLayer == nil {
 		// Respond with 200 OK while server initializes to ensure a distributed cluster
 		// is able to start on orchestration platforms like Docker Swarm.
 		// Refer https://github.com/minio/minio/issues/8140 for more details.

cmd/iam-object-store.go

@@ -49,7 +49,7 @@ func (iamOS *IAMObjectStore) getObjectAPI() ObjectLayer {
 	if iamOS.objAPI != nil {
 		return iamOS.objAPI
 	}
-	return globalObjectAPI
+	return newObjectLayerWithoutSafeModeFn()
 }
 
 func (iamOS *IAMObjectStore) setObjectAPI(objAPI ObjectLayer) {

cmd/iam.go

@@ -406,7 +406,7 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
 func (sys *IAMSys) DeletePolicy(policyName string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -431,7 +431,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 // InfoPolicy - expands the canned policy into its JSON structure.
 func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return nil, errServerNotInitialized
 	}
@@ -448,7 +448,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
 
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return nil, errServerNotInitialized
 	}
@@ -471,7 +471,7 @@ func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
 
 // SetPolicy - sets a new name policy.
 func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -492,7 +492,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 
 // DeleteUser - delete user (only for long-term users not STS users).
 func (sys *IAMSys) DeleteUser(accessKey string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -521,7 +521,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
 func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -561,7 +561,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return nil, errServerNotInitialized
 	}
@@ -587,7 +587,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return u, errServerNotInitialized
 	}
@@ -617,7 +617,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -653,7 +653,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -685,7 +685,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 // SetUserSecretKey - sets user secret key
 func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -724,7 +724,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -782,7 +782,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
 func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -863,7 +863,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 // SetGroupStatus - enable/disabled a group
 func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -952,7 +952,7 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 // PolicyDB. This function applies only long-term users. For STS
 // users, policy is set directly by called sys.policyDBSet().
 func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -1007,7 +1007,7 @@ func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
 		return nil, errInvalidArgument
 	}
 
-	objectAPI := globalObjectAPI
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return nil, errServerNotInitialized
 	}

cmd/lifecycle.go

@@ -59,7 +59,7 @@ func (sys *LifecycleSys) Get(bucketName string) (lifecycle lifecycle.Lifecycle,
 	if globalIsGateway {
 		// When gateway is enabled, no cached value
 		// is used to validate life cycle policies.
-		objAPI := globalObjectAPI
+		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			return
 		}

cmd/metrics.go

@@ -83,9 +83,9 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 	minioVersionInfo.WithLabelValues(Version, CommitID).Set(float64(1.0))
 
 	// Fetch disk space info
-	objLayer := globalObjectAPI
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
-	if objLayer == nil || globalSafeMode {
+	if objLayer == nil {
 		return
 	}
 

cmd/notification.go

@@ -628,6 +628,20 @@ func (sys *NotificationSys) ListenBucketNotification(ctx context.Context, bucket
 	}()
 }
 
+// AddNotificationTargetsFromConfig - adds notification targets from server config.
+func (sys *NotificationSys) AddNotificationTargetsFromConfig(cfg config.Config) error {
+	targetList, err := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, globalRootCAs)
+	if err != nil {
+		return err
+	}
+	for _, target := range targetList.Targets() {
+		if err = sys.targetList.Add(target); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // AddRemoteTarget - adds event rules map, HTTP/PeerRPC client target to bucket name.
 func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Target, rulesMap event.RulesMap) error {
 	if err := sys.targetList.Add(target); err != nil {
@@ -684,7 +698,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 
 	// Construct path to listener.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
-	transactionConfigFile := configFile + ".transaction"
+	transactionConfigFile := configFile + "/transaction.lock"
 
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
@@ -1098,22 +1112,14 @@ func (sys *NotificationSys) NetworkInfo() []madmin.ServerNetworkHardwareInfo {
 }
 
 // NewNotificationSys - creates new notification system object.
-func NewNotificationSys(cfg config.Config, endpoints EndpointList) (*NotificationSys, error) {
+func NewNotificationSys(endpoints EndpointList) *NotificationSys {
-	targetList, err := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, globalRootCAs)
-	if err != nil {
-		return nil, config.Errorf("Unable to start notification sub system: %s", err)
-	}
-
-	remoteHosts := getRemoteHosts(endpoints)
-	remoteClients := getRestClients(remoteHosts)
-
 	// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
 	return &NotificationSys{
-		targetList:                 targetList,
+		targetList:                 event.NewTargetList(),
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
-		peerClients:                remoteClients,
+		peerClients:                getRestClients(endpoints),
-	}, nil
+	}
 }
 
 type eventArgs struct {
@@ -1264,7 +1270,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 
 	// Construct path to listener.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
-	transactionConfigFile := configFile + ".transaction"
+	transactionConfigFile := configFile + "/transaction.lock"
 
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
@@ -1315,7 +1321,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 
 	// Construct path to listener.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketListenerConfig)
-	transactionConfigFile := configFile + ".transaction"
+	transactionConfigFile := configFile + "/transaction.lock"
 
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()

cmd/peer-rest-client.go

@@ -693,19 +693,24 @@ func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
 	var remoteHosts []*xnet.Host
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse peer Host")
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			continue
+		}
 		remoteHosts = append(remoteHosts, host)
 	}
 
 	return remoteHosts
 }
 
-func getRestClients(peerHosts []*xnet.Host) []*peerRESTClient {
+func getRestClients(endpoints EndpointList) []*peerRESTClient {
+	peerHosts := getRemoteHosts(endpoints)
 	restClients := make([]*peerRESTClient, len(peerHosts))
 	for i, host := range peerHosts {
 		client, err := newPeerRESTClient(host)
 		if err != nil {
 			logger.LogIf(context.Background(), err)
+			continue
 		}
 		restClients[i] = client
 	}

cmd/peer-rest-server.go

@@ -42,12 +42,8 @@ type peerRESTServer struct {
 }
 
 func getServerInfo() (*ServerInfoData, error) {
-	if globalBootTime.IsZero() {
+	objLayer := newObjectLayerWithoutSafeModeFn()
-		return nil, errServerNotInitialized
+	if objLayer == nil {
-	}
-
-	objLayer := globalObjectAPI
-	if objLayer == nil || globalSafeMode {
 		return nil, errServerNotInitialized
 	}
 
@@ -166,7 +162,7 @@ func (s *peerRESTServer) DeletePolicyHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -194,7 +190,7 @@ func (s *peerRESTServer) LoadPolicyHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -222,7 +218,7 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -251,7 +247,7 @@ func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -279,7 +275,7 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -329,7 +325,7 @@ func (s *peerRESTServer) LoadGroupHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -539,7 +535,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerWithoutSafeModeFn()
 	if objAPI == nil {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return

cmd/policy.go

@@ -69,7 +69,7 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 	if globalIsGateway {
 		// When gateway is enabled, no cached value
 		// is used to validate bucket policies.
-		objAPI := globalObjectAPI
+		objAPI := newObjectLayerFn()
 		if objAPI != nil {
 			config, err := objAPI.GetBucketPolicy(context.Background(), args.BucketName)
 			if err == nil {

cmd/server-main.go

@@ -180,53 +180,99 @@ func serverHandleEnvVars() {
 	handleCommonEnvVars()
 }
 
-func initAllSubsystems(newObject ObjectLayer) {
+func newAllSubsystems() {
+	// Create new notification system and initialize notification targets
+	globalNotificationSys = NewNotificationSys(globalEndpoints)
+
 	// Create a new config system.
 	globalConfigSys = NewConfigSys()
 
-	// Initialize config system.
-	if err := globalConfigSys.Init(newObject); err != nil {
-		logger.Fatal(err, "Unable to initialize config system")
-	}
-
 	// Create new IAM system.
 	globalIAMSys = NewIAMSys()
 
-	if err := globalIAMSys.Init(newObject); err != nil {
+	// Create new policy system.
-		logger.Fatal(err, "Unable to initialize IAM system")
+	globalPolicySys = NewPolicySys()
-	}
 
-	buckets, err := newObject.ListBuckets(context.Background())
+	// Create new lifecycle system.
+	globalLifecycleSys = NewLifecycleSys()
+}
+
+func initSafeModeInit(buckets []BucketInfo) (err error) {
+	defer func() {
 		if err != nil {
-		logger.Fatal(err, "Unable to list buckets")
+			// Enable logger
+			logger.Disable = false
+
+			// Prints the formatted startup message in safe mode operation.
+			printStartupSafeModeMessage(getAPIEndpoints(), err)
+
+			// Initialization returned error reaching safe mode and
+			// not proceeding waiting for admin action.
+			handleSignals()
 		}
+	}()
 
-	// Create new notification system and initialize notification targets
+	newObject := newObjectLayerWithoutSafeModeFn()
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
+
-	if err != nil {
+	// Calls New() for all sub-systems.
-		logger.Fatal(err, "Unable to initialize notification targets")
+	newAllSubsystems()
+
+	// Migrate all backend configs to encrypted backend, also handles rotation as well.
+	if err = handleEncryptedConfigBackend(newObject, true); err != nil {
+		return fmt.Errorf("Unable to handle encrypted backend for config, iam and policies: %v", err)
+	}
+
+	// ****  WARNING ****
+	// Migrating to encrypted backend should happen before initialization of any
+	// sub-systems, make sure that we do not move the above codeblock elsewhere.
+
+	// Validate and initialize all subsystems.
+	if err = initAllSubsystems(buckets, newObject); err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		// ****  WARNING ****
+		// Migrating to encrypted backend on etcd should happen before initialization of
+		// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
+		if err = migrateIAMConfigsEtcdToEncrypted(globalEtcdClient); err != nil {
+			return fmt.Errorf("Unable to handle encrypted backend for iam and policies: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error) {
+	// Initialize config system.
+	if err = globalConfigSys.Init(newObject); err != nil {
+		return fmt.Errorf("Unable to initialize config system: %v", err)
+	}
+
+	if err = globalNotificationSys.AddNotificationTargetsFromConfig(globalServerConfig); err != nil {
+		return fmt.Errorf("Unable to initialize notification target(s) from config: %v", err)
+	}
+
+	if err = globalIAMSys.Init(newObject); err != nil {
+		return fmt.Errorf("Unable to initialize IAM system: %v", err)
 	}
 
 	// Initialize notification system.
 	if err = globalNotificationSys.Init(buckets, newObject); err != nil {
-		logger.Fatal(err, "Unable to initialize notification system")
+		return fmt.Errorf("Unable to initialize notification system: %v", err)
 	}
 
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	// Initialize policy system.
 	if err = globalPolicySys.Init(buckets, newObject); err != nil {
-		logger.Fatal(err, "Unable to initialize policy system")
+		return fmt.Errorf("Unable to initialize policy system; %v", err)
 	}
 
-	// Create new lifecycle system.
-	globalLifecycleSys = NewLifecycleSys()
-
 	// Initialize lifecycle system.
 	if err = globalLifecycleSys.Init(buckets, newObject); err != nil {
-		logger.Fatal(err, "Unable to initialize lifecycle system")
+		return fmt.Errorf("Unable to initialize lifecycle system: %v", err)
 	}
+
+	return nil
 }
 
 // serverMain handler called for 'minio server' command.
@@ -327,42 +373,33 @@ func serverMain(ctx *cli.Context) {
 		globalTLSCerts.Stop()
 
 		globalHTTPServer.Shutdown()
-		logger.FatalIf(err, "Unable to initialize backend")
+		logger.Fatal(err, "Unable to initialize backend")
-	}
-
-	// Populate existing buckets to the etcd backend
-	if globalDNSConfig != nil {
-		initFederatorBackend(newObject)
 	}
 
 	// Re-enable logging
 	logger.Disable = false
 
-	// Migrate all backend configs to encrypted backend, also handles rotation as well.
+	// Once endpoints are finalized, initialize the new object api in safe mode.
-	logger.FatalIf(handleEncryptedConfigBackend(newObject, true),
+	globalObjLayerMutex.Lock()
-		"Unable to handle encrypted backend for config, iam and policies")
+	globalSafeMode = true
-
+	globalObjectAPI = newObject
-	// ****  WARNING ****
+	globalObjLayerMutex.Unlock()
-	// Migrating to encrypted backend should happen before initialization of any
-	// sub-systems, make sure that we do not move the above codeblock elsewhere.
 
-	// Validate and initialize all subsystems.
+	buckets, err := newObject.ListBuckets(context.Background())
-	initAllSubsystems(newObject)
+	if err != nil {
+		logger.Fatal(err, "Unable to list buckets")
+	}
 
-	if globalEtcdClient != nil {
+	// Populate existing buckets to the etcd backend
-		// ****  WARNING ****
+	if globalDNSConfig != nil {
-		// Migrating to encrypted backend on etcd should happen before initialization of
+		initFederatorBackend(buckets, newObject)
-		// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
-		logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(globalEtcdClient),
-			"Unable to handle encrypted backend for iam and policies")
 	}
 
-	if globalCacheConfig.Enabled {
+	initSafeModeInit(buckets)
-		logger.StartupMessage(color.Red(color.Bold("Disk caching is recommended only for gateway deployments")))
 
-		// initialize the new disk cache objects.
+	if globalCacheConfig.Enabled {
-		globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), globalCacheConfig)
+		msg := color.RedBold("Disk caching is disabled in 'server' mode, 'caching' is only supported in gateway deployments")
-		logger.FatalIf(err, "Unable to initialize disk caching")
+		logger.StartupMessage(msg)
 	}
 
 	initDailyLifecycle()
@@ -373,14 +410,17 @@ func serverMain(ctx *cli.Context) {
 		initGlobalHeal()
 	}
 
-	globalObjectAPI = newObject
+	// Disable safe mode operation, after all initialization is over.
+	globalObjLayerMutex.Lock()
+	globalSafeMode = false
+	globalObjLayerMutex.Unlock()
 
 	// Prints the formatted startup message once object layer is initialized.
 	printStartupMessage(getAPIEndpoints())
 
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred)
-		logger.StartupMessage(color.Red(color.Bold(msg)))
+		logger.StartupMessage(color.RedBold(msg))
 	}
 
 	// Set uptime time after object layer has initialized.

cmd/server-startup-msg.go

@@ -32,6 +32,7 @@ import (
 // Documentation links, these are part of message printing code.
 const (
 	mcQuickStartGuide      = "https://docs.min.io/docs/minio-client-quickstart-guide"
+	mcAdminQuickStartGuide = "https://docs.min.io/docs/minio-admin-complete-guide.html"
 	goQuickStartGuide      = "https://docs.min.io/docs/golang-client-quickstart-guide"
 	jsQuickStartGuide      = "https://docs.min.io/docs/javascript-client-quickstart-guide"
 	javaQuickStartGuide    = "https://docs.min.io/docs/java-client-quickstart-guide"
@@ -45,16 +46,75 @@ func getFormatStr(strLen int, padding int) string {
 	return "%" + formatStr
 }
 
+func printStartupSafeModeMessage(apiEndpoints []string, err error) {
+	logStartupMessage(color.RedBold("Server startup failed with '%v'", err))
+	logStartupMessage(color.RedBold("Server switching to safe mode"))
+	logStartupMessage(color.RedBold("Please use 'mc admin' commands to fix this issue"))
+
+	// Object layer is initialized then print StorageInfo in safe mode.
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI != nil {
+		if msg := getStorageInfoMsgSafeMode(objAPI.StorageInfo(context.Background())); msg != "" {
+			logStartupMessage(msg)
+		}
+	}
+
+	// Get saved credentials.
+	cred := globalActiveCred
+
+	// Get saved region.
+	region := globalServerRegion
+
+	strippedAPIEndpoints := stripStandardPorts(apiEndpoints)
+
+	apiEndpointStr := strings.Join(strippedAPIEndpoints, "  ")
+
+	// Colorize the message and print.
+	logStartupMessage(color.Red("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	if color.IsTerminal() && !globalCLIContext.Anonymous {
+		logStartupMessage(color.Red("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(color.Red("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		if region != "" {
+			logStartupMessage(color.Red("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+		}
+	}
+
+	// Prints `mc` cli configuration message chooses
+	// first endpoint as default.
+	alias := "myminio"
+	endPoint := strippedAPIEndpoints[0]
+
+	// Configure 'mc', following block prints platform specific information for minio client admin commands.
+	if color.IsTerminal() {
+		logStartupMessage(color.RedBold("\nCommand-line Access: ") + mcAdminQuickStartGuide)
+		if runtime.GOOS == globalWindowsOSName {
+			mcMessage := fmt.Sprintf("> mc.exe config host add %s %s %s %s --api s3v4", alias,
+				endPoint, cred.AccessKey, cred.SecretKey)
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			mcMessage = fmt.Sprintf("> mc.exe admin --help")
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		} else {
+			mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s --api s3v4", alias,
+				endPoint, cred.AccessKey, cred.SecretKey)
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			mcMessage = fmt.Sprintf("$ mc admin --help")
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		}
+	}
+}
+
 // Prints the formatted startup message.
-func printStartupMessage(apiEndPoints []string) {
+func printStartupMessage(apiEndpoints []string) {
 
-	strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
+	strippedAPIEndpoints := stripStandardPorts(apiEndpoints)
 	// If cache layer is enabled, print cache capacity.
-	if globalCacheObjectAPI != nil {
+	cachedObjAPI := newCachedObjectLayerFn()
-		printCacheStorageInfo(globalCacheObjectAPI.StorageInfo(context.Background()))
+	if cachedObjAPI != nil {
+		printCacheStorageInfo(cachedObjAPI.StorageInfo(context.Background()))
 	}
+
 	// Object layer is initialized then print StorageInfo.
-	objAPI := globalObjectAPI
+	objAPI := newObjectLayerFn()
 	if objAPI != nil {
 		printStorageInfo(objAPI.StorageInfo(context.Background()))
 	}
@@ -183,6 +243,16 @@ func printObjectAPIMsg() {
 	logStartupMessage(color.Blue("   .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide))
 }
 
+// Get formatted disk/storage info message.
+func getStorageInfoMsgSafeMode(storageInfo StorageInfo) string {
+	var msg string
+	if storageInfo.Backend.Type == BackendErasure {
+		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum())
+		msg += color.Red("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
+	}
+	return msg
+}
+
 // Get formatted disk/storage info message.
 func getStorageInfoMsg(storageInfo StorageInfo) string {
 	var msg string

cmd/signals.go

@@ -55,7 +55,7 @@ func handleSignals() {
 		// send signal to various go-routines that they need to quit.
 		close(GlobalServiceDoneCh)
 
-		if objAPI := globalObjectAPI; objAPI != nil {
+		if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 			oerr = objAPI.Shutdown(context.Background())
 			logger.LogIf(context.Background(), oerr)
 		}
@@ -66,7 +66,7 @@ func handleSignals() {
 	for {
 		select {
 		case err := <-globalHTTPServerErrorCh:
-			if objAPI := globalObjectAPI; objAPI != nil {
+			if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 				objAPI.Shutdown(context.Background())
 			}
 			if err != nil {

cmd/test-utils_test.go

@@ -370,11 +370,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalPolicySys = NewPolicySys()
 	globalPolicySys.Init(buckets, objLayer)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks)
+	globalNotificationSys = NewNotificationSys(testServer.Disks)
-	if err != nil {
-		t.Fatalf("Unable to initialize notification system %s", err)
-	}
-
 	globalNotificationSys.Init(buckets, objLayer)
 
 	globalLifecycleSys = NewLifecycleSys()
@@ -1640,10 +1636,10 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 	globalIAMSys.Init(xl)
 
 	globalPolicySys = NewPolicySys()
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints)
+	globalPolicySys.Init(nil, xl)
-	if err != nil {
+
-		return xl, err
+	globalNotificationSys = NewNotificationSys(endpoints)
-	}
+	globalNotificationSys.Init(nil, xl)
 
 	return xl, nil
 }

cmd/web-router.go

@@ -62,18 +62,8 @@ const specialAssets = "index_bundle.*.js|loader.css|logo.svg|firefox.png|safari.
 func registerWebRouter(router *mux.Router) error {
 	// Initialize Web.
 	web := &webAPIHandlers{
-		ObjectAPI: func() ObjectLayer {
+		ObjectAPI: newObjectLayerFn,
-			if !globalSafeMode {
+		CacheAPI:  newCachedObjectLayerFn,
-				return globalObjectAPI
-			}
-			return nil
-		},
-		CacheAPI: func() CacheObjectLayer {
-			if !globalSafeMode {
-				return globalCacheObjectAPI
-			}
-			return nil
-		},
 	}
 
 	// Initialize a new json2 codec.

cmd/xl-v1-multipart.go

@@ -839,12 +839,12 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
 	if err != nil {
-		return toObjectErr(err, bucket, object)
+		return toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Cleanup all uploaded parts.
 	if err = xl.deleteObject(ctx, minioMetaMultipartBucket, uploadIDPath, writeQuorum, false); err != nil {
-		return toObjectErr(err, bucket, object)
+		return toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Successfully purged.

cmd/xl-v1-utils.go

@@ -18,11 +18,11 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"hash/crc32"
 	"path"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -117,7 +117,6 @@ func hashOrder(key string, cardinality int) []int {
 
 // Constructs xlMetaV1 using `jsoniter` lib.
 func xlMetaV1UnmarshalJSON(ctx context.Context, xlMetaBuf []byte) (xlMeta xlMetaV1, err error) {
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err = json.Unmarshal(xlMetaBuf, &xlMeta)
 	return xlMeta, err
 }

docs/config/README.md

@@ -54,7 +54,9 @@ export MINIO_SECRET_KEY_OLD=minio123
 minio server /data
 ```
 
-Once the migration is complete and server has started successfully remove `MINIO_ACCESS_KEY_OLD` and `MINIO_SECRET_KEY_OLD` environment variables, restart the server.
+Once the migration is complete, server will automatically unset the `MINIO_ACCESS_KEY_OLD` and `MINIO_SECRET_KEY_OLD` with in the process namespace.
+
+> **NOTE: Make sure to remove `MINIO_ACCESS_KEY_OLD` and `MINIO_SECRET_KEY_OLD` in scripts or service files before next service restarts of the server to avoid double encryption of your existing contents.**
 
 #### Region
 | Field                     | Type     | Description                                                                                                                                                                             |

pkg/color/color.go

@@ -37,6 +37,13 @@ var (
 		return fmt.Sprint
 	}()
 
+	RedBold = func() func(format string, a ...interface{}) string {
+		if IsTerminal() {
+			return color.New(color.FgRed, color.Bold).SprintfFunc()
+		}
+		return fmt.Sprintf
+	}()
+
 	Red = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgRed).SprintfFunc()

pkg/event/targetlist.go

@@ -36,15 +36,17 @@ type TargetList struct {
 }
 
 // Add - adds unique target to target list.
-func (list *TargetList) Add(target Target) error {
+func (list *TargetList) Add(targets ...Target) error {
 	list.Lock()
 	defer list.Unlock()
 
+	for _, target := range targets {
 		if _, ok := list.targets[target.ID()]; ok {
 			return fmt.Errorf("target %v already exists", target.ID())
 		}
-
 		list.targets[target.ID()] = target
+	}
+
 	return nil
 }
 
@@ -102,6 +104,19 @@ func (list *TargetList) Remove(targetids ...TargetID) <-chan TargetIDErr {
 	return errCh
 }
 
+// Targets - list all targets
+func (list *TargetList) Targets() []Target {
+	list.RLock()
+	defer list.RUnlock()
+
+	targets := []Target{}
+	for _, tgt := range list.targets {
+		targets = append(targets, tgt)
+	}
+
+	return targets
+}
+
 // List - returns available target IDs.
 func (list *TargetList) List() []TargetID {
 	list.RLock()