fix: make single locks for both IAM and object-store (#9279)

Additionally add context support for IAM sub-system

cmd/admin-handlers_test.go

@@ -70,7 +70,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(GlobalContext, objLayer)
 
 	buckets, err := objLayer.ListBuckets(context.Background())
 	if err != nil {

cmd/config-encrypted.go

@@ -163,8 +163,8 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)

cmd/gateway-main.go

@@ -255,13 +255,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// ****  WARNING ****
 		// Migrating to encrypted backend on etcd should happen before initialization of
 		// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
-		logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(globalEtcdClient),
+		logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(GlobalContext, globalEtcdClient),
 			"Unable to handle encrypted backend for iam and policies")
 	}
 
 	if enableIAMOps {
 		// Initialize IAM sys.
-		logger.FatalIf(globalIAMSys.Init(newObject), "Unable to initialize IAM system")
+		logger.FatalIf(globalIAMSys.Init(GlobalContext, newObject), "Unable to initialize IAM system")
 	}
 
 	if globalCacheConfig.Enabled {

cmd/iam-etcd-store.go

@@ -74,37 +74,30 @@ func etcdKvsToSetPolicyDB(prefix string, kvs []*mvccpb.KeyValue) set.StringSet {
 // IAMEtcdStore implements IAMStorageAPI
 type IAMEtcdStore struct {
 	sync.RWMutex
+
 	ctx context.Context
 
 	client *etcd.Client
 }
 
-func newIAMEtcdStore() *IAMEtcdStore {
-	return &IAMEtcdStore{client: globalEtcdClient}
-}
-
-func (ies *IAMEtcdStore) getContext() context.Context {
-	ies.RLock()
-	defer ies.RUnlock()
-
-	if ies.ctx == nil {
-		return context.Background()
-	}
-	return ies.ctx
+func newIAMEtcdStore(ctx context.Context) *IAMEtcdStore {
+	return &IAMEtcdStore{client: globalEtcdClient, ctx: ctx}
 }
 
-func (ies *IAMEtcdStore) setContext(ctx context.Context) {
+func (ies *IAMEtcdStore) lock() {
 	ies.Lock()
-	defer ies.Unlock()
+}
 
-	ies.ctx = ctx
+func (ies *IAMEtcdStore) unlock() {
+	ies.Unlock()
 }
 
-func (ies *IAMEtcdStore) clearContext() {
-	ies.Lock()
-	defer ies.Unlock()
+func (ies *IAMEtcdStore) rlock() {
+	ies.RLock()
+}
 
-	ies.ctx = nil
+func (ies *IAMEtcdStore) runlock() {
+	ies.RUnlock()
 }
 
 func (ies *IAMEtcdStore) saveIAMConfig(item interface{}, path string) error {
@@ -118,11 +111,11 @@ func (ies *IAMEtcdStore) saveIAMConfig(item interface{}, path string) error {
 			return err
 		}
 	}
-	return saveKeyEtcd(ies.getContext(), ies.client, path, data)
+	return saveKeyEtcd(ies.ctx, ies.client, path, data)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfig(item interface{}, path string) error {
-	pdata, err := readKeyEtcd(ies.getContext(), ies.client, path)
+	pdata, err := readKeyEtcd(ies.ctx, ies.client, path)
 	if err != nil {
 		return err
 	}
@@ -138,19 +131,17 @@ func (ies *IAMEtcdStore) loadIAMConfig(item interface{}, path string) error {
 }
 
 func (ies *IAMEtcdStore) deleteIAMConfig(path string) error {
-	return deleteKeyEtcd(ies.getContext(), ies.client, path)
+	return deleteKeyEtcd(ies.ctx, ies.client, path)
 }
 
-func (ies *IAMEtcdStore) migrateUsersConfigToV1(isSTS bool) error {
+func (ies *IAMEtcdStore) migrateUsersConfigToV1(ctx context.Context, isSTS bool) error {
 	basePrefix := iamConfigUsersPrefix
 	if isSTS {
 		basePrefix = iamConfigSTSPrefix
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
-	ies.setContext(ctx)
-	defer ies.clearContext()
 	r, err := ies.client.Get(ctx, basePrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
@@ -216,7 +207,7 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(isSTS bool) error {
 		cred.AccessKey = user
 		u := newUserIdentity(cred)
 		if err := ies.saveIAMConfig(u, identityPath); err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(ctx, err)
 			return err
 		}
 
@@ -226,7 +217,7 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(isSTS bool) error {
 	return nil
 }
 
-func (ies *IAMEtcdStore) migrateToV1() error {
+func (ies *IAMEtcdStore) migrateToV1(ctx context.Context) error {
 	var iamFmt iamFormat
 	path := getIAMFormatFilePath()
 	if err := ies.loadIAMConfig(&iamFmt, path); err != nil {
@@ -248,29 +239,26 @@ func (ies *IAMEtcdStore) migrateToV1() error {
 	}
 
 	// Migrate long-term users
-	if err := ies.migrateUsersConfigToV1(false); err != nil {
-		logger.LogIf(context.Background(), err)
+	if err := ies.migrateUsersConfigToV1(ctx, false); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
 	// Migrate STS users
-	if err := ies.migrateUsersConfigToV1(true); err != nil {
-		logger.LogIf(context.Background(), err)
+	if err := ies.migrateUsersConfigToV1(ctx, true); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
 	// Save iam version file.
 	if err := ies.saveIAMConfig(newIAMFormatVersion1(), path); err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		return err
 	}
 	return nil
 }
 
 // Should be called under config migration lock
-func (ies *IAMEtcdStore) migrateBackendFormat(objAPI ObjectLayer) error {
-	if err := ies.migrateToV1(); err != nil {
-		return err
-	}
-	return nil
+func (ies *IAMEtcdStore) migrateBackendFormat(ctx context.Context) error {
+	return ies.migrateToV1(ctx)
 }
 
 func (ies *IAMEtcdStore) loadPolicyDoc(policy string, m map[string]iampolicy.Policy) error {
@@ -283,11 +271,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(policy string, m map[string]iampolicy.Pol
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadPolicyDocs(m map[string]iampolicy.Policy) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
-	ies.setContext(ctx)
-	defer ies.clearContext()
 	r, err := ies.client.Get(ctx, iamConfigPoliciesPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
@@ -317,9 +303,8 @@ func (ies *IAMEtcdStore) loadUser(user string, userType IAMUserType, m map[strin
 
 	if u.Credentials.IsExpired() {
 		// Delete expired identity.
-		ctx := ies.getContext()
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+		deleteKeyEtcd(ies.ctx, ies.client, getUserIdentityPath(user, userType))
+		deleteKeyEtcd(ies.ctx, ies.client, getMappedPolicyPath(user, userType, false))
 		return nil
 	}
 
@@ -351,7 +336,7 @@ func (ies *IAMEtcdStore) loadUser(user string, userType IAMUserType, m map[strin
 
 }
 
-func (ies *IAMEtcdStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
 	case srvAccUser:
@@ -362,10 +347,8 @@ func (ies *IAMEtcdStore) loadUsers(userType IAMUserType, m map[string]auth.Crede
 		basePrefix = iamConfigUsersPrefix
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
-	ies.setContext(ctx)
-	defer ies.clearContext()
 	r, err := ies.client.Get(ctx, basePrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
@@ -396,11 +379,9 @@ func (ies *IAMEtcdStore) loadGroup(group string, m map[string]GroupInfo) error {
 
 }
 
-func (ies *IAMEtcdStore) loadGroups(m map[string]GroupInfo) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
-	ies.setContext(ctx)
-	defer ies.clearContext()
 	r, err := ies.client.Get(ctx, iamConfigGroupsPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
@@ -432,11 +413,9 @@ func (ies *IAMEtcdStore) loadMappedPolicy(name string, userType IAMUserType, isG
 
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
-	ies.setContext(ctx)
-	defer ies.clearContext()
 	var basePrefix string
 	if isGroup {
 		basePrefix = iamConfigPolicyDBGroupsPrefix
@@ -467,7 +446,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(userType IAMUserType, isGroup bool,
 
 }
 
-func (ies *IAMEtcdStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
+func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamGroupsMap := make(map[string]GroupInfo)
 	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
@@ -475,51 +454,51 @@ func (ies *IAMEtcdStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
 	isMinIOUsersSys := false
-	sys.RLock()
+	ies.rlock()
 	if sys.usersSysType == MinIOUsersSysType {
 		isMinIOUsersSys = true
 	}
-	sys.RUnlock()
+	ies.runlock()
 
-	if err := ies.loadPolicyDocs(iamPolicyDocsMap); err != nil {
+	if err := ies.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
 		return err
 	}
 
 	// load STS temp users
-	if err := ies.loadUsers(stsUser, iamUsersMap); err != nil {
+	if err := ies.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
 		return err
 	}
 
 	if isMinIOUsersSys {
 		// load long term users
-		if err := ies.loadUsers(regularUser, iamUsersMap); err != nil {
+		if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
 			return err
 		}
-		if err := ies.loadUsers(srvAccUser, iamUsersMap); err != nil {
+		if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
-		if err := ies.loadGroups(iamGroupsMap); err != nil {
+		if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
 			return err
 		}
-		if err := ies.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
+		if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 
 	// load STS policy mappings into the same map
-	if err := ies.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
 	// Sets default canned policies, if none are set.
 	setDefaultCannedPolicies(iamPolicyDocsMap)
 
-	sys.Lock()
-	defer sys.Unlock()
+	ies.lock()
+	defer ies.Unlock()
 
 	sys.iamUsersMap = iamUsersMap
 	sys.iamGroupsMap = iamGroupsMap
@@ -579,17 +558,17 @@ func (ies *IAMEtcdStore) deleteGroupInfo(name string) error {
 	return err
 }
 
-func (ies *IAMEtcdStore) watch(sys *IAMSys) {
+func (ies *IAMEtcdStore) watch(ctx context.Context, sys *IAMSys) {
 	watchEtcd := func() {
 		for {
 		outerLoop:
 			// Refresh IAMSys with etcd watch.
-			watchCh := ies.client.Watch(context.Background(),
+			watchCh := ies.client.Watch(ctx,
 				iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 
 			for {
 				select {
-				case <-GlobalServiceDoneCh:
+				case <-ctx.Done():
 					return
 				case watchResp, ok := <-watchCh:
 					if !ok {
@@ -599,7 +578,7 @@ func (ies *IAMEtcdStore) watch(sys *IAMSys) {
 						goto outerLoop
 					}
 					if err := watchResp.Err(); err != nil {
-						logger.LogIf(context.Background(), err)
+						logger.LogIf(ctx, err)
 						// log and retry.
 						time.Sleep(1 * time.Second)
 						// Upon an error on watch channel
@@ -607,9 +586,9 @@ func (ies *IAMEtcdStore) watch(sys *IAMSys) {
 						goto outerLoop
 					}
 					for _, event := range watchResp.Events {
-						sys.Lock()
+						ies.lock()
 						ies.reloadFromEvent(sys, event)
-						sys.Unlock()
+						ies.unlock()
 					}
 				}
 			}

cmd/iam-object-store.go

@@ -38,32 +38,28 @@ type IAMObjectStore struct {
 	// Protect assignment to objAPI
 	sync.RWMutex
 
+	ctx    context.Context
 	objAPI ObjectLayer
 }
 
-func newIAMObjectStore() *IAMObjectStore {
-	return &IAMObjectStore{objAPI: nil}
+func newIAMObjectStore(ctx context.Context, objAPI ObjectLayer) *IAMObjectStore {
+	return &IAMObjectStore{ctx: ctx, objAPI: objAPI}
 }
 
-func (iamOS *IAMObjectStore) getObjectAPI() ObjectLayer {
-	iamOS.RLock()
-	defer iamOS.RUnlock()
-	if iamOS.objAPI != nil {
-		return iamOS.objAPI
-	}
-	return newObjectLayerWithoutSafeModeFn()
+func (iamOS *IAMObjectStore) lock() {
+	iamOS.Lock()
 }
 
-func (iamOS *IAMObjectStore) setObjectAPI(objAPI ObjectLayer) {
-	iamOS.Lock()
-	defer iamOS.Unlock()
-	iamOS.objAPI = objAPI
+func (iamOS *IAMObjectStore) unlock() {
+	iamOS.Unlock()
 }
 
-func (iamOS *IAMObjectStore) clearObjectAPI() {
-	iamOS.Lock()
-	defer iamOS.Unlock()
-	iamOS.objAPI = nil
+func (iamOS *IAMObjectStore) rlock() {
+	iamOS.RLock()
+}
+
+func (iamOS *IAMObjectStore) runlock() {
+	iamOS.RUnlock()
 }
 
 // Migrate users directory in a single scan.
@@ -80,17 +76,13 @@ func (iamOS *IAMObjectStore) clearObjectAPI() {
 // location.
 //
 // 3. Migrate user identity json file to include version info.
-func (iamOS *IAMObjectStore) migrateUsersConfigToV1(isSTS bool) error {
+func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context, isSTS bool) error {
 	basePrefix := iamConfigUsersPrefix
 	if isSTS {
 		basePrefix = iamConfigSTSPrefix
 	}
 
-	objAPI := iamOS.getObjectAPI()
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-	for item := range listIAMConfigItems(objAPI, basePrefix, true, doneCh) {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix, true) {
 		if item.Err != nil {
 			return item.Err
 		}
@@ -169,7 +161,7 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(isSTS bool) error {
 
 }
 
-func (iamOS *IAMObjectStore) migrateToV1() error {
+func (iamOS *IAMObjectStore) migrateToV1(ctx context.Context) error {
 	var iamFmt iamFormat
 	path := getIAMFormatFilePath()
 	if err := iamOS.loadIAMConfig(&iamFmt, path); err != nil {
@@ -190,35 +182,29 @@ func (iamOS *IAMObjectStore) migrateToV1() error {
 	}
 
 	// Migrate long-term users
-	if err := iamOS.migrateUsersConfigToV1(false); err != nil {
-		logger.LogIf(context.Background(), err)
+	if err := iamOS.migrateUsersConfigToV1(ctx, false); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
 	// Migrate STS users
-	if err := iamOS.migrateUsersConfigToV1(true); err != nil {
-		logger.LogIf(context.Background(), err)
+	if err := iamOS.migrateUsersConfigToV1(ctx, true); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
 	// Save iam format to version 1.
 	if err := iamOS.saveIAMConfig(newIAMFormatVersion1(), path); err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		return err
 	}
 	return nil
 }
 
 // Should be called under config migration lock
-func (iamOS *IAMObjectStore) migrateBackendFormat(objAPI ObjectLayer) error {
-	iamOS.setObjectAPI(objAPI)
-	defer iamOS.clearObjectAPI()
-	if err := iamOS.migrateToV1(); err != nil {
-		return err
-	}
-	return nil
+func (iamOS *IAMObjectStore) migrateBackendFormat(ctx context.Context) error {
+	return iamOS.migrateToV1(ctx)
 }
 
 func (iamOS *IAMObjectStore) saveIAMConfig(item interface{}, path string) error {
-	objectAPI := iamOS.getObjectAPI()
 	data, err := json.Marshal(item)
 	if err != nil {
 		return err
@@ -229,12 +215,11 @@ func (iamOS *IAMObjectStore) saveIAMConfig(item interface{}, path string) error
 			return err
 		}
 	}
-	return saveConfig(context.Background(), objectAPI, path, data)
+	return saveConfig(context.Background(), iamOS.objAPI, path, data)
 }
 
 func (iamOS *IAMObjectStore) loadIAMConfig(item interface{}, path string) error {
-	objectAPI := iamOS.getObjectAPI()
-	data, err := readConfig(context.Background(), objectAPI, path)
+	data, err := readConfig(iamOS.ctx, iamOS.objAPI, path)
 	if err != nil {
 		return err
 	}
@@ -248,7 +233,7 @@ func (iamOS *IAMObjectStore) loadIAMConfig(item interface{}, path string) error
 }
 
 func (iamOS *IAMObjectStore) deleteIAMConfig(path string) error {
-	err := deleteConfig(context.Background(), iamOS.getObjectAPI(), path)
+	err := deleteConfig(iamOS.ctx, iamOS.objAPI, path)
 	if _, ok := err.(ObjectNotFound); ok {
 		return errConfigNotFound
 	}
@@ -256,11 +241,6 @@ func (iamOS *IAMObjectStore) deleteIAMConfig(path string) error {
 }
 
 func (iamOS *IAMObjectStore) loadPolicyDoc(policy string, m map[string]iampolicy.Policy) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	var p iampolicy.Policy
 	err := iamOS.loadIAMConfig(&p, getPolicyDocPath(policy))
 	if err != nil {
@@ -273,15 +253,8 @@ func (iamOS *IAMObjectStore) loadPolicyDoc(policy string, m map[string]iampolicy
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDocs(m map[string]iampolicy.Policy) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-	for item := range listIAMConfigItems(objectAPI, iamConfigPoliciesPrefix, true, doneCh) {
+func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix, true) {
 		if item.Err != nil {
 			return item.Err
 		}
@@ -296,11 +269,6 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(m map[string]iampolicy.Policy) error
 }
 
 func (iamOS *IAMObjectStore) loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	var u UserIdentity
 	err := iamOS.loadIAMConfig(&u, getUserIdentityPath(user, userType))
 	if err != nil {
@@ -344,15 +312,7 @@ func (iamOS *IAMObjectStore) loadUser(user string, userType IAMUserType, m map[s
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
+func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
 	case srvAccUser:
@@ -363,7 +323,7 @@ func (iamOS *IAMObjectStore) loadUsers(userType IAMUserType, m map[string]auth.C
 		basePrefix = iamConfigUsersPrefix
 	}
 
-	for item := range listIAMConfigItems(objectAPI, basePrefix, true, doneCh) {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix, true) {
 		if item.Err != nil {
 			return item.Err
 		}
@@ -378,11 +338,6 @@ func (iamOS *IAMObjectStore) loadUsers(userType IAMUserType, m map[string]auth.C
 }
 
 func (iamOS *IAMObjectStore) loadGroup(group string, m map[string]GroupInfo) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	var g GroupInfo
 	err := iamOS.loadIAMConfig(&g, getGroupInfoPath(group))
 	if err != nil {
@@ -395,15 +350,8 @@ func (iamOS *IAMObjectStore) loadGroup(group string, m map[string]GroupInfo) err
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadGroups(m map[string]GroupInfo) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-	for item := range listIAMConfigItems(objectAPI, iamConfigGroupsPrefix, true, doneCh) {
+func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigGroupsPrefix, true) {
 		if item.Err != nil {
 			return item.Err
 		}
@@ -420,11 +368,6 @@ func (iamOS *IAMObjectStore) loadGroups(m map[string]GroupInfo) error {
 func (iamOS *IAMObjectStore) loadMappedPolicy(name string, userType IAMUserType, isGroup bool,
 	m map[string]MappedPolicy) error {
 
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	var p MappedPolicy
 	err := iamOS.loadIAMConfig(&p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
@@ -437,14 +380,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(name string, userType IAMUserType,
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
-	objectAPI := iamOS.getObjectAPI()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	doneCh := make(chan struct{})
-	defer close(doneCh)
+func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	var basePath string
 	if isGroup {
 		basePath = iamConfigPolicyDBGroupsPrefix
@@ -458,7 +394,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(userType IAMUserType, isGroup bo
 			basePath = iamConfigPolicyDBUsersPrefix
 		}
 	}
-	for item := range listIAMConfigItems(objectAPI, basePath, false, doneCh) {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePath, false) {
 		if item.Err != nil {
 			return item.Err
 		}
@@ -475,17 +411,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(userType IAMUserType, isGroup bo
 
 // Refresh IAMSys. If an object layer is passed in use that, otherwise
 // load from global.
-func (iamOS *IAMObjectStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
-	if objectAPI == nil {
-		objectAPI = iamOS.getObjectAPI()
-	}
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	// cache object layer for other load* functions
-	iamOS.setObjectAPI(objectAPI)
-	defer iamOS.clearObjectAPI()
-
+func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamGroupsMap := make(map[string]GroupInfo)
 	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
@@ -493,48 +419,48 @@ func (iamOS *IAMObjectStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
 	isMinIOUsersSys := false
-	sys.RLock()
+	iamOS.rlock()
 	if sys.usersSysType == MinIOUsersSysType {
 		isMinIOUsersSys = true
 	}
-	sys.RUnlock()
+	iamOS.runlock()
 
-	if err := iamOS.loadPolicyDocs(iamPolicyDocsMap); err != nil {
+	if err := iamOS.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
 		return err
 	}
 	// load STS temp users
-	if err := iamOS.loadUsers(stsUser, iamUsersMap); err != nil {
+	if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
 		return err
 	}
 	if isMinIOUsersSys {
-		if err := iamOS.loadUsers(regularUser, iamUsersMap); err != nil {
+		if err := iamOS.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
 			return err
 		}
-		if err := iamOS.loadUsers(srvAccUser, iamUsersMap); err != nil {
+		if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
-		if err := iamOS.loadGroups(iamGroupsMap); err != nil {
+		if err := iamOS.loadGroups(ctx, iamGroupsMap); err != nil {
 			return err
 		}
 
-		if err := iamOS.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
+		if err := iamOS.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
 	// Sets default canned policies, if none are set.
 	setDefaultCannedPolicies(iamPolicyDocsMap)
 
-	sys.Lock()
-	defer sys.Unlock()
+	iamOS.lock()
+	defer iamOS.unlock()
 
 	sys.iamUsersMap = iamUsersMap
 	sys.iamPolicyDocsMap = iamPolicyDocsMap
@@ -604,10 +530,9 @@ type itemOrErr struct {
 // prefix. If dirs is true, only directories are listed, otherwise
 // only objects are listed. All returned items have the pathPrefix
 // removed from their names.
-func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
-	doneCh <-chan struct{}) <-chan itemOrErr {
-
+func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string, dirs bool) <-chan itemOrErr {
 	ch := make(chan itemOrErr)
+
 	dirList := func(lo ListObjectsInfo) []string {
 		return lo.Prefixes
 	}
@@ -623,12 +548,12 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 
 		marker := ""
 		for {
-			lo, err := objectAPI.ListObjects(context.Background(),
+			lo, err := objAPI.ListObjects(context.Background(),
 				minioMetaBucket, pathPrefix, marker, SlashSeparator, maxObjectList)
 			if err != nil {
 				select {
 				case ch <- itemOrErr{Err: err}:
-				case <-doneCh:
+				case <-ctx.Done():
 				}
 				return
 			}
@@ -651,7 +576,7 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 				item = strings.TrimSuffix(item, SlashSeparator)
 				select {
 				case ch <- itemOrErr{Item: item}:
-				case <-doneCh:
+				case <-ctx.Done():
 					return
 				}
 			}
@@ -660,19 +585,18 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 			}
 		}
 	}()
+
 	return ch
 }
 
-func (iamOS *IAMObjectStore) watch(sys *IAMSys) {
-	ctx := GlobalContext
+func (iamOS *IAMObjectStore) watch(ctx context.Context, sys *IAMSys) {
 	watchDisk := func() {
 		for {
 			select {
 			case <-ctx.Done():
 				return
 			case <-time.NewTimer(globalRefreshIAMInterval).C:
-				err := iamOS.loadAll(sys, nil)
-				logger.LogIf(ctx, err)
+				logger.LogIf(ctx, iamOS.loadAll(ctx, sys))
 			}
 		}
 	}

cmd/iam.go

@@ -22,7 +22,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"sync"
 
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/config"
@@ -173,8 +172,6 @@ func newMappedPolicy(policy string) MappedPolicy {
 
 // IAMSys - config system.
 type IAMSys struct {
-	sync.RWMutex
-
 	usersSysType UsersSysType
 
 	// map of policy names to policy definitions
@@ -205,21 +202,27 @@ const (
 
 // IAMStorageAPI defines an interface for the IAM persistence layer
 type IAMStorageAPI interface {
-	migrateBackendFormat(ObjectLayer) error
+	lock()
+	unlock()
+
+	rlock()
+	runlock()
+
+	migrateBackendFormat(context.Context) error
 
 	loadPolicyDoc(policy string, m map[string]iampolicy.Policy) error
-	loadPolicyDocs(m map[string]iampolicy.Policy) error
+	loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error
 
 	loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error
-	loadUsers(userType IAMUserType, m map[string]auth.Credentials) error
+	loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error
 
 	loadGroup(group string, m map[string]GroupInfo) error
-	loadGroups(m map[string]GroupInfo) error
+	loadGroups(ctx context.Context, m map[string]GroupInfo) error
 
 	loadMappedPolicy(name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
-	loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
 
-	loadAll(*IAMSys, ObjectLayer) error
+	loadAll(context.Context, *IAMSys) error
 
 	saveIAMConfig(item interface{}, path string) error
 	loadIAMConfig(item interface{}, path string) error
@@ -235,7 +238,7 @@ type IAMStorageAPI interface {
 	deleteUserIdentity(name string, userType IAMUserType) error
 	deleteGroupInfo(name string) error
 
-	watch(*IAMSys)
+	watch(context.Context, *IAMSys)
 }
 
 // LoadGroup - loads a specific group from storage, and updates the
@@ -244,10 +247,7 @@ type IAMStorageAPI interface {
 // simplifies the implementation for group removal. This is called
 // only via IAM notifications.
 func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
-	sys.Lock()
-	defer sys.Unlock()
-
-	if objAPI == nil || sys.store == nil {
+	if objAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -256,6 +256,9 @@ func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
 		return nil
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	err := sys.store.loadGroup(group, sys.iamGroupsMap)
 	if err != nil && err != errConfigNotFound {
 		return err
@@ -285,13 +288,13 @@ func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
 
 // LoadPolicy - reloads a specific canned policy from backend disks or etcd.
 func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
-	sys.Lock()
-	defer sys.Unlock()
-
-	if objAPI == nil || sys.store == nil {
+	if objAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	if globalEtcdClient == nil {
 		return sys.store.loadPolicyDoc(policyName, sys.iamPolicyDocsMap)
 	}
@@ -303,13 +306,13 @@ func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
 // LoadPolicyMapping - loads the mapped policy for a user or group
 // from storage into server memory.
 func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isGroup bool) error {
-	sys.Lock()
-	defer sys.Unlock()
-
-	if objAPI == nil || sys.store == nil {
+	if objAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	if globalEtcdClient == nil {
 		var err error
 		if isGroup {
@@ -329,13 +332,13 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 
 // LoadUser - reloads a specific user from backend disks or etcd.
 func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUserType) error {
-	sys.Lock()
-	defer sys.Unlock()
-
-	if objAPI == nil || sys.store == nil {
+	if objAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	if globalEtcdClient == nil {
 		err := sys.store.loadUser(accessKey, userType, sys.iamUsersMap)
 		if err != nil {
@@ -352,42 +355,40 @@ func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUs
 }
 
 // Load - loads iam subsystem
-func (sys *IAMSys) Load() error {
+func (sys *IAMSys) Load(ctx context.Context) error {
 	// Pass nil objectlayer here - it will be loaded internally
 	// from the IAMStorageAPI.
-	return sys.store.loadAll(sys, nil)
+	return sys.store.loadAll(ctx, sys)
 }
 
 // Perform IAM configuration migration.
-func (sys *IAMSys) doIAMConfigMigration(objAPI ObjectLayer) error {
-	return sys.store.migrateBackendFormat(objAPI)
+func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
+	return sys.store.migrateBackendFormat(ctx)
 }
 
 // Init - initializes config system from iam.json
-func (sys *IAMSys) Init(objAPI ObjectLayer) error {
+func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
 
-	if globalLDAPConfig.Enabled {
-		sys.EnableLDAPSys()
-	}
-
-	sys.Lock()
 	if globalEtcdClient == nil {
-		sys.store = newIAMObjectStore()
+		sys.store = newIAMObjectStore(ctx, objAPI)
 	} else {
-		sys.store = newIAMEtcdStore()
+		sys.store = newIAMEtcdStore(ctx)
+	}
+
+	if globalLDAPConfig.Enabled {
+		sys.EnableLDAPSys()
 	}
-	sys.Unlock()
 
 	// Migrate IAM configuration
-	if err := sys.doIAMConfigMigration(objAPI); err != nil {
+	if err := sys.doIAMConfigMigration(ctx); err != nil {
 		return err
 	}
 
-	sys.store.watch(sys)
-	err := sys.store.loadAll(sys, objAPI)
+	sys.store.watch(ctx, sys)
+	err := sys.store.loadAll(ctx, sys)
 
 	// Invalidate the old cred after finishing IAM initialization
 	globalOldCred = auth.Credentials{}
@@ -398,7 +399,7 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 // DeletePolicy - deletes a canned policy from backend or etcd.
 func (sys *IAMSys) DeletePolicy(policyName string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -406,12 +407,8 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 		return errInvalidArgument
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	err := sys.store.deletePolicyDoc(policyName)
 	switch err.(type) {
@@ -462,12 +459,12 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 // InfoPolicy - expands the canned policy into its JSON structure.
 func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	v, ok := sys.iamPolicyDocsMap[policyName]
 	if !ok {
@@ -479,14 +476,14 @@ func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
 
 	var policyDocsMap = make(map[string][]byte)
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	for k, v := range sys.iamPolicyDocsMap {
 		data, err := json.Marshal(v)
@@ -502,7 +499,7 @@ func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
 // SetPolicy - sets a new name policy.
 func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -510,12 +507,8 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 		return errInvalidArgument
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if err := sys.store.savePolicyDoc(policyName, p); err != nil {
 		return err
@@ -528,7 +521,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 // DeleteUser - delete user (only for long-term users not STS users).
 func (sys *IAMSys) DeleteUser(accessKey string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -537,6 +530,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	if getErr != nil {
 		return getErr
 	}
+
 	for _, group := range userInfo.MemberOf {
 		removeErr := sys.RemoveUsersFromGroup(group, []string{accessKey})
 		if removeErr != nil {
@@ -545,17 +539,13 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	}
 
 	// Next we can remove the user from memory and IAM store
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	// It is ok to ignore deletion error on the mapped policy
 	sys.store.deleteMappedPolicy(accessKey, regularUser, false)
 	err := sys.store.deleteUserIdentity(accessKey, regularUser)
@@ -583,12 +573,12 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
 func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	// If OPA is not set we honor any policy claims for this
 	// temporary user which match with pre-configured canned
@@ -603,10 +593,6 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 			return nil
 		}
 
-		if sys.store == nil {
-			return errServerNotInitialized
-		}
-
 		mp := newMappedPolicy(policyName)
 		if err := sys.store.saveMappedPolicy(accessKey, stsUser, false, mp); err != nil {
 			return err
@@ -615,10 +601,6 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 		sys.iamUserPolicyMap[accessKey] = mp
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	u := newUserIdentity(cred)
 	if err := sys.store.saveUserIdentity(accessKey, stsUser, u); err != nil {
 		return err
@@ -631,14 +613,14 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
 
 	var users = make(map[string]madmin.UserInfo)
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
@@ -664,12 +646,12 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 // IsTempUser - returns if given key is a temporary user.
 func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return false, errServerNotInitialized
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	creds, found := sys.iamUsersMap[name]
 	if !found {
@@ -682,12 +664,12 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 // IsServiceAccount - returns if given key is a service account
 func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return false, "", errServerNotInitialized
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	creds, found := sys.iamUsersMap[name]
 	if !found {
@@ -704,12 +686,12 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return u, errServerNotInitialized
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return madmin.UserInfo{
@@ -743,7 +725,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -751,8 +733,8 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		return errInvalidArgument
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
@@ -778,10 +760,6 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		}(),
 	})
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	if err := sys.store.saveUserIdentity(accessKey, regularUser, uinfo); err != nil {
 		return err
 	}
@@ -793,7 +771,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 // NewServiceAccount - create a new service account
 func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser, sessionPolicy string) (auth.Credentials, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return auth.Credentials{}, errServerNotInitialized
 	}
 
@@ -812,17 +790,13 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser, sessionPol
 		}
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return auth.Credentials{}, errIAMActionNotAllowed
 	}
 
-	if sys.store == nil {
-		return auth.Credentials{}, errServerNotInitialized
-	}
-
 	if parentUser == globalActiveCred.AccessKey {
 		return auth.Credentials{}, errIAMActionNotAllowed
 	}
@@ -867,21 +841,17 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser, sessionPol
 // GetServiceAccount - returns the credentials of the given service account
 func (sys *IAMSys) GetServiceAccount(ctx context.Context, serviceAccountAccessKey string) (auth.Credentials, error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return auth.Credentials{}, errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return auth.Credentials{}, errIAMActionNotAllowed
 	}
 
-	if sys.store == nil {
-		return auth.Credentials{}, errServerNotInitialized
-	}
-
 	cr, ok := sys.iamUsersMap[serviceAccountAccessKey]
 	if !ok {
 		return auth.Credentials{}, errNoSuchUser
@@ -897,7 +867,7 @@ func (sys *IAMSys) GetServiceAccount(ctx context.Context, serviceAccountAccessKe
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -907,17 +877,13 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 		Status:    string(uinfo.Status),
 	})
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	cr, ok := sys.iamUsersMap[accessKey]
 	if cr.IsTemp() && ok {
 		return errIAMActionNotAllowed
@@ -939,12 +905,12 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 // SetUserSecretKey - sets user secret key
 func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
@@ -955,10 +921,6 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 		return errNoSuchUser
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
 	if err := sys.store.saveUserIdentity(accessKey, regularUser, u); err != nil {
@@ -971,8 +933,13 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
-	sys.RLock()
-	defer sys.RUnlock()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || sys == nil || sys.store == nil {
+		return cred, false
+	}
+
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	cred, ok = sys.iamUsersMap[accessKey]
 	return cred, ok && cred.IsValid()
@@ -982,7 +949,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -990,8 +957,8 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 		return errInvalidArgument
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
@@ -1019,10 +986,6 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 		gi.Members = uniqMembers
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	if err := sys.store.saveGroupInfo(group, gi); err != nil {
 		return err
 	}
@@ -1047,7 +1010,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 // given, and the group is empty, deletes the group as well.
 func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
@@ -1055,8 +1018,8 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 		return errInvalidArgument
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
@@ -1083,10 +1046,6 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 		return errGroupNotEmpty
 	}
 
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	if len(members) == 0 {
 		// len(gi.Members) == 0 here.
 
@@ -1135,16 +1094,12 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 // SetGroupStatus - enable/disabled a group
 func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
@@ -1174,6 +1129,11 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 
 // GetGroupDescription - builds up group description
 func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || sys == nil || sys.store == nil {
+		return gd, errServerNotInitialized
+	}
+
 	ps, err := sys.PolicyDBGet(group, true)
 	if err != nil {
 		return gd, err
@@ -1192,8 +1152,8 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 		}, nil
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	gi, ok := sys.iamGroupsMap[group]
 	if !ok {
@@ -1210,8 +1170,13 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 
 // ListGroups - lists groups.
 func (sys *IAMSys) ListGroups() (r []string, err error) {
-	sys.RLock()
-	defer sys.RUnlock()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || sys == nil || sys.store == nil {
+		return r, errServerNotInitialized
+	}
+
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
@@ -1228,12 +1193,12 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 // users, policy is set directly by called sys.policyDBSet().
 func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	// isSTS is always false when called via PolicyDBSet as policy
 	// is never set by an external API call for STS users.
@@ -1243,10 +1208,6 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 // policyDBSet - sets a policy for user in the policy db. Assumes that caller
 // has sys.Lock(). If policy == "", then policy mapping is removed.
 func (sys *IAMSys) policyDBSet(name, policy string, userType IAMUserType, isGroup bool) error {
-	if sys.store == nil {
-		return errServerNotInitialized
-	}
-
 	if name == "" {
 		return errInvalidArgument
 	}
@@ -1351,8 +1312,8 @@ func (sys *IAMSys) GetAccountAccess(accountName, bucket string) (rd, wr, o bool)
 	}
 
 	// Policies were found, evaluate all of them.
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	var availablePolicies []iampolicy.Policy
 	for _, pname := range policies {
@@ -1401,17 +1362,17 @@ func (sys *IAMSys) GetAccountAccess(accountName, bucket string) (rd, wr, o bool)
 // be a member of multiple groups, this function returns an array of
 // applicable policies (each group is mapped to at most one policy).
 func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
-	if name == "" {
-		return nil, errInvalidArgument
-	}
-
 	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
+	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	if name == "" {
+		return nil, errInvalidArgument
+	}
+
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	return sys.policyDBGet(name, isGroup)
 }
@@ -1499,14 +1460,14 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	var availablePolicies []iampolicy.Policy
 
 	// Policies were found, evaluate all of them.
-	sys.RLock()
+	sys.store.rlock()
 	for _, pname := range parentUserPolicies {
 		p, found := sys.iamPolicyDocsMap[pname]
 		if found {
 			availablePolicies = append(availablePolicies, p)
 		}
 	}
-	sys.RUnlock()
+	sys.store.runlock()
 
 	if len(availablePolicies) == 0 {
 		return false
@@ -1595,8 +1556,8 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 			return false
 		}
 
-		sys.RLock()
-		defer sys.RUnlock()
+		sys.store.rlock()
+		defer sys.store.runlock()
 
 		// We look up the policy mapping directly to bypass
 		// users exists, group exists validations that do not
@@ -1641,8 +1602,8 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 		return false
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	// If policy is available for given user, check the policy.
 	mp, ok := sys.iamUserPolicyMap[args.AccountName]
@@ -1742,8 +1703,8 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	}
 
 	// Policies were found, evaluate all of them.
-	sys.RLock()
-	defer sys.RUnlock()
+	sys.store.rlock()
+	defer sys.store.runlock()
 
 	var availablePolicies []iampolicy.Policy
 	for _, pname := range policies {
@@ -1822,8 +1783,8 @@ func (sys *IAMSys) removeGroupFromMembershipsMap(group string) {
 
 // EnableLDAPSys - enable ldap system users type.
 func (sys *IAMSys) EnableLDAPSys() {
-	sys.Lock()
-	defer sys.Unlock()
+	sys.store.lock()
+	defer sys.store.unlock()
 
 	sys.usersSysType = LDAPUsersSysType
 }

cmd/peer-rest-server.go

@@ -357,7 +357,7 @@ func (s *peerRESTServer) LoadUsersHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	err := globalIAMSys.Load()
+	err := globalIAMSys.Load(GlobalContext)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return

cmd/server-main.go

@@ -250,12 +250,12 @@ func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error)
 		// ****  WARNING ****
 		// Migrating to encrypted backend on etcd should happen before initialization of
 		// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
-		if err = migrateIAMConfigsEtcdToEncrypted(globalEtcdClient); err != nil {
+		if err = migrateIAMConfigsEtcdToEncrypted(GlobalContext, globalEtcdClient); err != nil {
 			return fmt.Errorf("Unable to handle encrypted backend for iam and policies: %w", err)
 		}
 	}
 
-	if err = globalIAMSys.Init(newObject); err != nil {
+	if err = globalIAMSys.Init(GlobalContext, newObject); err != nil {
 		return fmt.Errorf("Unable to initialize IAM system: %w", err)
 	}
 

cmd/test-utils_test.go

@@ -358,7 +358,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(GlobalContext, objLayer)
 
 	buckets, err := objLayer.ListBuckets(context.Background())
 	if err != nil {
@@ -1604,7 +1604,7 @@ func newTestObjectLayer(endpointZones EndpointZones) (newObject ObjectLayer, err
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(z)
+	globalIAMSys.Init(GlobalContext, z)
 
 	globalPolicySys = NewPolicySys()
 	globalPolicySys.Init(nil, z)
@@ -1908,7 +1908,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 
 	newAllSubsystems()
 
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(GlobalContext, objLayer)
 
 	buckets, err := objLayer.ListBuckets(context.Background())
 	if err != nil {
@@ -1964,7 +1964,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 	}
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(GlobalContext, objLayer)
 
 	buckets, err := objLayer.ListBuckets(context.Background())
 	if err != nil {