fix: load credentials from etcd directly when possible (#11339)

under large deployments loading credentials might be
time consuming, while this is okay and we will not
respond quickly for `mc admin user list` like queries
but it is possible to support `mc admin user info`

just like how we handle authentication by fetching
the user directly from persistent store.

additionally support service accounts properly,
reloaded from etcd during watch() - this was missing

This PR is also half way remedy for #11305
master
Harshavardhana 4 years ago committed by GitHub
parent 5f51ef0b40
commit 5c6bfae4c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      cmd/admin-handlers-users.go
  2. 8
      cmd/config/etcd/etcd.go
  3. 129
      cmd/iam-etcd-store.go
  4. 105
      cmd/iam-object-store.go
  5. 207
      cmd/iam.go
  6. 1
      go.mod

@ -426,7 +426,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
return
}
if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
if err = globalIAMSys.CreateUser(accessKey, uinfo); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}

@ -27,6 +27,7 @@ import (
xnet "github.com/minio/minio/pkg/net"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/namespace"
"go.uber.org/zap"
)
const (
@ -144,6 +145,13 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg.Enabled = true
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
// Disable etcd client SDK logging, etcd client
// incorrectly starts logging in unexpected data
// format.
cfg.LogConfig = &zap.Config{
Level: zap.NewAtomicLevelAt(zap.FatalLevel),
Encoding: "console",
}
cfg.Endpoints = etcdEndpoints
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
// Default path prefix for all keys on etcd, other than CoreDNSPath.

@ -264,6 +264,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
var p iampolicy.Policy
err := ies.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
if err != nil {
if err == errConfigNotFound {
return errNoSuchPolicy
}
return err
}
m[policy] = p
@ -282,8 +285,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampol
// Reload config and policies for all policys.
for _, policyName := range policies.ToSlice() {
err = ies.loadPolicyDoc(ctx, policyName, m)
if err != nil {
if err = ies.loadPolicyDoc(ctx, policyName, m); err != nil && err != errNoSuchPolicy {
return err
}
}
@ -358,7 +360,7 @@ func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m
// Reload config for all users.
for _, user := range users.ToSlice() {
if err = ies.loadUser(ctx, user, userType, m); err != nil {
if err = ies.loadUser(ctx, user, userType, m); err != nil && err != errNoSuchUser {
return err
}
}
@ -392,7 +394,7 @@ func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo)
// Reload config for all groups.
for _, group := range groups.ToSlice() {
if err = ies.loadGroup(ctx, group, m); err != nil {
if err = ies.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup {
return err
}
}
@ -440,7 +442,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
// Reload config and policies for all users.
for _, user := range users.ToSlice() {
if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil {
if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil && err != errNoSuchPolicy {
return err
}
}
@ -449,117 +451,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
}
func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
iamUsersMap := make(map[string]auth.Credentials)
iamGroupsMap := make(map[string]GroupInfo)
iamUserPolicyMap := make(map[string]MappedPolicy)
iamGroupPolicyMap := make(map[string]MappedPolicy)
ies.rlock()
isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
ies.runlock()
ies.lock()
if err := ies.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
ies.unlock()
return err
}
// Sets default canned policies, if none are set.
setDefaultCannedPolicies(sys.iamPolicyDocsMap)
ies.unlock()
if isMinIOUsersSys {
if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
return err
}
if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
return err
}
}
// load polices mapped to users
if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
return err
}
// load policies mapped to groups
if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
return err
}
if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
return err
}
// load STS temp users
if err := ies.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
return err
}
// load STS policy mappings
if err := ies.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
return err
}
ies.lock()
defer ies.Unlock()
// Merge the new reloaded entries into global map.
// See issue https://github.com/minio/minio/issues/9651
// where the present list of entries on disk are not yet
// latest, there is a small window where this can make
// valid users invalid.
for k, v := range iamUsersMap {
sys.iamUsersMap[k] = v
}
for k, v := range iamUserPolicyMap {
sys.iamUserPolicyMap[k] = v
}
// purge any expired entries which became expired now.
var expiredEntries []string
for k, v := range sys.iamUsersMap {
if v.IsExpired() {
delete(sys.iamUsersMap, k)
delete(sys.iamUserPolicyMap, k)
expiredEntries = append(expiredEntries, k)
// Deleting on the disk is taken care of in the next cycle
}
}
for _, v := range sys.iamUsersMap {
if v.IsServiceAccount() {
for _, accessKey := range expiredEntries {
if v.ParentUser == accessKey {
_ = ies.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
delete(sys.iamUsersMap, v.AccessKey)
}
}
}
}
// purge any expired entries which became expired now.
for k, v := range sys.iamUsersMap {
if v.IsExpired() {
delete(sys.iamUsersMap, k)
delete(sys.iamUserPolicyMap, k)
// Deleting on the etcd is taken care of in the next cycle
}
}
for k, v := range iamGroupPolicyMap {
sys.iamGroupPolicyMap[k] = v
}
for k, v := range iamGroupsMap {
sys.iamGroupsMap[k] = v
}
sys.buildUserGroupMemberships()
sys.storeFallback = false
return nil
return sys.Load(ctx, ies)
}
func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
@ -653,6 +545,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
usersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigUsersPrefix)
groupsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigGroupsPrefix)
stsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigSTSPrefix)
svcPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigServiceAccountsPrefix)
policyPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPoliciesPrefix)
policyDBUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBUsersPrefix)
policyDBSTSUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBSTSUsersPrefix)
@ -672,6 +565,10 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
iamConfigSTSPrefix))
ies.loadUser(ctx, accessKey, stsUser, sys.iamUsersMap)
case svcPrefix:
accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
iamConfigServiceAccountsPrefix))
ies.loadUser(ctx, accessKey, srvAccUser, sys.iamUsersMap)
case groupsPrefix:
group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
iamConfigGroupsPrefix))

@ -402,110 +402,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
return nil
}
// Refresh IAMSys. If an object layer is passed in use that, otherwise
// load from global.
// Refresh IAMSys. If an object layer is passed in use that, otherwise load from global.
func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
iamUsersMap := make(map[string]auth.Credentials)
iamGroupsMap := make(map[string]GroupInfo)
iamUserPolicyMap := make(map[string]MappedPolicy)
iamGroupPolicyMap := make(map[string]MappedPolicy)
iamOS.rlock()
isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
iamOS.runlock()
iamOS.lock()
if err := iamOS.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
iamOS.unlock()
return err
}
// Sets default canned policies, if none are set.
setDefaultCannedPolicies(sys.iamPolicyDocsMap)
iamOS.unlock()
if isMinIOUsersSys {
if err := iamOS.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
return err
}
if err := iamOS.loadGroups(ctx, iamGroupsMap); err != nil {
return err
}
}
// load polices mapped to users
if err := iamOS.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
return err
}
// load policies mapped to groups
if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
return err
}
if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
return err
}
// load STS temp users
if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
return err
}
// load STS policy mappings
if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
return err
}
iamOS.lock()
defer iamOS.unlock()
// Merge the new reloaded entries into global map.
// See issue https://github.com/minio/minio/issues/9651
// where the present list of entries on disk are not yet
// latest, there is a small window where this can make
// valid users invalid.
for k, v := range iamUsersMap {
sys.iamUsersMap[k] = v
}
for k, v := range iamUserPolicyMap {
sys.iamUserPolicyMap[k] = v
}
// purge any expired entries which became expired now.
var expiredEntries []string
for k, v := range sys.iamUsersMap {
if v.IsExpired() {
delete(sys.iamUsersMap, k)
delete(sys.iamUserPolicyMap, k)
expiredEntries = append(expiredEntries, k)
// Deleting on the disk is taken care of in the next cycle
}
}
for _, v := range sys.iamUsersMap {
if v.IsServiceAccount() {
for _, accessKey := range expiredEntries {
if v.ParentUser == accessKey {
_ = iamOS.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
delete(sys.iamUsersMap, v.AccessKey)
}
}
}
}
for k, v := range iamGroupPolicyMap {
sys.iamGroupPolicyMap[k] = v
}
for k, v := range iamGroupsMap {
sys.iamGroupsMap[k] = v
}
sys.buildUserGroupMemberships()
sys.storeFallback = false
return nil
return sys.Load(ctx, iamOS)
}
func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {

@ -442,6 +442,122 @@ func (sys *IAMSys) Initialized() bool {
return sys.store != nil
}
// Load - loads all credentials
func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
iamUsersMap := make(map[string]auth.Credentials)
iamGroupsMap := make(map[string]GroupInfo)
iamUserPolicyMap := make(map[string]MappedPolicy)
iamGroupPolicyMap := make(map[string]MappedPolicy)
iamPolicyDocsMap := make(map[string]iampolicy.Policy)
store.rlock()
isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
store.runlock()
if err := store.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
return err
}
// Sets default canned policies, if none are set.
setDefaultCannedPolicies(iamPolicyDocsMap)
if isMinIOUsersSys {
if err := store.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
return err
}
if err := store.loadGroups(ctx, iamGroupsMap); err != nil {
return err
}
}
// load polices mapped to users
if err := store.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
return err
}
// load policies mapped to groups
if err := store.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
return err
}
if err := store.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
return err
}
// load STS temp users
if err := store.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
return err
}
// load STS policy mappings
if err := store.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
return err
}
store.lock()
defer store.unlock()
for k, v := range iamPolicyDocsMap {
sys.iamPolicyDocsMap[k] = v
}
// Merge the new reloaded entries into global map.
// See issue https://github.com/minio/minio/issues/9651
// where the present list of entries on disk are not yet
// latest, there is a small window where this can make
// valid users invalid.
for k, v := range iamUsersMap {
sys.iamUsersMap[k] = v
}
for k, v := range iamUserPolicyMap {
sys.iamUserPolicyMap[k] = v
}
// purge any expired entries which became expired now.
var expiredEntries []string
for k, v := range sys.iamUsersMap {
if v.IsExpired() {
delete(sys.iamUsersMap, k)
delete(sys.iamUserPolicyMap, k)
expiredEntries = append(expiredEntries, k)
// Deleting on the disk is taken care of in the next cycle
}
}
for _, v := range sys.iamUsersMap {
if v.IsServiceAccount() {
for _, accessKey := range expiredEntries {
if v.ParentUser == accessKey {
_ = store.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
delete(sys.iamUsersMap, v.AccessKey)
}
}
}
}
// purge any expired entries which became expired now.
for k, v := range sys.iamUsersMap {
if v.IsExpired() {
delete(sys.iamUsersMap, k)
delete(sys.iamUserPolicyMap, k)
// Deleting on the etcd is taken care of in the next cycle
}
}
for k, v := range iamGroupPolicyMap {
sys.iamGroupPolicyMap[k] = v
}
for k, v := range iamGroupsMap {
sys.iamGroupsMap[k] = v
}
sys.buildUserGroupMemberships()
sys.storeFallback = false
return nil
}
// Init - initializes config system by reading entries from config/iam
func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
// Initialize IAM store
@ -664,6 +780,10 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
return errServerNotInitialized
}
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
// First we remove the user from their groups.
userInfo, getErr := sys.GetUserInfo(accessKey)
if getErr != nil {
@ -681,18 +801,21 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
sys.store.lock()
defer sys.store.unlock()
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
// Delete any service accounts if any first.
for _, u := range sys.iamUsersMap {
// Delete any service accounts if any first.
if u.IsServiceAccount() {
if u.ParentUser == accessKey {
_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, srvAccUser)
delete(sys.iamUsersMap, u.AccessKey)
}
}
// Delete any associated STS users.
if u.IsTemp() {
if u.ParentUser == accessKey {
_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, stsUser)
delete(sys.iamUsersMap, u.AccessKey)
}
}
}
// It is ok to ignore deletion error on the mapped policy
@ -869,13 +992,19 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
}
sys.store.rlock()
defer sys.store.runlock()
fallback := sys.storeFallback
sys.store.runlock()
if fallback {
sys.loadUserFromStore(name)
}
if sys.usersSysType != MinIOUsersSysType {
sys.store.rlock()
// If the user has a mapped policy or is a member of a group, we
// return that info. Otherwise we return error.
mappedPolicy, ok1 := sys.iamUserPolicyMap[name]
memberships, ok2 := sys.iamUserGroupMemberships[name]
sys.store.runlock()
if !ok1 && !ok2 {
return u, errNoSuchUser
}
@ -885,6 +1014,9 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
}, nil
}
sys.store.rlock()
defer sys.store.runlock()
cred, found := sys.iamUsersMap[name]
if !found {
return u, errNoSuchUser
@ -894,7 +1026,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
return u, errIAMActionNotAllowed
}
u = madmin.UserInfo{
return madmin.UserInfo{
PolicyName: sys.iamUserPolicyMap[name].Policies,
Status: func() madmin.AccountStatus {
if cred.IsValid() {
@ -903,8 +1035,8 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
return madmin.AccountDisabled
}(),
MemberOf: sys.iamUserGroupMemberships[name].ToSlice(),
}
return u, nil
}, nil
}
// SetUserStatus - sets current user status, supports disabled or enabled.
@ -913,6 +1045,10 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
return errServerNotInitialized
}
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
if status != madmin.AccountEnabled && status != madmin.AccountDisabled {
return errInvalidArgument
}
@ -920,10 +1056,6 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
sys.store.lock()
defer sys.store.unlock()
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
cred, ok := sys.iamUsersMap[accessKey]
if !ok {
return errNoSuchUser
@ -1091,30 +1223,31 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
return nil
}
// SetUser - set user credentials and policy.
func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
// CreateUser - create new user credentials and policy, if user already exists
// they shall be rewritten with new inputs.
func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
if !sys.Initialized() {
return errServerNotInitialized
}
u := newUserIdentity(auth.Credentials{
AccessKey: accessKey,
SecretKey: uinfo.SecretKey,
Status: string(uinfo.Status),
})
sys.store.lock()
defer sys.store.unlock()
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
sys.store.lock()
defer sys.store.unlock()
cr, ok := sys.iamUsersMap[accessKey]
if cr.IsTemp() && ok {
return errIAMActionNotAllowed
}
u := newUserIdentity(auth.Credentials{
AccessKey: accessKey,
SecretKey: uinfo.SecretKey,
Status: string(uinfo.Status),
})
if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
return err
}
@ -1156,13 +1289,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
return nil
}
// GetUser - get user credentials
func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
if !sys.Initialized() {
return cred, false
}
reloadUser := func() {
func (sys *IAMSys) loadUserFromStore(accessKey string) {
sys.store.lock()
// If user is already found proceed.
if _, found := sys.iamUsersMap[accessKey]; !found {
@ -1198,13 +1325,19 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
sys.buildUserGroupMemberships()
sys.store.unlock()
}
// GetUser - get user credentials
func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
if !sys.Initialized() {
return cred, false
}
sys.store.rlock()
fallback := sys.storeFallback
sys.store.runlock()
if fallback {
reloadUser()
sys.loadUserFromStore(accessKey)
}
sys.store.rlock()
@ -1217,7 +1350,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
// the IAM store and see if credential
// exists now. If it doesn't proceed to
// fail.
reloadUser()
sys.loadUserFromStore(accessKey)
sys.store.rlock()
cred, ok = sys.iamUsersMap[accessKey]
@ -1305,14 +1438,14 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
return errServerNotInitialized
}
if group == "" {
return errInvalidArgument
}
if sys.usersSysType != MinIOUsersSysType {
return errIAMActionNotAllowed
}
if group == "" {
return errInvalidArgument
}
sys.store.lock()
defer sys.store.unlock()

@ -81,6 +81,7 @@ require (
github.com/willf/bloom v2.0.3+incompatible
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
go.uber.org/zap v1.13.0
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
golang.org/x/net v0.0.0-20201216054612-986b41b23924
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68

Loading…
Cancel
Save