Add etcd support to support STS on gateway mode (#6531)

cmd/config-common.go

@@ -50,6 +50,20 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 	return buffer.Bytes(), nil
 }
 
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	_, err := client.Put(ctx, configFile, string(data))
+	return err
+}
+
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
 	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {

cmd/config-current.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
@@ -30,6 +31,7 @@ import (
 	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -270,6 +272,20 @@ func (s *serverConfig) loadFromEnvs() {
 	if globalIsEnvCompression {
 		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
 	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			s.OpenID.JWKS.PopulatePublicKey()
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
 }
 
 // TestNotificationTargets tries to establish connections to all notification

cmd/gateway-main.go

@@ -244,9 +244,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Create new IAM system.
 	globalIAMSys = NewIAMSys()
-
-	// Initialize IAM sys.
-	go globalIAMSys.Init(newObject)
+	if globalEtcdClient != nil {
+		// Initialize IAM sys.
+		go globalIAMSys.Init(newObject)
+	}
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()

cmd/iam.go

@@ -20,9 +20,12 @@ import (
 	"context"
 	"encoding/json"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/iam/policy"
@@ -64,25 +67,46 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	if err := sys.refresh(objAPI); err != nil {
-		return err
-	}
-
-	// Refresh IAMSys in background.
-	go func() {
-		ticker := time.NewTicker(globalRefreshIAMInterval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-globalServiceDoneCh:
-				return
-			case <-ticker.C:
-				logger.LogIf(context.Background(), sys.refresh(objAPI))
+	defer func() {
+		// Refresh IAMSys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshIAMInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-globalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
 			}
-		}
+		}()
 	}()
-	return nil
 
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing IAM needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			// Load IAMSys once during boot.
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for IAM subsystem to be initialized..")
+					continue
+				}
+				return err
+			}
+			return nil
+		}
+	}
 }
 
 // SetPolicy - sets policy to given user name.  If policy is empty,
@@ -99,13 +123,19 @@ func (sys *IAMSys) SetPolicy(accessKey string, p iampolicy.Policy) error {
 		return err
 	}
 
+	if globalEtcdClient != nil {
+		if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
+			return err
+		}
+	} else {
+		if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
+			return err
+		}
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
-	if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
-		return err
-	}
-
 	if p.IsEmpty() {
 		delete(sys.iamPolicyMap, accessKey)
 	} else {
@@ -128,13 +158,19 @@ func (sys *IAMSys) SaveTempPolicy(accessKey string, p iampolicy.Policy) error {
 		return err
 	}
 
+	if globalEtcdClient != nil {
+		if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
+			return err
+		}
+	} else {
+		if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
+			return err
+		}
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
-	if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
-		return err
-	}
-
 	if p.IsEmpty() {
 		delete(sys.iamPolicyMap, accessKey)
 	} else {
@@ -152,13 +188,17 @@ func (sys *IAMSys) DeletePolicy(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	var err error
 	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+	} else {
+		err = deleteConfig(context.Background(), objectAPI, configFile)
+	}
 
 	sys.Lock()
 	defer sys.Unlock()
 
-	err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile)
-
 	delete(sys.iamPolicyMap, accessKey)
 
 	return err
@@ -171,12 +211,17 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	var err error
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+	} else {
+		err = deleteConfig(context.Background(), objectAPI, configFile)
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
-	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
-	err := objectAPI.DeleteObject(context.Background(), minioMetaBucket, configFile)
-
 	delete(sys.iamUsersMap, accessKey)
 	return err
 }
@@ -188,19 +233,25 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error {
 		return errServerNotInitialized
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
 	configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile)
 	data, err := json.Marshal(cred)
 	if err != nil {
 		return err
 	}
 
-	if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
-		return err
+	if globalEtcdClient != nil {
+		if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
+			return err
+		}
+	} else {
+		if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
+			return err
+		}
 	}
 
+	sys.Lock()
+	defer sys.Unlock()
+
 	sys.iamUsersMap[accessKey] = cred
 	return nil
 }
@@ -218,13 +269,19 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 		return err
 	}
 
+	if globalEtcdClient != nil {
+		if err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data); err != nil {
+			return err
+		}
+	} else {
+		if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
+			return err
+		}
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
-	if err = saveConfig(context.Background(), objectAPI, configFile, data); err != nil {
-		return err
-	}
-
 	sys.iamUsersMap[accessKey] = auth.Credentials{
 		AccessKey: accessKey,
 		SecretKey: uinfo.SecretKey,
@@ -266,6 +323,76 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	return args.IsOwner
 }
 
+var defaultContextTimeout = 5 * time.Minute
+
+// Similar to reloadUsers but updates users, policies maps from etcd server,
+func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	users := set.NewStringSet()
+	for _, kv := range r.Kvs {
+		// Extract user by stripping off the `prefix` value as suffix,
+		// then strip off the remaining basename to obtain the prefix
+		// value, usually in the following form.
+		//
+		//  key := "config/iam/users/newuser/identity.json"
+		//  prefix := "config/iam/users/"
+		//  v := trim(trim(key, prefix), base(key)) == "newuser"
+		//
+		user := strings.TrimSuffix(strings.TrimSuffix(string(kv.Key), prefix), path.Base(string(kv.Key)))
+		if !users.Contains(user) {
+			users.Add(user)
+		}
+	}
+
+	// Reload config and policies for all users.
+	for _, user := range users.ToSlice() {
+		idFile := pathJoin(prefix, user, iamIdentityFile)
+		pFile := pathJoin(prefix, user, iamPolicyFile)
+		cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
+		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+		if cerr != nil && cerr != errConfigNotFound {
+			return cerr
+		}
+		if perr != nil && perr != errConfigNotFound {
+			return perr
+		}
+		if cerr == errConfigNotFound && perr == errConfigNotFound {
+			continue
+		}
+		if cerr == nil {
+			var cred auth.Credentials
+			if err = json.Unmarshal(cdata, &cred); err != nil {
+				return err
+			}
+			cred.AccessKey = user
+			if cred.IsExpired() {
+				deleteConfigEtcd(ctx, globalEtcdClient, idFile)
+				deleteConfigEtcd(ctx, globalEtcdClient, pFile)
+				continue
+			}
+			usersMap[cred.AccessKey] = cred
+		}
+		if perr == nil {
+			var p iampolicy.Policy
+			if err = json.Unmarshal(pdata, &p); err != nil {
+				return err
+			}
+			policyMap[path.Base(prefix)] = p
+		}
+	}
+	return nil
+}
+
 // reloadUsers reads an updates users, policies from object layer into user and policy maps.
 func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]iampolicy.Policy) error {
 	marker := ""
@@ -326,12 +453,20 @@ func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamPolicyMap := make(map[string]iampolicy.Policy)
 
-	if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
-		return err
-	}
-
-	if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
-		return err
+	if globalEtcdClient != nil {
+		if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+	} else {
+		if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
 	}
 
 	sys.Lock()

cmd/object-api-errors.go

@@ -383,15 +383,3 @@ func isErrObjectNotFound(err error) bool {
 	_, ok := err.(ObjectNotFound)
 	return ok
 }
-
-// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum.
-func isInsufficientReadQuorum(err error) bool {
-	_, ok := err.(InsufficientReadQuorum)
-	return ok
-}
-
-// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
-func isInsufficientWriteQuorum(err error) bool {
-	_, ok := err.(InsufficientWriteQuorum)
-	return ok
-}

docs/sts/README.md

@@ -19,6 +19,7 @@ In this document we will explain in detail on how to configure all the prerequis
 ### 1. Prerequisites
 - [Configuring wso2](./wso2.md)
 - [Configuring opa](./opa.md)
+- [Configuring etcd (optional needed only in gateway or federation mode)](./etcd.md)
 
 ### 2. Setup Minio with WSO2, OPA
 Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio server to use these credentials to perform object API operations.
@@ -31,7 +32,21 @@ export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
 minio server /mnt/data
 ```
 
-### 3. Test using full-example.go
+### 3. Setup Minio Gateway with WSO2, OPA, ETCD
+Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use Minio STS API and Minio gateway to use these credentials to perform object API operations.
+
+> NOTE: Minio gateway requires etcd to be configured to use STS API.
+
+```
+export MINIO_ACCESS_KEY=aws_access_key
+export MINIO_SECRET_KEY=aws_secret_key
+export MINIO_IAM_JWKS_URL=https://localhost:9443/oauth2/jwks
+export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
+export MINIO_ETCD_ENDPOINTS=localhost:2379
+minio gateway s3
+```
+
+### 4. Test using full-example.go
 On another terminal run `full-example.go` a sample client application which obtains JWT access tokens from an identity provider, in our case its WSO2. Uses the returned access token response to get new temporary credentials from the Minio server using the STS API call `AssumeRoleWithClientGrants`.
 
 ```

docs/sts/etcd.md

@@ -0,0 +1,57 @@
+# etcd V3 Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines.
+
+## Get started
+### 1. Prerequisites
+- Docker 18.03 or above, refer here for [installation](https://docs.docker.com/install/).
+
+### 2. Start etcd
+etcd uses [gcr.io/etcd-development/etcd](gcr.io/etcd-development/etcd) as a primary container registry.
+
+```
+rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \
+  docker rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \
+  docker run \
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \
+  --name etcd-gcr-v3.3.9 \
+  gcr.io/etcd-development/etcd:v3.3.9 \
+  /usr/local/bin/etcd \
+  --name s1 \
+  --data-dir /etcd-data \
+  --listen-client-urls http://0.0.0.0:2379 \
+  --advertise-client-urls http://0.0.0.0:2379 \
+  --listen-peer-urls http://0.0.0.0:2380 \
+  --initial-advertise-peer-urls http://0.0.0.0:2380 \
+  --initial-cluster s1=http://0.0.0.0:2380 \
+  --initial-cluster-token tkn \
+  --initial-cluster-state new
+```
+
+### 3. Setup Minio with etcd
+Minio server expects environment variable for etcd as `MINIO_ETCD_ENDPOINTS`, this environment variable takes many comma separated entries.
+```
+export MINIO_ETCD_ENDPOINTS=localhost:2379
+minio server /data
+```
+
+### 5. Test with Minio STS API
+Assuming that you have configured Minio server to support STS API by following the doc [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) and once you have obtained the JWT from WSO2 as mentioned in [WSO2 Quickstart Guide](https://docs.minio.io/docs/wso2-quickstart-guide).
+```
+go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga
+
+##### Credentials
+{
+	"accessKey": "IRBLVDGN5QGMDCMO1X8V",
+	"secretKey": "KzS3UZKE7xqNdtRbKyfcWgxBS6P1G4kwZn4DXKuY",
+	"expiration": "2018-08-21T15:49:38-07:00",
+	"sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJUkJMVkRHTjVRR01EQ01PMVg4ViIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODkxNzc4LCJpYXQiOjE1MzQ4ODgxNzgsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMTg0NDMyOWMtZDY1YS00OGEzLTgyMjgtOWRmNzNmZTgzZDU2In0.4rKsZ8VkZnIS_ALzfTJ9UbEKPFlQVvIyuHw6AWTJcDFDVgQA2ooQHmH9wUDnhXBi1M7o8yWJ47DXP-TLPhwCgQ"
+}
+```
+
+These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
+
+## Explore Further
+- [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
+- [The Minio documentation website](https://docs.minio.io)

docs/sts/wso2.md

@@ -83,6 +83,8 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW
 }
 ```
 
+These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
+
 ## Explore Further
 - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
 - [The Minio documentation website](https://docs.minio.io)