crypto: allow multiple KES endpoints (#10383)

This commit addresses a maintenance / automation problem when MinIO-KES
is deployed on bare-metal. In orchestrated env. the orchestrator (K8S)
will make sure that `n` KES servers (IPs) are available via the same DNS
name. There it is sufficient to provide just one endpoint.

cmd/admin-handlers.go

@@ -1582,12 +1582,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
 
-	if kmsInfo.Endpoint == "" {
+	if len(kmsInfo.Endpoints) == 0 {
 		vault.Status = "KMS configured using master key"
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
+	if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"

cmd/crypto/config.go

@@ -16,11 +16,14 @@ package crypto
 
 import (
 	"errors"
+	"math/rand"
 	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/ellipses"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -167,7 +170,8 @@ const (
 
 const (
 	// EnvKMSKesEndpoint is the environment variable used to specify
-	// the kes server HTTPS endpoint.
+	// one or multiple KES server HTTPS endpoints. The individual
+	// endpoints should be separated by ','.
 	EnvKMSKesEndpoint = "MINIO_KMS_KES_ENDPOINT"
 
 	// EnvKMSKesKeyFile is the environment variable used to specify
@@ -216,16 +220,30 @@ func LookupKesConfig(kvs config.KVS) (KesConfig, error) {
 	kesCfg := KesConfig{}
 
 	endpointStr := env.Get(EnvKMSKesEndpoint, kvs.Get(KMSKesEndpoint))
-	if endpointStr != "" {
-		// Lookup kes configuration & overwrite config entry if ENV var is present
-		endpoint, err := xnet.ParseHTTPURL(endpointStr)
+	var endpoints []string
+	for _, endpoint := range strings.Split(endpointStr, ",") {
+		if !ellipses.HasEllipses(endpoint) {
+			endpoints = append(endpoints, endpoint)
+			continue
+		}
+		pattern, err := ellipses.FindEllipsesPatterns(endpoint)
 		if err != nil {
 			return kesCfg, err
 		}
-		endpointStr = endpoint.String()
+		for _, p := range pattern {
+			endpoints = append(endpoints, p.Expand()...)
+		}
 	}
 
-	kesCfg.Endpoint = endpointStr
+	randNum := rand.Intn(len(endpoints) + 1) // We add 1 b/c len(endpoints) may be 0: See: rand.Intn docs
+	kesCfg.Endpoint = make([]string, len(endpoints))
+	for i, endpoint := range endpoints {
+		endpoint, err := xnet.ParseHTTPURL(endpoint)
+		if err != nil {
+			return kesCfg, err
+		}
+		kesCfg.Endpoint[(randNum+i)%len(endpoints)] = endpoint.String()
+	}
 	kesCfg.KeyFile = env.Get(EnvKMSKesKeyFile, kvs.Get(KMSKesKeyFile))
 	kesCfg.CertFile = env.Get(EnvKMSKesCertFile, kvs.Get(KMSKesCertFile))
 	kesCfg.CAPath = env.Get(EnvKMSKesCAPath, kvs.Get(KMSKesCAPath))

cmd/crypto/kes.go

@@ -46,8 +46,8 @@ var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist
 type KesConfig struct {
 	Enabled bool
 
-	// The kes server endpoint.
-	Endpoint string
+	// The KES server endpoints.
+	Endpoint []string
 
 	// The path to the TLS private key used
 	// by MinIO to authenticate to the kes
@@ -86,7 +86,7 @@ type KesConfig struct {
 // Verify verifies if the kes configuration is correct
 func (k KesConfig) Verify() (err error) {
 	switch {
-	case k.Endpoint == "":
+	case len(k.Endpoint) == 0:
 		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
 		err = Errorf("crypto: missing cert file")
@@ -101,7 +101,7 @@ func (k KesConfig) Verify() (err error) {
 type kesService struct {
 	client *kesClient
 
-	endpoint     string
+	endpoints    []string
 	defaultKeyID string
 }
 
@@ -141,12 +141,12 @@ func NewKes(cfg KesConfig) (KMS, error) {
 
 	return &kesService{
 		client: &kesClient{
-			addr: cfg.Endpoint,
+			endpoints: cfg.Endpoint,
 			httpClient: http.Client{
 				Transport: cfg.Transport,
 			},
 		},
-		endpoint:     cfg.Endpoint,
+		endpoints:    cfg.Endpoint,
 		defaultKeyID: cfg.DefaultKeyID,
 	}, nil
 }
@@ -163,9 +163,9 @@ func (kes *kesService) DefaultKeyID() string {
 // method.
 func (kes *kesService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: kes.endpoint,
-		Name:     kes.DefaultKeyID(),
-		AuthType: "TLS",
+		Endpoints: kes.endpoints,
+		Name:      kes.DefaultKeyID(),
+		AuthType:  "TLS",
 	}
 }
 
@@ -221,7 +221,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 //   • GenerateDataKey (API: /v1/key/generate/)
 //   • DecryptDataKey  (API: /v1/key/decrypt/)
 type kesClient struct {
-	addr       string
+	endpoints  []string
 	httpClient http.Client
 }
 
@@ -232,8 +232,8 @@ type kesClient struct {
 // application does not have the cryptographic key at
 // any point in time.
 func (c *kesClient) CreateKey(name string) error {
-	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
-	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
+	path := fmt.Sprintf("/v1/key/create/%s", url.PathEscape(name))
+	_, err := c.postRetry(path, nil, 0) // No request body and no response expected
 	if err != nil {
 		return err
 	}
@@ -265,8 +265,8 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
 	}
 
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/generate/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -302,8 +302,8 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b
 	}
 
 	const limit = 1 << 20 // A data key will never be larger than 1 MiB
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/decrypt/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, err
 	}
@@ -402,12 +402,14 @@ func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, er
 	return &respBody, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+	retryMax := 1 + len(c.endpoints)
 	for i := 0; ; i++ {
 		if body != nil {
 			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
 		}
-		response, err := c.post(url, body, limit)
+
+		response, err := c.post(c.endpoints[i%len(c.endpoints)]+path, body, limit)
 		if err == nil {
 			return response, nil
 		}

cmd/crypto/kms.go

@@ -109,9 +109,9 @@ type masterKeyKMS struct {
 // KMSInfo contains some describing information about
 // the KMS.
 type KMSInfo struct {
-	Endpoint string
-	Name     string
-	AuthType string
+	Endpoints []string
+	Name      string
+	AuthType  string
 }
 
 // NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
@@ -147,9 +147,9 @@ func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, s
 // KMS is configured directly using master key
 func (kms *masterKeyKMS) Info() (info KMSInfo) {
 	return KMSInfo{
-		Endpoint: "",
-		Name:     "",
-		AuthType: "master-key",
+		Endpoints: []string{},
+		Name:      "",
+		AuthType:  "master-key",
 	}
 }
 

cmd/crypto/retry.go

@@ -23,7 +23,6 @@ import (
 const (
 	retryWaitMin = 500 * time.Millisecond // minimum retry limit.
 	retryWaitMax = 3 * time.Second        // 3 secs worth of max retry.
-	retryMax     = 2
 )
 
 // LinearJitterBackoff provides the time.Duration for a caller to

cmd/crypto/vault.go

@@ -199,13 +199,13 @@ func (v *vaultService) DefaultKeyID() string {
 }
 
 // Info returns some information about the Vault,
-// configuration - like the endpoint or authentication
+// configuration - like the endpoints or authentication
 // method.
 func (v *vaultService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: v.config.Endpoint,
-		Name:     v.DefaultKeyID(),
-		AuthType: v.config.Auth.Type,
+		Endpoints: []string{v.config.Endpoint},
+		Name:      v.DefaultKeyID(),
+		AuthType:  v.config.Auth.Type,
 	}
 }