From 18725679c4f581683362ed814c98baa1fb8c53c0 Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Tue, 1 Sep 2020 03:10:52 +0200 Subject: [PATCH] crypto: allow multiple KES endpoints (#10383) This commit addresses a maintenance / automation problem when MinIO-KES is deployed on bare-metal. In orchestrated env. the orchestrator (K8S) will make sure that `n` KES servers (IPs) are available via the same DNS name. There it is sufficient to provide just one endpoint. --- cmd/admin-handlers.go | 4 ++-- cmd/crypto/config.go | 30 ++++++++++++++++++++++++------ cmd/crypto/kes.go | 38 ++++++++++++++++++++------------------ cmd/crypto/kms.go | 12 ++++++------ cmd/crypto/retry.go | 1 - cmd/crypto/vault.go | 8 ++++---- 6 files changed, 56 insertions(+), 37 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index f78d8c6ec..e9dab2125 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1582,12 +1582,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault { keyID := GlobalKMS.DefaultKeyID() kmsInfo := GlobalKMS.Info() - if kmsInfo.Endpoint == "" { + if len(kmsInfo.Endpoints) == 0 { vault.Status = "KMS configured using master key" return vault } - if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil { + if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil { vault.Status = "offline" } else { vault.Status = "online" diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go index 81a79743c..5761c254d 100644 --- a/cmd/crypto/config.go +++ b/cmd/crypto/config.go @@ -16,11 +16,14 @@ package crypto import ( "errors" + "math/rand" "net/http" "reflect" "strconv" + "strings" "github.com/minio/minio/cmd/config" + "github.com/minio/minio/pkg/ellipses" "github.com/minio/minio/pkg/env" xnet "github.com/minio/minio/pkg/net" ) @@ -167,7 +170,8 @@ const ( const ( // EnvKMSKesEndpoint is the environment variable used to specify - // the kes server HTTPS endpoint. + // one or multiple KES server HTTPS endpoints. The individual + // endpoints should be separated by ','. EnvKMSKesEndpoint = "MINIO_KMS_KES_ENDPOINT" // EnvKMSKesKeyFile is the environment variable used to specify @@ -216,16 +220,30 @@ func LookupKesConfig(kvs config.KVS) (KesConfig, error) { kesCfg := KesConfig{} endpointStr := env.Get(EnvKMSKesEndpoint, kvs.Get(KMSKesEndpoint)) - if endpointStr != "" { - // Lookup kes configuration & overwrite config entry if ENV var is present - endpoint, err := xnet.ParseHTTPURL(endpointStr) + var endpoints []string + for _, endpoint := range strings.Split(endpointStr, ",") { + if !ellipses.HasEllipses(endpoint) { + endpoints = append(endpoints, endpoint) + continue + } + pattern, err := ellipses.FindEllipsesPatterns(endpoint) if err != nil { return kesCfg, err } - endpointStr = endpoint.String() + for _, p := range pattern { + endpoints = append(endpoints, p.Expand()...) + } } - kesCfg.Endpoint = endpointStr + randNum := rand.Intn(len(endpoints) + 1) // We add 1 b/c len(endpoints) may be 0: See: rand.Intn docs + kesCfg.Endpoint = make([]string, len(endpoints)) + for i, endpoint := range endpoints { + endpoint, err := xnet.ParseHTTPURL(endpoint) + if err != nil { + return kesCfg, err + } + kesCfg.Endpoint[(randNum+i)%len(endpoints)] = endpoint.String() + } kesCfg.KeyFile = env.Get(EnvKMSKesKeyFile, kvs.Get(KMSKesKeyFile)) kesCfg.CertFile = env.Get(EnvKMSKesCertFile, kvs.Get(KMSKesCertFile)) kesCfg.CAPath = env.Get(EnvKMSKesCAPath, kvs.Get(KMSKesCAPath)) diff --git a/cmd/crypto/kes.go b/cmd/crypto/kes.go index 0d6323173..0a65be08b 100644 --- a/cmd/crypto/kes.go +++ b/cmd/crypto/kes.go @@ -46,8 +46,8 @@ var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist type KesConfig struct { Enabled bool - // The kes server endpoint. - Endpoint string + // The KES server endpoints. + Endpoint []string // The path to the TLS private key used // by MinIO to authenticate to the kes @@ -86,7 +86,7 @@ type KesConfig struct { // Verify verifies if the kes configuration is correct func (k KesConfig) Verify() (err error) { switch { - case k.Endpoint == "": + case len(k.Endpoint) == 0: err = Errorf("crypto: missing kes endpoint") case k.CertFile == "": err = Errorf("crypto: missing cert file") @@ -101,7 +101,7 @@ func (k KesConfig) Verify() (err error) { type kesService struct { client *kesClient - endpoint string + endpoints []string defaultKeyID string } @@ -141,12 +141,12 @@ func NewKes(cfg KesConfig) (KMS, error) { return &kesService{ client: &kesClient{ - addr: cfg.Endpoint, + endpoints: cfg.Endpoint, httpClient: http.Client{ Transport: cfg.Transport, }, }, - endpoint: cfg.Endpoint, + endpoints: cfg.Endpoint, defaultKeyID: cfg.DefaultKeyID, }, nil } @@ -163,9 +163,9 @@ func (kes *kesService) DefaultKeyID() string { // method. func (kes *kesService) Info() KMSInfo { return KMSInfo{ - Endpoint: kes.endpoint, - Name: kes.DefaultKeyID(), - AuthType: "TLS", + Endpoints: kes.endpoints, + Name: kes.DefaultKeyID(), + AuthType: "TLS", } } @@ -221,7 +221,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k // • GenerateDataKey (API: /v1/key/generate/) // • DecryptDataKey (API: /v1/key/decrypt/) type kesClient struct { - addr string + endpoints []string httpClient http.Client } @@ -232,8 +232,8 @@ type kesClient struct { // application does not have the cryptographic key at // any point in time. func (c *kesClient) CreateKey(name string) error { - url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name)) - _, err := c.postRetry(url, nil, 0) // No request body and no response expected + path := fmt.Sprintf("/v1/key/create/%s", url.PathEscape(name)) + _, err := c.postRetry(path, nil, 0) // No request body and no response expected if err != nil { return err } @@ -265,8 +265,8 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte } const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB - url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name)) - resp, err := c.postRetry(url, bytes.NewReader(body), limit) + path := fmt.Sprintf("/v1/key/generate/%s", url.PathEscape(name)) + resp, err := c.postRetry(path, bytes.NewReader(body), limit) if err != nil { return nil, nil, err } @@ -302,8 +302,8 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b } const limit = 1 << 20 // A data key will never be larger than 1 MiB - url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name)) - resp, err := c.postRetry(url, bytes.NewReader(body), limit) + path := fmt.Sprintf("/v1/key/decrypt/%s", url.PathEscape(name)) + resp, err := c.postRetry(path, bytes.NewReader(body), limit) if err != nil { return nil, err } @@ -402,12 +402,14 @@ func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, er return &respBody, nil } -func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) { +func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.Reader, error) { + retryMax := 1 + len(c.endpoints) for i := 0; ; i++ { if body != nil { body.Seek(0, io.SeekStart) // seek to the beginning of the body. } - response, err := c.post(url, body, limit) + + response, err := c.post(c.endpoints[i%len(c.endpoints)]+path, body, limit) if err == nil { return response, nil } diff --git a/cmd/crypto/kms.go b/cmd/crypto/kms.go index 8db9a8dd4..b52e6e4f0 100644 --- a/cmd/crypto/kms.go +++ b/cmd/crypto/kms.go @@ -109,9 +109,9 @@ type masterKeyKMS struct { // KMSInfo contains some describing information about // the KMS. type KMSInfo struct { - Endpoint string - Name string - AuthType string + Endpoints []string + Name string + AuthType string } // NewMasterKey returns a basic KMS implementation from a single 256 bit master key. @@ -147,9 +147,9 @@ func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, s // KMS is configured directly using master key func (kms *masterKeyKMS) Info() (info KMSInfo) { return KMSInfo{ - Endpoint: "", - Name: "", - AuthType: "master-key", + Endpoints: []string{}, + Name: "", + AuthType: "master-key", } } diff --git a/cmd/crypto/retry.go b/cmd/crypto/retry.go index b03116586..e5b8ca373 100644 --- a/cmd/crypto/retry.go +++ b/cmd/crypto/retry.go @@ -23,7 +23,6 @@ import ( const ( retryWaitMin = 500 * time.Millisecond // minimum retry limit. retryWaitMax = 3 * time.Second // 3 secs worth of max retry. - retryMax = 2 ) // LinearJitterBackoff provides the time.Duration for a caller to diff --git a/cmd/crypto/vault.go b/cmd/crypto/vault.go index 8674a6179..007623daa 100644 --- a/cmd/crypto/vault.go +++ b/cmd/crypto/vault.go @@ -199,13 +199,13 @@ func (v *vaultService) DefaultKeyID() string { } // Info returns some information about the Vault, -// configuration - like the endpoint or authentication +// configuration - like the endpoints or authentication // method. func (v *vaultService) Info() KMSInfo { return KMSInfo{ - Endpoint: v.config.Endpoint, - Name: v.DefaultKeyID(), - AuthType: v.config.Auth.Type, + Endpoints: []string{v.config.Endpoint}, + Name: v.DefaultKeyID(), + AuthType: v.config.Auth.Type, } }