crypto: allow multiple KES endpoints (#10383)

This commit addresses a maintenance / automation problem when MinIO-KES
is deployed on bare-metal. In orchestrated env. the orchestrator (K8S)
will make sure that `n` KES servers (IPs) are available via the same DNS
name. There it is sufficient to provide just one endpoint.
master
Andreas Auernhammer 4 years ago committed by GitHub
parent ba8a8ad818
commit 18725679c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      cmd/admin-handlers.go
  2. 30
      cmd/crypto/config.go
  3. 34
      cmd/crypto/kes.go
  4. 4
      cmd/crypto/kms.go
  5. 1
      cmd/crypto/retry.go
  6. 4
      cmd/crypto/vault.go

@ -1582,12 +1582,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
keyID := GlobalKMS.DefaultKeyID()
kmsInfo := GlobalKMS.Info()
if kmsInfo.Endpoint == "" {
if len(kmsInfo.Endpoints) == 0 {
vault.Status = "KMS configured using master key"
return vault
}
if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
vault.Status = "offline"
} else {
vault.Status = "online"

@ -16,11 +16,14 @@ package crypto
import (
"errors"
"math/rand"
"net/http"
"reflect"
"strconv"
"strings"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/ellipses"
"github.com/minio/minio/pkg/env"
xnet "github.com/minio/minio/pkg/net"
)
@ -167,7 +170,8 @@ const (
const (
// EnvKMSKesEndpoint is the environment variable used to specify
// the kes server HTTPS endpoint.
// one or multiple KES server HTTPS endpoints. The individual
// endpoints should be separated by ','.
EnvKMSKesEndpoint = "MINIO_KMS_KES_ENDPOINT"
// EnvKMSKesKeyFile is the environment variable used to specify
@ -216,16 +220,30 @@ func LookupKesConfig(kvs config.KVS) (KesConfig, error) {
kesCfg := KesConfig{}
endpointStr := env.Get(EnvKMSKesEndpoint, kvs.Get(KMSKesEndpoint))
if endpointStr != "" {
// Lookup kes configuration & overwrite config entry if ENV var is present
endpoint, err := xnet.ParseHTTPURL(endpointStr)
var endpoints []string
for _, endpoint := range strings.Split(endpointStr, ",") {
if !ellipses.HasEllipses(endpoint) {
endpoints = append(endpoints, endpoint)
continue
}
pattern, err := ellipses.FindEllipsesPatterns(endpoint)
if err != nil {
return kesCfg, err
}
endpointStr = endpoint.String()
for _, p := range pattern {
endpoints = append(endpoints, p.Expand()...)
}
}
kesCfg.Endpoint = endpointStr
randNum := rand.Intn(len(endpoints) + 1) // We add 1 b/c len(endpoints) may be 0: See: rand.Intn docs
kesCfg.Endpoint = make([]string, len(endpoints))
for i, endpoint := range endpoints {
endpoint, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
return kesCfg, err
}
kesCfg.Endpoint[(randNum+i)%len(endpoints)] = endpoint.String()
}
kesCfg.KeyFile = env.Get(EnvKMSKesKeyFile, kvs.Get(KMSKesKeyFile))
kesCfg.CertFile = env.Get(EnvKMSKesCertFile, kvs.Get(KMSKesCertFile))
kesCfg.CAPath = env.Get(EnvKMSKesCAPath, kvs.Get(KMSKesCAPath))

@ -46,8 +46,8 @@ var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist
type KesConfig struct {
Enabled bool
// The kes server endpoint.
Endpoint string
// The KES server endpoints.
Endpoint []string
// The path to the TLS private key used
// by MinIO to authenticate to the kes
@ -86,7 +86,7 @@ type KesConfig struct {
// Verify verifies if the kes configuration is correct
func (k KesConfig) Verify() (err error) {
switch {
case k.Endpoint == "":
case len(k.Endpoint) == 0:
err = Errorf("crypto: missing kes endpoint")
case k.CertFile == "":
err = Errorf("crypto: missing cert file")
@ -101,7 +101,7 @@ func (k KesConfig) Verify() (err error) {
type kesService struct {
client *kesClient
endpoint string
endpoints []string
defaultKeyID string
}
@ -141,12 +141,12 @@ func NewKes(cfg KesConfig) (KMS, error) {
return &kesService{
client: &kesClient{
addr: cfg.Endpoint,
endpoints: cfg.Endpoint,
httpClient: http.Client{
Transport: cfg.Transport,
},
},
endpoint: cfg.Endpoint,
endpoints: cfg.Endpoint,
defaultKeyID: cfg.DefaultKeyID,
}, nil
}
@ -163,7 +163,7 @@ func (kes *kesService) DefaultKeyID() string {
// method.
func (kes *kesService) Info() KMSInfo {
return KMSInfo{
Endpoint: kes.endpoint,
Endpoints: kes.endpoints,
Name: kes.DefaultKeyID(),
AuthType: "TLS",
}
@ -221,7 +221,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
// • GenerateDataKey (API: /v1/key/generate/)
// • DecryptDataKey (API: /v1/key/decrypt/)
type kesClient struct {
addr string
endpoints []string
httpClient http.Client
}
@ -232,8 +232,8 @@ type kesClient struct {
// application does not have the cryptographic key at
// any point in time.
func (c *kesClient) CreateKey(name string) error {
url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
_, err := c.postRetry(url, nil, 0) // No request body and no response expected
path := fmt.Sprintf("/v1/key/create/%s", url.PathEscape(name))
_, err := c.postRetry(path, nil, 0) // No request body and no response expected
if err != nil {
return err
}
@ -265,8 +265,8 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
}
const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
resp, err := c.postRetry(url, bytes.NewReader(body), limit)
path := fmt.Sprintf("/v1/key/generate/%s", url.PathEscape(name))
resp, err := c.postRetry(path, bytes.NewReader(body), limit)
if err != nil {
return nil, nil, err
}
@ -302,8 +302,8 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b
}
const limit = 1 << 20 // A data key will never be larger than 1 MiB
url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
resp, err := c.postRetry(url, bytes.NewReader(body), limit)
path := fmt.Sprintf("/v1/key/decrypt/%s", url.PathEscape(name))
resp, err := c.postRetry(path, bytes.NewReader(body), limit)
if err != nil {
return nil, err
}
@ -402,12 +402,14 @@ func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, er
return &respBody, nil
}
func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.Reader, error) {
retryMax := 1 + len(c.endpoints)
for i := 0; ; i++ {
if body != nil {
body.Seek(0, io.SeekStart) // seek to the beginning of the body.
}
response, err := c.post(url, body, limit)
response, err := c.post(c.endpoints[i%len(c.endpoints)]+path, body, limit)
if err == nil {
return response, nil
}

@ -109,7 +109,7 @@ type masterKeyKMS struct {
// KMSInfo contains some describing information about
// the KMS.
type KMSInfo struct {
Endpoint string
Endpoints []string
Name string
AuthType string
}
@ -147,7 +147,7 @@ func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, s
// KMS is configured directly using master key
func (kms *masterKeyKMS) Info() (info KMSInfo) {
return KMSInfo{
Endpoint: "",
Endpoints: []string{},
Name: "",
AuthType: "master-key",
}

@ -23,7 +23,6 @@ import (
const (
retryWaitMin = 500 * time.Millisecond // minimum retry limit.
retryWaitMax = 3 * time.Second // 3 secs worth of max retry.
retryMax = 2
)
// LinearJitterBackoff provides the time.Duration for a caller to

@ -199,11 +199,11 @@ func (v *vaultService) DefaultKeyID() string {
}
// Info returns some information about the Vault,
// configuration - like the endpoint or authentication
// configuration - like the endpoints or authentication
// method.
func (v *vaultService) Info() KMSInfo {
return KMSInfo{
Endpoint: v.config.Endpoint,
Endpoints: []string{v.config.Endpoint},
Name: v.DefaultKeyID(),
AuthType: v.config.Auth.Type,
}

Loading…
Cancel
Save