From e047ac52b852c9ff4b8465b282cf2230b35d599b Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Thu, 19 Dec 2019 00:10:57 +0100 Subject: [PATCH] remove github.com/minio/kes as a dependency (#8665) This commit removes github.com/minio/kes as a dependency and implements the necessary client-side functionality without relying on the KES project. This resolves the licensing issue since KES is licensed under AGPL while MinIO is licensed under Apache. --- cmd/config-current.go | 4 +- cmd/crypto/config.go | 4 +- cmd/crypto/kes.go | 131 +++++++++++++++++++++++++++++++++++++++--- go.mod | 1 - go.sum | 4 -- 5 files changed, 129 insertions(+), 15 deletions(-) diff --git a/cmd/config-current.go b/cmd/config-current.go index c9d13dadf..1d4b48c7b 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -251,7 +251,7 @@ func validateConfig(s config.Config) error { } } { - kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get()) + kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport()) if err != nil { return err } @@ -355,7 +355,7 @@ func lookupConfigs(s config.Config) { } } - kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get()) + kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport()) if err != nil { logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err)) } diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go index 1c6b3003a..6642aeaba 100644 --- a/cmd/crypto/config.go +++ b/cmd/crypto/config.go @@ -17,6 +17,7 @@ package crypto import ( "errors" "fmt" + "net/http" "reflect" "strconv" @@ -259,7 +260,7 @@ func lookupAutoEncryption() (bool, error) { // LookupConfig lookup vault or kes config, returns KMSConfig // to configure KMS object for object encryption -func LookupConfig(c config.Config, defaultRootCAsDir string) (KMSConfig, error) { +func LookupConfig(c config.Config, defaultRootCAsDir string, transport *http.Transport) (KMSConfig, error) { vcfg, err := LookupVaultConfig(c[config.KmsVaultSubSys][config.Default]) if err != nil { return KMSConfig{}, err @@ -268,6 +269,7 @@ func LookupConfig(c config.Config, defaultRootCAsDir string) (KMSConfig, error) if err != nil { return KMSConfig{}, err } + kesCfg.Transport = transport if kesCfg.Enabled && kesCfg.CAPath == "" { kesCfg.CAPath = defaultRootCAsDir } diff --git a/cmd/crypto/kes.go b/cmd/crypto/kes.go index 74aa9ef35..85c46300f 100644 --- a/cmd/crypto/kes.go +++ b/cmd/crypto/kes.go @@ -18,13 +18,16 @@ import ( "bytes" "crypto/tls" "crypto/x509" + "encoding/json" "errors" "fmt" + "io" "io/ioutil" + "net/http" + "net/url" "os" "path/filepath" - - "github.com/minio/kes" + "strings" ) // KesConfig contains the configuration required @@ -63,6 +66,10 @@ type KesConfig struct { // The default key ID returned by KMS.KeyID(). DefaultKeyID string + + // The HTTP transport configuration for + // the KES client. + Transport *http.Transport } // Verify verifies if the kes configuration is correct @@ -81,7 +88,7 @@ func (k KesConfig) Verify() (err error) { } type kesService struct { - client *kes.Client + client *kesClient endpoint string defaultKeyID string @@ -102,11 +109,17 @@ func NewKes(cfg KesConfig) (KMS, error) { if err != nil { return nil, err } + cfg.Transport.TLSClientConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: certPool, + } return &kesService{ - client: kes.NewClient(cfg.Endpoint, &tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: certPool, - }), + client: &kesClient{ + addr: cfg.Endpoint, + httpClient: http.Client{ + Transport: cfg.Transport, + }, + }, endpoint: cfg.Endpoint, defaultKeyID: cfg.DefaultKeyID, }, nil @@ -187,6 +200,110 @@ func (kes *kesService) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([ return sealedKey, nil } +// kesClient implements the bare minimum functionality needed for +// MinIO to talk to a KES server. In particular, it implements +// GenerateDataKey (API: /v1/key/generate/) and +// DecryptDataKey (API: /v1/key/decrypt/). +type kesClient struct { + addr string + httpClient http.Client +} + +// GenerateDataKey requests a new data key from the KES server. +// On success, the KES server will respond with the plaintext key +// and the ciphertext key as the plaintext key encrypted with +// the key specified by name. +// +// The optional context is crytpo. bound to the generated data key +// such that you have to provide the same context when decrypting +// the data key. +func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) { + type Request struct { + Context []byte `json:"context"` + } + body, err := json.Marshal(Request{ + Context: context, + }) + if err != nil { + return nil, nil, err + } + + url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name)) + resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, nil, c.parseErrorResponse(resp) + } + defer resp.Body.Close() + + type Response struct { + Plaintext []byte `json:"plaintext"` + Ciphertext []byte `json:"ciphertext"` + } + const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB + var response Response + if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil { + return nil, nil, err + } + return response.Plaintext, response.Ciphertext, nil +} + +// GenerateDataKey decrypts an encrypted data key with the key +// specified by name by talking to the KES server. +// On success, the KES server will respond with the plaintext key. +// +// The optional context must match the value you provided when +// generating the data key. +func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) { + type Request struct { + Ciphertext []byte `json:"ciphertext"` + Context []byte `json:"context"` + } + body, err := json.Marshal(Request{ + Ciphertext: ciphertext, + Context: context, + }) + if err != nil { + return nil, err + } + + url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name)) + resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, c.parseErrorResponse(resp) + } + defer resp.Body.Close() + + type Response struct { + Plaintext []byte `json:"plaintext"` + } + const limit = 32 * 1024 // A data key will never be larger than 32 KB + var response Response + if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil { + return nil, err + } + return response.Plaintext, nil +} + +func (c *kesClient) parseErrorResponse(resp *http.Response) error { + if resp.Body == nil { + return nil + } + defer resp.Body.Close() + + const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB + var errMsg strings.Builder + if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil { + return err + } + return fmt.Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String()) +} + // loadCACertificates returns a new CertPool // that contains all system root CA certificates // and any PEM-encoded certificate(s) found at diff --git a/go.mod b/go.mod index 603bce89e..0ff2752f1 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( github.com/minio/gokrb5/v7 v7.2.5 github.com/minio/hdfs/v3 v3.0.1 github.com/minio/highwayhash v1.0.0 - github.com/minio/kes v0.4.0 github.com/minio/lsync v1.0.1 github.com/minio/mc v0.0.0-20191012041914-735aa139b19c github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f diff --git a/go.sum b/go.sum index b4aafb2cc..1c7413c53 100644 --- a/go.sum +++ b/go.sum @@ -563,8 +563,6 @@ github.com/minio/hdfs/v3 v3.0.1/go.mod h1:6ALh9HsAwG9xAXdpdrZJcSY0vR6z3K+9XIz6Y9 github.com/minio/highwayhash v0.0.0-20181220011308-93ed73d64169/go.mod h1:NL8wme5P5MoscwAkXfGroz3VgpCdhBw3KYOu5mEsvpU= github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA= github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= -github.com/minio/kes v0.4.0 h1:N96zVEH/RnY4XfGf/gVz+j8DRyrYWDM2f24lF/K3LhA= -github.com/minio/kes v0.4.0/go.mod h1:Y1hLsM+dhq0azrJ+7nxG1QaD8gliTUthFnvOk99FasQ= github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887/go.mod h1:ni10+iSX7FO8N2rv41XM444V6w4rYO0dZo5KIkbn/YA= github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs= github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA= @@ -702,8 +700,6 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= -github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw=