kes: try to auto. create master key if not present (#9790)

This commit changes the data key generation such that
if a MinIO server/nodes tries to generate a new DEK
but the particular master key does not exist - then
MinIO asks KES to create a new master key and then
requests the DEK again.

From now on, a SSE-S3 master key must not be created
explicitly via: `kes key create <key-name>`.
Instead, it is sufficient to just set the env. var.
```
export MINIO_KMS_KES_KEY_NAME=<key-name>
```

However, the MinIO identity (mTLS client certificate)
must have the permission to access the `/v1/key/create/`
API. Therefore, KES policy for MinIO must look similar to:
```
[
  /v1/key/create/<key-name-pattern>
  /v1/key/generate/<key-name-pattern>
  /v1/key/decrypt/<key-name-pattern>
]
```
However, in our guides we already suggest that.
See e.g.: https://github.com/minio/kes/wiki/MinIO-Object-Storage#kes-server-setup

***

The ability to create master keys on request may also be
necessary / useful in case of SSE-KMS.

cmd/crypto/kes.go

@@ -34,6 +34,10 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+// ErrKESKeyNotFound is the error returned a KES server
+// when a master key does not exist.
+var ErrKESKeyNotFound = NewKESError(http.StatusNotFound, "key does not exist")
+
 // KesConfig contains the configuration required
 // to initialize and connect to a kes server.
 type KesConfig struct {
@@ -154,6 +158,12 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 
 	var plainKey []byte
 	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	if err == ErrKESKeyNotFound { // Try to create the key if it does not exist.
+		if err = kes.client.CreateKey(keyID); err != nil {
+			return key, nil, err
+		}
+		plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	}
 	if err != nil {
 		return key, nil, err
 	}
@@ -214,16 +224,19 @@ type kesClient struct {
 	httpClient http.Client
 }
 
-// Response KES response struct
+// CreateKey tries to create a new cryptographic key with
-type response struct {
+// the specified name.
-	Plaintext  []byte `json:"plaintext"`
+//
-	Ciphertext []byte `json:"ciphertext,omitempty"`
+// The key will be generated by the server. The client
-}
+// application does not have the cryptographic key at
-
+// any point in time.
-// Request KES request struct
+func (c *kesClient) CreateKey(name string) error {
-type request struct {
+	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
-	Ciphertext []byte `json:"ciphertext,omitempty"`
+	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
-	Context    []byte `json:"context"`
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // GenerateDataKey requests a new data key from the KES server.
@@ -235,104 +248,172 @@ type request struct {
 // such that you have to provide the same context when decrypting
 // the data key.
 func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) {
-	body, err := json.Marshal(request{
+	type Request struct {
+		Context []byte `json:"context"`
+	}
+	type Response struct {
+		Plaintext  []byte `json:"plaintext"`
+		Ciphertext []byte `json:"ciphertext"`
+	}
+
+	body, err := json.Marshal(Request{
 		Context: context,
 	})
 	if err != nil {
 		return nil, nil, err
 	}
 
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
+	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
 	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	return resp.Plaintext, resp.Ciphertext, nil
+	var response Response
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
+		return nil, nil, err
+	}
+	return response.Plaintext, response.Ciphertext, nil
 }
 
-func (c *kesClient) post(url string, body io.Reader, limit int64) (*response, error) {
+// GenerateDataKey decrypts an encrypted data key with the key
-	resp, err := c.httpClient.Post(url, "application/json", body)
+// specified by name by talking to the KES server.
+// On success, the KES server will respond with the plaintext key.
+//
+// The optional context must match the value you provided when
+// generating the data key.
+func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
+	type Request struct {
+		Ciphertext []byte `json:"ciphertext"`
+		Context    []byte `json:"context,omitempty"`
+	}
+	type Response struct {
+		Plaintext []byte `json:"plaintext"`
+	}
+
+	body, err := json.Marshal(Request{
+		Ciphertext: ciphertext,
+		Context:    context,
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	// Drain the entire body to make sure we have re-use connections
+	const limit = 1 << 20 // A data key will never be larger than 1 MiB
-	defer xhttp.DrainBody(resp.Body)
+	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-
+	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
-	if resp.StatusCode != http.StatusOK {
+	if err != nil {
-		return nil, c.parseErrorResponse(resp)
+		return nil, err
 	}
 
-	response := &response{}
+	var response Response
-	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(response); err != nil {
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
 		return nil, err
 	}
-	return response, nil
+	return response.Plaintext, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*response, error) {
+// NewKESError returns a new KES API error with the given
-	for i := 0; ; i++ {
+// HTTP status code and error message.
-		body.Seek(0, io.SeekStart) // seek to the beginning of the body.
+//
+// Two errors with the same status code and
+// error message are equal:
+//   e1 == e2 // true.
+func NewKESError(code int, text string) error {
+	return kesError{
+		code:    code,
+		message: text,
+	}
+}
 
-		response, err := c.post(url, body, limit)
+type kesError struct {
-		if err == nil {
+	code    int
-			return response, nil
+	message string
+}
+
+// Status returns the HTTP status code of the error.
+func (e kesError) Status() int { return e.code }
+
+// Status returns the error message of the error.
+func (e kesError) Error() string { return e.message }
+
+func parseErrorResponse(resp *http.Response) error {
+	if resp == nil || resp.StatusCode < 400 {
+		return nil
+	}
+	if resp.Body == nil {
+		return NewKESError(resp.StatusCode, "")
 	}
+	defer resp.Body.Close()
 
-		if !xnet.IsNetworkOrHostDown(err) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+	const MaxBodySize = 1 << 20
-			return nil, err
+	var size = resp.ContentLength
+	if size < 0 || size > MaxBodySize {
+		size = MaxBodySize
 	}
 
-		// retriable network errors.
+	contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
-		remain := retryMax - i
+	if strings.HasPrefix(contentType, "application/json") {
-		if remain <= 0 {
+		type Response struct {
-			return response, err
+			Message string `json:"message"`
+		}
+		var response Response
+		if err := json.NewDecoder(io.LimitReader(resp.Body, size)).Decode(&response); err != nil {
+			return err
+		}
+		return NewKESError(resp.StatusCode, response.Message)
 	}
 
-		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
+	var sb strings.Builder
+	if _, err := io.Copy(&sb, io.LimitReader(resp.Body, size)); err != nil {
+		return err
 	}
+	return NewKESError(resp.StatusCode, sb.String())
 }
 
-// GenerateDataKey decrypts an encrypted data key with the key
+func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, error) {
-// specified by name by talking to the KES server.
+	resp, err := c.httpClient.Post(url, "application/json", body)
-// On success, the KES server will respond with the plaintext key.
-//
-// The optional context must match the value you provided when
-// generating the data key.
-func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
-	body, err := json.Marshal(request{
-		Ciphertext: ciphertext,
-		Context:    context,
-	})
 	if err != nil {
 		return nil, err
 	}
+	// Drain the entire body to make sure we have re-use connections
+	defer xhttp.DrainBody(resp.Body)
 
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
+	if resp.StatusCode != http.StatusOK {
+		return nil, parseErrorResponse(resp)
+	}
 
-	const limit = 32 * 1024 // A data key will never be larger than 32 KB
+	// We have to copy the response body due to draining.
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	var respBody bytes.Buffer
-	if err != nil {
+	if _, err = io.Copy(&respBody, io.LimitReader(resp.Body, limit)); err != nil {
 		return nil, err
 	}
-	return resp.Plaintext, nil
+	return &respBody, nil
 }
 
-func (c *kesClient) parseErrorResponse(resp *http.Response) error {
+func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
-	if resp.Body == nil {
+	for i := 0; ; i++ {
-		return Errorf("%s: no body", http.StatusText(resp.StatusCode))
+		if body != nil {
+			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
+		}
+		response, err := c.post(url, body, limit)
+		if err == nil {
+			return response, nil
 		}
 
-	const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB
+		if !xnet.IsNetworkOrHostDown(err) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-	var errMsg strings.Builder
+			return nil, err
-	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
-		return Errorf("%s: %s", http.StatusText(resp.StatusCode), err)
 		}
 
-	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
+		// retriable network errors.
+		remain := retryMax - i
+		if remain <= 0 {
+			return response, err
+		}
+
+		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
+	}
 }
 
 // loadCACertificates returns a new CertPool