diff --git a/cmd/admin-rpc_test.go b/cmd/admin-rpc_test.go index 34f793ff4..1b86a0256 100644 --- a/cmd/admin-rpc_test.go +++ b/cmd/admin-rpc_test.go @@ -40,11 +40,8 @@ import ( /////////////////////////////////////////////////////////////////////////////// func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) { - tmpGlobalServiceSignalCh := globalServiceSignalCh + defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh) globalServiceSignalCh = make(chan serviceSignal, 10) - defer func() { - globalServiceSignalCh = tmpGlobalServiceSignalCh - }() testCases := []struct { signal serviceSignal diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index ff751e3b6..254c2e5b1 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -619,6 +619,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h rawReader := hashReader pReader := NewPutObjReader(rawReader, nil, nil) var objectEncryptionKey []byte + + if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { + r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + } if objectAPI.IsEncryptionSupported() { if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests var reader io.Reader diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go index b16e5037d..be5b70de2 100644 --- a/cmd/crypto/config.go +++ b/cmd/crypto/config.go @@ -16,5 +16,6 @@ package crypto // KMSConfig has the KMS config for hashicorp vault type KMSConfig struct { - Vault VaultConfig `json:"vault"` + AutoEncryption bool `json:"-"` + Vault VaultConfig `json:"vault"` } diff --git a/cmd/environment.go b/cmd/environment.go index e41abdcd6..293873a91 100644 --- a/cmd/environment.go +++ b/cmd/environment.go @@ -30,6 +30,13 @@ const ( // a KMS master key used to protect SSE-S3 per-object keys. // Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE". EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY" + + // EnvAutoEncryption is the environment variable used to en/disable + // SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled, + // requires a valid KMS configuration and turns any non-SSE-C + // request into an SSE-S3 request. + // If present EnvAutoEncryption must be either "on" or "off". + EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION" ) const ( @@ -141,6 +148,15 @@ func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) { } globalKMSKeyID = config.Vault.Key.Name } + + autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off")) + if err != nil { + return err + } + globalAutoEncryption = bool(autoEncryption) + if globalAutoEncryption && globalKMS == nil { // auto-encryption enabled but no KMS + return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present") + } return nil } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index f294124b3..e83bda226 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -18,6 +18,7 @@ package cmd import ( "context" + "errors" "fmt" "net/url" "os" @@ -222,7 +223,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) { globalHTTPServer.Shutdown() logger.FatalIf(err, "Unable to initialize gateway backend") } - // Create a new config system. globalConfigSys = NewConfigSys() if globalEtcdClient != nil && gatewayName == "nas" { @@ -279,6 +279,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) { if globalEtcdClient != nil && newObject.IsNotificationSupported() { _ = globalNotificationSys.Init(newObject) } + + if globalAutoEncryption && !newObject.IsEncryptionSupported() { + logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but gateway does not support encryption") + } + // Once endpoints are finalized, initialize the new object api. globalObjLayerMutex.Lock() globalObjectAPI = newObject diff --git a/cmd/globals.go b/cmd/globals.go index de9b0d057..3f6e2c03a 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -231,6 +231,11 @@ var ( // Allocated KMS globalKMS crypto.KMS + // Auto-Encryption, if enabled, turns any non-SSE-C request + // into an SSE-S3 request. If enabled a valid, non-empty KMS + // configuration must be present. + globalAutoEncryption bool + // Is compression include extensions/content-types set. globalIsEnvCompression bool diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 7a101a63a..6b6646fb6 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -857,6 +857,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re rawReader := srcInfo.Reader pReader := NewPutObjReader(srcInfo.Reader, nil, nil) + if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { + r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + } var encMetadata = make(map[string]string) if objectAPI.IsEncryptionSupported() && !isCompressed { // Encryption parameters not applicable for this object. @@ -1244,6 +1247,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } } + if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { + r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + } var objectEncryptionKey []byte if objectAPI.IsEncryptionSupported() { if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE requests @@ -1372,6 +1378,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r var encMetadata = map[string]string{} + if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { + r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + } if objectAPI.IsEncryptionSupported() { if hasServerSideEncryptionHeader(r.Header) { if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil { diff --git a/cmd/server-main.go b/cmd/server-main.go index fd5bc1c3f..e151a6876 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -18,6 +18,7 @@ package cmd import ( "context" + "errors" "fmt" "net/http" "os" @@ -383,6 +384,9 @@ func serverMain(ctx *cli.Context) { if err = globalNotificationSys.Init(newObject); err != nil { logger.LogIf(context.Background(), err) } + if globalAutoEncryption && !newObject.IsEncryptionSupported() { + logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but server does not support encryption") + } globalObjLayerMutex.Lock() globalObjectAPI = newObject diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 8d70f56a0..b616a7459 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -742,11 +742,6 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { writeWebErrorResponse(w, errServerNotInitialized) return } - - putObject := objectAPI.PutObject - if web.CacheAPI() != nil { - putObject = web.CacheAPI().PutObject - } vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -785,6 +780,9 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { return } } + if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) { + r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + } // Require Content-Length to be set in the request size := r.ContentLength @@ -800,9 +798,15 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { return } - reader := r.Body + var pReader *PutObjReader + var reader io.Reader = r.Body actualSize := size + hashReader, err := hash.NewReader(reader, size, "", "", actualSize) + if err != nil { + writeWebErrorResponse(w, err) + return + } if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 { // Storing the compression metadata. metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1 @@ -828,13 +832,35 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { // Set compression metrics. size = -1 // Since compressed size is un-predictable. reader = pipeReader + hashReader, err = hash.NewReader(reader, size, "", "", actualSize) + if err != nil { + writeWebErrorResponse(w, err) + return + } } + pReader = NewPutObjReader(hashReader, nil, nil) - hashReader, err := hash.NewReader(reader, size, "", "", actualSize) - if err != nil { - writeWebErrorResponse(w, err) - return + if objectAPI.IsEncryptionSupported() { + if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE requests + rawReader := hashReader + var objectEncryptionKey []byte + reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata) + if err != nil { + writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + info := ObjectInfo{Size: size} + hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content + if err != nil { + writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey) + } } + // Ensure that metadata does not contain sensitive information + crypto.RemoveSensitiveEntries(metadata) + var opts ObjectOptions // Deny if WORM is enabled if globalWORMEnabled { @@ -844,11 +870,26 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { } } - objInfo, err := putObject(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts) + putObject := objectAPI.PutObject + if !hasServerSideEncryptionHeader(r.Header) && web.CacheAPI() != nil { + putObject = web.CacheAPI().PutObject + } + objInfo, err := putObject(context.Background(), bucket, object, pReader, metadata, opts) if err != nil { writeWebErrorResponse(w, err) return } + if objectAPI.IsEncryptionSupported() { + if crypto.IsEncrypted(objInfo.UserDefined) { + switch { + case crypto.S3.IsEncrypted(objInfo.UserDefined): + w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256) + case crypto.SSEC.IsRequested(r.Header): + w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm)) + w.Header().Set(crypto.SSECKeyMD5, r.Header.Get(crypto.SSECKeyMD5)) + } + } + } // Get host and port from Request.RemoteAddr. host, port, err := net.SplitHostPort(handlers.GetSourceIP(r)) diff --git a/docs/kms/README.md b/docs/kms/README.md index bf5414c92..3fffa6180 100644 --- a/docs/kms/README.md +++ b/docs/kms/README.md @@ -18,6 +18,12 @@ Minio supports two different KMS concepts: Note: If the Minio server machine is ever compromised, then the master key must also be treated as compromised. +**Important:** +If multiple minio server are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md) +pointing to the *same* backend - for example the same NAS storage - than the KMS configuration **must** be equal for +all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It's the operators +responsibility to ensure that. + ## Get started ### 1. Prerequisites @@ -85,6 +91,12 @@ export MINIO_SSE_VAULT_KEY_NAME=my-minio-key minio server ~/export ``` +Optionally set `MINIO_SSE_VAULT_CAPATH` to a directory of PEM-encoded CA cert files to use mTLS for client-server authentication. + +``` +export MINIO_SSE_VAULT_CAPATH=/home/user/custom-certs +``` + Optionally set `MINIO_SSE_VAULT_NAMESPACE` if AppRole and Transit Secrets engine have been scoped to Vault Namespace ``` @@ -93,6 +105,8 @@ export MINIO_SSE_VAULT_NAMESPACE=ns1 Note: If [Vault Namespaces](https://learn.hashicorp.com/vault/operations/namespaces) are in use, MINIO_SSE_VAULT_NAMESPACE variable needs to be set before setting approle and transit secrets engine. + + #### 2.2 Specify a master key A KMS master key consists of a master-key ID (CMK) and the 256 bit master key encoded as HEX value separated by a `:`. @@ -106,6 +120,25 @@ export MINIO_SSE_MASTER_KEY=my-minio-key:6368616e676520746869732070617373776f726 To test this setup, start minio server with environment variables set in Step 3, and server is ready to handle SSE-S3 requests. +### Auto-Encryption + +Minio can also enable auto-encryption **if** a valid KMS configuration is specified and the storage backend supports +encrypted objects. Auto-Encryption, if enabled, ensures that all uploaded objects are encrypted using the specified +KMS configuration. + +Auto-Encryption is useful especially if the Minio operator wants to ensure that objects are **never** stored in +plaintext - for example if sensitive data is stored on public cloud storage. + +To enable auto-encryption either set the ENV. variable: + +```sh +export MINIO_SSE_AUTO_ENCRYPTION=on +``` + +Note: Auto-Encryption only affects non-SSE-C requests since objects uploaded using SSE-C are already encrypted +and S3 only allows either SSE-S3 or SSE-C but not both for the same object. + + # Explore Further - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)