add functions to remove confidential information (#6516)

This commit adds two functions for removing
confidential information - like SSE-C keys -
from HTTP headers / object metadata.

This creates a central point grouping all
headers/entries which must be filtered / removed.

See also https://github.com/minio/minio/pull/6489#discussion_r219797993
of #6489

cmd/crypto/header.go

@@ -74,6 +74,14 @@ const (
 	SSEAlgorithmKMS = "aws:kms"
 )
 
+// RemoveSensitiveHeaders removes confidential encryption
+// information - e.g. the SSE-C key - from the HTTP headers.
+// It has the same semantics as RemoveSensitiveEntires.
+func RemoveSensitiveHeaders(h http.Header) {
+	h.Del(SSECKey)
+	h.Del(SSECopyKey)
+}
+
 // S3 represents AWS SSE-S3. It provides functionality to handle
 // SSE-S3 requests.
 var S3 = s3{}

cmd/crypto/header_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"net/http"
+	"sort"
 	"testing"
 )
 
@@ -337,3 +338,102 @@ func TestSSECopyParse(t *testing.T) {
 		}
 	}
 }
+
+var removeSensitiveHeadersTests = []struct {
+	Header, ExpectedHeader http.Header
+}{
+	{
+		Header: http.Header{
+			SSECKey:    []string{""},
+			SSECopyKey: []string{""},
+		},
+		ExpectedHeader: http.Header{},
+	},
+	{ // Standard SSE-C request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + SSE-C-copy request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + metadata request headers
+		Header: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
+}
+
+func TestRemoveSensitiveHeaders(t *testing.T) {
+	isEqual := func(x, y http.Header) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			u, ok := y[k]
+			if !ok || len(v) != len(u) {
+				return false
+			}
+			sort.Strings(v)
+			sort.Strings(u)
+			for j := range v {
+				if v[j] != u[j] {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	areKeysEqual := func(h http.Header, metadata map[string]string) bool {
+		if len(h) != len(metadata) {
+			return false
+		}
+		for k := range h {
+			if _, ok := metadata[k]; !ok {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeSensitiveHeadersTests {
+		metadata := make(map[string]string, len(test.Header))
+		for k := range test.Header {
+			metadata[k] = "" // set metadata key - we don't care about the value
+		}
+
+		RemoveSensitiveHeaders(test.Header)
+		if !isEqual(test.ExpectedHeader, test.Header) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+		RemoveSensitiveEntries(metadata)
+		if !areKeysEqual(test.ExpectedHeader, metadata) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -32,6 +32,14 @@ func IsMultiPart(metadata map[string]string) bool {
 	return false
 }
 
+// RemoveSensitiveEntries removes confidential encryption
+// information - e.g. the SSE-C key - from the metadata map.
+// It has the same semantics as RemoveSensitiveHeaders.
+func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
+	delete(metadata, SSECKey)
+	delete(metadata, SSECopyKey)
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //

cmd/crypto/metadata_test.go

@@ -326,15 +326,20 @@ func TestS3CreateMetadata(t *testing.T) {
 	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
 
-var ssecCreateMetadataTests = []SealedKey{
+var ssecCreateMetadataTests = []struct {
-	{Algorithm: SealAlgorithm},
+	KeyID         string
-	{IV: [32]byte{0xff}, Key: [64]byte{0x7e}, Algorithm: SealAlgorithm},
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
 }
 
 func TestSSECCreateMetadata(t *testing.T) {
 	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
 	logger.Disable = true
-	for i, test := range s3CreateMetadataTests {
+	for i, test := range ssecCreateMetadataTests {
 		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
 		sealedKey, err := SSEC.ParseMetadata(metadata)
 		if err != nil {