diff --git a/cmd/crypto/doc.go b/cmd/crypto/doc.go new file mode 100644 index 000000000..a938f911a --- /dev/null +++ b/cmd/crypto/doc.go @@ -0,0 +1,116 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package crypto implements AWS S3 related cryptographic building blocks +// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption +// with customer provided keys (SSE-C). +// +// All objects are encrypted with an unique and randomly generated 'ObjectKey'. +// The ObjectKey itself is never stored in plaintext. Instead it is only stored +// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey' +// with an unique key-encryption-key. Given the correct key-encryption-key the +// sealed 'ObjectKey' can be unsealed and the object can be decrypted. +// +// +// ## SSE-C +// +// SSE-C computes the key-encryption-key from the client-provided key, an +// initialization vector (IV) and the bucket/object path. +// +// 1. Encrypt: +// Input: ClientKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: ClientKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +// +// +// ## SSE-S3 +// +// SSE-S3 can use either a master key or a KMS as root-of-trust. +// The en/decryption slightly depens upon which root-of-trust is used. +// +// ### SSE-S3 and single master key +// +// The master key is used to derive unique object- and key-encryption-keys. +// SSE-S3 with a single master key works as SSE-C where the master key is +// used as the client-provided key. +// +// 1. Encrypt: +// Input: MasterKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: MasterKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +// +// +// ### SSE-S3 and KMS +// +// SSE-S3 requires that the KMS provides two functions: +// 1. Generate(KeyID) -> (Key, EncKey) +// 2. Unseal(KeyID, EncKey) -> Key +// +// 1. Encrypt: +// Input: KeyID, bucket, object, metadata, object_data +// - Key, EncKey := Generate(KeyID) +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- KeyID +// - metadata <- EncKey +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: bucket, object, metadata, enc_object_data +// - KeyID <- metadata +// - EncKey <- metadata +// - IV <- metadata +// - SealedKey <- metadata +// - Key := Unseal(KeyID, EncKey) +// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +// +package crypto diff --git a/cmd/crypto/error.go b/cmd/crypto/error.go new file mode 100644 index 000000000..8dda24935 --- /dev/null +++ b/cmd/crypto/error.go @@ -0,0 +1,23 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import "errors" + +var ( + // ErrInvalidEncryptionMethod indicates that the specified SSE encryption method + // is not supported. + ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported") +) diff --git a/cmd/crypto/header.go b/cmd/crypto/header.go new file mode 100644 index 000000000..9227c246b --- /dev/null +++ b/cmd/crypto/header.go @@ -0,0 +1,49 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import ( + "net/http" +) + +// SSEHeader is the general AWS SSE HTTP header key. +const SSEHeader = "X-Amz-Server-Side-Encryption" + +// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header. +// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html +// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html +const SSEAlgorithmAES256 = "AES256" + +// S3 represents AWS SSE-S3. It provides functionality to handle +// SSE-S3 requests. +var S3 = s3{} + +type s3 struct{} + +// IsRequested returns true if the HTTP headers indicates that +// the S3 client requests SSE-S3. +func (s3) IsRequested(h http.Header) bool { + _, ok := h[SSEHeader] + return ok +} + +// Parse parses the SSE-S3 related HTTP headers and checks +// whether they contain valid values. +func (s3) Parse(h http.Header) (err error) { + if h.Get(SSEHeader) != SSEAlgorithmAES256 { + err = ErrInvalidEncryptionMethod + } + return +} diff --git a/cmd/crypto/header_test.go b/cmd/crypto/header_test.go new file mode 100644 index 000000000..eedc35c2b --- /dev/null +++ b/cmd/crypto/header_test.go @@ -0,0 +1,56 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import ( + "net/http" + "testing" +) + +var isRequestedTests = []struct { + Header http.Header + Expected bool +}{ + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3 +} + +func TestS3IsRequested(t *testing.T) { + for i, test := range isRequestedTests { + if got := S3.IsRequested(test.Header); got != test.Expected { + t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var parseTests = []struct { + Header http.Header + ExpectedErr error +}{ + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 3 +} + +func TestS3Parse(t *testing.T) { + for i, test := range parseTests { + if err := S3.Parse(test.Header); err != test.ExpectedErr { + t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err) + } + } +} diff --git a/cmd/crypto/key.go b/cmd/crypto/key.go new file mode 100644 index 000000000..edef2923e --- /dev/null +++ b/cmd/crypto/key.go @@ -0,0 +1,93 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "encoding/binary" + "errors" + "io" + "path/filepath" + + "github.com/minio/minio/cmd/logger" + sha256 "github.com/minio/sha256-simd" + "github.com/minio/sio" +) + +// ObjectKey is a 256 bit secret key used to encrypt the object. +// It must never be stored in plaintext. +type ObjectKey [32]byte + +// GenerateKey generates a unique ObjectKey from a 256 bit external key +// and a source of randomness. If random is nil the default PRNG of system +// (crypto/rand) is used. +func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) { + if random == nil { + random = rand.Reader + } + var nonce [32]byte + if _, err := io.ReadFull(random, nonce[:]); err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system")) + } + sha := sha256.New() + sha.Write(extKey[:]) + sha.Write(nonce[:]) + sha.Sum(key[:0]) + return +} + +// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed +// key is also cryptographically bound to the object's path (bucket/object). +func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte { + var sealedKey bytes.Buffer + mac := hmac.New(sha256.New, extKey[:]) + mac.Write(iv[:]) + mac.Write([]byte(filepath.Join(bucket, object))) + + if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key")) + } + return sealedKey.Bytes() +} + +// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key +// is cryptographically bound to the object's path the same bucket/object as during sealing +// must be provided. On success the ObjectKey contains the decrypted sealed key. +func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error { + var unsealedKey bytes.Buffer + mac := hmac.New(sha256.New, extKey[:]) + mac.Write(iv[:]) + mac.Write([]byte(filepath.Join(bucket, object))) + + if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil { + return err // TODO(aead): upgrade sio to use sio.Error + } + copy(key[:], unsealedKey.Bytes()) + return nil +} + +// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index. +func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) { + var bin [4]byte + binary.LittleEndian.PutUint32(bin[:], id) + + mac := hmac.New(sha256.New, key[:]) + mac.Write(bin[:]) + mac.Sum(partKey[:0]) + return +} diff --git a/cmd/crypto/key_test.go b/cmd/crypto/key_test.go new file mode 100644 index 000000000..c91415edf --- /dev/null +++ b/cmd/crypto/key_test.go @@ -0,0 +1,127 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "io" + "testing" +) + +var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) } + +func recoverTest(i int, shouldPass bool, t *testing.T) { + if err := recover(); err == nil && !shouldPass { + t.Errorf("Test %d should fail but passed successfully", i) + } else if err != nil && shouldPass { + t.Errorf("Test %d should pass but failed: %v", i, err) + } +} + +var generateKeyTests = []struct { + ExtKey [32]byte + Random io.Reader + ShouldPass bool +}{ + {ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0 + {ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1 + {ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2 + // {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064 +} + +func TestGenerateKey(t *testing.T) { + for i, test := range generateKeyTests { + func() { + defer recoverTest(i, test.ShouldPass, t) + key := GenerateKey(test.ExtKey, test.Random) + if [32]byte(key) == [32]byte{} { + t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key + } + }() + } +} + +var sealUnsealKeyTests = []struct { + SealExtKey, SealIV [32]byte + SealBucket, SealObject string + + UnsealExtKey, UnsealIV [32]byte + UnsealBucket, UnsealObject string + + ShouldPass bool +}{ + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object", + ShouldPass: true, + }, // 0 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object", + ShouldPass: false, + }, // 1 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object", + ShouldPass: false, + }, // 2 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object", + ShouldPass: false, + }, // 3 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object", + ShouldPass: false, + }, // 4 +} + +func TestSealUnsealKey(t *testing.T) { + for i, test := range sealUnsealKeyTests { + key := GenerateKey(test.SealExtKey, rand.Reader) + sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject) + if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass { + t.Errorf("Test %d should fail but passed successfully", i) + } else if err != nil && test.ShouldPass { + t.Errorf("Test %d should pass put failed: %v", i, err) + } + } +} + +var derivePartKeyTest = []struct { + PartID uint32 + PartKey string +}{ + {PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"}, + {PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"}, + {PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"}, +} + +func TestDerivePartKey(t *testing.T) { + var key ObjectKey + for i, test := range derivePartKeyTest { + expectedPartKey, err := hex.DecodeString(test.PartKey) + if err != nil { + t.Fatalf("Test %d failed to decode expected part-key: %v", i, err) + } + partKey := key.DerivePartKey(test.PartID) + if !bytes.Equal(partKey[:], expectedPartKey[:]) { + t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey) + } + } +} diff --git a/cmd/crypto/sse.go b/cmd/crypto/sse.go new file mode 100644 index 000000000..0cc1584fd --- /dev/null +++ b/cmd/crypto/sse.go @@ -0,0 +1,60 @@ +// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crypto + +import ( + "context" + "errors" + "io" + + "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/ioutil" + "github.com/minio/sio" +) + +const ( + // S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3. + S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key" + // S3KMSKeyID is the metadata key referencing the KMS key-id used to + // generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS. + S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id" + // S3KMSSealedKey is the metadata key referencing the encrypted key generated + // by KMS. It is only used for SSE-S3 + KMS. + S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key" +) + +// EncryptSinglePart encrypts an io.Reader which must be the +// the body of a single-part PUT request. +func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader { + r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]}) + if err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key")) + } + return r +} + +// DecryptSinglePart decrypts an io.Writer which must an object +// uploaded with the single-part PUT API. The offset and length +// specify the requested range. +func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser { + const PayloadSize = 1 << 16 // DARE 2.0 + w = ioutil.LimitedWriter(w, offset%PayloadSize, length) + + decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]}) + if err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key")) + } + return decWriter +}