oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new package: cmd/crypto (#6062)

Browse Source 
This commit introduces a new crypto package providing
AWS S3 related cryptographic building blocks to implement
SSE-S3 (master key or KMS) and SSE-C.

This change only adds some basic functionallity esp.
related to SSE-S3 and documents the general approach
for SSE-S3 and SSE-C.
master
Andreas Auernhammer 7 years ago committed by kannappanr
parent ea76e72054
commit 805186ab97
7 changed files with 524 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 116
      cmd/crypto/doc.go
  2. 23
      cmd/crypto/error.go
  3. 49
      cmd/crypto/header.go
  4. 56
      cmd/crypto/header_test.go
  5. 93
      cmd/crypto/key.go
  6. 127
      cmd/crypto/key_test.go
  7. 60
      cmd/crypto/sse.go

116
cmd/crypto/doc.go
Unescape View File

@ -0,0 +1,116 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package crypto implements AWS S3 related cryptographic building blocks
// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
// with customer provided keys (SSE-C).
//
// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
// The ObjectKey itself is never stored in plaintext. Instead it is only stored
// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
// with an unique key-encryption-key. Given the correct key-encryption-key the
// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
//
//
// ## SSE-C
//
// SSE-C computes the key-encryption-key from the client-provided key, an
// initialization vector (IV) and the bucket/object path.
//
// 1. Encrypt:
// Input: ClientKey, bucket, object, metadata, object_data
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: ClientKey, bucket, object, metadata, enc_object_data
// - IV <- metadata
// - SealedKey <- metadata
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
//
// ## SSE-S3
//
// SSE-S3 can use either a master key or a KMS as root-of-trust.
// The en/decryption slightly depens upon which root-of-trust is used.
//
// ### SSE-S3 and single master key
//
// The master key is used to derive unique object- and key-encryption-keys.
// SSE-S3 with a single master key works as SSE-C where the master key is
// used as the client-provided key.
//
// 1. Encrypt:
// Input: MasterKey, bucket, object, metadata, object_data
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: MasterKey, bucket, object, metadata, enc_object_data
// - IV <- metadata
// - SealedKey <- metadata
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
//
// ### SSE-S3 and KMS
//
// SSE-S3 requires that the KMS provides two functions:
// 1. Generate(KeyID) -> (Key, EncKey)
// 2. Unseal(KeyID, EncKey) -> Key
//
// 1. Encrypt:
// Input: KeyID, bucket, object, metadata, object_data
// - Key, EncKey := Generate(KeyID)
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- KeyID
// - metadata <- EncKey
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: bucket, object, metadata, enc_object_data
// - KeyID <- metadata
// - EncKey <- metadata
// - IV <- metadata
// - SealedKey <- metadata
// - Key := Unseal(KeyID, EncKey)
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
package crypto

23
cmd/crypto/error.go
Unescape View File

@ -0,0 +1,23 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import "errors"
var (
// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
// is not supported.
ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
)

49
cmd/crypto/header.go
Unescape View File

@ -0,0 +1,49 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"net/http"
)
// SSEHeader is the general AWS SSE HTTP header key.
const SSEHeader = "X-Amz-Server-Side-Encryption"
// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
const SSEAlgorithmAES256 = "AES256"
// S3 represents AWS SSE-S3. It provides functionality to handle
// SSE-S3 requests.
var S3 = s3{}
type s3 struct{}
// IsRequested returns true if the HTTP headers indicates that
// the S3 client requests SSE-S3.
func (s3) IsRequested(h http.Header) bool {
_, ok := h[SSEHeader]
return ok
}
// Parse parses the SSE-S3 related HTTP headers and checks
// whether they contain valid values.
func (s3) Parse(h http.Header) (err error) {
if h.Get(SSEHeader) != SSEAlgorithmAES256 {
err = ErrInvalidEncryptionMethod
}
return
}

56
cmd/crypto/header_test.go
Unescape View File

@ -0,0 +1,56 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"net/http"
"testing"
)
var isRequestedTests = []struct {
Header http.Header
Expected bool
}{
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3
}
func TestS3IsRequested(t *testing.T) {
for i, test := range isRequestedTests {
if got := S3.IsRequested(test.Header); got != test.Expected {
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
}
}
}
var parseTests = []struct {
Header http.Header
ExpectedErr error
}{
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil}, // 0
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 2
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 3
}
func TestS3Parse(t *testing.T) {
for i, test := range parseTests {
if err := S3.Parse(test.Header); err != test.ExpectedErr {
t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
}
}
}

93
cmd/crypto/key.go
Unescape View File

@ -0,0 +1,93 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"bytes"
"context"
"crypto/hmac"
"crypto/rand"
"encoding/binary"
"errors"
"io"
"path/filepath"
"github.com/minio/minio/cmd/logger"
sha256 "github.com/minio/sha256-simd"
"github.com/minio/sio"
)
// ObjectKey is a 256 bit secret key used to encrypt the object.
// It must never be stored in plaintext.
type ObjectKey [32]byte
// GenerateKey generates a unique ObjectKey from a 256 bit external key
// and a source of randomness. If random is nil the default PRNG of system
// (crypto/rand) is used.
func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
if random == nil {
random = rand.Reader
}
var nonce [32]byte
if _, err := io.ReadFull(random, nonce[:]); err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
}
sha := sha256.New()
sha.Write(extKey[:])
sha.Write(nonce[:])
sha.Sum(key[:0])
return
}
// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
// key is also cryptographically bound to the object's path (bucket/object).
func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
var sealedKey bytes.Buffer
mac := hmac.New(sha256.New, extKey[:])
mac.Write(iv[:])
mac.Write([]byte(filepath.Join(bucket, object)))
if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
}
return sealedKey.Bytes()
}
// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
// is cryptographically bound to the object's path the same bucket/object as during sealing
// must be provided. On success the ObjectKey contains the decrypted sealed key.
func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
var unsealedKey bytes.Buffer
mac := hmac.New(sha256.New, extKey[:])
mac.Write(iv[:])
mac.Write([]byte(filepath.Join(bucket, object)))
if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
return err // TODO(aead): upgrade sio to use sio.Error
}
copy(key[:], unsealedKey.Bytes())
return nil
}
// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
var bin [4]byte
binary.LittleEndian.PutUint32(bin[:], id)
mac := hmac.New(sha256.New, key[:])
mac.Write(bin[:])
mac.Sum(partKey[:0])
return
}

127
cmd/crypto/key_test.go
Unescape View File

@ -0,0 +1,127 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"bytes"
"crypto/rand"
"encoding/hex"
"io"
"testing"
)
var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
func recoverTest(i int, shouldPass bool, t *testing.T) {
if err := recover(); err == nil && !shouldPass {
t.Errorf("Test %d should fail but passed successfully", i)
} else if err != nil && shouldPass {
t.Errorf("Test %d should pass but failed: %v", i, err)
}
}
var generateKeyTests = []struct {
ExtKey [32]byte
Random io.Reader
ShouldPass bool
}{
{ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0
{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1
{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
}
func TestGenerateKey(t *testing.T) {
for i, test := range generateKeyTests {
func() {
defer recoverTest(i, test.ShouldPass, t)
key := GenerateKey(test.ExtKey, test.Random)
if [32]byte(key) == [32]byte{} {
t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
}
}()
}
}
var sealUnsealKeyTests = []struct {
SealExtKey, SealIV [32]byte
SealBucket, SealObject string
UnsealExtKey, UnsealIV [32]byte
UnsealBucket, UnsealObject string
ShouldPass bool
}{
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: true,
}, // 0
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: false,
}, // 1
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: false,
}, // 2
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
ShouldPass: false,
}, // 3
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
ShouldPass: false,
}, // 4
}
func TestSealUnsealKey(t *testing.T) {
for i, test := range sealUnsealKeyTests {
key := GenerateKey(test.SealExtKey, rand.Reader)
sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
t.Errorf("Test %d should fail but passed successfully", i)
} else if err != nil && test.ShouldPass {
t.Errorf("Test %d should pass put failed: %v", i, err)
}
}
}
var derivePartKeyTest = []struct {
PartID uint32
PartKey string
}{
{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
}
func TestDerivePartKey(t *testing.T) {
var key ObjectKey
for i, test := range derivePartKeyTest {
expectedPartKey, err := hex.DecodeString(test.PartKey)
if err != nil {
t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
}
partKey := key.DerivePartKey(test.PartID)
if !bytes.Equal(partKey[:], expectedPartKey[:]) {
t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
}
}
}

60
cmd/crypto/sse.go
Unescape View File

@ -0,0 +1,60 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"context"
"errors"
"io"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/ioutil"
"github.com/minio/sio"
)
const (
// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
// S3KMSKeyID is the metadata key referencing the KMS key-id used to
// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
// S3KMSSealedKey is the metadata key referencing the encrypted key generated
// by KMS. It is only used for SSE-S3 + KMS.
S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
)
// EncryptSinglePart encrypts an io.Reader which must be the
// the body of a single-part PUT request.
func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
if err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
}
return r
}
// DecryptSinglePart decrypts an io.Writer which must an object
// uploaded with the single-part PUT API. The offset and length
// specify the requested range.
func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
const PayloadSize = 1 << 16 // DARE 2.0
w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
if err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
}
return decWriter
}
Write Preview
Loading…
Cancel
Save