minio/cmd/encryption-v1_test.go

/*
 * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"encoding/base64"
	"net/http"
	"testing"

	humanize "github.com/dustin/go-humanize"
	"github.com/minio/minio-go/v7/pkg/encrypt"
	"github.com/minio/minio/cmd/crypto"
	"github.com/minio/sio"
)

var encryptRequestTests = []struct {
	header   map[string]string
	metadata map[string]string
}{
	{
		header: map[string]string{
			crypto.SSECAlgorithm: "AES256",
			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
		},
		metadata: map[string]string{},
	},
	{
		header: map[string]string{
			crypto.SSECAlgorithm: "AES256",
			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
		},
		metadata: map[string]string{
			crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
		},
	},
}

func TestEncryptRequest(t *testing.T) {
	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
	globalIsSSL = true
	for i, test := range encryptRequestTests {
		content := bytes.NewReader(make([]byte, 64))
		req := &http.Request{Header: http.Header{}}
		for k, v := range test.header {
			req.Header.Set(k, v)
		}
		_, _, err := EncryptRequest(content, req, "bucket", "object", test.metadata)

		if err != nil {
			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
		}
		if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
		}
		if iv, ok := test.metadata[crypto.SSEIV]; !ok {
			t.Errorf("Test %d: crypto.SSEIV must be part of metadata: %v", i, iv)
		}
		if mac, ok := test.metadata[crypto.SSECSealedKey]; !ok {
			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
		}
	}
}

var decryptObjectInfoTests = []struct {
	info    ObjectInfo
	request *http.Request
	expErr  error
}{
	{
		info:    ObjectInfo{Size: 100},
		request: &http.Request{Header: http.Header{}},
		expErr:  nil,
	},
	{
		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
		expErr:  nil,
	},
	{
		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
		expErr:  nil,
	},
	{
		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
		request: &http.Request{Header: http.Header{}},
		expErr:  errEncryptedObject,
	},
	{
		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
		request: &http.Request{Method: http.MethodGet, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
		expErr:  errInvalidEncryptionParameters,
	},
	{
		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
		request: &http.Request{Method: http.MethodHead, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
		expErr:  errInvalidEncryptionParameters,
	},
	{
		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
		expErr:  errObjectTampered,
	},
}

func TestDecryptObjectInfo(t *testing.T) {
	for i, test := range decryptObjectInfoTests {
		if encrypted, err := DecryptObjectInfo(&test.info, test.request); err != test.expErr {
			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
		} else if !encrypted && enc != encrypted {
			t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)
		}
	}
}

var decryptETagTests = []struct {
	ObjectKey  crypto.ObjectKey
	ObjectInfo ObjectInfo
	ShouldFail bool
	ETag       string
}{
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"},
		ETag:       "8ad3fe6b84bf38489e95c701c84355b6",
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d6935"},
		ETag:       "",
		ShouldFail: true, // ETag is not a valid hex value
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "00000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"},
		ETag:       "",
		ShouldFail: true, // modified ETag
	},

	// Special tests for ETags that end with a '-x'
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-1"},
		ETag:       "916516b396f0f4d4f2a0e7177557bec4-1",
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-738"},
		ETag:       "916516b396f0f4d4f2a0e7177557bec4-738",
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-Q"},
		ETag:       "",
		ShouldFail: true, // Q is not a number
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1"},
		ETag:       "",
		ShouldFail: true, // ETag prefix is not a valid hex value
	},
	{
		ObjectKey:  [32]byte{},
		ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1-2"},
		ETag:       "",
		ShouldFail: true, // ETag contains multiple: -
	},
}

func TestDecryptETag(t *testing.T) {
	for i, test := range decryptETagTests {
		etag, err := DecryptETag(test.ObjectKey, test.ObjectInfo)
		if err != nil && !test.ShouldFail {
			t.Fatalf("Test %d: should succeed but failed: %v", i, err)
		}
		if err == nil && test.ShouldFail {
			t.Fatalf("Test %d: should fail but succeeded", i)
		}
		if err == nil {
			if etag != test.ETag {
				t.Fatalf("Test %d: ETag mismatch: got %s - want %s", i, etag, test.ETag)
			}
		}
	}
}

// Tests for issue reproduced when getting the right encrypted
// offset of the object.
func TestGetDecryptedRange_Issue50(t *testing.T) {
	rs, err := parseRequestRangeSpec("bytes=594870256-594870263")
	if err != nil {
		t.Fatal(err)
	}

	objInfo := ObjectInfo{
		Bucket: "bucket",
		Name:   "object",
		Size:   595160760,
		UserDefined: map[string]string{
			crypto.SSEMultipart:                    "",
			crypto.SSEIV:                           "HTexa=",
			crypto.SSESealAlgorithm:                "DAREv2-HMAC-SHA256",
			crypto.SSECSealedKey:                   "IAA8PGAA==",
			ReservedMetadataPrefix + "actual-size": "594870264",
			"content-type":                         "application/octet-stream",
			"etag":                                 "166b1545b4c1535294ee0686678bea8c-2",
		},
		Parts: []ObjectPartInfo{
			{
				Number:     1,
				Size:       297580380,
				ActualSize: 297435132,
			},
			{
				Number:     2,
				Size:       297580380,
				ActualSize: 297435132,
			},
		},
	}

	encOff, encLength, skipLen, seqNumber, partStart, err := objInfo.GetDecryptedRange(rs)
	if err != nil {
		t.Fatalf("Test: failed %s", err)
	}
	if encOff != 595127964 {
		t.Fatalf("Test: expected %d, got %d", 595127964, encOff)
	}
	if encLength != 32796 {
		t.Fatalf("Test: expected %d, got %d", 32796, encLength)
	}
	if skipLen != 32756 {
		t.Fatalf("Test: expected %d, got %d", 32756, skipLen)
	}
	if seqNumber != 4538 {
		t.Fatalf("Test: expected %d, got %d", 4538, seqNumber)
	}
	if partStart != 1 {
		t.Fatalf("Test: expected %d, got %d", 1, partStart)
	}
}

func TestGetDecryptedRange(t *testing.T) {
	var (
		pkgSz     = int64(64) * humanize.KiByte
		minPartSz = int64(5) * humanize.MiByte
		maxPartSz = int64(5) * humanize.GiByte

		getEncSize = func(s int64) int64 {
			v, _ := sio.EncryptedSize(uint64(s))
			return int64(v)
		}
		udMap = func(isMulti bool) map[string]string {
			m := map[string]string{
				crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
				crypto.SSEMultipart:     "1",
			}
			if !isMulti {
				delete(m, crypto.SSEMultipart)
			}
			return m
		}
	)

	// Single part object tests
	var (
		mkSPObj = func(s int64) ObjectInfo {
			return ObjectInfo{
				Size:        getEncSize(s),
				UserDefined: udMap(false),
			}
		}
	)

	testSP := []struct {
		decSz int64
		oi    ObjectInfo
	}{
		{0, mkSPObj(0)},
		{1, mkSPObj(1)},
		{pkgSz - 1, mkSPObj(pkgSz - 1)},
		{pkgSz, mkSPObj(pkgSz)},
		{2*pkgSz - 1, mkSPObj(2*pkgSz - 1)},
		{minPartSz, mkSPObj(minPartSz)},
		{maxPartSz, mkSPObj(maxPartSz)},
	}

	for i, test := range testSP {
		{
			// nil range
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}
			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != getEncSize(test.decSz) {
				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
			}
		}

		if test.decSz >= 10 {
			// first 10 bytes
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 0, 9})
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}
			var rLen = pkgSz + 32
			if test.decSz < pkgSz {
				rLen = test.decSz + 32
			}
			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != rLen {
				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
			}
		}

		kb32 := int64(32) * humanize.KiByte
		if test.decSz >= (64+32)*humanize.KiByte {
			// Skip the first 32Kib, and read the next 64Kib
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, kb32, 3*kb32 - 1})
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}
			var rLen = (pkgSz + 32) * 2
			if test.decSz < 2*pkgSz {
				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32)
			}
			if skip != kb32 || sn != 0 || ps != 0 || o != 0 || l != rLen {
				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
			}
		}

		if test.decSz >= (64*2+32)*humanize.KiByte {
			// Skip the first 96Kib and read the next 64Kib
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 3 * kb32, 5*kb32 - 1})
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}
			var rLen = (pkgSz + 32) * 2
			if test.decSz-pkgSz < 2*pkgSz {
				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2)
			}
			if skip != kb32 || sn != 1 || ps != 0 || o != pkgSz+32 || l != rLen {
				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
			}
		}

	}

	// Multipart object tests
	var (
		// make a multipart object-info given part sizes
		mkMPObj = func(sizes []int64) ObjectInfo {
			r := make([]ObjectPartInfo, len(sizes))
			sum := int64(0)
			for i, s := range sizes {
				r[i].Number = i
				r[i].Size = getEncSize(s)
				sum += r[i].Size
			}
			return ObjectInfo{
				Size:        sum,
				UserDefined: udMap(true),
				Parts:       r,
			}
		}
		// Simple useful utilities
		repeat = func(k int64, n int) []int64 {
			a := []int64{}
			for i := 0; i < n; i++ {
				a = append(a, k)
			}
			return a
		}
		lsum = func(s []int64) int64 {
			sum := int64(0)
			for _, i := range s {
				if i < 0 {
					return -1
				}
				sum += i
			}
			return sum
		}
		esum = func(oi ObjectInfo) int64 {
			sum := int64(0)
			for _, i := range oi.Parts {
				sum += i.Size
			}
			return sum
		}
	)

	s1 := []int64{5487701, 5487799, 3}
	s2 := repeat(5487701, 5)
	s3 := repeat(maxPartSz, 10000)
	testMPs := []struct {
		decSizes []int64
		oi       ObjectInfo
	}{
		{s1, mkMPObj(s1)},
		{s2, mkMPObj(s2)},
		{s3, mkMPObj(s3)},
	}

	// This function is a reference (re-)implementation of
	// decrypted range computation, written solely for the purpose
	// of the unit tests.
	//
	// `s` gives the decrypted part sizes, and the other
	// parameters describe the desired read segment. When
	// `isFromEnd` is true, `skipLen` argument is ignored.
	decryptedRangeRef := func(s []int64, skipLen, readLen int64, isFromEnd bool) (o, l, skip int64, sn uint32, ps int) {
		oSize := lsum(s)
		if isFromEnd {
			skipLen = oSize - readLen
		}
		if skipLen < 0 || readLen < 0 || oSize < 0 || skipLen+readLen > oSize {
			t.Fatalf("Impossible read specified: %d %d %d", skipLen, readLen, oSize)
		}

		var cumulativeSum, cumulativeEncSum int64
		toRead := readLen
		readStart := false
		for i, v := range s {
			partOffset := int64(0)
			partDarePkgOffset := int64(0)
			if !readStart && cumulativeSum+v > skipLen {
				// Read starts at the current part
				readStart = true

				partOffset = skipLen - cumulativeSum

				// All return values except `l` are
				// calculated here.
				sn = uint32(partOffset / pkgSz)
				skip = partOffset % pkgSz
				ps = i
				o = cumulativeEncSum + int64(sn)*(pkgSz+32)

				partDarePkgOffset = partOffset - skip
			}
			if readStart {
				currentPartBytes := v - partOffset
				currentPartDareBytes := v - partDarePkgOffset
				if currentPartBytes < toRead {
					toRead -= currentPartBytes
					l += getEncSize(currentPartDareBytes)
				} else {
					// current part has the last
					// byte required
					lbPartOffset := partOffset + toRead - 1

					// round up the lbPartOffset
					// to the end of the
					// corresponding DARE package
					lbPkgEndOffset := lbPartOffset - (lbPartOffset % pkgSz) + pkgSz
					if lbPkgEndOffset > v {
						lbPkgEndOffset = v
					}
					bytesToDrop := v - lbPkgEndOffset

					// Last segment to update `l`
					l += getEncSize(currentPartDareBytes - bytesToDrop)
					break
				}
			}

			cumulativeSum += v
			cumulativeEncSum += getEncSize(v)
		}
		return
	}

	for i, test := range testMPs {
		{
			// nil range
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}
			if o != 0 || l != esum(test.oi) || skip != 0 || sn != 0 || ps != 0 {
				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
			}
		}

		// Skip 1Mib and read 1Mib (in the decrypted object)
		//
		// The check below ensures the object is large enough
		// for the read.
		if lsum(test.decSizes) >= 2*humanize.MiByte {
			skipLen, readLen := int64(1)*humanize.MiByte, int64(1)*humanize.MiByte
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, skipLen, skipLen + readLen - 1})
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}

			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, skipLen, readLen, false)
			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
			}
		}

		// Read the last 6Mib+1 bytes of the (decrypted)
		// object
		//
		// The check below ensures the object is large enough
		// for the read.
		readLen := int64(6)*humanize.MiByte + 1
		if lsum(test.decSizes) >= readLen {
			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{true, -readLen, -1})
			if err != nil {
				t.Errorf("Case %d: unexpected err: %v", i, err)
			}

			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, 0, readLen, true)
			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
			}
		}

	}
}

var getDefaultOptsTests = []struct {
	headers        http.Header
	copySource     bool
	metadata       map[string]string
	encryptionType encrypt.Type
	err            error
}{
	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
		copySource:     false,
		metadata:       nil,
		encryptionType: encrypt.SSEC,
		err:            nil}, // 0
	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
		copySource:     true,
		metadata:       nil,
		encryptionType: "",
		err:            nil}, // 1
	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
		crypto.SSECKey:    []string{"Mz"},
		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
		copySource:     false,
		metadata:       nil,
		encryptionType: "",
		err:            crypto.ErrInvalidCustomerKey}, // 2
	{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
		copySource:     false,
		metadata:       nil,
		encryptionType: encrypt.S3,
		err:            nil}, // 3
	{headers: http.Header{},
		copySource: false,
		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
			crypto.S3KMSKeyID:     "kms-key",
			crypto.S3KMSSealedKey: "m-key"},
		encryptionType: encrypt.S3,
		err:            nil}, // 4
	{headers: http.Header{},
		copySource: true,
		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
			crypto.S3KMSKeyID:     "kms-key",
			crypto.S3KMSSealedKey: "m-key"},
		encryptionType: "",
		err:            nil}, // 5
	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
		copySource:     true,
		metadata:       nil,
		encryptionType: encrypt.SSEC,
		err:            nil}, // 6
	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
		copySource:     false,
		metadata:       nil,
		encryptionType: "",
		err:            nil}, // 7
}

func TestGetDefaultOpts(t *testing.T) {
	for i, test := range getDefaultOptsTests {
		opts, err := getDefaultOpts(test.headers, test.copySource, test.metadata)
		if test.err != err {
			t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err)
		}
		if err == nil {
			if opts.ServerSideEncryption == nil && test.encryptionType != "" {
				t.Errorf("Case %d: expected opts to be of %v encryption type", i, test.encryptionType)

			}
			if opts.ServerSideEncryption != nil && test.encryptionType != opts.ServerSideEncryption.Type() {
				t.Errorf("Case %d: expected opts to have encryption type %v but was %v ", i, test.encryptionType, opts.ServerSideEncryption.Type())
			}
		}
	}
}