Allow Compression + encryption (#11103)

cmd/config/compress/compress.go

@@ -26,19 +26,22 @@ import (
 
 // Config represents the compression settings.
 type Config struct {
-	Enabled    bool     `json:"enabled"`
+	Enabled        bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
+	AllowEncrypted bool     `json:"allow_encryption"`
-	MimeTypes  []string `json:"mime-types"`
+	Extensions     []string `json:"extensions"`
+	MimeTypes      []string `json:"mime-types"`
 }
 
 // Compression environment variables
 const (
-	Extensions = "extensions"
+	Extensions     = "extensions"
-	MimeTypes  = "mime_types"
+	AllowEncrypted = "allow_encryption"
+	MimeTypes      = "mime_types"
 
-	EnvCompressState      = "MINIO_COMPRESS_ENABLE"
+	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
-	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
-	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@@ -52,6 +55,10 @@ var (
 			Key:   config.Enable,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   AllowEncrypted,
+			Value: config.EnableOff,
+		},
 		config.KV{
 			Key:   Extensions,
 			Value: DefaultExtensions,
@@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		return cfg, nil
 	}
 
+	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
+	if err != nil {
+		return cfg, err
+	}
+
 	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
 	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
 	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))

cmd/encryption-v1.go

@@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 
 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
-func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {
-	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
-	io.Reader, error) {
 
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case

cmd/object-api-getobject_test.go

@@ -31,7 +31,7 @@ import (
 
 // Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup.
 func TestGetObject(t *testing.T) {
-	ExecObjectLayerTest(t, testGetObject)
+	ExecExtendedObjectLayerTest(t, testGetObject)
 }
 
 // ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.

cmd/object-api-multipart_test.go

@@ -157,7 +157,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 
 // Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup.
 func TestObjectAPIPutObjectPart(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
+	ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart)
 }
 
 // Tests validate correctness of PutObjectPart.
@@ -289,7 +289,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 
 // Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup.
 func TestListMultipartUploads(t *testing.T) {
-	ExecObjectLayerTest(t, testListMultipartUploads)
+	ExecExtendedObjectLayerTest(t, testListMultipartUploads)
 }
 
 // testListMultipartUploads - Tests validate listing of multipart uploads.
@@ -1643,7 +1643,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 
 // Test for validating complete Multipart upload.
 func TestObjectCompleteMultipartUpload(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectCompleteMultipartUpload)
+	ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload)
 }
 
 // Tests validate CompleteMultipart functionality.

cmd/object-api-putobject_test.go

@@ -37,7 +37,7 @@ func md5Header(data []byte) map[string]string {
 
 // Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup.
 func TestObjectAPIPutObjectSingle(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectAPIPutObject)
+	ExecExtendedObjectLayerTest(t, testObjectAPIPutObject)
 }
 
 // Tests validate correctness of PutObject.

cmd/object-api-utils.go

@@ -394,9 +394,6 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 	if !ok {
 		return false, nil
 	}
-	if crypto.IsEncrypted(o.UserDefined) {
-		return true, fmt.Errorf("compression %q and encryption enabled on same object", scheme)
-	}
 	switch scheme {
 	case compressionAlgorithmV1, compressionAlgorithmV2:
 		return true, nil
@@ -415,9 +412,6 @@ func (o ObjectInfo) GetActualETag(h http.Header) string {
 
 // GetActualSize - returns the actual size of the stored object
 func (o ObjectInfo) GetActualSize() (int64, error) {
-	if crypto.IsEncrypted(o.UserDefined) {
-		return o.DecryptedSize()
-	}
 	if o.IsCompressed() {
 		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
 		if !ok {
@@ -429,6 +423,10 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
 		}
 		return size, nil
 	}
+	if crypto.IsEncrypted(o.UserDefined) {
+		return o.DecryptedSize()
+	}
+
 	return o.Size, nil
 }
 
@@ -441,7 +439,7 @@ func isCompressible(header http.Header, object string) bool {
 	globalCompressConfigMu.Unlock()
 
 	_, ok := crypto.IsRequested(header)
-	if !cfg.Enabled || ok || excludeForCompression(header, object, cfg) {
+	if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
 		return false
 	}
 	return true
@@ -461,16 +459,15 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
 	}
 
 	// Filter compression includes.
-	if len(cfg.Extensions) == 0 || len(cfg.MimeTypes) == 0 {
+	exclude := len(cfg.Extensions) > 0 || len(cfg.MimeTypes) > 0
-		return false
+	if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) {
+		exclude = false
 	}
 
-	extensions := cfg.Extensions
+	if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) {
-	mimeTypes := cfg.MimeTypes
+		exclude = false
-	if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
-		return false
 	}
-	return true
+	return exclude
 }
 
 // Utility which returns if a string is present in the list.
@@ -520,21 +517,29 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec {
 }
 
 // Returns the compressed offset which should be skipped.
-func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) {
+// If encrypted offsets are adjusted for encrypted block headers/trailers.
-	var compressedOffset int64
+// Since de-compression is after decryption encryption overhead is only added to compressedOffset.
+func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64) {
 	var skipLength int64
 	var cumulativeActualSize int64
+	var firstPartIdx int
 	if len(objectInfo.Parts) > 0 {
-		for _, part := range objectInfo.Parts {
+		for i, part := range objectInfo.Parts {
 			cumulativeActualSize += part.ActualSize
 			if cumulativeActualSize <= offset {
 				compressedOffset += part.Size
 			} else {
+				firstPartIdx = i
 				skipLength = cumulativeActualSize - part.ActualSize
 				break
 			}
 		}
 	}
+	if isEncryptedMultipart(objectInfo) && firstPartIdx > 0 {
+		off, _, _, _, _, err := objectInfo.GetDecryptedRange(partNumberToRangeSpec(objectInfo, firstPartIdx))
+		logger.LogIf(context.Background(), err)
+		compressedOffset += off
+	}
 	return compressedOffset, offset - skipLength
 }
 
@@ -604,71 +609,10 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 		isEncrypted = false
 	}
 	var skipLen int64
-	// Calculate range to read (different for
+	// Calculate range to read (different for encrypted/compressed objects)
-	// e.g. encrypted/compressed objects)
 	switch {
-	case isEncrypted:
-		var seqNumber uint32
-		var partStart int
-		off, length, skipLen, seqNumber, partStart, err = oi.GetDecryptedRange(rs)
-		if err != nil {
-			return nil, 0, 0, err
-		}
-		var decSize int64
-		decSize, err = oi.DecryptedSize()
-		if err != nil {
-			return nil, 0, 0, err
-		}
-		var decRangeLength int64
-		decRangeLength, err = rs.GetLength(decSize)
-		if err != nil {
-			return nil, 0, 0, err
-		}
-
-		// We define a closure that performs decryption given
-		// a reader that returns the desired range of
-		// encrypted bytes. The header parameter is used to
-		// provide encryption parameters.
-		fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
-			copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
-
-			cFns = append(cleanUpFns, cFns...)
-			// Attach decrypter on inputReader
-			var decReader io.Reader
-			decReader, err = DecryptBlocksRequestR(inputReader, h,
-				off, length, seqNumber, partStart, oi, copySource)
-			if err != nil {
-				// Call the cleanup funcs
-				for i := len(cFns) - 1; i >= 0; i-- {
-					cFns[i]()
-				}
-				return nil, err
-			}
-
-			if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
-				// Call the cleanup funcs
-				for i := len(cFns) - 1; i >= 0; i-- {
-					cFns[i]()
-				}
-				return nil, PreConditionFailed{}
-			}
-
-			oi.ETag = getDecryptedETag(h, oi, false)
-
-			// Apply the skipLen and limit on the
-			// decrypted stream
-			decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
-
-			// Assemble the GetObjectReader
-			r = &GetObjectReader{
-				ObjInfo:    oi,
-				pReader:    decReader,
-				cleanUpFns: cFns,
-				opts:       opts,
-			}
-			return r, nil
-		}
 	case isCompressed:
+		// If compressed, we start from the beginning of the part.
 		// Read the decompressed size from the meta.json.
 		actualSize, err := oi.GetActualSize()
 		if err != nil {
@@ -696,7 +640,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 				return nil, 0, 0, errInvalidRange
 			}
 		}
-		fn = func(inputReader io.Reader, _ http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
+		fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
 			cFns = append(cleanUpFns, cFns...)
 			if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
 				// Call the cleanup funcs
@@ -705,6 +649,18 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 				}
 				return nil, PreConditionFailed{}
 			}
+			if isEncrypted {
+				copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
+				// Attach decrypter on inputReader
+				inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, opts.PartNumber, oi, copySource)
+				if err != nil {
+					// Call the cleanup funcs
+					for i := len(cFns) - 1; i >= 0; i-- {
+						cFns[i]()
+					}
+					return nil, err
+				}
+			}
 			// Decompression reader.
 			s2Reader := s2.NewReader(inputReader)
 			// Apply the skipLen and limit on the decompressed stream.
@@ -739,6 +695,67 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 			return r, nil
 		}
 
+	case isEncrypted:
+		var seqNumber uint32
+		var partStart int
+		off, length, skipLen, seqNumber, partStart, err = oi.GetDecryptedRange(rs)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		var decSize int64
+		decSize, err = oi.DecryptedSize()
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		var decRangeLength int64
+		decRangeLength, err = rs.GetLength(decSize)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+
+		// We define a closure that performs decryption given
+		// a reader that returns the desired range of
+		// encrypted bytes. The header parameter is used to
+		// provide encryption parameters.
+		fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
+			copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
+
+			cFns = append(cleanUpFns, cFns...)
+			// Attach decrypter on inputReader
+			var decReader io.Reader
+			decReader, err = DecryptBlocksRequestR(inputReader, h, seqNumber, partStart, oi, copySource)
+			if err != nil {
+				// Call the cleanup funcs
+				for i := len(cFns) - 1; i >= 0; i-- {
+					cFns[i]()
+				}
+				return nil, err
+			}
+
+			if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
+				// Call the cleanup funcs
+				for i := len(cFns) - 1; i >= 0; i-- {
+					cFns[i]()
+				}
+				return nil, PreConditionFailed{}
+			}
+
+			oi.ETag = getDecryptedETag(h, oi, false)
+
+			// Apply the skipLen and limit on the
+			// decrypted stream
+			decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
+
+			// Assemble the GetObjectReader
+			r = &GetObjectReader{
+				ObjInfo:    oi,
+				pReader:    decReader,
+				cleanUpFns: cFns,
+				opts:       opts,
+			}
+			return r, nil
+		}
+
 	default:
 		off, length, err = rs.GetOffsetLength(oi.Size)
 		if err != nil {

cmd/object-api-utils_test.go

@@ -317,7 +317,7 @@ func TestIsCompressed(t *testing.T) {
 		result  bool
 		err     bool
 	}{
-		{
+		0: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1,
 					"content-type": "application/octet-stream",
@@ -325,7 +325,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: true,
 		},
-		{
+		1: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
 					"content-type": "application/octet-stream",
@@ -333,7 +333,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: true,
 		},
-		{
+		2: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type",
 					"content-type": "application/octet-stream",
@@ -342,7 +342,7 @@ func TestIsCompressed(t *testing.T) {
 			result: true,
 			err:    true,
 		},
-		{
+		3: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
 					"content-type": "application/octet-stream",
@@ -351,9 +351,9 @@ func TestIsCompressed(t *testing.T) {
 				},
 			},
 			result: true,
-			err:    true,
+			err:    false,
 		},
-		{
+		4: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2",
 					"content-type": "application/octet-stream",
@@ -361,7 +361,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: false,
 		},
-		{
+		5: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"content-type": "application/octet-stream",
 					"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},

cmd/object-handlers.go

@@ -977,18 +977,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	var reader io.Reader
-	var length = srcInfo.Size
 
-	// Set the actual size to the decrypted size if encrypted.
+	// Set the actual size to the compressed/decrypted size if encrypted.
-	actualSize := srcInfo.Size
+	actualSize, err := srcInfo.GetActualSize()
-	if crypto.IsEncrypted(srcInfo.UserDefined) {
+	if err != nil {
-		actualSize, err = srcInfo.DecryptedSize()
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		if err != nil {
+		return
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		length = actualSize
 	}
+	length := actualSize
 
 	if !cpSrcDstSame {
 		if err := enforceBucketQuota(ctx, dstBucket, actualSize); err != nil {
@@ -1000,8 +996,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	var compressMetadata map[string]string
 	// No need to compress for remote etcd calls
 	// Pass the decompressed stream to such calls.
-	isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
+	isDstCompressed := objectAPI.IsCompressionSupported() &&
-	if isCompressed {
+		isCompressible(r.Header, srcObject) &&
+		!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
+	if isDstCompressed {
 		compressMetadata = make(map[string]string, 2)
 		// Preserving the compression metadata.
 		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
@@ -1034,7 +1032,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	_, objectEncryption := crypto.IsRequested(r.Header)
 	objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined)
 	var encMetadata = make(map[string]string)
-	if objectAPI.IsEncryptionSupported() && !isCompressed {
+	if objectAPI.IsEncryptionSupported() {
 		// Encryption parameters not applicable for this object.
 		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
@@ -1105,8 +1103,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			var targetSize int64
 
 			switch {
+			case isDstCompressed:
+				targetSize = -1
 			case !isSourceEncrypted && !isTargetEncrypted:
-				targetSize = srcInfo.Size
+				targetSize, _ = srcInfo.GetActualSize()
 			case isSourceEncrypted && isTargetEncrypted:
 				objInfo := ObjectInfo{Size: actualSize}
 				targetSize = objInfo.EncryptedSize()
@@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			}
 
 			// do not try to verify encrypted content
-			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize, globalCLIContext.StrictS3Compat)
+			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1541,10 +1541,15 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			info := ObjectInfo{Size: size}
+
+			wantSize := int64(-1)
+			if size >= 0 {
+				info := ObjectInfo{Size: size}
+				wantSize = info.EncryptedSize()
+			}
 
 			// do not try to verify encrypted content
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
+			hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1564,10 +1569,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	switch {
-	case objInfo.IsCompressed():
-		if !strings.HasSuffix(objInfo.ETag, "-1") {
-			objInfo.ETag = objInfo.ETag + "-1"
-		}
 	case crypto.IsEncrypted(objInfo.UserDefined):
 		switch {
 		case crypto.S3.IsEncrypted(objInfo.UserDefined):
@@ -1581,6 +1582,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:]
 			}
 		}
+	case objInfo.IsCompressed():
+		if !strings.HasSuffix(objInfo.ETag, "-1") {
+			objInfo.ETag = objInfo.ETag + "-1"
+		}
 	}
 	if mustReplicate(ctx, r, bucket, object, metadata, "") {
 		globalReplicationState.queueReplicaTask(objInfo)
@@ -1892,7 +1897,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	actualPartSize := srcInfo.Size
 	if crypto.IsEncrypted(srcInfo.UserDefined) {
-		actualPartSize, err = srcInfo.DecryptedSize()
+		actualPartSize, err = srcInfo.GetActualSize()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -1991,7 +1996,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
-	if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
+	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
 			return
@@ -2022,8 +2027,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			return
 		}
 
-		info := ObjectInfo{Size: length}
+		wantSize := int64(-1)
-		srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length, globalCLIContext.StrictS3Compat)
+		if length >= 0 {
+			info := ObjectInfo{Size: length}
+			wantSize = info.EncryptedSize()
+		}
+
+		srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -2226,7 +2236,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
-	if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
+	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
 			return
@@ -2267,9 +2277,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		info := ObjectInfo{Size: size}
+		wantSize := int64(-1)
+		if size >= 0 {
+			info := ObjectInfo{Size: size}
+			wantSize = info.EncryptedSize()
+		}
 		// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
+		hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return

cmd/object-handlers_test.go

@@ -327,7 +327,7 @@ func TestAPIGetObjectHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()
 
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
 }
 
 func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -651,7 +651,7 @@ func TestAPIGetObjectWithMPHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()
 
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
 }
 
 func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -849,7 +849,7 @@ func TestAPIGetObjectWithPartNumberHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()
 
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
 }
 
 func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -971,7 +971,7 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
 // Wrapper for calling PutObject API handler tests using streaming signature v4 for both Erasure multiple disks and FS single drive setup.
 func TestAPIPutObjectStreamSigV4Handler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
 }
 
 func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1289,7 +1289,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 // Wrapper for calling PutObject API handler tests for both Erasure multiple disks and FS single drive setup.
 func TestAPIPutObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
 }
 
 func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1538,7 +1538,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 // expected.
 func TestAPICopyObjectPartHandlerSanity(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
 }
 
 func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1649,7 +1649,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 // Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup.
 func TestAPICopyObjectPartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
 }
 
 func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1965,7 +1965,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 // Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup.
 func TestAPICopyObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
 }
 
 func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2044,8 +2044,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// expected output.
 		expectedRespStatus int
 	}{
+		0: {
+			expectedRespStatus: http.StatusMethodNotAllowed,
+		},
 		// Test case - 1, copy metadata from newObject1, ignore request headers.
-		{
+		1: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2059,7 +2062,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 
 		// Test case - 2.
 		// Test case with invalid source object.
-		{
+		2: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator),
@@ -2071,7 +2074,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 
 		// Test case - 3.
 		// Test case with new object name is same as object to be copied.
-		{
+		3: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2084,7 +2087,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 4.
 		// Test case with new object name is same as object to be copied.
 		// But source copy is without leading slash
-		{
+		4: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(bucketName + SlashSeparator + objectName),
@@ -2097,7 +2100,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 5.
 		// Test case with new object name is same as object to be copied
 		// but metadata is updated.
-		{
+		5: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2113,7 +2116,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 
 		// Test case - 6.
 		// Test case with invalid metadata-directive.
-		{
+		6: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2130,7 +2133,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 7.
 		// Test case with new object name is same as object to be copied
 		// fail with BadRequest.
-		{
+		7: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2148,7 +2151,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case with non-existent source file.
 		// Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`.
 		// Expecting the response status code to http.StatusNotFound (404).
-		{
+		8: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"),
@@ -2162,7 +2165,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case with non-existent source file.
 		// Case for the purpose of failing `api.ObjectAPI.PutObject`.
 		// Expecting the response status code to http.StatusNotFound (404).
-		{
+		9: {
 			bucketName:       "non-existent-destination-bucket",
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2174,7 +2177,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 
 		// Test case - 10.
 		// Case with invalid AccessKey.
-		{
+		10: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2184,7 +2187,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 11, copy metadata from newObject1 with satisfying modified header.
-		{
+		11: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2194,7 +2197,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 12, copy metadata from newObject1 with unsatisfying modified header.
-		{
+		12: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2204,7 +2207,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusPreconditionFailed,
 		},
 		// Test case - 13, copy metadata from newObject1 with wrong modified header format
-		{
+		13: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2214,7 +2217,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 14, copy metadata from newObject1 with satisfying unmodified header.
-		{
+		14: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2224,7 +2227,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusOK,
 		},
 		// Test case - 15, copy metadata from newObject1 with unsatisfying unmodified header.
-		{
+		15: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2234,7 +2237,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusPreconditionFailed,
 		},
 		// Test case - 16, copy metadata from newObject1 with incorrect unmodified header format.
-		{
+		16: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2244,7 +2247,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusOK,
 		},
 		// Test case - 17, copy metadata from newObject1 with null versionId
-		{
+		17: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null",
@@ -2253,7 +2256,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 18, copy metadata from newObject1 with non null versionId
-		{
+		18: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17",
@@ -2273,7 +2276,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
+			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
 		}
 		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
 		if testCase.copySourceHeader != "" {
@@ -2303,25 +2306,35 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		apiRouter.ServeHTTP(rec, req)
 		// Assert the response code with the expected status.
 		if rec.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+			t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code)
 			continue
 		}
 		if rec.Code == http.StatusOK {
 			var cpObjResp CopyObjectResponse
 			if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
-				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
+				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i, instanceType, err)
 			}
 
 			// See if the new object is formed.
 			// testing whether the copy was successful.
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
+			// Note that this goes directly to the file system,
-			if err != nil {
+			// so encryption/compression may interfere at some point.
-				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
+
-			}
+			globalCompressConfigMu.Lock()
-			if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
+			cfg := globalCompressConfig
-				t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType)
+			globalCompressConfigMu.Unlock()
+			if !cfg.Enabled {
+				err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
+				if err != nil {
+					t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i, instanceType, err)
+				}
+				if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
+					t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i, instanceType)
+				}
+				buffers[0].Reset()
+			} else {
+				t.Log("object not validated due to compression")
 			}
-			buffers[0].Reset()
 		}
 
 		// Verify response of the V2 signed HTTP request.
@@ -2330,7 +2343,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 
 		reqV2, err = newTestRequest(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), 0, nil)
 		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
+			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
 		}
 		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
 		if testCase.copySourceHeader != "" {
@@ -2366,7 +2379,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV2, reqV2)
 		if recV2.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code)
 		}
 	}
 
@@ -3304,7 +3317,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 // when the request signature type is `streaming signature`.
 func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
 }
 
 func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3392,7 +3405,7 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
 //  for variety of inputs.
 func TestAPIPutObjectPartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
 }
 
 func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3797,7 +3810,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN
 //  for variety of success/failure cases.
 func TestAPIListObjectPartsHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
+	ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
 }
 
 func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,

cmd/object_api_suite_test.go

@@ -21,10 +21,12 @@ import (
 	"context"
 	"io"
 	"math/rand"
+	"os"
 	"strconv"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Return pointer to testOneByteReadEOF{}
@@ -68,10 +70,8 @@ func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-type ObjectLayerAPISuite struct{}
-
 // Wrapper for calling testMakeBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMakeBucket(t *testing.T) {
+func TestMakeBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testMakeBucket)
 }
 
@@ -84,8 +84,8 @@ func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }
 
 // Wrapper for calling testMultipartObjectCreation for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
+func TestMultipartObjectCreation(t *testing.T) {
-	ExecObjectLayerTest(t, testMultipartObjectCreation)
+	ExecExtendedObjectLayerTest(t, testMultipartObjectCreation)
 }
 
 // Tests validate creation of part files during Multipart operation.
@@ -128,7 +128,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 }
 
 // Wrapper for calling testMultipartObjectAbort for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
+func TestMultipartObjectAbort(t *testing.T) {
 	ExecObjectLayerTest(t, testMultipartObjectAbort)
 }
 
@@ -173,8 +173,8 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 }
 
 // Wrapper for calling testMultipleObjectCreation for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
+func TestMultipleObjectCreation(t *testing.T) {
-	ExecObjectLayerTest(t, testMultipleObjectCreation)
+	ExecExtendedObjectLayerTest(t, testMultipleObjectCreation)
 }
 
 // Tests validate object creation.
@@ -230,7 +230,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 }
 
 // Wrapper for calling TestPaging for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPaging(t *testing.T) {
+func TestPaging(t *testing.T) {
 	ExecObjectLayerTest(t, testPaging)
 }
 
@@ -434,7 +434,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }
 
 // Wrapper for calling testObjectOverwriteWorks for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(t *testing.T) {
+func TestObjectOverwriteWorks(t *testing.T) {
 	ExecObjectLayerTest(t, testObjectOverwriteWorks)
 }
 
@@ -471,7 +471,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 }
 
 // Wrapper for calling testNonExistantBucketOperations for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
+func TestNonExistantBucketOperations(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantBucketOperations)
 }
 
@@ -488,7 +488,7 @@ func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t Tes
 }
 
 // Wrapper for calling testBucketRecreateFails for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestBucketRecreateFails(t *testing.T) {
+func TestBucketRecreateFails(t *testing.T) {
 	ExecObjectLayerTest(t, testBucketRecreateFails)
 }
 
@@ -508,9 +508,68 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand
 	}
 }
 
+func execExtended(t *testing.T, fn func(t *testing.T)) {
+	// Exec with default settings...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = false
+	globalCompressConfigMu.Unlock()
+	t.Run("default", func(t *testing.T) {
+		fn(t)
+	})
+	if testing.Short() {
+		return
+	}
+
+	// Enable compression and exec...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = true
+	globalCompressConfig.MimeTypes = nil
+	globalCompressConfig.Extensions = nil
+	globalCompressConfigMu.Unlock()
+	t.Run("compressed", func(t *testing.T) {
+		fn(t)
+	})
+
+	globalAutoEncryption = true
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	var err error
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("encrypted", func(t *testing.T) {
+		fn(t)
+	})
+
+	// Enable compression of encrypted and exec...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.AllowEncrypted = true
+	globalCompressConfigMu.Unlock()
+	t.Run("compressed+encrypted", func(t *testing.T) {
+		fn(t)
+	})
+
+	// Reset...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = false
+	globalCompressConfig.AllowEncrypted = false
+	globalCompressConfigMu.Unlock()
+	globalAutoEncryption = false
+}
+
+// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
+// This can be used to test functionality when reading and writing data.
+func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) {
+	execExtended(t, func(t *testing.T) {
+		ExecObjectLayerTest(t, objTest)
+	})
+}
+
 // Wrapper for calling testPutObject for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPutObject(t *testing.T) {
+func TestPutObject(t *testing.T) {
-	ExecObjectLayerTest(t, testPutObject)
+	ExecExtendedObjectLayerTest(t, testPutObject)
 }
 
 // Tests validate PutObject without prefix.
@@ -553,8 +612,8 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }
 
 // Wrapper for calling testPutObjectInSubdir for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(t *testing.T) {
+func TestPutObjectInSubdir(t *testing.T) {
-	ExecObjectLayerTest(t, testPutObjectInSubdir)
+	ExecExtendedObjectLayerTest(t, testPutObjectInSubdir)
 }
 
 // Tests validate PutObject with subdirectory prefix.
@@ -585,7 +644,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 }
 
 // Wrapper for calling testListBuckets for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListBuckets(t *testing.T) {
+func TestListBuckets(t *testing.T) {
 	ExecObjectLayerTest(t, testListBuckets)
 }
 
@@ -644,7 +703,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }
 
 // Wrapper for calling testListBucketsOrder for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListBucketsOrder(t *testing.T) {
+func TestListBucketsOrder(t *testing.T) {
 	ExecObjectLayerTest(t, testListBucketsOrder)
 }
 
@@ -678,7 +737,7 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
 }
 
 // Wrapper for calling testListObjectsTestsForNonExistantBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(t *testing.T) {
+func TestListObjectsTestsForNonExistantBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testListObjectsTestsForNonExistantBucket)
 }
 
@@ -700,7 +759,7 @@ func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType stri
 }
 
 // Wrapper for calling testNonExistantObjectInBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(t *testing.T) {
+func TestNonExistantObjectInBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantObjectInBucket)
 }
 
@@ -716,8 +775,8 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s: Expected error but found nil", instanceType)
 	}
 	if isErrObjectNotFound(err) {
-		if err.Error() != "Object not found: bucket#dir1" {
+		if err.Error() != "Object not found: bucket/dir1" {
-			t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket#dir1", err.Error())
+			t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket/dir1", err.Error())
 		}
 	} else {
 		if err.Error() != "fails" {
@@ -727,7 +786,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 }
 
 // Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
+func TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
 	ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound)
 }
 
@@ -770,7 +829,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 }
 
 // Wrapper for calling testContentType for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestContentType(t *testing.T) {
+func TestContentType(t *testing.T) {
 	ExecObjectLayerTest(t, testContentType)
 }
 

cmd/test-utils_test.go

@@ -1866,6 +1866,14 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	removeRoots(append(erasureDisks, fsDir))
 }
 
+// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
+// This can be used to test functionality when reading and writing data.
+func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
+	execExtended(t, func(t *testing.T) {
+		ExecObjectLayerAPITest(t, objAPITest, endpoints)
+	})
+}
+
 // function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
 type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string,
 	apiRouter http.Handler, credentials auth.Credentials, t *testing.T)

docs/bucket/versioning/DESIGN.md

@@ -26,7 +26,7 @@ Once the header is validated, we proceed to the actual data structure of the `xl
 - LegacyObjectType (preserves existing deployments and older xl.json format)
 - DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility)
 
-A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta-to-json.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta-to-json.go) program.
+A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta.go) program.
 ```json
 {
   "Versions": [

docs/bucket/versioning/xl-meta.go

@@ -56,11 +56,12 @@ GLOBAL FLAGS:
 	}
 
 	app.Action = func(c *cli.Context) error {
-		if !c.Args().Present() {
+		files := c.Args()
-			cli.ShowAppHelp(c)
+		if len(files) == 0 {
-			return nil
+			// If no args, assume xl.meta
+			files = []string{"xl.meta"}
 		}
-		for _, file := range c.Args() {
+		for _, file := range files {
 			var r io.Reader
 			switch file {
 			case "-":

docs/compression/README.md

@@ -1,10 +1,19 @@
 # Compression Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 
-MinIO server allows streaming compression to ensure efficient disk space usage. Compression happens inflight, i.e objects are compressed before being written to disk(s). MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) streaming compression due to its stability and performance.
+MinIO server allows streaming compression to ensure efficient disk space usage. 
+Compression happens inflight, i.e objects are compressed before being written to disk(s). 
+MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) 
+streaming compression due to its stability and performance.
 
-This algorithm is specifically optimized for machine generated content. Write throughput is typically at least 300MB/s per CPU core. Decompression speed is typically at least 1GB/s.
+This algorithm is specifically optimized for machine generated content. 
-This means that in cases where raw IO is below these numbers compression will not only reduce disk usage but also help increase system throughput.
+Write throughput is typically at least 500MB/s per CPU core,
-Typically enabling compression on spinning disk systems will increase speed when the content can be compressed.
+and scales with the number of available CPU cores. 
+Decompression speed is typically at least 1GB/s.
+
+This means that in cases where raw IO is below these numbers 
+compression will not only reduce disk usage but also help increase system throughput.
+Typically, enabling compression on spinning disk systems 
+will increase speed when the content can be compressed.
 
 ## Get Started
 
@@ -14,40 +23,71 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta
 
 ### 2. Run MinIO with compression
 
-Compression can be enabled by updating the `compress` config settings for MinIO server config. Config `compress` settings take extensions and mime-types to be compressed.
+Compression can be enabled by updating the `compress` config settings for MinIO server config. 
+Config `compress` settings take extensions and mime-types to be compressed.
 
-```
+```bash
-$ mc admin config get myminio compression
+~ mc admin config get myminio compression
 compression extensions=".txt,.log,.csv,.json,.tar,.xml,.bin" mime_types="text/*,application/json,application/xml"
 ```
 
 Default config includes most common highly compressible content extensions and mime-types.
 
-```
+```bash
-$ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf"
+~ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf"
 ```
 
 To show help on setting compression config values.
-```
+```bash
 ~ mc admin config set myminio compression
 ```
 
-To enable compression for all content, with default extensions and mime-types.
+To enable compression for all content, no matter the extension and content type 
-```
+(except for the default excluded types) set BOTH extensions and mime types to empty.
-~ mc admin config set myminio compression enable="on"
+
+```bash
+~ mc admin config set myminio compression enable="on" extensions="" mime_types=""
 ```
 
-The compression settings may also be set through environment variables. When set, environment variables override the defined `compress` config settings in the server config.
+The compression settings may also be set through environment variables. 
+When set, environment variables override the defined `compress` config settings in the server config.
 
 ```bash
 export MINIO_COMPRESS="on"
-export MINIO_COMPRESS_EXTENSIONS=".pdf,.doc"
+export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
-export MINIO_COMPRESS_MIME_TYPES="application/pdf"
+export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml"
 ```
 
-### 3. Note
+### 3. Compression + Encryption
+
+Combining encryption and compression is not safe in all setups.
+This is particularly so if the compression ratio of your content reveals information about it.
+See [CRIME TLS](https://en.wikipedia.org/wiki/CRIME) as an example of this.
+
+Therefore, compression is disabled when encrypting by default, and must be enabled separately.
+
+Consult our security experts on [SUBNET](https://min.io/pricing) to help you evaluate if 
+your setup can use this feature combination safely.
+
+To enable compression+encryption use:
+
+```bash
+~ mc admin config set myminio compression allow_encryption=on
+```
+
+Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`.
+
+### 4. Excluded Types
+
+- Already compressed objects are not fit for compression since they do not have compressible patterns. 
+Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78)
+which is a fitness factor for a lossless data compression.
+
+Pre-compressed input typically compresses in excess of 2GiB/s per core, 
+so performance impact should be minimal even if precompressed data is re-compressed.
+Decompressing incompressible data has no significant performance impact.
 
-- Already compressed objects are not fit for compression since they do not have compressible patterns. Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78) which is a fitness factor for a lossless data compression. Below is a list of common files and content-types which are not suitable for compression.
+Below is a list of common files and content-types which are typically not suitable for compression.
 
     - Extensions
 
@@ -72,15 +112,17 @@ export MINIO_COMPRESS_MIME_TYPES="application/pdf"
       | `application/x-compress` |
       | `application/x-xz` |
 
-All files with these extensions and mime types are excluded from compression, even if compression is enabled for all types.
+All files with these extensions and mime types are excluded from compression, 
+even if compression is enabled for all types.
 
-- MinIO does not support encryption with compression because compression and encryption together potentially enables room for side channel attacks like [`CRIME and BREACH`](https://blog.minio.io/c-e-compression-encryption-cb6b7f04a369)
+### 5. Notes
 
 - MinIO does not support compression for Gateway (Azure/GCS/NAS) implementations.
 
 ## To test the setup
 
-To test this setup, practice put calls to the server using `mc` and use `mc ls` on the data directory to view the size of the object.
+To test this setup, practice put calls to the server using `mc` and use `mc ls` on 
+the data directory to view the size of the object.
 
 ## Explore Further