Set etag properly to calculated value if available (#5106)

Fixes #5100

cmd/fs-v1-multipart.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -571,7 +572,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
-	md5hex := data.MD5HexString()
+	md5hex := hex.EncodeToString(data.MD5Current())
 
 	// Save the object part info in `fs.json`.
 	fsMeta.AddObjectPart(partID, partSuffix, md5hex, data.Size())

cmd/fs-v1.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -593,8 +594,6 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 		return ObjectInfo{}, traceError(errInvalidArgument)
 	}
 
-	metadata["etag"] = data.MD5HexString()
-
 	var wlk *lock.LockedFile
 	if bucket != minioMetaBucket {
 		bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
@@ -634,6 +633,8 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	metadata["etag"] = hex.EncodeToString(data.MD5Current())
+
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {

cmd/xl-v1-multipart.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -694,7 +695,7 @@ func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 	// Once part is successfully committed, proceed with updating XL metadata.
 	xlMeta.Stat.ModTime = UTCNow()
 
-	md5hex := data.MD5HexString()
+	md5hex := hex.EncodeToString(data.MD5Current())
 
 	// Add the current part.
 	xlMeta.AddObjectPart(partID, partSuffix, md5hex, file.Size)

cmd/xl-v1-object.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"io"
 	"path"
 	"strconv"
@@ -579,7 +580,7 @@ func (xl xlObjects) PutObject(bucket string, object string, data *hash.Reader, m
 
 	// Save additional erasureMetadata.
 	modTime := UTCNow()
-	metadata["etag"] = data.MD5HexString()
+	metadata["etag"] = hex.EncodeToString(data.MD5Current())
 
 	// Guess content-type from the extension if possible.
 	if metadata["content-type"] == "" {

pkg/hash/reader.go

@@ -52,13 +52,8 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string) (*Reader, er
 	if err != nil {
 		return nil, err
 	}
-	var (
+
-		md5Hash    hash.Hash
+	var sha256Hash hash.Hash
-		sha256Hash hash.Hash
-	)
-	if len(md5sum) != 0 {
-		md5Hash = md5.New()
-	}
 	if len(sha256sum) != 0 {
 		sha256Hash = sha256.New()
 	}
@@ -68,7 +63,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string) (*Reader, er
 		sha256sum:  sha256sum,
 		src:        io.LimitReader(src, size),
 		size:       size,
-		md5Hash:    md5Hash,
+		md5Hash:    md5.New(),
 		sha256Hash: sha256Hash,
 	}, nil
 }
@@ -76,9 +71,7 @@ func NewReader(src io.Reader, size int64, md5Hex, sha256Hex string) (*Reader, er
 func (r *Reader) Read(p []byte) (n int, err error) {
 	n, err = r.src.Read(p)
 	if n > 0 {
-		if r.md5Hash != nil {
+		r.md5Hash.Write(p[:n])
-			r.md5Hash.Write(p[:n])
-		}
 		if r.sha256Hash != nil {
 			r.sha256Hash.Write(p[:n])
 		}
@@ -104,6 +97,14 @@ func (r *Reader) MD5() []byte {
 	return r.md5sum
 }
 
+// MD5Current - returns byte md5 value of the current state
+// of the md5 hash after reading the incoming content.
+// NOTE: Calling this function multiple times might yield
+// different results if they are intermixed with Reader.
+func (r *Reader) MD5Current() []byte {
+	return r.md5Hash.Sum(nil)
+}
+
 // SHA256 - returns byte sha256 value
 func (r *Reader) SHA256() []byte {
 	return r.sha256sum
@@ -122,12 +123,12 @@ func (r *Reader) SHA256HexString() string {
 // Verify verifies if the computed MD5 sum and SHA256 sum are
 // equal to the ones specified when creating the Reader.
 func (r *Reader) Verify() error {
-	if r.sha256Hash != nil {
+	if r.sha256Hash != nil && len(r.sha256sum) > 0 {
 		if sum := r.sha256Hash.Sum(nil); !bytes.Equal(r.sha256sum, sum) {
 			return SHA256Mismatch{hex.EncodeToString(r.sha256sum), hex.EncodeToString(sum)}
 		}
 	}
-	if r.md5Hash != nil {
+	if len(r.md5sum) > 0 {
 		if sum := r.md5Hash.Sum(nil); !bytes.Equal(r.md5sum, sum) {
 			return BadDigest{hex.EncodeToString(r.md5sum), hex.EncodeToString(sum)}
 		}

pkg/hash/reader_test.go

@@ -50,6 +50,9 @@ func TestHashReaderHelperMethods(t *testing.T) {
 	if !bytes.Equal(r.MD5(), expectedMD5) {
 		t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", r.MD5HexString())
 	}
+	if !bytes.Equal(r.MD5Current(), expectedMD5) {
+		t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(r.MD5Current()))
+	}
 	expectedSHA256, err := hex.DecodeString("88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589")
 	if !bytes.Equal(r.SHA256(), expectedSHA256) {
 		t.Errorf("Expected md5hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString())