Merge pull request #308 from fkautz/pr_out_metadata_now_stores_hex_md5_not_base64_minio_api_now_converts_md5_hex_to_base64

pkg/api/minioapi/api_test.go

@@ -32,6 +32,7 @@ import (
 	mstorage "github.com/minio-io/minio/pkg/storage"
 	"github.com/minio-io/minio/pkg/storage/memory"
 
+	"encoding/base64"
 	. "gopkg.in/check.v1"
 )
 
@@ -73,7 +74,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
 
 	metadata, err := storage.GetObjectMetadata("bucket", "object", "")
 	c.Assert(err, IsNil)
-	verifyHeaders(c, response.Header, metadata.Created, 0, "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, 0, "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 
 	// TODO Test Headers
 }
@@ -98,7 +99,7 @@ func (s *MySuite) TestObject(c *C) {
 
 	metadata, err := storage.GetObjectMetadata("bucket", "object", "")
 	c.Assert(err, IsNil)
-	verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 }
 
 func (s *MySuite) TestMultipleObjects(c *C) {
@@ -134,7 +135,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
 	c.Assert(response.StatusCode, Equals, http.StatusOK)
 
 	// verify headers
-	verifyHeaders(c, response.Header, metadata.Created, len("hello one"), "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, len("hello one"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 	c.Assert(err, IsNil)
 
 	// verify response data
@@ -153,7 +154,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
 	c.Assert(response.StatusCode, Equals, http.StatusOK)
 
 	// verify headers
-	verifyHeaders(c, response.Header, metadata.Created, len("hello two"), "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, len("hello two"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 	c.Assert(err, IsNil)
 
 	// verify response data
@@ -172,7 +173,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
 	c.Assert(response.StatusCode, Equals, http.StatusOK)
 
 	// verify headers
-	verifyHeaders(c, response.Header, metadata.Created, len("hello three"), "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, len("hello three"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 	c.Assert(err, IsNil)
 
 	// verify object
@@ -212,7 +213,7 @@ func (s *MySuite) TestHeader(c *C) {
 
 	metadata, err := storage.GetObjectMetadata("bucket", "object", "")
 	c.Assert(err, IsNil)
-	verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.ETag)
+	verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 }
 
 func (s *MySuite) TestPutBucket(c *C) {

pkg/api/minioapi/headers.go

@@ -24,6 +24,7 @@ import (
 	"strconv"
 	"time"
 
+	"encoding/base64"
 	mstorage "github.com/minio-io/minio/pkg/storage"
 )
 
@@ -62,7 +63,7 @@ func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata)
 	lastModified := metadata.Created.Format(time.RFC1123)
 	// common headers
 	writeCommonHeaders(w, metadata.ContentType)
-	w.Header().Set("ETag", metadata.ETag)
+	w.Header().Set("ETag", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 	w.Header().Set("Last-Modified", lastModified)
 	w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
 	w.Header().Set("Connection", "close")
@@ -73,7 +74,7 @@ func writeRangeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMeta
 	lastModified := metadata.Created.Format(time.RFC1123)
 	// common headers
 	writeCommonHeaders(w, metadata.ContentType)
-	w.Header().Set("ETag", metadata.ETag)
+	w.Header().Set("ETag", base64.StdEncoding.EncodeToString([]byte(metadata.Md5)))
 	w.Header().Set("Last-Modified", lastModified)
 	w.Header().Set("Content-Range", ra)
 	w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))

pkg/api/minioapi/response.go

@@ -19,6 +19,7 @@ package minioapi
 import (
 	"sort"
 
+	"encoding/hex"
 	mstorage "github.com/minio-io/minio/pkg/storage"
 )
 
@@ -89,7 +90,7 @@ func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata,
 		}
 		content.Key = object.Key
 		content.LastModified = object.Created.Format(iso8601Format)
-		content.ETag = object.ETag
+		content.ETag = hex.EncodeToString([]byte(object.Md5))
 		content.Size = object.Size
 		content.StorageClass = "STANDARD"
 		content.Owner = owner

pkg/storage/encoded/encoded.go

@@ -18,7 +18,6 @@ package encoded
 
 import (
 	"bytes"
-	"encoding/hex"
 	"errors"
 	"github.com/minio-io/minio/pkg/donutbox"
 	"github.com/minio-io/minio/pkg/encoding/erasure"
@@ -152,7 +151,7 @@ func (diskStorage StorageDriver) GetObjectMetadata(bucket, key string, prefix st
 		Key:         key,
 		ContentType: metadata["contentType"],
 		Created:     created,
-		ETag:        hex.EncodeToString([]byte(metadata["md5"])),
+		Md5:         metadata["md5"],
 		Size:        size,
 	}
 	return objectMetadata, nil
@@ -210,7 +209,7 @@ func (diskStorage StorageDriver) CreateObject(bucketKey string, objectKey string
 
 		ContentType: contentType,
 		Created:     time.Now(),
-		ETag:        "md5",
+		Md5:         "md5",
 		Size:        int64(totalLength),
 	}
 
@@ -248,7 +247,7 @@ func createMetadata(metadataObject storage.ObjectMetadata, blockSize int, k, m u
 	metadata["key"] = metadataObject.Key
 	metadata["contentType"] = metadataObject.ContentType
 	metadata["created"] = metadataObject.Created.Format(time.RFC3339Nano)
-	metadata["md5"] = metadataObject.ETag
+	metadata["md5"] = metadataObject.Md5
 	metadata["size"] = strconv.FormatInt(metadataObject.Size, 10)
 
 	metadata["blockSize"] = strconv.FormatUint(uint64(blockSize), 10)

pkg/storage/file/file_object.go

@@ -174,7 +174,7 @@ func (storage *Storage) GetObjectMetadata(bucket, object, prefix string) (mstora
 		Key:         trimmedObject,
 		Created:     stat.ModTime(),
 		Size:        stat.Size(),
-		ETag:        etag,
+		Md5:         etag,
 		ContentType: contentType,
 	}
 

pkg/storage/memory/memory.go

@@ -19,7 +19,6 @@ package memory
 import (
 	"bufio"
 	"bytes"
-	"crypto/sha256"
 	"fmt"
 	"io"
 	"sort"
@@ -27,6 +26,8 @@ import (
 	"sync"
 	"time"
 
+	"crypto/md5"
+	"encoding/hex"
 	mstorage "github.com/minio-io/minio/pkg/storage"
 )
 
@@ -114,14 +115,15 @@ func (storage *Storage) CreateObject(bucket, key, contentType string, data io.Re
 	var newObject = storedObject{}
 	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
 		size := bytesBuffer.Len()
-		etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes()))
+		md5SumBytes := md5.Sum(bytesBuffer.Bytes())
+		md5Sum := hex.EncodeToString(md5SumBytes[:])
 		newObject.metadata = mstorage.ObjectMetadata{
 			Bucket: bucket,
 			Key:    key,
 
 			ContentType: contentType,
 			Created:     time.Now(),
-			ETag:        etag,
+			Md5:         md5Sum,
 			Size:        int64(size),
 		}
 		newObject.data = bytesBuffer.Bytes()

pkg/storage/storage.go

@@ -52,7 +52,7 @@ type ObjectMetadata struct {
 
 	ContentType string
 	Created     time.Time
-	ETag        string
+	Md5         string
 	Size        int64
 }
 

pkg/storage/storage_api_suite.go

@@ -74,9 +74,9 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) {
 		c.Assert(err, check.IsNil)
 		c.Assert(metadata.Size, check.Equals, int64(len(value)))
 
-		_, ok := etags[metadata.ETag]
+		_, ok := etags[metadata.Md5]
 		c.Assert(ok, check.Equals, false)
-		etags[metadata.ETag] = metadata.ETag
+		etags[metadata.Md5] = metadata.Md5
 	}
 }