From 583c32c7742a3b5e11111cc3f4ce73c10ebe24e3 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Mon, 16 Mar 2015 18:03:03 -0700 Subject: [PATCH] Metadata now stores hex md5, not base64. Minio API now converts md5 hex to base64. --- pkg/api/minioapi/api_test.go | 13 +++++++------ pkg/api/minioapi/headers.go | 5 +++-- pkg/api/minioapi/response.go | 3 ++- pkg/storage/encoded/encoded.go | 7 +++---- pkg/storage/file/file_object.go | 2 +- pkg/storage/memory/memory.go | 8 +++++--- pkg/storage/storage.go | 2 +- pkg/storage/storage_api_suite.go | 4 ++-- 8 files changed, 24 insertions(+), 20 deletions(-) diff --git a/pkg/api/minioapi/api_test.go b/pkg/api/minioapi/api_test.go index 0cb200db7..a75fccad6 100644 --- a/pkg/api/minioapi/api_test.go +++ b/pkg/api/minioapi/api_test.go @@ -32,6 +32,7 @@ import ( mstorage "github.com/minio-io/minio/pkg/storage" "github.com/minio-io/minio/pkg/storage/memory" + "encoding/base64" . "gopkg.in/check.v1" ) @@ -73,7 +74,7 @@ func (s *MySuite) TestEmptyObject(c *C) { metadata, err := storage.GetObjectMetadata("bucket", "object", "") c.Assert(err, IsNil) - verifyHeaders(c, response.Header, metadata.Created, 0, "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, 0, "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) // TODO Test Headers } @@ -98,7 +99,7 @@ func (s *MySuite) TestObject(c *C) { metadata, err := storage.GetObjectMetadata("bucket", "object", "") c.Assert(err, IsNil) - verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) } func (s *MySuite) TestMultipleObjects(c *C) { @@ -134,7 +135,7 @@ func (s *MySuite) TestMultipleObjects(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello one"), "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, len("hello one"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) c.Assert(err, IsNil) // verify response data @@ -153,7 +154,7 @@ func (s *MySuite) TestMultipleObjects(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello two"), "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, len("hello two"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) c.Assert(err, IsNil) // verify response data @@ -172,7 +173,7 @@ func (s *MySuite) TestMultipleObjects(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello three"), "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, len("hello three"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) c.Assert(err, IsNil) // verify object @@ -212,7 +213,7 @@ func (s *MySuite) TestHeader(c *C) { metadata, err := storage.GetObjectMetadata("bucket", "object", "") c.Assert(err, IsNil) - verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.ETag) + verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) } func (s *MySuite) TestPutBucket(c *C) { diff --git a/pkg/api/minioapi/headers.go b/pkg/api/minioapi/headers.go index cb30964d3..3bdbd6199 100644 --- a/pkg/api/minioapi/headers.go +++ b/pkg/api/minioapi/headers.go @@ -24,6 +24,7 @@ import ( "strconv" "time" + "encoding/base64" mstorage "github.com/minio-io/minio/pkg/storage" ) @@ -62,7 +63,7 @@ func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata) lastModified := metadata.Created.Format(time.RFC1123) // common headers writeCommonHeaders(w, metadata.ContentType) - w.Header().Set("ETag", metadata.ETag) + w.Header().Set("ETag", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) w.Header().Set("Last-Modified", lastModified) w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) w.Header().Set("Connection", "close") @@ -73,7 +74,7 @@ func writeRangeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMeta lastModified := metadata.Created.Format(time.RFC1123) // common headers writeCommonHeaders(w, metadata.ContentType) - w.Header().Set("ETag", metadata.ETag) + w.Header().Set("ETag", base64.StdEncoding.EncodeToString([]byte(metadata.Md5))) w.Header().Set("Last-Modified", lastModified) w.Header().Set("Content-Range", ra) w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) diff --git a/pkg/api/minioapi/response.go b/pkg/api/minioapi/response.go index d7eea2ca8..9bca7fbca 100644 --- a/pkg/api/minioapi/response.go +++ b/pkg/api/minioapi/response.go @@ -19,6 +19,7 @@ package minioapi import ( "sort" + "encoding/hex" mstorage "github.com/minio-io/minio/pkg/storage" ) @@ -89,7 +90,7 @@ func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, } content.Key = object.Key content.LastModified = object.Created.Format(iso8601Format) - content.ETag = object.ETag + content.ETag = hex.EncodeToString([]byte(object.Md5)) content.Size = object.Size content.StorageClass = "STANDARD" content.Owner = owner diff --git a/pkg/storage/encoded/encoded.go b/pkg/storage/encoded/encoded.go index c9cb0a5dc..181d9977a 100644 --- a/pkg/storage/encoded/encoded.go +++ b/pkg/storage/encoded/encoded.go @@ -18,7 +18,6 @@ package encoded import ( "bytes" - "encoding/hex" "errors" "github.com/minio-io/minio/pkg/donutbox" "github.com/minio-io/minio/pkg/encoding/erasure" @@ -152,7 +151,7 @@ func (diskStorage StorageDriver) GetObjectMetadata(bucket, key string, prefix st Key: key, ContentType: metadata["contentType"], Created: created, - ETag: hex.EncodeToString([]byte(metadata["md5"])), + Md5: metadata["md5"], Size: size, } return objectMetadata, nil @@ -210,7 +209,7 @@ func (diskStorage StorageDriver) CreateObject(bucketKey string, objectKey string ContentType: contentType, Created: time.Now(), - ETag: "md5", + Md5: "md5", Size: int64(totalLength), } @@ -248,7 +247,7 @@ func createMetadata(metadataObject storage.ObjectMetadata, blockSize int, k, m u metadata["key"] = metadataObject.Key metadata["contentType"] = metadataObject.ContentType metadata["created"] = metadataObject.Created.Format(time.RFC3339Nano) - metadata["md5"] = metadataObject.ETag + metadata["md5"] = metadataObject.Md5 metadata["size"] = strconv.FormatInt(metadataObject.Size, 10) metadata["blockSize"] = strconv.FormatUint(uint64(blockSize), 10) diff --git a/pkg/storage/file/file_object.go b/pkg/storage/file/file_object.go index 719a83c8d..3301f5d6b 100644 --- a/pkg/storage/file/file_object.go +++ b/pkg/storage/file/file_object.go @@ -174,7 +174,7 @@ func (storage *Storage) GetObjectMetadata(bucket, object, prefix string) (mstora Key: trimmedObject, Created: stat.ModTime(), Size: stat.Size(), - ETag: etag, + Md5: etag, ContentType: contentType, } diff --git a/pkg/storage/memory/memory.go b/pkg/storage/memory/memory.go index b995466d6..16cf99705 100644 --- a/pkg/storage/memory/memory.go +++ b/pkg/storage/memory/memory.go @@ -19,7 +19,6 @@ package memory import ( "bufio" "bytes" - "crypto/sha256" "fmt" "io" "sort" @@ -27,6 +26,8 @@ import ( "sync" "time" + "crypto/md5" + "encoding/hex" mstorage "github.com/minio-io/minio/pkg/storage" ) @@ -114,14 +115,15 @@ func (storage *Storage) CreateObject(bucket, key, contentType string, data io.Re var newObject = storedObject{} if _, ok := io.Copy(&bytesBuffer, data); ok == nil { size := bytesBuffer.Len() - etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) + md5SumBytes := md5.Sum(bytesBuffer.Bytes()) + md5Sum := hex.EncodeToString(md5SumBytes[:]) newObject.metadata = mstorage.ObjectMetadata{ Bucket: bucket, Key: key, ContentType: contentType, Created: time.Now(), - ETag: etag, + Md5: md5Sum, Size: int64(size), } newObject.data = bytesBuffer.Bytes() diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f984f608b..1231efd45 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -52,7 +52,7 @@ type ObjectMetadata struct { ContentType string Created time.Time - ETag string + Md5 string Size int64 } diff --git a/pkg/storage/storage_api_suite.go b/pkg/storage/storage_api_suite.go index 1f5c62486..7939b27ba 100644 --- a/pkg/storage/storage_api_suite.go +++ b/pkg/storage/storage_api_suite.go @@ -74,9 +74,9 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) { c.Assert(err, check.IsNil) c.Assert(metadata.Size, check.Equals, int64(len(value))) - _, ok := etags[metadata.ETag] + _, ok := etags[metadata.Md5] c.Assert(ok, check.Equals, false) - etags[metadata.ETag] = metadata.ETag + etags[metadata.Md5] = metadata.Md5 } }