Merge pull request #133 from fkautz/pr_out_adding_content_type_to_backend_storage_drivers

pkg/api/minioapi/minioapi.go

@@ -192,7 +192,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques
 	vars := mux.Vars(req)
 	bucket := vars["bucket"]
 	object := vars["object"]
-	err := server.storage.StoreObject(bucket, object, req.Body)
+	err := server.storage.StoreObject(bucket, object, "", req.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		w.Write([]byte(err.Error()))

pkg/api/minioapi/minioapi_test.go

@@ -57,7 +57,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
 
 	buffer := bytes.NewBufferString("")
 	storage.StoreBucket("bucket")
-	storage.StoreObject("bucket", "object", buffer)
+	storage.StoreObject("bucket", "object", "", buffer)
 
 	response, err := http.Get(testServer.URL + "/bucket/object")
 	c.Assert(err, IsNil)
@@ -82,7 +82,7 @@ func (s *MySuite) TestObject(c *C) {
 
 	buffer := bytes.NewBufferString("hello world")
 	storage.StoreBucket("bucket")
-	storage.StoreObject("bucket", "object", buffer)
+	storage.StoreObject("bucket", "object", "", buffer)
 
 	response, err := http.Get(testServer.URL + "/bucket/object")
 	c.Assert(err, IsNil)
@@ -108,9 +108,9 @@ func (s *MySuite) TestMultipleObjects(c *C) {
 	buffer3 := bytes.NewBufferString("hello three")
 
 	storage.StoreBucket("bucket")
-	storage.StoreObject("bucket", "object1", buffer1)
-	storage.StoreObject("bucket", "object2", buffer2)
-	storage.StoreObject("bucket", "object3", buffer3)
+	storage.StoreObject("bucket", "object1", "", buffer1)
+	storage.StoreObject("bucket", "object2", "", buffer2)
+	storage.StoreObject("bucket", "object3", "", buffer3)
 
 	// test non-existant object
 	response, err := http.Get(testServer.URL + "/bucket/object")
@@ -200,7 +200,7 @@ func (s *MySuite) TestHeader(c *C) {
 
 	buffer := bytes.NewBufferString("hello world")
 	storage.StoreBucket("bucket")
-	storage.StoreObject("bucket", "object", buffer)
+	storage.StoreObject("bucket", "object", "", buffer)
 
 	response, err = http.Get(testServer.URL + "/bucket/object")
 	c.Assert(err, IsNil)

pkg/storage/fs/fs.go

@@ -1,6 +1,8 @@
 package fs
 
 import (
+	"bytes"
+	"encoding/json"
 	"io"
 	"io/ioutil"
 	"os"
@@ -16,6 +18,10 @@ type storage struct {
 	writeLock sync.Mutex
 }
 
+type SerializedMetadata struct {
+	ContentType string
+}
+
 type MkdirFailedError struct{}
 
 func (self MkdirFailedError) Error() string {
@@ -146,12 +152,41 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag
 		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
 	}
 
+	_, err = os.Stat(objectPath + "$metadata")
+	if os.IsNotExist(err) {
+		return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
+	}
+
+	file, err := os.Open(objectPath + "$metadata")
+	defer file.Close()
+	if err != nil {
+		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
+	}
+
+	metadataBuffer, err := ioutil.ReadAll(file)
+	if err != nil {
+		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
+	}
+
+	var deserializedMetadata SerializedMetadata
+	err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
+	if err != nil {
+		return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
+	}
+
+	contentType := "application/octet-stream"
+	if deserializedMetadata.ContentType != "" {
+		contentType = deserializedMetadata.ContentType
+	}
+	contentType = strings.TrimSpace(contentType)
+
 	metadata := mstorage.ObjectMetadata{
-		Bucket:  bucket,
-		Key:     object,
-		Created: stat.ModTime(),
-		Size:    stat.Size(),
-		ETag:    bucket + "#" + object,
+		Bucket:      bucket,
+		Key:         object,
+		Created:     stat.ModTime(),
+		Size:        stat.Size(),
+		ETag:        bucket + "#" + object,
+		ContentType: contentType,
 	}
 
 	return metadata, nil
@@ -179,24 +214,26 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag
 
 	var metadataList []mstorage.ObjectMetadata
 	for _, file := range files {
-		if len(metadataList) >= count {
-			return metadataList, true, nil
-		}
-		if strings.HasPrefix(file.Name(), prefix) {
-			metadata := mstorage.ObjectMetadata{
-				Bucket:  bucket,
-				Key:     file.Name(),
-				Created: file.ModTime(),
-				Size:    file.Size(),
-				ETag:    bucket + "#" + file.Name(),
+		if !strings.HasSuffix(file.Name(), "$metadata") {
+			if len(metadataList) >= count {
+				return metadataList, true, nil
+			}
+			if strings.HasPrefix(file.Name(), prefix) {
+				metadata := mstorage.ObjectMetadata{
+					Bucket:  bucket,
+					Key:     file.Name(),
+					Created: file.ModTime(),
+					Size:    file.Size(),
+					ETag:    bucket + "#" + file.Name(),
+				}
+				metadataList = append(metadataList, metadata)
 			}
-			metadataList = append(metadataList, metadata)
 		}
 	}
 	return metadataList, false, nil
 }
 
-func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
+func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
 	// TODO Commits should stage then move instead of writing directly
 	storage.writeLock.Lock()
 	defer storage.writeLock.Unlock()
@@ -216,6 +253,12 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
 		return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
 	}
 
+	// verify content type
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	contentType = strings.TrimSpace(contentType)
+
 	// get object path
 	objectPath := path.Join(storage.root, bucket, key)
 	objectDir := path.Dir(objectPath)
@@ -246,5 +289,24 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
 		return mstorage.EmbedError(bucket, key, err)
 	}
 
+	// serialize metadata to json
+
+	metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
+	if err != nil {
+		return mstorage.EmbedError(bucket, key, err)
+	}
+
+	//
+	file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
+	defer file.Close()
+	if err != nil {
+		return mstorage.EmbedError(bucket, key, err)
+	}
+
+	_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
+	if err != nil {
+		return mstorage.EmbedError(bucket, key, err)
+	}
+
 	return nil
 }

pkg/storage/inmemory/inmemory.go

@@ -41,7 +41,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
 	}
 }
 
-func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
+func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
 	objectKey := bucket + ":" + key
 
 	if _, ok := storage.bucketdata[bucket]; ok == false {
@@ -51,17 +51,26 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
 	if _, ok := storage.objectdata[objectKey]; ok == true {
 		return mstorage.ObjectExists{Bucket: bucket, Key: key}
 	}
+
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+
+	contentType = strings.TrimSpace(contentType)
+
 	var bytesBuffer bytes.Buffer
 	var newObject = storedObject{}
 	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
 		size := bytesBuffer.Len()
 		etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes()))
 		newObject.metadata = mstorage.ObjectMetadata{
-			Bucket:  bucket,
-			Key:     key,
-			Created: time.Now(),
-			Size:    int64(size),
-			ETag:    etag,
+			Bucket: bucket,
+			Key:    key,
+
+			ContentType: contentType,
+			Created:     time.Now(),
+			ETag:        etag,
+			Size:        int64(size),
 		}
 		newObject.data = bytesBuffer.Bytes()
 	}

pkg/storage/storage.go

@@ -31,7 +31,7 @@ type Storage interface {
 	CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error)
 	GetObjectMetadata(bucket string, object string) (ObjectMetadata, error)
 	ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error)
-	StoreObject(bucket string, key string, data io.Reader) error
+	StoreObject(bucket string, key string, contentType string, data io.Reader) error
 }
 
 type BucketMetadata struct {
@@ -40,11 +40,13 @@ type BucketMetadata struct {
 }
 
 type ObjectMetadata struct {
-	Bucket  string
-	Key     string
-	Created time.Time
-	Size    int64
-	ETag    string
+	Bucket string
+	Key    string
+
+	ContentType string
+	Created     time.Time
+	ETag        string
+	Size        int64
 }
 
 func IsValidBucket(bucket string) bool {

pkg/storage/storage_api_suite.go

@@ -21,6 +21,7 @@ func APITestSuite(c *C, create func() Storage) {
 	testListObjectsTestsForNonExistantBucket(c, create)
 	testNonExistantObjectInBucket(c, create)
 	testGetDirectoryReturnsObjectNotFound(c, create)
+	testDefaultContentType(c, create)
 }
 
 func testCreateBucket(c *C, create func() Storage) {
@@ -40,7 +41,7 @@ func testMultipleObjectCreation(c *C, create func() Storage) {
 		}
 		key := "obj" + strconv.Itoa(i)
 		objects[key] = []byte(randomString)
-		err := storage.StoreObject("bucket", key, bytes.NewBufferString(randomString))
+		err := storage.StoreObject("bucket", key, "", bytes.NewBufferString(randomString))
 		c.Assert(err, IsNil)
 	}
 
@@ -72,7 +73,7 @@ func testPaging(c *C, create func() Storage) {
 	// check before paging occurs
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		storage.StoreObject("bucket", key, bytes.NewBufferString(key))
+		storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
 		objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
 		c.Assert(len(objects), Equals, i+1)
 		c.Assert(isTruncated, Equals, false)
@@ -81,7 +82,7 @@ func testPaging(c *C, create func() Storage) {
 	// check after paging occurs pages work
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		storage.StoreObject("bucket", key, bytes.NewBufferString(key))
+		storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
 		objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
 		c.Assert(len(objects), Equals, 5)
 		c.Assert(isTruncated, Equals, true)
@@ -89,8 +90,8 @@ func testPaging(c *C, create func() Storage) {
 	}
 	// check paging with prefix at end returns less objects
 	{
-		storage.StoreObject("bucket", "newPrefix", bytes.NewBufferString("prefix1"))
-		storage.StoreObject("bucket", "newPrefix2", bytes.NewBufferString("prefix2"))
+		storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1"))
+		storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2"))
 		objects, isTruncated, err = storage.ListObjects("bucket", "new", 5)
 		c.Assert(len(objects), Equals, 2)
 	}
@@ -124,9 +125,9 @@ func testPaging(c *C, create func() Storage) {
 func testObjectOverwriteFails(c *C, create func() Storage) {
 	storage := create()
 	storage.StoreBucket("bucket")
-	err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
+	err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
 	c.Assert(err, IsNil)
-	err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three"))
+	err = storage.StoreObject("bucket", "object", "", bytes.NewBufferString("three"))
 	c.Assert(err, Not(IsNil))
 	var bytesBuffer bytes.Buffer
 	length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object")
@@ -137,7 +138,7 @@ func testObjectOverwriteFails(c *C, create func() Storage) {
 
 func testNonExistantBucketOperations(c *C, create func() Storage) {
 	storage := create()
-	err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
+	err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
 	c.Assert(err, Not(IsNil))
 }
 
@@ -153,7 +154,7 @@ func testPutObjectInSubdir(c *C, create func() Storage) {
 	storage := create()
 	err := storage.StoreBucket("bucket")
 	c.Assert(err, IsNil)
-	err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
+	err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
 	c.Assert(err, IsNil)
 	var bytesBuffer bytes.Buffer
 	length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "dir1/dir2/object")
@@ -250,7 +251,7 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
 	err := storage.StoreBucket("bucket")
 	c.Assert(err, IsNil)
 
-	err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
+	err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
 	c.Assert(err, IsNil)
 
 	var byteBuffer bytes.Buffer
@@ -287,3 +288,27 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
 	}
 	c.Assert(len(byteBuffer2.Bytes()), Equals, 0)
 }
+
+func testDefaultContentType(c *C, create func() Storage) {
+	storage := create()
+	err := storage.StoreBucket("bucket")
+	c.Assert(err, IsNil)
+
+	// test empty
+	err = storage.StoreObject("bucket", "one", "", bytes.NewBufferString("one"))
+	metadata, err := storage.GetObjectMetadata("bucket", "one")
+	c.Assert(err, IsNil)
+	c.Assert(metadata.ContentType, Equals, "application/octet-stream")
+
+	// test custom
+	storage.StoreObject("bucket", "two", "application/text", bytes.NewBufferString("two"))
+	metadata, err = storage.GetObjectMetadata("bucket", "two")
+	c.Assert(err, IsNil)
+	c.Assert(metadata.ContentType, Equals, "application/text")
+
+	// test trim space
+	storage.StoreObject("bucket", "three", "\tapplication/json    ", bytes.NewBufferString("three"))
+	metadata, err = storage.GetObjectMetadata("bucket", "three")
+	c.Assert(err, IsNil)
+	c.Assert(metadata.ContentType, Equals, "application/json")
+}