From 288bc3fcc751c33a5d9a1a429bd31b775ae390ae Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Wed, 4 Feb 2015 17:32:40 -0800 Subject: [PATCH] Adding content type to backend storage drivers --- pkg/api/minioapi/minioapi.go | 2 +- pkg/api/minioapi/minioapi_test.go | 12 ++-- pkg/storage/fs/fs.go | 96 +++++++++++++++++++++++++------ pkg/storage/inmemory/inmemory.go | 21 +++++-- pkg/storage/storage.go | 14 +++-- pkg/storage/storage_api_suite.go | 45 +++++++++++---- 6 files changed, 144 insertions(+), 46 deletions(-) diff --git a/pkg/api/minioapi/minioapi.go b/pkg/api/minioapi/minioapi.go index 364bfd495..232edaebd 100644 --- a/pkg/api/minioapi/minioapi.go +++ b/pkg/api/minioapi/minioapi.go @@ -192,7 +192,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques vars := mux.Vars(req) bucket := vars["bucket"] object := vars["object"] - err := server.storage.StoreObject(bucket, object, req.Body) + err := server.storage.StoreObject(bucket, object, "", req.Body) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) diff --git a/pkg/api/minioapi/minioapi_test.go b/pkg/api/minioapi/minioapi_test.go index 1c1aaa441..89d6c455b 100644 --- a/pkg/api/minioapi/minioapi_test.go +++ b/pkg/api/minioapi/minioapi_test.go @@ -57,7 +57,7 @@ func (s *MySuite) TestEmptyObject(c *C) { buffer := bytes.NewBufferString("") storage.StoreBucket("bucket") - storage.StoreObject("bucket", "object", buffer) + storage.StoreObject("bucket", "object", "", buffer) response, err := http.Get(testServer.URL + "/bucket/object") c.Assert(err, IsNil) @@ -82,7 +82,7 @@ func (s *MySuite) TestObject(c *C) { buffer := bytes.NewBufferString("hello world") storage.StoreBucket("bucket") - storage.StoreObject("bucket", "object", buffer) + storage.StoreObject("bucket", "object", "", buffer) response, err := http.Get(testServer.URL + "/bucket/object") c.Assert(err, IsNil) @@ -108,9 +108,9 @@ func (s *MySuite) TestMultipleObjects(c *C) { buffer3 := bytes.NewBufferString("hello three") storage.StoreBucket("bucket") - storage.StoreObject("bucket", "object1", buffer1) - storage.StoreObject("bucket", "object2", buffer2) - storage.StoreObject("bucket", "object3", buffer3) + storage.StoreObject("bucket", "object1", "", buffer1) + storage.StoreObject("bucket", "object2", "", buffer2) + storage.StoreObject("bucket", "object3", "", buffer3) // test non-existant object response, err := http.Get(testServer.URL + "/bucket/object") @@ -200,7 +200,7 @@ func (s *MySuite) TestHeader(c *C) { buffer := bytes.NewBufferString("hello world") storage.StoreBucket("bucket") - storage.StoreObject("bucket", "object", buffer) + storage.StoreObject("bucket", "object", "", buffer) response, err = http.Get(testServer.URL + "/bucket/object") c.Assert(err, IsNil) diff --git a/pkg/storage/fs/fs.go b/pkg/storage/fs/fs.go index 7b65b3a25..a3723e604 100644 --- a/pkg/storage/fs/fs.go +++ b/pkg/storage/fs/fs.go @@ -1,6 +1,8 @@ package fs import ( + "bytes" + "encoding/json" "io" "io/ioutil" "os" @@ -16,6 +18,10 @@ type storage struct { writeLock sync.Mutex } +type SerializedMetadata struct { + ContentType string +} + type MkdirFailedError struct{} func (self MkdirFailedError) Error() string { @@ -146,12 +152,41 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object} } + _, err = os.Stat(objectPath + "$metadata") + if os.IsNotExist(err) { + return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object} + } + + file, err := os.Open(objectPath + "$metadata") + defer file.Close() + if err != nil { + return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err) + } + + metadataBuffer, err := ioutil.ReadAll(file) + if err != nil { + return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err) + } + + var deserializedMetadata SerializedMetadata + err = json.Unmarshal(metadataBuffer, &deserializedMetadata) + if err != nil { + return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err) + } + + contentType := "application/octet-stream" + if deserializedMetadata.ContentType != "" { + contentType = deserializedMetadata.ContentType + } + contentType = strings.TrimSpace(contentType) + metadata := mstorage.ObjectMetadata{ - Bucket: bucket, - Key: object, - Created: stat.ModTime(), - Size: stat.Size(), - ETag: bucket + "#" + object, + Bucket: bucket, + Key: object, + Created: stat.ModTime(), + Size: stat.Size(), + ETag: bucket + "#" + object, + ContentType: contentType, } return metadata, nil @@ -179,24 +214,26 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag var metadataList []mstorage.ObjectMetadata for _, file := range files { - if len(metadataList) >= count { - return metadataList, true, nil - } - if strings.HasPrefix(file.Name(), prefix) { - metadata := mstorage.ObjectMetadata{ - Bucket: bucket, - Key: file.Name(), - Created: file.ModTime(), - Size: file.Size(), - ETag: bucket + "#" + file.Name(), + if !strings.HasSuffix(file.Name(), "$metadata") { + if len(metadataList) >= count { + return metadataList, true, nil + } + if strings.HasPrefix(file.Name(), prefix) { + metadata := mstorage.ObjectMetadata{ + Bucket: bucket, + Key: file.Name(), + Created: file.ModTime(), + Size: file.Size(), + ETag: bucket + "#" + file.Name(), + } + metadataList = append(metadataList, metadata) } - metadataList = append(metadataList, metadata) } } return metadataList, false, nil } -func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error { +func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error { // TODO Commits should stage then move instead of writing directly storage.writeLock.Lock() defer storage.writeLock.Unlock() @@ -216,6 +253,12 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key} } + // verify content type + if contentType == "" { + contentType = "application/octet-stream" + } + contentType = strings.TrimSpace(contentType) + // get object path objectPath := path.Join(storage.root, bucket, key) objectDir := path.Dir(objectPath) @@ -246,5 +289,24 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e return mstorage.EmbedError(bucket, key, err) } + // serialize metadata to json + + metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType}) + if err != nil { + return mstorage.EmbedError(bucket, key, err) + } + + // + file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600) + defer file.Close() + if err != nil { + return mstorage.EmbedError(bucket, key, err) + } + + _, err = io.Copy(file, bytes.NewBuffer(metadataBuffer)) + if err != nil { + return mstorage.EmbedError(bucket, key, err) + } + return nil } diff --git a/pkg/storage/inmemory/inmemory.go b/pkg/storage/inmemory/inmemory.go index bb3c9d673..a6b9691e9 100644 --- a/pkg/storage/inmemory/inmemory.go +++ b/pkg/storage/inmemory/inmemory.go @@ -41,7 +41,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st } } -func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error { +func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error { objectKey := bucket + ":" + key if _, ok := storage.bucketdata[bucket]; ok == false { @@ -51,17 +51,26 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e if _, ok := storage.objectdata[objectKey]; ok == true { return mstorage.ObjectExists{Bucket: bucket, Key: key} } + + if contentType == "" { + contentType = "application/octet-stream" + } + + contentType = strings.TrimSpace(contentType) + var bytesBuffer bytes.Buffer var newObject = storedObject{} if _, ok := io.Copy(&bytesBuffer, data); ok == nil { size := bytesBuffer.Len() etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) newObject.metadata = mstorage.ObjectMetadata{ - Bucket: bucket, - Key: key, - Created: time.Now(), - Size: int64(size), - ETag: etag, + Bucket: bucket, + Key: key, + + ContentType: contentType, + Created: time.Now(), + ETag: etag, + Size: int64(size), } newObject.data = bytesBuffer.Bytes() } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4f67a696f..59560c8e0 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -31,7 +31,7 @@ type Storage interface { CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) GetObjectMetadata(bucket string, object string) (ObjectMetadata, error) ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error) - StoreObject(bucket string, key string, data io.Reader) error + StoreObject(bucket string, key string, contentType string, data io.Reader) error } type BucketMetadata struct { @@ -40,11 +40,13 @@ type BucketMetadata struct { } type ObjectMetadata struct { - Bucket string - Key string - Created time.Time - Size int64 - ETag string + Bucket string + Key string + + ContentType string + Created time.Time + ETag string + Size int64 } func IsValidBucket(bucket string) bool { diff --git a/pkg/storage/storage_api_suite.go b/pkg/storage/storage_api_suite.go index 79a5554e2..01e88a174 100644 --- a/pkg/storage/storage_api_suite.go +++ b/pkg/storage/storage_api_suite.go @@ -21,6 +21,7 @@ func APITestSuite(c *C, create func() Storage) { testListObjectsTestsForNonExistantBucket(c, create) testNonExistantObjectInBucket(c, create) testGetDirectoryReturnsObjectNotFound(c, create) + testDefaultContentType(c, create) } func testCreateBucket(c *C, create func() Storage) { @@ -40,7 +41,7 @@ func testMultipleObjectCreation(c *C, create func() Storage) { } key := "obj" + strconv.Itoa(i) objects[key] = []byte(randomString) - err := storage.StoreObject("bucket", key, bytes.NewBufferString(randomString)) + err := storage.StoreObject("bucket", key, "", bytes.NewBufferString(randomString)) c.Assert(err, IsNil) } @@ -72,7 +73,7 @@ func testPaging(c *C, create func() Storage) { // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - storage.StoreObject("bucket", key, bytes.NewBufferString(key)) + storage.StoreObject("bucket", key, "", bytes.NewBufferString(key)) objects, isTruncated, err = storage.ListObjects("bucket", "", 5) c.Assert(len(objects), Equals, i+1) c.Assert(isTruncated, Equals, false) @@ -81,7 +82,7 @@ func testPaging(c *C, create func() Storage) { // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - storage.StoreObject("bucket", key, bytes.NewBufferString(key)) + storage.StoreObject("bucket", key, "", bytes.NewBufferString(key)) objects, isTruncated, err = storage.ListObjects("bucket", "", 5) c.Assert(len(objects), Equals, 5) c.Assert(isTruncated, Equals, true) @@ -89,8 +90,8 @@ func testPaging(c *C, create func() Storage) { } // check paging with prefix at end returns less objects { - storage.StoreObject("bucket", "newPrefix", bytes.NewBufferString("prefix1")) - storage.StoreObject("bucket", "newPrefix2", bytes.NewBufferString("prefix2")) + storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1")) + storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2")) objects, isTruncated, err = storage.ListObjects("bucket", "new", 5) c.Assert(len(objects), Equals, 2) } @@ -124,9 +125,9 @@ func testPaging(c *C, create func() Storage) { func testObjectOverwriteFails(c *C, create func() Storage) { storage := create() storage.StoreBucket("bucket") - err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one")) + err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one")) c.Assert(err, IsNil) - err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three")) + err = storage.StoreObject("bucket", "object", "", bytes.NewBufferString("three")) c.Assert(err, Not(IsNil)) var bytesBuffer bytes.Buffer length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object") @@ -137,7 +138,7 @@ func testObjectOverwriteFails(c *C, create func() Storage) { func testNonExistantBucketOperations(c *C, create func() Storage) { storage := create() - err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one")) + err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one")) c.Assert(err, Not(IsNil)) } @@ -153,7 +154,7 @@ func testPutObjectInSubdir(c *C, create func() Storage) { storage := create() err := storage.StoreBucket("bucket") c.Assert(err, IsNil) - err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world")) + err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world")) c.Assert(err, IsNil) var bytesBuffer bytes.Buffer length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "dir1/dir2/object") @@ -250,7 +251,7 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) { err := storage.StoreBucket("bucket") c.Assert(err, IsNil) - err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world")) + err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world")) c.Assert(err, IsNil) var byteBuffer bytes.Buffer @@ -287,3 +288,27 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) { } c.Assert(len(byteBuffer2.Bytes()), Equals, 0) } + +func testDefaultContentType(c *C, create func() Storage) { + storage := create() + err := storage.StoreBucket("bucket") + c.Assert(err, IsNil) + + // test empty + err = storage.StoreObject("bucket", "one", "", bytes.NewBufferString("one")) + metadata, err := storage.GetObjectMetadata("bucket", "one") + c.Assert(err, IsNil) + c.Assert(metadata.ContentType, Equals, "application/octet-stream") + + // test custom + storage.StoreObject("bucket", "two", "application/text", bytes.NewBufferString("two")) + metadata, err = storage.GetObjectMetadata("bucket", "two") + c.Assert(err, IsNil) + c.Assert(metadata.ContentType, Equals, "application/text") + + // test trim space + storage.StoreObject("bucket", "three", "\tapplication/json ", bytes.NewBufferString("three")) + metadata, err = storage.GetObjectMetadata("bucket", "three") + c.Assert(err, IsNil) + c.Assert(metadata.ContentType, Equals, "application/json") +}