oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #133 from fkautz/pr_out_adding_content_type_to_backend_storage_drivers

Browse Source
master
Frederick F. Kautz IV 10 years ago
parent 05665a9111 288bc3fcc7
commit 549cf6b854
6 changed files with 144 additions and 46 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      pkg/api/minioapi/minioapi.go
  2. 12
      pkg/api/minioapi/minioapi_test.go
  3. 64
      pkg/storage/fs/fs.go
  4. 13
      pkg/storage/inmemory/inmemory.go
  5. 6
      pkg/storage/storage.go
  6. 45
      pkg/storage/storage_api_suite.go

2
pkg/api/minioapi/minioapi.go
Unescape View File

@ -192,7 +192,7 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
err := server.storage.StoreObject(bucket, object, req.Body)
err := server.storage.StoreObject(bucket, object, "", req.Body)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))

12
pkg/api/minioapi/minioapi_test.go
Unescape View File

@ -57,7 +57,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
buffer := bytes.NewBufferString("")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -82,7 +82,7 @@ func (s *MySuite) TestObject(c *C) {
buffer := bytes.NewBufferString("hello world")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -108,9 +108,9 @@ func (s *MySuite) TestMultipleObjects(c *C) {
buffer3 := bytes.NewBufferString("hello three")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object1", buffer1)
storage.StoreObject("bucket", "object2", buffer2)
storage.StoreObject("bucket", "object3", buffer3)
storage.StoreObject("bucket", "object1", "", buffer1)
storage.StoreObject("bucket", "object2", "", buffer2)
storage.StoreObject("bucket", "object3", "", buffer3)
// test non-existant object
response, err := http.Get(testServer.URL + "/bucket/object")
@ -200,7 +200,7 @@ func (s *MySuite) TestHeader(c *C) {
buffer := bytes.NewBufferString("hello world")
storage.StoreBucket("bucket")
storage.StoreObject("bucket", "object", buffer)
storage.StoreObject("bucket", "object", "", buffer)
response, err = http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)

64
pkg/storage/fs/fs.go
Unescape View File

@ -1,6 +1,8 @@
package fs
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"os"
@ -16,6 +18,10 @@ type storage struct {
writeLock sync.Mutex
}
type SerializedMetadata struct {
ContentType string
}
type MkdirFailedError struct{}
func (self MkdirFailedError) Error() string {
@ -146,12 +152,41 @@ func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorag
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
}
file, err := os.Open(objectPath + "$metadata")
defer file.Close()
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
metadataBuffer, err := ioutil.ReadAll(file)
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
var deserializedMetadata SerializedMetadata
err = json.Unmarshal(metadataBuffer, &deserializedMetadata)
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
}
contentType := "application/octet-stream"
if deserializedMetadata.ContentType != "" {
contentType = deserializedMetadata.ContentType
}
contentType = strings.TrimSpace(contentType)
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: object,
Created: stat.ModTime(),
Size: stat.Size(),
ETag: bucket + "#" + object,
ContentType: contentType,
}
return metadata, nil
@ -179,6 +214,7 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag
var metadataList []mstorage.ObjectMetadata
for _, file := range files {
if !strings.HasSuffix(file.Name(), "$metadata") {
if len(metadataList) >= count {
return metadataList, true, nil
}
@ -193,10 +229,11 @@ func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorag
metadataList = append(metadataList, metadata)
}
}
}
return metadataList, false, nil
}
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
storage.writeLock.Lock()
defer storage.writeLock.Unlock()
@ -216,6 +253,12 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// verify content type
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := path.Join(storage.root, bucket, key)
objectDir := path.Dir(objectPath)
@ -246,5 +289,24 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
return mstorage.EmbedError(bucket, key, err)
}
// serialize metadata to json
metadataBuffer, err := json.Marshal(SerializedMetadata{ContentType: contentType})
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
//
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
_, err = io.Copy(file, bytes.NewBuffer(metadataBuffer))
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
return nil
}

13
pkg/storage/inmemory/inmemory.go
Unescape View File

@ -41,7 +41,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
}
}
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
objectKey := bucket + ":" + key
if _, ok := storage.bucketdata[bucket]; ok == false {
@ -51,6 +51,13 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
if _, ok := storage.objectdata[objectKey]; ok == true {
return mstorage.ObjectExists{Bucket: bucket, Key: key}
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
var bytesBuffer bytes.Buffer
var newObject = storedObject{}
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
@ -59,9 +66,11 @@ func (storage *storage) StoreObject(bucket string, key string, data io.Reader) e
newObject.metadata = mstorage.ObjectMetadata{
Bucket: bucket,
Key: key,
ContentType: contentType,
Created: time.Now(),
Size: int64(size),
ETag: etag,
Size: int64(size),
}
newObject.data = bytesBuffer.Bytes()
}

6
pkg/storage/storage.go
Unescape View File

@ -31,7 +31,7 @@ type Storage interface {
CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error)
GetObjectMetadata(bucket string, object string) (ObjectMetadata, error)
ListObjects(bucket, prefix string, count int) ([]ObjectMetadata, bool, error)
StoreObject(bucket string, key string, data io.Reader) error
StoreObject(bucket string, key string, contentType string, data io.Reader) error
}
type BucketMetadata struct {
@ -42,9 +42,11 @@ type BucketMetadata struct {
type ObjectMetadata struct {
Bucket string
Key string
ContentType string
Created time.Time
Size int64
ETag string
Size int64
}
func IsValidBucket(bucket string) bool {

45
pkg/storage/storage_api_suite.go
Unescape View File

@ -21,6 +21,7 @@ func APITestSuite(c *C, create func() Storage) {
testListObjectsTestsForNonExistantBucket(c, create)
testNonExistantObjectInBucket(c, create)
testGetDirectoryReturnsObjectNotFound(c, create)
testDefaultContentType(c, create)
}
func testCreateBucket(c *C, create func() Storage) {
@ -40,7 +41,7 @@ func testMultipleObjectCreation(c *C, create func() Storage) {
}
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := storage.StoreObject("bucket", key, bytes.NewBufferString(randomString))
err := storage.StoreObject("bucket", key, "", bytes.NewBufferString(randomString))
c.Assert(err, IsNil)
}
@ -72,7 +73,7 @@ func testPaging(c *C, create func() Storage) {
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Assert(len(objects), Equals, i+1)
c.Assert(isTruncated, Equals, false)
@ -81,7 +82,7 @@ func testPaging(c *C, create func() Storage) {
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
storage.StoreObject("bucket", key, "", bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Assert(len(objects), Equals, 5)
c.Assert(isTruncated, Equals, true)
@ -89,8 +90,8 @@ func testPaging(c *C, create func() Storage) {
}
// check paging with prefix at end returns less objects
{
storage.StoreObject("bucket", "newPrefix", bytes.NewBufferString("prefix1"))
storage.StoreObject("bucket", "newPrefix2", bytes.NewBufferString("prefix2"))
storage.StoreObject("bucket", "newPrefix", "", bytes.NewBufferString("prefix1"))
storage.StoreObject("bucket", "newPrefix2", "", bytes.NewBufferString("prefix2"))
objects, isTruncated, err = storage.ListObjects("bucket", "new", 5)
c.Assert(len(objects), Equals, 2)
}
@ -124,9 +125,9 @@ func testPaging(c *C, create func() Storage) {
func testObjectOverwriteFails(c *C, create func() Storage) {
storage := create()
storage.StoreBucket("bucket")
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three"))
err = storage.StoreObject("bucket", "object", "", bytes.NewBufferString("three"))
c.Assert(err, Not(IsNil))
var bytesBuffer bytes.Buffer
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object")
@ -137,7 +138,7 @@ func testObjectOverwriteFails(c *C, create func() Storage) {
func testNonExistantBucketOperations(c *C, create func() Storage) {
storage := create()
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
err := storage.StoreObject("bucket", "object", "", bytes.NewBufferString("one"))
c.Assert(err, Not(IsNil))
}
@ -153,7 +154,7 @@ func testPutObjectInSubdir(c *C, create func() Storage) {
storage := create()
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil)
var bytesBuffer bytes.Buffer
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "dir1/dir2/object")
@ -250,7 +251,7 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
err = storage.StoreObject("bucket", "dir1/dir2/object", bytes.NewBufferString("hello world"))
err = storage.StoreObject("bucket", "dir1/dir2/object", "", bytes.NewBufferString("hello world"))
c.Assert(err, IsNil)
var byteBuffer bytes.Buffer
@ -287,3 +288,27 @@ func testGetDirectoryReturnsObjectNotFound(c *C, create func() Storage) {
}
c.Assert(len(byteBuffer2.Bytes()), Equals, 0)
}
func testDefaultContentType(c *C, create func() Storage) {
storage := create()
err := storage.StoreBucket("bucket")
c.Assert(err, IsNil)
// test empty
err = storage.StoreObject("bucket", "one", "", bytes.NewBufferString("one"))
metadata, err := storage.GetObjectMetadata("bucket", "one")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/octet-stream")
// test custom
storage.StoreObject("bucket", "two", "application/text", bytes.NewBufferString("two"))
metadata, err = storage.GetObjectMetadata("bucket", "two")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/text")
// test trim space
storage.StoreObject("bucket", "three", "\tapplication/json ", bytes.NewBufferString("three"))
metadata, err = storage.GetObjectMetadata("bucket", "three")
c.Assert(err, IsNil)
c.Assert(metadata.ContentType, Equals, "application/json")
}

Write Preview
Loading…
Cancel
Save