From 19da9760b3c1a2d9dad57b0febdc21cf86e6bddd Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Wed, 10 Dec 2014 20:40:53 -0800 Subject: [PATCH] ObjectStorage refactor to take io.Reader instead of []byte --- cmd/erasure-demo/fs.go | 8 +--- pkgs/gateway/gateway.go | 3 +- pkgs/storage/appendstorage/append_storage.go | 15 ++++--- .../appendstorage/append_storage_test.go | 42 +++++++++++-------- pkgs/storage/fsstorage/fs_storage.go | 13 ++++-- pkgs/storage/fsstorage/fs_storage_test.go | 21 ++++++---- pkgs/storage/storage.go | 6 ++- 7 files changed, 65 insertions(+), 43 deletions(-) diff --git a/cmd/erasure-demo/fs.go b/cmd/erasure-demo/fs.go index 5cd3c9524..5a143e60c 100644 --- a/cmd/erasure-demo/fs.go +++ b/cmd/erasure-demo/fs.go @@ -35,8 +35,7 @@ func fsGet(config inputConfig, objectPath string) (io.Reader, error) { if err != nil { return nil, err } - objectBuffer := bytes.NewBuffer(object) - return objectBuffer, nil + return object, nil } func fsPut(config inputConfig, objectPath string, reader io.Reader) error { @@ -46,11 +45,8 @@ func fsPut(config inputConfig, objectPath string, reader io.Reader) error { return err } var objectStorage storage.ObjectStorage - buffer := new(bytes.Buffer) - buffer.ReadFrom(reader) - object := buffer.Bytes() objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} - if err = objectStorage.Put(objectPath, object); err != nil { + if err = objectStorage.Put(objectPath, reader); err != nil { return err } return nil diff --git a/pkgs/gateway/gateway.go b/pkgs/gateway/gateway.go index 61ae86b3b..59c5b1265 100644 --- a/pkgs/gateway/gateway.go +++ b/pkgs/gateway/gateway.go @@ -1,6 +1,7 @@ package gateway import ( + "bytes" "errors" "fmt" "io/ioutil" @@ -212,7 +213,7 @@ func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config Gat } case "PUT": objectPath := path.Join(bucket, request.path) - fileStorage.Put(objectPath, request.object) + fileStorage.Put(objectPath, bytes.NewBuffer(request.object)) request.callback <- nil default: request.callback <- errors.New("Unexpected message") diff --git a/pkgs/storage/appendstorage/append_storage.go b/pkgs/storage/appendstorage/append_storage.go index 524734817..806e8a9bb 100644 --- a/pkgs/storage/appendstorage/append_storage.go +++ b/pkgs/storage/appendstorage/append_storage.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "errors" + "io" "io/ioutil" "os" "path" @@ -58,7 +59,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) { }, nil } -func (storage *appendStorage) Get(objectPath string) ([]byte, error) { +func (storage *appendStorage) Get(objectPath string) (io.Reader, error) { header, ok := storage.objects[objectPath] if ok == false { return nil, nil @@ -72,10 +73,10 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) { if err != nil { return nil, err } - return object, nil + return bytes.NewBuffer(object), nil } -func (aStorage *appendStorage) Put(objectPath string, object []byte) error { +func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error { header := Header{ Path: objectPath, Offset: 0, @@ -86,11 +87,15 @@ func (aStorage *appendStorage) Put(objectPath string, object []byte) error { if err != nil { return err } - if _, err := aStorage.file.Write(object); err != nil { + objectBytes, err := ioutil.ReadAll(object) + if err != nil { + return err + } + if _, err := aStorage.file.Write(objectBytes); err != nil { return err } header.Offset = offset - header.Length = len(object) + header.Length = len(objectBytes) aStorage.objects[objectPath] = header var mapBuffer bytes.Buffer encoder := gob.NewEncoder(&mapBuffer) diff --git a/pkgs/storage/appendstorage/append_storage_test.go b/pkgs/storage/appendstorage/append_storage_test.go index 0ca4fb230..d526a2e8f 100644 --- a/pkgs/storage/appendstorage/append_storage_test.go +++ b/pkgs/storage/appendstorage/append_storage_test.go @@ -1,6 +1,7 @@ package appendstorage import ( + "bytes" "io/ioutil" "os" "testing" @@ -28,24 +29,27 @@ func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) { objectStorage, err = NewStorage(rootDir, 0) c.Assert(err, IsNil) - err = objectStorage.Put("path1", []byte("object1")) + err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1"))) c.Assert(err, IsNil) // assert object1 was created in correct path - object1, err := objectStorage.Get("path1") + objectResult1, err := objectStorage.Get("path1") c.Assert(err, IsNil) + object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") - err = objectStorage.Put("path2", []byte("object2")) + err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2"))) c.Assert(err, IsNil) // assert object1 was created in correct path - object2, err := objectStorage.Get("path2") + objectResult2, err := objectStorage.Get("path2") c.Assert(err, IsNil) + object2, _ := ioutil.ReadAll(objectResult2) c.Assert(string(object2), Equals, "object2") - object1, err = objectStorage.Get("path1") + objectResult1, err = objectStorage.Get("path1") c.Assert(err, IsNil) + object1, _ = ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") } @@ -59,19 +63,21 @@ func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) { c.Assert(err, IsNil) // add object 1 - objectStorage.Put("path1/path2/path3", []byte("object")) + objectStorage.Put("path1/path2/path3", bytes.NewBuffer([]byte("object"))) // assert object1 was created in correct path - object1, err := objectStorage.Get("path1/path2/path3") + objectResult1, err := objectStorage.Get("path1/path2/path3") c.Assert(err, IsNil) + object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object") // add object 2 - objectStorage.Put("path1/path1/path1", []byte("object2")) + objectStorage.Put("path1/path1/path1", bytes.NewBuffer([]byte("object2"))) // assert object1 was created in correct path - object2, err := objectStorage.Get("path1/path1/path1") + objectResult2, err := objectStorage.Get("path1/path1/path1") c.Assert(err, IsNil) + object2, _ := ioutil.ReadAll(objectResult2) c.Assert(string(object2), Equals, "object2") } @@ -83,11 +89,11 @@ func (s *AppendStorageSuite) TestSerialization(c *C) { objectStorage, err := NewStorage(rootDir, 0) c.Assert(err, IsNil) - err = objectStorage.Put("path1", []byte("object1")) + err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1"))) c.Assert(err, IsNil) - err = objectStorage.Put("path2", []byte("object2")) + err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2"))) c.Assert(err, IsNil) - err = objectStorage.Put("path3/obj3", []byte("object3")) + err = objectStorage.Put("path3/obj3", bytes.NewBuffer([]byte("object3"))) c.Assert(err, IsNil) es := objectStorage.(*appendStorage) @@ -96,18 +102,18 @@ func (s *AppendStorageSuite) TestSerialization(c *C) { objectStorage2, err := NewStorage(rootDir, 0) c.Assert(err, IsNil) - object1, err := objectStorage2.Get("path1") + objectResult1, err := objectStorage2.Get("path1") c.Assert(err, IsNil) + object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") - object2, err := objectStorage2.Get("path2") + objectResult2, err := objectStorage2.Get("path2") c.Assert(err, IsNil) + object2, _ := ioutil.ReadAll(objectResult2) c.Assert(string(object2), Equals, "object2") - object3, err := objectStorage2.Get("path3/obj3") + objectResult3, err := objectStorage2.Get("path3/obj3") c.Assert(err, IsNil) + object3, _ := ioutil.ReadAll(objectResult3) c.Assert(string(object3), Equals, "object3") } - -func (s *AppendStorageSuite) TestSlice(c *C) { -} diff --git a/pkgs/storage/fsstorage/fs_storage.go b/pkgs/storage/fsstorage/fs_storage.go index 65f6af7dd..b379e3d01 100644 --- a/pkgs/storage/fsstorage/fs_storage.go +++ b/pkgs/storage/fsstorage/fs_storage.go @@ -1,6 +1,7 @@ package fsstorage import ( + "io" "io/ioutil" "os" "path" @@ -32,14 +33,18 @@ func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescri return descriptions, nil } -func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) { - return ioutil.ReadFile(path.Join(storage.RootDir, objectPath)) +func (storage FileSystemStorage) Get(objectPath string) (io.Reader, error) { + return os.Open(path.Join(storage.RootDir, objectPath)) } -func (storage FileSystemStorage) Put(objectPath string, object []byte) error { +func (storage FileSystemStorage) Put(objectPath string, object io.Reader) error { err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700) if err != nil { return err } - return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600) + objectBytes, err := ioutil.ReadAll(object) + if err != nil { + return err + } + return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), objectBytes, 0600) } diff --git a/pkgs/storage/fsstorage/fs_storage_test.go b/pkgs/storage/fsstorage/fs_storage_test.go index beeafe9b1..89f3db91b 100644 --- a/pkgs/storage/fsstorage/fs_storage_test.go +++ b/pkgs/storage/fsstorage/fs_storage_test.go @@ -1,6 +1,7 @@ package fsstorage import ( + "bytes" "io/ioutil" "os" "testing" @@ -29,11 +30,13 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) { RootDir: rootDir, } - objectStorage.Put("path1", []byte("object1")) + objectBuffer := bytes.NewBuffer([]byte("object1")) + objectStorage.Put("path1", objectBuffer) // assert object1 was created in correct path - object1, err := objectStorage.Get("path1") + objectResult1, err := objectStorage.Get("path1") c.Assert(err, IsNil) + object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") objectList, err := objectStorage.List("/") @@ -51,19 +54,23 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) { RootDir: rootDir, } - objectStorage.Put("path1/path2/path3", []byte("object")) + objectBuffer1 := bytes.NewBuffer([]byte("object1")) + objectStorage.Put("path1/path2/path3", objectBuffer1) // assert object1 was created in correct path - object1, err := objectStorage.Get("path1/path2/path3") + objectResult1, err := objectStorage.Get("path1/path2/path3") c.Assert(err, IsNil) - c.Assert(string(object1), Equals, "object") + object1, _ := ioutil.ReadAll(objectResult1) + c.Assert(string(object1), Equals, "object1") // add second object - err = objectStorage.Put("path2/path2/path2", []byte("object2")) + objectBuffer2 := bytes.NewBuffer([]byte("object2")) + err = objectStorage.Put("path2/path2/path2", objectBuffer2) c.Assert(err, IsNil) // add third object - err = objectStorage.Put("object3", []byte("object3")) + objectBuffer3 := bytes.NewBuffer([]byte("object3")) + err = objectStorage.Put("object3", objectBuffer3) c.Assert(err, IsNil) objectList, err := objectStorage.List("/") diff --git a/pkgs/storage/storage.go b/pkgs/storage/storage.go index cc325df6d..b9bcc479f 100644 --- a/pkgs/storage/storage.go +++ b/pkgs/storage/storage.go @@ -1,9 +1,11 @@ package storage +import "io" + type ObjectStorage interface { List(path string) ([]ObjectDescription, error) - Get(path string) ([]byte, error) - Put(path string, object []byte) error + Get(path string) (io.Reader, error) + Put(path string, object io.Reader) error } type ObjectDescription struct {