Merge pull request #103 from fkautz/pr_out_objectstorage_refactor_to_take_io_reader_instead_of_byte

ObjectStorage refactor to take io.Reader instead of []byte

cmd/erasure-demo/fs.go

@@ -35,8 +35,7 @@ func fsGet(config inputConfig, objectPath string) (io.Reader, error) {
 	if err != nil {
 		return nil, err
 	}
-	objectBuffer := bytes.NewBuffer(object)
-	return objectBuffer, nil
+	return object, nil
 }
 
 func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
@@ -46,11 +45,8 @@ func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
 		return err
 	}
 	var objectStorage storage.ObjectStorage
-	buffer := new(bytes.Buffer)
-	buffer.ReadFrom(reader)
-	object := buffer.Bytes()
 	objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir}
-	if err = objectStorage.Put(objectPath, object); err != nil {
+	if err = objectStorage.Put(objectPath, reader); err != nil {
 		return err
 	}
 	return nil

pkgs/gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -212,7 +213,7 @@ func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config Gat
 			}
 		case "PUT":
 			objectPath := path.Join(bucket, request.path)
-			fileStorage.Put(objectPath, request.object)
+			fileStorage.Put(objectPath, bytes.NewBuffer(request.object))
 			request.callback <- nil
 		default:
 			request.callback <- errors.New("Unexpected message")

pkgs/storage/appendstorage/append_storage.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/gob"
 	"errors"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -58,7 +59,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
 	}, nil
 }
 
-func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
+func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
 	header, ok := storage.objects[objectPath]
 	if ok == false {
 		return nil, nil
@@ -72,10 +73,10 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return object, nil
+	return bytes.NewBuffer(object), nil
 }
 
-func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
+func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
 	header := Header{
 		Path:   objectPath,
 		Offset: 0,
@@ -86,11 +87,15 @@ func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
 	if err != nil {
 		return err
 	}
-	if _, err := aStorage.file.Write(object); err != nil {
+	objectBytes, err := ioutil.ReadAll(object)
+	if err != nil {
+		return err
+	}
+	if _, err := aStorage.file.Write(objectBytes); err != nil {
 		return err
 	}
 	header.Offset = offset
-	header.Length = len(object)
+	header.Length = len(objectBytes)
 	aStorage.objects[objectPath] = header
 	var mapBuffer bytes.Buffer
 	encoder := gob.NewEncoder(&mapBuffer)

pkgs/storage/appendstorage/append_storage_test.go

@@ -1,6 +1,7 @@
 package appendstorage
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -28,24 +29,27 @@ func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
 	objectStorage, err = NewStorage(rootDir, 0)
 	c.Assert(err, IsNil)
 
-	err = objectStorage.Put("path1", []byte("object1"))
+	err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
 	c.Assert(err, IsNil)
 
 	// assert object1 was created in correct path
-	object1, err := objectStorage.Get("path1")
+	objectResult1, err := objectStorage.Get("path1")
 	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 
-	err = objectStorage.Put("path2", []byte("object2"))
+	err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
 	c.Assert(err, IsNil)
 
 	// assert object1 was created in correct path
-	object2, err := objectStorage.Get("path2")
+	objectResult2, err := objectStorage.Get("path2")
 	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
 	c.Assert(string(object2), Equals, "object2")
 
-	object1, err = objectStorage.Get("path1")
+	objectResult1, err = objectStorage.Get("path1")
 	c.Assert(err, IsNil)
+	object1, _ = ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 }
 
@@ -59,19 +63,21 @@ func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
 	c.Assert(err, IsNil)
 
 	// add object 1
-	objectStorage.Put("path1/path2/path3", []byte("object"))
+	objectStorage.Put("path1/path2/path3", bytes.NewBuffer([]byte("object")))
 
 	// assert object1 was created in correct path
-	object1, err := objectStorage.Get("path1/path2/path3")
+	objectResult1, err := objectStorage.Get("path1/path2/path3")
 	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object")
 
 	// add object 2
-	objectStorage.Put("path1/path1/path1", []byte("object2"))
+	objectStorage.Put("path1/path1/path1", bytes.NewBuffer([]byte("object2")))
 
 	// assert object1 was created in correct path
-	object2, err := objectStorage.Get("path1/path1/path1")
+	objectResult2, err := objectStorage.Get("path1/path1/path1")
 	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
 	c.Assert(string(object2), Equals, "object2")
 }
 
@@ -83,11 +89,11 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
 	objectStorage, err := NewStorage(rootDir, 0)
 	c.Assert(err, IsNil)
 
-	err = objectStorage.Put("path1", []byte("object1"))
+	err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
 	c.Assert(err, IsNil)
-	err = objectStorage.Put("path2", []byte("object2"))
+	err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
 	c.Assert(err, IsNil)
-	err = objectStorage.Put("path3/obj3", []byte("object3"))
+	err = objectStorage.Put("path3/obj3", bytes.NewBuffer([]byte("object3")))
 	c.Assert(err, IsNil)
 
 	es := objectStorage.(*appendStorage)
@@ -96,18 +102,18 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
 	objectStorage2, err := NewStorage(rootDir, 0)
 	c.Assert(err, IsNil)
 
-	object1, err := objectStorage2.Get("path1")
+	objectResult1, err := objectStorage2.Get("path1")
 	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 
-	object2, err := objectStorage2.Get("path2")
+	objectResult2, err := objectStorage2.Get("path2")
 	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
 	c.Assert(string(object2), Equals, "object2")
 
-	object3, err := objectStorage2.Get("path3/obj3")
+	objectResult3, err := objectStorage2.Get("path3/obj3")
 	c.Assert(err, IsNil)
+	object3, _ := ioutil.ReadAll(objectResult3)
 	c.Assert(string(object3), Equals, "object3")
 }
-
-func (s *AppendStorageSuite) TestSlice(c *C) {
-}

pkgs/storage/fsstorage/fs_storage.go

@@ -1,6 +1,7 @@
 package fsstorage
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -32,14 +33,18 @@ func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescri
 	return descriptions, nil
 }
 
-func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) {
-	return ioutil.ReadFile(path.Join(storage.RootDir, objectPath))
+func (storage FileSystemStorage) Get(objectPath string) (io.Reader, error) {
+	return os.Open(path.Join(storage.RootDir, objectPath))
 }
 
-func (storage FileSystemStorage) Put(objectPath string, object []byte) error {
+func (storage FileSystemStorage) Put(objectPath string, object io.Reader) error {
 	err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600)
+	objectBytes, err := ioutil.ReadAll(object)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), objectBytes, 0600)
 }

pkgs/storage/fsstorage/fs_storage_test.go

@@ -1,6 +1,7 @@
 package fsstorage
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -29,11 +30,13 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) {
 		RootDir: rootDir,
 	}
 
-	objectStorage.Put("path1", []byte("object1"))
+	objectBuffer := bytes.NewBuffer([]byte("object1"))
+	objectStorage.Put("path1", objectBuffer)
 
 	// assert object1 was created in correct path
-	object1, err := objectStorage.Get("path1")
+	objectResult1, err := objectStorage.Get("path1")
 	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 
 	objectList, err := objectStorage.List("/")
@@ -51,19 +54,23 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) {
 		RootDir: rootDir,
 	}
 
-	objectStorage.Put("path1/path2/path3", []byte("object"))
+	objectBuffer1 := bytes.NewBuffer([]byte("object1"))
+	objectStorage.Put("path1/path2/path3", objectBuffer1)
 
 	// assert object1 was created in correct path
-	object1, err := objectStorage.Get("path1/path2/path3")
+	objectResult1, err := objectStorage.Get("path1/path2/path3")
 	c.Assert(err, IsNil)
-	c.Assert(string(object1), Equals, "object")
+	object1, _ := ioutil.ReadAll(objectResult1)
+	c.Assert(string(object1), Equals, "object1")
 
 	// add second object
-	err = objectStorage.Put("path2/path2/path2", []byte("object2"))
+	objectBuffer2 := bytes.NewBuffer([]byte("object2"))
+	err = objectStorage.Put("path2/path2/path2", objectBuffer2)
 	c.Assert(err, IsNil)
 
 	// add third object
-	err = objectStorage.Put("object3", []byte("object3"))
+	objectBuffer3 := bytes.NewBuffer([]byte("object3"))
+	err = objectStorage.Put("object3", objectBuffer3)
 	c.Assert(err, IsNil)
 
 	objectList, err := objectStorage.List("/")

pkgs/storage/storage.go

@@ -1,9 +1,11 @@
 package storage
 
+import "io"
+
 type ObjectStorage interface {
 	List(path string) ([]ObjectDescription, error)
-	Get(path string) ([]byte, error)
-	Put(path string, object []byte) error
+	Get(path string) (io.Reader, error)
+	Put(path string, object io.Reader) error
 }
 
 type ObjectDescription struct {