From e15edbf393cfa5816c7e4860017729c9fb1a8953 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 11 Dec 2014 16:49:35 -0800 Subject: [PATCH] Implement simple encoded storage in gateway --- pkgs/gateway/gateway.go | 36 ++++++++++++++++++++--- pkgs/gateway/gateway_test.go | 19 +++++++++--- pkgs/storage/fsstorage/fs_storage.go | 15 +++++++--- pkgs/storage/fsstorage/fs_storage_test.go | 16 ++++------ 4 files changed, 64 insertions(+), 22 deletions(-) diff --git a/pkgs/gateway/gateway.go b/pkgs/gateway/gateway.go index 59c5b1265..f9ab2ecce 100644 --- a/pkgs/gateway/gateway.go +++ b/pkgs/gateway/gateway.go @@ -9,6 +9,7 @@ import ( "path" "github.com/gorilla/mux" + "github.com/minio-io/minio/pkgs/storage/encodedstorage" "github.com/minio-io/minio/pkgs/storage/fsstorage" "github.com/tchap/go-patricia/patricia" ) @@ -18,7 +19,10 @@ type GatewayConfig struct { StorageDriver StorageDriver BucketDriver BucketDriver requestBucketChan chan BucketRequest - dataDir string + DataDir string + K, + M int + BlockSize uint64 } // Message for requesting a bucket @@ -197,10 +201,34 @@ func InMemoryStorageDriver(bucket string, input chan ObjectRequest, config Gatew } } -func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) { - fileStorage := fsstorage.FileSystemStorage{ - RootDir: config.dataDir, +func SimpleEncodedStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) { + eStorage, _ := encodedstorage.NewStorage(config.DataDir, config.K, config.M, config.BlockSize) + for request := range input { + switch request.requestType { + case "GET": + objectPath := path.Join(bucket, request.path) + object, err := eStorage.Get(objectPath) + if err != nil { + request.callback <- err + } else { + request.callback <- object + } + case "PUT": + objectPath := path.Join(bucket, request.path) + err := eStorage.Put(objectPath, bytes.NewBuffer(request.object)) + if err != nil { + request.callback <- err + } else { + request.callback <- nil + } + default: + request.callback <- errors.New("Unexpected message") + } } +} + +func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) { + fileStorage := fsstorage.NewStorage(config.DataDir) for request := range input { switch request.requestType { case "GET": diff --git a/pkgs/gateway/gateway_test.go b/pkgs/gateway/gateway_test.go index ae9e6fede..0ea62cd9d 100644 --- a/pkgs/gateway/gateway_test.go +++ b/pkgs/gateway/gateway_test.go @@ -104,9 +104,13 @@ func (s *GatewaySuite) TestBucketCreation(c *C) { } func (s *GatewaySuite) TestInMemoryBucketOperations(c *C) { - simpleFileStorageRootDir, err := miniotest.MakeTempTestDir() - c.Assert(err, IsNil) + simpleFileStorageRootDir, err1 := miniotest.MakeTempTestDir() + c.Assert(err1, IsNil) + simpleEncodedStorageRootDir, err2 := miniotest.MakeTempTestDir() + c.Assert(err2, IsNil) defer os.RemoveAll(simpleFileStorageRootDir) + defer os.RemoveAll(simpleEncodedStorageRootDir) + configs := []GatewayConfig{ GatewayConfig{ StorageDriver: InMemoryStorageDriver, @@ -115,7 +119,15 @@ func (s *GatewaySuite) TestInMemoryBucketOperations(c *C) { GatewayConfig{ StorageDriver: SimpleFileStorageDriver, requestBucketChan: make(chan BucketRequest), - dataDir: simpleFileStorageRootDir, + DataDir: simpleFileStorageRootDir, + }, + GatewayConfig{ + StorageDriver: SimpleEncodedStorageDriver, + requestBucketChan: make(chan BucketRequest), + DataDir: simpleEncodedStorageRootDir, + K: 10, + M: 6, + BlockSize: 1024 * 1024, }, } for _, config := range configs { @@ -147,6 +159,5 @@ func (s *GatewaySuite) TestInMemoryBucketOperations(c *C) { barResult, err := bucket.Get(context, "foo") c.Assert(err, IsNil) c.Assert(string(barResult), Equals, "bar") - } } diff --git a/pkgs/storage/fsstorage/fs_storage.go b/pkgs/storage/fsstorage/fs_storage.go index b379e3d01..63c1283ad 100644 --- a/pkgs/storage/fsstorage/fs_storage.go +++ b/pkgs/storage/fsstorage/fs_storage.go @@ -10,11 +10,18 @@ import ( "github.com/minio-io/minio/pkgs/storage" ) -type FileSystemStorage struct { +type fileSystemStorage struct { RootDir string } -func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) { +func NewStorage(rootDir string) (storage.ObjectStorage, error) { + newStorage := fileSystemStorage{ + RootDir: rootDir, + } + return &newStorage, nil +} + +func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) { fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath)) if err != nil { return nil, err @@ -33,11 +40,11 @@ func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescri return descriptions, nil } -func (storage FileSystemStorage) Get(objectPath string) (io.Reader, error) { +func (storage *fileSystemStorage) Get(objectPath string) (io.Reader, error) { return os.Open(path.Join(storage.RootDir, objectPath)) } -func (storage FileSystemStorage) Put(objectPath string, object io.Reader) error { +func (storage *fileSystemStorage) Put(objectPath string, object io.Reader) error { err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700) if err != nil { return err diff --git a/pkgs/storage/fsstorage/fs_storage_test.go b/pkgs/storage/fsstorage/fs_storage_test.go index 89f3db91b..dd9c89516 100644 --- a/pkgs/storage/fsstorage/fs_storage_test.go +++ b/pkgs/storage/fsstorage/fs_storage_test.go @@ -10,9 +10,9 @@ import ( . "gopkg.in/check.v1" ) -type FileSystemStorageSuite struct{} +type fileSystemStorageSuite struct{} -var _ = Suite(&FileSystemStorageSuite{}) +var _ = Suite(&fileSystemStorageSuite{}) func Test(t *testing.T) { TestingT(t) } @@ -20,15 +20,13 @@ func makeTempTestDir() (string, error) { return ioutil.TempDir("/tmp", "minio-test-") } -func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) { +func (s *fileSystemStorageSuite) TestfileStoragePutAtRootPath(c *C) { rootDir, err := makeTempTestDir() c.Assert(err, IsNil) defer os.RemoveAll(rootDir) var objectStorage storage.ObjectStorage - objectStorage = FileSystemStorage{ - RootDir: rootDir, - } + objectStorage, _ = NewStorage(rootDir) objectBuffer := bytes.NewBuffer([]byte("object1")) objectStorage.Put("path1", objectBuffer) @@ -44,15 +42,13 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) { c.Assert(objectList[0].Path, Equals, "path1") } -func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) { +func (s *fileSystemStorageSuite) TestfileStoragePutDirPath(c *C) { rootDir, err := makeTempTestDir() c.Assert(err, IsNil) defer os.RemoveAll(rootDir) var objectStorage storage.ObjectStorage - objectStorage = FileSystemStorage{ - RootDir: rootDir, - } + objectStorage, _ = NewStorage(rootDir) objectBuffer1 := bytes.NewBuffer([]byte("object1")) objectStorage.Put("path1/path2/path3", objectBuffer1)