From 44b28166f5c45e850ace462cdfcb0c65f70742ec Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Fri, 14 Nov 2014 18:22:50 -0700 Subject: [PATCH] Adding simple file storage driver for persistent storage --- file_storage.go | 22 ++++++++++++ file_storage_test.go | 51 ++++++++++++++++++++++++++ gateway.go | 35 ++++++++++++++---- gateway_test.go | 86 ++++++++++++++++++++++++-------------------- setup_test.go | 8 +++++ storage.go | 5 +++ 6 files changed, 163 insertions(+), 44 deletions(-) create mode 100644 file_storage.go create mode 100644 file_storage_test.go create mode 100644 setup_test.go diff --git a/file_storage.go b/file_storage.go new file mode 100644 index 000000000..03f51b5d5 --- /dev/null +++ b/file_storage.go @@ -0,0 +1,22 @@ +package minio + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" +) + +type FileStorage struct { + RootDir string +} + +func (storage FileStorage) Get(objectPath string) ([]byte, error) { + return ioutil.ReadFile(path.Join(storage.RootDir, objectPath)) + +} + +func (storage FileStorage) Put(objectPath string, object []byte) error { + os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700) + return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600) +} diff --git a/file_storage_test.go b/file_storage_test.go new file mode 100644 index 000000000..7ef52cd52 --- /dev/null +++ b/file_storage_test.go @@ -0,0 +1,51 @@ +package minio + +import ( + . "gopkg.in/check.v1" + "io/ioutil" + "os" +) + +type FileStorageSuite struct{} + +var _ = Suite(&FileStorageSuite{}) + +func makeTempTestDir() (string, error) { + return ioutil.TempDir("/tmp", "minio-test-") +} + +func (s *FileStorageSuite) TestFileStoragePutAtRootPath(c *C) { + rootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(rootDir) + + var storage ObjectStorage + storage = FileStorage{ + RootDir: rootDir, + } + + storage.Put("path1", []byte("object1")) + + // assert object1 was created in correct path + object1, err := storage.Get("path1") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object1") +} + +func (s *FileStorageSuite) TestFileStoragePutDirPath(c *C) { + rootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(rootDir) + + var storage ObjectStorage + storage = FileStorage{ + RootDir: rootDir, + } + + storage.Put("path1/path2/path3", []byte("object")) + + // assert object1 was created in correct path + object1, err := storage.Get("path1/path2/path3") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object") +} diff --git a/gateway.go b/gateway.go index 2112dd6f6..4a0b482fd 100644 --- a/gateway.go +++ b/gateway.go @@ -7,6 +7,7 @@ import ( "github.com/tchap/go-patricia/patricia" "io/ioutil" "net/http" + "path" ) // Stores system configuration, populated from CLI or test runner @@ -14,6 +15,7 @@ type GatewayConfig struct { StorageDriver StorageDriver BucketDriver BucketDriver requestBucketChan chan BucketRequest + dataDir string } // Message for requesting a bucket @@ -37,7 +39,7 @@ type Bucket interface { type BucketDriver func(config GatewayConfig) // Storage driver function, should read from a channel and respond through callback channels -type StorageDriver func(bucket string, input chan ObjectRequest) +type StorageDriver func(bucket string, input chan ObjectRequest, config GatewayConfig) // TODO remove when building real context type fakeContext struct{} @@ -103,7 +105,7 @@ func SynchronizedBucketDriver(config GatewayConfig) { for request := range config.requestBucketChan { if buckets[request.name] == nil { bucketChannel := make(chan ObjectRequest) - go config.StorageDriver(request.name, bucketChannel) + go config.StorageDriver(request.name, bucketChannel, config) buckets[request.name] = &SynchronizedBucket{ name: request.name, channel: bucketChannel, @@ -176,17 +178,14 @@ func (bucket *SynchronizedBucket) closeChannel() { close(bucket.channel) } -func InMemoryStorageDriver(bucket string, input chan ObjectRequest) { +func InMemoryStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) { objects := patricia.NewTrie() for request := range input { prefix := patricia.Prefix(request.path) - fmt.Println("objects:", objects) switch request.requestType { case "GET": - fmt.Println("GET: " + request.path) request.callback <- objects.Get(prefix) case "PUT": - fmt.Println("PUT: " + request.path) objects.Insert(prefix, request.object) request.callback <- nil default: @@ -194,3 +193,27 @@ func InMemoryStorageDriver(bucket string, input chan ObjectRequest) { } } } + +func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) { + storage := FileStorage{ + RootDir: config.dataDir, + } + for request := range input { + switch request.requestType { + case "GET": + objectPath := path.Join(bucket, request.path) + object, err := storage.Get(objectPath) + if err != nil { + request.callback <- nil + } else { + request.callback <- object + } + case "PUT": + objectPath := path.Join(bucket, request.path) + storage.Put(objectPath, request.object) + request.callback <- nil + default: + request.callback <- errors.New("Unexpected message") + } + } +} diff --git a/gateway_test.go b/gateway_test.go index c730a3cff..849fcd57a 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -6,17 +6,15 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "os" "strings" - "testing" ) -type MySuite struct{} +type GatewaySuite struct{} -var _ = Suite(&MySuite{}) +var _ = Suite(&GatewaySuite{}) -func Test(t *testing.T) { TestingT(t) } - -func (s *MySuite) TestPrintsGateway(c *C) { +func (s *GatewaySuite) TestPrintsGateway(c *C) { // set up router with in memory storage driver router := mux.NewRouter() config := GatewayConfig{ @@ -59,7 +57,7 @@ func (s *MySuite) TestPrintsGateway(c *C) { type TestContext struct{} -func (s *MySuite) TestBucketCreation(c *C) { +func (s *GatewaySuite) TestBucketCreation(c *C) { config := GatewayConfig{ StorageDriver: InMemoryStorageDriver, requestBucketChan: make(chan BucketRequest), @@ -103,38 +101,50 @@ func (s *MySuite) TestBucketCreation(c *C) { c.Assert(bucketB.GetName(context), Equals, "bucketB") } -func (s *MySuite) TestInMemoryBucketOperations(c *C) { - // Test in memory bucket operations - config := GatewayConfig{ - StorageDriver: InMemoryStorageDriver, - requestBucketChan: make(chan BucketRequest), +func (s *GatewaySuite) TestInMemoryBucketOperations(c *C) { + simpleFileStorageRootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(simpleFileStorageRootDir) + configs := []GatewayConfig{ + GatewayConfig{ + StorageDriver: InMemoryStorageDriver, + requestBucketChan: make(chan BucketRequest), + }, + GatewayConfig{ + StorageDriver: SimpleFileStorageDriver, + requestBucketChan: make(chan BucketRequest), + dataDir: simpleFileStorageRootDir, + }, } - defer close(config.requestBucketChan) - go SynchronizedBucketDriver(config) - context := TestContext{} + for _, config := range configs { + defer close(config.requestBucketChan) + go SynchronizedBucketDriver(config) + context := TestContext{} + + // get bucket + callback := make(chan Bucket) + config.requestBucketChan <- BucketRequest{ + name: "bucket", + context: context, + callback: callback, + } + bucket := <-callback + c.Assert(bucket.GetName(context), Equals, "bucket") + + // get missing value + nilResult, err := bucket.Get(context, "foo") + c.Assert(nilResult, IsNil) + c.Assert(err, Not(IsNil)) + c.Assert(err.Error(), Equals, "Object not found") + + // add new value + err = bucket.Put(context, "foo", []byte("bar")) + c.Assert(err, IsNil) + + // retrieve value + barResult, err := bucket.Get(context, "foo") + c.Assert(err, IsNil) + c.Assert(string(barResult), Equals, "bar") - // get bucket - callback := make(chan Bucket) - config.requestBucketChan <- BucketRequest{ - name: "bucket", - context: context, - callback: callback, } - bucket := <-callback - c.Assert(bucket.GetName(context), Equals, "bucket") - - // get missing value - nilResult, err := bucket.Get(context, "foo") - c.Assert(nilResult, IsNil) - c.Assert(err, Not(IsNil)) - c.Assert(err.Error(), Equals, "Object not found") - - // add new value - err = bucket.Put(context, "foo", []byte("bar")) - c.Assert(err, IsNil) - - // retrieve value - barResult, err := bucket.Get(context, "foo") - c.Assert(err, IsNil) - c.Assert(string(barResult), Equals, "bar") } diff --git a/setup_test.go b/setup_test.go new file mode 100644 index 000000000..9174d9647 --- /dev/null +++ b/setup_test.go @@ -0,0 +1,8 @@ +package minio + +import ( + . "gopkg.in/check.v1" + "testing" +) + +func Test(t *testing.T) { TestingT(t) } diff --git a/storage.go b/storage.go index 4861996d0..5892b2d33 100644 --- a/storage.go +++ b/storage.go @@ -6,6 +6,11 @@ import ( "net/http" ) +type ObjectStorage interface { + Get(path string) ([]byte, error) + Put(path string, object []byte) error +} + func RegisterStorageHandlers(router *mux.Router) { router.HandleFunc("/storage/rpc", StorageHandler) }