From cae9b288b05cbe765845fc2c6106c764ec40a9a0 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Wed, 10 Dec 2014 18:54:04 -0800 Subject: [PATCH] Adding List(path) to object storage definition and accompanying definition to fs --- cmd/erasure-demo/fs.go | 9 +++- pkgs/storage/appendstorage/append_storage.go | 14 +++--- pkgs/storage/fsstorage/fs_storage.go | 18 +++++-- pkgs/storage/fsstorage/fs_storage_test.go | 52 +++++++++++++++++--- pkgs/storage/storage.go | 8 ++- pkgs/storage/storage_test.go | 32 ------------ 6 files changed, 78 insertions(+), 55 deletions(-) delete mode 100644 pkgs/storage/storage_test.go diff --git a/cmd/erasure-demo/fs.go b/cmd/erasure-demo/fs.go index 942449942..5cd3c9524 100644 --- a/cmd/erasure-demo/fs.go +++ b/cmd/erasure-demo/fs.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "io" "os" "path" @@ -14,11 +15,15 @@ func fsGetList(config inputConfig) (io.Reader, error) { var objectStorage storage.ObjectStorage rootDir := path.Join(config.rootDir, config.storageDriver) objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} - objectlist, err := objectStorage.GetList() + objectList, err := objectStorage.List("/") if err != nil { return nil, err } - objectListBuffer := bytes.NewBuffer(objectlist) + var objectListBytes []byte + if objectListBytes, err = json.Marshal(objectList); err != nil { + return nil, err + } + objectListBuffer := bytes.NewBuffer(objectListBytes) return objectListBuffer, nil } diff --git a/pkgs/storage/appendstorage/append_storage.go b/pkgs/storage/appendstorage/append_storage.go index 3b02af52b..524734817 100644 --- a/pkgs/storage/appendstorage/append_storage.go +++ b/pkgs/storage/appendstorage/append_storage.go @@ -75,30 +75,30 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) { return object, nil } -func (storage *appendStorage) Put(objectPath string, object []byte) error { +func (aStorage *appendStorage) Put(objectPath string, object []byte) error { header := Header{ Path: objectPath, Offset: 0, Length: 0, Crc: nil, } - offset, err := storage.file.Seek(0, os.SEEK_END) + offset, err := aStorage.file.Seek(0, os.SEEK_END) if err != nil { return err } - if _, err := storage.file.Write(object); err != nil { + if _, err := aStorage.file.Write(object); err != nil { return err } header.Offset = offset header.Length = len(object) - storage.objects[objectPath] = header + aStorage.objects[objectPath] = header var mapBuffer bytes.Buffer encoder := gob.NewEncoder(&mapBuffer) - encoder.Encode(storage.objects) - ioutil.WriteFile(storage.objectsFile, mapBuffer.Bytes(), 0600) + encoder.Encode(aStorage.objects) + ioutil.WriteFile(aStorage.objectsFile, mapBuffer.Bytes(), 0600) return nil } -func (storage *appendStorage) GetList() ([]byte, error) { +func (aStorage *appendStorage) List(listPath string) ([]storage.ObjectDescription, error) { return nil, errors.New("Not Implemented") } diff --git a/pkgs/storage/fsstorage/fs_storage.go b/pkgs/storage/fsstorage/fs_storage.go index 9e8ee89f3..65f6af7dd 100644 --- a/pkgs/storage/fsstorage/fs_storage.go +++ b/pkgs/storage/fsstorage/fs_storage.go @@ -5,23 +5,31 @@ import ( "os" "path" "path/filepath" + + "github.com/minio-io/minio/pkgs/storage" ) type FileSystemStorage struct { RootDir string } -func (storage FileSystemStorage) GetList() ([]byte, error) { - fileInfos, err := ioutil.ReadDir(storage.RootDir) +func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) { + fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath)) if err != nil { return nil, err } - var list []byte + var descriptions []storage.ObjectDescription + for _, fi := range fileInfos { - list = append(list, "{"+fi.Name()+"}\n"...) + description := storage.ObjectDescription{ + Path: fi.Name(), + IsDir: fi.IsDir(), + Hash: "", // TODO + } + descriptions = append(descriptions, description) } - return list, nil + return descriptions, nil } func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) { diff --git a/pkgs/storage/fsstorage/fs_storage_test.go b/pkgs/storage/fsstorage/fs_storage_test.go index e8335bdf7..beeafe9b1 100644 --- a/pkgs/storage/fsstorage/fs_storage_test.go +++ b/pkgs/storage/fsstorage/fs_storage_test.go @@ -3,6 +3,7 @@ package fsstorage import ( "io/ioutil" "os" + "testing" "github.com/minio-io/minio/pkgs/storage" . "gopkg.in/check.v1" @@ -12,6 +13,8 @@ type FileSystemStorageSuite struct{} var _ = Suite(&FileSystemStorageSuite{}) +func Test(t *testing.T) { TestingT(t) } + func makeTempTestDir() (string, error) { return ioutil.TempDir("/tmp", "minio-test-") } @@ -21,17 +24,21 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) { c.Assert(err, IsNil) defer os.RemoveAll(rootDir) - var storage storage.ObjectStorage - storage = FileSystemStorage{ + var objectStorage storage.ObjectStorage + objectStorage = FileSystemStorage{ RootDir: rootDir, } - storage.Put("path1", []byte("object1")) + objectStorage.Put("path1", []byte("object1")) // assert object1 was created in correct path - object1, err := storage.Get("path1") + object1, err := objectStorage.Get("path1") c.Assert(err, IsNil) c.Assert(string(object1), Equals, "object1") + + objectList, err := objectStorage.List("/") + c.Assert(err, IsNil) + c.Assert(objectList[0].Path, Equals, "path1") } func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) { @@ -39,15 +46,44 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) { c.Assert(err, IsNil) defer os.RemoveAll(rootDir) - var storage storage.ObjectStorage - storage = FileSystemStorage{ + var objectStorage storage.ObjectStorage + objectStorage = FileSystemStorage{ RootDir: rootDir, } - storage.Put("path1/path2/path3", []byte("object")) + objectStorage.Put("path1/path2/path3", []byte("object")) // assert object1 was created in correct path - object1, err := storage.Get("path1/path2/path3") + object1, err := objectStorage.Get("path1/path2/path3") c.Assert(err, IsNil) c.Assert(string(object1), Equals, "object") + + // add second object + err = objectStorage.Put("path2/path2/path2", []byte("object2")) + c.Assert(err, IsNil) + + // add third object + err = objectStorage.Put("object3", []byte("object3")) + c.Assert(err, IsNil) + + objectList, err := objectStorage.List("/") + c.Assert(err, IsNil) + c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""}) + c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""}) + c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) + c.Assert(len(objectList), Equals, 3) + + objectList, err = objectStorage.List("/path1") + c.Assert(err, IsNil) + c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) + c.Assert(len(objectList), Equals, 1) + + objectList, err = objectStorage.List("/path1/path2") + c.Assert(err, IsNil) + c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""}) + c.Assert(len(objectList), Equals, 1) + + objectList, err = objectStorage.List("/path1/path2/path3") + c.Assert(err, Not(IsNil)) + c.Assert(objectList, IsNil) } diff --git a/pkgs/storage/storage.go b/pkgs/storage/storage.go index 6f86f968b..cc325df6d 100644 --- a/pkgs/storage/storage.go +++ b/pkgs/storage/storage.go @@ -1,7 +1,13 @@ package storage type ObjectStorage interface { - GetList() ([]byte, error) + List(path string) ([]ObjectDescription, error) Get(path string) ([]byte, error) Put(path string, object []byte) error } + +type ObjectDescription struct { + Path string + IsDir bool + Hash string +} diff --git a/pkgs/storage/storage_test.go b/pkgs/storage/storage_test.go deleted file mode 100644 index 816efa457..000000000 --- a/pkgs/storage/storage_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package storage - -import ( - "fmt" - "io/ioutil" - "log" - "net/http" - "net/http/httptest" - "testing" -) - -func TestPrintsStorage(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(storageHandler)) - defer server.Close() - res, err := http.Get(server.URL) - if err != nil { - log.Fatal(err) - } - body, err := ioutil.ReadAll(res.Body) - res.Body.Close() - if err != nil { - log.Fatal(err) - } - bodyString := string(body) - if bodyString != "Storage" { - log.Fatal("Expected 'Storage', Received '" + bodyString + "'") - } -} - -func storageHandler(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Storage") -}