From 0a0e1111cd56eb36954e97128caf6e133272e750 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 12 Dec 2014 02:50:47 -0800 Subject: [PATCH] Add list() object support for erasure and append storage drivers - Reply back objects with their protectionlevel and md5sum - // TODO hash value - Calculate md5sum after "storeBlocks()", to make sure data is committed --- cmd/erasure-demo/erasure.go | 22 ++++++-- cmd/erasure-demo/fs.go | 2 +- pkgs/storage/appendstorage/append_storage.go | 15 +++++- .../storage/encodedstorage/encoded_storage.go | 21 ++++++-- .../encodedstorage/encoded_storage_test.go | 28 ++-------- pkgs/storage/fsstorage/fs_storage.go | 10 ++-- pkgs/storage/fsstorage/fs_storage_test.go | 53 +------------------ pkgs/storage/storage.go | 9 ++-- 8 files changed, 65 insertions(+), 95 deletions(-) diff --git a/cmd/erasure-demo/erasure.go b/cmd/erasure-demo/erasure.go index b9c775f5b..6c6004288 100644 --- a/cmd/erasure-demo/erasure.go +++ b/cmd/erasure-demo/erasure.go @@ -1,7 +1,8 @@ package main import ( - "errors" + "bytes" + "encoding/json" "io" "os" "path" @@ -11,8 +12,23 @@ import ( ) func erasureGetList(config inputConfig) (io.Reader, error) { - // do nothing - return nil, errors.New("Not Implemented") + var objectStorage storage.ObjectStorage + rootDir := path.Join(config.rootDir, config.storageDriver) + objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize) + if err != nil { + return nil, err + } + objectDescList, err := objectStorage.List() + if err != nil { + return nil, err + } + var objectDescListBytes []byte + if objectDescListBytes, err = json.Marshal(objectDescList); err != nil { + return nil, err + } + objectDescListBuffer := bytes.NewBuffer(objectDescListBytes) + + return objectDescListBuffer, nil } func erasureGet(config inputConfig, objectPath string) (io.Reader, error) { diff --git a/cmd/erasure-demo/fs.go b/cmd/erasure-demo/fs.go index c5737b803..b5c0f26d3 100644 --- a/cmd/erasure-demo/fs.go +++ b/cmd/erasure-demo/fs.go @@ -15,7 +15,7 @@ func fsGetList(config inputConfig) (io.Reader, error) { var objectStorage storage.ObjectStorage rootDir := path.Join(config.rootDir, config.storageDriver) objectStorage, _ = fsstorage.NewStorage(rootDir) - objectList, err := objectStorage.List("/") + objectList, err := objectStorage.List() if err != nil { return nil, err } diff --git a/pkgs/storage/appendstorage/append_storage.go b/pkgs/storage/appendstorage/append_storage.go index 651da0b5f..95f22ccaf 100644 --- a/pkgs/storage/appendstorage/append_storage.go +++ b/pkgs/storage/appendstorage/append_storage.go @@ -117,6 +117,17 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error { return nil } -func (aStorage *appendStorage) List(listPath string) ([]storage.ObjectDescription, error) { - return nil, errors.New("Not Implemented") +func (aStorage *appendStorage) List() ([]storage.ObjectDescription, error) { + var objectDescList []storage.ObjectDescription + for objectName, _ := range aStorage.objects { + var objectDescription storage.ObjectDescription + objectDescription.Name = objectName + objectDescription.Md5sum = "" + objectDescription.Protectionlevel = "" + objectDescList = append(objectDescList, objectDescription) + } + if len(objectDescList) == 0 { + return nil, errors.New("No objects found") + } + return objectDescList, nil } diff --git a/pkgs/storage/encodedstorage/encoded_storage.go b/pkgs/storage/encodedstorage/encoded_storage.go index 2909ad49e..b582f4560 100644 --- a/pkgs/storage/encodedstorage/encoded_storage.go +++ b/pkgs/storage/encodedstorage/encoded_storage.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/md5" "encoding/gob" + "encoding/hex" "errors" "io" "io/ioutil" @@ -93,8 +94,20 @@ func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) { return reader, nil } -func (eStorage *encodedStorage) List(listPath string) ([]storage.ObjectDescription, error) { - return nil, errors.New("Not Implemented") +func (eStorage *encodedStorage) List() ([]storage.ObjectDescription, error) { + var objectDescList []storage.ObjectDescription + for objectName, objectEntry := range eStorage.objects { + var objectDescription storage.ObjectDescription + protectionLevel := strconv.Itoa(objectEntry.Encoderparams.K) + "," + strconv.Itoa(objectEntry.Encoderparams.M) + objectDescription.Name = objectName + objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum) + objectDescription.Protectionlevel = protectionLevel + objectDescList = append(objectDescList, objectDescription) + } + if len(objectDescList) == 0 { + return nil, errors.New("No objects found") + } + return objectDescList, nil } func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error { @@ -124,8 +137,6 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error { // encode for chunk := range chunks { if chunk.Err == nil { - // md5sum on chunk - hash.Write(chunk.Data) // encode each blocks, length := encoder.Encode(chunk.Data) // store each @@ -135,6 +146,8 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error { return err } } + // md5sum only after chunk is committed to disk + hash.Write(chunk.Data) blockEntry := StorageBlockEntry{ Index: i, Length: length, diff --git a/pkgs/storage/encodedstorage/encoded_storage_test.go b/pkgs/storage/encodedstorage/encoded_storage_test.go index 73b589baa..b788eee22 100644 --- a/pkgs/storage/encodedstorage/encoded_storage_test.go +++ b/pkgs/storage/encodedstorage/encoded_storage_test.go @@ -38,9 +38,9 @@ func (s *EncodedStorageSuite) TestFileStoragePutAtRootPath(c *C) { object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") - // objectList, err := objectStorage.List("/") - // c.Assert(err, IsNil) - // c.Assert(objectList[0].Path, Equals, "path1") + objectList, err := objectStorage.List() + c.Assert(err, IsNil) + c.Assert(objectList[0].Name, Equals, "path1") } func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) { @@ -70,28 +70,6 @@ func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) { objectBuffer3 := bytes.NewBuffer([]byte("object3")) err = objectStorage.Put("object3", objectBuffer3) c.Assert(err, IsNil) - - // TODO support list - // objectList, err := objectStorage.List("/") - // c.Assert(err, IsNil) - // c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""}) - // c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""}) - // c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) - // c.Assert(len(objectList), Equals, 3) - // - // objectList, err = objectStorage.List("/path1") - // c.Assert(err, IsNil) - // c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) - // c.Assert(len(objectList), Equals, 1) - // - // objectList, err = objectStorage.List("/path1/path2") - // c.Assert(err, IsNil) - // c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""}) - // c.Assert(len(objectList), Equals, 1) - // - // objectList, err = objectStorage.List("/path1/path2/path3") - // c.Assert(err, Not(IsNil)) - // c.Assert(objectList, IsNil) } func (s *EncodedStorageSuite) TestObjectWithChunking(c *C) { diff --git a/pkgs/storage/fsstorage/fs_storage.go b/pkgs/storage/fsstorage/fs_storage.go index 63c1283ad..db82df11b 100644 --- a/pkgs/storage/fsstorage/fs_storage.go +++ b/pkgs/storage/fsstorage/fs_storage.go @@ -21,8 +21,8 @@ func NewStorage(rootDir string) (storage.ObjectStorage, error) { return &newStorage, nil } -func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) { - fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath)) +func (fsStorage *fileSystemStorage) List() ([]storage.ObjectDescription, error) { + fileInfos, err := ioutil.ReadDir(fsStorage.RootDir) if err != nil { return nil, err } @@ -31,9 +31,9 @@ func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescr for _, fi := range fileInfos { description := storage.ObjectDescription{ - Path: fi.Name(), - IsDir: fi.IsDir(), - Hash: "", // TODO + Name: fi.Name(), + Md5sum: "", + Protectionlevel: "", } descriptions = append(descriptions, description) } diff --git a/pkgs/storage/fsstorage/fs_storage_test.go b/pkgs/storage/fsstorage/fs_storage_test.go index dd9c89516..1603e1c35 100644 --- a/pkgs/storage/fsstorage/fs_storage_test.go +++ b/pkgs/storage/fsstorage/fs_storage_test.go @@ -37,56 +37,7 @@ func (s *fileSystemStorageSuite) TestfileStoragePutAtRootPath(c *C) { object1, _ := ioutil.ReadAll(objectResult1) c.Assert(string(object1), Equals, "object1") - objectList, err := objectStorage.List("/") + objectList, err := objectStorage.List() c.Assert(err, IsNil) - c.Assert(objectList[0].Path, Equals, "path1") -} - -func (s *fileSystemStorageSuite) TestfileStoragePutDirPath(c *C) { - rootDir, err := makeTempTestDir() - c.Assert(err, IsNil) - defer os.RemoveAll(rootDir) - - var objectStorage storage.ObjectStorage - objectStorage, _ = NewStorage(rootDir) - - objectBuffer1 := bytes.NewBuffer([]byte("object1")) - objectStorage.Put("path1/path2/path3", objectBuffer1) - - // assert object1 was created in correct path - objectResult1, err := objectStorage.Get("path1/path2/path3") - c.Assert(err, IsNil) - object1, _ := ioutil.ReadAll(objectResult1) - c.Assert(string(object1), Equals, "object1") - - // add second object - objectBuffer2 := bytes.NewBuffer([]byte("object2")) - err = objectStorage.Put("path2/path2/path2", objectBuffer2) - c.Assert(err, IsNil) - - // add third object - objectBuffer3 := bytes.NewBuffer([]byte("object3")) - err = objectStorage.Put("object3", objectBuffer3) - c.Assert(err, IsNil) - - objectList, err := objectStorage.List("/") - c.Assert(err, IsNil) - c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""}) - c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""}) - c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) - c.Assert(len(objectList), Equals, 3) - - objectList, err = objectStorage.List("/path1") - c.Assert(err, IsNil) - c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""}) - c.Assert(len(objectList), Equals, 1) - - objectList, err = objectStorage.List("/path1/path2") - c.Assert(err, IsNil) - c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""}) - c.Assert(len(objectList), Equals, 1) - - objectList, err = objectStorage.List("/path1/path2/path3") - c.Assert(err, Not(IsNil)) - c.Assert(objectList, IsNil) + c.Assert(objectList[0].Name, Equals, "path1") } diff --git a/pkgs/storage/storage.go b/pkgs/storage/storage.go index b9bcc479f..3520e5ee4 100644 --- a/pkgs/storage/storage.go +++ b/pkgs/storage/storage.go @@ -3,13 +3,14 @@ package storage import "io" type ObjectStorage interface { - List(path string) ([]ObjectDescription, error) + List() ([]ObjectDescription, error) Get(path string) (io.Reader, error) Put(path string, object io.Reader) error } type ObjectDescription struct { - Path string - IsDir bool - Hash string + Name string + Md5sum string + Protectionlevel string + // Hash string - TODO }