Add list() object support for erasure and append storage drivers

- Reply back objects with their protectionlevel and md5sum
  - // TODO hash value
  - Calculate md5sum after "storeBlocks()", to make sure data is committed

cmd/erasure-demo/erasure.go

@@ -1,7 +1,8 @@
 package main
 
 import (
-	"errors"
+	"bytes"
+	"encoding/json"
 	"io"
 	"os"
 	"path"
@@ -11,8 +12,23 @@ import (
 )
 
 func erasureGetList(config inputConfig) (io.Reader, error) {
-	// do nothing
-	return nil, errors.New("Not Implemented")
+	var objectStorage storage.ObjectStorage
+	rootDir := path.Join(config.rootDir, config.storageDriver)
+	objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
+	if err != nil {
+		return nil, err
+	}
+	objectDescList, err := objectStorage.List()
+	if err != nil {
+		return nil, err
+	}
+	var objectDescListBytes []byte
+	if objectDescListBytes, err = json.Marshal(objectDescList); err != nil {
+		return nil, err
+	}
+	objectDescListBuffer := bytes.NewBuffer(objectDescListBytes)
+
+	return objectDescListBuffer, nil
 }
 
 func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {

cmd/erasure-demo/fs.go

@@ -15,7 +15,7 @@ func fsGetList(config inputConfig) (io.Reader, error) {
 	var objectStorage storage.ObjectStorage
 	rootDir := path.Join(config.rootDir, config.storageDriver)
 	objectStorage, _ = fsstorage.NewStorage(rootDir)
-	objectList, err := objectStorage.List("/")
+	objectList, err := objectStorage.List()
 	if err != nil {
 		return nil, err
 	}

pkgs/storage/appendstorage/append_storage.go

@@ -117,6 +117,17 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
 	return nil
 }
 
-func (aStorage *appendStorage) List(listPath string) ([]storage.ObjectDescription, error) {
-	return nil, errors.New("Not Implemented")
+func (aStorage *appendStorage) List() ([]storage.ObjectDescription, error) {
+	var objectDescList []storage.ObjectDescription
+	for objectName, _ := range aStorage.objects {
+		var objectDescription storage.ObjectDescription
+		objectDescription.Name = objectName
+		objectDescription.Md5sum = ""
+		objectDescription.Protectionlevel = ""
+		objectDescList = append(objectDescList, objectDescription)
+	}
+	if len(objectDescList) == 0 {
+		return nil, errors.New("No objects found")
+	}
+	return objectDescList, nil
 }

pkgs/storage/encodedstorage/encoded_storage.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/gob"
+	"encoding/hex"
 	"errors"
 	"io"
 	"io/ioutil"
@@ -93,8 +94,20 @@ func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) {
 	return reader, nil
 }
 
-func (eStorage *encodedStorage) List(listPath string) ([]storage.ObjectDescription, error) {
-	return nil, errors.New("Not Implemented")
+func (eStorage *encodedStorage) List() ([]storage.ObjectDescription, error) {
+	var objectDescList []storage.ObjectDescription
+	for objectName, objectEntry := range eStorage.objects {
+		var objectDescription storage.ObjectDescription
+		protectionLevel := strconv.Itoa(objectEntry.Encoderparams.K) + "," + strconv.Itoa(objectEntry.Encoderparams.M)
+		objectDescription.Name = objectName
+		objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum)
+		objectDescription.Protectionlevel = protectionLevel
+		objectDescList = append(objectDescList, objectDescription)
+	}
+	if len(objectDescList) == 0 {
+		return nil, errors.New("No objects found")
+	}
+	return objectDescList, nil
 }
 
 func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
@@ -124,8 +137,6 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
 	// encode
 	for chunk := range chunks {
 		if chunk.Err == nil {
-			// md5sum on chunk
-			hash.Write(chunk.Data)
 			// encode each
 			blocks, length := encoder.Encode(chunk.Data)
 			// store each
@@ -135,6 +146,8 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
 					return err
 				}
 			}
+			// md5sum only after chunk is committed to disk
+			hash.Write(chunk.Data)
 			blockEntry := StorageBlockEntry{
 				Index:  i,
 				Length: length,

pkgs/storage/encodedstorage/encoded_storage_test.go

@@ -38,9 +38,9 @@ func (s *EncodedStorageSuite) TestFileStoragePutAtRootPath(c *C) {
 	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 
-	//	objectList, err := objectStorage.List("/")
-	//	c.Assert(err, IsNil)
-	//	c.Assert(objectList[0].Path, Equals, "path1")
+	objectList, err := objectStorage.List()
+	c.Assert(err, IsNil)
+	c.Assert(objectList[0].Name, Equals, "path1")
 }
 
 func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) {
@@ -70,28 +70,6 @@ func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) {
 	objectBuffer3 := bytes.NewBuffer([]byte("object3"))
 	err = objectStorage.Put("object3", objectBuffer3)
 	c.Assert(err, IsNil)
-
-	// TODO support list
-	//	objectList, err := objectStorage.List("/")
-	//	c.Assert(err, IsNil)
-	//	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""})
-	//	c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""})
-	//	c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
-	//	c.Assert(len(objectList), Equals, 3)
-	//
-	//	objectList, err = objectStorage.List("/path1")
-	//	c.Assert(err, IsNil)
-	//	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
-	//	c.Assert(len(objectList), Equals, 1)
-	//
-	//	objectList, err = objectStorage.List("/path1/path2")
-	//	c.Assert(err, IsNil)
-	//	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""})
-	//	c.Assert(len(objectList), Equals, 1)
-	//
-	//	objectList, err = objectStorage.List("/path1/path2/path3")
-	//	c.Assert(err, Not(IsNil))
-	//	c.Assert(objectList, IsNil)
 }
 
 func (s *EncodedStorageSuite) TestObjectWithChunking(c *C) {

pkgs/storage/fsstorage/fs_storage.go

@@ -21,8 +21,8 @@ func NewStorage(rootDir string) (storage.ObjectStorage, error) {
 	return &newStorage, nil
 }
 
-func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) {
-	fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath))
+func (fsStorage *fileSystemStorage) List() ([]storage.ObjectDescription, error) {
+	fileInfos, err := ioutil.ReadDir(fsStorage.RootDir)
 	if err != nil {
 		return nil, err
 	}
@@ -31,9 +31,9 @@ func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescr
 
 	for _, fi := range fileInfos {
 		description := storage.ObjectDescription{
-			Path:  fi.Name(),
-			IsDir: fi.IsDir(),
-			Hash:  "", // TODO
+			Name:            fi.Name(),
+			Md5sum:          "",
+			Protectionlevel: "",
 		}
 		descriptions = append(descriptions, description)
 	}

pkgs/storage/fsstorage/fs_storage_test.go

@@ -37,56 +37,7 @@ func (s *fileSystemStorageSuite) TestfileStoragePutAtRootPath(c *C) {
 	object1, _ := ioutil.ReadAll(objectResult1)
 	c.Assert(string(object1), Equals, "object1")
 
-	objectList, err := objectStorage.List("/")
+	objectList, err := objectStorage.List()
 	c.Assert(err, IsNil)
-	c.Assert(objectList[0].Path, Equals, "path1")
-}
-
-func (s *fileSystemStorageSuite) TestfileStoragePutDirPath(c *C) {
-	rootDir, err := makeTempTestDir()
-	c.Assert(err, IsNil)
-	defer os.RemoveAll(rootDir)
-
-	var objectStorage storage.ObjectStorage
-	objectStorage, _ = NewStorage(rootDir)
-
-	objectBuffer1 := bytes.NewBuffer([]byte("object1"))
-	objectStorage.Put("path1/path2/path3", objectBuffer1)
-
-	// assert object1 was created in correct path
-	objectResult1, err := objectStorage.Get("path1/path2/path3")
-	c.Assert(err, IsNil)
-	object1, _ := ioutil.ReadAll(objectResult1)
-	c.Assert(string(object1), Equals, "object1")
-
-	// add second object
-	objectBuffer2 := bytes.NewBuffer([]byte("object2"))
-	err = objectStorage.Put("path2/path2/path2", objectBuffer2)
-	c.Assert(err, IsNil)
-
-	// add third object
-	objectBuffer3 := bytes.NewBuffer([]byte("object3"))
-	err = objectStorage.Put("object3", objectBuffer3)
-	c.Assert(err, IsNil)
-
-	objectList, err := objectStorage.List("/")
-	c.Assert(err, IsNil)
-	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""})
-	c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""})
-	c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
-	c.Assert(len(objectList), Equals, 3)
-
-	objectList, err = objectStorage.List("/path1")
-	c.Assert(err, IsNil)
-	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
-	c.Assert(len(objectList), Equals, 1)
-
-	objectList, err = objectStorage.List("/path1/path2")
-	c.Assert(err, IsNil)
-	c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""})
-	c.Assert(len(objectList), Equals, 1)
-
-	objectList, err = objectStorage.List("/path1/path2/path3")
-	c.Assert(err, Not(IsNil))
-	c.Assert(objectList, IsNil)
+	c.Assert(objectList[0].Name, Equals, "path1")
 }

pkgs/storage/storage.go

@@ -3,13 +3,14 @@ package storage
 import "io"
 
 type ObjectStorage interface {
-	List(path string) ([]ObjectDescription, error)
+	List() ([]ObjectDescription, error)
 	Get(path string) (io.Reader, error)
 	Put(path string, object io.Reader) error
 }
 
 type ObjectDescription struct {
-	Path  string
-	IsDir bool
-	Hash  string
+	Name            string
+	Md5sum          string
+	Protectionlevel string
+	//	Hash   string - TODO
 }