Merge pull request #102 from fkautz/pr_out_adding_list_path_to_object_storage_definition_and_accompanying_definition_to_fs

master
Frederick F. Kautz IV 10 years ago
commit c7b4e14f64
  1. 9
      cmd/erasure-demo/fs.go
  2. 14
      pkgs/storage/appendstorage/append_storage.go
  3. 18
      pkgs/storage/fsstorage/fs_storage.go
  4. 52
      pkgs/storage/fsstorage/fs_storage_test.go
  5. 8
      pkgs/storage/storage.go
  6. 32
      pkgs/storage/storage_test.go

@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json"
"io" "io"
"os" "os"
"path" "path"
@ -14,11 +15,15 @@ func fsGetList(config inputConfig) (io.Reader, error) {
var objectStorage storage.ObjectStorage var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver) rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir}
objectlist, err := objectStorage.GetList() objectList, err := objectStorage.List("/")
if err != nil { if err != nil {
return nil, err return nil, err
} }
objectListBuffer := bytes.NewBuffer(objectlist) var objectListBytes []byte
if objectListBytes, err = json.Marshal(objectList); err != nil {
return nil, err
}
objectListBuffer := bytes.NewBuffer(objectListBytes)
return objectListBuffer, nil return objectListBuffer, nil
} }

@ -75,30 +75,30 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
return object, nil return object, nil
} }
func (storage *appendStorage) Put(objectPath string, object []byte) error { func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
header := Header{ header := Header{
Path: objectPath, Path: objectPath,
Offset: 0, Offset: 0,
Length: 0, Length: 0,
Crc: nil, Crc: nil,
} }
offset, err := storage.file.Seek(0, os.SEEK_END) offset, err := aStorage.file.Seek(0, os.SEEK_END)
if err != nil { if err != nil {
return err return err
} }
if _, err := storage.file.Write(object); err != nil { if _, err := aStorage.file.Write(object); err != nil {
return err return err
} }
header.Offset = offset header.Offset = offset
header.Length = len(object) header.Length = len(object)
storage.objects[objectPath] = header aStorage.objects[objectPath] = header
var mapBuffer bytes.Buffer var mapBuffer bytes.Buffer
encoder := gob.NewEncoder(&mapBuffer) encoder := gob.NewEncoder(&mapBuffer)
encoder.Encode(storage.objects) encoder.Encode(aStorage.objects)
ioutil.WriteFile(storage.objectsFile, mapBuffer.Bytes(), 0600) ioutil.WriteFile(aStorage.objectsFile, mapBuffer.Bytes(), 0600)
return nil return nil
} }
func (storage *appendStorage) GetList() ([]byte, error) { func (aStorage *appendStorage) List(listPath string) ([]storage.ObjectDescription, error) {
return nil, errors.New("Not Implemented") return nil, errors.New("Not Implemented")
} }

@ -5,23 +5,31 @@ import (
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"github.com/minio-io/minio/pkgs/storage"
) )
type FileSystemStorage struct { type FileSystemStorage struct {
RootDir string RootDir string
} }
func (storage FileSystemStorage) GetList() ([]byte, error) { func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) {
fileInfos, err := ioutil.ReadDir(storage.RootDir) fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath))
if err != nil { if err != nil {
return nil, err return nil, err
} }
var list []byte var descriptions []storage.ObjectDescription
for _, fi := range fileInfos { for _, fi := range fileInfos {
list = append(list, "{"+fi.Name()+"}\n"...) description := storage.ObjectDescription{
Path: fi.Name(),
IsDir: fi.IsDir(),
Hash: "", // TODO
}
descriptions = append(descriptions, description)
} }
return list, nil return descriptions, nil
} }
func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) { func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) {

@ -3,6 +3,7 @@ package fsstorage
import ( import (
"io/ioutil" "io/ioutil"
"os" "os"
"testing"
"github.com/minio-io/minio/pkgs/storage" "github.com/minio-io/minio/pkgs/storage"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
@ -12,6 +13,8 @@ type FileSystemStorageSuite struct{}
var _ = Suite(&FileSystemStorageSuite{}) var _ = Suite(&FileSystemStorageSuite{})
func Test(t *testing.T) { TestingT(t) }
func makeTempTestDir() (string, error) { func makeTempTestDir() (string, error) {
return ioutil.TempDir("/tmp", "minio-test-") return ioutil.TempDir("/tmp", "minio-test-")
} }
@ -21,17 +24,21 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
defer os.RemoveAll(rootDir) defer os.RemoveAll(rootDir)
var storage storage.ObjectStorage var objectStorage storage.ObjectStorage
storage = FileSystemStorage{ objectStorage = FileSystemStorage{
RootDir: rootDir, RootDir: rootDir,
} }
storage.Put("path1", []byte("object1")) objectStorage.Put("path1", []byte("object1"))
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := storage.Get("path1") object1, err := objectStorage.Get("path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object1") c.Assert(string(object1), Equals, "object1")
objectList, err := objectStorage.List("/")
c.Assert(err, IsNil)
c.Assert(objectList[0].Path, Equals, "path1")
} }
func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) { func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) {
@ -39,15 +46,44 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
defer os.RemoveAll(rootDir) defer os.RemoveAll(rootDir)
var storage storage.ObjectStorage var objectStorage storage.ObjectStorage
storage = FileSystemStorage{ objectStorage = FileSystemStorage{
RootDir: rootDir, RootDir: rootDir,
} }
storage.Put("path1/path2/path3", []byte("object")) objectStorage.Put("path1/path2/path3", []byte("object"))
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := storage.Get("path1/path2/path3") object1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object") c.Assert(string(object1), Equals, "object")
// add second object
err = objectStorage.Put("path2/path2/path2", []byte("object2"))
c.Assert(err, IsNil)
// add third object
err = objectStorage.Put("object3", []byte("object3"))
c.Assert(err, IsNil)
objectList, err := objectStorage.List("/")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""})
c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""})
c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
c.Assert(len(objectList), Equals, 3)
objectList, err = objectStorage.List("/path1")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
c.Assert(len(objectList), Equals, 1)
objectList, err = objectStorage.List("/path1/path2")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""})
c.Assert(len(objectList), Equals, 1)
objectList, err = objectStorage.List("/path1/path2/path3")
c.Assert(err, Not(IsNil))
c.Assert(objectList, IsNil)
} }

@ -1,7 +1,13 @@
package storage package storage
type ObjectStorage interface { type ObjectStorage interface {
GetList() ([]byte, error) List(path string) ([]ObjectDescription, error)
Get(path string) ([]byte, error) Get(path string) ([]byte, error)
Put(path string, object []byte) error Put(path string, object []byte) error
} }
type ObjectDescription struct {
Path string
IsDir bool
Hash string
}

@ -1,32 +0,0 @@
package storage
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"testing"
)
func TestPrintsStorage(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(storageHandler))
defer server.Close()
res, err := http.Get(server.URL)
if err != nil {
log.Fatal(err)
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
log.Fatal(err)
}
bodyString := string(body)
if bodyString != "Storage" {
log.Fatal("Expected 'Storage', Received '" + bodyString + "'")
}
}
func storageHandler(w http.ResponseWriter, req *http.Request) {
fmt.Fprintf(w, "Storage")
}
Loading…
Cancel
Save