From 1d7b3866cb6c81a5ef58468d325937e096e4ca20 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Mon, 8 Dec 2014 23:31:19 -0800 Subject: [PATCH] Adding append storage driver --- pkgs/storage/appendstorage/append_storage.go | 99 +++++++++++++++ .../appendstorage/append_storage_test.go | 113 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 pkgs/storage/appendstorage/append_storage.go create mode 100644 pkgs/storage/appendstorage/append_storage_test.go diff --git a/pkgs/storage/appendstorage/append_storage.go b/pkgs/storage/appendstorage/append_storage.go new file mode 100644 index 000000000..b38c20ad5 --- /dev/null +++ b/pkgs/storage/appendstorage/append_storage.go @@ -0,0 +1,99 @@ +package appendstorage + +import ( + "bytes" + "encoding/gob" + "io/ioutil" + "os" + "path" + "strconv" + + "github.com/minio-io/minio/pkgs/storage" +) + +type appendStorage struct { + RootDir string + file *os.File + objects map[string]Header + objectsFile string +} + +type Header struct { + Path string + Offset int64 + Length int + Crc []byte +} + +func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) { + rootPath := path.Join(rootDir, strconv.Itoa(slice)) + // TODO verify and fix partial writes + file, err := os.OpenFile(rootPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return &appendStorage{}, err + } + objectsFile := path.Join(rootDir, strconv.Itoa(slice)+".map") + objects := make(map[string]Header) + if _, err := os.Stat(objectsFile); err == nil { + mapFile, err := os.Open(objectsFile) + defer mapFile.Close() + if err != nil { + return &appendStorage{}, nil + } + dec := gob.NewDecoder(mapFile) + err = dec.Decode(&objects) + if err != nil { + return &appendStorage{}, nil + } + } + if err != nil { + return &appendStorage{}, err + } + return &appendStorage{ + RootDir: rootDir, + file: file, + objects: objects, + objectsFile: objectsFile, + }, nil +} + +func (storage *appendStorage) Get(objectPath string) ([]byte, error) { + header, ok := storage.objects[objectPath] + if ok == false { + return nil, nil + } + + offset := header.Offset + length := header.Length + + object := make([]byte, length) + _, err := storage.file.ReadAt(object, offset) + if err != nil { + return nil, err + } + return object, nil +} + +func (storage *appendStorage) Put(objectPath string, object []byte) error { + header := Header{ + Path: objectPath, + Offset: 0, + Length: 0, + Crc: nil, + } + offset, err := storage.file.Seek(0, os.SEEK_END) + if err != nil { + return err + } + if _, err := storage.file.Write(object); err != nil { + return err + } + header.Offset = offset + header.Length = len(object) + storage.objects[objectPath] = header + var mapBuffer bytes.Buffer + encoder := gob.NewEncoder(&mapBuffer) + encoder.Encode(storage.objects) + ioutil.WriteFile(storage.objectsFile, mapBuffer.Bytes(), 0600) + return nil +} diff --git a/pkgs/storage/appendstorage/append_storage_test.go b/pkgs/storage/appendstorage/append_storage_test.go new file mode 100644 index 000000000..0ca4fb230 --- /dev/null +++ b/pkgs/storage/appendstorage/append_storage_test.go @@ -0,0 +1,113 @@ +package appendstorage + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/minio-io/minio/pkgs/storage" + . "gopkg.in/check.v1" +) + +type AppendStorageSuite struct{} + +var _ = Suite(&AppendStorageSuite{}) + +func Test(t *testing.T) { TestingT(t) } + +func makeTempTestDir() (string, error) { + return ioutil.TempDir("/tmp", "minio-test-") +} + +func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) { + rootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(rootDir) + + var objectStorage storage.ObjectStorage + objectStorage, err = NewStorage(rootDir, 0) + c.Assert(err, IsNil) + + err = objectStorage.Put("path1", []byte("object1")) + c.Assert(err, IsNil) + + // assert object1 was created in correct path + object1, err := objectStorage.Get("path1") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object1") + + err = objectStorage.Put("path2", []byte("object2")) + c.Assert(err, IsNil) + + // assert object1 was created in correct path + object2, err := objectStorage.Get("path2") + c.Assert(err, IsNil) + c.Assert(string(object2), Equals, "object2") + + object1, err = objectStorage.Get("path1") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object1") +} + +func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) { + rootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(rootDir) + + var objectStorage storage.ObjectStorage + objectStorage, err = NewStorage(rootDir, 0) + c.Assert(err, IsNil) + + // add object 1 + objectStorage.Put("path1/path2/path3", []byte("object")) + + // assert object1 was created in correct path + object1, err := objectStorage.Get("path1/path2/path3") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object") + + // add object 2 + objectStorage.Put("path1/path1/path1", []byte("object2")) + + // assert object1 was created in correct path + object2, err := objectStorage.Get("path1/path1/path1") + c.Assert(err, IsNil) + c.Assert(string(object2), Equals, "object2") +} + +func (s *AppendStorageSuite) TestSerialization(c *C) { + rootDir, err := makeTempTestDir() + c.Assert(err, IsNil) + defer os.RemoveAll(rootDir) + + objectStorage, err := NewStorage(rootDir, 0) + c.Assert(err, IsNil) + + err = objectStorage.Put("path1", []byte("object1")) + c.Assert(err, IsNil) + err = objectStorage.Put("path2", []byte("object2")) + c.Assert(err, IsNil) + err = objectStorage.Put("path3/obj3", []byte("object3")) + c.Assert(err, IsNil) + + es := objectStorage.(*appendStorage) + es.file.Close() + + objectStorage2, err := NewStorage(rootDir, 0) + c.Assert(err, IsNil) + + object1, err := objectStorage2.Get("path1") + c.Assert(err, IsNil) + c.Assert(string(object1), Equals, "object1") + + object2, err := objectStorage2.Get("path2") + c.Assert(err, IsNil) + c.Assert(string(object2), Equals, "object2") + + object3, err := objectStorage2.Get("path3/obj3") + c.Assert(err, IsNil) + c.Assert(string(object3), Equals, "object3") +} + +func (s *AppendStorageSuite) TestSlice(c *C) { +}