You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
133 lines
3.0 KiB
133 lines
3.0 KiB
package appendstorage
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/gob"
|
|
"errors"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"strconv"
|
|
|
|
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
|
"github.com/minio-io/minio/pkgs/storage"
|
|
)
|
|
|
|
type appendStorage struct {
|
|
RootDir string
|
|
file *os.File
|
|
objects map[string]Header
|
|
objectsFile string
|
|
}
|
|
|
|
type Header struct {
|
|
Path string
|
|
Offset int64
|
|
Length int
|
|
Crc uint32
|
|
}
|
|
|
|
func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
|
|
rootPath := path.Join(rootDir, strconv.Itoa(slice))
|
|
// TODO verify and fix partial writes
|
|
file, err := os.OpenFile(rootPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return &appendStorage{}, err
|
|
}
|
|
objectsFile := path.Join(rootDir, strconv.Itoa(slice)+".map")
|
|
objects := make(map[string]Header)
|
|
if _, err := os.Stat(objectsFile); err == nil {
|
|
mapFile, err := os.Open(objectsFile)
|
|
defer mapFile.Close()
|
|
if err != nil {
|
|
return &appendStorage{}, nil
|
|
}
|
|
dec := gob.NewDecoder(mapFile)
|
|
err = dec.Decode(&objects)
|
|
if err != nil && err != io.EOF {
|
|
return &appendStorage{}, nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
return &appendStorage{}, err
|
|
}
|
|
return &appendStorage{
|
|
RootDir: rootDir,
|
|
file: file,
|
|
objects: objects,
|
|
objectsFile: objectsFile,
|
|
}, nil
|
|
}
|
|
|
|
func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
|
|
header, ok := storage.objects[objectPath]
|
|
if ok == false {
|
|
return nil, nil
|
|
}
|
|
|
|
offset := header.Offset
|
|
length := header.Length
|
|
crc := header.Crc
|
|
|
|
object := make([]byte, length)
|
|
_, err := storage.file.ReadAt(object, offset)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
newcrc, err := crc32c.Crc32c(object)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if newcrc != crc {
|
|
return nil, err
|
|
}
|
|
return bytes.NewBuffer(object), nil
|
|
}
|
|
|
|
func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
|
|
header := Header{
|
|
Path: objectPath,
|
|
Offset: 0,
|
|
Length: 0,
|
|
Crc: 0,
|
|
}
|
|
offset, err := aStorage.file.Seek(0, os.SEEK_END)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
objectBytes, err := ioutil.ReadAll(object)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := aStorage.file.Write(objectBytes); err != nil {
|
|
return err
|
|
}
|
|
header.Offset = offset
|
|
header.Length = len(objectBytes)
|
|
header.Crc, err = crc32c.Crc32c(objectBytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
aStorage.objects[objectPath] = header
|
|
var mapBuffer bytes.Buffer
|
|
encoder := gob.NewEncoder(&mapBuffer)
|
|
encoder.Encode(aStorage.objects)
|
|
ioutil.WriteFile(aStorage.objectsFile, mapBuffer.Bytes(), 0600)
|
|
return nil
|
|
}
|
|
|
|
func (aStorage *appendStorage) List() ([]storage.ObjectDescription, error) {
|
|
var objectDescList []storage.ObjectDescription
|
|
for objectName, _ := range aStorage.objects {
|
|
var objectDescription storage.ObjectDescription
|
|
objectDescription.Name = objectName
|
|
objectDescription.Md5sum = ""
|
|
objectDescription.Protectionlevel = ""
|
|
objectDescList = append(objectDescList, objectDescription)
|
|
}
|
|
if len(objectDescList) == 0 {
|
|
return nil, errors.New("No objects found")
|
|
}
|
|
return objectDescList, nil
|
|
}
|
|
|