oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #677 from harshavardhana/pr_out_moving_to_more_typed_responses_this_removes_all_the_necessity_for_strconv

Browse Source 
Donut moves to typed metadata, removing the necessity for strconv, parsing and string converstions
master
Harshavardhana 9 years ago
parent e3d8a9e0f1 68974918ac
commit aab4937084
7 changed files with 154 additions and 183 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 132
      pkg/storage/donut/bucket.go
  2. 23
      pkg/storage/donut/definitions.go
  3. 84
      pkg/storage/donut/donut.go
  4. 17
      pkg/storage/donut/donut_test.go
  5. 6
      pkg/storage/donut/interfaces.go
  6. 28
      pkg/storage/donut/object.go
  7. 47
      pkg/storage/drivers/donut/donut.go

132
pkg/storage/donut/bucket.go
Unescape View File

@ -49,20 +49,18 @@ type bucket struct {
} }
// newBucket - instantiate a new bucket // newBucket - instantiate a new bucket
func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, map[string]string, error) { func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bucket, BucketMetadata, error) {
errParams := map[string]string{ errParams := map[string]string{
"bucketName": bucketName, "bucketName": bucketName,
"donutName": donutName, "donutName": donutName,
"aclType": aclType, "aclType": aclType,
} }
if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" { if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" {
return bucket{}, nil, iodine.New(InvalidArgument{}, errParams) return bucket{}, BucketMetadata{}, iodine.New(InvalidArgument{}, errParams)
} }
bucketMetadata := make(map[string]string)
bucketMetadata["acl"] = aclType
t := time.Now().UTC()
bucketMetadata["created"] = t.Format(time.RFC3339Nano)
b := bucket{} b := bucket{}
t := time.Now().UTC()
b.name = bucketName b.name = bucketName
b.acl = aclType b.acl = aclType
b.time = t b.time = t
@ -70,7 +68,13 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu
b.nodes = nodes b.nodes = nodes
b.objects = make(map[string]object) b.objects = make(map[string]object)
b.lock = new(sync.RWMutex) b.lock = new(sync.RWMutex)
return b, bucketMetadata, nil
metadata := BucketMetadata{}
metadata.Name = bucketName
metadata.ACL = aclType
metadata.Created = t
return b, metadata, nil
} }
func (b bucket) getBucketName() string { func (b bucket) getBucketName() string {
@ -93,7 +97,7 @@ func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, er
return newObjectMetadata.Object, nil return newObjectMetadata.Object, nil
} }
func (b bucket) GetObjectMetadata(objectName string) (*objectMetadata, error) { func (b bucket) GetObjectMetadata(objectName string) (ObjectMetadata, error) {
return b.objects[objectName].GetObjectMetadata() return b.objects[objectName].GetObjectMetadata()
} }
@ -183,14 +187,14 @@ func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64,
if !ok { if !ok {
return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil) return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil)
} }
// verify if donutObjectMetadata is readable, before we server the request // verify if sysObjectMetadata is readable, before we server the request
donutObjMetadata, err := object.GetDonutObjectMetadata() sysObjMetadata, err := object.GetSystemObjectMetadata()
if err != nil { if err != nil {
return nil, 0, iodine.New(err, nil) return nil, 0, iodine.New(err, nil)
} }
// read and reply back to GetObject() request in a go-routine // read and reply back to GetObject() request in a go-routine
go b.readEncodedData(b.normalizeObjectName(objectName), writer, donutObjMetadata) go b.readEncodedData(b.normalizeObjectName(objectName), writer, sysObjMetadata)
return reader, donutObjMetadata.Size, nil return reader, sysObjMetadata.Size, nil
} }
// WriteObject - write a new object into bucket // WriteObject - write a new object into bucket
@ -207,10 +211,10 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
sumMD5 := md5.New() sumMD5 := md5.New()
sum512 := sha512.New() sum512 := sha512.New()
objectMetadata := new(objectMetadata) objMetadata := new(ObjectMetadata)
donutObjectMetadata := new(donutObjectMetadata) sysObjMetadata := new(SystemObjectMetadata)
objectMetadata.Version = objectMetadataVersion objMetadata.Version = objectMetadataVersion
donutObjectMetadata.Version = donutObjectMetadataVersion sysObjMetadata.Version = systemObjectMetadataVersion
size := metadata["contentLength"] size := metadata["contentLength"]
sizeInt, err := strconv.ParseInt(size, 10, 64) sizeInt, err := strconv.ParseInt(size, 10, 64)
if err != nil { if err != nil {
@ -225,8 +229,8 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
donutObjectMetadata.Size = totalLength sysObjMetadata.Size = totalLength
objectMetadata.Size = totalLength objMetadata.Size = totalLength
case false: case false:
// calculate data and parity dictated by total number of writers // calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers)) k, m, err := b.getDataAndParity(len(writers))
@ -239,50 +243,50 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
/// donutMetadata section /// donutMetadata section
donutObjectMetadata.BlockSize = 10 * 1024 * 1024 sysObjMetadata.BlockSize = 10 * 1024 * 1024
donutObjectMetadata.ChunkCount = chunkCount sysObjMetadata.ChunkCount = chunkCount
donutObjectMetadata.DataDisks = k sysObjMetadata.DataDisks = k
donutObjectMetadata.ParityDisks = m sysObjMetadata.ParityDisks = m
donutObjectMetadata.ErasureTechnique = "Cauchy" sysObjMetadata.ErasureTechnique = "Cauchy"
donutObjectMetadata.Size = int64(totalLength) sysObjMetadata.Size = int64(totalLength)
// keep size inside objectMetadata as well for Object API requests // keep size inside ObjectMetadata as well for Object API requests
objectMetadata.Size = int64(totalLength) objMetadata.Size = int64(totalLength)
} }
objectMetadata.Bucket = b.getBucketName() objMetadata.Bucket = b.getBucketName()
objectMetadata.Object = objectName objMetadata.Object = objectName
objectMetadata.Metadata = metadata objMetadata.Metadata = metadata
dataMD5sum := sumMD5.Sum(nil) dataMD5sum := sumMD5.Sum(nil)
dataSHA512sum := sum512.Sum(nil) dataSHA512sum := sum512.Sum(nil)
objectMetadata.Created = time.Now().UTC() objMetadata.Created = time.Now().UTC()
// keeping md5sum for the object in two different places // keeping md5sum for the object in two different places
// one for object storage and another is for internal use // one for object storage and another is for internal use
hexMD5Sum := hex.EncodeToString(dataMD5sum) hexMD5Sum := hex.EncodeToString(dataMD5sum)
hex512Sum := hex.EncodeToString(dataSHA512sum) hex512Sum := hex.EncodeToString(dataSHA512sum)
objectMetadata.MD5Sum = hexMD5Sum objMetadata.MD5Sum = hexMD5Sum
objectMetadata.SHA512Sum = hex512Sum objMetadata.SHA512Sum = hex512Sum
donutObjectMetadata.MD5Sum = hexMD5Sum sysObjMetadata.MD5Sum = hexMD5Sum
donutObjectMetadata.SHA512Sum = hex512Sum sysObjMetadata.SHA512Sum = hex512Sum
// Verify if the written object is equal to what is expected, only if it is requested as such // Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" { if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata.MD5Sum); err != nil { if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objMetadata.MD5Sum); err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
} }
// write donut specific metadata // write donut specific metadata
if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil { if err := b.writeSystemObjectMetadata(b.normalizeObjectName(objectName), sysObjMetadata); err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// write object specific metadata // write object specific metadata
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil { if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objMetadata); err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// close all writers, when control flow reaches here // close all writers, when control flow reaches here
for _, writer := range writers { for _, writer := range writers {
writer.Close() writer.Close()
} }
return objectMetadata.MD5Sum, nil return objMetadata.MD5Sum, nil
} }
// isMD5SumEqual - returns error if md5sum mismatches, other its `nil` // isMD5SumEqual - returns error if md5sum mismatches, other its `nil`
@ -305,41 +309,41 @@ func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
} }
// writeObjectMetadata - write additional object metadata // writeObjectMetadata - write additional object metadata
func (b bucket) writeObjectMetadata(objectName string, objectMetadata *objectMetadata) error { func (b bucket) writeObjectMetadata(objectName string, objMetadata *ObjectMetadata) error {
if objectMetadata == nil { if objMetadata == nil {
return iodine.New(InvalidArgument{}, nil) return iodine.New(InvalidArgument{}, nil)
} }
objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig) objMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig)
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
for _, objectMetadataWriter := range objectMetadataWriters { for _, objMetadataWriter := range objMetadataWriters {
defer objectMetadataWriter.Close() defer objMetadataWriter.Close()
} }
for _, objectMetadataWriter := range objectMetadataWriters { for _, objMetadataWriter := range objMetadataWriters {
jenc := json.NewEncoder(objectMetadataWriter) jenc := json.NewEncoder(objMetadataWriter)
if err := jenc.Encode(objectMetadata); err != nil { if err := jenc.Encode(objMetadata); err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
} }
return nil return nil
} }
// writeDonutObjectMetadata - write donut related object metadata // writeSystemObjectMetadata - write donut related object metadata
func (b bucket) writeDonutObjectMetadata(objectName string, donutObjectMetadata *donutObjectMetadata) error { func (b bucket) writeSystemObjectMetadata(objectName string, sysObjMetadata *SystemObjectMetadata) error {
if donutObjectMetadata == nil { if sysObjMetadata == nil {
return iodine.New(InvalidArgument{}, nil) return iodine.New(InvalidArgument{}, nil)
} }
donutObjectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig) sysObjMetadataWriters, err := b.getDiskWriters(objectName, sysObjectMetadataConfig)
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { for _, sysObjMetadataWriter := range sysObjMetadataWriters {
defer donutObjectMetadataWriter.Close() defer sysObjMetadataWriter.Close()
} }
for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { for _, sysObjMetadataWriter := range sysObjMetadataWriters {
jenc := json.NewEncoder(donutObjectMetadataWriter) jenc := json.NewEncoder(sysObjMetadataWriter)
if err := jenc.Encode(donutObjectMetadata); err != nil { if err := jenc.Encode(sysObjMetadata); err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
} }
@ -403,8 +407,8 @@ func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectDat
} }
// readEncodedData - // readEncodedData -
func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutObjMetadata *donutObjectMetadata) { func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, sysObjMetadata SystemObjectMetadata) {
expectedMd5sum, err := hex.DecodeString(donutObjMetadata.MD5Sum) expectedMd5sum, err := hex.DecodeString(sysObjMetadata.MD5Sum)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
@ -421,18 +425,18 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
mwriter := io.MultiWriter(writer, hasher) mwriter := io.MultiWriter(writer, hasher)
switch len(readers) == 1 { switch len(readers) == 1 {
case false: case false:
if donutObjMetadata.ErasureTechnique == "" { if sysObjMetadata.ErasureTechnique == "" {
writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil)) writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil))
return return
} }
encoder, err := newEncoder(donutObjMetadata.DataDisks, donutObjMetadata.ParityDisks, donutObjMetadata.ErasureTechnique) encoder, err := newEncoder(sysObjMetadata.DataDisks, sysObjMetadata.ParityDisks, sysObjMetadata.ErasureTechnique)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
} }
totalLeft := donutObjMetadata.Size totalLeft := sysObjMetadata.Size
for i := 0; i < donutObjMetadata.ChunkCount; i++ { for i := 0; i < sysObjMetadata.ChunkCount; i++ {
decodedData, err := b.decodeEncodedData(totalLeft, int64(donutObjMetadata.BlockSize), readers, encoder, writer) decodedData, err := b.decodeEncodedData(totalLeft, int64(sysObjMetadata.BlockSize), readers, encoder, writer)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
@ -442,7 +446,7 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
} }
totalLeft = totalLeft - int64(donutObjMetadata.BlockSize) totalLeft = totalLeft - int64(sysObjMetadata.BlockSize)
} }
case true: case true:
_, err := io.Copy(writer, readers[0]) _, err := io.Copy(writer, readers[0])

23
pkg/storage/donut/definitions.go
Unescape View File

@ -18,8 +18,8 @@ package donut
import "time" import "time"
// objectMetadata object specific metadata per object // ObjectMetadata object specific metadata per object
type objectMetadata struct { type ObjectMetadata struct {
// version // version
Version string `json:"version"` Version string `json:"version"`
@ -37,8 +37,8 @@ type objectMetadata struct {
Metadata map[string]string `json:"metadata"` Metadata map[string]string `json:"metadata"`
} }
// donutObjectMetadata container for donut specific internal metadata per object // SystemObjectMetadata container for donut system specific metadata per object
type donutObjectMetadata struct { type SystemObjectMetadata struct {
// version // version
Version string `json:"version"` Version string `json:"version"`
@ -57,14 +57,21 @@ type donutObjectMetadata struct {
SHA512Sum string `json:"sys.sha512sum"` SHA512Sum string `json:"sys.sha512sum"`
} }
// donutMetadata container for donut level metadata // Metadata container for donut metadata
type donutMetadata struct { type Metadata struct {
Version string `json:"version"` Version string `json:"version"`
} }
// bucketMetadata container for bucket level metadata // AllBuckets container for all buckets
type bucketMetadata struct { type AllBuckets struct {
Buckets map[string]BucketMetadata
}
// BucketMetadata container for bucket level metadata
type BucketMetadata struct {
Version string `json:"version"` Version string `json:"version"`
Name string `json:"name"`
ACL string `json:"acl"` ACL string `json:"acl"`
Created time.Time `json:"created"`
Metadata map[string]string `json:"metadata"` Metadata map[string]string `json:"metadata"`
} }

84
pkg/storage/donut/donut.go
Unescape View File

@ -25,7 +25,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time"
"github.com/minio/minio/pkg/iodine" "github.com/minio/minio/pkg/iodine"
) )
@ -40,17 +39,18 @@ type donut struct {
// config files used inside Donut // config files used inside Donut
const ( const (
// donut object metadata and config // donut system object metadata
donutObjectMetadataConfig = "donutObjectMetadata.json" sysObjectMetadataConfig = "sysObjectMetadata.json"
donutConfig = "donutMetadata.json" // donut system config
donutConfig = "donutConfig.json"
// bucket, object metadata // bucket, object metadata
bucketMetadataConfig = "bucketMetadata.json" bucketMetadataConfig = "bucketMetadata.json"
objectMetadataConfig = "objectMetadata.json" objectMetadataConfig = "objectMetadata.json"
// versions // versions
objectMetadataVersion = "1.0" objectMetadataVersion = "1.0.0"
donutObjectMetadataVersion = "1.0" systemObjectMetadataVersion = "1.0.0"
) )
// attachDonutNode - wrapper function to instantiate a new node for associatedt donut // attachDonutNode - wrapper function to instantiate a new node for associatedt donut
@ -98,24 +98,24 @@ func (dt donut) MakeBucket(bucket, acl string) error {
} }
// GetBucketMetadata - get bucket metadata // GetBucketMetadata - get bucket metadata
func (dt donut) GetBucketMetadata(bucket string) (map[string]string, error) { func (dt donut) GetBucketMetadata(bucketName string) (BucketMetadata, error) {
dt.lock.RLock() dt.lock.RLock()
defer dt.lock.RUnlock() defer dt.lock.RUnlock()
if err := dt.listDonutBuckets(); err != nil { if err := dt.listDonutBuckets(); err != nil {
return nil, iodine.New(err, nil) return BucketMetadata{}, iodine.New(err, nil)
} }
if _, ok := dt.buckets[bucket]; !ok { if _, ok := dt.buckets[bucketName]; !ok {
return nil, iodine.New(BucketNotFound{Bucket: bucket}, nil) return BucketMetadata{}, iodine.New(BucketNotFound{Bucket: bucketName}, nil)
} }
metadata, err := dt.getDonutBucketMetadata() metadata, err := dt.getDonutBucketMetadata()
if err != nil { if err != nil {
return nil, iodine.New(err, nil) return BucketMetadata{}, iodine.New(err, nil)
} }
return metadata[bucket], nil return metadata.Buckets[bucketName], nil
} }
// SetBucketMetadata - set bucket metadata // SetBucketMetadata - set bucket metadata
func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]string) error { func (dt donut) SetBucketMetadata(bucketName string, bucketMetadata map[string]string) error {
dt.lock.Lock() dt.lock.Lock()
defer dt.lock.Unlock() defer dt.lock.Unlock()
if err := dt.listDonutBuckets(); err != nil { if err := dt.listDonutBuckets(); err != nil {
@ -125,28 +125,31 @@ func (dt donut) SetBucketMetadata(bucket string, bucketMetadata map[string]strin
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
oldBucketMetadata := metadata[bucket] oldBucketMetadata := metadata.Buckets[bucketName]
// TODO ignore rest of the keys for now, only mutable data is "acl" acl, ok := bucketMetadata["acl"]
oldBucketMetadata["acl"] = bucketMetadata["acl"] if !ok {
metadata[bucket] = oldBucketMetadata return iodine.New(InvalidArgument{}, nil)
}
oldBucketMetadata.ACL = acl
metadata.Buckets[bucketName] = oldBucketMetadata
return dt.setDonutBucketMetadata(metadata) return dt.setDonutBucketMetadata(metadata)
} }
// ListBuckets - return list of buckets // ListBuckets - return list of buckets
func (dt donut) ListBuckets() (metadata map[string]map[string]string, err error) { func (dt donut) ListBuckets() (map[string]BucketMetadata, error) {
dt.lock.RLock() dt.lock.RLock()
defer dt.lock.RUnlock() defer dt.lock.RUnlock()
if err := dt.listDonutBuckets(); err != nil { if err := dt.listDonutBuckets(); err != nil {
return nil, iodine.New(err, nil) return nil, iodine.New(err, nil)
} }
metadata, err = dt.getDonutBucketMetadata() metadata, err := dt.getDonutBucketMetadata()
if err != nil { if err != nil {
// intentionally left out the error when Donut is empty // intentionally left out the error when Donut is empty
// but we need to revisit this area in future - since we need // but we need to revisit this area in future - since we need
// to figure out between acceptable and unacceptable errors // to figure out between acceptable and unacceptable errors
return make(map[string]map[string]string), nil return make(map[string]BucketMetadata), nil
} }
return metadata, nil return metadata.Buckets, nil
} }
// ListObjects - return list of objects // ListObjects - return list of objects
@ -233,7 +236,7 @@ func (dt donut) GetObject(bucket, object string) (reader io.ReadCloser, size int
} }
// GetObjectMetadata - get object metadata // GetObjectMetadata - get object metadata
func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, error) { func (dt donut) GetObjectMetadata(bucket, object string) (ObjectMetadata, error) {
dt.lock.RLock() dt.lock.RLock()
defer dt.lock.RUnlock() defer dt.lock.RUnlock()
errParams := map[string]string{ errParams := map[string]string{
@ -241,10 +244,10 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err
"object": object, "object": object,
} }
if err := dt.listDonutBuckets(); err != nil { if err := dt.listDonutBuckets(); err != nil {
return nil, iodine.New(err, errParams) return ObjectMetadata{}, iodine.New(err, errParams)
} }
if _, ok := dt.buckets[bucket]; !ok { if _, ok := dt.buckets[bucket]; !ok {
return nil, iodine.New(BucketNotFound{Bucket: bucket}, errParams) return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, errParams)
} }
// //
// there is a potential issue here, if the object comes after the truncated list // there is a potential issue here, if the object comes after the truncated list
@ -253,26 +256,14 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err
// will fix it when we bring in persistent json into Donut - TODO // will fix it when we bring in persistent json into Donut - TODO
objectList, _, _, err := dt.buckets[bucket].ListObjects("", "", "", 1000) objectList, _, _, err := dt.buckets[bucket].ListObjects("", "", "", 1000)
if err != nil { if err != nil {
return nil, iodine.New(err, errParams) return ObjectMetadata{}, iodine.New(err, errParams)
} }
for _, objectName := range objectList { for _, objectName := range objectList {
if objectName == object { if objectName == object {
objectMetadataMap := make(map[string]string) return dt.buckets[bucket].GetObjectMetadata(object)
objectMetadata, err := dt.buckets[bucket].GetObjectMetadata(object)
if err != nil {
return nil, iodine.New(err, nil)
}
objectMetadataMap["created"] = objectMetadata.Created.Format(time.RFC3339Nano)
objectMetadataMap["size"] = strconv.FormatInt(objectMetadata.Size, 10)
objectMetadataMap["md5"] = objectMetadata.MD5Sum
objectMetadataMap["version"] = objectMetadata.Version
for k, v := range objectMetadata.Metadata {
objectMetadataMap[k] = v
}
return objectMetadataMap, nil
} }
} }
return nil, iodine.New(ObjectNotFound{Object: object}, errParams) return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: object}, errParams)
} }
// getDiskWriters - // getDiskWriters -
@ -315,7 +306,7 @@ func (dt donut) getBucketMetadataReaders() ([]io.ReadCloser, error) {
} }
// //
func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) error { func (dt donut) setDonutBucketMetadata(metadata *AllBuckets) error {
writers, err := dt.getBucketMetadataWriters() writers, err := dt.getBucketMetadataWriters()
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
@ -332,8 +323,8 @@ func (dt donut) setDonutBucketMetadata(metadata map[string]map[string]string) er
return nil return nil
} }
func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) { func (dt donut) getDonutBucketMetadata() (*AllBuckets, error) {
metadata := make(map[string]map[string]string) metadata := new(AllBuckets)
readers, err := dt.getBucketMetadataReaders() readers, err := dt.getBucketMetadataReaders()
if err != nil { if err != nil {
return nil, iodine.New(err, nil) return nil, iodine.New(err, nil)
@ -343,7 +334,7 @@ func (dt donut) getDonutBucketMetadata() (map[string]map[string]string, error) {
} }
for _, reader := range readers { for _, reader := range readers {
jenc := json.NewDecoder(reader) jenc := json.NewDecoder(reader)
if err := jenc.Decode(&metadata); err != nil { if err := jenc.Decode(metadata); err != nil {
return nil, iodine.New(err, nil) return nil, iodine.New(err, nil)
} }
} }
@ -380,8 +371,9 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error {
metadata, err := dt.getDonutBucketMetadata() metadata, err := dt.getDonutBucketMetadata()
if err != nil { if err != nil {
if os.IsNotExist(iodine.ToError(err)) { if os.IsNotExist(iodine.ToError(err)) {
metadata := make(map[string]map[string]string) metadata := new(AllBuckets)
metadata[bucketName] = bucketMetadata metadata.Buckets = make(map[string]BucketMetadata)
metadata.Buckets[bucketName] = bucketMetadata
err = dt.setDonutBucketMetadata(metadata) err = dt.setDonutBucketMetadata(metadata)
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
@ -390,7 +382,7 @@ func (dt donut) makeDonutBucket(bucketName, acl string) error {
} }
return iodine.New(err, nil) return iodine.New(err, nil)
} }
metadata[bucketName] = bucketMetadata metadata.Buckets[bucketName] = bucketMetadata
err = dt.setDonutBucketMetadata(metadata) err = dt.setDonutBucketMetadata(metadata)
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)

17
pkg/storage/donut/donut_test.go
Unescape View File

@ -26,7 +26,6 @@ import (
"path/filepath" "path/filepath"
"strconv" "strconv"
"testing" "testing"
"time"
. "github.com/minio/check" . "github.com/minio/check"
) )
@ -113,7 +112,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) {
buckets, err := donut.ListBuckets() buckets, err := donut.ListBuckets()
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 1) c.Assert(len(buckets), Equals, 1)
c.Assert(buckets["foo"]["acl"], Equals, "private") c.Assert(buckets["foo"].ACL, Equals, "private")
} }
// test re-create bucket // test re-create bucket
@ -202,9 +201,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
objectMetadata, err := donut.GetObjectMetadata("foo", "obj") objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(objectMetadata["contentType"], Equals, metadata["contentType"]) c.Assert(objectMetadata.Metadata["contentType"], Equals, metadata["contentType"])
c.Assert(objectMetadata["foo"], Equals, metadata["foo"]) c.Assert(objectMetadata.Metadata["foo"], Equals, metadata["foo"])
c.Assert(objectMetadata["hello"], Equals, metadata["hello"]) c.Assert(objectMetadata.Metadata["hello"], Equals, metadata["hello"])
} }
// test create object fails without name // test create object fails without name
@ -255,11 +254,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
actualMetadata, err := donut.GetObjectMetadata("foo", "obj") actualMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(expectedMd5Sum, Equals, actualMetadata["md5"]) c.Assert(expectedMd5Sum, Equals, actualMetadata.MD5Sum)
c.Assert(strconv.Itoa(len(data)), Equals, actualMetadata["size"]) c.Assert(int64(len(data)), Equals, actualMetadata.Size)
c.Assert("1.0", Equals, actualMetadata["version"]) c.Assert("1.0.0", Equals, actualMetadata.Version)
_, err = time.Parse(time.RFC3339Nano, actualMetadata["created"])
c.Assert(err, IsNil)
} }
// test list objects // test list objects

6
pkg/storage/donut/interfaces.go
Unescape View File

@ -29,9 +29,9 @@ type Donut interface {
// ObjectStorage is a donut object storage interface // ObjectStorage is a donut object storage interface
type ObjectStorage interface { type ObjectStorage interface {
// Storage service operations // Storage service operations
GetBucketMetadata(bucket string) (map[string]string, error) GetBucketMetadata(bucket string) (BucketMetadata, error)
SetBucketMetadata(bucket string, metadata map[string]string) error SetBucketMetadata(bucket string, metadata map[string]string) error
ListBuckets() (map[string]map[string]string, error) ListBuckets() (map[string]BucketMetadata, error)
MakeBucket(bucket, acl string) error MakeBucket(bucket, acl string) error
// Bucket operations // Bucket operations
@ -39,7 +39,7 @@ type ObjectStorage interface {
// Object operations // Object operations
GetObject(bucket, object string) (io.ReadCloser, int64, error) GetObject(bucket, object string) (io.ReadCloser, int64, error)
GetObjectMetadata(bucket, object string) (map[string]string, error) GetObjectMetadata(bucket, object string) (ObjectMetadata, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error)
} }

28
pkg/storage/donut/object.go
Unescape View File

@ -41,26 +41,26 @@ func newObject(objectName, p string) (object, error) {
return o, nil return o, nil
} }
func (o object) GetObjectMetadata() (*objectMetadata, error) { func (o object) GetObjectMetadata() (ObjectMetadata, error) {
objectMetadata := new(objectMetadata) objMetadata := ObjectMetadata{}
objectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig)) objMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig))
if err != nil { if err != nil {
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil)
} }
if err := json.Unmarshal(objectMetadataBytes, objectMetadata); err != nil { if err := json.Unmarshal(objMetadataBytes, &objMetadata); err != nil {
return nil, iodine.New(err, nil) return ObjectMetadata{}, iodine.New(err, nil)
} }
return objectMetadata, nil return objMetadata, nil
} }
func (o object) GetDonutObjectMetadata() (*donutObjectMetadata, error) { func (o object) GetSystemObjectMetadata() (SystemObjectMetadata, error) {
donutObjectMetadata := new(donutObjectMetadata) sysObjMetadata := SystemObjectMetadata{}
donutObjectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, donutObjectMetadataConfig)) sysObjMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, sysObjectMetadataConfig))
if err != nil { if err != nil {
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) return SystemObjectMetadata{}, iodine.New(ObjectNotFound{Object: o.name}, nil)
} }
if err := json.Unmarshal(donutObjectMetadataBytes, donutObjectMetadata); err != nil { if err := json.Unmarshal(sysObjMetadataBytes, &sysObjMetadata); err != nil {
return nil, iodine.New(err, nil) return SystemObjectMetadata{}, iodine.New(err, nil)
} }
return donutObjectMetadata, nil return sysObjMetadata, nil
} }

47
pkg/storage/drivers/donut/donut.go
Unescape View File

@ -26,7 +26,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time"
"io/ioutil" "io/ioutil"
@ -141,13 +140,9 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error)
return nil, err return nil, err
} }
for name, metadata := range buckets { for name, metadata := range buckets {
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
if err != nil {
return nil, iodine.New(err, nil)
}
result := drivers.BucketMetadata{ result := drivers.BucketMetadata{
Name: name, Name: name,
Created: created, Created: metadata.Created,
} }
results = append(results, result) results = append(results, result)
} }
@ -195,18 +190,10 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
if err != nil { if err != nil {
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucketName}, nil) return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucketName}, nil)
} }
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
if err != nil {
return drivers.BucketMetadata{}, iodine.New(err, nil)
}
acl, ok := metadata["acl"]
if !ok {
return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{}, nil)
}
bucketMetadata := drivers.BucketMetadata{ bucketMetadata := drivers.BucketMetadata{
Name: bucketName, Name: bucketName,
Created: created, Created: metadata.Created,
ACL: drivers.BucketACL(acl), ACL: drivers.BucketACL(metadata.ACL),
} }
return bucketMetadata, nil return bucketMetadata, nil
} }
@ -332,22 +319,14 @@ func (d donutDriver) GetObjectMetadata(bucketName, objectName string) (drivers.O
Object: objectName, Object: objectName,
}, errParams) }, errParams)
} }
created, err := time.Parse(time.RFC3339Nano, metadata["created"])
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
}
size, err := strconv.ParseInt(metadata["size"], 10, 64)
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
}
objectMetadata := drivers.ObjectMetadata{ objectMetadata := drivers.ObjectMetadata{
Bucket: bucketName, Bucket: bucketName,
Key: objectName, Key: objectName,
ContentType: metadata["contentType"], ContentType: metadata.Metadata["contentType"],
Created: created, Created: metadata.Created,
Md5: metadata["md5"], Md5: metadata.MD5Sum,
Size: size, Size: metadata.Size,
} }
return objectMetadata, nil return objectMetadata, nil
} }
@ -390,18 +369,10 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
if err != nil { if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
} }
t, err := time.Parse(time.RFC3339Nano, objectMetadata["created"])
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
}
size, err := strconv.ParseInt(objectMetadata["size"], 10, 64)
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
}
metadata := drivers.ObjectMetadata{ metadata := drivers.ObjectMetadata{
Key: objectName, Key: objectName,
Created: t, Created: objectMetadata.Created,
Size: size, Size: objectMetadata.Size,
} }
results = append(results, metadata) results = append(results, metadata)
} }

Write Preview
Loading…
Cancel
Save