From c99d96dbc28e3ea2ed9f3f32d670146f36d4fd77 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 27 Apr 2015 02:04:05 -0700 Subject: [PATCH] Now donut supports bucket ACL's, bringing in this support for dl.minio.io --- pkg/api/api_bucket_handlers.go | 4 +- pkg/storage/donut/donut_bucket.go | 9 +- pkg/storage/donut/donut_disk.go | 2 +- pkg/storage/donut/objectstorage.go | 57 ++++++++--- pkg/storage/donut/objectstorage_internal.go | 104 +++++++++++++++++++- pkg/storage/drivers/api_testsuite.go | 2 +- pkg/storage/drivers/donut/donut.go | 5 +- pkg/storage/drivers/memory/memory.go | 4 + 8 files changed, 165 insertions(+), 22 deletions(-) diff --git a/pkg/api/api_bucket_handlers.go b/pkg/api/api_bucket_handlers.go index 3950101b9..bb9584d15 100644 --- a/pkg/api/api_bucket_handlers.go +++ b/pkg/api/api_bucket_handlers.go @@ -45,13 +45,13 @@ func (server *minioAPI) isValidOp(w http.ResponseWriter, req *http.Request, acce if stripAccessKey(req) == "" && bucketMetadata.ACL.IsPrivate() { return true - // Uncomment this when we have webCli + // Uncomment this before release // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) // return false } if bucketMetadata.ACL.IsPublicRead() && req.Method == "PUT" { return true - // Uncomment this when we have webCli + // Uncomment this before release // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) // return false } diff --git a/pkg/storage/donut/donut_bucket.go b/pkg/storage/donut/donut_bucket.go index 7733a6485..c61c8015a 100644 --- a/pkg/storage/donut/donut_bucket.go +++ b/pkg/storage/donut/donut_bucket.go @@ -43,15 +43,18 @@ type bucket struct { } // NewBucket - instantiate a new bucket -func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bucket, error) { +func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bucket, map[string]string, error) { errParams := map[string]string{ "bucketName": bucketName, "donutName": donutName, "aclType": aclType, } if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" { - return nil, iodine.New(errors.New("invalid argument"), errParams) + return nil, nil, iodine.New(errors.New("invalid argument"), errParams) } + bucketMetadata := make(map[string]string) + bucketMetadata["acl"] = aclType + bucketMetadata["created"] = time.Now().Format(time.RFC3339Nano) b := bucket{} b.name = bucketName b.acl = aclType @@ -59,7 +62,7 @@ func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bu b.donutName = donutName b.objects = make(map[string]Object) b.nodes = nodes - return b, nil + return b, bucketMetadata, nil } // ListObjects - list all objects diff --git a/pkg/storage/donut/donut_disk.go b/pkg/storage/donut/donut_disk.go index 447910b28..53c47d3b3 100644 --- a/pkg/storage/donut/donut_disk.go +++ b/pkg/storage/donut/donut_disk.go @@ -133,7 +133,7 @@ func (d disk) MakeFile(filename string) (*os.File, error) { if err := os.MkdirAll(path.Dir(filePath), 0700); err != nil { return nil, iodine.New(err, nil) } - dataFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + dataFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return nil, iodine.New(err, nil) } diff --git a/pkg/storage/donut/objectstorage.go b/pkg/storage/donut/objectstorage.go index b32512d87..d42e5731d 100644 --- a/pkg/storage/donut/objectstorage.go +++ b/pkg/storage/donut/objectstorage.go @@ -19,10 +19,10 @@ package donut import ( "errors" "io" + "os" "sort" "strconv" "strings" - "time" "github.com/minio-io/minio/pkg/iodine" ) @@ -44,17 +44,25 @@ func (d donut) GetBucketMetadata(bucket string) (map[string]string, error) { if _, ok := d.buckets[bucket]; !ok { return nil, iodine.New(errors.New("bucket does not exist"), nil) } - // TODO get this, from whatever is written from SetBucketMetadata - metadata := make(map[string]string) - metadata["name"] = bucket - metadata["created"] = time.Now().Format(time.RFC3339Nano) - metadata["acl"] = "private" - return metadata, nil + metadata, err := d.getDonutBucketMetadata() + if err != nil { + return nil, iodine.New(err, nil) + } + return metadata[bucket], nil } // SetBucketMetadata - set bucket metadata -func (d donut) SetBucketMetadata(bucket string, metadata map[string]string) error { - return errors.New("Not implemented") +func (d donut) SetBucketMetadata(bucket string, bucketMetadata map[string]string) error { + err := d.getDonutBuckets() + if err != nil { + return iodine.New(err, nil) + } + metadata, err := d.getDonutBucketMetadata() + if err != nil { + return iodine.New(err, nil) + } + metadata[bucket] = bucketMetadata + return d.setDonutBucketMetadata(metadata) } // ListBuckets - return list of buckets @@ -63,7 +71,16 @@ func (d donut) ListBuckets() (results []string, err error) { if err != nil { return nil, iodine.New(err, nil) } - for name := range d.buckets { + metadata, err := d.getDonutBucketMetadata() + if err != nil { + err = iodine.ToError(err) + if os.IsNotExist(err) { + // valid case + return nil, nil + } + return nil, iodine.New(err, nil) + } + for name := range metadata { results = append(results, name) } sort.Strings(results) @@ -151,6 +168,15 @@ func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCl if _, ok := d.buckets[bucket]; !ok { return iodine.New(errors.New("bucket does not exist"), nil) } + objectList, err := d.buckets[bucket].ListObjects() + if err != nil { + return iodine.New(err, nil) + } + for objectName := range objectList { + if objectName == object { + return iodine.New(errors.New("object exists"), nil) + } + } err = d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata) if err != nil { return iodine.New(err, errParams) @@ -177,7 +203,16 @@ func (d donut) GetObject(bucket, object string) (reader io.ReadCloser, size int6 if _, ok := d.buckets[bucket]; !ok { return nil, 0, iodine.New(errors.New("bucket does not exist"), errParams) } - return d.buckets[bucket].GetObject(object) + objectList, err := d.buckets[bucket].ListObjects() + if err != nil { + return nil, 0, iodine.New(err, nil) + } + for objectName := range objectList { + if objectName == object { + return d.buckets[bucket].GetObject(object) + } + } + return nil, 0, iodine.New(errors.New("object not found"), nil) } // GetObjectMetadata - get object metadata diff --git a/pkg/storage/donut/objectstorage_internal.go b/pkg/storage/donut/objectstorage_internal.go index acb0df821..9511dba9e 100644 --- a/pkg/storage/donut/objectstorage_internal.go +++ b/pkg/storage/donut/objectstorage_internal.go @@ -17,15 +17,94 @@ package donut import ( + "encoding/json" "errors" "fmt" + "io" + "os" "path" "strings" "github.com/minio-io/minio/pkg/iodine" ) -// TODO we have to store the acl's +/// This file contains all the internal functions used by Object interface + +// getDiskWriters - +func (d donut) getBucketMetadataWriters() ([]io.WriteCloser, error) { + var writers []io.WriteCloser + for _, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, iodine.New(err, nil) + } + writers = make([]io.WriteCloser, len(disks)) + for _, disk := range disks { + bucketMetaDataWriter, err := disk.MakeFile(path.Join(d.name, bucketMetadataConfig)) + if err != nil { + return nil, iodine.New(err, nil) + } + writers[disk.GetOrder()] = bucketMetaDataWriter + } + } + return writers, nil +} + +func (d donut) getBucketMetadataReaders() ([]io.ReadCloser, error) { + var readers []io.ReadCloser + for _, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, iodine.New(err, nil) + } + readers = make([]io.ReadCloser, len(disks)) + for _, disk := range disks { + bucketMetaDataReader, err := disk.OpenFile(path.Join(d.name, bucketMetadataConfig)) + if err != nil { + return nil, iodine.New(err, nil) + } + readers[disk.GetOrder()] = bucketMetaDataReader + } + } + return readers, nil +} + +// +func (d donut) setDonutBucketMetadata(metadata map[string]map[string]string) error { + writers, err := d.getBucketMetadataWriters() + if err != nil { + return iodine.New(err, nil) + } + for _, writer := range writers { + defer writer.Close() + } + for _, writer := range writers { + jenc := json.NewEncoder(writer) + if err := jenc.Encode(metadata); err != nil { + return iodine.New(err, nil) + } + } + return nil +} + +func (d donut) getDonutBucketMetadata() (map[string]map[string]string, error) { + metadata := make(map[string]map[string]string) + readers, err := d.getBucketMetadataReaders() + if err != nil { + return nil, iodine.New(err, nil) + } + for _, reader := range readers { + defer reader.Close() + } + for _, reader := range readers { + jenc := json.NewDecoder(reader) + if err := jenc.Decode(&metadata); err != nil { + return nil, iodine.New(err, nil) + } + } + return metadata, nil +} + func (d donut) makeDonutBucket(bucketName, acl string) error { err := d.getDonutBuckets() if err != nil { @@ -34,7 +113,7 @@ func (d donut) makeDonutBucket(bucketName, acl string) error { if _, ok := d.buckets[bucketName]; ok { return iodine.New(errors.New("bucket exists"), nil) } - bucket, err := NewBucket(bucketName, acl, d.name, d.nodes) + bucket, bucketMetadata, err := NewBucket(bucketName, acl, d.name, d.nodes) if err != nil { return iodine.New(err, nil) } @@ -54,6 +133,25 @@ func (d donut) makeDonutBucket(bucketName, acl string) error { } nodeNumber = nodeNumber + 1 } + metadata, err := d.getDonutBucketMetadata() + if err != nil { + err = iodine.ToError(err) + if os.IsNotExist(err) { + metadata := make(map[string]map[string]string) + metadata[bucketName] = bucketMetadata + err = d.setDonutBucketMetadata(metadata) + if err != nil { + return iodine.New(err, nil) + } + return nil + } + return iodine.New(err, nil) + } + metadata[bucketName] = bucketMetadata + err = d.setDonutBucketMetadata(metadata) + if err != nil { + return iodine.New(err, nil) + } return nil } @@ -75,7 +173,7 @@ func (d donut) getDonutBuckets() error { } bucketName := splitDir[0] // we dont need this NewBucket once we cache from makeDonutBucket() - bucket, err := NewBucket(bucketName, "private", d.name, d.nodes) + bucket, _, err := NewBucket(bucketName, "private", d.name, d.nodes) if err != nil { return iodine.New(err, nil) } diff --git a/pkg/storage/drivers/api_testsuite.go b/pkg/storage/drivers/api_testsuite.go index 5206af404..f9d8d1931 100644 --- a/pkg/storage/drivers/api_testsuite.go +++ b/pkg/storage/drivers/api_testsuite.go @@ -235,7 +235,7 @@ func testBucketMetadata(c *check.C, create func() Driver) { metadata, err := drivers.GetBucketMetadata("string") c.Assert(err, check.IsNil) - c.Assert(metadata.Name, check.Equals, "string") + c.Assert(metadata.ACL, check.Equals, BucketACL("private")) } func testBucketRecreateFails(c *check.C, create func() Driver) { diff --git a/pkg/storage/drivers/donut/donut.go b/pkg/storage/drivers/donut/donut.go index 147d7bfbe..6d2b5bef5 100644 --- a/pkg/storage/drivers/donut/donut.go +++ b/pkg/storage/drivers/donut/donut.go @@ -150,6 +150,9 @@ func (d donutDriver) CreateBucket(bucketName, acl string) error { return iodine.New(drivers.InvalidACL{ACL: acl}, nil) } if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") { + if strings.TrimSpace(acl) == "" { + acl = "private" + } return d.donut.MakeBucket(bucketName, acl) } return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil) @@ -173,7 +176,7 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{}, nil) } bucketMetadata := drivers.BucketMetadata{ - Name: metadata["name"], + Name: bucketName, Created: created, ACL: drivers.BucketACL(acl), } diff --git a/pkg/storage/drivers/memory/memory.go b/pkg/storage/drivers/memory/memory.go index a7c2ca99b..458ae9990 100644 --- a/pkg/storage/drivers/memory/memory.go +++ b/pkg/storage/drivers/memory/memory.go @@ -285,6 +285,10 @@ func (memory *memoryDriver) CreateBucket(bucketName, acl string) error { } memory.lock.RUnlock() + if strings.TrimSpace(acl) == "" { + // default is private + acl = "private" + } var newBucket = storedBucket{} newBucket.metadata = drivers.BucketMetadata{} newBucket.metadata.Name = bucketName