From 1c0ff2c7581fcfeb5329c7c86da66e9ef553c16c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Apr 2015 18:19:53 -0700 Subject: [PATCH] ACL driver/storage layer support --- pkg/api/api_generic_handlers.go | 4 ++ pkg/storage/donut/donut.go | 6 ++- pkg/storage/donut/donut_bucket.go | 7 ++- pkg/storage/donut/donut_public_interfaces.go | 2 +- pkg/storage/donut/donut_test.go | 24 +++++----- pkg/storage/donut/objectstorage.go | 5 +- pkg/storage/donut/objectstorage_internal.go | 7 +-- pkg/storage/drivers/api_testsuite.go | 36 +++++++------- pkg/storage/drivers/donut/donut.go | 16 +++++-- pkg/storage/drivers/driver.go | 49 +++++++++++++++++++- pkg/storage/drivers/errors.go | 11 +++++ pkg/storage/drivers/memory/memory.go | 12 +++-- pkg/storage/drivers/mocks/Driver.go | 4 +- 13 files changed, 133 insertions(+), 50 deletions(-) diff --git a/pkg/api/api_generic_handlers.go b/pkg/api/api_generic_handlers.go index a4c5f3d23..f26a313e6 100644 --- a/pkg/api/api_generic_handlers.go +++ b/pkg/api/api_generic_handlers.go @@ -79,6 +79,10 @@ func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path) return } + if r.Method == "PUT" && bucketMetadata.ACL.IsPublicRead() { + writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path) + return + } } switch true { diff --git a/pkg/storage/donut/donut.go b/pkg/storage/donut/donut.go index e9888516f..165629d60 100644 --- a/pkg/storage/donut/donut.go +++ b/pkg/storage/donut/donut.go @@ -31,9 +31,13 @@ type donut struct { // config files used inside Donut const ( + // donut object metadata and config donutObjectMetadataConfig = "donutObjectMetadata.json" - objectMetadataConfig = "objectMetadata.json" donutConfig = "donutMetadata.json" + + // bucket, object metadata + bucketMetadataConfig = "bucketMetadata.json" + objectMetadataConfig = "objectMetadata.json" ) // attachDonutNode - wrapper function to instantiate a new node for associated donut diff --git a/pkg/storage/donut/donut_bucket.go b/pkg/storage/donut/donut_bucket.go index 5566cc83d..7733a6485 100644 --- a/pkg/storage/donut/donut_bucket.go +++ b/pkg/storage/donut/donut_bucket.go @@ -35,22 +35,27 @@ import ( // internal struct carrying bucket specific information type bucket struct { name string + acl string + time time.Time donutName string nodes map[string]Node objects map[string]Object } // NewBucket - instantiate a new bucket -func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) { +func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bucket, error) { errParams := map[string]string{ "bucketName": bucketName, "donutName": donutName, + "aclType": aclType, } if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" { return nil, iodine.New(errors.New("invalid argument"), errParams) } b := bucket{} b.name = bucketName + b.acl = aclType + b.time = time.Now() b.donutName = donutName b.objects = make(map[string]Object) b.nodes = nodes diff --git a/pkg/storage/donut/donut_public_interfaces.go b/pkg/storage/donut/donut_public_interfaces.go index e2a2ff7a3..0aecef3b8 100644 --- a/pkg/storage/donut/donut_public_interfaces.go +++ b/pkg/storage/donut/donut_public_interfaces.go @@ -32,7 +32,7 @@ type ObjectStorage interface { GetBucketMetadata(bucket string) (map[string]string, error) SetBucketMetadata(bucket string, metadata map[string]string) error ListBuckets() ([]string, error) - MakeBucket(bucket string) error + MakeBucket(bucket, acl string) error // Bucket Operations ListObjects(bucket, prefix, marker, delim string, maxKeys int) (result []string, prefixes []string, isTruncated bool, err error) diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go index 8c7195d48..3fb5f2e0d 100644 --- a/pkg/storage/donut/donut_test.go +++ b/pkg/storage/donut/donut_test.go @@ -75,10 +75,10 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) // fail to create new bucket without a name - err = donut.MakeBucket("") + err = donut.MakeBucket("", "private") c.Assert(err, Not(IsNil)) - err = donut.MakeBucket(" ") + err = donut.MakeBucket(" ", "private") c.Assert(err, Not(IsNil)) } @@ -90,7 +90,7 @@ func (s *MySuite) TestEmptyBucket(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) - c.Assert(donut.MakeBucket("foo"), IsNil) + c.Assert(donut.MakeBucket("foo", "private"), IsNil) // check if bucket is empty objects, _, istruncated, err := donut.ListObjects("foo", "", "", "", 1) c.Assert(err, IsNil) @@ -106,7 +106,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) // create bucket - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, IsNil) // check bucket exists @@ -122,10 +122,10 @@ func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) { defer os.RemoveAll(root) donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, IsNil) - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, Not(IsNil)) } @@ -137,17 +137,17 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) // add a second bucket - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, IsNil) - err = donut.MakeBucket("bar") + err = donut.MakeBucket("bar", "private") c.Assert(err, IsNil) buckets, err := donut.ListBuckets() c.Assert(err, IsNil) c.Assert(buckets, DeepEquals, []string{"bar", "foo"}) - err = donut.MakeBucket("foobar") + err = donut.MakeBucket("foobar", "private") c.Assert(err, IsNil) buckets, err = donut.ListBuckets() @@ -185,7 +185,7 @@ func (s *MySuite) TestNewObjectMetadata(c *C) { expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil)) reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, IsNil) err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata) @@ -222,7 +222,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) - err = donut.MakeBucket("foo") + err = donut.MakeBucket("foo", "private") c.Assert(err, IsNil) metadata := make(map[string]string) @@ -263,7 +263,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { donut, err := NewDonut("test", createTestNodeDiskMap(root)) c.Assert(err, IsNil) - c.Assert(donut.MakeBucket("foo"), IsNil) + c.Assert(donut.MakeBucket("foo", "private"), IsNil) one := ioutil.NopCloser(bytes.NewReader([]byte("one"))) err = donut.PutObject("foo", "obj1", "", one, nil) diff --git a/pkg/storage/donut/objectstorage.go b/pkg/storage/donut/objectstorage.go index 58ef79684..2fd37efed 100644 --- a/pkg/storage/donut/objectstorage.go +++ b/pkg/storage/donut/objectstorage.go @@ -28,11 +28,11 @@ import ( ) // MakeBucket - make a new bucket -func (d donut) MakeBucket(bucket string) error { +func (d donut) MakeBucket(bucket, acl string) error { if bucket == "" || strings.TrimSpace(bucket) == "" { return iodine.New(errors.New("invalid argument"), nil) } - return d.makeDonutBucket(bucket) + return d.makeDonutBucket(bucket, acl) } // GetBucketMetadata - get bucket metadata @@ -47,6 +47,7 @@ func (d donut) GetBucketMetadata(bucket string) (map[string]string, error) { metadata := make(map[string]string) metadata["name"] = bucket metadata["created"] = time.Now().Format(time.RFC3339Nano) // TODO get this, from whatever is written from SetBucketMetadata + metadata["acl"] = "private" return metadata, nil } diff --git a/pkg/storage/donut/objectstorage_internal.go b/pkg/storage/donut/objectstorage_internal.go index d679df606..acb0df821 100644 --- a/pkg/storage/donut/objectstorage_internal.go +++ b/pkg/storage/donut/objectstorage_internal.go @@ -25,7 +25,8 @@ import ( "github.com/minio-io/minio/pkg/iodine" ) -func (d donut) makeDonutBucket(bucketName string) error { +// TODO we have to store the acl's +func (d donut) makeDonutBucket(bucketName, acl string) error { err := d.getDonutBuckets() if err != nil { return iodine.New(err, nil) @@ -33,7 +34,7 @@ func (d donut) makeDonutBucket(bucketName string) error { if _, ok := d.buckets[bucketName]; ok { return iodine.New(errors.New("bucket exists"), nil) } - bucket, err := NewBucket(bucketName, d.name, d.nodes) + bucket, err := NewBucket(bucketName, acl, d.name, d.nodes) if err != nil { return iodine.New(err, nil) } @@ -74,7 +75,7 @@ func (d donut) getDonutBuckets() error { } bucketName := splitDir[0] // we dont need this NewBucket once we cache from makeDonutBucket() - bucket, err := NewBucket(bucketName, d.name, d.nodes) + bucket, err := NewBucket(bucketName, "private", d.name, d.nodes) if err != nil { return iodine.New(err, nil) } diff --git a/pkg/storage/drivers/api_testsuite.go b/pkg/storage/drivers/api_testsuite.go index aa561a76b..157a29f90 100644 --- a/pkg/storage/drivers/api_testsuite.go +++ b/pkg/storage/drivers/api_testsuite.go @@ -48,14 +48,14 @@ func APITestSuite(c *check.C, create func() Driver) { func testCreateBucket(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) } func testMultipleObjectCreation(c *check.C, create func() Driver) { objects := make(map[string][]byte) drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) for i := 0; i < 10; i++ { randomPerm := rand.Perm(10) @@ -94,7 +94,7 @@ func testMultipleObjectCreation(c *check.C, create func() Driver) { func testPaging(c *check.C, create func() Driver) { drivers := create() - drivers.CreateBucket("bucket") + drivers.CreateBucket("bucket", "") resources := BucketResourcesMetadata{} objects, resources, err := drivers.ListObjects("bucket", resources) c.Assert(err, check.IsNil) @@ -198,7 +198,7 @@ func testPaging(c *check.C, create func() Driver) { func testObjectOverwriteFails(c *check.C, create func() Driver) { drivers := create() - drivers.CreateBucket("bucket") + drivers.CreateBucket("bucket", "") hasher1 := md5.New() hasher1.Write([]byte("one")) @@ -227,7 +227,7 @@ func testNonExistantBucketOperations(c *check.C, create func() Driver) { func testBucketMetadata(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("string") + err := drivers.CreateBucket("string", "") c.Assert(err, check.IsNil) metadata, err := drivers.GetBucketMetadata("string") @@ -237,15 +237,15 @@ func testBucketMetadata(c *check.C, create func() Driver) { func testBucketRecreateFails(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("string") + err := drivers.CreateBucket("string", "") c.Assert(err, check.IsNil) - err = drivers.CreateBucket("string") + err = drivers.CreateBucket("string", "") c.Assert(err, check.Not(check.IsNil)) } func testPutObjectInSubdir(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) hasher := md5.New() @@ -270,7 +270,7 @@ func testListBuckets(c *check.C, create func() Driver) { c.Assert(len(buckets), check.Equals, 0) // add one and test exists - err = drivers.CreateBucket("bucket1") + err = drivers.CreateBucket("bucket1", "") c.Assert(err, check.IsNil) buckets, err = drivers.ListBuckets() @@ -278,7 +278,7 @@ func testListBuckets(c *check.C, create func() Driver) { c.Assert(err, check.IsNil) // add two and test exists - err = drivers.CreateBucket("bucket2") + err = drivers.CreateBucket("bucket2", "") c.Assert(err, check.IsNil) buckets, err = drivers.ListBuckets() @@ -286,7 +286,7 @@ func testListBuckets(c *check.C, create func() Driver) { c.Assert(err, check.IsNil) // add three and test exists + prefix - err = drivers.CreateBucket("bucket22") + err = drivers.CreateBucket("bucket22", "") buckets, err = drivers.ListBuckets() c.Assert(len(buckets), check.Equals, 3) @@ -299,8 +299,8 @@ func testListBucketsOrder(c *check.C, create func() Driver) { for i := 0; i < 10; i++ { drivers := create() // add one and test exists - drivers.CreateBucket("bucket1") - drivers.CreateBucket("bucket2") + drivers.CreateBucket("bucket1", "") + drivers.CreateBucket("bucket2", "") buckets, err := drivers.ListBuckets() c.Assert(len(buckets), check.Equals, 2) @@ -321,7 +321,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver) func testNonExistantObjectInBucket(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) var byteBuffer bytes.Buffer @@ -343,7 +343,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Driver) { func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world")) @@ -386,7 +386,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) { func testDefaultContentType(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) // test empty @@ -408,10 +408,9 @@ func testDefaultContentType(c *check.C, create func() Driver) { c.Assert(metadata.ContentType, check.Equals, "application/json") } -/* func testContentMd5Set(c *check.C, create func() Driver) { drivers := create() - err := drivers.CreateBucket("bucket") + err := drivers.CreateBucket("bucket", "") c.Assert(err, check.IsNil) // test md5 invalid @@ -420,4 +419,3 @@ func testContentMd5Set(c *check.C, create func() Driver) { err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one")) c.Assert(err, check.IsNil) } -*/ diff --git a/pkg/storage/drivers/donut/donut.go b/pkg/storage/drivers/donut/donut.go index e2ce2f223..8227f91e8 100644 --- a/pkg/storage/drivers/donut/donut.go +++ b/pkg/storage/drivers/donut/donut.go @@ -39,6 +39,7 @@ import ( // donutDriver - creates a new single disk drivers driver using donut type donutDriver struct { donut donut.Donut + path string } const ( @@ -82,6 +83,7 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) { s := new(donutDriver) s.donut = donut + s.path = path go start(ctrlChannel, errorChannel, s) return ctrlChannel, errorChannel, s @@ -117,11 +119,14 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) } // CreateBucket creates a new bucket -func (d donutDriver) CreateBucket(bucketName string) error { +func (d donutDriver) CreateBucket(bucketName, acl string) error { + if !drivers.IsValidBucketACL(acl) { + return iodine.New(drivers.InvalidACL{ACL: acl}, nil) + } if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") { - return d.donut.MakeBucket(bucketName) + return d.donut.MakeBucket(bucketName, acl) } - return iodine.New(errors.New("Invalid bucket"), map[string]string{"bucket": bucketName}) + return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil) } // GetBucketMetadata retrieves an bucket's metadata @@ -137,9 +142,14 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat if err != nil { return drivers.BucketMetadata{}, iodine.New(err, nil) } + acl, ok := metadata["acl"] + if !ok { + return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{Path: d.path}, nil) + } bucketMetadata := drivers.BucketMetadata{ Name: metadata["name"], Created: created, + ACL: drivers.BucketACL(acl), } return bucketMetadata, nil } diff --git a/pkg/storage/drivers/driver.go b/pkg/storage/drivers/driver.go index e6eb3571e..40792daaa 100644 --- a/pkg/storage/drivers/driver.go +++ b/pkg/storage/drivers/driver.go @@ -27,7 +27,7 @@ import ( type Driver interface { // Bucket Operations ListBuckets() ([]BucketMetadata, error) - CreateBucket(bucket string) error + CreateBucket(bucket, acl string) error GetBucketMetadata(bucket string) (BucketMetadata, error) // Object Operations @@ -38,10 +38,40 @@ type Driver interface { CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error } +// BucketACL - bucket level access control +type BucketACL string + +// different types of ACL's currently supported for buckets +const ( + BucketPrivate = BucketACL("private") + BucketPublicRead = BucketACL("public-read") + BucketPublicReadWrite = BucketACL("public-read-write") +) + +func (b BucketACL) String() string { + return string(b) +} + +// IsPrivate - is acl Private +func (b BucketACL) IsPrivate() bool { + return b == BucketACL("private") +} + +// IsPublicRead - is acl PublicRead +func (b BucketACL) IsPublicRead() bool { + return b == BucketACL("public-read") +} + +// IsPublicReadWrite - is acl PublicReadWrite +func (b BucketACL) IsPublicReadWrite() bool { + return b == BucketACL("public-read-write") +} + // BucketMetadata - name and create date type BucketMetadata struct { Name string Created time.Time + ACL BucketACL } // ObjectMetadata - object key and its relevant metadata @@ -98,6 +128,23 @@ func GetMode(resources BucketResourcesMetadata) FilterMode { return f } +// IsValidBucketACL - is provided acl string supported +func IsValidBucketACL(acl string) bool { + switch acl { + case "private": + fallthrough + case "public-read": + fallthrough + case "public-read-write": + return true + case "": + // by default its "private" + return true + default: + return false + } +} + // IsDelimiterPrefixSet Delimiter and Prefix set func (b BucketResourcesMetadata) IsDelimiterPrefixSet() bool { return b.Mode == DelimiterPrefixMode diff --git a/pkg/storage/drivers/errors.go b/pkg/storage/drivers/errors.go index fa0a51031..bcd3a2cd3 100644 --- a/pkg/storage/drivers/errors.go +++ b/pkg/storage/drivers/errors.go @@ -54,6 +54,17 @@ type DigestError struct { Md5 string } +/// ACL related errors + +// InvalidACL - acl invalid +type InvalidACL struct { + ACL string +} + +func (e InvalidACL) Error() string { + return "Requested ACL is " + e.ACL + " invalid" +} + /// Bucket related errors // BucketNameInvalid - bucketname provided is invalid diff --git a/pkg/storage/drivers/memory/memory.go b/pkg/storage/drivers/memory/memory.go index 9545521e3..62dcb365f 100644 --- a/pkg/storage/drivers/memory/memory.go +++ b/pkg/storage/drivers/memory/memory.go @@ -188,13 +188,16 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, md5sum string } // CreateBucket - create bucket in memory -func (memory *memoryDriver) CreateBucket(bucketName string) error { +func (memory *memoryDriver) CreateBucket(bucketName, acl string) error { memory.lock.RLock() if !drivers.IsValidBucket(bucketName) { memory.lock.RUnlock() return drivers.BucketNameInvalid{Bucket: bucketName} } - + if !drivers.IsValidBucketACL(acl) { + memory.lock.RUnlock() + return drivers.InvalidACL{ACL: acl} + } if _, ok := memory.bucketMetadata[bucketName]; ok == true { memory.lock.RUnlock() return drivers.BucketExists{Bucket: bucketName} @@ -205,6 +208,7 @@ func (memory *memoryDriver) CreateBucket(bucketName string) error { newBucket.metadata = drivers.BucketMetadata{} newBucket.metadata.Name = bucketName newBucket.metadata.Created = time.Now() + newBucket.metadata.ACL = drivers.BucketACL(acl) memory.lock.Lock() defer memory.lock.Unlock() memory.bucketMetadata[bucketName] = newBucket @@ -234,7 +238,7 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedN switch true { case key == resources.Prefix: keys = appendUniq(keys, key) - // DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow + // DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow case key == resources.Prefix+delimitedName: keys = appendUniq(keys, key) case delimitedName != "": @@ -339,8 +343,6 @@ func (memory *memoryDriver) evictObject(key lru.Key, value interface{}) { k := key.(string) memory.totalSize = memory.totalSize - memory.objectMetadata[k].metadata.Size - log.Println("evicting:", k) - delete(memory.objectMetadata, k) } diff --git a/pkg/storage/drivers/mocks/Driver.go b/pkg/storage/drivers/mocks/Driver.go index 5995328fd..e94472951 100644 --- a/pkg/storage/drivers/mocks/Driver.go +++ b/pkg/storage/drivers/mocks/Driver.go @@ -27,8 +27,8 @@ func (m *Driver) ListBuckets() ([]drivers.BucketMetadata, error) { } // CreateBucket is a mock -func (m *Driver) CreateBucket(bucket string) error { - ret := m.Called(bucket) +func (m *Driver) CreateBucket(bucket, acl string) error { + ret := m.Called(bucket, acl) r0 := ret.Error(0)