oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ACL driver/storage layer support

Browse Source
master
Harshavardhana 9 years ago
parent c8713fd650
commit 1c0ff2c758
13 changed files with 133 additions and 50 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      pkg/api/api_generic_handlers.go
  2. 6
      pkg/storage/donut/donut.go
  3. 7
      pkg/storage/donut/donut_bucket.go
  4. 2
      pkg/storage/donut/donut_public_interfaces.go
  5. 24
      pkg/storage/donut/donut_test.go
  6. 5
      pkg/storage/donut/objectstorage.go
  7. 7
      pkg/storage/donut/objectstorage_internal.go
  8. 36
      pkg/storage/drivers/api_testsuite.go
  9. 16
      pkg/storage/drivers/donut/donut.go
  10. 49
      pkg/storage/drivers/driver.go
  11. 11
      pkg/storage/drivers/errors.go
  12. 12
      pkg/storage/drivers/memory/memory.go
  13. 4
      pkg/storage/drivers/mocks/Driver.go

4
pkg/api/api_generic_handlers.go
Unescape View File

@ -79,6 +79,10 @@ func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path)
return
}
if r.Method == "PUT" && bucketMetadata.ACL.IsPublicRead() {
writeErrorResponse(w, r, AccessDenied, acceptsContentType, r.URL.Path)
return
}
}
switch true {

6
pkg/storage/donut/donut.go
Unescape View File

@ -31,9 +31,13 @@ type donut struct {
// config files used inside Donut
const (
// donut object metadata and config
donutObjectMetadataConfig = "donutObjectMetadata.json"
objectMetadataConfig = "objectMetadata.json"
donutConfig = "donutMetadata.json"
// bucket, object metadata
bucketMetadataConfig = "bucketMetadata.json"
objectMetadataConfig = "objectMetadata.json"
)
// attachDonutNode - wrapper function to instantiate a new node for associated donut

7
pkg/storage/donut/donut_bucket.go
Unescape View File

@ -35,22 +35,27 @@ import (
// internal struct carrying bucket specific information
type bucket struct {
name string
acl string
time time.Time
donutName string
nodes map[string]Node
objects map[string]Object
}
// NewBucket - instantiate a new bucket
func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) {
func NewBucket(bucketName, aclType, donutName string, nodes map[string]Node) (Bucket, error) {
errParams := map[string]string{
"bucketName": bucketName,
"donutName": donutName,
"aclType": aclType,
}
if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(donutName) == "" {
return nil, iodine.New(errors.New("invalid argument"), errParams)
}
b := bucket{}
b.name = bucketName
b.acl = aclType
b.time = time.Now()
b.donutName = donutName
b.objects = make(map[string]Object)
b.nodes = nodes

2
pkg/storage/donut/donut_public_interfaces.go
Unescape View File

@ -32,7 +32,7 @@ type ObjectStorage interface {
GetBucketMetadata(bucket string) (map[string]string, error)
SetBucketMetadata(bucket string, metadata map[string]string) error
ListBuckets() ([]string, error)
MakeBucket(bucket string) error
MakeBucket(bucket, acl string) error
// Bucket Operations
ListObjects(bucket, prefix, marker, delim string, maxKeys int) (result []string, prefixes []string, isTruncated bool, err error)

24
pkg/storage/donut/donut_test.go
Unescape View File

@ -75,10 +75,10 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
// fail to create new bucket without a name
err = donut.MakeBucket("")
err = donut.MakeBucket("", "private")
c.Assert(err, Not(IsNil))
err = donut.MakeBucket(" ")
err = donut.MakeBucket(" ", "private")
c.Assert(err, Not(IsNil))
}
@ -90,7 +90,7 @@ func (s *MySuite) TestEmptyBucket(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
c.Assert(donut.MakeBucket("foo"), IsNil)
c.Assert(donut.MakeBucket("foo", "private"), IsNil)
// check if bucket is empty
objects, _, istruncated, err := donut.ListObjects("foo", "", "", "", 1)
c.Assert(err, IsNil)
@ -106,7 +106,7 @@ func (s *MySuite) TestMakeBucketAndList(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
// create bucket
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
// check bucket exists
@ -122,10 +122,10 @@ func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) {
defer os.RemoveAll(root)
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, Not(IsNil))
}
@ -137,17 +137,17 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
// add a second bucket
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
err = donut.MakeBucket("bar")
err = donut.MakeBucket("bar", "private")
c.Assert(err, IsNil)
buckets, err := donut.ListBuckets()
c.Assert(err, IsNil)
c.Assert(buckets, DeepEquals, []string{"bar", "foo"})
err = donut.MakeBucket("foobar")
err = donut.MakeBucket("foobar", "private")
c.Assert(err, IsNil)
buckets, err = donut.ListBuckets()
@ -185,7 +185,7 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil))
reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
@ -222,7 +222,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.MakeBucket("foo")
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
metadata := make(map[string]string)
@ -263,7 +263,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
c.Assert(donut.MakeBucket("foo"), IsNil)
c.Assert(donut.MakeBucket("foo", "private"), IsNil)
one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
err = donut.PutObject("foo", "obj1", "", one, nil)

5
pkg/storage/donut/objectstorage.go
Unescape View File

@ -28,11 +28,11 @@ import (
)
// MakeBucket - make a new bucket
func (d donut) MakeBucket(bucket string) error {
func (d donut) MakeBucket(bucket, acl string) error {
if bucket == "" || strings.TrimSpace(bucket) == "" {
return iodine.New(errors.New("invalid argument"), nil)
}
return d.makeDonutBucket(bucket)
return d.makeDonutBucket(bucket, acl)
}
// GetBucketMetadata - get bucket metadata
@ -47,6 +47,7 @@ func (d donut) GetBucketMetadata(bucket string) (map[string]string, error) {
metadata := make(map[string]string)
metadata["name"] = bucket
metadata["created"] = time.Now().Format(time.RFC3339Nano) // TODO get this, from whatever is written from SetBucketMetadata
metadata["acl"] = "private"
return metadata, nil
}

7
pkg/storage/donut/objectstorage_internal.go
Unescape View File

@ -25,7 +25,8 @@ import (
"github.com/minio-io/minio/pkg/iodine"
)
func (d donut) makeDonutBucket(bucketName string) error {
// TODO we have to store the acl's
func (d donut) makeDonutBucket(bucketName, acl string) error {
err := d.getDonutBuckets()
if err != nil {
return iodine.New(err, nil)
@ -33,7 +34,7 @@ func (d donut) makeDonutBucket(bucketName string) error {
if _, ok := d.buckets[bucketName]; ok {
return iodine.New(errors.New("bucket exists"), nil)
}
bucket, err := NewBucket(bucketName, d.name, d.nodes)
bucket, err := NewBucket(bucketName, acl, d.name, d.nodes)
if err != nil {
return iodine.New(err, nil)
}
@ -74,7 +75,7 @@ func (d donut) getDonutBuckets() error {
}
bucketName := splitDir[0]
// we dont need this NewBucket once we cache from makeDonutBucket()
bucket, err := NewBucket(bucketName, d.name, d.nodes)
bucket, err := NewBucket(bucketName, "private", d.name, d.nodes)
if err != nil {
return iodine.New(err, nil)
}

36
pkg/storage/drivers/api_testsuite.go
Unescape View File

@ -48,14 +48,14 @@ func APITestSuite(c *check.C, create func() Driver) {
func testCreateBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
}
func testMultipleObjectCreation(c *check.C, create func() Driver) {
objects := make(map[string][]byte)
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
@ -94,7 +94,7 @@ func testMultipleObjectCreation(c *check.C, create func() Driver) {
func testPaging(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
drivers.CreateBucket("bucket", "")
resources := BucketResourcesMetadata{}
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
@ -198,7 +198,7 @@ func testPaging(c *check.C, create func() Driver) {
func testObjectOverwriteFails(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
drivers.CreateBucket("bucket", "")
hasher1 := md5.New()
hasher1.Write([]byte("one"))
@ -227,7 +227,7 @@ func testNonExistantBucketOperations(c *check.C, create func() Driver) {
func testBucketMetadata(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("string")
err := drivers.CreateBucket("string", "")
c.Assert(err, check.IsNil)
metadata, err := drivers.GetBucketMetadata("string")
@ -237,15 +237,15 @@ func testBucketMetadata(c *check.C, create func() Driver) {
func testBucketRecreateFails(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("string")
err := drivers.CreateBucket("string", "")
c.Assert(err, check.IsNil)
err = drivers.CreateBucket("string")
err = drivers.CreateBucket("string", "")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
hasher := md5.New()
@ -270,7 +270,7 @@ func testListBuckets(c *check.C, create func() Driver) {
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = drivers.CreateBucket("bucket1")
err = drivers.CreateBucket("bucket1", "")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
@ -278,7 +278,7 @@ func testListBuckets(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil)
// add two and test exists
err = drivers.CreateBucket("bucket2")
err = drivers.CreateBucket("bucket2", "")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
@ -286,7 +286,7 @@ func testListBuckets(c *check.C, create func() Driver) {
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = drivers.CreateBucket("bucket22")
err = drivers.CreateBucket("bucket22", "")
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
@ -299,8 +299,8 @@ func testListBucketsOrder(c *check.C, create func() Driver) {
for i := 0; i < 10; i++ {
drivers := create()
// add one and test exists
drivers.CreateBucket("bucket1")
drivers.CreateBucket("bucket2")
drivers.CreateBucket("bucket1", "")
drivers.CreateBucket("bucket2", "")
buckets, err := drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
@ -321,7 +321,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver)
func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
@ -343,7 +343,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
@ -386,7 +386,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
func testDefaultContentType(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
// test empty
@ -408,10 +408,9 @@ func testDefaultContentType(c *check.C, create func() Driver) {
c.Assert(metadata.ContentType, check.Equals, "application/json")
}
/*
func testContentMd5Set(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
err := drivers.CreateBucket("bucket", "")
c.Assert(err, check.IsNil)
// test md5 invalid
@ -420,4 +419,3 @@ func testContentMd5Set(c *check.C, create func() Driver) {
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
}
*/

16
pkg/storage/drivers/donut/donut.go
Unescape View File

@ -39,6 +39,7 @@ import (
// donutDriver - creates a new single disk drivers driver using donut
type donutDriver struct {
donut donut.Donut
path string
}
const (
@ -82,6 +83,7 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
s := new(donutDriver)
s.donut = donut
s.path = path
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
@ -117,11 +119,14 @@ func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error)
}
// CreateBucket creates a new bucket
func (d donutDriver) CreateBucket(bucketName string) error {
func (d donutDriver) CreateBucket(bucketName, acl string) error {
if !drivers.IsValidBucketACL(acl) {
return iodine.New(drivers.InvalidACL{ACL: acl}, nil)
}
if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") {
return d.donut.MakeBucket(bucketName)
return d.donut.MakeBucket(bucketName, acl)
}
return iodine.New(errors.New("Invalid bucket"), map[string]string{"bucket": bucketName})
return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil)
}
// GetBucketMetadata retrieves an bucket's metadata
@ -137,9 +142,14 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat
if err != nil {
return drivers.BucketMetadata{}, iodine.New(err, nil)
}
acl, ok := metadata["acl"]
if !ok {
return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{Path: d.path}, nil)
}
bucketMetadata := drivers.BucketMetadata{
Name: metadata["name"],
Created: created,
ACL: drivers.BucketACL(acl),
}
return bucketMetadata, nil
}

49
pkg/storage/drivers/driver.go
Unescape View File

@ -27,7 +27,7 @@ import (
type Driver interface {
// Bucket Operations
ListBuckets() ([]BucketMetadata, error)
CreateBucket(bucket string) error
CreateBucket(bucket, acl string) error
GetBucketMetadata(bucket string) (BucketMetadata, error)
// Object Operations
@ -38,10 +38,40 @@ type Driver interface {
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error
}
// BucketACL - bucket level access control
type BucketACL string
// different types of ACL's currently supported for buckets
const (
BucketPrivate = BucketACL("private")
BucketPublicRead = BucketACL("public-read")
BucketPublicReadWrite = BucketACL("public-read-write")
)
func (b BucketACL) String() string {
return string(b)
}
// IsPrivate - is acl Private
func (b BucketACL) IsPrivate() bool {
return b == BucketACL("private")
}
// IsPublicRead - is acl PublicRead
func (b BucketACL) IsPublicRead() bool {
return b == BucketACL("public-read")
}
// IsPublicReadWrite - is acl PublicReadWrite
func (b BucketACL) IsPublicReadWrite() bool {
return b == BucketACL("public-read-write")
}
// BucketMetadata - name and create date
type BucketMetadata struct {
Name string
Created time.Time
ACL BucketACL
}
// ObjectMetadata - object key and its relevant metadata
@ -98,6 +128,23 @@ func GetMode(resources BucketResourcesMetadata) FilterMode {
return f
}
// IsValidBucketACL - is provided acl string supported
func IsValidBucketACL(acl string) bool {
switch acl {
case "private":
fallthrough
case "public-read":
fallthrough
case "public-read-write":
return true
case "":
// by default its "private"
return true
default:
return false
}
}
// IsDelimiterPrefixSet Delimiter and Prefix set
func (b BucketResourcesMetadata) IsDelimiterPrefixSet() bool {
return b.Mode == DelimiterPrefixMode

11
pkg/storage/drivers/errors.go
Unescape View File

@ -54,6 +54,17 @@ type DigestError struct {
Md5 string
}
/// ACL related errors
// InvalidACL - acl invalid
type InvalidACL struct {
ACL string
}
func (e InvalidACL) Error() string {
return "Requested ACL is " + e.ACL + " invalid"
}
/// Bucket related errors
// BucketNameInvalid - bucketname provided is invalid

12
pkg/storage/drivers/memory/memory.go
Unescape View File

@ -188,13 +188,16 @@ func (memory *memoryDriver) CreateObject(bucket, key, contentType, md5sum string
}
// CreateBucket - create bucket in memory
func (memory *memoryDriver) CreateBucket(bucketName string) error {
func (memory *memoryDriver) CreateBucket(bucketName, acl string) error {
memory.lock.RLock()
if !drivers.IsValidBucket(bucketName) {
memory.lock.RUnlock()
return drivers.BucketNameInvalid{Bucket: bucketName}
}
if !drivers.IsValidBucketACL(acl) {
memory.lock.RUnlock()
return drivers.InvalidACL{ACL: acl}
}
if _, ok := memory.bucketMetadata[bucketName]; ok == true {
memory.lock.RUnlock()
return drivers.BucketExists{Bucket: bucketName}
@ -205,6 +208,7 @@ func (memory *memoryDriver) CreateBucket(bucketName string) error {
newBucket.metadata = drivers.BucketMetadata{}
newBucket.metadata.Name = bucketName
newBucket.metadata.Created = time.Now()
newBucket.metadata.ACL = drivers.BucketACL(acl)
memory.lock.Lock()
defer memory.lock.Unlock()
memory.bucketMetadata[bucketName] = newBucket
@ -234,7 +238,7 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedN
switch true {
case key == resources.Prefix:
keys = appendUniq(keys, key)
// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
case key == resources.Prefix+delimitedName:
keys = appendUniq(keys, key)
case delimitedName != "":
@ -339,8 +343,6 @@ func (memory *memoryDriver) evictObject(key lru.Key, value interface{}) {
k := key.(string)
memory.totalSize = memory.totalSize - memory.objectMetadata[k].metadata.Size
log.Println("evicting:", k)
delete(memory.objectMetadata, k)
}

4
pkg/storage/drivers/mocks/Driver.go
Unescape View File

@ -27,8 +27,8 @@ func (m *Driver) ListBuckets() ([]drivers.BucketMetadata, error) {
}
// CreateBucket is a mock
func (m *Driver) CreateBucket(bucket string) error {
ret := m.Called(bucket)
func (m *Driver) CreateBucket(bucket, acl string) error {
ret := m.Called(bucket, acl)
r0 := ret.Error(0)

Write Preview
Loading…
Cancel
Save