From ab15f56a61bfc2fd1fd70089563253968e79a893 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 1 Nov 2015 20:56:54 -0800 Subject: [PATCH] fs/bucket: Move bucket metadata into buckets.json --- pkg/fs/acl.go | 29 +++------ pkg/fs/api_suite_windows_test.go | 3 +- pkg/fs/config.go | 63 +++++++++++++++--- pkg/fs/fs-bucket.go | 108 +++++++++++++++++++------------ pkg/fs/fs.go | 23 +++++++ pkg/fs/fs_test.go | 3 +- server_fs_test.go | 10 ++- 7 files changed, 165 insertions(+), 74 deletions(-) diff --git a/pkg/fs/acl.go b/pkg/fs/acl.go index 51954ef8d..b7f38fede 100644 --- a/pkg/fs/acl.go +++ b/pkg/fs/acl.go @@ -16,46 +16,35 @@ package fs -import ( - "os" - "path/filepath" -) - // IsPrivateBucket - is private bucket func (fs Filesystem) IsPrivateBucket(bucket string) bool { fs.lock.Lock() defer fs.lock.Unlock() - // get bucket path - bucketDir := filepath.Join(fs.path, bucket) - fi, err := os.Stat(bucketDir) - if err != nil { + bucketMetadata, ok := fs.buckets.Metadata[bucket] + if !ok { return true } - return permToACL(fi.Mode()).IsPrivate() + return bucketMetadata.ACL.IsPrivate() } // IsPublicBucket - is public bucket func (fs Filesystem) IsPublicBucket(bucket string) bool { fs.lock.Lock() defer fs.lock.Unlock() - // get bucket path - bucketDir := filepath.Join(fs.path, bucket) - fi, err := os.Stat(bucketDir) - if err != nil { + bucketMetadata, ok := fs.buckets.Metadata[bucket] + if !ok { return true } - return permToACL(fi.Mode()).IsPublicReadWrite() + return bucketMetadata.ACL.IsPublicReadWrite() } // IsReadOnlyBucket - is read only bucket func (fs Filesystem) IsReadOnlyBucket(bucket string) bool { fs.lock.Lock() defer fs.lock.Unlock() - // get bucket path - bucketDir := filepath.Join(fs.path, bucket) - fi, err := os.Stat(bucketDir) - if err != nil { + bucketMetadata, ok := fs.buckets.Metadata[bucket] + if !ok { return true } - return permToACL(fi.Mode()).IsPublicRead() + return bucketMetadata.ACL.IsPublicRead() } diff --git a/pkg/fs/api_suite_windows_test.go b/pkg/fs/api_suite_windows_test.go index 21a3bc541..22ebc52dc 100644 --- a/pkg/fs/api_suite_windows_test.go +++ b/pkg/fs/api_suite_windows_test.go @@ -333,8 +333,7 @@ func testBucketMetadata(c *check.C, create func() Filesystem) { metadata, err := fs.GetBucketMetadata("string") c.Assert(err, check.IsNil) - // On windows everything directory is always in public-read-write mode -- TODO need to handle this - c.Assert(metadata.ACL, check.Equals, BucketACL("public-read-write")) + c.Assert(metadata.ACL, check.Equals, BucketACL("private")) } func testBucketRecreateFails(c *check.C, create func() Filesystem) { diff --git a/pkg/fs/config.go b/pkg/fs/config.go index 628c9c413..c85decec4 100644 --- a/pkg/fs/config.go +++ b/pkg/fs/config.go @@ -24,7 +24,19 @@ import ( "github.com/minio/minio-xl/pkg/quick" ) -func getFSMultipartConfigPath() (string, *probe.Error) { +func getFSBucketsConfigPath() (string, *probe.Error) { + if customBucketsConfigPath != "" { + return customBucketsConfigPath, nil + } + u, err := user.Current() + if err != nil { + return "", probe.NewError(err) + } + fsBucketsConfigPath := filepath.Join(u.HomeDir, ".minio", "buckets.json") + return fsBucketsConfigPath, nil +} + +func getFSMultipartsSessionConfigPath() (string, *probe.Error) { if customMultipartsConfigPath != "" { return customMultipartsConfigPath, nil } @@ -32,16 +44,16 @@ func getFSMultipartConfigPath() (string, *probe.Error) { if err != nil { return "", probe.NewError(err) } - fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts.json") + fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts-session.json") return fsMultipartsConfigPath, nil } // internal variable only accessed via get/set methods -var customConfigPath, customMultipartsConfigPath string +var customMultipartsConfigPath, customBucketsConfigPath string -// SetFSConfigPath - set custom fs config path -func SetFSConfigPath(configPath string) { - customConfigPath = configPath +// SetFSBucketsConfigPath - set custom fs buckets config path +func SetFSBucketsConfigPath(configPath string) { + customBucketsConfigPath = configPath } // SetFSMultipartsConfigPath - set custom multiparts session config path @@ -51,7 +63,7 @@ func SetFSMultipartsConfigPath(configPath string) { // SaveMultipartsSession - save multiparts func SaveMultipartsSession(multiparts *Multiparts) *probe.Error { - fsMultipartsConfigPath, err := getFSMultipartConfigPath() + fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath() if err != nil { return err.Trace() } @@ -65,9 +77,25 @@ func SaveMultipartsSession(multiparts *Multiparts) *probe.Error { return nil } +// SaveBucketsMetadata - save metadata of all buckets +func SaveBucketsMetadata(buckets *Buckets) *probe.Error { + fsBucketsConfigPath, err := getFSBucketsConfigPath() + if err != nil { + return err.Trace() + } + qc, err := quick.New(buckets) + if err != nil { + return err.Trace() + } + if err := qc.Save(fsBucketsConfigPath); err != nil { + return err.Trace() + } + return nil +} + // loadMultipartsSession load multipart session file func loadMultipartsSession() (*Multiparts, *probe.Error) { - fsMultipartsConfigPath, err := getFSMultipartConfigPath() + fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath() if err != nil { return nil, err.Trace() } @@ -83,3 +111,22 @@ func loadMultipartsSession() (*Multiparts, *probe.Error) { } return qc.Data().(*Multiparts), nil } + +// loadBucketsMetadata load buckets metadata file +func loadBucketsMetadata() (*Buckets, *probe.Error) { + fsBucketsConfigPath, err := getFSBucketsConfigPath() + if err != nil { + return nil, err.Trace() + } + buckets := &Buckets{} + buckets.Version = "1" + buckets.Metadata = make(map[string]*BucketMetadata) + qc, err := quick.New(buckets) + if err != nil { + return nil, err.Trace() + } + if err := qc.Load(fsBucketsConfigPath); err != nil { + return nil, err.Trace() + } + return qc.Data().(*Buckets), nil +} diff --git a/pkg/fs/fs-bucket.go b/pkg/fs/fs-bucket.go index 5f95109e7..5f96bbd7b 100644 --- a/pkg/fs/fs-bucket.go +++ b/pkg/fs/fs-bucket.go @@ -38,7 +38,13 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error { } bucketDir := filepath.Join(fs.path, bucket) // check bucket exists - if _, err := os.Stat(bucketDir); os.IsNotExist(err) { + if _, err := os.Stat(bucketDir); err != nil { + if os.IsNotExist(err) { + return probe.NewError(BucketNotFound{Bucket: bucket}) + } + return probe.NewError(err) + } + if _, ok := fs.buckets.Metadata[bucket]; !ok { return probe.NewError(BucketNotFound{Bucket: bucket}) } if err := os.Remove(bucketDir); err != nil { @@ -47,6 +53,10 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error { } return probe.NewError(err) } + delete(fs.buckets.Metadata, bucket) + if err := SaveBucketsMetadata(fs.buckets); err != nil { + return err.Trace(bucket) + } return nil } @@ -102,9 +112,12 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } + if !IsValidBucketACL(acl) { + return probe.NewError(InvalidACL{ACL: acl}) + } + // get bucket path bucketDir := filepath.Join(fs.path, bucket) - // check if bucket exists if _, err = os.Stat(bucketDir); err == nil { return probe.NewError(BucketExists{ @@ -113,10 +126,30 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error { } // make bucket - err = os.Mkdir(bucketDir, aclToPerm(acl)) + err = os.Mkdir(bucketDir, 0700) if err != nil { return probe.NewError(err) } + + bucketMetadata := &BucketMetadata{} + fi, err := os.Stat(bucketDir) + // check if bucket exists + if err != nil { + if os.IsNotExist(err) { + return probe.NewError(BucketNotFound{Bucket: bucket}) + } + return probe.NewError(err) + } + if strings.TrimSpace(acl) == "" { + acl = "private" + } + bucketMetadata.Name = fi.Name() + bucketMetadata.Created = fi.ModTime() + bucketMetadata.ACL = BucketACL(acl) + fs.buckets.Metadata[bucket] = bucketMetadata + if err := SaveBucketsMetadata(fs.buckets); err != nil { + return err.Trace(bucket) + } return nil } @@ -129,48 +162,22 @@ func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Er } // get bucket path bucketDir := filepath.Join(fs.path, bucket) - bucketMetadata := BucketMetadata{} fi, err := os.Stat(bucketDir) - // check if bucket exists - if os.IsNotExist(err) { - return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket}) - } if err != nil { + // check if bucket exists + if os.IsNotExist(err) { + return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket}) + } return BucketMetadata{}, probe.NewError(err) } - - bucketMetadata.Name = fi.Name() - bucketMetadata.Created = fi.ModTime() - bucketMetadata.ACL = permToACL(fi.Mode()) - return bucketMetadata, nil -} - -// permToACL - convert perm to meaningful ACL -func permToACL(mode os.FileMode) BucketACL { - switch mode.Perm() { - case os.FileMode(0700): - return BucketACL("private") - case os.FileMode(0500): - return BucketACL("public-read") - case os.FileMode(0777): - return BucketACL("public-read-write") - default: - return BucketACL("private") - } -} - -// aclToPerm - convert acl to filesystem mode -func aclToPerm(acl string) os.FileMode { - switch acl { - case "private": - return os.FileMode(0700) - case "public-read": - return os.FileMode(0500) - case "public-read-write": - return os.FileMode(0777) - default: - return os.FileMode(0700) + bucketMetadata, ok := fs.buckets.Metadata[bucket] + if !ok { + bucketMetadata = &BucketMetadata{} + bucketMetadata.Name = fi.Name() + bucketMetadata.Created = fi.ModTime() + bucketMetadata.ACL = BucketACL("private") } + return *bucketMetadata, nil } // SetBucketMetadata - set bucket metadata @@ -184,11 +191,28 @@ func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string if !IsValidBucketACL(acl) { return probe.NewError(InvalidACL{ACL: acl}) } - // get bucket path + if strings.TrimSpace(acl) == "" { + acl = "private" + } bucketDir := filepath.Join(fs.path, bucket) - err := os.Chmod(bucketDir, aclToPerm(acl)) + fi, err := os.Stat(bucketDir) if err != nil { + // check if bucket exists + if os.IsNotExist(err) { + return probe.NewError(BucketNotFound{Bucket: bucket}) + } return probe.NewError(err) } + bucketMetadata, ok := fs.buckets.Metadata[bucket] + if !ok { + bucketMetadata = &BucketMetadata{} + bucketMetadata.Name = fi.Name() + bucketMetadata.Created = fi.ModTime() + } + bucketMetadata.ACL = BucketACL(acl) + fs.buckets.Metadata[bucket] = bucketMetadata + if err := SaveBucketsMetadata(fs.buckets); err != nil { + return err.Trace(bucket) + } return nil } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 93dfd66e1..2b1198882 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -30,6 +30,13 @@ type Filesystem struct { minFreeDisk int64 lock *sync.Mutex multiparts *Multiparts + buckets *Buckets +} + +// Buckets holds acl information +type Buckets struct { + Version string `json:"version"` + Metadata map[string]*BucketMetadata } // MultipartSession holds active session information @@ -65,8 +72,24 @@ func New() (Filesystem, *probe.Error) { return Filesystem{}, err.Trace() } } + var buckets *Buckets + buckets, err = loadBucketsMetadata() + if err != nil { + if os.IsNotExist(err.ToGoError()) { + buckets = &Buckets{ + Version: "1", + Metadata: make(map[string]*BucketMetadata), + } + if err := SaveBucketsMetadata(buckets); err != nil { + return Filesystem{}, err.Trace() + } + } else { + return Filesystem{}, err.Trace() + } + } a := Filesystem{lock: new(sync.Mutex)} a.multiparts = multiparts + a.buckets = buckets return a, nil } diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go index 086fbbb26..24360c1f0 100644 --- a/pkg/fs/fs_test.go +++ b/pkg/fs/fs_test.go @@ -38,7 +38,8 @@ func (s *MySuite) TestAPISuite(c *C) { c.Check(err, IsNil) path, err := ioutil.TempDir(os.TempDir(), "minio-") c.Check(err, IsNil) - SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts.json")) + SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts-session.json")) + SetFSBucketsConfigPath(filepath.Join(configPath, "buckets.json")) storageList = append(storageList, path) store, perr := New() store.SetRootPath(path) diff --git a/server_fs_test.go b/server_fs_test.go index 1e32e3c6b..da62c779a 100644 --- a/server_fs_test.go +++ b/server_fs_test.go @@ -59,12 +59,20 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) { fsroot, err := ioutil.TempDir(os.TempDir(), "api-") c.Assert(err, IsNil) - fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts.json")) + fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts-session.json")) + fs.SetFSBucketsConfigPath(filepath.Join(root, "buckets.json")) multiparts := &fs.Multiparts{} + multiparts.Version = "1" multiparts.ActiveSession = make(map[string]*fs.MultipartSession) perr := fs.SaveMultipartsSession(multiparts) c.Assert(perr, IsNil) + buckets := &fs.Buckets{} + buckets.Version = "1" + buckets.Metadata = make(map[string]*fs.BucketMetadata) + perr = fs.SaveBucketsMetadata(buckets) + c.Assert(perr, IsNil) + accessKeyID, perr := generateAccessKeyID() c.Assert(perr, IsNil) secretAccessKey, perr := generateSecretAccessKey()