oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fs/bucket: Move bucket metadata into buckets.json

Browse Source
master
Harshavardhana 9 years ago
parent 38aa7e1678
commit ab15f56a61
7 changed files with 165 additions and 74 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 29
      pkg/fs/acl.go
  2. 3
      pkg/fs/api_suite_windows_test.go
  3. 63
      pkg/fs/config.go
  4. 108
      pkg/fs/fs-bucket.go
  5. 23
      pkg/fs/fs.go
  6. 3
      pkg/fs/fs_test.go
  7. 10
      server_fs_test.go

29
pkg/fs/acl.go
Unescape View File

@ -16,46 +16,35 @@
package fs
import (
"os"
"path/filepath"
)
// IsPrivateBucket - is private bucket
func (fs Filesystem) IsPrivateBucket(bucket string) bool {
fs.lock.Lock()
defer fs.lock.Unlock()
// get bucket path
bucketDir := filepath.Join(fs.path, bucket)
fi, err := os.Stat(bucketDir)
if err != nil {
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return permToACL(fi.Mode()).IsPrivate()
return bucketMetadata.ACL.IsPrivate()
}
// IsPublicBucket - is public bucket
func (fs Filesystem) IsPublicBucket(bucket string) bool {
fs.lock.Lock()
defer fs.lock.Unlock()
// get bucket path
bucketDir := filepath.Join(fs.path, bucket)
fi, err := os.Stat(bucketDir)
if err != nil {
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return permToACL(fi.Mode()).IsPublicReadWrite()
return bucketMetadata.ACL.IsPublicReadWrite()
}
// IsReadOnlyBucket - is read only bucket
func (fs Filesystem) IsReadOnlyBucket(bucket string) bool {
fs.lock.Lock()
defer fs.lock.Unlock()
// get bucket path
bucketDir := filepath.Join(fs.path, bucket)
fi, err := os.Stat(bucketDir)
if err != nil {
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
return true
}
return permToACL(fi.Mode()).IsPublicRead()
return bucketMetadata.ACL.IsPublicRead()
}

3
pkg/fs/api_suite_windows_test.go
Unescape View File

@ -333,8 +333,7 @@ func testBucketMetadata(c *check.C, create func() Filesystem) {
metadata, err := fs.GetBucketMetadata("string")
c.Assert(err, check.IsNil)
// On windows everything directory is always in public-read-write mode -- TODO need to handle this
c.Assert(metadata.ACL, check.Equals, BucketACL("public-read-write"))
c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
}
func testBucketRecreateFails(c *check.C, create func() Filesystem) {

63
pkg/fs/config.go
Unescape View File

@ -24,7 +24,19 @@ import (
"github.com/minio/minio-xl/pkg/quick"
)
func getFSMultipartConfigPath() (string, *probe.Error) {
func getFSBucketsConfigPath() (string, *probe.Error) {
if customBucketsConfigPath != "" {
return customBucketsConfigPath, nil
}
u, err := user.Current()
if err != nil {
return "", probe.NewError(err)
}
fsBucketsConfigPath := filepath.Join(u.HomeDir, ".minio", "buckets.json")
return fsBucketsConfigPath, nil
}
func getFSMultipartsSessionConfigPath() (string, *probe.Error) {
if customMultipartsConfigPath != "" {
return customMultipartsConfigPath, nil
}
@ -32,16 +44,16 @@ func getFSMultipartConfigPath() (string, *probe.Error) {
if err != nil {
return "", probe.NewError(err)
}
fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts.json")
fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts-session.json")
return fsMultipartsConfigPath, nil
}
// internal variable only accessed via get/set methods
var customConfigPath, customMultipartsConfigPath string
var customMultipartsConfigPath, customBucketsConfigPath string
// SetFSConfigPath - set custom fs config path
func SetFSConfigPath(configPath string) {
customConfigPath = configPath
// SetFSBucketsConfigPath - set custom fs buckets config path
func SetFSBucketsConfigPath(configPath string) {
customBucketsConfigPath = configPath
}
// SetFSMultipartsConfigPath - set custom multiparts session config path
@ -51,7 +63,7 @@ func SetFSMultipartsConfigPath(configPath string) {
// SaveMultipartsSession - save multiparts
func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
fsMultipartsConfigPath, err := getFSMultipartConfigPath()
fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath()
if err != nil {
return err.Trace()
}
@ -65,9 +77,25 @@ func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
return nil
}
// SaveBucketsMetadata - save metadata of all buckets
func SaveBucketsMetadata(buckets *Buckets) *probe.Error {
fsBucketsConfigPath, err := getFSBucketsConfigPath()
if err != nil {
return err.Trace()
}
qc, err := quick.New(buckets)
if err != nil {
return err.Trace()
}
if err := qc.Save(fsBucketsConfigPath); err != nil {
return err.Trace()
}
return nil
}
// loadMultipartsSession load multipart session file
func loadMultipartsSession() (*Multiparts, *probe.Error) {
fsMultipartsConfigPath, err := getFSMultipartConfigPath()
fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath()
if err != nil {
return nil, err.Trace()
}
@ -83,3 +111,22 @@ func loadMultipartsSession() (*Multiparts, *probe.Error) {
}
return qc.Data().(*Multiparts), nil
}
// loadBucketsMetadata load buckets metadata file
func loadBucketsMetadata() (*Buckets, *probe.Error) {
fsBucketsConfigPath, err := getFSBucketsConfigPath()
if err != nil {
return nil, err.Trace()
}
buckets := &Buckets{}
buckets.Version = "1"
buckets.Metadata = make(map[string]*BucketMetadata)
qc, err := quick.New(buckets)
if err != nil {
return nil, err.Trace()
}
if err := qc.Load(fsBucketsConfigPath); err != nil {
return nil, err.Trace()
}
return qc.Data().(*Buckets), nil
}

108
pkg/fs/fs-bucket.go
Unescape View File

@ -38,7 +38,13 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
}
bucketDir := filepath.Join(fs.path, bucket)
// check bucket exists
if _, err := os.Stat(bucketDir); os.IsNotExist(err) {
if _, err := os.Stat(bucketDir); err != nil {
if os.IsNotExist(err) {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
return probe.NewError(err)
}
if _, ok := fs.buckets.Metadata[bucket]; !ok {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
if err := os.Remove(bucketDir); err != nil {
@ -47,6 +53,10 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
}
return probe.NewError(err)
}
delete(fs.buckets.Metadata, bucket)
if err := SaveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
}
@ -102,9 +112,12 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidBucketACL(acl) {
return probe.NewError(InvalidACL{ACL: acl})
}
// get bucket path
bucketDir := filepath.Join(fs.path, bucket)
// check if bucket exists
if _, err = os.Stat(bucketDir); err == nil {
return probe.NewError(BucketExists{
@ -113,10 +126,30 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
}
// make bucket
err = os.Mkdir(bucketDir, aclToPerm(acl))
err = os.Mkdir(bucketDir, 0700)
if err != nil {
return probe.NewError(err)
}
bucketMetadata := &BucketMetadata{}
fi, err := os.Stat(bucketDir)
// check if bucket exists
if err != nil {
if os.IsNotExist(err) {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
return probe.NewError(err)
}
if strings.TrimSpace(acl) == "" {
acl = "private"
}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = BucketACL(acl)
fs.buckets.Metadata[bucket] = bucketMetadata
if err := SaveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
}
@ -129,48 +162,22 @@ func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Er
}
// get bucket path
bucketDir := filepath.Join(fs.path, bucket)
bucketMetadata := BucketMetadata{}
fi, err := os.Stat(bucketDir)
// check if bucket exists
if os.IsNotExist(err) {
return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
}
if err != nil {
// check if bucket exists
if os.IsNotExist(err) {
return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
}
return BucketMetadata{}, probe.NewError(err)
}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = permToACL(fi.Mode())
return bucketMetadata, nil
}
// permToACL - convert perm to meaningful ACL
func permToACL(mode os.FileMode) BucketACL {
switch mode.Perm() {
case os.FileMode(0700):
return BucketACL("private")
case os.FileMode(0500):
return BucketACL("public-read")
case os.FileMode(0777):
return BucketACL("public-read-write")
default:
return BucketACL("private")
}
}
// aclToPerm - convert acl to filesystem mode
func aclToPerm(acl string) os.FileMode {
switch acl {
case "private":
return os.FileMode(0700)
case "public-read":
return os.FileMode(0500)
case "public-read-write":
return os.FileMode(0777)
default:
return os.FileMode(0700)
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
bucketMetadata = &BucketMetadata{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = BucketACL("private")
}
return *bucketMetadata, nil
}
// SetBucketMetadata - set bucket metadata
@ -184,11 +191,28 @@ func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string
if !IsValidBucketACL(acl) {
return probe.NewError(InvalidACL{ACL: acl})
}
// get bucket path
if strings.TrimSpace(acl) == "" {
acl = "private"
}
bucketDir := filepath.Join(fs.path, bucket)
err := os.Chmod(bucketDir, aclToPerm(acl))
fi, err := os.Stat(bucketDir)
if err != nil {
// check if bucket exists
if os.IsNotExist(err) {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
return probe.NewError(err)
}
bucketMetadata, ok := fs.buckets.Metadata[bucket]
if !ok {
bucketMetadata = &BucketMetadata{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
}
bucketMetadata.ACL = BucketACL(acl)
fs.buckets.Metadata[bucket] = bucketMetadata
if err := SaveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
}

23
pkg/fs/fs.go
Unescape View File

@ -30,6 +30,13 @@ type Filesystem struct {
minFreeDisk int64
lock *sync.Mutex
multiparts *Multiparts
buckets *Buckets
}
// Buckets holds acl information
type Buckets struct {
Version string `json:"version"`
Metadata map[string]*BucketMetadata
}
// MultipartSession holds active session information
@ -65,8 +72,24 @@ func New() (Filesystem, *probe.Error) {
return Filesystem{}, err.Trace()
}
}
var buckets *Buckets
buckets, err = loadBucketsMetadata()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
buckets = &Buckets{
Version: "1",
Metadata: make(map[string]*BucketMetadata),
}
if err := SaveBucketsMetadata(buckets); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
a := Filesystem{lock: new(sync.Mutex)}
a.multiparts = multiparts
a.buckets = buckets
return a, nil
}

3
pkg/fs/fs_test.go
Unescape View File

@ -38,7 +38,8 @@ func (s *MySuite) TestAPISuite(c *C) {
c.Check(err, IsNil)
path, err := ioutil.TempDir(os.TempDir(), "minio-")
c.Check(err, IsNil)
SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts.json"))
SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts-session.json"))
SetFSBucketsConfigPath(filepath.Join(configPath, "buckets.json"))
storageList = append(storageList, path)
store, perr := New()
store.SetRootPath(path)

10
server_fs_test.go
Unescape View File

@ -59,12 +59,20 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
fsroot, err := ioutil.TempDir(os.TempDir(), "api-")
c.Assert(err, IsNil)
fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts.json"))
fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts-session.json"))
fs.SetFSBucketsConfigPath(filepath.Join(root, "buckets.json"))
multiparts := &fs.Multiparts{}
multiparts.Version = "1"
multiparts.ActiveSession = make(map[string]*fs.MultipartSession)
perr := fs.SaveMultipartsSession(multiparts)
c.Assert(perr, IsNil)
buckets := &fs.Buckets{}
buckets.Version = "1"
buckets.Metadata = make(map[string]*fs.BucketMetadata)
perr = fs.SaveBucketsMetadata(buckets)
c.Assert(perr, IsNil)
accessKeyID, perr := generateAccessKeyID()
c.Assert(perr, IsNil)
secretAccessKey, perr := generateSecretAccessKey()

Write Preview
Loading…
Cancel
Save