/* * Minio Cloud Storage, (C) 2015 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package fs import ( "os" "runtime" "strings" "sync" "time" "io/ioutil" "path/filepath" "github.com/minio/minio-xl/pkg/probe" ) // API - local variables type API struct { path string lock *sync.Mutex multiparts *Multiparts } // MultipartSession holds active session information type MultipartSession struct { TotalParts int UploadID string Initiated time.Time Parts []*PartMetadata } // Multiparts collection of many parts type Multiparts struct { Version string `json:"version"` ActiveSession map[string]*MultipartSession `json:"activeSessions"` } // New instantiate a new donut func New(path string) (CloudStorage, *probe.Error) { var err *probe.Error // load multiparts session from disk var multiparts *Multiparts multiparts, err = loadMultipartsSession() if err != nil { if os.IsNotExist(err.ToGoError()) { multiparts = &Multiparts{ Version: "1", ActiveSession: make(map[string]*MultipartSession), } if err := SaveMultipartsSession(multiparts); err != nil { return nil, err.Trace() } } else { return nil, err.Trace() } } a := API{ path: path, lock: new(sync.Mutex), } a.multiparts = multiparts return a, nil } /// Bucket Operations // DeleteBucket - delete bucket func (fs API) DeleteBucket(bucket string) *probe.Error { fs.lock.Lock() defer fs.lock.Unlock() // verify bucket path legal if !IsValidBucket(bucket) { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } bucketDir := filepath.Join(fs.path, bucket) // check bucket exists if _, err := os.Stat(bucketDir); os.IsNotExist(err) { return probe.NewError(BucketNotFound{Bucket: bucket}) } if err := RemoveAllDirs(bucketDir); err != nil { if err == ErrDirNotEmpty || strings.Contains(err.Error(), "directory not empty") { return probe.NewError(BucketNotEmpty{Bucket: bucket}) } return probe.NewError(err) } if err := os.Remove(bucketDir); err != nil { if strings.Contains(err.Error(), "directory not empty") { return probe.NewError(BucketNotEmpty{Bucket: bucket}) } return probe.NewError(err) } return nil } // ListBuckets - Get service func (fs API) ListBuckets() ([]BucketMetadata, *probe.Error) { fs.lock.Lock() defer fs.lock.Unlock() files, err := ioutil.ReadDir(fs.path) if err != nil { return []BucketMetadata{}, probe.NewError(err) } var metadataList []BucketMetadata for _, file := range files { if !file.IsDir() { // if files found ignore them continue } if file.IsDir() { // if directories found with odd names, skip them too if !IsValidBucket(file.Name()) { continue } } metadata := BucketMetadata{ Name: file.Name(), Created: file.ModTime(), } metadataList = append(metadataList, metadata) } return metadataList, nil } // MakeBucket - PUT Bucket func (fs API) MakeBucket(bucket, acl string) *probe.Error { fs.lock.Lock() defer fs.lock.Unlock() // verify bucket path legal if !IsValidBucket(bucket) { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } // get bucket path bucketDir := filepath.Join(fs.path, bucket) // check if bucket exists if _, err := os.Stat(bucketDir); err == nil { return probe.NewError(BucketExists{ Bucket: bucket, }) } // make bucket err := os.Mkdir(bucketDir, aclToPerm(acl)) if err != nil { return probe.NewError(err) } return nil } // GetBucketMetadata - func (fs API) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) { fs.lock.Lock() defer fs.lock.Unlock() if !IsValidBucket(bucket) { return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) } // get bucket path bucketDir := filepath.Join(fs.path, bucket) bucketMetadata := BucketMetadata{} fi, err := os.Stat(bucketDir) // check if bucket exists if os.IsNotExist(err) { return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket}) } if err != nil { return BucketMetadata{}, probe.NewError(err) } bucketMetadata.Name = fi.Name() bucketMetadata.Created = fi.ModTime() bucketMetadata.ACL = permToACL(fi.Mode()) return bucketMetadata, nil } // permToACL - convert perm to meaningful ACL func permToACL(mode os.FileMode) BucketACL { switch mode.Perm() { case os.FileMode(0700): return BucketACL("private") case os.FileMode(0500): return BucketACL("public-read") case os.FileMode(0777): return BucketACL("public-read-write") default: return BucketACL("private") } } // aclToPerm - convert acl to filesystem mode func aclToPerm(acl string) os.FileMode { switch acl { case "private": return os.FileMode(0700) case "public-read": return os.FileMode(0500) case "public-read-write": return os.FileMode(0777) default: return os.FileMode(0700) } } // SetBucketMetadata - func (fs API) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error { fs.lock.Lock() defer fs.lock.Unlock() if !IsValidBucket(bucket) { return probe.NewError(BucketNameInvalid{Bucket: bucket}) } acl := metadata["acl"] if !IsValidBucketACL(acl) { return probe.NewError(InvalidACL{ACL: acl}) } // get bucket path bucketDir := filepath.Join(fs.path, bucket) err := os.Chmod(bucketDir, aclToPerm(acl)) if err != nil { return probe.NewError(err) } return nil } // ListObjects - GET bucket (list objects) func (fs API) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) { if !IsValidBucket(bucket) { return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket}) } if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false { return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}) } p := bucketDir{} rootPrefix := filepath.Join(fs.path, bucket) // check bucket exists if _, err := os.Stat(rootPrefix); os.IsNotExist(err) { return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket}) } p.root = rootPrefix /// automatically treat "/" delimiter as "\\" delimiter on windows due to its path constraints. if resources.Delimiter == "/" { if runtime.GOOS == "windows" { resources.Delimiter = string(os.PathSeparator) } } // if delimiter is supplied and not prefix then we are the very top level, list everything and move on. if resources.Delimiter != "" && resources.Prefix == "" { files, err := ioutil.ReadDir(rootPrefix) if err != nil { if os.IsNotExist(err) { return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket}) } return nil, resources, probe.NewError(err) } for _, fl := range files { p.files = append(p.files, contentInfo{ Prefix: fl.Name(), Size: fl.Size(), Mode: fl.Mode(), ModTime: fl.ModTime(), FileInfo: fl, }) } } // If delimiter and prefix is supplied make sure that paging doesn't go deep, treat it as simple directory listing. if resources.Delimiter != "" && resources.Prefix != "" { if !strings.HasSuffix(resources.Prefix, resources.Delimiter) { fl, err := os.Stat(filepath.Join(rootPrefix, resources.Prefix)) if err != nil { if os.IsNotExist(err) { return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) } return nil, resources, probe.NewError(err) } p.files = append(p.files, contentInfo{ Prefix: resources.Prefix, Size: fl.Size(), Mode: os.ModeDir, ModTime: fl.ModTime(), FileInfo: fl, }) } else { files, err := ioutil.ReadDir(filepath.Join(rootPrefix, resources.Prefix)) if err != nil { if os.IsNotExist(err) { return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) } return nil, resources, probe.NewError(err) } for _, fl := range files { prefix := fl.Name() if resources.Prefix != "" { prefix = filepath.Join(resources.Prefix, fl.Name()) } p.files = append(p.files, contentInfo{ Prefix: prefix, Size: fl.Size(), Mode: fl.Mode(), ModTime: fl.ModTime(), FileInfo: fl, }) } } } if resources.Delimiter == "" { var files []contentInfo getAllFiles := func(fp string, fl os.FileInfo, err error) error { // If any error return back quickly if err != nil { return err } if strings.HasSuffix(fp, "$multiparts") { return nil } // if file pointer equals to rootPrefix - discard it if fp == p.root { return nil } if len(files) > resources.Maxkeys { return ErrSkipFile } // Split the root prefix from the incoming file pointer realFp := "" if runtime.GOOS == "windows" { if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 { realFp = splits[1] } } else { if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 { realFp = splits[1] } } // If path is a directory and has a prefix verify if the file pointer // has the prefix if it does not skip the directory. if fl.Mode().IsDir() { if resources.Prefix != "" { if !strings.HasPrefix(fp, filepath.Join(p.root, resources.Prefix)) { return ErrSkipDir } } } // If path is a directory and has a marker verify if the file split file pointer // is lesser than the Marker top level directory if yes skip it. if fl.Mode().IsDir() { if resources.Marker != "" { if realFp != "" { if runtime.GOOS == "windows" { if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] { return ErrSkipDir } } else { if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] { return ErrSkipDir } } } } } // If regular file verify if fl.Mode().IsRegular() { // If marker is present this will be used to check if filepointer is // lexically higher than then Marker if realFp != "" { if resources.Marker != "" { if realFp > resources.Marker { files = append(files, contentInfo{ Prefix: realFp, Size: fl.Size(), Mode: fl.Mode(), ModTime: fl.ModTime(), FileInfo: fl, }) } } else { files = append(files, contentInfo{ Prefix: realFp, Size: fl.Size(), Mode: fl.Mode(), ModTime: fl.ModTime(), FileInfo: fl, }) } } } // If file is a symlink follow it and populate values. if fl.Mode()&os.ModeSymlink == os.ModeSymlink { st, err := os.Stat(fp) if err != nil { return nil } // If marker is present this will be used to check if filepointer is // lexically higher than then Marker if realFp != "" { if resources.Marker != "" { if realFp > resources.Marker { files = append(files, contentInfo{ Prefix: realFp, Size: st.Size(), Mode: st.Mode(), ModTime: st.ModTime(), FileInfo: st, }) } } else { files = append(files, contentInfo{ Prefix: realFp, Size: st.Size(), Mode: st.Mode(), ModTime: st.ModTime(), FileInfo: st, }) } } } p.files = files return nil } // If no delimiter is specified, crawl through everything. err := Walk(rootPrefix, getAllFiles) if err != nil { if os.IsNotExist(err) { return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) } return nil, resources, probe.NewError(err) } } var metadataList []ObjectMetadata var metadata ObjectMetadata // Filter objects for _, content := range p.files { if len(metadataList) == resources.Maxkeys { resources.IsTruncated = true if resources.IsTruncated && resources.Delimiter != "" { resources.NextMarker = metadataList[len(metadataList)-1].Object } break } if content.Prefix > resources.Marker { var err *probe.Error metadata, resources, err = fs.filterObjects(bucket, content, resources) if err != nil { return nil, resources, err.Trace() } if metadata.Bucket != "" { metadataList = append(metadataList, metadata) } } } return metadataList, resources, nil }