oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Filesystem goes the high road *again*

Browse Source
master
Harshavardhana 9 years ago
parent c4c67581dc
commit 2571342451
10 changed files with 0 additions and 1450 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 33
      commands.go
  2. 9
      pkg/api/api_test.go
  3. 18
      pkg/server/server.go
  4. 42
      pkg/storage/drivers/fs/fs.go
  5. 205
      pkg/storage/drivers/fs/fs_bucket.go
  6. 88
      pkg/storage/drivers/fs/fs_common.go
  7. 88
      pkg/storage/drivers/fs/fs_filter.go
  8. 618
      pkg/storage/drivers/fs/fs_multipart.go
  9. 295
      pkg/storage/drivers/fs/fs_object.go
  10. 54
      pkg/storage/drivers/fs/fs_test.go

33
commands.go
Unescape View File

@ -27,7 +27,6 @@ var commands = []cli.Command{
var modeCommands = []cli.Command{
memoryCmd,
fsCmd,
donutCmd,
}
@ -56,23 +55,6 @@ EXAMPLES:
`,
}
var fsCmd = cli.Command{
Name: "fs",
Description: "Path to filesystem volume.",
Action: runFilesystem,
CustomHelpTemplate: `NAME:
minio mode {{.Name}} - {{.Description}}
USAGE:
minio mode {{.Name}} limit SIZE expire TIME
EXAMPLES:
1. Export an existing filesystem path
$ minio mode {{.Name}} /var/www
`,
}
var donutCmd = cli.Command{
Name: "donut",
Description: "[status: EXPERIMENTAL]. Path to donut volume.",
@ -250,18 +232,3 @@ func runDonut(c *cli.Context) {
servers := []server.StartServerFunc{apiServer} //, webServer}
server.StartMinio(servers)
}
func runFilesystem(c *cli.Context) {
if len(c.Args()) != 1 {
cli.ShowCommandHelpAndExit(c, "fs", 1) // last argument is exit code
}
apiServerConfig := getAPIServerConfig(c)
fsDriver := server.FilesystemFactory{
Config: apiServerConfig,
Path: c.Args()[0],
}
apiServer := fsDriver.GetStartServerFunc()
// webServer := getWebServerConfigFunc(c)
servers := []server.StartServerFunc{apiServer} //, webServer}
server.StartMinio(servers)
}

9
pkg/api/api_test.go
Unescape View File

@ -34,7 +34,6 @@ import (
"github.com/minio/minio/pkg/storage/drivers"
"github.com/minio/minio/pkg/storage/drivers/donut"
"github.com/minio/minio/pkg/storage/drivers/fs"
"github.com/minio/minio/pkg/storage/drivers/memory"
"github.com/minio/minio/pkg/storage/drivers/mocks"
"github.com/stretchr/testify/mock"
@ -74,14 +73,6 @@ var _ = Suite(&MySuite{
},
})
var _ = Suite(&MySuite{
initDriver: func() (drivers.Driver, string) {
root, _ := ioutil.TempDir(os.TempDir(), "minio-fs-api")
driver, _ := filesystem.NewDriver(root)
return driver, root
},
})
func (s *MySuite) SetUpSuite(c *C) {
driver, root := s.initDriver()
if root != "" {

18
pkg/server/server.go
Unescape View File

@ -27,7 +27,6 @@ import (
"github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/server/httpserver"
"github.com/minio/minio/pkg/storage/drivers/donut"
fs "github.com/minio/minio/pkg/storage/drivers/fs"
"github.com/minio/minio/pkg/storage/drivers/memory"
"github.com/minio/minio/pkg/utils/log"
)
@ -50,23 +49,6 @@ func (f MemoryFactory) GetStartServerFunc() StartServerFunc {
}
}
// FilesystemFactory is used to build filesystem api server
type FilesystemFactory struct {
httpserver.Config
Path string
}
// GetStartServerFunc builds memory api server
func (f FilesystemFactory) GetStartServerFunc() StartServerFunc {
return func() (chan<- string, <-chan error) {
driver, _ := fs.NewDriver(f.Path)
conf := api.Config{RateLimit: f.RateLimit}
conf.SetDriver(driver)
ctrl, status, _ := httpserver.Start(api.HTTPHandler(conf), f.Config)
return ctrl, status
}
}
// WebFactory is used to build web cli server
type WebFactory struct {
httpserver.Config

42
pkg/storage/drivers/fs/fs.go
Unescape View File

@ -1,42 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"os"
"sync"
"github.com/minio/minio/pkg/storage/drivers"
)
type fsDriver struct {
root string
lock *sync.Mutex
multiparts *Multiparts
}
// NewDriver instantiate a new filesystem driver
func NewDriver(root string) (drivers.Driver, error) {
fs := new(fsDriver)
fs.root = root
fs.lock = new(sync.Mutex)
// internal related to multiparts
fs.multiparts = new(Multiparts)
fs.multiparts.ActiveSession = make(map[string]*MultipartSession)
err := os.MkdirAll(fs.root, 0700)
return fs, err
}

205
pkg/storage/drivers/fs/fs_bucket.go
Unescape View File

@ -1,205 +0,0 @@
/*
* Mini Object Fs, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"os"
"sort"
"io/ioutil"
"path/filepath"
"github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/storage/drivers"
)
/// Bucket Operations
// ListBuckets - Get service
func (fs *fsDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
files, err := ioutil.ReadDir(fs.root)
if err != nil {
return []drivers.BucketMetadata{}, iodine.New(err, nil)
}
var metadataList []drivers.BucketMetadata
for _, file := range files {
if !file.IsDir() {
// if files found ignore them
continue
}
if file.IsDir() {
// if directories found with odd names, skip them too
if !drivers.IsValidBucket(file.Name()) {
continue
}
}
metadata := drivers.BucketMetadata{
Name: file.Name(),
Created: file.ModTime(),
}
metadataList = append(metadataList, metadata)
}
return metadataList, nil
}
// CreateBucket - PUT Bucket
func (fs *fsDriver) CreateBucket(bucket, acl string) error {
fs.lock.Lock()
defer fs.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// get bucket path
bucketDir := filepath.Join(fs.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return iodine.New(drivers.BucketExists{
Bucket: bucket,
}, nil)
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return iodine.New(err, nil)
}
return nil
}
// GetBucketMetadata -
func (fs *fsDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
if !drivers.IsValidBucket(bucket) {
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// get bucket path
bucketDir := filepath.Join(fs.root, bucket)
bucketMetadata := drivers.BucketMetadata{}
fi, err := os.Stat(bucketDir)
// check if bucket exists
if os.IsNotExist(err) {
return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return drivers.BucketMetadata{}, iodine.New(err, nil)
}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
// TODO convert os.FileMode to meaningful ACL's
bucketMetadata.ACL = drivers.BucketACL("private")
return bucketMetadata, nil
}
// aclToPerm - convert acl to filesystem mode
func aclToPerm(acl string) os.FileMode {
switch acl {
case "private":
return os.FileMode(0700)
case "public-read":
return os.FileMode(0500)
case "public-read-write":
return os.FileMode(0777)
case "authenticated-read":
return os.FileMode(0770)
default:
return os.FileMode(0700)
}
}
// SetBucketMetadata -
func (fs *fsDriver) SetBucketMetadata(bucket, acl string) error {
fs.lock.Lock()
defer fs.lock.Unlock()
if !drivers.IsValidBucket(bucket) {
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
if !drivers.IsValidBucketACL(acl) {
return iodine.New(drivers.InvalidACL{ACL: acl}, nil)
}
// get bucket path
bucketDir := filepath.Join(fs.root, bucket)
err := os.Chmod(bucketDir, aclToPerm(acl))
if err != nil {
return iodine.New(err, nil)
}
return nil
}
// ListObjects - GET bucket (list objects)
func (fs *fsDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
p := bucketDir{}
p.files = make(map[string]os.FileInfo)
if drivers.IsValidBucket(bucket) == false {
return []drivers.ObjectMetadata{}, resources, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
if resources.Prefix != "" && drivers.IsValidObjectName(resources.Prefix) == false {
return []drivers.ObjectMetadata{}, resources, iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}, nil)
}
rootPrefix := filepath.Join(fs.root, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return []drivers.ObjectMetadata{}, resources, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
p.root = rootPrefix
err := filepath.Walk(rootPrefix, p.getAllFiles)
if err != nil {
return []drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
var metadataList []drivers.ObjectMetadata
var metadata drivers.ObjectMetadata
// Populate filtering mode
resources.Mode = drivers.GetMode(resources)
var fileNames []string
for name := range p.files {
fileNames = append(fileNames, name)
}
sort.Strings(fileNames)
for _, name := range fileNames {
if len(metadataList) >= resources.Maxkeys {
resources.IsTruncated = true
if resources.IsTruncated && resources.IsDelimiterSet() {
resources.NextMarker = metadataList[len(metadataList)-1].Key
}
break
}
if name > resources.Marker {
metadata, resources, err = fs.filterObjects(bucket, name, p.files[name], resources)
if err != nil {
return []drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
}
}
sort.Sort(byObjectKey(metadataList))
return metadataList, resources, nil
}

88
pkg/storage/drivers/fs/fs_common.go
Unescape View File

@ -1,88 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"bufio"
"bytes"
"os"
"regexp"
"strings"
"github.com/minio/minio/pkg/storage/drivers"
)
// Metadata - carries metadata about object
type Metadata struct {
Md5sum []byte
ContentType string
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
type bucketDir struct {
files map[string]os.FileInfo
root string
}
func (p *bucketDir) getAllFiles(object string, fl os.FileInfo, err error) error {
if err != nil {
return err
}
if fl.Mode().IsRegular() {
if strings.HasSuffix(object, "$metadata") {
return nil
}
if strings.HasSuffix(object, "$multiparts") {
return nil
}
matched, err := regexp.MatchString("\\$[0-9].*$", object)
if err != nil {
return nil
}
if matched {
return nil
}
_p := strings.Split(object, p.root+"/")
if len(_p) > 1 {
p.files[_p[1]] = fl
}
}
return nil
}
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
type byObjectKey []drivers.ObjectMetadata
func (b byObjectKey) Len() int { return len(b) }
func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }

88
pkg/storage/drivers/fs/fs_filter.go
Unescape View File

@ -1,88 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"os"
"strings"
"github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/storage/drivers"
)
func (fs *fsDriver) filterObjects(bucket, name string, file os.FileInfo, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
var err error
var metadata drivers.ObjectMetadata
switch true {
// Both delimiter and Prefix is present
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
trimmedName := strings.TrimPrefix(name, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
switch true {
case name == resources.Prefix:
// Use resources.Prefix to filter out delimited files
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
case delimitedName == file.Name():
// Use resources.Prefix to filter out delimited files
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
}
}
// Delimiter present and Prefix is absent
case resources.IsDelimiterSet():
delimitedName := delimiter(name, resources.Delimiter)
switch true {
case delimitedName == "":
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
case delimitedName == file.Name():
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Delimiter is absent and only Prefix is present
case resources.IsPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
// Do not strip prefix object output
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
}
case resources.IsDefault():
metadata, err = fs.GetObjectMetadata(bucket, name)
if err != nil {
return drivers.ObjectMetadata{}, resources, iodine.New(err, nil)
}
}
return metadata, resources, nil
}

618
pkg/storage/drivers/fs/fs_multipart.go
Unescape View File

@ -1,618 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"bytes"
"crypto/md5"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/storage/drivers"
)
// MultipartSession holds active session information
type MultipartSession struct {
TotalParts int
UploadID string
Initiated time.Time
Parts []*drivers.PartMetadata
}
// Multiparts collection of many parts
type Multiparts struct {
ActiveSession map[string]*MultipartSession
}
func (fs *fsDriver) loadActiveSessions(bucket string) {
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
if err != nil {
return
}
activeSessionFile, err := os.OpenFile(bucketPath+"$activeSession", os.O_RDONLY, 0600)
if err != nil {
return
}
defer activeSessionFile.Close()
var deserializedActiveSession map[string]*MultipartSession
decoder := json.NewDecoder(activeSessionFile)
err = decoder.Decode(&deserializedActiveSession)
if err != nil {
return
}
for key, value := range deserializedActiveSession {
fs.multiparts.ActiveSession[key] = value
}
return
}
func (fs *fsDriver) isValidUploadID(key, uploadID string) bool {
s, ok := fs.multiparts.ActiveSession[key]
if !ok {
return false
}
if uploadID == s.UploadID {
return true
}
return false
}
func (fs *fsDriver) writePart(objectPath string, partID int, size int64, data io.Reader) (drivers.PartMetadata, error) {
partPath := objectPath + fmt.Sprintf("$%d", partID)
// write part
partFile, err := os.OpenFile(partPath, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return drivers.PartMetadata{}, iodine.New(err, nil)
}
defer partFile.Close()
h := md5.New()
mw := io.MultiWriter(partFile, h)
_, err = io.CopyN(mw, data, size)
if err != nil {
return drivers.PartMetadata{}, iodine.New(err, nil)
}
fi, err := os.Stat(partPath)
if err != nil {
return drivers.PartMetadata{}, iodine.New(err, nil)
}
partMetadata := drivers.PartMetadata{}
partMetadata.ETag = hex.EncodeToString(h.Sum(nil))
partMetadata.PartNumber = partID
partMetadata.Size = fi.Size()
partMetadata.LastModified = fi.ModTime()
return partMetadata, nil
}
// byKey is a sortable interface for UploadMetadata slice
type byKey []*drivers.UploadMetadata
func (a byKey) Len() int { return len(a) }
func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
func (fs *fsDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
if !drivers.IsValidBucket(bucket) {
return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.InternalError{}, nil)
}
// load from disk
fs.loadActiveSessions(bucket)
var uploads []*drivers.UploadMetadata
for key, session := range fs.multiparts.ActiveSession {
if strings.HasPrefix(key, resources.Prefix) {
if len(uploads) > resources.MaxUploads {
sort.Sort(byKey(uploads))
resources.Upload = uploads
resources.NextKeyMarker = key
resources.NextUploadIDMarker = session.UploadID
resources.IsTruncated = true
return resources, nil
}
// uploadIDMarker is ignored if KeyMarker is empty
switch {
case resources.KeyMarker != "" && resources.UploadIDMarker == "":
if key > resources.KeyMarker {
upload := new(drivers.UploadMetadata)
upload.Key = key
upload.UploadID = session.UploadID
upload.Initiated = session.Initiated
uploads = append(uploads, upload)
}
case resources.KeyMarker != "" && resources.UploadIDMarker != "":
if session.UploadID > resources.UploadIDMarker {
if key >= resources.KeyMarker {
upload := new(drivers.UploadMetadata)
upload.Key = key
upload.UploadID = session.UploadID
upload.Initiated = session.Initiated
uploads = append(uploads, upload)
}
}
default:
upload := new(drivers.UploadMetadata)
upload.Key = key
upload.UploadID = session.UploadID
upload.Initiated = session.Initiated
uploads = append(uploads, upload)
}
}
}
sort.Sort(byKey(uploads))
resources.Upload = uploads
return resources, nil
}
func (fs *fsDriver) concatParts(parts map[int]string, objectPath string, mw io.Writer) error {
for i := 1; i <= len(parts); i++ {
recvMD5 := parts[i]
partFile, err := os.OpenFile(objectPath+fmt.Sprintf("$%d", i), os.O_RDONLY, 0600)
if err != nil {
return iodine.New(err, nil)
}
obj, err := ioutil.ReadAll(partFile)
if err != nil {
return iodine.New(err, nil)
}
calcMD5Bytes := md5.Sum(obj)
// complete multi part request header md5sum per part is hex encoded
recvMD5Bytes, err := hex.DecodeString(strings.Trim(recvMD5, "\""))
if err != nil {
return iodine.New(drivers.InvalidDigest{Md5: recvMD5}, nil)
}
if !bytes.Equal(recvMD5Bytes, calcMD5Bytes[:]) {
return iodine.New(drivers.BadDigest{Md5: recvMD5}, nil)
}
_, err = io.Copy(mw, bytes.NewBuffer(obj))
if err != nil {
return iodine.New(err, nil)
}
}
return nil
}
func (fs *fsDriver) NewMultipartUpload(bucket, key, contentType string) (string, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
if !drivers.IsValidBucket(bucket) {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
if !drivers.IsValidObjectName(key) {
return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil)
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return "", iodine.New(drivers.InternalError{}, nil)
}
objectPath := filepath.Join(bucketPath, key)
objectDir := filepath.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return "", iodine.New(err, nil)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return "", iodine.New(drivers.ObjectExists{
Bucket: bucket,
Object: key,
}, nil)
}
var activeSessionFile *os.File
_, err = os.Stat(bucketPath + "$activeSession")
switch {
case os.IsNotExist(err):
activeSessionFile, err = os.OpenFile(bucketPath+"$activeSession", os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
default:
activeSessionFile, err = os.OpenFile(bucketPath+"$activeSession", os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
}
defer activeSessionFile.Close()
id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + key + time.Now().String())
uploadIDSum := sha512.Sum512(id)
uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
multiPartfile, err := os.OpenFile(objectPath+"$multiparts", os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer multiPartfile.Close()
mpartSession := new(MultipartSession)
mpartSession.TotalParts = 0
mpartSession.UploadID = uploadID
mpartSession.Initiated = time.Now().UTC()
var parts []*drivers.PartMetadata
mpartSession.Parts = parts
fs.multiparts.ActiveSession[key] = mpartSession
encoder := json.NewEncoder(multiPartfile)
err = encoder.Encode(mpartSession)
if err != nil {
return "", iodine.New(err, nil)
}
encoder = json.NewEncoder(activeSessionFile)
err = encoder.Encode(fs.multiparts.ActiveSession)
if err != nil {
return "", iodine.New(err, nil)
}
return uploadID, nil
}
// partNumber is a sortable interface for Part slice
type partNumber []*drivers.PartMetadata
func (a partNumber) Len() int { return len(a) }
func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
func (fs *fsDriver) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
if partID <= 0 {
return "", iodine.New(errors.New("invalid part id, cannot be zero or less than zero"), nil)
}
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// verify object path legal
if drivers.IsValidObjectName(key) == false {
return "", iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: key}, nil)
}
if !fs.isValidUploadID(key, uploadID) {
return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
}
if strings.TrimSpace(expectedMD5Sum) != "" {
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil {
// pro-actively close the connection
return "", iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil)
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return "", iodine.New(drivers.InternalError{}, nil)
}
objectPath := filepath.Join(bucketPath, key)
objectDir := filepath.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return "", iodine.New(err, nil)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return "", iodine.New(drivers.ObjectExists{
Bucket: bucket,
Object: key,
}, nil)
}
partMetadata, err := fs.writePart(objectPath, partID, size, data)
if err != nil {
return "", iodine.New(err, nil)
}
// Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" {
if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), partMetadata.ETag); err != nil {
return "", iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil)
}
}
multiPartfile, err := os.OpenFile(objectPath+"$multiparts", os.O_RDWR|os.O_APPEND, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer multiPartfile.Close()
var deserializedMultipartSession MultipartSession
decoder := json.NewDecoder(multiPartfile)
err = decoder.Decode(&deserializedMultipartSession)
if err != nil {
return "", iodine.New(err, nil)
}
deserializedMultipartSession.Parts = append(deserializedMultipartSession.Parts, &partMetadata)
deserializedMultipartSession.TotalParts++
fs.multiparts.ActiveSession[key] = &deserializedMultipartSession
sort.Sort(partNumber(deserializedMultipartSession.Parts))
encoder := json.NewEncoder(multiPartfile)
err = encoder.Encode(&deserializedMultipartSession)
if err != nil {
return "", iodine.New(err, nil)
}
return partMetadata.ETag, nil
}
func (fs *fsDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// verify object path legal
if drivers.IsValidObjectName(key) == false {
return "", iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: key}, nil)
}
if !fs.isValidUploadID(key, uploadID) {
return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return "", iodine.New(drivers.InternalError{}, nil)
}
objectPath := filepath.Join(bucketPath, key)
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return "", iodine.New(drivers.ObjectExists{
Bucket: bucket,
Object: key,
}, nil)
}
file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer file.Close()
h := md5.New()
mw := io.MultiWriter(file, h)
err = fs.concatParts(parts, objectPath, mw)
if err != nil {
return "", iodine.New(err, nil)
}
md5sum := hex.EncodeToString(h.Sum(nil))
delete(fs.multiparts.ActiveSession, key)
for partNumber := range parts {
err = os.Remove(objectPath + fmt.Sprintf("$%d", partNumber))
if err != nil {
return "", iodine.New(err, nil)
}
}
err = os.Remove(objectPath + "$multiparts")
if err != nil {
return "", iodine.New(err, nil)
}
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer file.Close()
metadata := &Metadata{
ContentType: "application/octet-stream",
Md5sum: h.Sum(nil),
}
// serialize metadata to json
encoder := json.NewEncoder(file)
err = encoder.Encode(metadata)
if err != nil {
return "", iodine.New(err, nil)
}
activeSessionFile, err := os.OpenFile(bucketPath+"$activeSession", os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer activeSessionFile.Close()
encoder = json.NewEncoder(activeSessionFile)
err = encoder.Encode(fs.multiparts.ActiveSession)
if err != nil {
return "", iodine.New(err, nil)
}
return md5sum, nil
}
func (fs *fsDriver) ListObjectParts(bucket, key string, resources drivers.ObjectResourcesMetadata) (drivers.ObjectResourcesMetadata, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
// load from disk
fs.loadActiveSessions(bucket)
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// verify object path legal
if drivers.IsValidObjectName(key) == false {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: key}, nil)
}
if !fs.isValidUploadID(key, resources.UploadID) {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.InvalidUploadID{UploadID: resources.UploadID}, nil)
}
objectResourcesMetadata := resources
objectResourcesMetadata.Bucket = bucket
objectResourcesMetadata.Key = key
var startPartNumber int
switch {
case objectResourcesMetadata.PartNumberMarker == 0:
startPartNumber = 1
default:
startPartNumber = objectResourcesMetadata.PartNumberMarker
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.InternalError{}, nil)
}
objectPath := filepath.Join(bucketPath, key)
multiPartfile, err := os.OpenFile(objectPath+"$multiparts", os.O_RDONLY, 0600)
if err != nil {
return drivers.ObjectResourcesMetadata{}, iodine.New(err, nil)
}
defer multiPartfile.Close()
var deserializedMultipartSession MultipartSession
decoder := json.NewDecoder(multiPartfile)
err = decoder.Decode(&deserializedMultipartSession)
if err != nil {
return drivers.ObjectResourcesMetadata{}, iodine.New(err, nil)
}
var parts []*drivers.PartMetadata
for i := startPartNumber; i <= deserializedMultipartSession.TotalParts; i++ {
if len(parts) > objectResourcesMetadata.MaxParts {
sort.Sort(partNumber(parts))
objectResourcesMetadata.IsTruncated = true
objectResourcesMetadata.Part = parts
objectResourcesMetadata.NextPartNumberMarker = i
return objectResourcesMetadata, nil
}
parts = append(parts, deserializedMultipartSession.Parts[i-1])
}
sort.Sort(partNumber(parts))
objectResourcesMetadata.Part = parts
return objectResourcesMetadata, nil
}
func (fs *fsDriver) AbortMultipartUpload(bucket, key, uploadID string) error {
fs.lock.Lock()
defer fs.lock.Unlock()
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// verify object path legal
if drivers.IsValidObjectName(key) == false {
return iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: key}, nil)
}
if !fs.isValidUploadID(key, uploadID) {
return iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil)
}
bucketPath := filepath.Join(fs.root, bucket)
_, err := os.Stat(bucketPath)
// check bucket exists
if os.IsNotExist(err) {
return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
if err != nil {
return iodine.New(drivers.InternalError{}, nil)
}
objectPath := filepath.Join(bucketPath, key)
multiPartfile, err := os.OpenFile(objectPath+"$multiparts", os.O_RDWR, 0600)
if err != nil {
return iodine.New(err, nil)
}
var deserializedMultipartSession MultipartSession
decoder := json.NewDecoder(multiPartfile)
err = decoder.Decode(&deserializedMultipartSession)
if err != nil {
return iodine.New(err, nil)
}
multiPartfile.Close() // close it right here, since we will delete it subsequently
delete(fs.multiparts.ActiveSession, key)
for _, part := range deserializedMultipartSession.Parts {
err = os.RemoveAll(objectPath + fmt.Sprintf("$%d", part.PartNumber))
if err != nil {
return iodine.New(err, nil)
}
}
err = os.RemoveAll(objectPath + "$multiparts")
if err != nil {
return iodine.New(err, nil)
}
return nil
}

295
pkg/storage/drivers/fs/fs_object.go
Unescape View File

@ -1,295 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"bytes"
"io"
"os"
"path/filepath"
"strings"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/storage/drivers"
)
/// Object Operations
// GetPartialObject - GET object from range
func (fs *fsDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// validate object
if drivers.IsValidObjectName(object) == false {
return 0, iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: object}, nil)
}
objectPath := filepath.Join(fs.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
}
default:
{
if os.IsNotExist(err) {
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
return 0, iodine.New(err, nil)
}
}
file, err := os.Open(objectPath)
if err != nil {
return 0, iodine.New(err, nil)
}
defer file.Close()
_, err = file.Seek(start, os.SEEK_SET)
if err != nil {
return 0, iodine.New(err, nil)
}
count, err := io.CopyN(w, file, length)
if err != nil {
return count, iodine.New(err, nil)
}
return count, nil
}
// GetObject - GET object from key
func (fs *fsDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// validate object
if drivers.IsValidObjectName(object) == false {
return 0, iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: object}, nil)
}
objectPath := filepath.Join(fs.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
}
default:
{
if os.IsNotExist(err) {
return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
return 0, iodine.New(err, nil)
}
}
file, err := os.Open(objectPath)
defer file.Close()
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.Copy(w, file)
if err != nil {
return count, iodine.New(err, nil)
}
return count, nil
}
// GetObjectMetadata - HEAD object
func (fs *fsDriver) GetObjectMetadata(bucket, object string) (drivers.ObjectMetadata, error) {
if drivers.IsValidBucket(bucket) == false {
return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
if drivers.IsValidObjectName(object) == false {
return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket}, nil)
}
// Do not use filepath.Join() since filepath.Join strips off any object names with '/', use them as is
// in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat()
objectPath := fs.root + "/" + bucket + "/" + object
stat, err := os.Stat(objectPath)
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil)
}
file, err := os.Open(objectPath + "$metadata")
defer file.Close()
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, nil)
}
var deserializedMetadata Metadata
decoder := json.NewDecoder(file)
err = decoder.Decode(&deserializedMetadata)
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, nil)
}
contentType := "application/octet-stream"
if deserializedMetadata.ContentType != "" {
contentType = deserializedMetadata.ContentType
}
contentType = strings.TrimSpace(contentType)
etag := bucket + "#" + filepath.Base(object)
if len(deserializedMetadata.Md5sum) != 0 {
etag = hex.EncodeToString(deserializedMetadata.Md5sum)
}
metadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: object,
Created: stat.ModTime(),
Size: stat.Size(),
Md5: etag,
ContentType: contentType,
}
return metadata, nil
}
// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
if strings.TrimSpace(expectedMD5Sum) != "" && strings.TrimSpace(actualMD5Sum) != "" {
expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
if err != nil {
return iodine.New(err, nil)
}
actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
if err != nil {
return iodine.New(err, nil)
}
if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
return iodine.New(errors.New("bad digest, md5sum mismatch"), nil)
}
return nil
}
return iodine.New(errors.New("invalid argument"), nil)
}
// CreateObject - PUT object
func (fs *fsDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
fs.lock.Lock()
defer fs.lock.Unlock()
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil)
}
// check bucket exists
if _, err := os.Stat(filepath.Join(fs.root, bucket)); os.IsNotExist(err) {
return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil)
}
// verify object path legal
if drivers.IsValidObjectName(key) == false {
return "", iodine.New(drivers.ObjectNameInvalid{Bucket: bucket, Object: key}, nil)
}
// verify content type
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := filepath.Join(fs.root, bucket, key)
objectDir := filepath.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return "", iodine.New(err, nil)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return "", iodine.New(drivers.ObjectExists{
Bucket: bucket,
Object: key,
}, nil)
}
if strings.TrimSpace(expectedMD5Sum) != "" {
expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
if err != nil {
// pro-actively close the connection
return "", iodine.New(drivers.InvalidDigest{Md5: expectedMD5Sum}, nil)
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
// write object
file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer file.Close()
h := md5.New()
mw := io.MultiWriter(file, h)
_, err = io.CopyN(mw, data, size)
if err != nil {
return "", iodine.New(err, nil)
}
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return "", iodine.New(err, nil)
}
defer file.Close()
metadata := &Metadata{
ContentType: contentType,
Md5sum: h.Sum(nil),
}
// serialize metadata to json
encoder := json.NewEncoder(file)
err = encoder.Encode(metadata)
md5Sum := hex.EncodeToString(metadata.Md5sum)
// Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" {
if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum); err != nil {
return "", iodine.New(drivers.BadDigest{Md5: expectedMD5Sum, Bucket: bucket, Key: key}, nil)
}
}
return md5Sum, nil
}

54
pkg/storage/drivers/fs/fs_test.go
Unescape View File

@ -1,54 +0,0 @@
/*
* Mini Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package filesystem
import (
"io/ioutil"
"os"
"testing"
. "github.com/minio/check"
"github.com/minio/minio/pkg/storage/drivers"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
c.Check(err, IsNil)
storageList = append(storageList, path)
store, err := NewDriver(path)
c.Check(err, IsNil)
return store
}
drivers.APITestSuite(c, create)
defer removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}
Write Preview
Loading…
Cancel
Save