- Drivers contain * donut/* * file/* * memory/* - Storage format contains * donut/* - GetObject() --> renamed to GetObjectReader() - Deleted stale objectwriter.go, renamed donutwriter.go to object_writer.go Simplifying, and documenting codebase furthermaster
parent
6dd7a9d654
commit
0474439b43
@ -1,4 +1,20 @@ |
||||
package storage |
||||
/* |
||||
* Minimalist Object Storage, (C) 2015 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package drivers |
||||
|
||||
import ( |
||||
"errors" |
@ -0,0 +1,133 @@ |
||||
/* |
||||
* Minimalist Object File, (C) 2015 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package file |
||||
|
||||
import ( |
||||
"os" |
||||
"path" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"io/ioutil" |
||||
"path/filepath" |
||||
|
||||
"github.com/minio-io/minio/pkg/drivers" |
||||
) |
||||
|
||||
/// Bucket Operations
|
||||
|
||||
// ListBuckets - Get service
|
||||
func (file *fileDriver) ListBuckets() ([]drivers.BucketMetadata, error) { |
||||
files, err := ioutil.ReadDir(file.root) |
||||
if err != nil { |
||||
return []drivers.BucketMetadata{}, drivers.EmbedError("bucket", "", err) |
||||
} |
||||
|
||||
var metadataList []drivers.BucketMetadata |
||||
for _, fileName := range files { |
||||
// Skip policy files
|
||||
if strings.HasSuffix(fileName.Name(), "_policy.json") { |
||||
continue |
||||
} |
||||
if !fileName.IsDir() { |
||||
return []drivers.BucketMetadata{}, drivers.BackendCorrupted{Path: file.root} |
||||
} |
||||
metadata := drivers.BucketMetadata{ |
||||
Name: fileName.Name(), |
||||
Created: fileName.ModTime(), // TODO - provide real created time
|
||||
} |
||||
metadataList = append(metadataList, metadata) |
||||
} |
||||
return metadataList, nil |
||||
} |
||||
|
||||
// CreateBucket - PUT Bucket
|
||||
func (file *fileDriver) CreateBucket(bucket string) error { |
||||
file.lock.Lock() |
||||
defer file.lock.Unlock() |
||||
|
||||
// verify bucket path legal
|
||||
if drivers.IsValidBucket(bucket) == false { |
||||
return drivers.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(file.root, bucket) |
||||
|
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err == nil { |
||||
return drivers.BucketExists{ |
||||
Bucket: bucket, |
||||
} |
||||
} |
||||
|
||||
// make bucket
|
||||
err := os.Mkdir(bucketDir, 0700) |
||||
if err != nil { |
||||
return drivers.EmbedError(bucket, "", err) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// ListObjects - GET bucket (list objects)
|
||||
func (file *fileDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { |
||||
p := bucketDir{} |
||||
p.files = make(map[string]os.FileInfo) |
||||
|
||||
if drivers.IsValidBucket(bucket) == false { |
||||
return []drivers.ObjectMetadata{}, resources, drivers.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
if resources.Prefix != "" && drivers.IsValidObject(resources.Prefix) == false { |
||||
return []drivers.ObjectMetadata{}, resources, drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix} |
||||
} |
||||
|
||||
rootPrefix := path.Join(file.root, bucket) |
||||
// check bucket exists
|
||||
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) { |
||||
return []drivers.ObjectMetadata{}, resources, drivers.BucketNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
p.root = rootPrefix |
||||
err := filepath.Walk(rootPrefix, p.getAllFiles) |
||||
if err != nil { |
||||
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err) |
||||
} |
||||
|
||||
var metadataList []drivers.ObjectMetadata |
||||
var metadata drivers.ObjectMetadata |
||||
|
||||
// Populate filtering mode
|
||||
resources.Mode = drivers.GetMode(resources) |
||||
|
||||
for name, f := range p.files { |
||||
if len(metadataList) >= resources.Maxkeys { |
||||
resources.IsTruncated = true |
||||
goto ret |
||||
} |
||||
metadata, resources, err = file.filter(bucket, name, f, resources) |
||||
if err != nil { |
||||
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err) |
||||
} |
||||
if metadata.Bucket != "" { |
||||
metadataList = append(metadataList, metadata) |
||||
} |
||||
} |
||||
|
||||
ret: |
||||
sort.Sort(byObjectKey(metadataList)) |
||||
return metadataList, resources, nil |
||||
} |
@ -0,0 +1,111 @@ |
||||
/* |
||||
* Minimalist Object Storage, (C) 2015 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package file |
||||
|
||||
import ( |
||||
"os" |
||||
"path" |
||||
|
||||
"encoding/json" |
||||
"github.com/minio-io/minio/pkg/drivers" |
||||
) |
||||
|
||||
// GetBucketPolicy - GET bucket policy
|
||||
func (file *fileDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) { |
||||
file.lock.Lock() |
||||
defer file.lock.Unlock() |
||||
|
||||
var p drivers.BucketPolicy |
||||
// verify bucket path legal
|
||||
if drivers.IsValidBucket(bucket) == false { |
||||
return drivers.BucketPolicy{}, drivers.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(file.root, bucket) |
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err != nil { |
||||
return drivers.BucketPolicy{}, drivers.BucketNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
// get policy path
|
||||
bucketPolicy := path.Join(file.root, bucket+"_policy.json") |
||||
filestat, err := os.Stat(bucketPolicy) |
||||
|
||||
if os.IsNotExist(err) { |
||||
return drivers.BucketPolicy{}, drivers.BucketPolicyNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
if filestat.IsDir() { |
||||
return drivers.BucketPolicy{}, drivers.BackendCorrupted{Path: bucketPolicy} |
||||
} |
||||
|
||||
f, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666) |
||||
defer f.Close() |
||||
if err != nil { |
||||
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err) |
||||
} |
||||
encoder := json.NewDecoder(f) |
||||
err = encoder.Decode(&p) |
||||
if err != nil { |
||||
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err) |
||||
} |
||||
|
||||
return p, nil |
||||
|
||||
} |
||||
|
||||
// CreateBucketPolicy - PUT bucket policy
|
||||
func (file *fileDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error { |
||||
file.lock.Lock() |
||||
defer file.lock.Unlock() |
||||
|
||||
// verify bucket path legal
|
||||
if drivers.IsValidBucket(bucket) == false { |
||||
return drivers.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(file.root, bucket) |
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err != nil { |
||||
return drivers.BucketNotFound{ |
||||
Bucket: bucket, |
||||
} |
||||
} |
||||
|
||||
// get policy path
|
||||
bucketPolicy := path.Join(file.root, bucket+"_policy.json") |
||||
filestat, ret := os.Stat(bucketPolicy) |
||||
if !os.IsNotExist(ret) { |
||||
if filestat.IsDir() { |
||||
return drivers.BackendCorrupted{Path: bucketPolicy} |
||||
} |
||||
} |
||||
|
||||
f, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600) |
||||
defer f.Close() |
||||
if err != nil { |
||||
return drivers.EmbedError(bucket, "", err) |
||||
} |
||||
encoder := json.NewEncoder(f) |
||||
err = encoder.Encode(p) |
||||
if err != nil { |
||||
return drivers.EmbedError(bucket, "", err) |
||||
} |
||||
return nil |
||||
} |
@ -1,11 +1,12 @@ |
||||
package donut |
||||
|
||||
type bucketDriver struct { |
||||
type donutBucket struct { |
||||
nodes []string |
||||
objects map[string][]byte |
||||
} |
||||
|
||||
func (b bucketDriver) GetNodes() ([]string, error) { |
||||
// GetNodes - get list of associated nodes for a given bucket
|
||||
func (b donutBucket) GetNodes() ([]string, error) { |
||||
var nodes []string |
||||
for _, node := range b.nodes { |
||||
nodes = append(nodes, node) |
@ -1,50 +1,160 @@ |
||||
package donut |
||||
|
||||
import ( |
||||
"errors" |
||||
"io" |
||||
"sort" |
||||
"strconv" |
||||
"strings" |
||||
) |
||||
|
||||
// INTERFACES
|
||||
type donut struct { |
||||
buckets map[string]Bucket |
||||
nodes map[string]Node |
||||
} |
||||
|
||||
// NewDonut - instantiate new donut
|
||||
func NewDonut(root string) Donut { |
||||
nodes := make(map[string]Node) |
||||
nodes["localhost"] = localDirectoryNode{root: root} |
||||
d := donut{ |
||||
buckets: make(map[string]Bucket), |
||||
nodes: nodes, |
||||
} |
||||
return d |
||||
} |
||||
|
||||
// Donut interface
|
||||
type Donut interface { |
||||
CreateBucket(bucket string) error |
||||
GetObject(bucket, object string) (io.ReadCloser, error) |
||||
GetObjectMetadata(bucket, object string) (map[string]string, error) |
||||
GetObjectWriter(bucket, object string) (ObjectWriter, error) |
||||
ListBuckets() ([]string, error) |
||||
ListObjects(bucket string) ([]string, error) |
||||
// CreateBucket - create a new bucket
|
||||
func (d donut) CreateBucket(bucketName string) error { |
||||
if _, ok := d.buckets[bucketName]; ok == false { |
||||
bucketName = strings.TrimSpace(bucketName) |
||||
if bucketName == "" { |
||||
return errors.New("Cannot create bucket with no name") |
||||
} |
||||
// assign nodes
|
||||
// TODO assign other nodes
|
||||
nodes := make([]string, 16) |
||||
for i := 0; i < 16; i++ { |
||||
nodes[i] = "localhost" |
||||
if node, ok := d.nodes["localhost"]; ok { |
||||
node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i)) |
||||
} |
||||
} |
||||
bucket := donutBucket{ |
||||
nodes: nodes, |
||||
} |
||||
d.buckets[bucketName] = bucket |
||||
return nil |
||||
} |
||||
return errors.New("Bucket exists") |
||||
} |
||||
|
||||
// Bucket interface
|
||||
type Bucket interface { |
||||
GetNodes() ([]string, error) |
||||
// ListBuckets - list all buckets
|
||||
func (d donut) ListBuckets() ([]string, error) { |
||||
var buckets []string |
||||
for bucket := range d.buckets { |
||||
buckets = append(buckets, bucket) |
||||
} |
||||
sort.Strings(buckets) |
||||
return buckets, nil |
||||
} |
||||
|
||||
// Node interface
|
||||
type Node interface { |
||||
CreateBucket(bucket string) error |
||||
GetBuckets() ([]string, error) |
||||
GetDonutMetadata(bucket, object string) (map[string]string, error) |
||||
GetMetadata(bucket, object string) (map[string]string, error) |
||||
GetReader(bucket, object string) (io.ReadCloser, error) |
||||
GetWriter(bucket, object string) (Writer, error) |
||||
ListObjects(bucket string) ([]string, error) |
||||
// GetObjectWriter - get a new writer interface for a new object
|
||||
func (d donut) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) { |
||||
if bucket, ok := d.buckets[bucketName]; ok == true { |
||||
writers := make([]Writer, 16) |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
for i, nodeID := range nodes { |
||||
if node, ok := d.nodes[nodeID]; ok == true { |
||||
writer, err := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName) |
||||
if err != nil { |
||||
for _, writerToClose := range writers { |
||||
if writerToClose != nil { |
||||
writerToClose.CloseWithError(err) |
||||
} |
||||
} |
||||
return nil, err |
||||
} |
||||
writers[i] = writer |
||||
} |
||||
} |
||||
return newErasureWriter(writers), nil |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
// ObjectWriter interface
|
||||
type ObjectWriter interface { |
||||
Close() error |
||||
CloseWithError(error) error |
||||
GetMetadata() (map[string]string, error) |
||||
SetMetadata(map[string]string) error |
||||
Write([]byte) (int, error) |
||||
// GetObjectReader - get a new reader interface for a new object
|
||||
func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, error) { |
||||
r, w := io.Pipe() |
||||
if bucket, ok := d.buckets[bucketName]; ok == true { |
||||
readers := make([]io.ReadCloser, 16) |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
var metadata map[string]string |
||||
for i, nodeID := range nodes { |
||||
if node, ok := d.nodes[nodeID]; ok == true { |
||||
bucketID := bucketName + ":0:" + strconv.Itoa(i) |
||||
reader, err := node.GetReader(bucketID, objectName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
readers[i] = reader |
||||
if metadata == nil { |
||||
metadata, err = node.GetDonutMetadata(bucketID, objectName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
} |
||||
} |
||||
go erasureReader(readers, metadata, w) |
||||
return r, nil |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
// Writer interface
|
||||
type Writer interface { |
||||
ObjectWriter |
||||
// GetObjectMetadata returns metadata for a given object in a bucket
|
||||
func (d donut) GetObjectMetadata(bucketName, object string) (map[string]string, error) { |
||||
if bucket, ok := d.buckets[bucketName]; ok { |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if node, ok := d.nodes[nodes[0]]; ok { |
||||
bucketID := bucketName + ":0:0" |
||||
metadata, err := node.GetMetadata(bucketID, object) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
donutMetadata, err := node.GetDonutMetadata(bucketID, object) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
metadata["sys.created"] = donutMetadata["created"] |
||||
metadata["sys.md5"] = donutMetadata["md5"] |
||||
metadata["sys.size"] = donutMetadata["size"] |
||||
return metadata, nil |
||||
} |
||||
return nil, errors.New("Cannot connect to node: " + nodes[0]) |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
GetDonutMetadata() (map[string]string, error) |
||||
SetDonutMetadata(map[string]string) error |
||||
// ListObjects - list all the available objects in a bucket
|
||||
func (d donut) ListObjects(bucketName string) ([]string, error) { |
||||
if bucket, ok := d.buckets[bucketName]; ok { |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if node, ok := d.nodes[nodes[0]]; ok { |
||||
return node.ListObjects(bucketName + ":0:0") |
||||
} |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
@ -1,155 +0,0 @@ |
||||
package donut |
||||
|
||||
import ( |
||||
"errors" |
||||
"io" |
||||
"sort" |
||||
"strconv" |
||||
"strings" |
||||
) |
||||
|
||||
type donutDriver struct { |
||||
buckets map[string]Bucket |
||||
nodes map[string]Node |
||||
} |
||||
|
||||
// NewDonutDriver - instantiate new donut driver
|
||||
func NewDonutDriver(root string) Donut { |
||||
nodes := make(map[string]Node) |
||||
nodes["localhost"] = localDirectoryNode{root: root} |
||||
driver := donutDriver{ |
||||
buckets: make(map[string]Bucket), |
||||
nodes: nodes, |
||||
} |
||||
return driver |
||||
} |
||||
|
||||
func (driver donutDriver) CreateBucket(bucketName string) error { |
||||
if _, ok := driver.buckets[bucketName]; ok == false { |
||||
bucketName = strings.TrimSpace(bucketName) |
||||
if bucketName == "" { |
||||
return errors.New("Cannot create bucket with no name") |
||||
} |
||||
// assign nodes
|
||||
// TODO assign other nodes
|
||||
nodes := make([]string, 16) |
||||
for i := 0; i < 16; i++ { |
||||
nodes[i] = "localhost" |
||||
if node, ok := driver.nodes["localhost"]; ok { |
||||
node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i)) |
||||
} |
||||
} |
||||
bucket := bucketDriver{ |
||||
nodes: nodes, |
||||
} |
||||
driver.buckets[bucketName] = bucket |
||||
return nil |
||||
} |
||||
return errors.New("Bucket exists") |
||||
} |
||||
|
||||
func (driver donutDriver) ListBuckets() ([]string, error) { |
||||
var buckets []string |
||||
for bucket := range driver.buckets { |
||||
buckets = append(buckets, bucket) |
||||
} |
||||
sort.Strings(buckets) |
||||
return buckets, nil |
||||
} |
||||
|
||||
func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) { |
||||
if bucket, ok := driver.buckets[bucketName]; ok == true { |
||||
writers := make([]Writer, 16) |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
for i, nodeID := range nodes { |
||||
if node, ok := driver.nodes[nodeID]; ok == true { |
||||
writer, err := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName) |
||||
if err != nil { |
||||
for _, writerToClose := range writers { |
||||
if writerToClose != nil { |
||||
writerToClose.CloseWithError(err) |
||||
} |
||||
} |
||||
return nil, err |
||||
} |
||||
writers[i] = writer |
||||
} |
||||
} |
||||
return newErasureWriter(writers), nil |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
func (driver donutDriver) GetObject(bucketName, objectName string) (io.ReadCloser, error) { |
||||
r, w := io.Pipe() |
||||
if bucket, ok := driver.buckets[bucketName]; ok == true { |
||||
readers := make([]io.ReadCloser, 16) |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
var metadata map[string]string |
||||
for i, nodeID := range nodes { |
||||
if node, ok := driver.nodes[nodeID]; ok == true { |
||||
bucketID := bucketName + ":0:" + strconv.Itoa(i) |
||||
reader, err := node.GetReader(bucketID, objectName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
readers[i] = reader |
||||
if metadata == nil { |
||||
metadata, err = node.GetDonutMetadata(bucketID, objectName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
} |
||||
} |
||||
go erasureReader(readers, metadata, w) |
||||
return r, nil |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
// GetObjectMetadata returns metadata for a given object in a bucket
|
||||
func (driver donutDriver) GetObjectMetadata(bucketName, object string) (map[string]string, error) { |
||||
if bucket, ok := driver.buckets[bucketName]; ok { |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if node, ok := driver.nodes[nodes[0]]; ok { |
||||
bucketID := bucketName + ":0:0" |
||||
metadata, err := node.GetMetadata(bucketID, object) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
donutMetadata, err := node.GetDonutMetadata(bucketID, object) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
metadata["sys.created"] = donutMetadata["created"] |
||||
metadata["sys.md5"] = donutMetadata["md5"] |
||||
metadata["sys.size"] = donutMetadata["size"] |
||||
return metadata, nil |
||||
} |
||||
return nil, errors.New("Cannot connect to node: " + nodes[0]) |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
||||
|
||||
func (driver donutDriver) ListObjects(bucketName string) ([]string, error) { |
||||
if bucket, ok := driver.buckets[bucketName]; ok { |
||||
nodes, err := bucket.GetNodes() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if node, ok := driver.nodes[nodes[0]]; ok { |
||||
return node.ListObjects(bucketName + ":0:0") |
||||
} |
||||
} |
||||
return nil, errors.New("Bucket not found") |
||||
} |
@ -0,0 +1,50 @@ |
||||
package donut |
||||
|
||||
import ( |
||||
"io" |
||||
) |
||||
|
||||
// Collection of Donut specification interfaces
|
||||
|
||||
// Donut interface
|
||||
type Donut interface { |
||||
CreateBucket(bucket string) error |
||||
GetObjectReader(bucket, object string) (io.ReadCloser, error) |
||||
GetObjectWriter(bucket, object string) (ObjectWriter, error) |
||||
GetObjectMetadata(bucket, object string) (map[string]string, error) |
||||
ListBuckets() ([]string, error) |
||||
ListObjects(bucket string) ([]string, error) |
||||
} |
||||
|
||||
// Bucket interface
|
||||
type Bucket interface { |
||||
GetNodes() ([]string, error) |
||||
} |
||||
|
||||
// Node interface
|
||||
type Node interface { |
||||
CreateBucket(bucket string) error |
||||
GetBuckets() ([]string, error) |
||||
GetDonutMetadata(bucket, object string) (map[string]string, error) |
||||
GetMetadata(bucket, object string) (map[string]string, error) |
||||
GetReader(bucket, object string) (io.ReadCloser, error) |
||||
GetWriter(bucket, object string) (Writer, error) |
||||
ListObjects(bucket string) ([]string, error) |
||||
} |
||||
|
||||
// ObjectWriter interface
|
||||
type ObjectWriter interface { |
||||
Close() error |
||||
CloseWithError(error) error |
||||
GetMetadata() (map[string]string, error) |
||||
SetMetadata(map[string]string) error |
||||
Write([]byte) (int, error) |
||||
} |
||||
|
||||
// Writer interface
|
||||
type Writer interface { |
||||
ObjectWriter |
||||
|
||||
GetDonutMetadata() (map[string]string, error) |
||||
SetDonutMetadata(map[string]string) error |
||||
} |
@ -1,39 +0,0 @@ |
||||
package donut |
||||
|
||||
import ( |
||||
"errors" |
||||
) |
||||
|
||||
type objectWriter struct { |
||||
metadata map[string]string |
||||
} |
||||
|
||||
func (obj objectWriter) Write(data []byte) (length int, err error) { |
||||
return 11, nil |
||||
} |
||||
|
||||
func (obj objectWriter) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
func (obj objectWriter) CloseWithError(err error) error { |
||||
return errors.New("Not Implemented") |
||||
} |
||||
|
||||
func (obj objectWriter) SetMetadata(metadata map[string]string) error { |
||||
for k := range obj.metadata { |
||||
delete(obj.metadata, k) |
||||
} |
||||
for k, v := range metadata { |
||||
obj.metadata[k] = v |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (obj objectWriter) GetMetadata() (map[string]string, error) { |
||||
ret := make(map[string]string) |
||||
for k, v := range obj.metadata { |
||||
ret[k] = v |
||||
} |
||||
return ret, nil |
||||
} |
@ -1,133 +0,0 @@ |
||||
/* |
||||
* Minimalist Object Storage, (C) 2015 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package file |
||||
|
||||
import ( |
||||
"os" |
||||
"path" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"io/ioutil" |
||||
"path/filepath" |
||||
|
||||
mstorage "github.com/minio-io/minio/pkg/storage" |
||||
) |
||||
|
||||
/// Bucket Operations
|
||||
|
||||
// ListBuckets - Get service
|
||||
func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) { |
||||
files, err := ioutil.ReadDir(storage.root) |
||||
if err != nil { |
||||
return []mstorage.BucketMetadata{}, mstorage.EmbedError("bucket", "", err) |
||||
} |
||||
|
||||
var metadataList []mstorage.BucketMetadata |
||||
for _, file := range files { |
||||
// Skip policy files
|
||||
if strings.HasSuffix(file.Name(), "_policy.json") { |
||||
continue |
||||
} |
||||
if !file.IsDir() { |
||||
return []mstorage.BucketMetadata{}, mstorage.BackendCorrupted{Path: storage.root} |
||||
} |
||||
metadata := mstorage.BucketMetadata{ |
||||
Name: file.Name(), |
||||
Created: file.ModTime(), // TODO - provide real created time
|
||||
} |
||||
metadataList = append(metadataList, metadata) |
||||
} |
||||
return metadataList, nil |
||||
} |
||||
|
||||
// CreateBucket - PUT Bucket
|
||||
func (storage *Storage) CreateBucket(bucket string) error { |
||||
storage.lock.Lock() |
||||
defer storage.lock.Unlock() |
||||
|
||||
// verify bucket path legal
|
||||
if mstorage.IsValidBucket(bucket) == false { |
||||
return mstorage.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(storage.root, bucket) |
||||
|
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err == nil { |
||||
return mstorage.BucketExists{ |
||||
Bucket: bucket, |
||||
} |
||||
} |
||||
|
||||
// make bucket
|
||||
err := os.Mkdir(bucketDir, 0700) |
||||
if err != nil { |
||||
return mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// ListObjects - GET bucket (list objects)
|
||||
func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) { |
||||
p := bucketDir{} |
||||
p.files = make(map[string]os.FileInfo) |
||||
|
||||
if mstorage.IsValidBucket(bucket) == false { |
||||
return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
if resources.Prefix != "" && mstorage.IsValidObject(resources.Prefix) == false { |
||||
return []mstorage.ObjectMetadata{}, resources, mstorage.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix} |
||||
} |
||||
|
||||
rootPrefix := path.Join(storage.root, bucket) |
||||
// check bucket exists
|
||||
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) { |
||||
return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
p.root = rootPrefix |
||||
err := filepath.Walk(rootPrefix, p.getAllFiles) |
||||
if err != nil { |
||||
return []mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
|
||||
var metadataList []mstorage.ObjectMetadata |
||||
var metadata mstorage.ObjectMetadata |
||||
|
||||
// Populate filtering mode
|
||||
resources.Mode = mstorage.GetMode(resources) |
||||
|
||||
for name, file := range p.files { |
||||
if len(metadataList) >= resources.Maxkeys { |
||||
resources.IsTruncated = true |
||||
goto ret |
||||
} |
||||
metadata, resources, err = storage.filter(bucket, name, file, resources) |
||||
if err != nil { |
||||
return []mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
if metadata.Bucket != "" { |
||||
metadataList = append(metadataList, metadata) |
||||
} |
||||
} |
||||
|
||||
ret: |
||||
sort.Sort(byObjectKey(metadataList)) |
||||
return metadataList, resources, nil |
||||
} |
@ -1,111 +0,0 @@ |
||||
/* |
||||
* Minimalist Object Storage, (C) 2015 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package file |
||||
|
||||
import ( |
||||
"os" |
||||
"path" |
||||
|
||||
"encoding/json" |
||||
mstorage "github.com/minio-io/minio/pkg/storage" |
||||
) |
||||
|
||||
// GetBucketPolicy - GET bucket policy
|
||||
func (storage *Storage) GetBucketPolicy(bucket string) (mstorage.BucketPolicy, error) { |
||||
storage.lock.Lock() |
||||
defer storage.lock.Unlock() |
||||
|
||||
var p mstorage.BucketPolicy |
||||
// verify bucket path legal
|
||||
if mstorage.IsValidBucket(bucket) == false { |
||||
return mstorage.BucketPolicy{}, mstorage.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(storage.root, bucket) |
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err != nil { |
||||
return mstorage.BucketPolicy{}, mstorage.BucketNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
// get policy path
|
||||
bucketPolicy := path.Join(storage.root, bucket+"_policy.json") |
||||
filestat, err := os.Stat(bucketPolicy) |
||||
|
||||
if os.IsNotExist(err) { |
||||
return mstorage.BucketPolicy{}, mstorage.BucketPolicyNotFound{Bucket: bucket} |
||||
} |
||||
|
||||
if filestat.IsDir() { |
||||
return mstorage.BucketPolicy{}, mstorage.BackendCorrupted{Path: bucketPolicy} |
||||
} |
||||
|
||||
file, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666) |
||||
defer file.Close() |
||||
if err != nil { |
||||
return mstorage.BucketPolicy{}, mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
encoder := json.NewDecoder(file) |
||||
err = encoder.Decode(&p) |
||||
if err != nil { |
||||
return mstorage.BucketPolicy{}, mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
|
||||
return p, nil |
||||
|
||||
} |
||||
|
||||
// CreateBucketPolicy - PUT bucket policy
|
||||
func (storage *Storage) CreateBucketPolicy(bucket string, p mstorage.BucketPolicy) error { |
||||
storage.lock.Lock() |
||||
defer storage.lock.Unlock() |
||||
|
||||
// verify bucket path legal
|
||||
if mstorage.IsValidBucket(bucket) == false { |
||||
return mstorage.BucketNameInvalid{Bucket: bucket} |
||||
} |
||||
|
||||
// get bucket path
|
||||
bucketDir := path.Join(storage.root, bucket) |
||||
// check if bucket exists
|
||||
if _, err := os.Stat(bucketDir); err != nil { |
||||
return mstorage.BucketNotFound{ |
||||
Bucket: bucket, |
||||
} |
||||
} |
||||
|
||||
// get policy path
|
||||
bucketPolicy := path.Join(storage.root, bucket+"_policy.json") |
||||
filestat, ret := os.Stat(bucketPolicy) |
||||
if !os.IsNotExist(ret) { |
||||
if filestat.IsDir() { |
||||
return mstorage.BackendCorrupted{Path: bucketPolicy} |
||||
} |
||||
} |
||||
|
||||
file, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600) |
||||
defer file.Close() |
||||
if err != nil { |
||||
return mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
encoder := json.NewEncoder(file) |
||||
err = encoder.Encode(p) |
||||
if err != nil { |
||||
return mstorage.EmbedError(bucket, "", err) |
||||
} |
||||
return nil |
||||
} |
Loading…
Reference in new issue