From c23fa2683060755c468a4cb38eb9fd2fd1ef5687 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Wed, 21 Jan 2015 12:44:09 -0800 Subject: [PATCH] Extracting storage api to interface --- pkg/server/server.go | 9 +- pkg/storage/inmemory/inmemory.go | 152 +++++++++++++++++++++++++++++++ pkg/storage/storage.go | 151 ++---------------------------- pkg/storage/storage_errors.go | 20 ++-- pkg/webapi/minioapi/minioapi.go | 4 +- 5 files changed, 179 insertions(+), 157 deletions(-) create mode 100644 pkg/storage/inmemory/inmemory.go diff --git a/pkg/server/server.go b/pkg/server/server.go index f532d3e91..0f6992736 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,7 +21,8 @@ import ( "reflect" "github.com/minio-io/minio/pkg/httpserver" - storageModule "github.com/minio-io/minio/pkg/storage" + mstorage "github.com/minio-io/minio/pkg/storage" + "github.com/minio-io/minio/pkg/storage/inmemory" "github.com/minio-io/minio/pkg/webapi/minioapi" ) @@ -29,7 +30,11 @@ func Start() { var ctrlChans []chan<- string var statusChans []<-chan error - ctrlChan, statusChan, storage := storageModule.Start() + var ctrlChan chan<- string + var statusChan <-chan error + var storage mstorage.Storage + + ctrlChan, statusChan, storage = inmemory.Start() ctrlChans = append(ctrlChans, ctrlChan) statusChans = append(statusChans, statusChan) diff --git a/pkg/storage/inmemory/inmemory.go b/pkg/storage/inmemory/inmemory.go new file mode 100644 index 000000000..913542ca3 --- /dev/null +++ b/pkg/storage/inmemory/inmemory.go @@ -0,0 +1,152 @@ +package inmemory + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io" + "log" + "strings" + "time" + + mstorage "github.com/minio-io/minio/pkg/storage" +) + +type Storage struct { + bucketdata map[string]storedBucket + objectdata map[string]storedObject +} + +type storedBucket struct { + metadata mstorage.BucketMetadata + // owner string // TODO + // id string // TODO +} + +type storedObject struct { + metadata mstorage.ObjectMetadata + data []byte +} + +func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { + // TODO synchronize access + // get object + key := bucket + ":" + object + if val, ok := storage.objectdata[key]; ok { + objectBuffer := bytes.NewBuffer(val.data) + written, err := io.Copy(w, objectBuffer) + return written, err + } else { + return 0, mstorage.ObjectNotFound{Bucket: bucket, Path: object} + } +} + +func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error { + objectKey := bucket + ":" + key + if _, ok := storage.objectdata[objectKey]; ok == true { + return mstorage.ObjectExists{Bucket: bucket, Key: key} + } + var bytesBuffer bytes.Buffer + newObject := storedObject{} + if _, ok := io.Copy(&bytesBuffer, data); ok == nil { + size := bytesBuffer.Len() + etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) + newObject.metadata = mstorage.ObjectMetadata{ + Key: key, + SecCreated: time.Now().Unix(), + Size: size, + ETag: etag, + } + newObject.data = bytesBuffer.Bytes() + } + storage.objectdata[objectKey] = newObject + return nil +} + +func (storage *Storage) StoreBucket(bucketName string) error { + if !isValidBucket(bucketName) { + return mstorage.BucketNameInvalid{Bucket: bucketName} + } + + if _, ok := storage.bucketdata[bucketName]; ok == true { + return mstorage.BucketExists{Bucket: bucketName} + } + newBucket := storedBucket{} + newBucket.metadata = mstorage.BucketMetadata{ + Name: bucketName, + Created: time.Now().Unix(), + } + log.Println(bucketName) + storage.bucketdata[bucketName] = newBucket + return nil +} + +func (storage *Storage) ListObjects(bucket, prefix string, count int) []mstorage.ObjectMetadata { + // TODO prefix and count handling + var results []mstorage.ObjectMetadata + for key, object := range storage.objectdata { + if strings.HasPrefix(key, bucket+":") { + results = append(results, object.metadata) + } + } + return results +} + +func (storage *Storage) ListBuckets(prefix string) []mstorage.BucketMetadata { + // TODO prefix handling + var results []mstorage.BucketMetadata + for _, bucket := range storage.bucketdata { + results = append(results, bucket.metadata) + } + return results +} + +func Start() (chan<- string, <-chan error, *Storage) { + ctrlChannel := make(chan string) + errorChannel := make(chan error) + go start(ctrlChannel, errorChannel) + return ctrlChannel, errorChannel, &Storage{ + bucketdata: make(map[string]storedBucket), + objectdata: make(map[string]storedObject), + } +} + +func start(ctrlChannel <-chan string, errorChannel chan<- error) { + close(errorChannel) +} + +func isValidBucket(bucket string) bool { + l := len(bucket) + if l < 3 || l > 63 { + return false + } + + valid := false + prev := byte('.') + for i := 0; i < len(bucket); i++ { + c := bucket[i] + switch { + default: + return false + case 'a' <= c && c <= 'z': + valid = true + case '0' <= c && c <= '9': + // Is allowed, but bucketname can't be just numbers. + // Therefore, don't set valid to true + case c == '-': + if prev == '.' { + return false + } + case c == '.': + if prev == '.' || prev == '-' { + return false + } + } + prev = c + } + + if prev == '-' || prev == '.' { + return false + } + return valid +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 3978bd989..226e0e4b9 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -17,30 +17,18 @@ package storage import ( - "bytes" - "fmt" "io" - "log" - "strings" - "time" - - "crypto/sha256" ) -type Storage struct { - bucketdata map[string]storedBucket - objectdata map[string]storedObject -} - -type storedObject struct { - metadata ObjectMetadata - data []byte -} +type Storage interface { + // Bucket Operations + StoreBucket(bucket string) error + ListBuckets(prefix string) []BucketMetadata -type storedBucket struct { - metadata BucketMetadata - // owner string // TODO - // id string // TODO + // Object Operations + CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) + StoreObject(bucket string, key string, data io.Reader) error + ListObjects(bucket, prefix string, count int) []ObjectMetadata } type BucketMetadata struct { @@ -54,126 +42,3 @@ type ObjectMetadata struct { Size int ETag string } - -func isValidBucket(bucket string) bool { - l := len(bucket) - if l < 3 || l > 63 { - return false - } - - valid := false - prev := byte('.') - for i := 0; i < len(bucket); i++ { - c := bucket[i] - switch { - default: - return false - case 'a' <= c && c <= 'z': - valid = true - case '0' <= c && c <= '9': - // Is allowed, but bucketname can't be just numbers. - // Therefore, don't set valid to true - case c == '-': - if prev == '.' { - return false - } - case c == '.': - if prev == '.' || prev == '-' { - return false - } - } - prev = c - } - - if prev == '-' || prev == '.' { - return false - } - return valid -} - -func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { - // TODO synchronize access - // get object - key := bucket + ":" + object - if val, ok := storage.objectdata[key]; ok { - objectBuffer := bytes.NewBuffer(val.data) - written, err := io.Copy(w, objectBuffer) - return written, err - } else { - return 0, ObjectNotFound{bucket: bucket, path: object} - } -} - -func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error { - objectKey := bucket + ":" + key - if _, ok := storage.objectdata[objectKey]; ok == true { - return ObjectExists{bucket: bucket, key: key} - } - var bytesBuffer bytes.Buffer - newObject := storedObject{} - if _, ok := io.Copy(&bytesBuffer, data); ok == nil { - size := bytesBuffer.Len() - etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) - newObject.metadata = ObjectMetadata{ - Key: key, - SecCreated: time.Now().Unix(), - Size: size, - ETag: etag, - } - newObject.data = bytesBuffer.Bytes() - } - storage.objectdata[objectKey] = newObject - return nil -} - -func (storage *Storage) StoreBucket(bucketName string) error { - if !isValidBucket(bucketName) { - return BucketNameInvalid{bucket: bucketName} - } - - if _, ok := storage.bucketdata[bucketName]; ok == true { - return BucketExists{bucket: bucketName} - } - newBucket := storedBucket{} - newBucket.metadata = BucketMetadata{ - Name: bucketName, - Created: time.Now().Unix(), - } - log.Println(bucketName) - storage.bucketdata[bucketName] = newBucket - return nil -} - -func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMetadata { - // TODO prefix and count handling - var results []ObjectMetadata - for key, object := range storage.objectdata { - if strings.HasPrefix(key, bucket+":") { - results = append(results, object.metadata) - } - } - return results -} - -func (storage *Storage) ListBuckets(prefix string) []BucketMetadata { - // TODO prefix handling - var results []BucketMetadata - for _, bucket := range storage.bucketdata { - results = append(results, bucket.metadata) - } - return results -} - -func Start() (chan<- string, <-chan error, *Storage) { - ctrlChannel := make(chan string) - errorChannel := make(chan error) - go start(ctrlChannel, errorChannel) - return ctrlChannel, errorChannel, &Storage{ - bucketdata: make(map[string]storedBucket), - objectdata: make(map[string]storedObject), - } -} - -func start(ctrlChannel <-chan string, errorChannel chan<- error) { - close(errorChannel) -} diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go index 84eb3c916..a6ddfd865 100644 --- a/pkg/storage/storage_errors.go +++ b/pkg/storage/storage_errors.go @@ -1,37 +1,37 @@ package storage type GenericError struct { - bucket string - path string + Bucket string + Path string } type ObjectExists struct { - bucket string - key string + Bucket string + Key string } type BucketNameInvalid struct { - bucket string + Bucket string } type BucketExists struct { - bucket string + Bucket string } type ObjectNotFound GenericError func (self ObjectNotFound) Error() string { - return "Object not Found: " + self.bucket + "#" + self.path + return "Object not Found: " + self.Bucket + "#" + self.Path } func (self ObjectExists) Error() string { - return "Object exists: " + self.bucket + "#" + self.key + return "Object exists: " + self.Bucket + "#" + self.Key } func (self BucketNameInvalid) Error() string { - return "Bucket name invalid: " + self.bucket + return "Bucket name invalid: " + self.Bucket } func (self BucketExists) Error() string { - return "Bucket exists: " + self.bucket + return "Bucket exists: " + self.Bucket } diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index 5089ed6db..5afde58e1 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -29,14 +29,14 @@ import ( ) type minioApi struct { - storage *mstorage.Storage + storage mstorage.Storage } type encoder interface { Encode(v interface{}) error } -func HttpHandler(storage *mstorage.Storage) http.Handler { +func HttpHandler(storage mstorage.Storage) http.Handler { mux := mux.NewRouter() api := minioApi{ storage: storage,