From 5b67da7d965fb736125c5192ee60e9c470d053de Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 21 Jan 2015 00:50:23 -0800 Subject: [PATCH] Add PutBucket and ListBuckets service --- Makefile | 5 +- pkg/httpserver/httpserver.go | 11 +++- pkg/storage/storage.go | 90 +++++++++++++++++++++++++++--- pkg/storage/storage_errors.go | 18 +++++- pkg/webapi/minioapi/definitions.go | 26 ++++++--- pkg/webapi/minioapi/minioapi.go | 72 +++++++++++++++++++++--- 6 files changed, 197 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 633480636..8b764fe1a 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,10 @@ build-storage: @$(MAKE) $(MAKE_OPTIONS) -C pkg/storage/erasure/isal lib @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/storage/erasure -cover: build-storage build-os build-utils +build-minioapi: + @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/webapi/minioapi + +cover: build-storage build-os build-utils build-minioapi install: cover diff --git a/pkg/httpserver/httpserver.go b/pkg/httpserver/httpserver.go index 15edca078..6fc5d971c 100644 --- a/pkg/httpserver/httpserver.go +++ b/pkg/httpserver/httpserver.go @@ -19,6 +19,7 @@ package httpserver import ( "log" "net/http" + "time" ) func Start(handler http.Handler, address string) (chan<- string, <-chan error) { @@ -30,7 +31,15 @@ func Start(handler http.Handler, address string) (chan<- string, <-chan error) { func start(ctrlChannel <-chan string, errorChannel chan<- error, router http.Handler, address string) { log.Println("Starting HTTP Server on " + address) - err := http.ListenAndServe(address, router) + // Minio server config + server := &http.Server{ + Addr: address, + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + err := server.ListenAndServe() errorChannel <- err close(errorChannel) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 0937ddbab..ea8301139 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -25,7 +25,8 @@ import ( ) type Storage struct { - data map[string]storedObject + bucketdata map[string]storedBucket + objectdata map[string]storedObject } type storedObject struct { @@ -33,17 +34,64 @@ type storedObject struct { data []byte } +type storedBucket struct { + metadata BucketMetadata + // owner string // TODO + // id string // TODO +} + +type BucketMetadata struct { + Name string + Created int64 +} + type ObjectMetadata struct { Key string SecCreated int64 Size int } +func isValidBucket(bucket string) bool { + l := len(bucket) + if l < 3 || l > 63 { + return false + } + + valid := false + prev := byte('.') + for i := 0; i < len(bucket); i++ { + c := bucket[i] + switch { + default: + return false + case 'a' <= c && c <= 'z': + valid = true + case '0' <= c && c <= '9': + // Is allowed, but bucketname can't be just numbers. + // Therefore, don't set valid to true + case c == '-': + if prev == '.' { + return false + } + case c == '.': + if prev == '.' || prev == '-' { + return false + } + } + prev = c + } + + if prev == '-' || prev == '.' { + return false + } + return valid +} + func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // TODO synchronize access // get object key := bucket + ":" + object - if val, ok := storage.data[key]; ok { + if val, ok := storage.objectdata[key]; ok { objectBuffer := bytes.NewBuffer(val.data) written, err := io.Copy(w, objectBuffer) return written, err @@ -54,7 +102,7 @@ func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object st func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error { objectKey := bucket + ":" + key - if _, ok := storage.data[objectKey]; ok == true { + if _, ok := storage.objectdata[objectKey]; ok == true { return ObjectExists{bucket: bucket, key: key} } var bytesBuffer bytes.Buffer @@ -67,14 +115,32 @@ func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) e } newObject.data = bytesBuffer.Bytes() } - storage.data[objectKey] = newObject + storage.objectdata[objectKey] = newObject + return nil +} + +func (storage *Storage) StoreBucket(bucketName string) error { + if !isValidBucket(bucketName) { + return BucketNameInvalid{bucket: bucketName} + } + + if _, ok := storage.bucketdata[bucketName]; ok == true { + return BucketExists{bucket: bucketName} + } + newBucket := storedBucket{} + newBucket.metadata = BucketMetadata{ + Name: bucketName, + Created: time.Now().Unix(), + } + log.Println(bucketName) + storage.bucketdata[bucketName] = newBucket return nil } func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMetadata { + // TODO prefix and count handling var results []ObjectMetadata - for key, object := range storage.data { - log.Println(key) + for key, object := range storage.objectdata { if strings.HasPrefix(key, bucket+":") { results = append(results, object.metadata) } @@ -82,12 +148,22 @@ func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMe return results } +func (storage *Storage) ListBuckets(prefix string) []BucketMetadata { + // TODO prefix handling + var results []BucketMetadata + for _, bucket := range storage.bucketdata { + results = append(results, bucket.metadata) + } + return results +} + func Start() (chan<- string, <-chan error, *Storage) { ctrlChannel := make(chan string) errorChannel := make(chan error) go start(ctrlChannel, errorChannel) return ctrlChannel, errorChannel, &Storage{ - data: make(map[string]storedObject), + bucketdata: make(map[string]storedBucket), + objectdata: make(map[string]storedObject), } } diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go index b81a299f3..84eb3c916 100644 --- a/pkg/storage/storage_errors.go +++ b/pkg/storage/storage_errors.go @@ -10,12 +10,28 @@ type ObjectExists struct { key string } +type BucketNameInvalid struct { + bucket string +} + +type BucketExists struct { + bucket string +} + type ObjectNotFound GenericError func (self ObjectNotFound) Error() string { - return "Not Found: " + self.bucket + "#" + self.path + return "Object not Found: " + self.bucket + "#" + self.path } func (self ObjectExists) Error() string { return "Object exists: " + self.bucket + "#" + self.key } + +func (self BucketNameInvalid) Error() string { + return "Bucket name invalid: " + self.bucket +} + +func (self BucketExists) Error() string { + return "Bucket exists: " + self.bucket +} diff --git a/pkg/webapi/minioapi/definitions.go b/pkg/webapi/minioapi/definitions.go index cfae223dc..6f80d2ce6 100644 --- a/pkg/webapi/minioapi/definitions.go +++ b/pkg/webapi/minioapi/definitions.go @@ -20,14 +20,24 @@ import ( "encoding/xml" ) -type ListResponse struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - storagerefix string - Marker string - MaxKeys int - IsTruncated bool - Contents []Content `xml:"Contents",innerxml` +type ObjectListResponse struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Marker string + MaxKeys int + IsTruncated bool + Contents []Content `xml:"Contents",innerxml` +} + +type BucketListResponse struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner + Buckets []Bucket `xml:"Buckets",innerxml` +} + +type Bucket struct { + Name string + CreationDate string } type Content struct { diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index e7b53349b..9555e952f 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -42,7 +42,7 @@ func HttpHandler(storage *mstorage.Storage) http.Handler { storage: storage, } mux.HandleFunc("/", api.listBucketsHandler).Methods("GET") - mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET") + mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT") mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT") @@ -74,7 +74,31 @@ func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Reques } func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) { - w.Write([]byte("/")) + vars := mux.Vars(req) + prefix, ok := vars["prefix"] + if ok == false { + prefix = "" + } + contentType := "xml" + if req.Header["Accept"][0] == "application/json" { + contentType = "json" + } + buckets := server.storage.ListBuckets(prefix) + response := generateBucketsListResult(buckets) + + var bytesBuffer bytes.Buffer + var encoder encoder + if contentType == "json" { + w.Header().Set("Content-Type", "application/json") + encoder = json.NewEncoder(&bytesBuffer) + } else { + w.Header().Set("Content-Type", "application/xml") + encoder = xml.NewEncoder(&bytesBuffer) + } + encoder.Encode(response) + + w.Write(bytesBuffer.Bytes()) + } func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) { @@ -102,7 +126,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ } objects := server.storage.ListObjects(bucket, prefix, 1000) - response := generateListResult(objects) + response := generateObjectsListResult(bucket, objects) var bytesBuffer bytes.Buffer var encoder encoder @@ -130,7 +154,41 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques } } -func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { +func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + bucket := vars["bucket"] + err := server.storage.StoreBucket(bucket) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + return + } +} + +func generateBucketsListResult(buckets []mstorage.BucketMetadata) (data BucketListResponse) { + listbuckets := []Bucket{} + + owner := Owner{ + ID: "minio", + DisplayName: "minio", + } + + for _, bucket := range buckets { + listbucket := Bucket{ + Name: bucket.Name, + CreationDate: formatDate(bucket.Created), + } + listbuckets = append(listbuckets, listbucket) + } + + data = BucketListResponse{ + Owner: owner, + Buckets: listbuckets, + } + return +} + +func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata) (data ObjectListResponse) { contents := []Content{} owner := Owner{ @@ -149,13 +207,13 @@ func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { } contents = append(contents, content) } - data := ListResponse{ - Name: "name", + data = ObjectListResponse{ + Name: bucket, Contents: contents, MaxKeys: len(objects), IsTruncated: false, } - return data + return } func formatDate(sec int64) string {