From e22ae2475a8678e19314c8860e31ab330fa24bbb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 20 Jan 2015 23:16:06 -0800 Subject: [PATCH 1/3] Add license headers and other cleanup --- pkg/httpserver/httpserver.go | 16 +++++++++++++ pkg/server/server.go | 16 +++++++++++++ pkg/storage/storage.go | 36 +++++++++++++--------------- pkg/storage/storage_errors.go | 21 ++++++++++++++++ pkg/utils/execpipe.go | 16 +++++++++++++ pkg/utils/execpipe_test.go | 16 ++++++++++++- pkg/webapi/minioapi/definitions.go | 30 +++++++++++++++++------ pkg/webapi/minioapi/minioapi.go | 16 +++++++++++++ pkg/webapi/minioapi/minioapi_test.go | 16 +++++++++++++ 9 files changed, 155 insertions(+), 28 deletions(-) create mode 100644 pkg/storage/storage_errors.go diff --git a/pkg/httpserver/httpserver.go b/pkg/httpserver/httpserver.go index 5919cc5b0..15edca078 100644 --- a/pkg/httpserver/httpserver.go +++ b/pkg/httpserver/httpserver.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package httpserver import ( diff --git a/pkg/server/server.go b/pkg/server/server.go index dc4a0cd26..f532d3e91 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package server import ( diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4d0e875cd..0937ddbab 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package storage import ( @@ -23,26 +39,6 @@ type ObjectMetadata struct { Size int } -type GenericError struct { - bucket string - path string -} - -type ObjectNotFound GenericError - -func (self ObjectNotFound) Error() string { - return "Not Found: " + self.bucket + "#" + self.path -} - -type ObjectExists struct { - bucket string - key string -} - -func (self ObjectExists) Error() string { - return "Object exists: " + self.bucket + "#" + self.key -} - func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // TODO synchronize access // get object diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go new file mode 100644 index 000000000..b81a299f3 --- /dev/null +++ b/pkg/storage/storage_errors.go @@ -0,0 +1,21 @@ +package storage + +type GenericError struct { + bucket string + path string +} + +type ObjectExists struct { + bucket string + key string +} + +type ObjectNotFound GenericError + +func (self ObjectNotFound) Error() string { + return "Not Found: " + self.bucket + "#" + self.path +} + +func (self ObjectExists) Error() string { + return "Object exists: " + self.bucket + "#" + self.key +} diff --git a/pkg/utils/execpipe.go b/pkg/utils/execpipe.go index 74be379f5..848abed3d 100644 --- a/pkg/utils/execpipe.go +++ b/pkg/utils/execpipe.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package utils import ( diff --git a/pkg/utils/execpipe_test.go b/pkg/utils/execpipe_test.go index 7d6451a2f..d7eda90eb 100644 --- a/pkg/utils/execpipe_test.go +++ b/pkg/utils/execpipe_test.go @@ -1,4 +1,18 @@ -// !build linux,amd64 +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package utils diff --git a/pkg/webapi/minioapi/definitions.go b/pkg/webapi/minioapi/definitions.go index bf8216237..cfae223dc 100644 --- a/pkg/webapi/minioapi/definitions.go +++ b/pkg/webapi/minioapi/definitions.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( @@ -5,13 +21,13 @@ import ( ) type ListResponse struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - toragerefix string - Marker string - MaxKeys int - IsTruncated bool - Contents []Content `xml:"Contents",innerxml` + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + storagerefix string + Marker string + MaxKeys int + IsTruncated bool + Contents []Content `xml:"Contents",innerxml` } type Content struct { diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index ef99463ec..e7b53349b 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( diff --git a/pkg/webapi/minioapi/minioapi_test.go b/pkg/webapi/minioapi/minioapi_test.go index 4abbd659a..7a632d69b 100644 --- a/pkg/webapi/minioapi/minioapi_test.go +++ b/pkg/webapi/minioapi/minioapi_test.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( From 5b67da7d965fb736125c5192ee60e9c470d053de Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 21 Jan 2015 00:50:23 -0800 Subject: [PATCH 2/3] Add PutBucket and ListBuckets service --- Makefile | 5 +- pkg/httpserver/httpserver.go | 11 +++- pkg/storage/storage.go | 90 +++++++++++++++++++++++++++--- pkg/storage/storage_errors.go | 18 +++++- pkg/webapi/minioapi/definitions.go | 26 ++++++--- pkg/webapi/minioapi/minioapi.go | 72 +++++++++++++++++++++--- 6 files changed, 197 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 633480636..8b764fe1a 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,10 @@ build-storage: @$(MAKE) $(MAKE_OPTIONS) -C pkg/storage/erasure/isal lib @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/storage/erasure -cover: build-storage build-os build-utils +build-minioapi: + @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/webapi/minioapi + +cover: build-storage build-os build-utils build-minioapi install: cover diff --git a/pkg/httpserver/httpserver.go b/pkg/httpserver/httpserver.go index 15edca078..6fc5d971c 100644 --- a/pkg/httpserver/httpserver.go +++ b/pkg/httpserver/httpserver.go @@ -19,6 +19,7 @@ package httpserver import ( "log" "net/http" + "time" ) func Start(handler http.Handler, address string) (chan<- string, <-chan error) { @@ -30,7 +31,15 @@ func Start(handler http.Handler, address string) (chan<- string, <-chan error) { func start(ctrlChannel <-chan string, errorChannel chan<- error, router http.Handler, address string) { log.Println("Starting HTTP Server on " + address) - err := http.ListenAndServe(address, router) + // Minio server config + server := &http.Server{ + Addr: address, + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + err := server.ListenAndServe() errorChannel <- err close(errorChannel) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 0937ddbab..ea8301139 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -25,7 +25,8 @@ import ( ) type Storage struct { - data map[string]storedObject + bucketdata map[string]storedBucket + objectdata map[string]storedObject } type storedObject struct { @@ -33,17 +34,64 @@ type storedObject struct { data []byte } +type storedBucket struct { + metadata BucketMetadata + // owner string // TODO + // id string // TODO +} + +type BucketMetadata struct { + Name string + Created int64 +} + type ObjectMetadata struct { Key string SecCreated int64 Size int } +func isValidBucket(bucket string) bool { + l := len(bucket) + if l < 3 || l > 63 { + return false + } + + valid := false + prev := byte('.') + for i := 0; i < len(bucket); i++ { + c := bucket[i] + switch { + default: + return false + case 'a' <= c && c <= 'z': + valid = true + case '0' <= c && c <= '9': + // Is allowed, but bucketname can't be just numbers. + // Therefore, don't set valid to true + case c == '-': + if prev == '.' { + return false + } + case c == '.': + if prev == '.' || prev == '-' { + return false + } + } + prev = c + } + + if prev == '-' || prev == '.' { + return false + } + return valid +} + func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // TODO synchronize access // get object key := bucket + ":" + object - if val, ok := storage.data[key]; ok { + if val, ok := storage.objectdata[key]; ok { objectBuffer := bytes.NewBuffer(val.data) written, err := io.Copy(w, objectBuffer) return written, err @@ -54,7 +102,7 @@ func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object st func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error { objectKey := bucket + ":" + key - if _, ok := storage.data[objectKey]; ok == true { + if _, ok := storage.objectdata[objectKey]; ok == true { return ObjectExists{bucket: bucket, key: key} } var bytesBuffer bytes.Buffer @@ -67,14 +115,32 @@ func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) e } newObject.data = bytesBuffer.Bytes() } - storage.data[objectKey] = newObject + storage.objectdata[objectKey] = newObject + return nil +} + +func (storage *Storage) StoreBucket(bucketName string) error { + if !isValidBucket(bucketName) { + return BucketNameInvalid{bucket: bucketName} + } + + if _, ok := storage.bucketdata[bucketName]; ok == true { + return BucketExists{bucket: bucketName} + } + newBucket := storedBucket{} + newBucket.metadata = BucketMetadata{ + Name: bucketName, + Created: time.Now().Unix(), + } + log.Println(bucketName) + storage.bucketdata[bucketName] = newBucket return nil } func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMetadata { + // TODO prefix and count handling var results []ObjectMetadata - for key, object := range storage.data { - log.Println(key) + for key, object := range storage.objectdata { if strings.HasPrefix(key, bucket+":") { results = append(results, object.metadata) } @@ -82,12 +148,22 @@ func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMe return results } +func (storage *Storage) ListBuckets(prefix string) []BucketMetadata { + // TODO prefix handling + var results []BucketMetadata + for _, bucket := range storage.bucketdata { + results = append(results, bucket.metadata) + } + return results +} + func Start() (chan<- string, <-chan error, *Storage) { ctrlChannel := make(chan string) errorChannel := make(chan error) go start(ctrlChannel, errorChannel) return ctrlChannel, errorChannel, &Storage{ - data: make(map[string]storedObject), + bucketdata: make(map[string]storedBucket), + objectdata: make(map[string]storedObject), } } diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go index b81a299f3..84eb3c916 100644 --- a/pkg/storage/storage_errors.go +++ b/pkg/storage/storage_errors.go @@ -10,12 +10,28 @@ type ObjectExists struct { key string } +type BucketNameInvalid struct { + bucket string +} + +type BucketExists struct { + bucket string +} + type ObjectNotFound GenericError func (self ObjectNotFound) Error() string { - return "Not Found: " + self.bucket + "#" + self.path + return "Object not Found: " + self.bucket + "#" + self.path } func (self ObjectExists) Error() string { return "Object exists: " + self.bucket + "#" + self.key } + +func (self BucketNameInvalid) Error() string { + return "Bucket name invalid: " + self.bucket +} + +func (self BucketExists) Error() string { + return "Bucket exists: " + self.bucket +} diff --git a/pkg/webapi/minioapi/definitions.go b/pkg/webapi/minioapi/definitions.go index cfae223dc..6f80d2ce6 100644 --- a/pkg/webapi/minioapi/definitions.go +++ b/pkg/webapi/minioapi/definitions.go @@ -20,14 +20,24 @@ import ( "encoding/xml" ) -type ListResponse struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - storagerefix string - Marker string - MaxKeys int - IsTruncated bool - Contents []Content `xml:"Contents",innerxml` +type ObjectListResponse struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Marker string + MaxKeys int + IsTruncated bool + Contents []Content `xml:"Contents",innerxml` +} + +type BucketListResponse struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner + Buckets []Bucket `xml:"Buckets",innerxml` +} + +type Bucket struct { + Name string + CreationDate string } type Content struct { diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index e7b53349b..9555e952f 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -42,7 +42,7 @@ func HttpHandler(storage *mstorage.Storage) http.Handler { storage: storage, } mux.HandleFunc("/", api.listBucketsHandler).Methods("GET") - mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET") + mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT") mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT") @@ -74,7 +74,31 @@ func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Reques } func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) { - w.Write([]byte("/")) + vars := mux.Vars(req) + prefix, ok := vars["prefix"] + if ok == false { + prefix = "" + } + contentType := "xml" + if req.Header["Accept"][0] == "application/json" { + contentType = "json" + } + buckets := server.storage.ListBuckets(prefix) + response := generateBucketsListResult(buckets) + + var bytesBuffer bytes.Buffer + var encoder encoder + if contentType == "json" { + w.Header().Set("Content-Type", "application/json") + encoder = json.NewEncoder(&bytesBuffer) + } else { + w.Header().Set("Content-Type", "application/xml") + encoder = xml.NewEncoder(&bytesBuffer) + } + encoder.Encode(response) + + w.Write(bytesBuffer.Bytes()) + } func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) { @@ -102,7 +126,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ } objects := server.storage.ListObjects(bucket, prefix, 1000) - response := generateListResult(objects) + response := generateObjectsListResult(bucket, objects) var bytesBuffer bytes.Buffer var encoder encoder @@ -130,7 +154,41 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques } } -func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { +func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + bucket := vars["bucket"] + err := server.storage.StoreBucket(bucket) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + return + } +} + +func generateBucketsListResult(buckets []mstorage.BucketMetadata) (data BucketListResponse) { + listbuckets := []Bucket{} + + owner := Owner{ + ID: "minio", + DisplayName: "minio", + } + + for _, bucket := range buckets { + listbucket := Bucket{ + Name: bucket.Name, + CreationDate: formatDate(bucket.Created), + } + listbuckets = append(listbuckets, listbucket) + } + + data = BucketListResponse{ + Owner: owner, + Buckets: listbuckets, + } + return +} + +func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata) (data ObjectListResponse) { contents := []Content{} owner := Owner{ @@ -149,13 +207,13 @@ func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { } contents = append(contents, content) } - data := ListResponse{ - Name: "name", + data = ObjectListResponse{ + Name: bucket, Contents: contents, MaxKeys: len(objects), IsTruncated: false, } - return data + return } func formatDate(sec int64) string { From d44404dd816717a443bc4cdfec3671b0212d6f34 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 21 Jan 2015 10:45:22 -0800 Subject: [PATCH 3/3] Provide ETag with sha256Sum of input object data --- pkg/storage/storage.go | 9 ++++++++- pkg/webapi/minioapi/minioapi.go | 3 ++- pkg/webapi/minioapi/minioapi_test.go | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index ea8301139..3978bd989 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,10 +18,13 @@ package storage import ( "bytes" + "fmt" "io" "log" "strings" "time" + + "crypto/sha256" ) type Storage struct { @@ -49,6 +52,7 @@ type ObjectMetadata struct { Key string SecCreated int64 Size int + ETag string } func isValidBucket(bucket string) bool { @@ -108,10 +112,13 @@ func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) e var bytesBuffer bytes.Buffer newObject := storedObject{} if _, ok := io.Copy(&bytesBuffer, data); ok == nil { + size := bytesBuffer.Len() + etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) newObject.metadata = ObjectMetadata{ Key: key, SecCreated: time.Now().Unix(), - Size: len(bytesBuffer.Bytes()), + Size: size, + ETag: etag, } newObject.data = bytesBuffer.Bytes() } diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index 9555e952f..5089ed6db 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -43,6 +43,7 @@ func HttpHandler(storage *mstorage.Storage) http.Handler { } mux.HandleFunc("/", api.listBucketsHandler).Methods("GET") mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT") + mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT") @@ -200,7 +201,7 @@ func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata) content := Content{ Key: object.Key, LastModified: formatDate(object.SecCreated), - ETag: object.Key, + ETag: object.ETag, Size: object.Size, StorageClass: "STANDARD", Owner: owner, diff --git a/pkg/webapi/minioapi/minioapi_test.go b/pkg/webapi/minioapi/minioapi_test.go index 7a632d69b..96cdb512b 100644 --- a/pkg/webapi/minioapi/minioapi_test.go +++ b/pkg/webapi/minioapi/minioapi_test.go @@ -47,7 +47,7 @@ func TestMinioApi(t *testing.T) { Owner: owner, }, } - data := &ListResponse{ + data := &ObjectListResponse{ Name: "name", Contents: contents, }