diff --git a/Makefile b/Makefile index 633480636..8b764fe1a 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,10 @@ build-storage: @$(MAKE) $(MAKE_OPTIONS) -C pkg/storage/erasure/isal lib @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/storage/erasure -cover: build-storage build-os build-utils +build-minioapi: + @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/webapi/minioapi + +cover: build-storage build-os build-utils build-minioapi install: cover diff --git a/pkg/httpserver/httpserver.go b/pkg/httpserver/httpserver.go index 5919cc5b0..6fc5d971c 100644 --- a/pkg/httpserver/httpserver.go +++ b/pkg/httpserver/httpserver.go @@ -1,8 +1,25 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package httpserver import ( "log" "net/http" + "time" ) func Start(handler http.Handler, address string) (chan<- string, <-chan error) { @@ -14,7 +31,15 @@ func Start(handler http.Handler, address string) (chan<- string, <-chan error) { func start(ctrlChannel <-chan string, errorChannel chan<- error, router http.Handler, address string) { log.Println("Starting HTTP Server on " + address) - err := http.ListenAndServe(address, router) + // Minio server config + server := &http.Server{ + Addr: address, + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + err := server.ListenAndServe() errorChannel <- err close(errorChannel) } diff --git a/pkg/server/server.go b/pkg/server/server.go index dc4a0cd26..f532d3e91 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package server import ( diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4d0e875cd..3978bd989 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,15 +1,35 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package storage import ( "bytes" + "fmt" "io" "log" "strings" "time" + + "crypto/sha256" ) type Storage struct { - data map[string]storedObject + bucketdata map[string]storedBucket + objectdata map[string]storedObject } type storedObject struct { @@ -17,37 +37,65 @@ type storedObject struct { data []byte } +type storedBucket struct { + metadata BucketMetadata + // owner string // TODO + // id string // TODO +} + +type BucketMetadata struct { + Name string + Created int64 +} + type ObjectMetadata struct { Key string SecCreated int64 Size int + ETag string } -type GenericError struct { - bucket string - path string -} - -type ObjectNotFound GenericError - -func (self ObjectNotFound) Error() string { - return "Not Found: " + self.bucket + "#" + self.path -} +func isValidBucket(bucket string) bool { + l := len(bucket) + if l < 3 || l > 63 { + return false + } -type ObjectExists struct { - bucket string - key string -} + valid := false + prev := byte('.') + for i := 0; i < len(bucket); i++ { + c := bucket[i] + switch { + default: + return false + case 'a' <= c && c <= 'z': + valid = true + case '0' <= c && c <= '9': + // Is allowed, but bucketname can't be just numbers. + // Therefore, don't set valid to true + case c == '-': + if prev == '.' { + return false + } + case c == '.': + if prev == '.' || prev == '-' { + return false + } + } + prev = c + } -func (self ObjectExists) Error() string { - return "Object exists: " + self.bucket + "#" + self.key + if prev == '-' || prev == '.' { + return false + } + return valid } func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // TODO synchronize access // get object key := bucket + ":" + object - if val, ok := storage.data[key]; ok { + if val, ok := storage.objectdata[key]; ok { objectBuffer := bytes.NewBuffer(val.data) written, err := io.Copy(w, objectBuffer) return written, err @@ -58,27 +106,48 @@ func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object st func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error { objectKey := bucket + ":" + key - if _, ok := storage.data[objectKey]; ok == true { + if _, ok := storage.objectdata[objectKey]; ok == true { return ObjectExists{bucket: bucket, key: key} } var bytesBuffer bytes.Buffer newObject := storedObject{} if _, ok := io.Copy(&bytesBuffer, data); ok == nil { + size := bytesBuffer.Len() + etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes())) newObject.metadata = ObjectMetadata{ Key: key, SecCreated: time.Now().Unix(), - Size: len(bytesBuffer.Bytes()), + Size: size, + ETag: etag, } newObject.data = bytesBuffer.Bytes() } - storage.data[objectKey] = newObject + storage.objectdata[objectKey] = newObject + return nil +} + +func (storage *Storage) StoreBucket(bucketName string) error { + if !isValidBucket(bucketName) { + return BucketNameInvalid{bucket: bucketName} + } + + if _, ok := storage.bucketdata[bucketName]; ok == true { + return BucketExists{bucket: bucketName} + } + newBucket := storedBucket{} + newBucket.metadata = BucketMetadata{ + Name: bucketName, + Created: time.Now().Unix(), + } + log.Println(bucketName) + storage.bucketdata[bucketName] = newBucket return nil } func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMetadata { + // TODO prefix and count handling var results []ObjectMetadata - for key, object := range storage.data { - log.Println(key) + for key, object := range storage.objectdata { if strings.HasPrefix(key, bucket+":") { results = append(results, object.metadata) } @@ -86,12 +155,22 @@ func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMe return results } +func (storage *Storage) ListBuckets(prefix string) []BucketMetadata { + // TODO prefix handling + var results []BucketMetadata + for _, bucket := range storage.bucketdata { + results = append(results, bucket.metadata) + } + return results +} + func Start() (chan<- string, <-chan error, *Storage) { ctrlChannel := make(chan string) errorChannel := make(chan error) go start(ctrlChannel, errorChannel) return ctrlChannel, errorChannel, &Storage{ - data: make(map[string]storedObject), + bucketdata: make(map[string]storedBucket), + objectdata: make(map[string]storedObject), } } diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go new file mode 100644 index 000000000..84eb3c916 --- /dev/null +++ b/pkg/storage/storage_errors.go @@ -0,0 +1,37 @@ +package storage + +type GenericError struct { + bucket string + path string +} + +type ObjectExists struct { + bucket string + key string +} + +type BucketNameInvalid struct { + bucket string +} + +type BucketExists struct { + bucket string +} + +type ObjectNotFound GenericError + +func (self ObjectNotFound) Error() string { + return "Object not Found: " + self.bucket + "#" + self.path +} + +func (self ObjectExists) Error() string { + return "Object exists: " + self.bucket + "#" + self.key +} + +func (self BucketNameInvalid) Error() string { + return "Bucket name invalid: " + self.bucket +} + +func (self BucketExists) Error() string { + return "Bucket exists: " + self.bucket +} diff --git a/pkg/utils/execpipe.go b/pkg/utils/execpipe.go index 74be379f5..848abed3d 100644 --- a/pkg/utils/execpipe.go +++ b/pkg/utils/execpipe.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package utils import ( diff --git a/pkg/utils/execpipe_test.go b/pkg/utils/execpipe_test.go index 7d6451a2f..d7eda90eb 100644 --- a/pkg/utils/execpipe_test.go +++ b/pkg/utils/execpipe_test.go @@ -1,4 +1,18 @@ -// !build linux,amd64 +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package utils diff --git a/pkg/webapi/minioapi/definitions.go b/pkg/webapi/minioapi/definitions.go index bf8216237..6f80d2ce6 100644 --- a/pkg/webapi/minioapi/definitions.go +++ b/pkg/webapi/minioapi/definitions.go @@ -1,19 +1,45 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( "encoding/xml" ) -type ListResponse struct { +type ObjectListResponse struct { XMLName xml.Name `xml:"ListBucketResult"` Name string `xml:"Name"` - toragerefix string Marker string MaxKeys int IsTruncated bool Contents []Content `xml:"Contents",innerxml` } +type BucketListResponse struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner + Buckets []Bucket `xml:"Buckets",innerxml` +} + +type Bucket struct { + Name string + CreationDate string +} + type Content struct { Key string LastModified string diff --git a/pkg/webapi/minioapi/minioapi.go b/pkg/webapi/minioapi/minioapi.go index ef99463ec..5089ed6db 100644 --- a/pkg/webapi/minioapi/minioapi.go +++ b/pkg/webapi/minioapi/minioapi.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( @@ -26,6 +42,7 @@ func HttpHandler(storage *mstorage.Storage) http.Handler { storage: storage, } mux.HandleFunc("/", api.listBucketsHandler).Methods("GET") + mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT") mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET") @@ -58,7 +75,31 @@ func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Reques } func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) { - w.Write([]byte("/")) + vars := mux.Vars(req) + prefix, ok := vars["prefix"] + if ok == false { + prefix = "" + } + contentType := "xml" + if req.Header["Accept"][0] == "application/json" { + contentType = "json" + } + buckets := server.storage.ListBuckets(prefix) + response := generateBucketsListResult(buckets) + + var bytesBuffer bytes.Buffer + var encoder encoder + if contentType == "json" { + w.Header().Set("Content-Type", "application/json") + encoder = json.NewEncoder(&bytesBuffer) + } else { + w.Header().Set("Content-Type", "application/xml") + encoder = xml.NewEncoder(&bytesBuffer) + } + encoder.Encode(response) + + w.Write(bytesBuffer.Bytes()) + } func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) { @@ -86,7 +127,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ } objects := server.storage.ListObjects(bucket, prefix, 1000) - response := generateListResult(objects) + response := generateObjectsListResult(bucket, objects) var bytesBuffer bytes.Buffer var encoder encoder @@ -114,7 +155,41 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques } } -func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { +func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + bucket := vars["bucket"] + err := server.storage.StoreBucket(bucket) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + return + } +} + +func generateBucketsListResult(buckets []mstorage.BucketMetadata) (data BucketListResponse) { + listbuckets := []Bucket{} + + owner := Owner{ + ID: "minio", + DisplayName: "minio", + } + + for _, bucket := range buckets { + listbucket := Bucket{ + Name: bucket.Name, + CreationDate: formatDate(bucket.Created), + } + listbuckets = append(listbuckets, listbucket) + } + + data = BucketListResponse{ + Owner: owner, + Buckets: listbuckets, + } + return +} + +func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata) (data ObjectListResponse) { contents := []Content{} owner := Owner{ @@ -126,20 +201,20 @@ func generateListResult(objects []mstorage.ObjectMetadata) ListResponse { content := Content{ Key: object.Key, LastModified: formatDate(object.SecCreated), - ETag: object.Key, + ETag: object.ETag, Size: object.Size, StorageClass: "STANDARD", Owner: owner, } contents = append(contents, content) } - data := ListResponse{ - Name: "name", + data = ObjectListResponse{ + Name: bucket, Contents: contents, MaxKeys: len(objects), IsTruncated: false, } - return data + return } func formatDate(sec int64) string { diff --git a/pkg/webapi/minioapi/minioapi_test.go b/pkg/webapi/minioapi/minioapi_test.go index 4abbd659a..96cdb512b 100644 --- a/pkg/webapi/minioapi/minioapi_test.go +++ b/pkg/webapi/minioapi/minioapi_test.go @@ -1,3 +1,19 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package minioapi import ( @@ -31,7 +47,7 @@ func TestMinioApi(t *testing.T) { Owner: owner, }, } - data := &ListResponse{ + data := &ObjectListResponse{ Name: "name", Contents: contents, }