Merge pull request #30 from harshavardhana/pr_out_provide_etag_with_sha256sum_of_input_object_data

Makefile

@@ -28,7 +28,10 @@ build-storage:
 	@$(MAKE) $(MAKE_OPTIONS) -C pkg/storage/erasure/isal lib
 	@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/storage/erasure
 
-cover: build-storage build-os build-utils
+build-minioapi:
+	@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkg/webapi/minioapi
+
+cover: build-storage build-os build-utils build-minioapi
 
 install: cover
 

pkg/httpserver/httpserver.go

@@ -1,8 +1,25 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package httpserver
 
 import (
 	"log"
 	"net/http"
+	"time"
 )
 
 func Start(handler http.Handler, address string) (chan<- string, <-chan error) {
@@ -14,7 +31,15 @@ func Start(handler http.Handler, address string) (chan<- string, <-chan error) {
 
 func start(ctrlChannel <-chan string, errorChannel chan<- error, router http.Handler, address string) {
 	log.Println("Starting HTTP Server on " + address)
-	err := http.ListenAndServe(address, router)
+	// Minio server config
+	server := &http.Server{
+		Addr:           address,
+		Handler:        router,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+	err := server.ListenAndServe()
 	errorChannel <- err
 	close(errorChannel)
 }

pkg/server/server.go

@@ -1,3 +1,19 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package server
 
 import (

pkg/storage/storage.go

@@ -1,15 +1,35 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package storage
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"log"
 	"strings"
 	"time"
+
+	"crypto/sha256"
 )
 
 type Storage struct {
-	data map[string]storedObject
+	bucketdata map[string]storedBucket
+	objectdata map[string]storedObject
 }
 
 type storedObject struct {
@@ -17,37 +37,65 @@ type storedObject struct {
 	data     []byte
 }
 
+type storedBucket struct {
+	metadata BucketMetadata
+	//	owner    string // TODO
+	//	id       string // TODO
+}
+
+type BucketMetadata struct {
+	Name    string
+	Created int64
+}
+
 type ObjectMetadata struct {
 	Key        string
 	SecCreated int64
 	Size       int
+	ETag       string
 }
 
-type GenericError struct {
-	bucket string
-	path   string
-}
-
-type ObjectNotFound GenericError
-
-func (self ObjectNotFound) Error() string {
-	return "Not Found: " + self.bucket + "#" + self.path
-}
+func isValidBucket(bucket string) bool {
+	l := len(bucket)
+	if l < 3 || l > 63 {
+		return false
+	}
 
-type ObjectExists struct {
-	bucket string
-	key    string
-}
+	valid := false
+	prev := byte('.')
+	for i := 0; i < len(bucket); i++ {
+		c := bucket[i]
+		switch {
+		default:
+			return false
+		case 'a' <= c && c <= 'z':
+			valid = true
+		case '0' <= c && c <= '9':
+			// Is allowed, but bucketname can't be just numbers.
+			// Therefore, don't set valid to true
+		case c == '-':
+			if prev == '.' {
+				return false
+			}
+		case c == '.':
+			if prev == '.' || prev == '-' {
+				return false
+			}
+		}
+		prev = c
+	}
 
-func (self ObjectExists) Error() string {
-	return "Object exists: " + self.bucket + "#" + self.key
+	if prev == '-' || prev == '.' {
+		return false
+	}
+	return valid
 }
 
 func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
 	// TODO synchronize access
 	// get object
 	key := bucket + ":" + object
-	if val, ok := storage.data[key]; ok {
+	if val, ok := storage.objectdata[key]; ok {
 		objectBuffer := bytes.NewBuffer(val.data)
 		written, err := io.Copy(w, objectBuffer)
 		return written, err
@@ -58,27 +106,48 @@ func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object st
 
 func (storage *Storage) StoreObject(bucket string, key string, data io.Reader) error {
 	objectKey := bucket + ":" + key
-	if _, ok := storage.data[objectKey]; ok == true {
+	if _, ok := storage.objectdata[objectKey]; ok == true {
 		return ObjectExists{bucket: bucket, key: key}
 	}
 	var bytesBuffer bytes.Buffer
 	newObject := storedObject{}
 	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
+		size := bytesBuffer.Len()
+		etag := fmt.Sprintf("%x", sha256.Sum256(bytesBuffer.Bytes()))
 		newObject.metadata = ObjectMetadata{
 			Key:        key,
 			SecCreated: time.Now().Unix(),
-			Size:       len(bytesBuffer.Bytes()),
+			Size:       size,
+			ETag:       etag,
 		}
 		newObject.data = bytesBuffer.Bytes()
 	}
-	storage.data[objectKey] = newObject
+	storage.objectdata[objectKey] = newObject
+	return nil
+}
+
+func (storage *Storage) StoreBucket(bucketName string) error {
+	if !isValidBucket(bucketName) {
+		return BucketNameInvalid{bucket: bucketName}
+	}
+
+	if _, ok := storage.bucketdata[bucketName]; ok == true {
+		return BucketExists{bucket: bucketName}
+	}
+	newBucket := storedBucket{}
+	newBucket.metadata = BucketMetadata{
+		Name:    bucketName,
+		Created: time.Now().Unix(),
+	}
+	log.Println(bucketName)
+	storage.bucketdata[bucketName] = newBucket
 	return nil
 }
 
 func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMetadata {
+	// TODO prefix and count handling
 	var results []ObjectMetadata
-	for key, object := range storage.data {
-		log.Println(key)
+	for key, object := range storage.objectdata {
 		if strings.HasPrefix(key, bucket+":") {
 			results = append(results, object.metadata)
 		}
@@ -86,12 +155,22 @@ func (storage *Storage) ListObjects(bucket, prefix string, count int) []ObjectMe
 	return results
 }
 
+func (storage *Storage) ListBuckets(prefix string) []BucketMetadata {
+	// TODO prefix handling
+	var results []BucketMetadata
+	for _, bucket := range storage.bucketdata {
+		results = append(results, bucket.metadata)
+	}
+	return results
+}
+
 func Start() (chan<- string, <-chan error, *Storage) {
 	ctrlChannel := make(chan string)
 	errorChannel := make(chan error)
 	go start(ctrlChannel, errorChannel)
 	return ctrlChannel, errorChannel, &Storage{
-		data: make(map[string]storedObject),
+		bucketdata: make(map[string]storedBucket),
+		objectdata: make(map[string]storedObject),
 	}
 }
 

pkg/storage/storage_errors.go

@@ -0,0 +1,37 @@
+package storage
+
+type GenericError struct {
+	bucket string
+	path   string
+}
+
+type ObjectExists struct {
+	bucket string
+	key    string
+}
+
+type BucketNameInvalid struct {
+	bucket string
+}
+
+type BucketExists struct {
+	bucket string
+}
+
+type ObjectNotFound GenericError
+
+func (self ObjectNotFound) Error() string {
+	return "Object not Found: " + self.bucket + "#" + self.path
+}
+
+func (self ObjectExists) Error() string {
+	return "Object exists: " + self.bucket + "#" + self.key
+}
+
+func (self BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + self.bucket
+}
+
+func (self BucketExists) Error() string {
+	return "Bucket exists: " + self.bucket
+}

pkg/utils/execpipe.go

@@ -1,3 +1,19 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package utils
 
 import (

pkg/utils/execpipe_test.go

@@ -1,4 +1,18 @@
-// !build linux,amd64
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package utils
 

pkg/webapi/minioapi/definitions.go

@@ -1,19 +1,45 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package minioapi
 
 import (
 	"encoding/xml"
 )
 
-type ListResponse struct {
+type ObjectListResponse struct {
 	XMLName     xml.Name `xml:"ListBucketResult"`
 	Name        string   `xml:"Name"`
-	toragerefix string
 	Marker      string
 	MaxKeys     int
 	IsTruncated bool
 	Contents    []Content `xml:"Contents",innerxml`
 }
 
+type BucketListResponse struct {
+	XMLName xml.Name `xml:"ListAllMyBucketsResult"`
+	Owner   Owner
+	Buckets []Bucket `xml:"Buckets",innerxml`
+}
+
+type Bucket struct {
+	Name         string
+	CreationDate string
+}
+
 type Content struct {
 	Key          string
 	LastModified string

pkg/webapi/minioapi/minioapi.go

@@ -1,3 +1,19 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package minioapi
 
 import (
@@ -26,6 +42,7 @@ func HttpHandler(storage *mstorage.Storage) http.Handler {
 		storage: storage,
 	}
 	mux.HandleFunc("/", api.listBucketsHandler).Methods("GET")
+	mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT")
 	mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET")
 	mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET")
 	mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET")
@@ -58,7 +75,31 @@ func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Reques
 }
 
 func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) {
-	w.Write([]byte("/"))
+	vars := mux.Vars(req)
+	prefix, ok := vars["prefix"]
+	if ok == false {
+		prefix = ""
+	}
+	contentType := "xml"
+	if req.Header["Accept"][0] == "application/json" {
+		contentType = "json"
+	}
+	buckets := server.storage.ListBuckets(prefix)
+	response := generateBucketsListResult(buckets)
+
+	var bytesBuffer bytes.Buffer
+	var encoder encoder
+	if contentType == "json" {
+		w.Header().Set("Content-Type", "application/json")
+		encoder = json.NewEncoder(&bytesBuffer)
+	} else {
+		w.Header().Set("Content-Type", "application/xml")
+		encoder = xml.NewEncoder(&bytesBuffer)
+	}
+	encoder.Encode(response)
+
+	w.Write(bytesBuffer.Bytes())
+
 }
 
 func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) {
@@ -86,7 +127,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ
 	}
 
 	objects := server.storage.ListObjects(bucket, prefix, 1000)
-	response := generateListResult(objects)
+	response := generateObjectsListResult(bucket, objects)
 
 	var bytesBuffer bytes.Buffer
 	var encoder encoder
@@ -114,7 +155,41 @@ func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Reques
 	}
 }
 
-func generateListResult(objects []mstorage.ObjectMetadata) ListResponse {
+func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	err := server.storage.StoreBucket(bucket)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(err.Error()))
+		return
+	}
+}
+
+func generateBucketsListResult(buckets []mstorage.BucketMetadata) (data BucketListResponse) {
+	listbuckets := []Bucket{}
+
+	owner := Owner{
+		ID:          "minio",
+		DisplayName: "minio",
+	}
+
+	for _, bucket := range buckets {
+		listbucket := Bucket{
+			Name:         bucket.Name,
+			CreationDate: formatDate(bucket.Created),
+		}
+		listbuckets = append(listbuckets, listbucket)
+	}
+
+	data = BucketListResponse{
+		Owner:   owner,
+		Buckets: listbuckets,
+	}
+	return
+}
+
+func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata) (data ObjectListResponse) {
 	contents := []Content{}
 
 	owner := Owner{
@@ -126,20 +201,20 @@ func generateListResult(objects []mstorage.ObjectMetadata) ListResponse {
 		content := Content{
 			Key:          object.Key,
 			LastModified: formatDate(object.SecCreated),
-			ETag:         object.Key,
+			ETag:         object.ETag,
 			Size:         object.Size,
 			StorageClass: "STANDARD",
 			Owner:        owner,
 		}
 		contents = append(contents, content)
 	}
-	data := ListResponse{
-		Name:        "name",
+	data = ObjectListResponse{
+		Name:        bucket,
 		Contents:    contents,
 		MaxKeys:     len(objects),
 		IsTruncated: false,
 	}
-	return data
+	return
 }
 
 func formatDate(sec int64) string {

pkg/webapi/minioapi/minioapi_test.go

@@ -1,3 +1,19 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package minioapi
 
 import (
@@ -31,7 +47,7 @@ func TestMinioApi(t *testing.T) {
 			Owner:        owner,
 		},
 	}
-	data := &ListResponse{
+	data := &ObjectListResponse{
 		Name:     "name",
 		Contents: contents,
 	}