From 72572d6c7120fb3a86b9b8a6bac97d8990ea3900 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 30 Jun 2015 14:42:29 -0700 Subject: [PATCH] Remove some api server code bringing in new cleanup --- pkg/api/api-bucket-handlers.go | 155 ++ ...{api_definitions.go => api-definitions.go} | 0 ...ic_handlers.go => api-generic-handlers.go} | 4 +- .../logging.go => api-logging-handlers.go} | 54 +- ...ect_handlers.go => api-object-handlers.go} | 229 +-- ...e_limiter.go => api-ratelimit-handlers.go} | 14 +- pkg/api/{api_response.go => api-response.go} | 0 pkg/api/{api_router.go => api-router.go} | 48 +- pkg/api/api_bucket_handlers.go | 320 --- pkg/api/api_test.go | 1793 ----------------- pkg/api/quota/bandwidth_cap.go | 152 -- pkg/api/quota/conn_limit.go | 89 - pkg/api/quota/errors.go | 127 -- pkg/api/quota/quota_handler.go | 96 - pkg/api/quota/request_limit.go | 65 - .../httpserver.go => api/server.go} | 58 +- pkg/server/server.go | 129 -- 17 files changed, 243 insertions(+), 3090 deletions(-) create mode 100644 pkg/api/api-bucket-handlers.go rename pkg/api/{api_definitions.go => api-definitions.go} (100%) rename pkg/api/{api_generic_handlers.go => api-generic-handlers.go} (98%) rename pkg/api/{logging/logging.go => api-logging-handlers.go} (67%) rename pkg/api/{api_object_handlers.go => api-object-handlers.go} (51%) rename pkg/api/{quota/rate_limiter.go => api-ratelimit-handlers.go} (78%) rename pkg/api/{api_response.go => api-response.go} (100%) rename pkg/api/{api_router.go => api-router.go} (64%) delete mode 100644 pkg/api/api_bucket_handlers.go delete mode 100644 pkg/api/api_test.go delete mode 100644 pkg/api/quota/bandwidth_cap.go delete mode 100644 pkg/api/quota/conn_limit.go delete mode 100644 pkg/api/quota/errors.go delete mode 100644 pkg/api/quota/quota_handler.go delete mode 100644 pkg/api/quota/request_limit.go rename pkg/{server/httpserver/httpserver.go => api/server.go} (67%) delete mode 100644 pkg/server/server.go diff --git a/pkg/api/api-bucket-handlers.go b/pkg/api/api-bucket-handlers.go new file mode 100644 index 000000000..cec239e98 --- /dev/null +++ b/pkg/api/api-bucket-handlers.go @@ -0,0 +1,155 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "log" + "net/http" + + "github.com/gorilla/mux" +) + +func (server *minioAPI) isValidOp(w http.ResponseWriter, req *http.Request, acceptsContentType contentType) bool { + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) + return true +} + +// GET Bucket (List Multipart uploads) +// ------------------------- +// This operation lists in-progress multipart uploads. An in-progress +// multipart upload is a multipart upload that has been initiated, +// using the Initiate Multipart Upload request, but has not yet been completed or aborted. +// This operation returns at most 1,000 multipart uploads in the response. +// +func (server *minioAPI) listMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + log.Println(acceptsContentType) + + resources := getBucketMultipartResources(req.URL.Query()) + if resources.MaxUploads == 0 { + resources.MaxUploads = maxObjectList + } + + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) +} + +// GET Bucket (List Objects) +// ------------------------- +// This implementation of the GET operation returns some or all (up to 1000) +// of the objects in a bucket. You can use the request parameters as selection +// criteria to return a subset of the objects in a bucket. +// +func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + // verify if bucket allows this operation + if !server.isValidOp(w, req, acceptsContentType) { + return + } + + if isRequestUploads(req.URL.Query()) { + server.listMultipartUploadsHandler(w, req) + return + } + + resources := getBucketResources(req.URL.Query()) + if resources.Maxkeys == 0 { + resources.Maxkeys = maxObjectList + } + + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) + +} + +// GET Service +// ----------- +// This implementation of the GET operation returns a list of all buckets +// owned by the authenticated sender of the request. +func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + // uncomment this when we have webcli + // without access key credentials one cannot list buckets + // if _, err := stripAuth(req); err != nil { + // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) + // return + // } + log.Println(acceptsContentType) +} + +// PUT Bucket +// ---------- +// This implementation of the PUT operation creates a new bucket for authenticated request +func (server *minioAPI) putBucketHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + // uncomment this when we have webcli + // without access key credentials one cannot create a bucket + // if _, err := stripAuth(req); err != nil { + // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) + // return + // } + if isRequestBucketACL(req.URL.Query()) { + server.putBucketACLHandler(w, req) + return + } + // read from 'x-amz-acl' + aclType := getACLType(req) + if aclType == unsupportedACLType { + writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path) + return + } + + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) +} + +// PUT Bucket ACL +// ---------- +// This implementation of the PUT operation modifies the bucketACL for authenticated request +func (server *minioAPI) putBucketACLHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + // read from 'x-amz-acl' + aclType := getACLType(req) + if aclType == unsupportedACLType { + writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path) + return + } + + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) +} + +// HEAD Bucket +// ---------- +// This operation is useful to determine if a bucket exists. +// The operation returns a 200 OK if the bucket exists and you +// have permission to access it. Otherwise, the operation might +// return responses such as 404 Not Found and 403 Forbidden. +func (server *minioAPI) headBucketHandler(w http.ResponseWriter, req *http.Request) { + acceptsContentType := getContentType(req) + log.Println(acceptsContentType) + + vars := mux.Vars(req) + bucket := vars["bucket"] + log.Println(bucket) +} diff --git a/pkg/api/api_definitions.go b/pkg/api/api-definitions.go similarity index 100% rename from pkg/api/api_definitions.go rename to pkg/api/api-definitions.go diff --git a/pkg/api/api_generic_handlers.go b/pkg/api/api-generic-handlers.go similarity index 98% rename from pkg/api/api_generic_handlers.go rename to pkg/api/api-generic-handlers.go index 49c5021c5..2f0cf251e 100644 --- a/pkg/api/api_generic_handlers.go +++ b/pkg/api/api-generic-handlers.go @@ -98,7 +98,7 @@ func stripAuth(r *http.Request) (*auth, error) { return a, nil } -func getDate(req *http.Request) (time.Time, error) { +func parseDate(req *http.Request) (time.Time, error) { amzDate := req.Header.Get("X-Amz-Date") switch { case amzDate != "": @@ -154,7 +154,7 @@ func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeErrorResponse(w, r, RequestTimeTooSkewed, acceptsContentType, r.URL.Path) return } - date, err := getDate(r) + date, err := parseDate(r) if err != nil { // there is no way to knowing if this is a valid request, could be a attack reject such clients writeErrorResponse(w, r, RequestTimeTooSkewed, acceptsContentType, r.URL.Path) diff --git a/pkg/api/logging/logging.go b/pkg/api/api-logging-handlers.go similarity index 67% rename from pkg/api/logging/logging.go rename to pkg/api/api-logging-handlers.go index 825ef90c7..87e12b98d 100644 --- a/pkg/api/logging/logging.go +++ b/pkg/api/api-logging-handlers.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package logging +package api import ( "bytes" @@ -31,12 +31,12 @@ import ( ) type logHandler struct { - http.Handler - Logger chan<- []byte + handler http.Handler + logger chan<- []byte } -// LogMessage is a serializable json log message -type LogMessage struct { +// logMessage is a serializable json log message +type logMessage struct { StartTime time.Time Duration time.Duration StatusMessage string // human readable http status message @@ -49,38 +49,38 @@ type LogMessage struct { } } -// LogWriter is used to capture status for log messages -type LogWriter struct { - ResponseWriter http.ResponseWriter - LogMessage *LogMessage +// logWriter is used to capture status for log messages +type logWriter struct { + responseWriter http.ResponseWriter + logMessage *logMessage } // WriteHeader writes headers and stores status in LogMessage -func (w *LogWriter) WriteHeader(status int) { - w.LogMessage.StatusMessage = http.StatusText(status) - w.ResponseWriter.WriteHeader(status) +func (w *logWriter) WriteHeader(status int) { + w.logMessage.StatusMessage = http.StatusText(status) + w.responseWriter.WriteHeader(status) } // Header Dummy wrapper for LogWriter -func (w *LogWriter) Header() http.Header { - return w.ResponseWriter.Header() +func (w *logWriter) Header() http.Header { + return w.responseWriter.Header() } // Write Dummy wrapper for LogWriter -func (w *LogWriter) Write(data []byte) (int, error) { - return w.ResponseWriter.Write(data) +func (w *logWriter) Write(data []byte) (int, error) { + return w.responseWriter.Write(data) } func (h *logHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - logMessage := &LogMessage{ + logMessage := &logMessage{ StartTime: time.Now().UTC(), } - logWriter := &LogWriter{ResponseWriter: w, LogMessage: logMessage} - h.Handler.ServeHTTP(logWriter, req) - h.Logger <- getLogMessage(logMessage, w, req) + logWriter := &logWriter{responseWriter: w, logMessage: logMessage} + h.handler.ServeHTTP(logWriter, req) + h.logger <- getLogMessage(logMessage, w, req) } -func getLogMessage(logMessage *LogMessage, w http.ResponseWriter, req *http.Request) []byte { +func getLogMessage(logMessage *logMessage, w http.ResponseWriter, req *http.Request) []byte { // store lower level details logMessage.HTTP.ResponseHeaders = w.Header() logMessage.HTTP.Request = req @@ -94,14 +94,14 @@ func getLogMessage(logMessage *LogMessage, w http.ResponseWriter, req *http.Requ return js } -// LogHandler logs requests -func LogHandler(h http.Handler) http.Handler { - logger, _ := FileLogger("access.log") - return &logHandler{Handler: h, Logger: logger} +// loggingHandler logs requests +func loggingHandler(h http.Handler) http.Handler { + logger, _ := fileLogger("access.log") + return &logHandler{handler: h, logger: logger} } -// FileLogger returns a channel that is used to write to the logger -func FileLogger(filename string) (chan<- []byte, error) { +// fileLogger returns a channel that is used to write to the logger +func fileLogger(filename string) (chan<- []byte, error) { ch := make(chan []byte) file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { diff --git a/pkg/api/api_object_handlers.go b/pkg/api/api-object-handlers.go similarity index 51% rename from pkg/api/api_object_handlers.go rename to pkg/api/api-object-handlers.go index 41f4e0aea..409c4d571 100644 --- a/pkg/api/api_object_handlers.go +++ b/pkg/api/api-object-handlers.go @@ -25,7 +25,6 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio/pkg/iodine" - "github.com/minio/minio/pkg/storage/drivers" "github.com/minio/minio/pkg/utils/log" ) @@ -48,47 +47,8 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques vars := mux.Vars(req) bucket = vars["bucket"] object = vars["object"] + log.Println(bucket, object) - metadata, err := server.driver.GetObjectMetadata(bucket, object) - switch iodine.ToError(err).(type) { - case nil: // success - { - httpRange, err := getRequestedRange(req, metadata.Size) - if err != nil { - writeErrorResponse(w, req, InvalidRange, acceptsContentType, req.URL.Path) - return - } - switch httpRange.start == 0 && httpRange.length == 0 { - case true: - setObjectHeaders(w, metadata) - if _, err := server.driver.GetObject(w, bucket, object); err != nil { - // unable to write headers, we've already printed data. Just close the connection. - log.Error.Println(iodine.New(err, nil)) - } - case false: - metadata.Size = httpRange.length - setRangeObjectHeaders(w, metadata, httpRange) - w.WriteHeader(http.StatusPartialContent) - if _, err := server.driver.GetPartialObject(w, bucket, object, httpRange.start, httpRange.length); err != nil { - // unable to write headers, we've already printed data. Just close the connection. - log.Error.Println(iodine.New(err, nil)) - } - } - } - case drivers.ObjectNotFound: - { - writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path) - } - case drivers.ObjectNameInvalid: - { - writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } } // HEAD Object @@ -105,34 +65,7 @@ func (server *minioAPI) headObjectHandler(w http.ResponseWriter, req *http.Reque vars := mux.Vars(req) bucket = vars["bucket"] object = vars["object"] - - metadata, err := server.driver.GetObjectMetadata(bucket, object) - switch iodine.ToError(err).(type) { - case nil: - { - setObjectHeaders(w, metadata) - w.WriteHeader(http.StatusOK) - } - case drivers.ObjectNotFound: - { - error := getErrorCode(NoSuchKey) - w.Header().Set("Server", "Minio") - w.WriteHeader(error.HTTPStatusCode) - } - case drivers.ObjectNameInvalid: - { - error := getErrorCode(NoSuchKey) - w.Header().Set("Server", "Minio") - w.WriteHeader(error.HTTPStatusCode) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - error := getErrorCode(InternalError) - w.Header().Set("Server", "Minio") - w.WriteHeader(error.HTTPStatusCode) - } - } + log.Println(bucket, object) } // PUT Object @@ -182,36 +115,7 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques writeErrorResponse(w, req, InvalidRequest, acceptsContentType, req.URL.Path) return } - calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, sizeInt64, req.Body) - switch iodine.ToError(err).(type) { - case nil: - { - w.Header().Set("ETag", calculatedMD5) - writeSuccessResponse(w, acceptsContentType) - - } - case drivers.ObjectExists: - { - writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path) - } - case drivers.BadDigest: - { - writeErrorResponse(w, req, BadDigest, acceptsContentType, req.URL.Path) - } - case drivers.EntityTooLarge: - { - writeErrorResponse(w, req, EntityTooLarge, acceptsContentType, req.URL.Path) - } - case drivers.InvalidDigest: - { - writeErrorResponse(w, req, InvalidDigest, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } + log.Println(bucket, object, sizeInt64) } /// Multipart API @@ -233,27 +137,7 @@ func (server *minioAPI) newMultipartUploadHandler(w http.ResponseWriter, req *ht vars := mux.Vars(req) bucket = vars["bucket"] object = vars["object"] - uploadID, err := server.driver.NewMultipartUpload(bucket, object, "") - switch iodine.ToError(err).(type) { - case nil: - { - response := generateInitiateMultipartUploadResult(bucket, object, uploadID) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write body - w.Write(encodedSuccessResponse) - } - case drivers.ObjectExists: - { - writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } + log.Println(bucket, object) } // Upload part @@ -293,6 +177,7 @@ func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Re vars := mux.Vars(req) bucket := vars["bucket"] object := vars["object"] + log.Println(bucket, object, sizeInt64) uploadID := req.URL.Query().Get("uploadId") partIDString := req.URL.Query().Get("partNumber") @@ -301,40 +186,7 @@ func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Re if err != nil { writeErrorResponse(w, req, InvalidPart, acceptsContentType, req.URL.Path) } - calculatedMD5, err := server.driver.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body) - switch iodine.ToError(err).(type) { - case nil: - { - w.Header().Set("ETag", calculatedMD5) - writeSuccessResponse(w, acceptsContentType) - - } - case drivers.InvalidUploadID: - { - writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path) - } - case drivers.ObjectExists: - { - writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path) - } - case drivers.BadDigest: - { - writeErrorResponse(w, req, BadDigest, acceptsContentType, req.URL.Path) - } - case drivers.EntityTooLarge: - { - writeErrorResponse(w, req, EntityTooLarge, acceptsContentType, req.URL.Path) - } - case drivers.InvalidDigest: - { - writeErrorResponse(w, req, InvalidDigest, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } + log.Println(uploadID, partID) } // Abort multipart upload @@ -349,25 +201,8 @@ func (server *minioAPI) abortMultipartUploadHandler(w http.ResponseWriter, req * bucket := vars["bucket"] object := vars["object"] - objectResourcesMetadata := getObjectResources(req.URL.Query()) - - err := server.driver.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID) - switch iodine.ToError(err).(type) { - case nil: - { - setCommonHeaders(w, getContentTypeString(acceptsContentType), 0) - w.WriteHeader(http.StatusNoContent) - } - case drivers.InvalidUploadID: - { - writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } + //objectResourcesMetadata := getObjectResources(req.URL.Query()) + log.Println(bucket, object) } // List object parts @@ -386,28 +221,7 @@ func (server *minioAPI) listObjectPartsHandler(w http.ResponseWriter, req *http. vars := mux.Vars(req) bucket := vars["bucket"] object := vars["object"] - - objectResourcesMetadata, err := server.driver.ListObjectParts(bucket, object, objectResourcesMetadata) - switch iodine.ToError(err).(type) { - case nil: - { - response := generateListPartsResult(objectResourcesMetadata) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write body - w.Write(encodedSuccessResponse) - } - case drivers.InvalidUploadID: - { - writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } + log.Println(bucket, object) } // Complete multipart upload @@ -434,34 +248,15 @@ func (server *minioAPI) completeMultipartUploadHandler(w http.ResponseWriter, re vars := mux.Vars(req) bucket := vars["bucket"] object := vars["object"] - objectResourcesMetadata := getObjectResources(req.URL.Query()) + log.Println(bucket, object) + + //objectResourcesMetadata := getObjectResources(req.URL.Query()) partMap := make(map[int]string) for _, part := range parts.Part { partMap[part.PartNumber] = part.ETag } - etag, err := server.driver.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, partMap) - switch iodine.ToError(err).(type) { - case nil: - { - response := generateCompleteMultpartUploadResult(bucket, object, "", etag) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write body - w.Write(encodedSuccessResponse) - } - case drivers.InvalidUploadID: - { - writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } } /// Delete API diff --git a/pkg/api/quota/rate_limiter.go b/pkg/api/api-ratelimit-handlers.go similarity index 78% rename from pkg/api/quota/rate_limiter.go rename to pkg/api/api-ratelimit-handlers.go index 5649538c1..03e31102c 100644 --- a/pkg/api/quota/rate_limiter.go +++ b/pkg/api/api-ratelimit-handlers.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package quota +package api import "net/http" @@ -24,26 +24,26 @@ type rateLimit struct { rateQueue chan bool } -func (c *rateLimit) Add() { +func (c rateLimit) Add() { c.rateQueue <- true // fill in the queue return } -func (c *rateLimit) Remove() { +func (c rateLimit) Remove() { <-c.rateQueue // invalidate the queue, after the request is served return } // ServeHTTP is an http.Handler ServeHTTP method -func (c *rateLimit) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (c rateLimit) ServeHTTP(w http.ResponseWriter, req *http.Request) { c.Add() // add c.handler.ServeHTTP(w, req) // serve c.Remove() // remove } -// RateLimit limits the number of concurrent http requests -func RateLimit(handle http.Handler, limit int) http.Handler { - return &rateLimit{ +// rateLimitHandler limits the number of concurrent http requests +func rateLimitHandler(handle http.Handler, limit int) http.Handler { + return rateLimit{ handler: handle, rateQueue: make(chan bool, limit), } diff --git a/pkg/api/api_response.go b/pkg/api/api-response.go similarity index 100% rename from pkg/api/api_response.go rename to pkg/api/api-response.go diff --git a/pkg/api/api_router.go b/pkg/api/api-router.go similarity index 64% rename from pkg/api/api_router.go rename to pkg/api/api-router.go index 7d6db5265..c0e77aeca 100644 --- a/pkg/api/api_router.go +++ b/pkg/api/api-router.go @@ -16,42 +16,15 @@ package api -import ( - "net/http" +import router "github.com/gorilla/mux" - router "github.com/gorilla/mux" - "github.com/minio/minio/pkg/api/logging" - "github.com/minio/minio/pkg/api/quota" - "github.com/minio/minio/pkg/storage/drivers" -) +type minioAPI struct{} -type minioAPI struct { - driver drivers.Driver -} - -// Config api configurable parameters -type Config struct { - RateLimit int - driver drivers.Driver -} - -// GetDriver - get a an existing set driver -func (c Config) GetDriver() drivers.Driver { - return c.driver -} - -// SetDriver - set a new driver -func (c *Config) SetDriver(driver drivers.Driver) { - c.driver = driver -} - -// HTTPHandler - http wrapper handler -func HTTPHandler(config Config) http.Handler { - var mux *router.Router +// Handler - api wrapper handler +func New(config Config) API { var api = minioAPI{} - api.driver = config.GetDriver() - mux = router.NewRouter() + mux := router.NewRouter() mux.HandleFunc("/", api.listBucketsHandler).Methods("GET") mux.HandleFunc("/{bucket}", api.listObjectsHandler).Methods("GET") mux.HandleFunc("/{bucket}", api.putBucketHandler).Methods("PUT") @@ -75,12 +48,7 @@ func HTTPHandler(config Config) http.Handler { handler = timeValidityHandler(handler) handler = ignoreResourcesHandler(handler) handler = validateAuthHeaderHandler(handler) - // handler = quota.BandwidthCap(h, 25*1024*1024, time.Duration(30*time.Minute)) - // handler = quota.BandwidthCap(h, 100*1024*1024, time.Duration(24*time.Hour)) - // handler = quota.RequestLimit(h, 100, time.Duration(30*time.Minute)) - // handler = quota.RequestLimit(h, 1000, time.Duration(24*time.Hour)) - // handler = quota.ConnectionLimit(handler, config.ConnectionLimit) - handler = quota.RateLimit(handler, config.RateLimit) - handler = logging.LogHandler(handler) - return handler + handler = rateLimitHandler(handler, config.RateLimit) + handler = loggingHandler(handler) + return API{config, handler} } diff --git a/pkg/api/api_bucket_handlers.go b/pkg/api/api_bucket_handlers.go deleted file mode 100644 index 38934c7bc..000000000 --- a/pkg/api/api_bucket_handlers.go +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "net/http" - - "github.com/gorilla/mux" - "github.com/minio/minio/pkg/iodine" - "github.com/minio/minio/pkg/storage/drivers" - "github.com/minio/minio/pkg/utils/log" -) - -func (server *minioAPI) isValidOp(w http.ResponseWriter, req *http.Request, acceptsContentType contentType) bool { - vars := mux.Vars(req) - bucket := vars["bucket"] - - bucketMetadata, err := server.driver.GetBucketMetadata(bucket) - switch iodine.ToError(err).(type) { - case drivers.BucketNotFound: - { - writeErrorResponse(w, req, NoSuchBucket, acceptsContentType, req.URL.Path) - return false - } - case drivers.BucketNameInvalid: - { - writeErrorResponse(w, req, InvalidBucketName, acceptsContentType, req.URL.Path) - return false - } - case nil: - if _, err := stripAuth(req); err != nil { - if bucketMetadata.ACL.IsPrivate() { - return true - //uncomment this when we have webcli - //writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) - //return false - } - if bucketMetadata.ACL.IsPublicRead() && req.Method == "PUT" { - return true - //uncomment this when we have webcli - //writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) - //return false - } - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } - return true -} - -// GET Bucket (List Multipart uploads) -// ------------------------- -// This operation lists in-progress multipart uploads. An in-progress -// multipart upload is a multipart upload that has been initiated, -// using the Initiate Multipart Upload request, but has not yet been completed or aborted. -// This operation returns at most 1,000 multipart uploads in the response. -// -func (server *minioAPI) listMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - - resources := getBucketMultipartResources(req.URL.Query()) - if resources.MaxUploads == 0 { - resources.MaxUploads = maxObjectList - } - - vars := mux.Vars(req) - bucket := vars["bucket"] - - resources, err := server.driver.ListMultipartUploads(bucket, resources) - switch iodine.ToError(err).(type) { - case nil: // success - { - // generate response - response := generateListMultipartUploadsResult(bucket, resources) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write body - w.Write(encodedSuccessResponse) - } - case drivers.BucketNotFound: - { - writeErrorResponse(w, req, NoSuchBucket, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } - -} - -// GET Bucket (List Objects) -// ------------------------- -// This implementation of the GET operation returns some or all (up to 1000) -// of the objects in a bucket. You can use the request parameters as selection -// criteria to return a subset of the objects in a bucket. -// -func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - // verify if bucket allows this operation - if !server.isValidOp(w, req, acceptsContentType) { - return - } - - if isRequestUploads(req.URL.Query()) { - server.listMultipartUploadsHandler(w, req) - return - } - - resources := getBucketResources(req.URL.Query()) - if resources.Maxkeys == 0 { - resources.Maxkeys = maxObjectList - } - - vars := mux.Vars(req) - bucket := vars["bucket"] - - objects, resources, err := server.driver.ListObjects(bucket, resources) - switch iodine.ToError(err).(type) { - case nil: // success - { - // generate response - response := generateListObjectsResponse(bucket, objects, resources) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write body - w.Write(encodedSuccessResponse) - } - case drivers.ObjectNotFound: - { - writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path) - } - case drivers.ObjectNameInvalid: - { - writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } -} - -// GET Service -// ----------- -// This implementation of the GET operation returns a list of all buckets -// owned by the authenticated sender of the request. -func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - // uncomment this when we have webcli - // without access key credentials one cannot list buckets - // if _, err := stripAuth(req); err != nil { - // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) - // return - // } - buckets, err := server.driver.ListBuckets() - switch iodine.ToError(err).(type) { - case nil: - { - // generate response - response := generateListBucketsResponse(buckets) - encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType) - // write headers - setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse)) - // write response - w.Write(encodedSuccessResponse) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } -} - -// PUT Bucket -// ---------- -// This implementation of the PUT operation creates a new bucket for authenticated request -func (server *minioAPI) putBucketHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - // uncomment this when we have webcli - // without access key credentials one cannot create a bucket - // if _, err := stripAuth(req); err != nil { - // writeErrorResponse(w, req, AccessDenied, acceptsContentType, req.URL.Path) - // return - // } - if isRequestBucketACL(req.URL.Query()) { - server.putBucketACLHandler(w, req) - return - } - // read from 'x-amz-acl' - aclType := getACLType(req) - if aclType == unsupportedACLType { - writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path) - return - } - - vars := mux.Vars(req) - bucket := vars["bucket"] - err := server.driver.CreateBucket(bucket, getACLTypeString(aclType)) - switch iodine.ToError(err).(type) { - case nil: - { - // Make sure to add Location information here only for bucket - w.Header().Set("Location", "/"+bucket) - writeSuccessResponse(w, acceptsContentType) - } - case drivers.TooManyBuckets: - { - writeErrorResponse(w, req, TooManyBuckets, acceptsContentType, req.URL.Path) - } - case drivers.BucketNameInvalid: - { - writeErrorResponse(w, req, InvalidBucketName, acceptsContentType, req.URL.Path) - } - case drivers.BucketExists: - { - writeErrorResponse(w, req, BucketAlreadyExists, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } -} - -// PUT Bucket ACL -// ---------- -// This implementation of the PUT operation modifies the bucketACL for authenticated request -func (server *minioAPI) putBucketACLHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - // read from 'x-amz-acl' - aclType := getACLType(req) - if aclType == unsupportedACLType { - writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path) - return - } - - vars := mux.Vars(req) - bucket := vars["bucket"] - err := server.driver.SetBucketMetadata(bucket, getACLTypeString(aclType)) - switch iodine.ToError(err).(type) { - case nil: - { - writeSuccessResponse(w, acceptsContentType) - } - case drivers.BucketNameInvalid: - { - writeErrorResponse(w, req, InvalidBucketName, acceptsContentType, req.URL.Path) - } - case drivers.BucketNotFound: - { - writeErrorResponse(w, req, NoSuchBucket, acceptsContentType, req.URL.Path) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path) - } - } -} - -// HEAD Bucket -// ---------- -// This operation is useful to determine if a bucket exists. -// The operation returns a 200 OK if the bucket exists and you -// have permission to access it. Otherwise, the operation might -// return responses such as 404 Not Found and 403 Forbidden. -func (server *minioAPI) headBucketHandler(w http.ResponseWriter, req *http.Request) { - acceptsContentType := getContentType(req) - - vars := mux.Vars(req) - bucket := vars["bucket"] - - _, err := server.driver.GetBucketMetadata(bucket) - switch iodine.ToError(err).(type) { - case nil: - { - writeSuccessResponse(w, acceptsContentType) - } - case drivers.BucketNotFound: - { - error := getErrorCode(NoSuchBucket) - w.WriteHeader(error.HTTPStatusCode) - } - case drivers.BucketNameInvalid: - { - error := getErrorCode(InvalidBucketName) - w.WriteHeader(error.HTTPStatusCode) - } - default: - { - log.Error.Println(iodine.New(err, nil)) - error := getErrorCode(InternalError) - w.WriteHeader(error.HTTPStatusCode) - } - } -} diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go deleted file mode 100644 index e53a1cbe6..000000000 --- a/pkg/api/api_test.go +++ /dev/null @@ -1,1793 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "bytes" - "io" - "io/ioutil" - "log" - "os" - "reflect" - "strconv" - "strings" - "testing" - "time" - - "encoding/xml" - "net/http" - "net/http/httptest" - - "github.com/minio/minio/pkg/storage/drivers" - "github.com/minio/minio/pkg/storage/drivers/cache" - "github.com/minio/minio/pkg/storage/drivers/donut" - "github.com/minio/minio/pkg/storage/drivers/mocks" - "github.com/stretchr/testify/mock" - - . "github.com/minio/check" -) - -func Test(t *testing.T) { TestingT(t) } - -type MySuite struct { - Driver drivers.Driver - MockDriver *mocks.Driver - initDriver func() (drivers.Driver, string) - Root string -} - -var _ = Suite(&MySuite{ - initDriver: func() (drivers.Driver, string) { - return startMockDriver(), "" - }, -}) - -var _ = Suite(&MySuite{ - initDriver: func() (drivers.Driver, string) { - root, _ := ioutil.TempDir(os.TempDir(), "minio-api") - var roots []string - roots = append(roots, root) - driver, _ := donut.NewDriver(roots) - driver, _ = cache.NewDriver(10000, 3*time.Hour, driver) - return driver, root - }, -}) - -func (s *MySuite) SetUpSuite(c *C) { - driver, root := s.initDriver() - if root != "" { - defer os.RemoveAll(root) - } - log.Println("Running API Suite:", reflect.TypeOf(driver)) -} - -func (s *MySuite) SetUpTest(c *C) { - driver, root := s.initDriver() - var typedDriver *mocks.Driver - switch driver := driver.(type) { - case *mocks.Driver: - { - typedDriver = driver - } - default: - { - typedDriver = startMockDriver() - } - } - s.Driver = driver - s.Root = root - s.MockDriver = typedDriver -} - -func (s *MySuite) TearDownTest(c *C) { - root := strings.TrimSpace(s.Root) - if root != "" { - os.RemoveAll(s.Root) - } - s.Driver = nil - s.Root = "" -} - -func setDummyAuthHeader(req *http.Request) { - authDummy := "AWS4-HMAC-SHA256 Credential=AC5NH40NQLTL4DUMMY/20130524/us-east-1/s3/aws4_request, SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd" - req.Header.Set("Authorization", authDummy) - req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) -} - -func setConfig(driver drivers.Driver) Config { - conf := Config{RateLimit: 16} - conf.SetDriver(driver) - return conf -} - -func (s *MySuite) TestNonExistantBucket(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - s.MockDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: "bucket"}).Once() - request, err := http.NewRequest("HEAD", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNotFound) -} - -func (s *MySuite) TestEmptyObject(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - metadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "key", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "d41d8cd98f00b204e9800998ecf8427e", - Size: 0, - } - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - typedDriver.On("CreateObject", "bucket", "object", "", "", 0, mock.Anything).Return(metadata.Md5, nil).Once() - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(metadata, nil).Once() - typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(metadata, nil).Once() - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - buffer := bytes.NewBufferString("") - driver.CreateBucket("bucket", "private") - driver.CreateObject("bucket", "object", "", "", 0, buffer) - - request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes())) - - resMetadata, err := driver.GetObjectMetadata("bucket", "object") - c.Assert(err, IsNil) - verifyHeaders(c, response.Header, resMetadata.Created, 0, "application/octet-stream", resMetadata.Md5) -} - -func (s *MySuite) TestBucket(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - metadata := drivers.BucketMetadata{ - Name: "bucket", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - driver.CreateBucket("bucket", "private") - - request, err := http.NewRequest("HEAD", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -func (s *MySuite) TestObject(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - metadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "key", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "5eb63bbbe01eeed093cb22bb8f5acdc3", - Size: 11, - } - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything, mock.Anything).Return(metadata.Md5, nil).Once() - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(metadata, nil).Twice() - typedDriver.SetGetObjectWriter("bucket", "object", []byte("hello world")) - typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once() - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - buffer := bytes.NewBufferString("hello world") - driver.CreateBucket("bucket", "private") - driver.CreateObject("bucket", "object", "", "", int64(buffer.Len()), buffer) - - request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(responseBody, DeepEquals, []byte("hello world")) - - resMetadata, err := driver.GetObjectMetadata("bucket", "object") - c.Assert(err, IsNil) - verifyHeaders(c, response.Header, resMetadata.Created, len("hello world"), "application/octet-stream", metadata.Md5) -} - -func (s *MySuite) TestMultipleObjects(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - metadata1 := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object1", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f11ac20bf1d3c85c586fa793fa03186", - Size: 9, - } - metadata2 := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object2", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "c1c7f5decb9ff01edf1af096ebb8f4a4", - Size: 9, - } - metadata3 := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object3", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "4e74ad3b92e2843e208a13ae1cf0d52c", - Size: 11, - } - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - buffer1 := bytes.NewBufferString("hello one") - buffer2 := bytes.NewBufferString("hello two") - buffer3 := bytes.NewBufferString("hello three") - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - driver.CreateBucket("bucket", "private") - typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything, mock.Anything).Return(metadata1.Md5, nil).Once() - driver.CreateObject("bucket", "object1", "", "", int64(buffer1.Len()), buffer1) - typedDriver.On("CreateObject", "bucket", "object2", "", "", mock.Anything, mock.Anything).Return(metadata2.Md5, nil).Once() - driver.CreateObject("bucket", "object2", "", "", int64(buffer2.Len()), buffer2) - typedDriver.On("CreateObject", "bucket", "object3", "", "", mock.Anything, mock.Anything).Return(metadata3.Md5, nil).Once() - driver.CreateObject("bucket", "object3", "", "", int64(buffer3.Len()), buffer3) - - // test non-existant object - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(drivers.ObjectMetadata{}, drivers.ObjectNotFound{}).Once() - request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - //// test object 1 - - // get object - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object1").Return(metadata1, nil).Once() - typedDriver.SetGetObjectWriter("bucket", "object1", []byte("hello one")) - typedDriver.On("GetObject", mock.Anything, "bucket", "object1").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/object1", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - - // get metadata - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object1").Return(metadata1, nil).Once() - metadata, err := driver.GetObjectMetadata("bucket", "object1") - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello one"), "application/octet-stream", metadata.Md5) - c.Assert(err, IsNil) - - // verify response data - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello one"))) - - // test object 2 - // get object - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object2").Return(metadata2, nil).Once() - typedDriver.SetGetObjectWriter("bucket", "object2", []byte("hello two")) - typedDriver.On("GetObject", mock.Anything, "bucket", "object2").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/object2", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - - // get metadata - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object2").Return(metadata2, nil).Once() - metadata, err = driver.GetObjectMetadata("bucket", "object2") - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello two"), "application/octet-stream", metadata.Md5) - c.Assert(err, IsNil) - - // verify response data - responseBody, err = ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello two"))) - - // test object 3 - // get object - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object3").Return(metadata3, nil).Once() - typedDriver.SetGetObjectWriter("bucket", "object3", []byte("hello three")) - typedDriver.On("GetObject", mock.Anything, "bucket", "object3").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/object3", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - - // get metadata - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object3").Return(metadata3, nil).Once() - metadata, err = driver.GetObjectMetadata("bucket", "object3") - - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // verify headers - verifyHeaders(c, response.Header, metadata.Created, len("hello three"), "application/octet-stream", metadata.Md5) - c.Assert(err, IsNil) - - // verify object - responseBody, err = ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello three"))) -} - -func (s *MySuite) TestNotImplemented(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - request, err := http.NewRequest("GET", testServer.URL+"/bucket/object?policy", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNotImplemented) - -} - -func (s *MySuite) TestHeader(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - typedDriver.AssertExpectations(c) - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - err := driver.CreateBucket("bucket", "private") - c.Assert(err, IsNil) - - bucketMetadata := drivers.BucketMetadata{ - Name: "bucket", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(drivers.ObjectMetadata{}, drivers.ObjectNotFound{}).Once() - request, err := http.NewRequest("GET", testServer.URL+"/bucket/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - objectMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f5902ac237024bdd0c176cb93063dc4", - Size: 11, - } - - buffer := bytes.NewBufferString("hello world") - typedDriver.On("GetBucketMetadata", "foo").Return(bucketMetadata, nil).Once() - typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything, mock.Anything).Return(objectMetadata.Md5, nil).Once() - driver.CreateObject("bucket", "object", "", "", int64(buffer.Len()), buffer) - - typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(objectMetadata, nil).Once() - typedDriver.SetGetObjectWriter("", "", []byte("hello world")) - typedDriver.On("GetObject", mock.Anything, "bucket", "object").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(bucketMetadata, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object").Return(objectMetadata, nil).Once() - metadata, err := driver.GetObjectMetadata("bucket", "object") - c.Assert(err, IsNil) - verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.Md5) -} - -func (s *MySuite) TestPutBucket(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("ListBuckets").Return(make([]drivers.BucketMetadata, 0), nil).Once() - buckets, err := driver.ListBuckets() - c.Assert(len(buckets), Equals, 0) - c.Assert(err, IsNil) - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket", bytes.NewBufferString("")) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // check bucket exists - typedDriver.On("ListBuckets").Return([]drivers.BucketMetadata{{Name: "bucket"}}, nil).Once() - buckets, err = driver.ListBuckets() - c.Assert(len(buckets), Equals, 1) - c.Assert(err, IsNil) - c.Assert(buckets[0].Name, Equals, "bucket") -} - -func (s *MySuite) TestPutObject(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - resources := drivers.BucketResourcesMetadata{} - - resources.Maxkeys = 1000 - resources.Prefix = "" - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("ListObjects", "bucket", mock.Anything).Return([]drivers.ObjectMetadata{}, - drivers.BucketResourcesMetadata{}, drivers.BucketNotFound{}).Once() - objects, resources, err := driver.ListObjects("bucket", resources) - c.Assert(len(objects), Equals, 0) - c.Assert(resources.IsTruncated, Equals, false) - c.Assert(err, Not(IsNil)) - - // breaks on fs driver,// breaks on fs driver, so we subtract one second - // date1 := time.Now().UTC() - date1 := time.Now().UTC().Add(-time.Second) - - // Put Bucket before - Put Object into a bucket - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - twoMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "two", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f5902ac237024bdd0c176cb93063dc4", - Size: 11, - } - - typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything, mock.Anything).Return(twoMetadata.Md5, nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - date2 := time.Now().UTC() - - resources.Maxkeys = 1000 - resources.Prefix = "" - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Twice() - typedDriver.On("ListObjects", "bucket", mock.Anything).Return([]drivers.ObjectMetadata{{}}, drivers.BucketResourcesMetadata{}, nil).Once() - objects, resources, err = driver.ListObjects("bucket", resources) - c.Assert(len(objects), Equals, 1) - c.Assert(resources.IsTruncated, Equals, false) - c.Assert(err, IsNil) - - var writer bytes.Buffer - - typedDriver.On("GetObjectMetadata", "bucket", "two").Return(twoMetadata, nil).Once() - typedDriver.SetGetObjectWriter("bucket", "two", []byte("hello world")) - typedDriver.On("GetObject", mock.Anything, "bucket", "two").Return(int64(11), nil).Once() - driver.GetObject(&writer, "bucket", "two") - - c.Assert(bytes.Equal(writer.Bytes(), []byte("hello world")), Equals, true) - - metadata, err := driver.GetObjectMetadata("bucket", "two") - c.Assert(err, IsNil) - lastModified := metadata.Created - - c.Assert(date1.Before(lastModified), Equals, true) - c.Assert(lastModified.Before(date2), Equals, true) -} - -func (s *MySuite) TestListBuckets(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("ListBuckets").Return([]drivers.BucketMetadata{}, nil).Once() - request, err := http.NewRequest("GET", testServer.URL+"/", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - listResponse, err := readListBucket(response.Body) - c.Assert(err, IsNil) - c.Assert(len(listResponse.Buckets.Bucket), Equals, 0) - - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - err = driver.CreateBucket("foo", "private") - c.Assert(err, IsNil) - - bucketMetadata := []drivers.BucketMetadata{ - {Name: "foo", Created: time.Now().UTC()}, - } - typedDriver.On("ListBuckets").Return(bucketMetadata, nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - listResponse, err = readListBucket(response.Body) - c.Assert(err, IsNil) - c.Assert(len(listResponse.Buckets.Bucket), Equals, 1) - c.Assert(listResponse.Buckets.Bucket[0].Name, Equals, "foo") - - typedDriver.On("CreateBucket", "bar", "private").Return(nil).Once() - err = driver.CreateBucket("bar", "private") - c.Assert(err, IsNil) - - bucketMetadata = []drivers.BucketMetadata{ - {Name: "bar", Created: time.Now().UTC()}, - bucketMetadata[0], - } - - typedDriver.On("ListBuckets").Return(bucketMetadata, nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - listResponse, err = readListBucket(response.Body) - c.Assert(err, IsNil) - c.Assert(len(listResponse.Buckets.Bucket), Equals, 2) - - c.Assert(listResponse.Buckets.Bucket[0].Name, Equals, "bar") - c.Assert(listResponse.Buckets.Bucket[1].Name, Equals, "foo") -} - -func readListBucket(reader io.Reader) (ListBucketsResponse, error) { - var results ListBucketsResponse - decoder := xml.NewDecoder(reader) - err := decoder.Decode(&results) - return results, err -} - -func (s *MySuite) TestListObjects(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - // TODO Implement -} - -func (s *MySuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - objectMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object1", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f5902ac237024bdd0c176cb93063dc4", - Size: 11, - } - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, drivers.BucketNotFound{}).Once() - typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything, mock.Anything).Return(objectMetadata.Md5, nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket/object1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) -} - -func (s *MySuite) TestHeadOnObject(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - objectMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "object1", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f5902ac237024bdd0c176cb93063dc4", - Size: 11, - } - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObject", "bucket", "object1", "", "", mock.Anything, mock.Anything).Return(objectMetadata.Md5, nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/bucket/object1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "object1").Return(objectMetadata, nil).Once() - request, err = http.NewRequest("HEAD", testServer.URL+"/bucket/object1", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -func (s *MySuite) TestHeadOnBucket(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - request, err = http.NewRequest("HEAD", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -func (s *MySuite) TestDateFormat(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket", nil) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - // set an invalid date - request.Header.Set("Date", "asfasdfadf") - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "RequestTimeTooSkewed", - "The difference between the request time and the server's time is too large.", http.StatusForbidden) - - request.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -func verifyHeaders(c *C, header http.Header, date time.Time, size int, contentType string, etag string) { - // Verify date - c.Assert(header.Get("Last-Modified"), Equals, date.Format(http.TimeFormat)) - - // verify size - c.Assert(header.Get("Content-Length"), Equals, strconv.Itoa(size)) - - // verify content type - c.Assert(header.Get("Content-Type"), Equals, contentType) - - // verify etag - c.Assert(header.Get("Etag"), Equals, "\""+etag+"\"") -} - -func (s *MySuite) TestXMLNameNotInBucketListJson(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - err := driver.CreateBucket("foo", "private") - c.Assert(err, IsNil) - - typedDriver.On("ListBuckets").Return([]drivers.BucketMetadata{{Name: "foo", Created: time.Now().UTC()}}, nil) - request, err := http.NewRequest("GET", testServer.URL+"/", nil) - c.Assert(err, IsNil) - request.Header.Add("Accept", "application/json") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - byteResults, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) -} - -func (s *MySuite) TestXMLNameNotInObjectListJson(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - err := driver.CreateBucket("foo", "private") - c.Assert(err, IsNil) - - resources := drivers.BucketResourcesMetadata{} - resources.Maxkeys = 1000 - resources.Prefix = "" - - metadata := drivers.BucketMetadata{ - Name: "foo", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - - typedDriver.On("GetBucketMetadata", "foo").Return(metadata, nil).Once() - typedDriver.On("ListObjects", "foo", resources).Return([]drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{}, nil).Once() - request, err := http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - request.Header.Add("Accept", "application/json") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - byteResults, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) -} - -func (s *MySuite) TestContentTypePersists(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - typedDriver.On("CreateBucket", "bucket", "private").Return(nil).Once() - err := driver.CreateBucket("bucket", "private") - c.Assert(err, IsNil) - - metadata := drivers.BucketMetadata{ - Name: "bucket", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - // test head - oneMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "one", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "d41d8cd98f00b204e9800998ecf8427e", - Size: 0, - } - - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() - typedDriver.On("CreateObject", "bucket", "one", "", "", mock.Anything, mock.Anything).Return(oneMetadata.Md5, nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/bucket/one", bytes.NewBufferString("hello world")) - delete(request.Header, "Content-Type") - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "one").Return(oneMetadata, nil).Once() - request, err = http.NewRequest("HEAD", testServer.URL+"/bucket/one", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") - - // test get object - typedDriver.SetGetObjectWriter("bucket", "once", []byte("")) - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Twice() - typedDriver.On("GetObjectMetadata", "bucket", "one").Return(oneMetadata, nil).Once() - typedDriver.On("GetObject", mock.Anything, "bucket", "one").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/one", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") - - twoMetadata := drivers.ObjectMetadata{ - Bucket: "bucket", - Key: "one", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - // Fix MD5 - Md5: "d41d8cd98f00b204e9800998ecf8427e", - Size: 0, - } - - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() - typedDriver.On("CreateObject", "bucket", "two", "", "", mock.Anything, mock.Anything).Return(twoMetadata.Md5, nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world")) - delete(request.Header, "Content-Type") - request.Header.Add("Content-Type", "application/json") - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Once() - typedDriver.On("GetObjectMetadata", "bucket", "two").Return(twoMetadata, nil).Once() - request, err = http.NewRequest("HEAD", testServer.URL+"/bucket/two", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") - - // test get object - typedDriver.On("GetBucketMetadata", "bucket").Return(metadata, nil).Twice() - typedDriver.On("GetObjectMetadata", "bucket", "two").Return(twoMetadata, nil).Once() - typedDriver.On("GetObject", mock.Anything, "bucket", "two").Return(int64(0), nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/bucket/two", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") -} - -func (s *MySuite) TestPartialContent(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - - metadata := drivers.ObjectMetadata{ - Bucket: "foo", - Key: "bar", - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "6f5902ac237024bdd0c176cb93063dc4", // even for range requests, md5sum is returned for the full object - Size: 11, - } - - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - typedDriver.On("CreateObject", "foo", "bar", "", "", mock.Anything, mock.Anything).Return(metadata.Md5, nil).Once() - err := driver.CreateBucket("foo", "private") - c.Assert(err, IsNil) - - driver.CreateObject("foo", "bar", "", "", int64(len("hello world")), bytes.NewBufferString("hello world")) - - // prepare for GET on range request - typedDriver.SetGetObjectWriter("foo", "bar", []byte("hello world")) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "bar").Return(metadata, nil).Once() - typedDriver.On("GetPartialObject", mock.Anything, "foo", "bar", int64(6), int64(2)).Return(int64(2), nil).Once() - - // prepare request - request, err := http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - request.Header.Add("Accept", "application/json") - request.Header.Add("Range", "bytes=6-7") - setDummyAuthHeader(request) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - partialObject, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - c.Assert(string(partialObject), Equals, "wo") -} - -func (s *MySuite) TestListObjectsHandlerErrors(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - default: - { - return - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, drivers.BucketNameInvalid{}).Once() - request, err := http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, drivers.BucketNotFound{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("ListObjects", "foo", mock.Anything).Return(make([]drivers.ObjectMetadata, 0), drivers.BucketResourcesMetadata{}, drivers.ObjectNameInvalid{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("ListObjects", "foo", mock.Anything).Return(make([]drivers.ObjectMetadata, 0), drivers.BucketResourcesMetadata{}, drivers.ObjectNotFound{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, drivers.BackendCorrupted{}).Once() - typedDriver.On("ListObjects", "foo", mock.Anything).Return(make([]drivers.ObjectMetadata, 0), drivers.BucketResourcesMetadata{}, drivers.BackendCorrupted{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InternalError", "We encountered an internal error, please try again.", http.StatusInternalServerError) -} - -func (s *MySuite) TestListBucketsErrors(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - default: - { - return - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - metadata := drivers.BucketMetadata{ - Name: "foo", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - - typedDriver.On("GetBucketMetadata", "foo").Return(metadata, nil).Once() - typedDriver.On("ListObjects", "foo", mock.Anything).Return(make([]drivers.ObjectMetadata, 0), - drivers.BucketResourcesMetadata{}, drivers.BackendCorrupted{}).Once() - request, err := http.NewRequest("GET", testServer.URL+"/foo", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InternalError", "We encountered an internal error, please try again.", http.StatusInternalServerError) -} - -func (s *MySuite) TestPutBucketErrors(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - default: - { - return - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - typedDriver.On("CreateBucket", "foo", "private").Return(drivers.BucketNameInvalid{}).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) - - typedDriver.On("CreateBucket", "foo", "private").Return(drivers.BucketExists{}).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "BucketAlreadyExists", "The requested bucket name is not available.", http.StatusConflict) - - typedDriver.On("CreateBucket", "foo", "private").Return(drivers.BackendCorrupted{}).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "private") - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InternalError", "We encountered an internal error, please try again.", http.StatusInternalServerError) - - typedDriver.On("CreateBucket", "foo", "unknown").Return(nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - request.Header.Add("x-amz-acl", "unknown") - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) -} - -func (s *MySuite) TestGetObjectErrors(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - default: - { - return - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - metadata := drivers.BucketMetadata{ - Name: "foo", - Created: time.Now().UTC(), - ACL: drivers.BucketACL("private"), - } - typedDriver.On("GetBucketMetadata", "foo").Return(metadata, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "bar").Return(drivers.ObjectMetadata{}, drivers.ObjectNotFound{}).Once() - request, err := http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, drivers.BucketNotFound{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(metadata, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "bar").Return(drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, drivers.BucketNameInvalid{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) - - typedDriver.On("GetBucketMetadata", "foo").Return(metadata, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "bar").Return(drivers.ObjectMetadata{}, drivers.BackendCorrupted{}).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InternalError", "We encountered an internal error, please try again.", http.StatusInternalServerError) -} - -func (s *MySuite) TestGetObjectRangeErrors(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - default: - { - return - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - metadata := drivers.ObjectMetadata{ - Bucket: "foo", - Key: "bar", - - ContentType: "application/octet-stream", - Created: time.Now().UTC(), - Md5: "e81c4e4f2b7b93b481e13a8553c2ae1b", - Size: 11, - } - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "bar").Return(metadata, nil).Once() - request, err := http.NewRequest("GET", testServer.URL+"/foo/bar", nil) - request.Header.Add("Range", "bytes=7-6") - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) -} - -func (s *MySuite) TestObjectMultipartAbort(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - // create bucket - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - // Initiate multipart upload - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() - request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResult{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - uploadID := newResponse.UploadID - - // put part one - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // put part two - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("AbortMultipartUpload", "foo", "object", "uploadid").Return(nil).Once() - request, err = http.NewRequest("DELETE", testServer.URL+"/foo/object?uploadId="+uploadID, nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response3, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response3.StatusCode, Equals, http.StatusNoContent) -} - -func (s *MySuite) TestBucketMultipartList(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - // create bucket - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - // Initiate multipart upload - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() - request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResult{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - uploadID := newResponse.UploadID - - // put part one - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // put part two - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("ListMultipartUploads", "foo", mock.Anything).Return(drivers.BucketMultipartResourcesMetadata{}, nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo?uploads", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response3, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response3.StatusCode, Equals, http.StatusOK) - - decoder = xml.NewDecoder(response3.Body) - newResponse3 := &ListMultipartUploadsResponse{} - err = decoder.Decode(newResponse3) - c.Assert(err, IsNil) - c.Assert(newResponse3.Bucket, Equals, "foo") -} - -func (s *MySuite) TestObjectMultipartList(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - // create bucket - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - // Initiate multipart upload - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() - request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResult{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - uploadID := newResponse.UploadID - - // put part one - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // put part two - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("ListObjectParts", "foo", "object", mock.Anything).Return(drivers.ObjectResourcesMetadata{}, nil).Once() - request, err = http.NewRequest("GET", testServer.URL+"/foo/object?uploadId="+uploadID, nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response3, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response3.StatusCode, Equals, http.StatusOK) - -} - -func (s *MySuite) TestObjectMultipart(c *C) { - switch driver := s.Driver.(type) { - case *mocks.Driver: - { - driver.AssertExpectations(c) - } - } - driver := s.Driver - typedDriver := s.MockDriver - - httpHandler := HTTPHandler(setConfig(driver)) - testServer := httptest.NewServer(httpHandler) - defer testServer.Close() - client := http.Client{} - - // create bucket - typedDriver.On("CreateBucket", "foo", "private").Return(nil).Once() - request, err := http.NewRequest("PUT", testServer.URL+"/foo", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - // Initiate multipart upload - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("NewMultipartUpload", "foo", "object", "").Return("uploadid", nil).Once() - request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploads", bytes.NewBufferString("")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResult{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - uploadID := newResponse.UploadID - - // put part one - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 1, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=1", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // put part two - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CreateObjectPart", "foo", "object", "uploadid", 2, "", "", 11, mock.Anything).Return("5eb63bbbe01eeed093cb22bb8f5acdc3", nil).Once() - request, err = http.NewRequest("PUT", testServer.URL+"/foo/object?uploadId="+uploadID+"&partNumber=2", bytes.NewBufferString("hello world")) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - // complete multipart upload - completeUploads := &CompleteMultipartUpload{ - Part: []Part{ - { - PartNumber: 1, - ETag: response1.Header.Get("ETag"), - }, - { - PartNumber: 2, - ETag: response2.Header.Get("ETag"), - }, - }, - } - - var completeBuffer bytes.Buffer - encoder := xml.NewEncoder(&completeBuffer) - encoder.Encode(completeUploads) - - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("CompleteMultipartUpload", "foo", "object", "uploadid", mock.Anything).Return("etag", nil).Once() - request, err = http.NewRequest("POST", testServer.URL+"/foo/object?uploadId="+uploadID, &completeBuffer) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // get data - typedDriver.On("GetBucketMetadata", "foo").Return(drivers.BucketMetadata{}, nil).Once() - typedDriver.On("GetObjectMetadata", "foo", "object").Return(drivers.ObjectMetadata{Size: 22}, nil).Once() - typedDriver.On("GetObject", mock.Anything, "foo", "object").Return(int64(22), nil).Once() - typedDriver.SetGetObjectWriter("foo", "object", []byte("hello worldhello world")) - request, err = http.NewRequest("GET", testServer.URL+"/foo/object", nil) - c.Assert(err, IsNil) - setDummyAuthHeader(request) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - object, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(string(object), Equals, ("hello worldhello world")) -} - -func verifyError(c *C, response *http.Response, code, description string, statusCode int) { - data, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - errorResponse := ErrorResponse{} - err = xml.Unmarshal(data, &errorResponse) - c.Assert(err, IsNil) - c.Assert(errorResponse.Code, Equals, code) - c.Assert(errorResponse.Message, Equals, description) - c.Assert(response.StatusCode, Equals, statusCode) -} - -func startMockDriver() *mocks.Driver { - return &mocks.Driver{ - ObjectWriterData: make(map[string][]byte), - } -} diff --git a/pkg/api/quota/bandwidth_cap.go b/pkg/api/quota/bandwidth_cap.go deleted file mode 100644 index 33ddcac81..000000000 --- a/pkg/api/quota/bandwidth_cap.go +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package quota - -import ( - "errors" - "io" - "net" - "net/http" - "time" - - "sync" - - "github.com/minio/minio/pkg/iodine" - "github.com/minio/minio/pkg/utils/log" -) - -// bandwidthQuotaHandler -type bandwidthQuotaHandler struct { - handler http.Handler - quotas *quotaMap -} - -// ServeHTTP is an http.Handler ServeHTTP method -func (h *bandwidthQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - host, _, _ := net.SplitHostPort(req.RemoteAddr) - longIP := longIP{net.ParseIP(host)}.IptoUint32() - if h.quotas.WillExceedQuota(longIP, req.ContentLength) { - hosts, _ := net.LookupAddr(uint32ToIP(longIP).String()) - log.Debug.Printf("Offending Host: %s, BandwidthUsed: %d", hosts, h.quotas.GetQuotaUsed(longIP)) - writeErrorResponse(w, req, BandWidthInsufficientToProceed, req.URL.Path) - return - } - qr := "aReader{ - ReadCloser: req.Body, - quotas: h.quotas, - ip: longIP, - w: w, - req: req, - lock: &sync.RWMutex{}, - } - req.Body = qr - w = "aWriter{ - ResponseWriter: w, - quotas: h.quotas, - ip: longIP, - quotaReader: qr, - } - h.handler.ServeHTTP(w, req) -} - -// BandwidthCap sets a quote based upon bandwidth used -func BandwidthCap(h http.Handler, limit int64, duration time.Duration) http.Handler { - return &bandwidthQuotaHandler{ - handler: h, - quotas: "aMap{ - data: make(map[int64]map[uint32]int64), - limit: int64(limit), - duration: duration, - segmentSize: segmentSize(duration), - }, - } -} - -type quotaReader struct { - io.ReadCloser - quotas *quotaMap - ip uint32 - w http.ResponseWriter - req *http.Request - err bool - lock *sync.RWMutex -} - -func (q *quotaReader) Read(b []byte) (int, error) { - log.Println(q.quotas.GetQuotaUsed(q.ip)) - log.Println(q.quotas.limit) - q.lock.Lock() - defer q.lock.Unlock() - if q.err { - return 0, iodine.New(errors.New("Quota Met"), nil) - } - if q.err == false && q.quotas.IsQuotaMet(q.ip) { - defer q.lock.Unlock() - q.err = true - hosts, _ := net.LookupAddr(uint32ToIP(q.ip).String()) - log.Debug.Printf("Offending Host: %s, BandwidthUsed: %d", hosts, q.quotas.GetQuotaUsed(q.ip)) - writeErrorResponse(q.w, q.req, BandWidthQuotaExceeded, q.req.URL.Path) - return 0, iodine.New(errors.New("Quota Met"), nil) - } - n, err := q.ReadCloser.Read(b) - q.quotas.Add(q.ip, int64(n)) - return n, iodine.New(err, nil) -} - -func (q *quotaReader) Close() error { - return iodine.New(q.ReadCloser.Close(), nil) -} - -type quotaWriter struct { - ResponseWriter http.ResponseWriter - quotas *quotaMap - ip uint32 - quotaReader *quotaReader -} - -func (q *quotaWriter) Write(b []byte) (int, error) { - q.quotaReader.lock.RLock() - defer q.quotaReader.lock.RUnlock() - if q.quotas.IsQuotaMet(q.ip) { - return 0, iodine.New(errors.New("Quota Met"), nil) - } - q.quotas.Add(q.ip, int64(len(b))) - n, err := q.ResponseWriter.Write(b) - // remove from quota if a full write isn't performed - q.quotas.Add(q.ip, int64(n-len(b))) - return n, iodine.New(err, nil) -} -func (q *quotaWriter) Header() http.Header { - return q.ResponseWriter.Header() -} - -func (q *quotaWriter) WriteHeader(status int) { - q.quotaReader.lock.RLock() - defer q.quotaReader.lock.RUnlock() - if q.quotas.IsQuotaMet(q.ip) || q.quotaReader.err { - return - } - q.ResponseWriter.WriteHeader(status) -} - -func segmentSize(duration time.Duration) time.Duration { - var segmentSize time.Duration - for i := int64(1); i < duration.Nanoseconds(); i = i * 10 { - segmentSize = time.Duration(i) - } - return segmentSize -} diff --git a/pkg/api/quota/conn_limit.go b/pkg/api/quota/conn_limit.go deleted file mode 100644 index 03f905486..000000000 --- a/pkg/api/quota/conn_limit.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package quota - -import ( - "net" - "net/http" - "sync" - - "github.com/minio/minio/pkg/utils/log" -) - -// requestLimitHandler -type connLimit struct { - sync.RWMutex - handler http.Handler - connections map[uint32]int - limit int -} - -func (c *connLimit) IsLimitExceeded(ip uint32) bool { - if c.connections[ip] >= c.limit { - return true - } - return false -} - -func (c *connLimit) GetUsed(ip uint32) int { - return c.connections[ip] -} - -func (c *connLimit) Add(ip uint32) { - c.Lock() - defer c.Unlock() - count := c.connections[ip] - count = count + 1 - c.connections[ip] = count - return -} - -func (c *connLimit) Remove(ip uint32) { - c.Lock() - defer c.Unlock() - count, _ := c.connections[ip] - count = count - 1 - if count <= 0 { - delete(c.connections, ip) - return - } - c.connections[ip] = count -} - -// ServeHTTP is an http.Handler ServeHTTP method -func (c *connLimit) ServeHTTP(w http.ResponseWriter, req *http.Request) { - host, _, _ := net.SplitHostPort(req.RemoteAddr) - longIP := longIP{net.ParseIP(host)}.IptoUint32() - if c.IsLimitExceeded(longIP) { - hosts, _ := net.LookupAddr(uint32ToIP(longIP).String()) - log.Debug.Printf("Connection limit reached - Host: %s, Total Connections: %d\n", hosts, c.GetUsed(longIP)) - writeErrorResponse(w, req, ConnectionLimitExceeded, req.URL.Path) - return - } - c.Add(longIP) - defer c.Remove(longIP) - c.handler.ServeHTTP(w, req) -} - -// ConnectionLimit limits the number of concurrent connections -func ConnectionLimit(h http.Handler, limit int) http.Handler { - return &connLimit{ - handler: h, - connections: make(map[uint32]int), - limit: limit, - } -} diff --git a/pkg/api/quota/errors.go b/pkg/api/quota/errors.go deleted file mode 100644 index c4ad019e5..000000000 --- a/pkg/api/quota/errors.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package quota - -import ( - "bytes" - "encoding/xml" - "net/http" -) - -// copied from api, no cyclic deps allowed - -// Error structure -type Error struct { - Code string - Description string - HTTPStatusCode int -} - -// ErrorResponse - error response format -type ErrorResponse struct { - XMLName xml.Name `xml:"Error" json:"-"` - Code string - Message string - Resource string - RequestID string - HostID string -} - -// Quota standard errors non exhaustive list -const ( - RequestTimeTooSkewed = iota - BandWidthQuotaExceeded - BandWidthInsufficientToProceed - ConnectionLimitExceeded - SlowDown -) - -// Golang http doesn't implement these -const ( - StatusTooManyRequests = 429 -) - -func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorType int, resource string) { - error := getErrorCode(errorType) - errorResponse := getErrorResponse(error, resource) - encodedErrorResponse := encodeErrorResponse(errorResponse) - // set headers - writeErrorHeaders(w) - w.WriteHeader(error.HTTPStatusCode) - // write body - w.Write(encodedErrorResponse) -} - -func writeErrorHeaders(w http.ResponseWriter) { - w.Header().Set("Server", "Minio") - w.Header().Set("Accept-Ranges", "bytes") - w.Header().Set("Content-Type", "application/xml") - w.Header().Set("Connection", "close") -} - -// Error code to Error structure map -var errorCodeResponse = map[int]Error{ - BandWidthQuotaExceeded: { - Code: "BandwidthQuotaExceeded", - Description: "Bandwidth Quota Exceeded.", - HTTPStatusCode: StatusTooManyRequests, - }, - BandWidthInsufficientToProceed: { - Code: "BandwidthQuotaWillBeExceeded", - Description: "Bandwidth quota will be exceeded with this request.", - HTTPStatusCode: StatusTooManyRequests, - }, - ConnectionLimitExceeded: { - Code: "ConnectionLimitExceeded", - Description: "Connections Limit Exceeded.", - HTTPStatusCode: StatusTooManyRequests, - }, - SlowDown: { - Code: "SlowDown", - Description: "Reduce your request rate.", - HTTPStatusCode: StatusTooManyRequests, - }, -} - -// Write error response headers -func encodeErrorResponse(response interface{}) []byte { - var bytesBuffer bytes.Buffer - encoder := xml.NewEncoder(&bytesBuffer) - encoder.Encode(response) - return bytesBuffer.Bytes() -} - -// errorCodeError provides errorCode to Error. It returns empty if the code provided is unknown -func getErrorCode(code int) Error { - return errorCodeResponse[code] -} - -// getErrorResponse gets in standard error and resource value and -// provides a encodable populated response values -func getErrorResponse(err Error, resource string) ErrorResponse { - var data = ErrorResponse{} - data.Code = err.Code - data.Message = err.Description - if resource != "" { - data.Resource = resource - } - // TODO implement this in future - data.RequestID = "3L137" - data.HostID = "3L137" - - return data -} diff --git a/pkg/api/quota/quota_handler.go b/pkg/api/quota/quota_handler.go deleted file mode 100644 index 4b56c1510..000000000 --- a/pkg/api/quota/quota_handler.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package quota - -import ( - "encoding/binary" - "net" - "sync" - "time" -) - -// map[minute][address] = current quota -type quotaMap struct { - sync.RWMutex - data map[int64]map[uint32]int64 - limit int64 - duration time.Duration - segmentSize time.Duration -} - -func (q *quotaMap) CanExpire() { - q.Lock() - defer q.Unlock() - currentMinute := time.Now().UTC().UnixNano() / q.segmentSize.Nanoseconds() - // divide by segmentSize, otherwise expiredQuotas will always be negative - expiredQuotas := currentMinute - (q.duration.Nanoseconds() / q.segmentSize.Nanoseconds()) - for time := range q.data { - if time < expiredQuotas { - delete(q.data, time) - } - } -} - -func (q *quotaMap) Add(ip uint32, size int64) { - q.CanExpire() - q.Lock() - defer q.Unlock() - currentMinute := time.Now().UTC().UnixNano() / q.segmentSize.Nanoseconds() - if _, ok := q.data[currentMinute]; !ok { - q.data[currentMinute] = make(map[uint32]int64) - } - currentData, _ := q.data[currentMinute][ip] - proposedDataSize := currentData + size - q.data[currentMinute][ip] = proposedDataSize -} - -func (q *quotaMap) IsQuotaMet(ip uint32) bool { - q.CanExpire() - if q.GetQuotaUsed(ip) >= q.limit { - return true - } - return false -} - -func (q *quotaMap) GetQuotaUsed(ip uint32) (total int64) { - q.CanExpire() - q.RLock() - defer q.RUnlock() - for _, segment := range q.data { - if used, ok := segment[ip]; ok { - total += used - } - } - return -} - -func (q *quotaMap) WillExceedQuota(ip uint32, size int64) (result bool) { - return q.GetQuotaUsed(ip)+size > q.limit -} - -type longIP struct { - net.IP -} - -// []byte to uint32 representation -func (p longIP) IptoUint32() (result uint32) { - ip := p.To4() - if ip == nil { - return 0 - } - return binary.BigEndian.Uint32(ip) -} diff --git a/pkg/api/quota/request_limit.go b/pkg/api/quota/request_limit.go deleted file mode 100644 index 976b017aa..000000000 --- a/pkg/api/quota/request_limit.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package quota - -import ( - "encoding/binary" - "net" - "net/http" - "time" - - "github.com/minio/minio/pkg/utils/log" -) - -// requestLimitHandler -type requestLimitHandler struct { - handler http.Handler - quotas *quotaMap -} - -//convert a uint32 to an ipv4 -func uint32ToIP(ip uint32) net.IP { - addr := net.IP{0, 0, 0, 0} - binary.BigEndian.PutUint32(addr, ip) - return addr -} - -// ServeHTTP is an http.Handler ServeHTTP method -func (h *requestLimitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - host, _, _ := net.SplitHostPort(req.RemoteAddr) - longIP := longIP{net.ParseIP(host)}.IptoUint32() - if h.quotas.IsQuotaMet(longIP) { - hosts, _ := net.LookupAddr(uint32ToIP(longIP).String()) - log.Debug.Printf("Offending Host: %s, RequestUSED: %d\n", hosts, h.quotas.GetQuotaUsed(longIP)) - writeErrorResponse(w, req, SlowDown, req.URL.Path) - } - h.quotas.Add(longIP, 1) - h.handler.ServeHTTP(w, req) -} - -// RequestLimit sets a quote based upon number of requests allowed over a time period -func RequestLimit(h http.Handler, limit int64, duration time.Duration) http.Handler { - return &requestLimitHandler{ - handler: h, - quotas: "aMap{ - data: make(map[int64]map[uint32]int64), - limit: int64(limit), - duration: duration, - segmentSize: segmentSize(duration), - }, - } -} diff --git a/pkg/server/httpserver/httpserver.go b/pkg/api/server.go similarity index 67% rename from pkg/server/httpserver/httpserver.go rename to pkg/api/server.go index 31aac54b3..4aaf179b8 100644 --- a/pkg/server/httpserver/httpserver.go +++ b/pkg/api/server.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package httpserver +package api import ( "fmt" @@ -32,38 +32,27 @@ type Config struct { RateLimit int } -// Server - http server related -type Server struct { - config Config - handler http.Handler -} - // Start http server -func Start(handler http.Handler, config Config) (chan<- string, <-chan error, Server) { - ctrlChannel := make(chan string) - errorChannel := make(chan error) - server := Server{ - config: config, - handler: handler, - } - go start(ctrlChannel, errorChannel, server) - return ctrlChannel, errorChannel, server +func Start(a API) <-chan error { + errCh := make(chan error) + go start(errCh, a) + return errCh } -func start(ctrlChannel <-chan string, errorChannel chan<- error, server Server) { - defer close(errorChannel) +func start(errCh chan error, a API) { + defer close(errCh) var err error // Minio server config httpServer := &http.Server{ - Addr: server.config.Address, - Handler: server.handler, + Addr: a.config.Address, + Handler: a.handler, MaxHeaderBytes: 1 << 20, } - host, port, err := net.SplitHostPort(server.config.Address) + host, port, err := net.SplitHostPort(a.config.Address) if err != nil { - errorChannel <- err + errCh <- err return } @@ -74,7 +63,7 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, server Server) default: addrs, err := net.InterfaceAddrs() if err != nil { - errorChannel <- err + errCh <- err return } for _, addr := range addrs { @@ -92,14 +81,31 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, server Server) fmt.Printf("Starting minio server on: http://%s:%s\n", host, port) } err = httpServer.ListenAndServe() - case server.config.TLS == true: + case a.config.TLS == true: for _, host := range hosts { fmt.Printf("Starting minio server on: https://%s:%s\n", host, port) } - err = httpServer.ListenAndServeTLS(server.config.CertFile, server.config.KeyFile) + err = httpServer.ListenAndServeTLS(a.config.CertFile, a.config.KeyFile) } if err != nil { - errorChannel <- err + errCh <- err } + errCh <- nil + return +} +// API is used to build api server +type API struct { + config Config + handler http.Handler +} + +// StartServer APIFactory builds api server +func StartServer(conf Config) error { + for err := range Start(New(conf)) { + if err != nil { + return err + } + } + return nil } diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index 1913ce66a..000000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package server - -import ( - "errors" - "fmt" - "reflect" - "time" - - "github.com/minio/minio/pkg/api" - "github.com/minio/minio/pkg/api/web" - "github.com/minio/minio/pkg/iodine" - "github.com/minio/minio/pkg/server/httpserver" - "github.com/minio/minio/pkg/storage/drivers" - "github.com/minio/minio/pkg/storage/drivers/cache" - "github.com/minio/minio/pkg/storage/drivers/donut" - "github.com/minio/minio/pkg/utils/log" -) - -// WebFactory is used to build web cli server -type WebFactory struct { - httpserver.Config -} - -// GetStartServerFunc builds web cli server -func (f WebFactory) GetStartServerFunc() StartServerFunc { - return func() (chan<- string, <-chan error) { - ctrl, status, _ := httpserver.Start(web.HTTPHandler(), f.Config) - return ctrl, status - } -} - -// Factory is used to build api server -type Factory struct { - httpserver.Config - Paths []string - MaxMemory uint64 - Expiration time.Duration -} - -// GetStartServerFunc Factory builds api server -func (f Factory) GetStartServerFunc() StartServerFunc { - return func() (chan<- string, <-chan error) { - conf := api.Config{RateLimit: f.RateLimit} - var driver drivers.Driver - var err error - if len(f.Paths) != 0 { - driver, err = donut.NewDriver(f.Paths) - if err != nil { - log.Fatalln(err) - } - driver, err = cache.NewDriver(f.MaxMemory, f.Expiration, driver) - if err != nil { - log.Fatalln(err) - } - } - conf.SetDriver(driver) - ctrl, status, _ := httpserver.Start(api.HTTPHandler(conf), f.Config) - return ctrl, status - } -} - -// StartServerFunc describes a function that can be used to start a server with StartMinio -type StartServerFunc func() (chan<- string, <-chan error) - -// StartMinio starts minio server -func StartMinio(servers []StartServerFunc) { - var ctrlChannels []chan<- string - var errChannels []<-chan error - for _, server := range servers { - ctrlChannel, errChannel := server() - ctrlChannels = append(ctrlChannels, ctrlChannel) - errChannels = append(errChannels, errChannel) - } - cases := createSelectCases(errChannels) - for len(cases) > 0 { - chosen, value, recvOk := reflect.Select(cases) - switch recvOk { - case true: - // Status Message Received - switch true { - case value.Interface() != nil: - // For any error received cleanup all existing channels and fail - for _, ch := range ctrlChannels { - close(ch) - } - msg := fmt.Sprintf("%q", value.Interface()) - log.Fatal(iodine.New(errors.New(msg), nil)) - } - case false: - // Channel closed, remove from list - var aliveStatusChans []<-chan error - for i, ch := range errChannels { - if i != chosen { - aliveStatusChans = append(aliveStatusChans, ch) - } - } - // create new select cases without defunct channel - errChannels = aliveStatusChans - cases = createSelectCases(errChannels) - } - } -} - -func createSelectCases(channels []<-chan error) []reflect.SelectCase { - cases := make([]reflect.SelectCase, len(channels)) - for i, ch := range channels { - cases[i] = reflect.SelectCase{ - Dir: reflect.SelectRecv, - Chan: reflect.ValueOf(ch), - } - } - return cases -}