From a42a48a455d8998948fac4f1df09b9debe70eaac Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 11 Mar 2015 01:01:49 -0700 Subject: [PATCH] Get object range request added --- pkg/api/minioapi/api_test.go | 17 ++-- pkg/api/minioapi/headers.go | 12 +++ pkg/api/minioapi/object_handlers.go | 36 ++++++++- pkg/api/minioapi/range.go | 118 ++++++++++++++++++++++++++++ pkg/storage/fs/fs_object.go | 48 +++++++++++ pkg/storage/inmemory/inmemory.go | 5 ++ pkg/storage/storage.go | 1 + 7 files changed, 226 insertions(+), 11 deletions(-) create mode 100644 pkg/api/minioapi/range.go diff --git a/pkg/api/minioapi/api_test.go b/pkg/api/minioapi/api_test.go index 903edff69..007f51fd3 100644 --- a/pkg/api/minioapi/api_test.go +++ b/pkg/api/minioapi/api_test.go @@ -364,16 +364,17 @@ func (s *MySuite) TestDateFormat(c *C) { func verifyHeaders(c *C, header http.Header, date time.Time, size int, contentType string, etag string) { // Verify date - c.Assert(header["Last-Modified"][0], Equals, date.Format(time.RFC1123)) + c.Log(header) + c.Assert(header.Get("Last-Modified"), Equals, date.Format(time.RFC1123)) // verify size - c.Assert(header["Content-Length"][0], Equals, strconv.Itoa(size)) + c.Assert(header.Get("Content-Length"), Equals, strconv.Itoa(size)) // verify content type - c.Assert(header["Content-Type"][0], Equals, contentType) + c.Assert(header.Get("Content-Type"), Equals, contentType) // verify etag - c.Assert(header["Etag"][0], Equals, etag) + c.Assert(header.Get("Etag"), Equals, etag) } func (s *MySuite) TestXMLNameNotInBucketListJson(c *C) { @@ -448,11 +449,11 @@ func (s *MySuite) TestContentTypePersists(c *C) { c.Assert(err, IsNil) response, err = client.Do(request) c.Assert(err, IsNil) - c.Assert(response.Header["Content-Type"][0], Equals, "application/octet-stream") + c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") // test get object response, err = http.Get(testServer.URL + "/bucket/one") - c.Assert(response.Header["Content-Type"][0], Equals, "application/octet-stream") + c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") request, err = http.NewRequest("PUT", testServer.URL+"/bucket/two", bytes.NewBufferString("hello world")) delete(request.Header, "Content-Type") @@ -466,9 +467,9 @@ func (s *MySuite) TestContentTypePersists(c *C) { c.Assert(err, IsNil) response, err = client.Do(request) c.Assert(err, IsNil) - c.Assert(response.Header["Content-Type"][0], Equals, "application/octet-stream") + c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") // test get object response, err = http.Get(testServer.URL + "/bucket/two") - c.Assert(response.Header["Content-Type"][0], Equals, "application/octet-stream") + c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") } diff --git a/pkg/api/minioapi/headers.go b/pkg/api/minioapi/headers.go index f6b2c9fc7..cb30964d3 100644 --- a/pkg/api/minioapi/headers.go +++ b/pkg/api/minioapi/headers.go @@ -37,6 +37,7 @@ type encoder interface { // Write http common headers func writeCommonHeaders(w http.ResponseWriter, acceptsType string) { w.Header().Set("Server", "Minio") + w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("Content-Type", acceptsType) } @@ -67,6 +68,17 @@ func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata) w.Header().Set("Connection", "close") } +// Write range object header +func writeRangeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata, ra string) { + lastModified := metadata.Created.Format(time.RFC1123) + // common headers + writeCommonHeaders(w, metadata.ContentType) + w.Header().Set("ETag", metadata.ETag) + w.Header().Set("Last-Modified", lastModified) + w.Header().Set("Content-Range", ra) + w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) +} + // Write object header and response func writeObjectHeadersAndResponse(w http.ResponseWriter, response interface{}, acceptsType contentType) []byte { var bytesBuffer bytes.Buffer diff --git a/pkg/api/minioapi/object_handlers.go b/pkg/api/minioapi/object_handlers.go index de3390969..5038b2f7a 100644 --- a/pkg/api/minioapi/object_handlers.go +++ b/pkg/api/minioapi/object_handlers.go @@ -30,7 +30,6 @@ import ( // you must have READ access to the object. func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Request) { var object, bucket string - acceptsContentType := getContentType(req) vars := mux.Vars(req) bucket = vars["bucket"] @@ -41,9 +40,40 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques case nil: // success { log.Println("Found: " + bucket + "#" + object) - writeObjectHeaders(w, metadata) - if _, err := server.storage.CopyObjectToWriter(w, bucket, object); err != nil { + httpRange, err := newRange(req, metadata.Size) + if err != nil { log.Println(err) + error := errorCodeError(InvalidRange) + errorResponse := getErrorResponse(error, "/"+bucket+"/"+object) + w.WriteHeader(error.HTTPStatusCode) + w.Write(writeErrorResponse(w, errorResponse, acceptsContentType)) + return + } + switch httpRange.start == 0 && httpRange.length == 0 { + case true: + writeObjectHeaders(w, metadata) + if _, err := server.storage.CopyObjectToWriter(w, bucket, object); err != nil { + log.Println(err) + error := errorCodeError(InternalError) + errorResponse := getErrorResponse(error, "/"+bucket+"/"+object) + w.WriteHeader(error.HTTPStatusCode) + w.Write(writeErrorResponse(w, errorResponse, acceptsContentType)) + return + } + case false: + metadata.Size = httpRange.length + writeRangeObjectHeaders(w, metadata, httpRange.getContentRange()) + w.WriteHeader(http.StatusPartialContent) + _, err := server.storage.CopyObjectToWriterRange(w, bucket, object, httpRange.start, httpRange.length) + if err != nil { + log.Println(err) + error := errorCodeError(InternalError) + errorResponse := getErrorResponse(error, "/"+bucket+"/"+object) + w.WriteHeader(error.HTTPStatusCode) + w.Write(writeErrorResponse(w, errorResponse, acceptsContentType)) + return + } + } } case mstorage.ObjectNotFound: diff --git a/pkg/api/minioapi/range.go b/pkg/api/minioapi/range.go new file mode 100644 index 000000000..885090bee --- /dev/null +++ b/pkg/api/minioapi/range.go @@ -0,0 +1,118 @@ +/* + * Mini Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minioapi + +import ( + "fmt" + "strconv" + "strings" + + "net/http" +) + +const ( + b = "bytes=" +) + +// HttpRange specifies the byte range to be sent to the client. +type httpRange struct { + start, length, size int64 +} + +// GetContentRange populate range header +func (r *httpRange) getContentRange() string { + return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, r.size) +} + +// Grab new range from request header +func newRange(req *http.Request, size int64) (*httpRange, error) { + r := &httpRange{ + start: 0, + length: 0, + size: 0, + } + r.size = size + if s := req.Header.Get("Range"); s != "" { + err := r.parseRange(s) + if err != nil { + return nil, err + } + } + return r, nil +} + +// parseRange parses a Range header string as per RFC 2616. +func (r *httpRange) parseRange(s string) error { + if s == "" { + return fmt.Errorf("header not present") + } + if !strings.HasPrefix(s, b) { + return fmt.Errorf("invalid range") + } + + ras := strings.Split(s[len(b):], ",") + if len(ras) == 0 { + return fmt.Errorf("invalid request") + } + // Just pick the first one and ignore the rest, we only support one range per object + if len(ras) > 1 { + return fmt.Errorf("multiple ranges specified") + } + + ra := strings.TrimSpace(ras[0]) + if ra == "" { + return fmt.Errorf("invalid range") + } + i := strings.Index(ra, "-") + if i < 0 { + return fmt.Errorf("invalid range") + } + start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file. + i, err := strconv.ParseInt(end, 10, 64) + if err != nil { + return fmt.Errorf("invalid range") + } + if i > r.size { + i = r.size + } + r.start = r.size - i + r.length = r.size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i > r.size || i < 0 { + return fmt.Errorf("invalid range") + } + r.start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.length = r.size - r.start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > i { + return fmt.Errorf("invalid range") + } + if i >= r.size { + i = r.size - 1 + } + r.length = i - r.start + 1 + } + } + return nil +} diff --git a/pkg/storage/fs/fs_object.go b/pkg/storage/fs/fs_object.go index cfb0a6ced..e423712a6 100644 --- a/pkg/storage/fs/fs_object.go +++ b/pkg/storage/fs/fs_object.go @@ -31,6 +31,54 @@ import ( /// Object Operations +// CopyObjectToWriterRange - GET object from range +func (storage *Storage) CopyObjectToWriterRange(w io.Writer, bucket, object string, start, length int64) (int64, error) { + // validate bucket + if mstorage.IsValidBucket(bucket) == false { + return 0, mstorage.BucketNameInvalid{Bucket: bucket} + } + + // validate object + if mstorage.IsValidObject(object) == false { + return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object} + } + + objectPath := path.Join(storage.root, bucket, object) + filestat, err := os.Stat(objectPath) + switch err := err.(type) { + case nil: + { + if filestat.IsDir() { + return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object} + } + } + default: + { + if os.IsNotExist(err) { + return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object} + } + return 0, mstorage.EmbedError(bucket, object, err) + } + } + file, err := os.Open(objectPath) + defer file.Close() + if err != nil { + return 0, mstorage.EmbedError(bucket, object, err) + } + + _, err = file.Seek(start, os.SEEK_SET) + if err != nil { + return 0, mstorage.EmbedError(bucket, object, err) + } + + count, err := io.CopyN(w, file, length) + if err != nil { + return count, mstorage.EmbedError(bucket, object, err) + } + + return count, nil +} + // CopyObjectToWriter - GET object func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // validate bucket diff --git a/pkg/storage/inmemory/inmemory.go b/pkg/storage/inmemory/inmemory.go index 2cf3f5dc2..55ea670e3 100644 --- a/pkg/storage/inmemory/inmemory.go +++ b/pkg/storage/inmemory/inmemory.go @@ -76,6 +76,11 @@ func (storage *Storage) CopyObjectToWriter(w io.Writer, bucket string, object st return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object} } +// CopyObjectToWriterRange - GET object from memory buffer range +func (storage *Storage) CopyObjectToWriterRange(w io.Writer, bucket, object string, start, end int64) (int64, error) { + return 0, mstorage.APINotImplemented{API: "GetObjectRange"} +} + // StoreBucketPolicy - Not implemented func (storage *Storage) StoreBucketPolicy(bucket string, policy mstorage.BucketPolicy) error { return mstorage.APINotImplemented{API: "PutBucketPolicy"} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 29333eea3..4e9bef361 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -33,6 +33,7 @@ type Storage interface { // Object Operations CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) + CopyObjectToWriterRange(w io.Writer, bucket string, object string, start, length int64) (int64, error) GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error) StoreObject(bucket string, key string, contentType string, data io.Reader) error