Merge pull request #780 from harshavardhana/pr_out_collapse_getpartialobject_into_getobject_

Collapse GetPartialObject() into GetObject()
master
Harshavardhana 9 years ago
commit d70695fc6f
  1. 8
      pkg/donut/donut-v1_test.go
  2. 98
      pkg/donut/donut-v2.go
  3. 8
      pkg/donut/donut-v2_test.go
  4. 3
      pkg/donut/interfaces.go
  5. 3
      pkg/donut/signature-v4.go
  6. 29
      pkg/server/api/headers.go
  7. 24
      pkg/server/api/object-handlers.go
  8. 34
      pkg/server/api/range.go

@ -200,7 +200,7 @@ func (s *MyDonutSuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
var buffer bytes.Buffer
size, err := dd.GetObject(&buffer, "foo", "obj")
size, err := dd.GetObject(&buffer, "foo", "obj", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len(data)))
c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
@ -225,13 +225,13 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
c.Assert(err, IsNil)
var buffer1 bytes.Buffer
size, err := dd.GetObject(&buffer1, "foo5", "obj1")
size, err := dd.GetObject(&buffer1, "foo5", "obj1", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("one"))))
c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
var buffer2 bytes.Buffer
size, err = dd.GetObject(&buffer2, "foo5", "obj2")
size, err = dd.GetObject(&buffer2, "foo5", "obj2", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("two"))))
@ -274,7 +274,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
c.Assert(err, IsNil)
var buffer bytes.Buffer
size, err = dd.GetObject(&buffer, "foo5", "obj3")
size, err = dd.GetObject(&buffer, "foo5", "obj3", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("three"))))
c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))

@ -21,7 +21,6 @@ import (
"crypto/md5"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"io/ioutil"
"log"
@ -127,53 +126,7 @@ func New() (Interface, error) {
/// V2 API functions
// GetObject - GET object from cache buffer
func (donut API) GetObject(w io.Writer, bucket string, object string) (int64, error) {
donut.lock.Lock()
defer donut.lock.Unlock()
if !IsValidBucket(bucket) {
return 0, iodine.New(BucketNameInvalid{Bucket: bucket}, nil)
}
if !IsValidObjectName(object) {
return 0, iodine.New(ObjectNameInvalid{Object: object}, nil)
}
if !donut.storedBuckets.Exists(bucket) {
return 0, iodine.New(BucketNotFound{Bucket: bucket}, nil)
}
objectKey := bucket + "/" + object
data, ok := donut.objects.Get(objectKey)
if !ok {
if len(donut.config.NodeDiskMap) > 0 {
reader, size, err := donut.getObject(bucket, object)
if err != nil {
return 0, iodine.New(err, nil)
}
// new proxy writer to capture data read from disk
pw := NewProxyWriter(w)
written, err := io.CopyN(pw, reader, size)
if err != nil {
return 0, iodine.New(err, nil)
}
/// cache object read from disk
ok := donut.objects.Append(objectKey, pw.writtenBytes)
pw.writtenBytes = nil
go debug.FreeOSMemory()
if !ok {
return 0, iodine.New(InternalError{}, nil)
}
return written, nil
}
return 0, iodine.New(ObjectNotFound{Object: object}, nil)
}
written, err := io.CopyN(w, bytes.NewBuffer(data), int64(donut.objects.Len(objectKey)))
if err != nil {
return 0, iodine.New(err, nil)
}
return written, nil
}
// GetPartialObject - GET object from cache buffer range
func (donut API) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
func (donut API) GetObject(w io.Writer, bucket string, object string, start, length int64) (int64, error) {
donut.lock.Lock()
defer donut.lock.Unlock()
@ -196,35 +149,58 @@ func (donut API) GetPartialObject(w io.Writer, bucket, object string, start, len
Length: length,
}, errParams)
}
if !donut.storedBuckets.Exists(bucket) {
return 0, iodine.New(BucketNotFound{Bucket: bucket}, errParams)
}
objectKey := bucket + "/" + object
data, ok := donut.objects.Get(objectKey)
var written int64
var err error
if !ok {
if len(donut.config.NodeDiskMap) > 0 {
reader, _, err := donut.getObject(bucket, object)
reader, size, err := donut.getObject(bucket, object)
if err != nil {
return 0, iodine.New(err, nil)
}
if _, err := io.CopyN(ioutil.Discard, reader, start); err != nil {
return 0, iodine.New(err, nil)
if start > 0 {
if _, err := io.CopyN(ioutil.Discard, reader, start); err != nil {
return 0, iodine.New(err, errParams)
}
}
// new proxy writer to capture data read from disk
pw := NewProxyWriter(w)
written, err := io.CopyN(w, reader, length)
if err != nil {
return 0, iodine.New(err, nil)
if length > 0 {
written, err = io.CopyN(pw, reader, length)
if err != nil {
return 0, iodine.New(err, errParams)
}
} else {
written, err = io.CopyN(pw, reader, size)
if err != nil {
return 0, iodine.New(err, errParams)
}
}
/// cache object read from disk
ok := donut.objects.Append(objectKey, pw.writtenBytes)
pw.writtenBytes = nil
go debug.FreeOSMemory()
if !ok {
return 0, iodine.New(InternalError{}, nil)
return 0, iodine.New(InternalError{}, errParams)
}
return written, nil
}
return 0, iodine.New(ObjectNotFound{Object: object}, nil)
return 0, iodine.New(ObjectNotFound{Object: object}, errParams)
}
written, err := io.CopyN(w, bytes.NewBuffer(data[start:]), length)
if err != nil {
return 0, iodine.New(err, nil)
if start == 0 && length == 0 {
written, err = io.CopyN(w, bytes.NewBuffer(data), int64(donut.objects.Len(objectKey)))
if err != nil {
return 0, iodine.New(err, nil)
}
} else {
written, err = io.CopyN(w, bytes.NewBuffer(data[start:]), length)
if err != nil {
return 0, iodine.New(err, nil)
}
}
return written, nil
}
@ -306,11 +282,11 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
return iodine.New(err, nil)
}
if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
return iodine.New(errors.New("bad digest, md5sum mismatch"), nil)
return iodine.New(BadDigest{}, nil)
}
return nil
}
return iodine.New(errors.New("invalid argument"), nil)
return iodine.New(InvalidArgument{}, nil)
}
// CreateObject - create an object

@ -175,7 +175,7 @@ func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
var buffer bytes.Buffer
size, err := dc.GetObject(&buffer, "foo", "obj")
size, err := dc.GetObject(&buffer, "foo", "obj", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len(data)))
c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
@ -200,13 +200,13 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
c.Assert(err, IsNil)
var buffer1 bytes.Buffer
size, err := dc.GetObject(&buffer1, "foo5", "obj1")
size, err := dc.GetObject(&buffer1, "foo5", "obj1", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("one"))))
c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
var buffer2 bytes.Buffer
size, err = dc.GetObject(&buffer2, "foo5", "obj2")
size, err = dc.GetObject(&buffer2, "foo5", "obj2", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("two"))))
@ -249,7 +249,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
c.Assert(err, IsNil)
var buffer bytes.Buffer
size, err = dc.GetObject(&buffer, "foo5", "obj3")
size, err = dc.GetObject(&buffer, "foo5", "obj3", 0, 0)
c.Assert(err, IsNil)
c.Assert(size, Equals, int64(len([]byte("three"))))
c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))

@ -38,8 +38,7 @@ type CloudStorage interface {
ListObjects(string, BucketResourcesMetadata, *Signature) ([]ObjectMetadata, BucketResourcesMetadata, error)
// Object operations
GetObject(w io.Writer, bucket, object string) (int64, error)
GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObjectMetadata(bucket, object string, signature *Signature) (ObjectMetadata, error)
// bucket, object, expectedMD5Sum, size, reader, metadata, signature
CreateObject(string, string, string, int64, io.Reader, map[string]string, *Signature) (ObjectMetadata, error)

@ -20,7 +20,6 @@ import (
"bytes"
"crypto/hmac"
"encoding/hex"
"errors"
"net/http"
"regexp"
"sort"
@ -79,7 +78,7 @@ func urlEncodeName(name string) (string, error) {
default:
len := utf8.RuneLen(s)
if len < 0 {
return "", errors.New("invalid utf-8")
return "", iodine.New(InvalidArgument{}, nil)
}
u := make([]byte, len)
utf8.EncodeRune(u, s)

@ -78,23 +78,30 @@ func encodeErrorResponse(response interface{}, acceptsType contentType) []byte {
}
// Write object header
func setObjectHeaders(w http.ResponseWriter, metadata donut.ObjectMetadata) {
func setObjectHeaders(w http.ResponseWriter, metadata donut.ObjectMetadata, contentRange *httpRange) {
// set common headers
if contentRange != nil {
if contentRange.length > 0 {
setCommonHeaders(w, metadata.Metadata["contentType"], int(contentRange.length))
} else {
setCommonHeaders(w, metadata.Metadata["contentType"], int(metadata.Size))
}
} else {
setCommonHeaders(w, metadata.Metadata["contentType"], int(metadata.Size))
}
// set object headers
lastModified := metadata.Created.Format(http.TimeFormat)
// common headers
setCommonHeaders(w, metadata.Metadata["contentType"], int(metadata.Size))
// object related headers
w.Header().Set("ETag", "\""+metadata.MD5Sum+"\"")
w.Header().Set("Last-Modified", lastModified)
}
// Write range object header
func setRangeObjectHeaders(w http.ResponseWriter, metadata donut.ObjectMetadata, contentRange *httpRange) {
// set common headers
setCommonHeaders(w, metadata.Metadata["contentType"], int(metadata.Size))
// set object headers
setObjectHeaders(w, metadata)
// set content range
w.Header().Set("Content-Range", contentRange.getContentRange())
if contentRange != nil {
if contentRange.start > 0 || contentRange.length > 0 {
w.Header().Set("Content-Range", contentRange.String())
w.WriteHeader(http.StatusPartialContent)
}
}
}
func encodeSuccessResponse(response interface{}, acceptsType contentType) []byte {

@ -69,26 +69,16 @@ func (api Minio) GetObjectHandler(w http.ResponseWriter, req *http.Request) {
switch iodine.ToError(err).(type) {
case nil: // success
{
httpRange, err := getRequestedRange(req, metadata.Size)
httpRange, err := getRequestedRange(req.Header.Get("Range"), metadata.Size)
if err != nil {
writeErrorResponse(w, req, InvalidRange, acceptsContentType, req.URL.Path)
return
}
switch httpRange.start == 0 && httpRange.length == 0 {
case true:
setObjectHeaders(w, metadata)
if _, err := api.Donut.GetObject(w, bucket, object); err != nil {
// unable to write headers, we've already printed data. Just close the connection.
log.Error.Println(iodine.New(err, nil))
}
case false:
metadata.Size = httpRange.length
setRangeObjectHeaders(w, metadata, httpRange)
w.WriteHeader(http.StatusPartialContent)
if _, err := api.Donut.GetPartialObject(w, bucket, object, httpRange.start, httpRange.length); err != nil {
// unable to write headers, we've already printed data. Just close the connection.
log.Error.Println(iodine.New(err, nil))
}
setObjectHeaders(w, metadata, httpRange)
if _, err := api.Donut.GetObject(w, bucket, object, httpRange.start, httpRange.length); err != nil {
// unable to write headers, we've already printed data. Just close the connection.
log.Error.Println(iodine.New(err, nil))
return
}
}
case donut.SignatureDoesNotMatch:
@ -144,7 +134,7 @@ func (api Minio) HeadObjectHandler(w http.ResponseWriter, req *http.Request) {
metadata, err := api.Donut.GetObjectMetadata(bucket, object, signature)
switch iodine.ToError(err).(type) {
case nil:
setObjectHeaders(w, metadata)
setObjectHeaders(w, metadata, nil)
w.WriteHeader(http.StatusOK)
case donut.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)

@ -19,9 +19,11 @@ package api
import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"github.com/minio/minio/pkg/donut"
"github.com/minio/minio/pkg/iodine"
)
const (
@ -33,23 +35,23 @@ type httpRange struct {
start, length, size int64
}
// GetContentRange populate range header
func (r *httpRange) getContentRange() string {
// String populate range stringer interface
func (r *httpRange) String() string {
return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, r.size)
}
// Grab new range from request header
func getRequestedRange(req *http.Request, size int64) (*httpRange, error) {
func getRequestedRange(hrange string, size int64) (*httpRange, error) {
r := &httpRange{
start: 0,
length: 0,
size: 0,
}
r.size = size
if s := req.Header.Get("Range"); s != "" {
err := r.parseRange(s)
if hrange != "" {
err := r.parseRange(hrange)
if err != nil {
return nil, err
return nil, iodine.New(err, nil)
}
}
return r, nil
@ -58,7 +60,7 @@ func getRequestedRange(req *http.Request, size int64) (*httpRange, error) {
func (r *httpRange) parse(ra string) error {
i := strings.Index(ra, "-")
if i < 0 {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
if start == "" {
@ -66,7 +68,7 @@ func (r *httpRange) parse(ra string) error {
// range start relative to the end of the file.
i, err := strconv.ParseInt(end, 10, 64)
if err != nil {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
if i > r.size {
i = r.size
@ -76,7 +78,7 @@ func (r *httpRange) parse(ra string) error {
} else {
i, err := strconv.ParseInt(start, 10, 64)
if err != nil || i > r.size || i < 0 {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
r.start = i
if end == "" {
@ -85,7 +87,7 @@ func (r *httpRange) parse(ra string) error {
} else {
i, err := strconv.ParseInt(end, 10, 64)
if err != nil || r.start > i {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
if i >= r.size {
i = r.size - 1
@ -99,24 +101,24 @@ func (r *httpRange) parse(ra string) error {
// parseRange parses a Range header string as per RFC 2616.
func (r *httpRange) parseRange(s string) error {
if s == "" {
return errors.New("header not present")
return iodine.New(errors.New("header not present"), nil)
}
if !strings.HasPrefix(s, b) {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
ras := strings.Split(s[len(b):], ",")
if len(ras) == 0 {
return errors.New("invalid request")
return iodine.New(errors.New("invalid request"), nil)
}
// Just pick the first one and ignore the rest, we only support one range per object
if len(ras) > 1 {
return errors.New("multiple ranges specified")
return iodine.New(errors.New("multiple ranges specified"), nil)
}
ra := strings.TrimSpace(ras[0])
if ra == "" {
return errors.New("invalid range")
return iodine.New(donut.InvalidRange{}, nil)
}
return r.parse(ra)
}

Loading…
Cancel
Save