oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

CompleteMultipartUpload and CreateObjectPart now fully support signature v4

Browse Source
master
Harshavardhana 9 years ago
parent 89c1215194
commit 00890c254e
11 changed files with 156 additions and 90 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 18
      pkg/donut/definitions.go
  2. 5
      pkg/donut/donut-v2.go
  3. 16
      pkg/donut/errors.go
  4. 4
      pkg/donut/interfaces.go
  5. 60
      pkg/donut/multipart.go
  6. 2
      pkg/server/api/bucket-handlers.go
  7. 36
      pkg/server/api/definitions.go
  8. 62
      pkg/server/api/object-handlers.go
  9. 18
      pkg/server/api/response.go
  10. 13
      pkg/server/api_donut_cache_test.go
  11. 12
      pkg/server/api_donut_test.go

18
pkg/donut/definitions.go
Unescape View File

@ -87,6 +87,24 @@ type PartMetadata struct {
Size int64
}
// CompletePart - completed part container
type CompletePart struct {
PartNumber int
ETag string
}
// completedParts is a sortable interface for Part slice
type completedParts []CompletePart
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CompleteMultipartUpload container for completing multipart upload
type CompleteMultipartUpload struct {
Part []CompletePart
}
// ObjectResourcesMetadata - various types of object resources
type ObjectResourcesMetadata struct {
Bucket string

5
pkg/donut/donut-v2.go
Unescape View File

@ -371,11 +371,6 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s
var length int
byteBuffer := make([]byte, 1024*1024)
length, err = data.Read(byteBuffer)
// While hash.Write() wouldn't mind a Nil byteBuffer
// It is necessary for us to verify this and break
if length == 0 {
break
}
hash.Write(byteBuffer[0:length])
sha256hash.Write(byteBuffer[0:length])
ok := donut.objects.Append(objectKey, byteBuffer[0:length])

16
pkg/donut/errors.go
Unescape View File

@ -334,3 +334,19 @@ type MissingDateHeader struct{}
func (e MissingDateHeader) Error() string {
return "Missing date header"
}
// InvalidPartOrder parts are not ordered as Requested
type InvalidPartOrder struct {
UploadID string
}
func (e InvalidPartOrder) Error() string {
return "Invalid part order sent for " + e.UploadID
}
// MalformedXML invalid xml format
type MalformedXML struct{}
func (e MalformedXML) Error() string {
return "Malformed XML"
}

4
pkg/donut/interfaces.go
Unescape View File

@ -51,8 +51,8 @@ type ObjectStorage interface {
type Multipart interface {
NewMultipartUpload(bucket, key, contentType string) (string, error)
AbortMultipartUpload(bucket, key, uploadID string) error
CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error)
CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (ObjectMetadata, error)
CreateObjectPart(string, string, string, int, string, string, int64, io.Reader, *Signature) (string, error)
CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, error)
ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, error)
ListObjectParts(bucket, key string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, error)
}

60
pkg/donut/multipart.go
Unescape View File

@ -22,8 +22,10 @@ import (
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"errors"
"io"
"io/ioutil"
"math/rand"
"runtime/debug"
"sort"
@ -31,6 +33,7 @@ import (
"strings"
"time"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/donut/cache/data"
"github.com/minio/minio/pkg/iodine"
)
@ -90,11 +93,11 @@ func (donut API) AbortMultipartUpload(bucket, key, uploadID string) error {
}
// CreateObjectPart - create a part in a multipart session
func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, error) {
donut.lock.Lock()
defer donut.lock.Unlock()
etag, err := donut.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data)
etag, err := donut.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data, signature)
// possible free
debug.FreeOSMemory()
@ -102,7 +105,7 @@ func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, cont
}
// createObject - internal wrapper function called by CreateObjectPart
func (donut API) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
func (donut API) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, error) {
if !IsValidBucket(bucket) {
return "", iodine.New(BucketNameInvalid{Bucket: bucket}, nil)
}
@ -138,6 +141,7 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
// calculate md5
hash := md5.New()
sha256hash := sha256.New()
var err error
var totalLength int64
@ -145,12 +149,8 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
var length int
byteBuffer := make([]byte, 1024*1024)
length, err = data.Read(byteBuffer)
// While hash.Write() wouldn't mind a Nil byteBuffer
// It is necessary for us to verify this and break
if length == 0 {
break
}
hash.Write(byteBuffer[0:length])
sha256hash.Write(byteBuffer[0:length])
ok := donut.multiPartObjects[uploadID].Append(partID, byteBuffer[0:length])
if !ok {
return "", iodine.New(InternalError{}, nil)
@ -174,6 +174,17 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
return "", iodine.New(BadDigest{}, nil)
}
}
if signature != nil {
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil)))
if err != nil {
return "", iodine.New(err, nil)
}
if !ok {
return "", iodine.New(SignatureDoesNotMatch{}, nil)
}
}
newPart := PartMetadata{
PartNumber: partID,
LastModified: time.Now().UTC(),
@ -200,7 +211,7 @@ func (donut API) cleanupMultipartSession(bucket, key, uploadID string) {
}
// CompleteMultipartUpload - complete a multipart upload and persist the data
func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (ObjectMetadata, error) {
func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, error) {
donut.lock.Lock()
if !IsValidBucket(bucket) {
@ -221,11 +232,36 @@ func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, parts map
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(InvalidUploadID{UploadID: uploadID}, nil)
}
partBytes, err := ioutil.ReadAll(data)
if err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(err, nil)
}
if signature != nil {
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256(partBytes)[:]))
if err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(err, nil)
}
if !ok {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(SignatureDoesNotMatch{}, nil)
}
}
parts := &CompleteMultipartUpload{}
if err := xml.Unmarshal(partBytes, parts); err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(MalformedXML{}, nil)
}
if !sort.IsSorted(completedParts(parts.Part)) {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(InvalidPartOrder{}, nil)
}
var size int64
var fullObject bytes.Buffer
for i := 1; i <= len(parts); i++ {
recvMD5 := parts[i]
object, ok := donut.multiPartObjects[uploadID].Get(i)
for i := 1; i <= len(parts.Part); i++ {
recvMD5 := parts.Part[i-1].ETag
object, ok := donut.multiPartObjects[uploadID].Get(parts.Part[i-1].PartNumber)
if ok == false {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)

2
pkg/server/api/bucket-handlers.go
Unescape View File

@ -100,7 +100,7 @@ func (api Minio) ListMultipartUploadsHandler(w http.ResponseWriter, req *http.Re
case nil: // success
{
// generate response
response := generateListMultipartUploadsResult(bucket, resources)
response := generateListMultipartUploadsResponse(bucket, resources)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))

36
pkg/server/api/definitions.go
Unescape View File

@ -62,6 +62,14 @@ type ListObjectsResponse struct {
Prefix string
}
// Part container for part metadata
type Part struct {
PartNumber int
ETag string
LastModified string
Size int64
}
// ListPartsResponse - format for list parts response
type ListPartsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
@ -134,14 +142,6 @@ type Bucket struct {
CreationDate string
}
// Part container for part metadata
type Part struct {
PartNumber int
ETag string
LastModified string
Size int64
}
// Object container for object metadata
type Object struct {
ETag string
@ -164,8 +164,8 @@ type Owner struct {
DisplayName string
}
// InitiateMultipartUploadResult container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
type InitiateMultipartUploadResult struct {
// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
type InitiateMultipartUploadResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
Bucket string
@ -173,20 +173,8 @@ type InitiateMultipartUploadResult struct {
UploadID string `xml:"UploadId"`
}
// completedParts is a sortable interface for Part slice
type completedParts []Part
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CompleteMultipartUpload container for completing multipart upload
type CompleteMultipartUpload struct {
Part []Part
}
// CompleteMultipartUploadResult container for completed multipart upload response
type CompleteMultipartUploadResult struct {
// CompleteMultipartUploadResponse container for completed multipart upload response
type CompleteMultipartUploadResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
Location string

62
pkg/server/api/object-handlers.go
Unescape View File

@ -18,11 +18,8 @@ package api
import (
"net/http"
"sort"
"strconv"
"encoding/xml"
"github.com/gorilla/mux"
"github.com/minio/minio/pkg/donut"
"github.com/minio/minio/pkg/iodine"
@ -277,7 +274,7 @@ func (api Minio) NewMultipartUploadHandler(w http.ResponseWriter, req *http.Requ
switch iodine.ToError(err).(type) {
case nil:
{
response := generateInitiateMultipartUploadResult(bucket, object, uploadID)
response := generateInitiateMultipartUploadResponse(bucket, object, uploadID)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -346,7 +343,18 @@ func (api Minio) PutObjectPartHandler(w http.ResponseWriter, req *http.Request)
writeErrorResponse(w, req, InvalidPart, acceptsContentType, req.URL.Path)
}
calculatedMD5, err := api.Donut.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body)
var signature *donut.Signature
if _, ok := req.Header["Authorization"]; ok {
// Init signature V4 verification
var err error
signature, err = InitSignatureV4(req)
if err != nil {
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
}
calculatedMD5, err := api.Donut.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body, signature)
switch iodine.ToError(err).(type) {
case nil:
w.Header().Set("ETag", calculatedMD5)
@ -357,6 +365,8 @@ func (api Minio) PutObjectPartHandler(w http.ResponseWriter, req *http.Request)
writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path)
case donut.BadDigest:
writeErrorResponse(w, req, BadDigest, acceptsContentType, req.URL.Path)
case donut.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
case donut.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, acceptsContentType, req.URL.Path)
case donut.EntityTooLarge:
@ -433,7 +443,7 @@ func (api Minio) ListObjectPartsHandler(w http.ResponseWriter, req *http.Request
switch iodine.ToError(err).(type) {
case nil:
{
response := generateListPartsResult(objectResourcesMetadata)
response := generateListPartsResponse(objectResourcesMetadata)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -464,35 +474,27 @@ func (api Minio) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http
return
}
decoder := xml.NewDecoder(req.Body)
parts := &CompleteMultipartUpload{}
err := decoder.Decode(parts)
if err != nil {
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
if !sort.IsSorted(completedParts(parts.Part)) {
writeErrorResponse(w, req, InvalidPartOrder, acceptsContentType, req.URL.Path)
return
}
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
objectResourcesMetadata := getObjectResources(req.URL.Query())
partMap := make(map[int]string)
for _, part := range parts.Part {
partMap[part.PartNumber] = part.ETag
var signature *donut.Signature
if _, ok := req.Header["Authorization"]; ok {
// Init signature V4 verification
var err error
signature, err = InitSignatureV4(req)
if err != nil {
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
}
metadata, err := api.Donut.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, partMap)
metadata, err := api.Donut.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, req.Body, signature)
switch iodine.ToError(err).(type) {
case nil:
{
response := generateCompleteMultpartUploadResult(bucket, object, "", metadata.MD5Sum)
response := generateCompleteMultpartUploadResponse(bucket, object, "", metadata.MD5Sum)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -501,6 +503,16 @@ func (api Minio) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http
}
case donut.InvalidUploadID:
writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path)
case donut.InvalidPartOrder:
writeErrorResponse(w, req, InvalidPartOrder, acceptsContentType, req.URL.Path)
case donut.MissingDateHeader:
writeErrorResponse(w, req, RequestTimeTooSkewed, acceptsContentType, req.URL.Path)
case donut.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
case donut.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, acceptsContentType, req.URL.Path)
case donut.MalformedXML:
writeErrorResponse(w, req, MalformedXML, acceptsContentType, req.URL.Path)
default:
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)

18
pkg/server/api/response.go
Unescape View File

@ -111,18 +111,18 @@ func generateListObjectsResponse(bucket string, objects []donut.ObjectMetadata,
return data
}
// generateInitiateMultipartUploadResult
func generateInitiateMultipartUploadResult(bucket, key, uploadID string) InitiateMultipartUploadResult {
return InitiateMultipartUploadResult{
// generateInitiateMultipartUploadResponse
func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
return InitiateMultipartUploadResponse{
Bucket: bucket,
Key: key,
UploadID: uploadID,
}
}
// generateCompleteMultipartUploadResult
func generateCompleteMultpartUploadResult(bucket, key, location, etag string) CompleteMultipartUploadResult {
return CompleteMultipartUploadResult{
// generateCompleteMultipartUploadResponse
func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
return CompleteMultipartUploadResponse{
Location: location,
Bucket: bucket,
Key: key,
@ -131,7 +131,7 @@ func generateCompleteMultpartUploadResult(bucket, key, location, etag string) Co
}
// generateListPartsResult
func generateListPartsResult(objectMetadata donut.ObjectResourcesMetadata) ListPartsResponse {
func generateListPartsResponse(objectMetadata donut.ObjectResourcesMetadata) ListPartsResponse {
// TODO - support EncodingType in xml decoding
listPartsResponse := ListPartsResponse{}
listPartsResponse.Bucket = objectMetadata.Bucket
@ -160,8 +160,8 @@ func generateListPartsResult(objectMetadata donut.ObjectResourcesMetadata) ListP
return listPartsResponse
}
// generateListMultipartUploadsResult
func generateListMultipartUploadsResult(bucket string, metadata donut.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
// generateListMultipartUploadsResponse
func generateListMultipartUploadsResponse(bucket string, metadata donut.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
listMultipartUploadsResponse := ListMultipartUploadsResponse{}
listMultipartUploadsResponse.Bucket = bucket
listMultipartUploadsResponse.Delimiter = metadata.Delimiter

13
pkg/server/api_donut_cache_test.go
Unescape View File

@ -27,6 +27,7 @@ import (
"net/http/httptest"
. "github.com/minio/check"
"github.com/minio/minio/pkg/donut"
"github.com/minio/minio/pkg/server/api"
)
@ -627,7 +628,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipartAbort(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -672,7 +673,7 @@ func (s *MyAPIDonutCacheSuite) TestBucketMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -723,7 +724,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -771,7 +772,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipart(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -795,8 +796,8 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipart(c *C) {
c.Assert(response2.StatusCode, Equals, http.StatusOK)
// complete multipart upload
completeUploads := &api.CompleteMultipartUpload{
Part: []api.Part{
completeUploads := &donut.CompleteMultipartUpload{
Part: []donut.CompletePart{
{
PartNumber: 1,
ETag: response1.Header.Get("ETag"),

12
pkg/server/api_donut_test.go
Unescape View File

@ -664,7 +664,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipartAbort(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -709,7 +709,7 @@ func (s *MyAPIDonutSuite) TestBucketMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -760,7 +760,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -808,7 +808,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipart(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -832,8 +832,8 @@ func (s *MyAPIDonutSuite) TestObjectMultipart(c *C) {
c.Assert(response2.StatusCode, Equals, http.StatusOK)
// complete multipart upload
completeUploads := &api.CompleteMultipartUpload{
Part: []api.Part{
completeUploads := &donut.CompleteMultipartUpload{
Part: []donut.CompletePart{
{
PartNumber: 1,
ETag: response1.Header.Get("ETag"),

Write Preview
Loading…
Cancel
Save