oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

s3: Add support of encodingType parameter (#7265)

Browse Source 
This commit honors encoding-type parameter in object listing,
parts listing and multipart uploads listing.
master
Anis Elleuch 6 years ago committed by Nitish Tiwari
parent dab314900d
commit 5efbe8a1b3
7 changed files with 124 additions and 93 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      cmd/api-errors.go
  2. 64
      cmd/api-response.go
  3. 22
      cmd/bucket-handlers-listobjects.go
  4. 4
      cmd/bucket-handlers.go
  5. 4
      cmd/object-handlers.go
  6. 103
      cmd/server_test.go
  7. 14
      cmd/test-utils_test.go

6
cmd/api-errors.go
Unescape View File

@ -73,6 +73,7 @@ const (
ErrInvalidCopyPartRange
ErrInvalidCopyPartRangeSource
ErrInvalidMaxKeys
ErrInvalidEncodingMethod
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidPartNumberMarker
@ -357,6 +358,11 @@ var errorCodes = errorCodeMap{
Description: "Argument maxKeys must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidEncodingMethod: {
Code: "InvalidArgument",
Description: "Invalid Encoding Method specified in Request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647",

64
cmd/api-response.go
Unescape View File

@ -305,6 +305,21 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
return u.String()
}
// s3EncodeName encodes string in response when encodingType
// is specified in AWS S3 requests.
func s3EncodeName(name string, encodingType string) (result string) {
// Quick path to exit
if encodingType == "" {
return name
}
encodingType = strings.ToLower(encodingType)
switch encodingType {
case "url":
return url.QueryEscape(name)
}
return name
}
// generates ListBucketsResponse from array of BucketInfo which can be
// serialized to match XML and JSON API spec output.
func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
@ -327,7 +342,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
}
// generates an ListObjectsV1 response for the said bucket with other enumerated options.
func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
var contents []Object
var prefixes []CommonPrefix
var owner = Owner{}
@ -339,7 +354,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
if object.Name == "" {
continue
}
content.Key = object.Name
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
@ -349,20 +364,20 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
content.Owner = owner
contents = append(contents, content)
}
// TODO - support EncodingType in xml decoding
data.Name = bucket
data.Contents = contents
data.Prefix = prefix
data.Marker = marker
data.Delimiter = delimiter
data.EncodingType = encodingType
data.Prefix = s3EncodeName(prefix, encodingType)
data.Marker = s3EncodeName(marker, encodingType)
data.Delimiter = s3EncodeName(delimiter, encodingType)
data.MaxKeys = maxKeys
data.NextMarker = resp.NextMarker
data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
data.IsTruncated = resp.IsTruncated
for _, prefix := range resp.Prefixes {
var prefixItem = CommonPrefix{}
prefixItem.Prefix = prefix
prefixItem.Prefix = s3EncodeName(prefix, encodingType)
prefixes = append(prefixes, prefixItem)
}
data.CommonPrefixes = prefixes
@ -370,7 +385,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
}
// generates an ListObjectsV2 response for the said bucket with other enumerated options.
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
var contents []Object
var commonPrefixes []CommonPrefix
var owner = Owner{}
@ -385,7 +400,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
if object.Name == "" {
continue
}
content.Key = object.Name
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
@ -395,20 +410,20 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
content.Owner = owner
contents = append(contents, content)
}
// TODO - support EncodingType in xml decoding
data.Name = bucket
data.Contents = contents
data.EncodingType = encodingType
data.StartAfter = startAfter
data.Delimiter = delimiter
data.Prefix = prefix
data.Delimiter = s3EncodeName(delimiter, encodingType)
data.Prefix = s3EncodeName(prefix, encodingType)
data.MaxKeys = maxKeys
data.ContinuationToken = token
data.NextContinuationToken = nextToken
data.IsTruncated = isTruncated
for _, prefix := range prefixes {
var prefixItem = CommonPrefix{}
prefixItem.Prefix = prefix
prefixItem.Prefix = s3EncodeName(prefix, encodingType)
commonPrefixes = append(commonPrefixes, prefixItem)
}
data.CommonPrefixes = commonPrefixes
@ -452,11 +467,10 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
}
// generates ListPartsResponse from ListPartsInfo.
func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
// TODO - support EncodingType in xml decoding
func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse {
listPartsResponse := ListPartsResponse{}
listPartsResponse.Bucket = partsInfo.Bucket
listPartsResponse.Key = partsInfo.Object
listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
listPartsResponse.UploadID = partsInfo.UploadID
listPartsResponse.StorageClass = globalMinioDefaultStorageClass
listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
@ -480,29 +494,29 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
}
// generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo.
func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse {
listMultipartUploadsResponse := ListMultipartUploadsResponse{}
listMultipartUploadsResponse.Bucket = bucket
listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType)
listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
listMultipartUploadsResponse.EncodingType = encodingType
listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType)
listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType)
listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType)
listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
for index, commonPrefix := range multipartsInfo.CommonPrefixes {
listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
Prefix: commonPrefix,
Prefix: s3EncodeName(commonPrefix, encodingType),
}
}
listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
for index, upload := range multipartsInfo.Uploads {
newUpload := Upload{}
newUpload.UploadID = upload.UploadID
newUpload.Key = upload.Object
newUpload.Key = s3EncodeName(upload.Object, encodingType)
newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
listMultipartUploadsResponse.Uploads[index] = newUpload
}

22
cmd/bucket-handlers-listobjects.go
Unescape View File

@ -18,6 +18,7 @@ package cmd
import (
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/crypto"
@ -32,12 +33,19 @@ import (
// - delimiter if set should be equal to '/', otherwise the request is rejected.
// - marker if set should have a common prefix with 'prefix' param, otherwise
// the request is rejected.
func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
// Max keys cannot be negative.
if maxKeys < 0 {
return ErrInvalidMaxKeys
}
if encodingType != "" {
// Only url encoding type is supported
if strings.ToLower(encodingType) != "url" {
return ErrInvalidEncodingMethod
}
}
/// Minio special conditions for ListObjects.
// Verify if delimiter is anything other than '/', which we do not support.
@ -78,7 +86,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
urlValues := r.URL.Query()
// Extract all the listObjectsV2 query params to their native values.
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
if errCode != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
return
@ -86,7 +94,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
// Validate the query params before beginning to serve the request.
// fetch-owner is not validated since it is a boolean
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
@ -126,7 +134,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
}
response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
// Write success response.
writeSuccessResponseXML(w, encodeResponse(response))
@ -158,14 +166,14 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
}
// Extract all the litsObjectsV1 query params to their native values.
prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
// Validate all the query params before beginning to serve the request.
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
@ -204,7 +212,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
}
}
}
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
// Write success response.
writeSuccessResponseXML(w, encodeResponse(response))

4
cmd/bucket-handlers.go
Unescape View File

@ -155,7 +155,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
return
}
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, errCode := getBucketMultipartResources(r.URL.Query())
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query())
if errCode != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
return
@ -180,7 +180,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
return
}
// generate response
response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType)
encodedSuccessResponse := encodeResponse(response)
// write success response.

4
cmd/object-handlers.go
Unescape View File

@ -2094,7 +2094,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
return
}
uploadID, partNumberMarker, maxParts, _, s3Error := getObjectResources(r.URL.Query())
uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.URL.Query())
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
@ -2145,7 +2145,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
}
}
response := generateListPartsResponse(listPartsInfo)
response := generateListPartsResponse(listPartsInfo, encodingType)
encodedSuccessResponse := encodeResponse(response)
// Write success response.

103
cmd/server_test.go
Unescape View File

@ -1658,62 +1658,59 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
buffer1 := bytes.NewReader([]byte("Hello World"))
request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, "bar"),
int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
// create listObjectsV1 request with valid parameters
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "1000"),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
getContent, err := ioutil.ReadAll(response.Body)
c.Assert(err, nil)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
// create listObjectsV2 request with valid parameters
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
for _, objectName := range []string{"foo bar 1", "foo bar 2"} {
buffer := bytes.NewReader([]byte("Hello World"))
request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName),
int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
getContent, err = ioutil.ReadAll(response.Body)
c.Assert(err, nil)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
c.Assert(strings.Contains(string(getContent), "<Owner><ID></ID><DisplayName></DisplayName></Owner>"), true)
client = http.Client{Transport: s.transport}
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
}
// create listObjectsV2 request with valid parameters and fetch-owner activated
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", "true"),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
var testCases = []struct {
getURL string
expectedStrings []string
}{
{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"<Key>foo bar 1</Key>", "<Key>foo bar 2</Key>"}},
{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", ""),
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
"<Owner><ID></ID><DisplayName></DisplayName></Owner>",
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", ""),
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
fmt.Sprintf("<Owner><ID>%s</ID><DisplayName></DisplayName></Owner>", globalMinioDefaultOwnerID),
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
}
getContent, err = ioutil.ReadAll(response.Body)
c.Assert(err, nil)
for i, testCase := range testCases {
// create listObjectsV1 request with valid parameters
request, err = newTestSignedRequest("GET", testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
c.Assert(strings.Contains(string(getContent), fmt.Sprintf("<Owner><ID>%s</ID><DisplayName></DisplayName></Owner>",
globalMinioDefaultOwnerID)), true)
getContent, err := ioutil.ReadAll(response.Body)
c.Assert(err, nil)
fmt.Printf("Test %d: %+v vs %+v\n", i+1, string(getContent), testCase.expectedStrings)
for _, expectedStr := range testCase.expectedStrings {
c.Assert(strings.Contains(string(getContent), expectedStr), true)
}
}
}
// TestListObjectsHandlerErrors - Setting invalid parameters to List Objects
@ -1733,7 +1730,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) {
c.Assert(response.StatusCode, http.StatusOK)
// create listObjectsV1 request with invalid value of max-keys parameter. max-keys is set to -2.
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "-2"),
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "", "-2", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
@ -1744,7 +1741,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) {
verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest)
// create listObjectsV2 request with invalid value of max-keys parameter. max-keys is set to -2.
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "-2", ""),
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "", "-2", "", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}

14
cmd/test-utils_test.go
Unescape View File

@ -1458,16 +1458,19 @@ func getBucketLocationURL(endPoint, bucketName string) string {
}
// return URL for listing objects in the bucket with V1 legacy API.
func getListObjectsV1URL(endPoint, bucketName string, maxKeys string) string {
func getListObjectsV1URL(endPoint, bucketName, prefix, maxKeys, encodingType string) string {
queryValue := url.Values{}
if maxKeys != "" {
queryValue.Set("max-keys", maxKeys)
}
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
if encodingType != "" {
queryValue.Set("encoding-type", encodingType)
}
return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
}
// return URL for listing objects in the bucket with V2 API.
func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner string) string {
func getListObjectsV2URL(endPoint, bucketName, prefix, maxKeys, fetchOwner, encodingType string) string {
queryValue := url.Values{}
queryValue.Set("list-type", "2") // Enables list objects V2 URL.
if maxKeys != "" {
@ -1476,7 +1479,10 @@ func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner
if fetchOwner != "" {
queryValue.Set("fetch-owner", fetchOwner)
}
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
if encodingType != "" {
queryValue.Set("encoding-type", encodingType)
}
return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
}
// return URL for a new multipart upload.

Write Preview
Loading…
Cancel
Save