From 58833711e0f5a4d8af975b009bdf90d76ebee22d Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Fri, 16 Jun 2017 22:17:00 -0700 Subject: [PATCH] Added ListObjectsV2 and ListObjectsV2 Anon support to Gateway S3 and Azure. (#4547) --- cmd/api-response.go | 16 +++--- cmd/bucket-handlers-listobjects.go | 2 +- cmd/gateway-azure-anonymous.go | 60 ++++++++++++++++++++++ cmd/gateway-azure.go | 35 +++++++++++++ cmd/gateway-azure_test.go | 79 +++++++++++++++++++++++++++++ cmd/gateway-handlers.go | 81 ++++++++++++++++++++++++++++++ cmd/gateway-router.go | 2 + cmd/gateway-s3-anonymous.go | 10 ++++ 8 files changed, 276 insertions(+), 9 deletions(-) diff --git a/cmd/api-response.go b/cmd/api-response.go index 794178c2b..a75d7325c 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -352,9 +352,9 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max } // generates an ListObjectsV2 response for the said bucket with other enumerated options. -func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response { +func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response { var contents []Object - var prefixes []CommonPrefix + var commonPrefixes []CommonPrefix var owner = Owner{} var data = ListObjectsV2Response{} @@ -363,7 +363,7 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter owner.DisplayName = globalMinioDefaultOwnerID } - for _, object := range resp.Objects { + for _, object := range objects { var content = Object{} if object.Name == "" { continue @@ -387,14 +387,14 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter data.Prefix = prefix data.MaxKeys = maxKeys data.ContinuationToken = token - data.NextContinuationToken = resp.NextMarker - data.IsTruncated = resp.IsTruncated - for _, prefix := range resp.Prefixes { + data.NextContinuationToken = nextToken + data.IsTruncated = isTruncated + for _, prefix := range prefixes { var prefixItem = CommonPrefix{} prefixItem.Prefix = prefix - prefixes = append(prefixes, prefixItem) + commonPrefixes = append(commonPrefixes, prefixItem) } - data.CommonPrefixes = prefixes + data.CommonPrefixes = commonPrefixes data.KeyCount = len(data.Contents) + len(data.CommonPrefixes) return data } diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index 6381ce837..d69666d53 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http return } - response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo) + response := generateListObjectsV2Response(bucket, prefix, token, listObjectsInfo.NextMarker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes) // Write success response. writeSuccessResponseXML(w, encodeResponse(response)) diff --git a/cmd/gateway-azure-anonymous.go b/cmd/gateway-azure-anonymous.go index c8f35e265..890071ec3 100644 --- a/cmd/gateway-azure-anonymous.go +++ b/cmd/gateway-azure-anonymous.go @@ -190,3 +190,63 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, result.Prefixes = listResp.BlobPrefixes return result, nil } + +// AnonListObjectsV2 - List objects in V2 mode, anonymously +func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) { + params := storage.ListBlobsParameters{ + Prefix: prefix, + Marker: continuationToken, + Delimiter: delimiter, + MaxResults: uint(maxKeys), + } + + q := azureListBlobsGetParameters(params) + q.Set("restype", "container") + q.Set("comp", "list") + + url, err := url.Parse(a.client.GetBlobURL(bucket, "")) + if err != nil { + return result, azureToObjectError(traceError(err)) + } + url.RawQuery = q.Encode() + + resp, err := http.Get(url.String()) + if err != nil { + return result, azureToObjectError(traceError(err)) + } + defer resp.Body.Close() + + var listResp storage.BlobListResponse + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return result, azureToObjectError(traceError(err)) + } + err = xml.Unmarshal(data, &listResp) + if err != nil { + return result, azureToObjectError(traceError(err)) + } + + // If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set + if listResp.NextMarker != "" { + result.IsTruncated = true + result.NextContinuationToken = listResp.NextMarker + } + for _, object := range listResp.Blobs { + t, e := time.Parse(time.RFC1123, object.Properties.LastModified) + if e != nil { + continue + } + result.Objects = append(result.Objects, ObjectInfo{ + Bucket: bucket, + Name: object.Name, + ModTime: t, + Size: object.Properties.ContentLength, + ETag: canonicalizeETag(object.Properties.Etag), + ContentType: object.Properties.ContentType, + ContentEncoding: object.Properties.ContentEncoding, + }) + } + result.Prefixes = listResp.BlobPrefixes + return result, nil +} diff --git a/cmd/gateway-azure.go b/cmd/gateway-azure.go index 16f762f52..1ef0b7c9e 100644 --- a/cmd/gateway-azure.go +++ b/cmd/gateway-azure.go @@ -269,6 +269,41 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max return result, nil } +// ListObjectsV2 - list all blobs in Azure bucket filtered by prefix +func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) { + resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{ + Prefix: prefix, + Marker: continuationToken, + Delimiter: delimiter, + MaxResults: uint(maxKeys), + }) + if err != nil { + return result, azureToObjectError(traceError(err), bucket, prefix) + } + // If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set + if resp.NextMarker != "" { + result.IsTruncated = true + result.NextContinuationToken = resp.NextMarker + } + for _, object := range resp.Blobs { + t, e := time.Parse(time.RFC1123, object.Properties.LastModified) + if e != nil { + continue + } + result.Objects = append(result.Objects, ObjectInfo{ + Bucket: bucket, + Name: object.Name, + ModTime: t, + Size: object.Properties.ContentLength, + ETag: canonicalizeETag(object.Properties.Etag), + ContentType: object.Properties.ContentType, + ContentEncoding: object.Properties.ContentEncoding, + }) + } + result.Prefixes = resp.BlobPrefixes + return result, nil +} + // GetObject - reads an object from azure. Supports additional // parameters like offset and length which are synonymous with // HTTP Range requests. diff --git a/cmd/gateway-azure_test.go b/cmd/gateway-azure_test.go index 7e5639b7f..5a989b020 100644 --- a/cmd/gateway-azure_test.go +++ b/cmd/gateway-azure_test.go @@ -18,6 +18,7 @@ package cmd import ( "net/http" + "net/url" "reflect" "testing" @@ -155,3 +156,81 @@ func TestAzureParseBlockID(t *testing.T) { t.Fatal("Expected azureParseBlockID() to return error") } } + +// Test azureListBlobsGetParameters() +func TestAzureListBlobsGetParameters(t *testing.T) { + + // Test values set 1 + expectedURLValues := url.Values{} + expectedURLValues.Set("prefix", "test") + expectedURLValues.Set("delimiter", "_") + expectedURLValues.Set("marker", "marker") + expectedURLValues.Set("include", "hello") + expectedURLValues.Set("maxresults", "20") + expectedURLValues.Set("timeout", "10") + + setBlobParameters := storage.ListBlobsParameters{"test", "_", "marker", "hello", 20, 10} + + // Test values set 2 + expectedURLValues1 := url.Values{} + + setBlobParameters1 := storage.ListBlobsParameters{"", "", "", "", 0, 0} + + testCases := []struct { + name string + args storage.ListBlobsParameters + want url.Values + }{ + {"TestIfValuesSet", setBlobParameters, expectedURLValues}, + {"TestIfValuesNotSet", setBlobParameters1, expectedURLValues1}, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if got := azureListBlobsGetParameters(test.args); !reflect.DeepEqual(got, test.want) { + t.Errorf("azureListBlobsGetParameters() = %v, want %v", got, test.want) + } + }) + } +} + +func TestAnonErrToObjectErr(t *testing.T) { + testCases := []struct { + name string + statusCode int + params []string + wantErr error + }{ + {"ObjectNotFound", + http.StatusNotFound, + []string{"testBucket", "testObject"}, + ObjectNotFound{Bucket: "testBucket", Object: "testObject"}, + }, + {"BucketNotFound", + http.StatusNotFound, + []string{"testBucket", ""}, + BucketNotFound{Bucket: "testBucket"}, + }, + {"ObjectNameInvalid", + http.StatusBadRequest, + []string{"testBucket", "testObject"}, + ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"}, + }, + {"BucketNameInvalid", + http.StatusBadRequest, + []string{"testBucket", ""}, + BucketNameInvalid{Bucket: "testBucket"}, + }, + {"UnexpectedError", + http.StatusBadGateway, + []string{"testBucket", "testObject"}, + errUnexpected, + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + if err := anonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) { + t.Errorf("anonErrToObjectErr() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/cmd/gateway-handlers.go b/cmd/gateway-handlers.go index 4ce311f72..27571c939 100644 --- a/cmd/gateway-handlers.go +++ b/cmd/gateway-handlers.go @@ -769,6 +769,87 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt return } response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo) + // Write success response. + writeSuccessResponseXML(w, encodeResponse(response)) +} + +// ListObjectsV2Handler - GET Bucket (List Objects) Version 2. +// -------------------------- +// This implementation of the GET operation returns some or all (up to 1000) +// of the objects in a bucket. You can use the request parameters as selection +// criteria to return a subset of the objects in a bucket. +// +// NOTE: It is recommended that this API to be used for application development. +// Minio continues to support ListObjectsV1 for supporting legacy tools. +func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { + vars := router.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(w, ErrServerNotInitialized, r.URL) + return + } + + reqAuthType := getRequestAuthType(r) + + switch reqAuthType { + case authTypePresignedV2, authTypeSignedV2: + // Signature V2 validation. + s3Error := isReqAuthenticatedV2(r) + if s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, s3Error, r.URL) + return + } + case authTypeSigned, authTypePresigned: + s3Error := isReqAuthenticated(r, serverConfig.GetRegion()) + if s3Error != ErrNone { + errorIf(errSignatureMismatch, dumpRequest(r)) + writeErrorResponse(w, s3Error, r.URL) + return + } + case authTypeAnonymous: + // No verification needed for anonymous requests. + default: + // For all unknown auth types return error. + writeErrorResponse(w, ErrAccessDenied, r.URL) + return + } + + // Extract all the listObjectsV2 query params to their native values. + prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query()) + + // In ListObjectsV2 'continuation-token' is the marker. + marker := token + // Check if 'continuation-token' is empty. + if token == "" { + // Then we need to use 'start-after' as marker instead. + marker = startAfter + } + + listObjectsV2 := objectAPI.ListObjectsV2 + if reqAuthType == authTypeAnonymous { + listObjectsV2 = objectAPI.AnonListObjectsV2 + } + + // Validate the query params before beginning to serve the request. + // fetch-owner is not validated since it is a boolean + if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone { + writeErrorResponse(w, s3Error, r.URL) + return + } + // Inititate a list objects operation based on the input params. + // On success would return back ListObjectsV2Info object to be + // serialized as XML and sent as S3 compatible response body. + listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, fetchOwner, delimiter, maxKeys) + if err != nil { + errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter) + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.ContinuationToken, startAfter, delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes) // Write success response. writeSuccessResponseXML(w, encodeResponse(response)) diff --git a/cmd/gateway-router.go b/cmd/gateway-router.go index b286ed6ed..02c32d01a 100644 --- a/cmd/gateway-router.go +++ b/cmd/gateway-router.go @@ -36,6 +36,8 @@ type GatewayLayer interface { GetBucketPolicies(string) (policy.BucketAccessPolicy, error) DeleteBucketPolicies(string) error AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) + AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) + ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) } diff --git a/cmd/gateway-s3-anonymous.go b/cmd/gateway-s3-anonymous.go index 147e8ea43..0c7ef53ea 100644 --- a/cmd/gateway-s3-anonymous.go +++ b/cmd/gateway-s3-anonymous.go @@ -94,6 +94,16 @@ func (l *s3Objects) AnonListObjects(bucket string, prefix string, marker string, return fromMinioClientListBucketResult(bucket, result), nil } +// AnonListObjectsV2 - List objects in V2 mode, anonymously +func (l *s3Objects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) { + result, err := l.anonClient.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) + if err != nil { + return ListObjectsV2Info{}, s3ToObjectError(traceError(err), bucket) + } + + return fromMinioClientListBucketV2Result(bucket, result), nil +} + // AnonGetBucketInfo - Get bucket metadata anonymously. func (l *s3Objects) AnonGetBucketInfo(bucket string) (BucketInfo, error) { if exists, err := l.anonClient.BucketExists(bucket); err != nil {