From 614c770b5d040599be01d29dc1d5d401a69c0cd5 Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Wed, 1 Jun 2016 10:40:55 +0530 Subject: [PATCH] List Objects version 2. (#1815) object: List Objects v2 support --- api-resources.go | 34 ++++++++++++++++++- api-response.go | 76 +++++++++++++++++++++++++++++++++++++++++++ bucket-handlers.go | 28 +++++++++++++--- xl-v1-list-objects.go | 11 +++---- 4 files changed, 137 insertions(+), 12 deletions(-) diff --git a/api-resources.go b/api-resources.go index a3e6012a7..7966b5014 100644 --- a/api-resources.go +++ b/api-resources.go @@ -22,7 +22,39 @@ import ( ) // Parse bucket url queries -func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) { +func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) { + prefix = values.Get("prefix") + marker = values.Get("marker") + delimiter = values.Get("delimiter") + if values.Get("max-keys") != "" { + maxkeys, _ = strconv.Atoi(values.Get("max-keys")) + } else { + maxkeys = maxObjectList + } + encodingType = values.Get("encoding-type") + return +} + +// Parse bucket url queries for ListObjects V2. +func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) { + prefix = values.Get("prefix") + startAfter = values.Get("start-after") + delimiter = values.Get("delimiter") + if values.Get("max-keys") != "" { + maxkeys, _ = strconv.Atoi(values.Get("max-keys")) + } else { + maxkeys = maxObjectList + } + encodingType = values.Get("encoding-type") + token = values.Get("continuation-token") + return +} + +// Parse bucket url queries +func getBucketResources(values url.Values) (listType int, prefix, marker, delimiter string, maxkeys int, encodingType string) { + if values.Get("list-type") != "" { + listType, _ = strconv.Atoi(values.Get("list-type")) + } prefix = values.Get("prefix") marker = values.Get("marker") delimiter = values.Get("delimiter") diff --git a/api-response.go b/api-response.go index 820680df7..690da75e9 100644 --- a/api-response.go +++ b/api-response.go @@ -65,6 +65,37 @@ type ListObjectsResponse struct { Prefix string } +// ListObjectsV2Response - format for list objects response. +type ListObjectsV2Response struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + CommonPrefixes []CommonPrefix + Contents []Object + + Delimiter string + + // Encoding type used to encode object keys in the response. + EncodingType string + + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + StartAfter string + MaxKeys int + Name string + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + ContinuationToken string + NextContinuationToken string + Prefix string +} + // Part container for part metadata. type Part struct { PartNumber int @@ -304,6 +335,51 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe return data } +// generates an ListObjects response for the said bucket with other enumerated options. +func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response { + var contents []Object + var prefixes []CommonPrefix + var owner = Owner{} + var data = ListObjectsV2Response{} + + owner.ID = "minio" + owner.DisplayName = "minio" + + for _, object := range resp.Objects { + var content = Object{} + if object.Name == "" { + continue + } + content.Key = object.Name + content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ) + if object.MD5Sum != "" { + content.ETag = "\"" + object.MD5Sum + "\"" + } + content.Size = object.Size + content.StorageClass = "STANDARD" + content.Owner = owner + contents = append(contents, content) + } + // TODO - support EncodingType in xml decoding + data.Name = bucket + data.Contents = contents + + data.StartAfter = startAfter + data.Delimiter = delimiter + data.Prefix = prefix + data.MaxKeys = maxKeys + data.ContinuationToken = token + data.NextContinuationToken = resp.NextMarker + data.IsTruncated = resp.IsTruncated + for _, prefix := range resp.Prefixes { + var prefixItem = CommonPrefix{} + prefixItem.Prefix = prefix + prefixes = append(prefixes, prefixItem) + } + data.CommonPrefixes = prefixes + return data +} + // generateCopyObjectResponse func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { return CopyObjectResponse{ diff --git a/bucket-handlers.go b/bucket-handlers.go index 645136401..afe0b192f 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -220,9 +220,22 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R return } } - + var prefix, marker, token, delimiter, startAfter string + var maxkeys int + var listV2 bool // TODO handle encoding type. - prefix, marker, delimiter, maxkeys, _ := getBucketResources(r.URL.Query()) + if r.URL.Query().Get("list-type") == "2" { + listV2 = true + prefix, token, startAfter, delimiter, maxkeys, _ = getListObjectsV2Args(r.URL.Query()) + // For ListV2 "start-after" is considered only if "continuation-token" is empty. + if token == "" { + marker = startAfter + } else { + marker = token + } + } else { + prefix, marker, delimiter, maxkeys, _ = getListObjectsV1Args(r.URL.Query()) + } if maxkeys < 0 { writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path) return @@ -242,10 +255,17 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R } listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys) + if err == nil { + var encodedSuccessResponse []byte // generate response - response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo) - encodedSuccessResponse := encodeResponse(response) + if listV2 { + response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxkeys, listObjectsInfo) + encodedSuccessResponse = encodeResponse(response) + } else { + response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo) + encodedSuccessResponse = encodeResponse(response) + } // Write headers setCommonHeaders(w) // Write success response. diff --git a/xl-v1-list-objects.go b/xl-v1-list-objects.go index e30e4d8f4..2d5e8a71e 100644 --- a/xl-v1-list-objects.go +++ b/xl-v1-list-objects.go @@ -79,13 +79,10 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey result := ListObjectsInfo{IsTruncated: !eof} for _, objInfo := range objInfos { - // With delimiter set we fill in NextMarker and Prefixes. - if delimiter == slashSeparator { - result.NextMarker = objInfo.Name - if objInfo.IsDir { - result.Prefixes = append(result.Prefixes, objInfo.Name) - continue - } + result.NextMarker = objInfo.Name + if objInfo.IsDir { + result.Prefixes = append(result.Prefixes, objInfo.Name) + continue } result.Objects = append(result.Objects, ObjectInfo{ Name: objInfo.Name,