Handle ListObjectsV2 start-after parameter in ObjectLayer (#6078)

master
wd256 6 years ago committed by Nitish Tiwari
parent a5453c307f
commit 25f9b0bc3b
  1. 12
      cmd/bucket-handlers-listobjects.go
  2. 7
      cmd/fs-v1.go
  3. 2
      cmd/gateway/azure/gateway-azure.go
  4. 12
      cmd/gateway/b2/gateway-b2.go
  5. 4
      cmd/gateway/gcs/gateway-gcs.go
  6. 21
      cmd/gateway/manta/gateway-manta.go
  7. 5
      cmd/gateway/oss/gateway-oss.go
  8. 7
      cmd/xl-sets.go
  9. 7
      cmd/xl-v1-object.go

@ -80,17 +80,9 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
return
}
// In ListObjectsV2 'continuation-token' is the marker.
marker := token
// Check if 'continuation-token' is empty.
if token == "" {
// Then we need to use 'start-after' as marker instead.
marker = startAfter
}
// Validate the query params before beginning to serve the request.
// fetch-owner is not validated since it is a boolean
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
writeErrorResponse(w, s3Error, r.URL)
return
}
@ -101,7 +93,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
// Inititate a list objects operation based on the input params.
// On success would return back ListObjectsInfo object to be
// marshaled into S3 compatible XML header.
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
if err != nil {
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return

@ -1250,7 +1250,12 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
// ListObjectsV2 lists all blobs in bucket filtered by prefix
func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
loi, err := fs.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
marker := continuationToken
if marker == "" {
marker = startAfter
}
loi, err := fs.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
if err != nil {
return result, err
}

@ -611,7 +611,7 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
// ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
marker := continuationToken
if startAfter != "" {
if marker == "" {
marker = startAfter
}

@ -355,12 +355,20 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
// fetchOwner, startAfter are not supported and unused.
// fetchOwner is not supported and unused.
marker := continuationToken
if marker == "" {
// B2's continuation token is an object name to "start at" rather than "start after"
// startAfter plus the lowest character B2 supports is used so that the startAfter
// object isn't included in the results
marker = startAfter + " "
}
bkt, err := l.Bucket(ctx, bucket)
if err != nil {
return loi, err
}
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
if lerr != nil {
logger.LogIf(ctx, lerr)
return loi, b2ToObjectError(lerr, bucket)

@ -658,6 +658,10 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
// ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
if continuationToken == "" && startAfter != "" {
continuationToken = toGCSPageToken(startAfter)
}
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
Delimiter: delimiter,
Prefix: prefix,

@ -347,6 +347,13 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
dirName = path.Join(mantaRoot, bucket, pathDir)
}
if marker != "" {
// Manta uses the marker as the key to start at rather than start after
// A space is appended to the marker so that the corresponding object is not
// included in the results
marker += " "
}
input = &storage.ListDirectoryInput{
DirectoryName: dirName,
Limit: uint64(maxKeys),
@ -419,6 +426,18 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
pathBase = path.Base(prefix)
)
marker := continuationToken
if marker == "" {
marker = startAfter
}
if marker != "" {
// Manta uses the marker as the key to start at rather than start after.
// A space is appended to the marker so that the corresponding object is not
// included in the results
marker += " "
}
if pathDir := path.Dir(prefix); pathDir == "." {
dirName = path.Join(mantaRoot, bucket)
} else {
@ -428,7 +447,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
input = &storage.ListDirectoryInput{
DirectoryName: dirName,
Limit: uint64(maxKeys),
Marker: continuationToken,
Marker: marker,
}
objs, err = t.client.Dir().List(ctx, input)
if err != nil {

@ -479,8 +479,11 @@ func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, mar
// ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix.
func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
// fetchOwner and startAfter are not supported and unused.
// fetchOwner is not supported and unused.
marker := continuationToken
if marker == "" {
marker = startAfter
}
resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys)
if err != nil {

@ -454,7 +454,12 @@ func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo B
// ListObjectsV2 lists all objects in bucket filtered by prefix
func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
loi, err := s.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
marker := continuationToken
if marker == "" {
marker = startAfter
}
loi, err := s.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
if err != nil {
return result, err
}

@ -855,7 +855,12 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
// ListObjectsV2 lists all blobs in bucket filtered by prefix
func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
loi, err := xl.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
marker := continuationToken
if marker == "" {
marker = startAfter
}
loi, err := xl.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
if err != nil {
return result, err
}

Loading…
Cancel
Save