diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index 017c10cc5..0eb7e9e6d 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -46,13 +46,6 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max } } - /// MinIO special conditions for ListObjects. - - // Verify if delimiter is anything other than '/', which we do not support. - if delimiter != "" && delimiter != "/" { - return ErrNotImplemented - } - // Success. return ErrNone } diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index a083f21a7..d94af00b9 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -21,6 +21,8 @@ import ( "path" "sync" + "strings" + humanize "github.com/dustin/go-humanize" "github.com/minio/minio/cmd/logger" ) @@ -234,7 +236,94 @@ func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath) } +func listObjectsNonSlash(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) { + endWalkCh := make(chan struct{}) + defer close(endWalkCh) + recursive := true + walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, endWalkCh) + + var objInfos []ObjectInfo + var eof bool + var prevPrefix string + + for { + if len(objInfos) == maxKeys { + break + } + result, ok := <-walkResultCh + if !ok { + eof = true + break + } + + var objInfo ObjectInfo + var err error + + index := strings.Index(strings.TrimPrefix(result.entry, prefix), delimiter) + if index == -1 { + objInfo, err = getObjInfo(ctx, bucket, result.entry) + if err != nil { + // Ignore errFileNotFound as the object might have got + // deleted in the interim period of listing and getObjectInfo(), + // ignore quorum error as it might be an entry from an outdated disk. + if IsErrIgnored(err, []error{ + errFileNotFound, + errXLReadQuorum, + }...) { + continue + } + return loi, toObjectErr(err, bucket, prefix) + } + } else { + index = len(prefix) + index + len(delimiter) + currPrefix := result.entry[:index] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + + objInfo = ObjectInfo{ + Bucket: bucket, + Name: currPrefix, + IsDir: true, + } + } + + if objInfo.Name <= marker { + continue + } + + objInfos = append(objInfos, objInfo) + if result.end { + eof = true + break + } + } + + result := ListObjectsInfo{} + for _, objInfo := range objInfos { + if objInfo.IsDir { + result.Prefixes = append(result.Prefixes, objInfo.Name) + continue + } + result.Objects = append(result.Objects, objInfo) + } + + if !eof { + result.IsTruncated = true + if len(objInfos) > 0 { + result.NextMarker = objInfos[len(objInfos)-1].Name + } + } + + return result, nil +} + func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) { + if delimiter != slashSeparator && delimiter != "" { + return listObjectsNonSlash(ctx, obj, bucket, prefix, marker, delimiter, maxKeys, tpool, listDir, getObjInfo, getObjectInfoDirs...) + } + if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil { return loi, err } diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 7a29b094c..31ae30460 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -471,10 +471,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) { {"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, {"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, {"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, - // Valid, existing bucket, but sending invalid delimeter values (9-10). - // Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter. - {"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false}, - {"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false}, // Testing for failure cases with both perfix and marker (11). // The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix). {"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false}, diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index b933205b7..9c53f458e 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -23,6 +23,7 @@ import ( "io" "net/http" "sort" + "strings" "sync" "time" @@ -907,10 +908,108 @@ func (s *xlSets) startMergeWalks(ctx context.Context, bucket, prefix, marker str return entryChs } +func (s *xlSets) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) { + endWalkCh := make(chan struct{}) + defer close(endWalkCh) + recursive := true + entryChs := s.startMergeWalks(context.Background(), bucket, prefix, "", recursive, endWalkCh) + + readQuorum := s.drivesPerSet / 2 + var objInfos []ObjectInfo + var eof bool + var prevPrefix string + + for { + if len(objInfos) == maxKeys { + break + } + result, ok := leastEntry(entryChs, readQuorum) + if !ok { + eof = true + break + } + + var objInfo ObjectInfo + + index := strings.Index(strings.TrimPrefix(result.Name, prefix), delimiter) + if index == -1 { + objInfo = ObjectInfo{ + IsDir: false, + Bucket: bucket, + Name: result.Name, + ModTime: result.ModTime, + Size: result.Size, + ContentType: result.Metadata["content-type"], + ContentEncoding: result.Metadata["content-encoding"], + } + + // Extract etag from metadata. + objInfo.ETag = extractETag(result.Metadata) + + // All the parts per object. + objInfo.Parts = result.Parts + + // etag/md5Sum has already been extracted. We need to + // remove to avoid it from appearing as part of + // response headers. e.g, X-Minio-* or X-Amz-*. + objInfo.UserDefined = cleanMetadata(result.Metadata) + + // Update storage class + if sc, ok := result.Metadata[amzStorageClass]; ok { + objInfo.StorageClass = sc + } else { + objInfo.StorageClass = globalMinioDefaultStorageClass + } + } else { + index = len(prefix) + index + len(delimiter) + currPrefix := result.Name[:index] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + + objInfo = ObjectInfo{ + Bucket: bucket, + Name: currPrefix, + IsDir: true, + } + } + + if objInfo.Name <= marker { + continue + } + + objInfos = append(objInfos, objInfo) + } + + result := ListObjectsInfo{} + for _, objInfo := range objInfos { + if objInfo.IsDir { + result.Prefixes = append(result.Prefixes, objInfo.Name) + continue + } + result.Objects = append(result.Objects, objInfo) + } + + if !eof { + result.IsTruncated = true + if len(objInfos) > 0 { + result.NextMarker = objInfos[len(objInfos)-1].Name + } + } + + return result, nil +} + // ListObjects - implements listing of objects across disks, each disk is indepenently // walked and merged at this layer. Resulting value through the merge process sends // the data in lexically sorted order. func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, heal bool) (loi ListObjectsInfo, err error) { + if delimiter != slashSeparator && delimiter != "" { + // "heal" option passed can be ignored as the heal-listing does not send non-standard delimiter. + return s.listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys) + } + if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil { return loi, err } diff --git a/go.mod b/go.mod index 775c336d8..d78c085e1 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( github.com/nsqio/go-nsq v1.0.7 github.com/pkg/profile v1.3.0 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 + github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect github.com/rjeczalik/notify v0.9.2 github.com/rs/cors v1.6.0 github.com/ryanuber/go-glob v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7536af835..cd5be1046 100644 --- a/go.sum +++ b/go.sum @@ -565,6 +565,8 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 h1:zQTtDd7fQiF9e80lbl+ShnD9/5NSq5r1EhcS8955ECg= +github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=