diff --git a/cmd/gateway-gcs.go b/cmd/gateway-gcs.go index f0acb7897..45a3b9db9 100644 --- a/cmd/gateway-gcs.go +++ b/cmd/gateway-gcs.go @@ -42,6 +42,9 @@ const ( // ZZZZMinioPrefix is used for metadata and multiparts. The prefix is being filtered out, // hence the naming of ZZZZ (last prefix) ZZZZMinioPrefix = "ZZZZ-Minio" + // token prefixed with GCS returned marker to differentiate + // from user supplied marker. + gcsTokenPrefix = "##minio" ) // Check if object prefix is "ZZZZ_Minio". @@ -303,6 +306,12 @@ func toGCSPageToken(name string) string { return base64.StdEncoding.EncodeToString(b) } +// Returns true if marker was returned by GCS, i.e prefixed with +// ##minio by minio gcs gateway. +func isGCSMarker(marker string) bool { + return strings.HasPrefix(marker, gcsTokenPrefix) +} + // ListObjects - lists all blobs in GCS bucket filtered by prefix func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) { it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Delimiter: delimiter, Prefix: prefix, Versions: false}) @@ -311,8 +320,26 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de nextMarker := "" prefixes := []string{} - // we'll set marker to continue - it.PageInfo().Token = marker + // To accommodate S3-compatible applications using + // ListObjectsV1 to use object keys as markers to control the + // listing of objects, we use the following encoding scheme to + // distinguish between GCS continuation tokens and application + // supplied markers. + // + // - NextMarker in ListObjectsV1 response is constructed by + // prefixing "##minio" to the GCS continuation token, + // e.g, "##minioCgRvYmoz" + // + // - Application supplied markers are used as-is to list + // object keys that appear after it in the lexicographical order. + + // If application is using GCS continuation token we should + // strip the gcsTokenPrefix we added. + gcsMarker := isGCSMarker(marker) + if gcsMarker { + it.PageInfo().Token = strings.TrimPrefix(marker, gcsTokenPrefix) + } + it.PageInfo().MaxSize = maxKeys objects := []ObjectInfo{} @@ -348,6 +375,10 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de } else if attrs.Prefix != "" { prefixes = append(prefixes, attrs.Prefix) continue + } else if !gcsMarker && attrs.Name <= marker { + // if user supplied a marker don't append + // objects until we reach marker (and skip it). + continue } objects = append(objects, ObjectInfo{ @@ -364,7 +395,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de return ListObjectsInfo{ IsTruncated: isTruncated, - NextMarker: nextMarker, + NextMarker: gcsTokenPrefix + nextMarker, Prefixes: prefixes, Objects: objects, }, nil diff --git a/cmd/gateway-gcs_test.go b/cmd/gateway-gcs_test.go index 1b3dab9a3..739690414 100644 --- a/cmd/gateway-gcs_test.go +++ b/cmd/gateway-gcs_test.go @@ -164,3 +164,35 @@ func TestIsGCSPrefix(t *testing.T) { } } } + +// Test for isGCSMarker. +func TestIsGCSMarker(t *testing.T) { + testCases := []struct { + marker string + expected bool + }{ + { + marker: "##miniogcs123", + expected: true, + }, + { + marker: "##mini_notgcs123", + expected: false, + }, + { + marker: "#minioagainnotgcs123", + expected: false, + }, + { + marker: "obj1", + expected: false, + }, + } + + for i, tc := range testCases { + if actual := isGCSMarker(tc.marker); actual != tc.expected { + t.Errorf("Test %d: marker is %s, expected %v but got %v", + i+1, tc.marker, tc.expected, actual) + } + } +} diff --git a/cmd/gateway-handlers.go b/cmd/gateway-handlers.go index 9d9974a7c..d343f6e61 100644 --- a/cmd/gateway-handlers.go +++ b/cmd/gateway-handlers.go @@ -741,15 +741,11 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt return } - // Extract all the litsObjectsV1 query params to their native values. + // Extract all the listObjectsV1 query params to their native + // values. N B We delegate validation of params to respective + // gateway backends. prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query()) - // Validate all the query params before beginning to serve the request. - if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone { - writeErrorResponse(w, s3Error, r.URL) - return - } - listObjects := objectAPI.ListObjects if reqAuthType == authTypeAnonymous { listObjects = objectAPI.AnonListObjects