From 682020ef2fbc0ee1a1126f238d7a61b1f2c2b262 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 19 Jan 2016 17:49:48 -0800 Subject: [PATCH] listObjects: Channel based changes. Supports: - prefixes - marker --- api-errors.go | 1 - api-resources.go | 12 +-- api-response.go | 11 +-- bucket-handlers.go | 21 ++--- pkg/fs/api_suite_nix_test.go | 121 +++++++++--------------- pkg/fs/api_suite_windows_test.go | 123 +++++++++---------------- pkg/fs/definitions.go | 37 +++----- pkg/fs/fs-bucket-listobjects.go | 152 +++++++++++++++---------------- pkg/fs/fs.go | 11 ++- server-config.go | 18 ---- server_fs_test.go | 7 +- update-main.go | 6 -- 12 files changed, 204 insertions(+), 316 deletions(-) diff --git a/api-errors.go b/api-errors.go index b1d5b76b2..cd1d1c8c7 100644 --- a/api-errors.go +++ b/api-errors.go @@ -53,7 +53,6 @@ const ( InvalidBucketName InvalidDigest InvalidRange - InvalidRequest InvalidMaxKeys InvalidMaxUploads InvalidMaxParts diff --git a/api-resources.go b/api-resources.go index e76963b91..1a1ed4d39 100644 --- a/api-resources.go +++ b/api-resources.go @@ -24,12 +24,12 @@ import ( ) // parse bucket url queries -func getBucketResources(values url.Values) (v fs.BucketResourcesMetadata) { - v.Prefix = values.Get("prefix") - v.Marker = values.Get("marker") - v.Maxkeys, _ = strconv.Atoi(values.Get("max-keys")) - v.Delimiter = values.Get("delimiter") - v.EncodingType = values.Get("encoding-type") +func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) { + prefix = values.Get("prefix") + marker = values.Get("marker") + delimiter = values.Get("delimiter") + maxkeys, _ = strconv.Atoi(values.Get("max-keys")) + encodingType = values.Get("encoding-type") return } diff --git a/api-response.go b/api-response.go index c3dfb0bfd..f29013754 100644 --- a/api-response.go +++ b/api-response.go @@ -92,8 +92,7 @@ func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyRe } // generates an ListObjects response for the said bucket with other enumerated options. -// func generateListObjectsResponse(bucket string, objects []fs.ObjectMetadata, bucketResources fs.BucketResourcesMetadata) ListObjectsResponse { -func generateListObjectsResponse(bucket string, req fs.ListObjectsReq, resp fs.ListObjectsResp) ListObjectsResponse { +func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse { var contents []*Object var prefixes []*CommonPrefix var owner = Owner{} @@ -119,10 +118,10 @@ func generateListObjectsResponse(bucket string, req fs.ListObjectsReq, resp fs.L data.Name = bucket data.Contents = contents - data.MaxKeys = req.MaxKeys - data.Prefix = req.Prefix - data.Delimiter = req.Delimiter - data.Marker = req.Marker + data.Prefix = prefix + data.Marker = marker + data.Delimiter = delimiter + data.MaxKeys = maxKeys data.NextMarker = resp.NextMarker data.IsTruncated = resp.IsTruncated diff --git a/bucket-handlers.go b/bucket-handlers.go index 146143d70..d413f7851 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -52,6 +52,7 @@ func (api CloudStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, req * default: writeErrorResponse(w, req, InternalError, req.URL.Path) } + return } // TODO: Location value for LocationResponse is deliberately not used, until @@ -128,25 +129,21 @@ func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.R } } } - resources := getBucketResources(req.URL.Query()) - if resources.Maxkeys < 0 { + + // TODO handle encoding type. + prefix, marker, delimiter, maxkeys, _ := getBucketResources(req.URL.Query()) + if maxkeys < 0 { writeErrorResponse(w, req, InvalidMaxKeys, req.URL.Path) return } - if resources.Maxkeys == 0 { - resources.Maxkeys = maxObjectList + if maxkeys == 0 { + maxkeys = maxObjectList } - listReq := fs.ListObjectsReq{ - Prefix: resources.Prefix, - Marker: resources.Marker, - Delimiter: resources.Delimiter, - MaxKeys: resources.Maxkeys, - } - listResp, err := api.Filesystem.ListObjects(bucket, listReq) + listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys) if err == nil { // generate response - response := generateListObjectsResponse(bucket, listReq, listResp) + response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp) encodedSuccessResponse := encodeSuccessResponse(response) // Write headers setCommonHeaders(w) diff --git a/pkg/fs/api_suite_nix_test.go b/pkg/fs/api_suite_nix_test.go index c3b50281c..0cc108c47 100644 --- a/pkg/fs/api_suite_nix_test.go +++ b/pkg/fs/api_suite_nix_test.go @@ -165,59 +165,50 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) { func testPaging(c *check.C, create func() Filesystem) { fs := create() fs.MakeBucket("bucket", "") - resources := BucketResourcesMetadata{} - objects, resources, err := fs.ListObjects("bucket", resources) + result, err := fs.ListObjects("bucket", "", "", "", 0) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 0) - c.Assert(resources.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, 0) + c.Assert(result.IsTruncated, check.Equals, false) // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - resources.Maxkeys = 5 - resources.Prefix = "" - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, i+1) - c.Assert(resources.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, i+1) + c.Assert(result.IsTruncated, check.Equals, false) } // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - resources.Maxkeys = 5 - resources.Prefix = "" - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 5) - c.Assert(resources.IsTruncated, check.Equals, true) + c.Assert(len(result.Objects), check.Equals, 5) + c.Assert(result.IsTruncated, check.Equals, true) } // check paging with prefix at end returns less objects { _, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - resources.Prefix = "new" - resources.Maxkeys = 5 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 2) + c.Assert(len(result.Objects), check.Equals, 2) } // check ordering of pages { - resources.Prefix = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") - c.Assert(objects[2].Object, check.Equals, "obj0") - c.Assert(objects[3].Object, check.Equals, "obj1") - c.Assert(objects[4].Object, check.Equals, "obj10") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[2].Object, check.Equals, "obj0") + c.Assert(result.Objects[3].Object, check.Equals, "obj1") + c.Assert(result.Objects[4].Object, check.Equals, "obj10") } // check delimited results with delimiter and prefix @@ -226,72 +217,49 @@ func testPaging(c *check.C, create func() Filesystem) { c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Delimiter = "/" - resources.Prefix = "this/is/" - resources.Maxkeys = 10 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 1) - c.Assert(resources.CommonPrefixes[0], check.Equals, "this/is/also/") + c.Assert(len(result.Objects), check.Equals, 1) + c.Assert(result.Prefixes[0], check.Equals, "this/is/also/") } time.Sleep(time.Second) // check delimited results with delimiter without prefix { - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Delimiter = "/" - resources.Prefix = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "/", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") - c.Assert(objects[2].Object, check.Equals, "obj0") - c.Assert(objects[3].Object, check.Equals, "obj1") - c.Assert(objects[4].Object, check.Equals, "obj10") - c.Assert(resources.CommonPrefixes[0], check.Equals, "this/") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[2].Object, check.Equals, "obj0") + c.Assert(result.Objects[3].Object, check.Equals, "obj1") + c.Assert(result.Objects[4].Object, check.Equals, "obj10") + c.Assert(result.Prefixes[0], check.Equals, "this/") } // check results with Marker { - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Prefix = "" - resources.Marker = "newPrefix" - resources.Delimiter = "" - resources.Maxkeys = 3 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix2") - c.Assert(objects[1].Object, check.Equals, "obj0") - c.Assert(objects[2].Object, check.Equals, "obj1") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[1].Object, check.Equals, "obj0") + c.Assert(result.Objects[2].Object, check.Equals, "obj1") } // check ordering of results with prefix { - resources.Prefix = "obj" - resources.Delimiter = "" - resources.Marker = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "obj", "", "", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "obj0") - c.Assert(objects[1].Object, check.Equals, "obj1") - c.Assert(objects[2].Object, check.Equals, "obj10") - c.Assert(objects[3].Object, check.Equals, "obj2") - c.Assert(objects[4].Object, check.Equals, "obj3") + c.Assert(result.Objects[0].Object, check.Equals, "obj0") + c.Assert(result.Objects[1].Object, check.Equals, "obj1") + c.Assert(result.Objects[2].Object, check.Equals, "obj10") + c.Assert(result.Objects[3].Object, check.Equals, "obj2") + c.Assert(result.Objects[4].Object, check.Equals, "obj3") } // check ordering of results with prefix and no paging { - resources.Prefix = "new" - resources.Marker = "" - resources.Maxkeys = 5 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") } } @@ -417,11 +385,10 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) { func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) { fs := create() - resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000} - objects, resources, err := fs.ListObjects("bucket", resources) + result, err := fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) - c.Assert(resources.IsTruncated, check.Equals, false) - c.Assert(len(objects), check.Equals, 0) + c.Assert(result.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, 0) } func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { diff --git a/pkg/fs/api_suite_windows_test.go b/pkg/fs/api_suite_windows_test.go index 22ebc52dc..be83e07b7 100644 --- a/pkg/fs/api_suite_windows_test.go +++ b/pkg/fs/api_suite_windows_test.go @@ -26,7 +26,6 @@ import ( "encoding/xml" "math/rand" "strconv" - "time" "gopkg.in/check.v1" ) @@ -165,59 +164,50 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) { func testPaging(c *check.C, create func() Filesystem) { fs := create() fs.MakeBucket("bucket", "") - resources := BucketResourcesMetadata{} - objects, resources, err := fs.ListObjects("bucket", resources) + result, err := fs.ListObjects("bucket", "", "", "", 0) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 0) - c.Assert(resources.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, 0) + c.Assert(result.IsTruncated, check.Equals, false) // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - resources.Maxkeys = 5 - resources.Prefix = "" - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, i+1) - c.Assert(resources.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, i+1) + c.Assert(result.IsTruncated, check.Equals, false) } // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) _, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - resources.Maxkeys = 5 - resources.Prefix = "" - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 5) - c.Assert(resources.IsTruncated, check.Equals, true) + c.Assert(len(result.Objects), check.Equals, 5) + c.Assert(result.IsTruncated, check.Equals, true) } // check paging with prefix at end returns less objects { _, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - resources.Prefix = "new" - resources.Maxkeys = 5 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 2) + c.Assert(len(result.Objects), check.Equals, 2) } // check ordering of pages { - resources.Prefix = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") - c.Assert(objects[2].Object, check.Equals, "obj0") - c.Assert(objects[3].Object, check.Equals, "obj1") - c.Assert(objects[4].Object, check.Equals, "obj10") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[2].Object, check.Equals, "obj0") + c.Assert(result.Objects[3].Object, check.Equals, "obj1") + c.Assert(result.Objects[4].Object, check.Equals, "obj10") } // check delimited results with delimiter and prefix @@ -226,72 +216,48 @@ func testPaging(c *check.C, create func() Filesystem) { c.Assert(err, check.IsNil) _, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Delimiter = "/" - resources.Prefix = "this/is/" - resources.Maxkeys = 10 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) - c.Assert(len(objects), check.Equals, 1) - c.Assert(resources.CommonPrefixes[0], check.Equals, "this/is/also/") + c.Assert(len(result.Objects), check.Equals, 1) + c.Assert(result.Prefixes[0], check.Equals, "this/is/also/") } - time.Sleep(time.Second) // check delimited results with delimiter without prefix { - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Delimiter = "/" - resources.Prefix = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "", "/", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") - c.Assert(objects[2].Object, check.Equals, "obj0") - c.Assert(objects[3].Object, check.Equals, "obj1") - c.Assert(objects[4].Object, check.Equals, "obj10") - c.Assert(resources.CommonPrefixes[0], check.Equals, "this/") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[2].Object, check.Equals, "obj0") + c.Assert(result.Objects[3].Object, check.Equals, "obj1") + c.Assert(result.Objects[4].Object, check.Equals, "obj10") + c.Assert(result.Prefixes[0], check.Equals, "this/") } // check results with Marker { - var prefixes []string - resources.CommonPrefixes = prefixes // allocate new everytime - resources.Prefix = "" - resources.Marker = "newPrefix" - resources.Delimiter = "" - resources.Maxkeys = 3 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix2") - c.Assert(objects[1].Object, check.Equals, "obj0") - c.Assert(objects[2].Object, check.Equals, "obj1") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[1].Object, check.Equals, "obj0") + c.Assert(result.Objects[2].Object, check.Equals, "obj1") } // check ordering of results with prefix { - resources.Prefix = "obj" - resources.Delimiter = "" - resources.Marker = "" - resources.Maxkeys = 1000 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "obj", "", "", 1000) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "obj0") - c.Assert(objects[1].Object, check.Equals, "obj1") - c.Assert(objects[2].Object, check.Equals, "obj10") - c.Assert(objects[3].Object, check.Equals, "obj2") - c.Assert(objects[4].Object, check.Equals, "obj3") + c.Assert(result.Objects[0].Object, check.Equals, "obj0") + c.Assert(result.Objects[1].Object, check.Equals, "obj1") + c.Assert(result.Objects[2].Object, check.Equals, "obj10") + c.Assert(result.Objects[3].Object, check.Equals, "obj2") + c.Assert(result.Objects[4].Object, check.Equals, "obj3") } // check ordering of results with prefix and no paging { - resources.Prefix = "new" - resources.Marker = "" - resources.Maxkeys = 5 - objects, resources, err = fs.ListObjects("bucket", resources) + result, err = fs.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) - c.Assert(objects[0].Object, check.Equals, "newPrefix") - c.Assert(objects[1].Object, check.Equals, "newPrefix2") + c.Assert(result.Objects[0].Object, check.Equals, "newPrefix") + c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2") } } @@ -416,11 +382,10 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) { func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) { fs := create() - resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000} - objects, resources, err := fs.ListObjects("bucket", resources) + result, err := fs.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) - c.Assert(resources.IsTruncated, check.Equals, false) - c.Assert(len(objects), check.Equals, 0) + c.Assert(result.IsTruncated, check.Equals, false) + c.Assert(len(result.Objects), check.Equals, 0) } func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { diff --git a/pkg/fs/definitions.go b/pkg/fs/definitions.go index 6c656167f..93924cdc5 100644 --- a/pkg/fs/definitions.go +++ b/pkg/fs/definitions.go @@ -118,19 +118,15 @@ type BucketMultipartResourcesMetadata struct { CommonPrefixes []string } -// BucketResourcesMetadata - various types of bucket resources -type BucketResourcesMetadata struct { - Prefix string - Marker string - NextMarker string - Maxkeys int - EncodingType string - Delimiter string - IsTruncated bool - CommonPrefixes []string -} - -type ListObjectsReq struct { +// ListObjectsResult - container for list object request results. +type ListObjectsResult struct { + IsTruncated bool + NextMarker string + Objects []ObjectMetadata + Prefixes []string +} + +type listObjectsReq struct { Bucket string Prefix string Marker string @@ -138,21 +134,14 @@ type ListObjectsReq struct { MaxKeys int } -type ListObjectsResp struct { - IsTruncated bool - NextMarker string - Objects []ObjectMetadata - Prefixes []string -} - type listServiceReq struct { - req ListObjectsReq - respCh chan ListObjectsResp + req listObjectsReq + respCh chan ListObjectsResult } type listWorkerReq struct { - req ListObjectsReq - respCh chan ListObjectsResp + req listObjectsReq + respCh chan ListObjectsResult } // CompletePart - completed part container diff --git a/pkg/fs/fs-bucket-listobjects.go b/pkg/fs/fs-bucket-listobjects.go index 041067727..54a5cd01d 100644 --- a/pkg/fs/fs-bucket-listobjects.go +++ b/pkg/fs/fs-bucket-listobjects.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015-2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ package fs import ( "errors" - "fmt" "os" "path/filepath" "strings" @@ -27,74 +26,88 @@ import ( "github.com/minio/minio-xl/pkg/probe" ) -func (fs Filesystem) listWorker(startReq ListObjectsReq) (chan<- listWorkerReq, *probe.Error) { - Separator := string(os.PathSeparator) +func (fs Filesystem) listWorker(startReq listObjectsReq) (chan<- listWorkerReq, *probe.Error) { bucket := startReq.Bucket prefix := startReq.Prefix marker := startReq.Marker delimiter := startReq.Delimiter - quit := make(chan bool) - if marker != "" { - return nil, probe.NewError(errors.New("Not supported")) - } - if delimiter != "" && delimiter != Separator { - return nil, probe.NewError(errors.New("Not supported")) - } + quitWalker := make(chan bool) reqCh := make(chan listWorkerReq) walkerCh := make(chan ObjectMetadata) go func() { - rootPath := filepath.Join(fs.path, bucket, prefix) - stripPath := filepath.Join(fs.path, bucket) + Separator + var rootPath string + bucketPath := filepath.Join(fs.path, bucket) + trimBucketPathPrefix := bucketPath + string(os.PathSeparator) + prefixPath := trimBucketPathPrefix + prefix + st, err := os.Stat(prefixPath) + if err != nil && os.IsNotExist(err) { + rootPath = bucketPath + } else { + if st.IsDir() && !strings.HasSuffix(prefix, delimiter) { + rootPath = bucketPath + } else { + rootPath = prefixPath + } + } filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { if path == rootPath { return nil } if info.IsDir() { - path = path + Separator - } - objectName := strings.TrimPrefix(path, stripPath) - object := ObjectMetadata{ - Object: objectName, - Created: info.ModTime(), - Mode: info.Mode(), - Size: info.Size(), - } - select { - case walkerCh <- object: - // do nothings - case <-quit: - fmt.Println("walker got quit") - // returning error ends the Walk() - return errors.New("Ending") + path = path + string(os.PathSeparator) } - if delimiter == Separator && info.IsDir() { - return filepath.SkipDir + objectName := strings.TrimPrefix(path, trimBucketPathPrefix) + if strings.HasPrefix(objectName, prefix) { + if marker >= objectName { + return nil + } + object := ObjectMetadata{ + Object: objectName, + Created: info.ModTime(), + Mode: info.Mode(), + Size: info.Size(), + } + select { + case walkerCh <- object: + // Do nothing + case <-quitWalker: + // Returning error ends the Walk() + return errors.New("Ending") + } + if delimiter != "" && info.IsDir() { + return filepath.SkipDir + } } return nil }) close(walkerCh) }() go func() { - resp := ListObjectsResp{} + resp := ListObjectsResult{} for { select { case <-time.After(10 * time.Second): - fmt.Println("worker got timeout") - quit <- true - timeoutReq := ListObjectsReq{bucket, prefix, marker, delimiter, 0} - fmt.Println("after timeout", fs) + quitWalker <- true + timeoutReq := listObjectsReq{bucket, prefix, marker, delimiter, 0} fs.timeoutReqCh <- timeoutReq // FIXME: can there be a race such that sender on reqCh panics? return - case req := <-reqCh: - resp = ListObjectsResp{} + case req, ok := <-reqCh: + if !ok { + return + } + resp = ListObjectsResult{} resp.Objects = make([]ObjectMetadata, 0) resp.Prefixes = make([]string, 0) count := 0 for object := range walkerCh { + if count == req.req.MaxKeys { + resp.IsTruncated = true + break + } if object.Mode.IsDir() { if delimiter == "" { - // skip directories for recursive list + // Skip directories for recursive list continue } resp.Prefixes = append(resp.Prefixes, object.Object) @@ -103,13 +116,7 @@ func (fs Filesystem) listWorker(startReq ListObjectsReq) (chan<- listWorkerReq, } resp.NextMarker = object.Object count++ - if count == req.req.MaxKeys { - resp.IsTruncated = true - break - } } - fmt.Println("response objects: ", len(resp.Objects)) - marker = resp.NextMarker req.respCh <- resp } } @@ -118,9 +125,8 @@ func (fs Filesystem) listWorker(startReq ListObjectsReq) (chan<- listWorkerReq, } func (fs *Filesystem) startListService() *probe.Error { - fmt.Println("startListService starting") listServiceReqCh := make(chan listServiceReq) - timeoutReqCh := make(chan ListObjectsReq) + timeoutReqCh := make(chan listObjectsReq) reqToListWorkerReqCh := make(map[string](chan<- listWorkerReq)) reqToStr := func(bucket string, prefix string, marker string, delimiter string) string { return strings.Join([]string{bucket, prefix, marker, delimiter}, ":") @@ -129,7 +135,6 @@ func (fs *Filesystem) startListService() *probe.Error { for { select { case timeoutReq := <-timeoutReqCh: - fmt.Println("listservice got timeout on ", timeoutReq) reqStr := reqToStr(timeoutReq.Bucket, timeoutReq.Prefix, timeoutReq.Marker, timeoutReq.Delimiter) listWorkerReqCh, ok := reqToListWorkerReqCh[reqStr] if ok { @@ -137,27 +142,22 @@ func (fs *Filesystem) startListService() *probe.Error { } delete(reqToListWorkerReqCh, reqStr) case serviceReq := <-listServiceReqCh: - fmt.Println("serviceReq received", serviceReq) - fmt.Println("sending to listservicereqch", fs) - reqStr := reqToStr(serviceReq.req.Bucket, serviceReq.req.Prefix, serviceReq.req.Marker, serviceReq.req.Delimiter) listWorkerReqCh, ok := reqToListWorkerReqCh[reqStr] if !ok { var err *probe.Error listWorkerReqCh, err = fs.listWorker(serviceReq.req) if err != nil { - fmt.Println("listWorker returned error", err) - serviceReq.respCh <- ListObjectsResp{} + serviceReq.respCh <- ListObjectsResult{} return } reqToListWorkerReqCh[reqStr] = listWorkerReqCh } - respCh := make(chan ListObjectsResp) + respCh := make(chan ListObjectsResult) listWorkerReqCh <- listWorkerReq{serviceReq.req, respCh} resp, ok := <-respCh if !ok { - serviceReq.respCh <- ListObjectsResp{} - fmt.Println("listWorker resp was not ok") + serviceReq.respCh <- ListObjectsResult{} return } delete(reqToListWorkerReqCh, reqStr) @@ -177,13 +177,12 @@ func (fs *Filesystem) startListService() *probe.Error { } // ListObjects - -func (fs Filesystem) ListObjects(bucket string, req ListObjectsReq) (ListObjectsResp, *probe.Error) { +func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) { fs.lock.Lock() defer fs.lock.Unlock() - Separator := string(os.PathSeparator) if !IsValidBucketName(bucket) { - return ListObjectsResp{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return ListObjectsResult{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) } bucket = fs.denormalizeBucket(bucket) @@ -191,39 +190,34 @@ func (fs Filesystem) ListObjects(bucket string, req ListObjectsReq) (ListObjects // check bucket exists if _, e := os.Stat(rootPrefix); e != nil { if os.IsNotExist(e) { - return ListObjectsResp{}, probe.NewError(BucketNotFound{Bucket: bucket}) + return ListObjectsResult{}, probe.NewError(BucketNotFound{Bucket: bucket}) } - return ListObjectsResp{}, probe.NewError(e) - } - - canonicalize := func(str string) string { - return strings.Replace(str, "/", string(os.PathSeparator), -1) - } - decanonicalize := func(str string) string { - return strings.Replace(str, string(os.PathSeparator), "/", -1) + return ListObjectsResult{}, probe.NewError(e) } + req := listObjectsReq{} req.Bucket = bucket - req.Prefix = canonicalize(req.Prefix) - req.Marker = canonicalize(req.Marker) - req.Delimiter = canonicalize(req.Delimiter) - - if req.Delimiter != "" && req.Delimiter != Separator { - return ListObjectsResp{}, probe.NewError(errors.New("not supported")) - } + req.Prefix = filepath.FromSlash(prefix) + req.Marker = filepath.FromSlash(marker) + req.Delimiter = filepath.FromSlash(delimiter) + req.MaxKeys = maxKeys - respCh := make(chan ListObjectsResp) + respCh := make(chan ListObjectsResult) fs.listServiceReqCh <- listServiceReq{req, respCh} resp := <-respCh for i := 0; i < len(resp.Prefixes); i++ { - resp.Prefixes[i] = decanonicalize(resp.Prefixes[i]) + resp.Prefixes[i] = filepath.ToSlash(resp.Prefixes[i]) } for i := 0; i < len(resp.Objects); i++ { - resp.Objects[i].Object = decanonicalize(resp.Objects[i].Object) + resp.Objects[i].Object = filepath.ToSlash(resp.Objects[i].Object) } if req.Delimiter == "" { - // unset NextMaker for recursive list + // This element is set only if you have delimiter set. + // If response does not include the NextMaker and it is + // truncated, you can use the value of the last Key in the + // response as the marker in the subsequent request to get the + // next set of object keys. resp.NextMarker = "" } return resp, nil diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 8f6f7571c..39f6a44a1 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -34,7 +34,7 @@ type Filesystem struct { multiparts *Multiparts buckets *Buckets listServiceReqCh chan<- listServiceReq - timeoutReqCh chan<- ListObjectsReq + timeoutReqCh chan<- listObjectsReq } // Buckets holds acl information @@ -94,10 +94,10 @@ func New(rootPath string) (Filesystem, *probe.Error) { return Filesystem{}, err.Trace() } } - a := Filesystem{lock: new(sync.Mutex)} - a.path = rootPath - a.multiparts = multiparts - a.buckets = buckets + fs := Filesystem{lock: new(sync.Mutex)} + fs.path = rootPath + fs.multiparts = multiparts + fs.buckets = buckets /// Defaults // maximum buckets to be listed from list buckets. @@ -105,6 +105,7 @@ func New(rootPath string) (Filesystem, *probe.Error) { // minium free disk required for i/o operations to succeed. fs.minFreeDisk = 10 + // Start list goroutine. err = fs.startListService() if err != nil { return Filesystem{}, err.Trace(rootPath) diff --git a/server-config.go b/server-config.go index c6c1c3368..b6dab99b7 100644 --- a/server-config.go +++ b/server-config.go @@ -159,24 +159,6 @@ func createConfigPath() *probe.Error { return nil } -// isAuthConfigFileExists is auth config file exists? -func isConfigFileExists() bool { - if _, err := os.Stat(mustGetConfigFile()); err != nil { - if os.IsNotExist(err) { - return false - } - panic(err) - } - return true -} - -// mustGetConfigFile always get users config file, if not panic -func mustGetConfigFile() string { - configFile, err := getConfigFile() - fatalIf(err.Trace(), "Unable to get config file.", nil) - return configFile -} - // getConfigFile get users config file func getConfigFile() (string, *probe.Error) { configPath, err := getConfigPath() diff --git a/server_fs_test.go b/server_fs_test.go index fb500d870..d833515f0 100644 --- a/server_fs_test.go +++ b/server_fs_test.go @@ -126,16 +126,17 @@ var ignoredHeaders = map[string]bool{ } func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { + if method == "" { + method = "POST" + } t := time.Now().UTC() + req, err := http.NewRequest(method, urlStr, nil) if err != nil { return nil, err } req.Header.Set("x-amz-date", t.Format(iso8601Format)) - if method == "" { - method = "POST" - } // add Content-Length req.ContentLength = contentLength diff --git a/update-main.go b/update-main.go index a4817f646..4fc972d96 100644 --- a/update-main.go +++ b/update-main.go @@ -74,12 +74,6 @@ const ( minioUpdateExperimentalURL = "https://dl.minio.io/server/minio/experimental/" ) -// minioUpdates container to hold updates json. -type minioUpdates struct { - BuildDate string - Platforms map[string]string -} - // updateMessage container to hold update messages. type updateMessage struct { Status string `json:"status"`