From 0c2d58bc6df16c9319ab143f9e7b91ef0384a313 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 28 Feb 2015 14:44:26 -0800 Subject: [PATCH] Implement proper delimiter and prefix handling With this change Minio server now responds with, delimited 'object names' in conjunction with prefix filtering ~~~ example-bucket 1000 / false sample.html 2011-02-26T01:56:20.000Z example-bucket#sample.html 142863 minio minio STANDARD photos/ ~~~ ~~~ example-bucket photos/2006/ 1000 / false photos/2006/feb/ photos/2006/jan/ ~~~ --- pkg/api/minioapi/bucket_handlers.go | 2 +- pkg/api/minioapi/definitions.go | 10 ++-- pkg/api/minioapi/response.go | 19 +++++-- pkg/storage/fs/fs.go | 82 +++++++++++++++++++++-------- pkg/storage/storage.go | 8 +-- 5 files changed, 84 insertions(+), 37 deletions(-) diff --git a/pkg/api/minioapi/bucket_handlers.go b/pkg/api/minioapi/bucket_handlers.go index cd7ace526..f98ad6e3d 100644 --- a/pkg/api/minioapi/bucket_handlers.go +++ b/pkg/api/minioapi/bucket_handlers.go @@ -45,7 +45,7 @@ func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Requ switch err := err.(type) { case nil: // success { - response := generateObjectsListResult(bucket, objects, resources.IsTruncated) + response := generateObjectsListResult(bucket, objects, resources) w.Write(writeObjectHeadersAndResponse(w, response, acceptsContentType)) } case mstorage.BucketNotFound: diff --git a/pkg/api/minioapi/definitions.go b/pkg/api/minioapi/definitions.go index 8192f05b1..1c35dc989 100644 --- a/pkg/api/minioapi/definitions.go +++ b/pkg/api/minioapi/definitions.go @@ -34,10 +34,8 @@ type ObjectListResponse struct { MaxKeys int Delimiter string IsTruncated bool - Contents []*Item `xml:,innerxml` - CommonPrefixes struct { - Prefix string - } `xml:,innerxml` + Contents []*Item `xml:,innerxml` + CommonPrefixes []*Prefix `xml:,innerxml` } // Bucket list response format @@ -49,6 +47,10 @@ type BucketListResponse struct { } `xml:,innerxml` // Buckets are nested } +type Prefix struct { + Prefix string +} + // Bucket struct type Bucket struct { Name string diff --git a/pkg/api/minioapi/response.go b/pkg/api/minioapi/response.go index 17a29a8ea..37cae1080 100644 --- a/pkg/api/minioapi/response.go +++ b/pkg/api/minioapi/response.go @@ -68,8 +68,9 @@ func (b ItemKey) Less(i, j int) bool { return b[i].Key < b[j].Key } // // output: // populated struct that can be serialized to match xml and json api spec output -func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, isTruncated bool) ObjectListResponse { +func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, bucketResources mstorage.BucketResourcesMetadata) ObjectListResponse { var contents []*Item + var prefixes []*Prefix var owner = Owner{} var data = ObjectListResponse{} @@ -78,6 +79,9 @@ func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, for _, object := range objects { var content = &Item{} + if object.Key == "" { + continue + } content.Key = object.Key content.LastModified = object.Created.Format(dateFormat) content.ETag = object.ETag @@ -89,7 +93,16 @@ func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, sort.Sort(ItemKey(contents)) data.Name = bucket data.Contents = contents - data.MaxKeys = MAX_OBJECT_LIST - data.IsTruncated = isTruncated + data.MaxKeys = bucketResources.Maxkeys + data.Prefix = bucketResources.Prefix + data.Delimiter = bucketResources.Delimiter + data.Marker = bucketResources.Marker + data.IsTruncated = bucketResources.IsTruncated + for _, prefix := range bucketResources.CommonPrefixes { + var prefixItem = &Prefix{} + prefixItem.Prefix = prefix + prefixes = append(prefixes, prefixItem) + } + data.CommonPrefixes = prefixes return data } diff --git a/pkg/storage/fs/fs.go b/pkg/storage/fs/fs.go index 06a818e7e..bb839a664 100644 --- a/pkg/storage/fs/fs.go +++ b/pkg/storage/fs/fs.go @@ -314,6 +314,29 @@ func (p *Path) getAllFiles(path string, fl os.FileInfo, err error) error { return nil } +func delimiter(path, delimiter string) string { + delimited := "" + if !strings.Contains(path, delimiter) { + return delimited + } + index := strings.Index(path, delimiter) + if index == -1 { + return delimited + } + delimitedIndex := index + len(delimiter) + delimited = path[:delimitedIndex] + return delimited +} + +func appendU(slice []string, i string) []string { + for _, ele := range slice { + if ele == i { + return slice + } + } + return append(slice, i) +} + type ByObjectKey []mstorage.ObjectMetadata func (b ByObjectKey) Len() int { return len(b) } @@ -350,31 +373,44 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso goto ret } // TODO handle resources.Marker - if resources.Delimiter != "" { - metadata := mstorage.ObjectMetadata{ - Bucket: bucket, - Maxkeys: resources.Maxkeys, - Prefix: resources.Prefix, - Marker: resources.Marker, - Delimiter: resources.Delimiter, + switch true { + case resources.Delimiter != "" && resources.Prefix == "": + delimited := delimiter(name, resources.Delimiter) + switch true { + case delimited == "": + metadata := mstorage.ObjectMetadata{ + Bucket: bucket, + Key: name, + Created: file.ModTime(), + Size: file.Size(), + ETag: bucket + "#" + name, + } + metadataList = append(metadataList, metadata) + case delimited != "": + resources.CommonPrefixes = appendU(resources.CommonPrefixes, delimited) } - metadataList = append(metadataList, metadata) - } - if resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix) { - metadata := mstorage.ObjectMetadata{} - metadataList = append(metadataList, metadata) - } - if strings.HasPrefix(name, resources.Prefix) { + case resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix): + delimited := delimiter(name, resources.Delimiter) + switch true { + case delimited == "": + metadata := mstorage.ObjectMetadata{ + Bucket: bucket, + Key: name, + Created: file.ModTime(), + Size: file.Size(), + ETag: bucket + "#" + name, + } + metadataList = append(metadataList, metadata) + case delimited != "": + resources.CommonPrefixes = appendU(resources.CommonPrefixes, delimited) + } + case strings.HasPrefix(name, resources.Prefix): metadata := mstorage.ObjectMetadata{ - Bucket: bucket, - Maxkeys: resources.Maxkeys, - Prefix: resources.Prefix, - Marker: resources.Marker, - Delimiter: resources.Delimiter, - Key: name, - Created: file.ModTime(), - Size: file.Size(), - ETag: bucket + "#" + name, + Bucket: bucket, + Key: name, + Created: file.ModTime(), + Size: file.Size(), + ETag: bucket + "#" + name, } metadataList = append(metadataList, metadata) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 5814b2bad..01fd041aa 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -43,12 +43,8 @@ type BucketMetadata struct { } type ObjectMetadata struct { - Bucket string - Key string - Maxkeys int - Prefix string - Marker string - Delimiter string + Bucket string + Key string ContentType string Created time.Time