|
|
|
|
@ -1,5 +1,5 @@ |
|
|
|
|
/* |
|
|
|
|
* Minio Cloud Storage, (C) 2015 Minio, Inc. |
|
|
|
|
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc. |
|
|
|
|
* |
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
|
|
|
* you may not use this file except in compliance with the License. |
|
|
|
|
@ -17,363 +17,272 @@ |
|
|
|
|
package fs |
|
|
|
|
|
|
|
|
|
import ( |
|
|
|
|
"io/ioutil" |
|
|
|
|
"errors" |
|
|
|
|
"hash/fnv" |
|
|
|
|
"os" |
|
|
|
|
"path/filepath" |
|
|
|
|
"runtime" |
|
|
|
|
"sort" |
|
|
|
|
"strings" |
|
|
|
|
"time" |
|
|
|
|
|
|
|
|
|
"github.com/minio/minio-xl/pkg/probe" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
// ListObjects - GET bucket (list objects)
|
|
|
|
|
func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) { |
|
|
|
|
fs.lock.Lock() |
|
|
|
|
defer fs.lock.Unlock() |
|
|
|
|
if !IsValidBucketName(bucket) { |
|
|
|
|
return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket}) |
|
|
|
|
} |
|
|
|
|
if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false { |
|
|
|
|
return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}) |
|
|
|
|
// listObjectsParams - list objects input parameters.
|
|
|
|
|
type listObjectsParams struct { |
|
|
|
|
// Bucket name to list the objects for.
|
|
|
|
|
Bucket string |
|
|
|
|
// list all objects with this parameter as common prefix.
|
|
|
|
|
Prefix string |
|
|
|
|
// list all objects starting with object after marker in
|
|
|
|
|
// lexicographical order.
|
|
|
|
|
Marker string |
|
|
|
|
// list all objects until the first occurrence of the delimtier
|
|
|
|
|
// after the prefix.
|
|
|
|
|
Delimiter string |
|
|
|
|
// maximum number of objects returned per listObjects()
|
|
|
|
|
// operation.
|
|
|
|
|
MaxKeys int |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
bucket = fs.denormalizeBucket(bucket) |
|
|
|
|
|
|
|
|
|
p := bucketDir{} |
|
|
|
|
rootPrefix := filepath.Join(fs.path, bucket) |
|
|
|
|
// check bucket exists
|
|
|
|
|
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) { |
|
|
|
|
return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket}) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
p.root = rootPrefix |
|
|
|
|
/// automatically treat incoming "/" as "\\" on windows due to its path constraints.
|
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
if resources.Prefix != "" { |
|
|
|
|
resources.Prefix = strings.Replace(resources.Prefix, "/", string(os.PathSeparator), -1) |
|
|
|
|
} |
|
|
|
|
if resources.Delimiter != "" { |
|
|
|
|
resources.Delimiter = strings.Replace(resources.Delimiter, "/", string(os.PathSeparator), -1) |
|
|
|
|
} |
|
|
|
|
if resources.Marker != "" { |
|
|
|
|
resources.Marker = strings.Replace(resources.Marker, "/", string(os.PathSeparator), -1) |
|
|
|
|
} |
|
|
|
|
// listServiceReq
|
|
|
|
|
type listServiceReq struct { |
|
|
|
|
reqParams listObjectsParams |
|
|
|
|
respCh chan ListObjectsResult |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// if delimiter is supplied and not prefix then we are the very top level, list everything and move on.
|
|
|
|
|
if resources.Delimiter != "" && resources.Prefix == "" { |
|
|
|
|
files, err := ioutil.ReadDir(rootPrefix) |
|
|
|
|
if err != nil { |
|
|
|
|
if os.IsNotExist(err) { |
|
|
|
|
return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket}) |
|
|
|
|
} |
|
|
|
|
return nil, resources, probe.NewError(err) |
|
|
|
|
} |
|
|
|
|
for _, fl := range files { |
|
|
|
|
if strings.HasSuffix(fl.Name(), "$multiparts") { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
p.files = append(p.files, contentInfo{ |
|
|
|
|
Prefix: fl.Name(), |
|
|
|
|
Size: fl.Size(), |
|
|
|
|
Mode: fl.Mode(), |
|
|
|
|
ModTime: fl.ModTime(), |
|
|
|
|
FileInfo: fl, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
type listWorkerReq struct { |
|
|
|
|
respCh chan ListObjectsResult |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// If delimiter and prefix is supplied make sure that paging doesn't go deep, treat it as simple directory listing.
|
|
|
|
|
if resources.Delimiter != "" && resources.Prefix != "" { |
|
|
|
|
if !strings.HasSuffix(resources.Prefix, resources.Delimiter) { |
|
|
|
|
fl, err := os.Stat(filepath.Join(rootPrefix, resources.Prefix)) |
|
|
|
|
if err != nil { |
|
|
|
|
if os.IsNotExist(err) { |
|
|
|
|
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) |
|
|
|
|
} |
|
|
|
|
return nil, resources, probe.NewError(err) |
|
|
|
|
} |
|
|
|
|
p.files = append(p.files, contentInfo{ |
|
|
|
|
Prefix: resources.Prefix, |
|
|
|
|
Size: fl.Size(), |
|
|
|
|
Mode: os.ModeDir, |
|
|
|
|
ModTime: fl.ModTime(), |
|
|
|
|
FileInfo: fl, |
|
|
|
|
}) |
|
|
|
|
// listObjects - list objects lists objects upto maxKeys for a given prefix.
|
|
|
|
|
func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) { |
|
|
|
|
quitWalker := make(chan bool) |
|
|
|
|
reqCh := make(chan listWorkerReq) |
|
|
|
|
walkerCh := make(chan ObjectMetadata) |
|
|
|
|
go func() { |
|
|
|
|
defer close(walkerCh) |
|
|
|
|
var walkPath string |
|
|
|
|
bucketPath := filepath.Join(fs.path, bucket) |
|
|
|
|
// Bucket path prefix should always end with a separator.
|
|
|
|
|
bucketPathPrefix := bucketPath + string(os.PathSeparator) |
|
|
|
|
prefixPath := bucketPathPrefix + prefix |
|
|
|
|
st, err := os.Stat(prefixPath) |
|
|
|
|
if err != nil && os.IsNotExist(err) { |
|
|
|
|
walkPath = bucketPath |
|
|
|
|
} else { |
|
|
|
|
var prefixPath string |
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix |
|
|
|
|
if st.IsDir() && !strings.HasSuffix(prefix, delimiter) { |
|
|
|
|
walkPath = bucketPath |
|
|
|
|
} else { |
|
|
|
|
prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix |
|
|
|
|
} |
|
|
|
|
files, err := ioutil.ReadDir(prefixPath) |
|
|
|
|
if err != nil { |
|
|
|
|
switch err := err.(type) { |
|
|
|
|
case *os.PathError: |
|
|
|
|
if err.Op == "open" { |
|
|
|
|
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) |
|
|
|
|
walkPath = prefixPath |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return nil, resources, probe.NewError(err) |
|
|
|
|
} |
|
|
|
|
for _, fl := range files { |
|
|
|
|
if strings.HasSuffix(fl.Name(), "$multiparts") { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
prefix := fl.Name() |
|
|
|
|
if resources.Prefix != "" { |
|
|
|
|
prefix = filepath.Join(resources.Prefix, fl.Name()) |
|
|
|
|
} |
|
|
|
|
p.files = append(p.files, contentInfo{ |
|
|
|
|
Prefix: prefix, |
|
|
|
|
Size: fl.Size(), |
|
|
|
|
Mode: fl.Mode(), |
|
|
|
|
ModTime: fl.ModTime(), |
|
|
|
|
FileInfo: fl, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if resources.Delimiter == "" { |
|
|
|
|
var files []contentInfo |
|
|
|
|
getAllFiles := func(fp string, fl os.FileInfo, err error) error { |
|
|
|
|
// If any error return back quickly
|
|
|
|
|
if err != nil { |
|
|
|
|
return err |
|
|
|
|
} |
|
|
|
|
if strings.HasSuffix(fp, "$multiparts") { |
|
|
|
|
Walk(walkPath, func(path string, info os.FileInfo, err error) error { |
|
|
|
|
// We don't need to list the walk path.
|
|
|
|
|
if path == walkPath { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
// if file pointer equals to rootPrefix - discard it
|
|
|
|
|
if fp == p.root { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
if len(files) > resources.Maxkeys { |
|
|
|
|
return ErrSkipFile |
|
|
|
|
} |
|
|
|
|
// Split the root prefix from the incoming file pointer
|
|
|
|
|
realFp := "" |
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 { |
|
|
|
|
realFp = splits[1] |
|
|
|
|
// For all incoming directories add a ending separator.
|
|
|
|
|
if info.IsDir() { |
|
|
|
|
path = path + string(os.PathSeparator) |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 { |
|
|
|
|
realFp = splits[1] |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// If path is a directory and has a prefix verify if the file pointer
|
|
|
|
|
// has the prefix if it does not skip the directory.
|
|
|
|
|
if fl.Mode().IsDir() { |
|
|
|
|
if resources.Prefix != "" { |
|
|
|
|
// Skip the directory on following situations
|
|
|
|
|
// - when prefix is part of file pointer along with the root path
|
|
|
|
|
// - when file pointer is part of the prefix along with root path
|
|
|
|
|
if !strings.HasPrefix(fp, filepath.Join(p.root, resources.Prefix)) && |
|
|
|
|
!strings.HasPrefix(filepath.Join(p.root, resources.Prefix), fp) { |
|
|
|
|
return ErrSkipDir |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// If path is a directory and has a marker verify if the file split file pointer
|
|
|
|
|
// is lesser than the Marker top level directory if yes skip it.
|
|
|
|
|
if fl.Mode().IsDir() { |
|
|
|
|
if resources.Marker != "" { |
|
|
|
|
if realFp != "" { |
|
|
|
|
// For windows split with its own os.PathSeparator
|
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] { |
|
|
|
|
return ErrSkipDir |
|
|
|
|
// Extract object name.
|
|
|
|
|
objectName := strings.TrimPrefix(path, bucketPathPrefix) |
|
|
|
|
if strings.HasPrefix(objectName, prefix) { |
|
|
|
|
// For objectName lesser than marker, ignore.
|
|
|
|
|
if marker >= objectName { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] { |
|
|
|
|
object := ObjectMetadata{ |
|
|
|
|
Object: objectName, |
|
|
|
|
Created: info.ModTime(), |
|
|
|
|
Mode: info.Mode(), |
|
|
|
|
Size: info.Size(), |
|
|
|
|
} |
|
|
|
|
select { |
|
|
|
|
// Send object on walker channel.
|
|
|
|
|
case walkerCh <- object: |
|
|
|
|
case <-quitWalker: |
|
|
|
|
// Returning error ends the file tree Walk().
|
|
|
|
|
return errors.New("Quit list worker.") |
|
|
|
|
} |
|
|
|
|
// If delimiter is set, we stop if current path is a
|
|
|
|
|
// directory.
|
|
|
|
|
if delimiter != "" && info.IsDir() { |
|
|
|
|
return ErrSkipDir |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// If regular file verify
|
|
|
|
|
if fl.Mode().IsRegular() { |
|
|
|
|
// If marker is present this will be used to check if filepointer is
|
|
|
|
|
// lexically higher than then Marker
|
|
|
|
|
if realFp != "" { |
|
|
|
|
if resources.Marker != "" { |
|
|
|
|
if realFp > resources.Marker { |
|
|
|
|
files = append(files, contentInfo{ |
|
|
|
|
Prefix: realFp, |
|
|
|
|
Size: fl.Size(), |
|
|
|
|
Mode: fl.Mode(), |
|
|
|
|
ModTime: fl.ModTime(), |
|
|
|
|
FileInfo: fl, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
files = append(files, contentInfo{ |
|
|
|
|
Prefix: realFp, |
|
|
|
|
Size: fl.Size(), |
|
|
|
|
Mode: fl.Mode(), |
|
|
|
|
ModTime: fl.ModTime(), |
|
|
|
|
FileInfo: fl, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// If file is a symlink follow it and populate values.
|
|
|
|
|
if fl.Mode()&os.ModeSymlink == os.ModeSymlink { |
|
|
|
|
st, err := os.Stat(fp) |
|
|
|
|
if err != nil { |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
// If marker is present this will be used to check if filepointer is
|
|
|
|
|
// lexically higher than then Marker
|
|
|
|
|
if realFp != "" { |
|
|
|
|
if resources.Marker != "" { |
|
|
|
|
if realFp > resources.Marker { |
|
|
|
|
files = append(files, contentInfo{ |
|
|
|
|
Prefix: realFp, |
|
|
|
|
Size: st.Size(), |
|
|
|
|
Mode: st.Mode(), |
|
|
|
|
ModTime: st.ModTime(), |
|
|
|
|
FileInfo: st, |
|
|
|
|
}) |
|
|
|
|
}() |
|
|
|
|
|
|
|
|
|
go func() { |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
// Timeout after 10 seconds if request did not arrive for
|
|
|
|
|
// the given list parameters.
|
|
|
|
|
case <-time.After(10 * time.Second): |
|
|
|
|
quitWalker <- true // Quit file path walk if running.
|
|
|
|
|
// Send back the hash for this request.
|
|
|
|
|
fs.timeoutReqCh <- fnvSum(bucket, prefix, marker, delimiter) |
|
|
|
|
return |
|
|
|
|
case req, ok := <-reqCh: |
|
|
|
|
if !ok { |
|
|
|
|
// If the request channel is closed, no more
|
|
|
|
|
// requests return here.
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
resp := ListObjectsResult{} |
|
|
|
|
var count int |
|
|
|
|
for object := range walkerCh { |
|
|
|
|
if count == maxKeys { |
|
|
|
|
resp.IsTruncated = true |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
// If object is a directory.
|
|
|
|
|
if object.Mode.IsDir() { |
|
|
|
|
if delimiter == "" { |
|
|
|
|
// Skip directories for recursive listing.
|
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
resp.Prefixes = append(resp.Prefixes, object.Object) |
|
|
|
|
} else { |
|
|
|
|
files = append(files, contentInfo{ |
|
|
|
|
Prefix: realFp, |
|
|
|
|
Size: st.Size(), |
|
|
|
|
Mode: st.Mode(), |
|
|
|
|
ModTime: st.ModTime(), |
|
|
|
|
FileInfo: st, |
|
|
|
|
}) |
|
|
|
|
resp.Objects = append(resp.Objects, object) |
|
|
|
|
} |
|
|
|
|
// Set the next marker for the next request.
|
|
|
|
|
resp.NextMarker = object.Object |
|
|
|
|
count++ |
|
|
|
|
} |
|
|
|
|
req.respCh <- resp |
|
|
|
|
} |
|
|
|
|
p.files = files |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
// If no delimiter is specified, crawl through everything.
|
|
|
|
|
err := Walk(rootPrefix, getAllFiles) |
|
|
|
|
if err != nil { |
|
|
|
|
if os.IsNotExist(err) { |
|
|
|
|
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix}) |
|
|
|
|
}() |
|
|
|
|
return reqCh, nil |
|
|
|
|
} |
|
|
|
|
return nil, resources, probe.NewError(err) |
|
|
|
|
|
|
|
|
|
// fnvSum calculates a hash for concatenation of all input strings.
|
|
|
|
|
func fnvSum(elements ...string) uint32 { |
|
|
|
|
fnvHash := fnv.New32a() |
|
|
|
|
for _, element := range elements { |
|
|
|
|
fnvHash.Write([]byte(element)) |
|
|
|
|
} |
|
|
|
|
return fnvHash.Sum32() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
var metadataList []ObjectMetadata |
|
|
|
|
var metadata ObjectMetadata |
|
|
|
|
// listObjectsService - list objects service manages various incoming
|
|
|
|
|
// list object requests by delegating them to an existing listObjects
|
|
|
|
|
// routine or initializes a new listObjects routine.
|
|
|
|
|
func (fs *Filesystem) listObjectsService() *probe.Error { |
|
|
|
|
// Initialize list service request channel.
|
|
|
|
|
listServiceReqCh := make(chan listServiceReq) |
|
|
|
|
fs.listServiceReqCh = listServiceReqCh |
|
|
|
|
|
|
|
|
|
// Filter objects
|
|
|
|
|
for _, content := range p.files { |
|
|
|
|
if len(metadataList) == resources.Maxkeys { |
|
|
|
|
resources.IsTruncated = true |
|
|
|
|
if resources.IsTruncated && resources.Delimiter != "" { |
|
|
|
|
resources.NextMarker = metadataList[len(metadataList)-1].Object |
|
|
|
|
} |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
if content.Prefix > resources.Marker { |
|
|
|
|
// Initialize timeout request channel to receive request hashes of
|
|
|
|
|
// timed-out requests.
|
|
|
|
|
timeoutReqCh := make(chan uint32) |
|
|
|
|
fs.timeoutReqCh = timeoutReqCh |
|
|
|
|
|
|
|
|
|
// Initialize request hash to list worker map.
|
|
|
|
|
reqToListWorkerReqCh := make(map[uint32]chan<- listWorkerReq) |
|
|
|
|
|
|
|
|
|
// Start service in a go routine.
|
|
|
|
|
go func() { |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
case reqHash := <-timeoutReqCh: |
|
|
|
|
// For requests which have timed-out, close the worker
|
|
|
|
|
// channels proactively, this may happen for idle
|
|
|
|
|
// workers once in 10seconds.
|
|
|
|
|
listWorkerReqCh, ok := reqToListWorkerReqCh[reqHash] |
|
|
|
|
if ok { |
|
|
|
|
close(listWorkerReqCh) |
|
|
|
|
} |
|
|
|
|
delete(reqToListWorkerReqCh, reqHash) |
|
|
|
|
case srvReq := <-listServiceReqCh: |
|
|
|
|
// Save the params for readability.
|
|
|
|
|
bucket := srvReq.reqParams.Bucket |
|
|
|
|
prefix := srvReq.reqParams.Prefix |
|
|
|
|
marker := srvReq.reqParams.Marker |
|
|
|
|
delimiter := srvReq.reqParams.Delimiter |
|
|
|
|
maxKeys := srvReq.reqParams.MaxKeys |
|
|
|
|
|
|
|
|
|
// Generate hash.
|
|
|
|
|
reqHash := fnvSum(bucket, prefix, marker, delimiter) |
|
|
|
|
listWorkerReqCh, ok := reqToListWorkerReqCh[reqHash] |
|
|
|
|
if !ok { |
|
|
|
|
var err *probe.Error |
|
|
|
|
metadata, resources, err = fs.filterObjects(bucket, content, resources) |
|
|
|
|
listWorkerReqCh, err = fs.listObjects(bucket, prefix, marker, delimiter, maxKeys) |
|
|
|
|
if err != nil { |
|
|
|
|
return nil, resources, err.Trace() |
|
|
|
|
srvReq.respCh <- ListObjectsResult{} |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
// If windows replace all the incoming paths to API compatible paths
|
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
metadata.Object = sanitizeWindowsPath(metadata.Object) |
|
|
|
|
reqToListWorkerReqCh[reqHash] = listWorkerReqCh |
|
|
|
|
} |
|
|
|
|
if metadata.Bucket != "" { |
|
|
|
|
metadataList = append(metadataList, metadata) |
|
|
|
|
respCh := make(chan ListObjectsResult) |
|
|
|
|
listWorkerReqCh <- listWorkerReq{respCh} |
|
|
|
|
resp, ok := <-respCh |
|
|
|
|
if !ok { |
|
|
|
|
srvReq.respCh <- ListObjectsResult{} |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
delete(reqToListWorkerReqCh, reqHash) |
|
|
|
|
if !resp.IsTruncated { |
|
|
|
|
close(listWorkerReqCh) |
|
|
|
|
} else { |
|
|
|
|
nextMarker := resp.NextMarker |
|
|
|
|
reqHash = fnvSum(bucket, prefix, nextMarker, delimiter) |
|
|
|
|
reqToListWorkerReqCh[reqHash] = listWorkerReqCh |
|
|
|
|
} |
|
|
|
|
srvReq.respCh <- resp |
|
|
|
|
} |
|
|
|
|
// Sanitize common prefixes back into API compatible paths
|
|
|
|
|
if runtime.GOOS == "windows" { |
|
|
|
|
resources.CommonPrefixes = sanitizeWindowsPaths(resources.CommonPrefixes...) |
|
|
|
|
} |
|
|
|
|
return metadataList, resources, nil |
|
|
|
|
}() |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (fs Filesystem) filterObjects(bucket string, content contentInfo, resources BucketResourcesMetadata) (ObjectMetadata, BucketResourcesMetadata, *probe.Error) { |
|
|
|
|
var err *probe.Error |
|
|
|
|
var metadata ObjectMetadata |
|
|
|
|
// ListObjects - lists all objects for a given prefix, returns upto
|
|
|
|
|
// maxKeys number of objects per call.
|
|
|
|
|
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) { |
|
|
|
|
fs.lock.Lock() |
|
|
|
|
defer fs.lock.Unlock() |
|
|
|
|
|
|
|
|
|
name := content.Prefix |
|
|
|
|
switch true { |
|
|
|
|
// Both delimiter and Prefix is present
|
|
|
|
|
case resources.Delimiter != "" && resources.Prefix != "": |
|
|
|
|
if strings.HasPrefix(name, resources.Prefix) { |
|
|
|
|
trimmedName := strings.TrimPrefix(name, resources.Prefix) |
|
|
|
|
delimitedName := delimiter(trimmedName, resources.Delimiter) |
|
|
|
|
switch true { |
|
|
|
|
case name == resources.Prefix: |
|
|
|
|
// Use resources.Prefix to filter out delimited file
|
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
} |
|
|
|
|
if metadata.Mode.IsDir() { |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter) |
|
|
|
|
return ObjectMetadata{}, resources, nil |
|
|
|
|
} |
|
|
|
|
case delimitedName == content.FileInfo.Name(): |
|
|
|
|
// Use resources.Prefix to filter out delimited files
|
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
} |
|
|
|
|
if metadata.Mode.IsDir() { |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter) |
|
|
|
|
return ObjectMetadata{}, resources, nil |
|
|
|
|
} |
|
|
|
|
case delimitedName != "": |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, resources.Prefix+delimitedName) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
// Delimiter present and Prefix is absent
|
|
|
|
|
case resources.Delimiter != "" && resources.Prefix == "": |
|
|
|
|
delimitedName := delimiter(name, resources.Delimiter) |
|
|
|
|
switch true { |
|
|
|
|
case delimitedName == "": |
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
} |
|
|
|
|
if metadata.Mode.IsDir() { |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter) |
|
|
|
|
return ObjectMetadata{}, resources, nil |
|
|
|
|
} |
|
|
|
|
case delimitedName == content.FileInfo.Name(): |
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
} |
|
|
|
|
if metadata.Mode.IsDir() { |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter) |
|
|
|
|
return ObjectMetadata{}, resources, nil |
|
|
|
|
if !IsValidBucketName(bucket) { |
|
|
|
|
return ListObjectsResult{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) |
|
|
|
|
} |
|
|
|
|
case delimitedName != "": |
|
|
|
|
resources.CommonPrefixes = append(resources.CommonPrefixes, delimitedName) |
|
|
|
|
|
|
|
|
|
bucket = fs.denormalizeBucket(bucket) |
|
|
|
|
rootPrefix := filepath.Join(fs.path, bucket) |
|
|
|
|
// check bucket exists
|
|
|
|
|
if _, e := os.Stat(rootPrefix); e != nil { |
|
|
|
|
if os.IsNotExist(e) { |
|
|
|
|
return ListObjectsResult{}, probe.NewError(BucketNotFound{Bucket: bucket}) |
|
|
|
|
} |
|
|
|
|
// Delimiter is absent and only Prefix is present
|
|
|
|
|
case resources.Delimiter == "" && resources.Prefix != "": |
|
|
|
|
if strings.HasPrefix(name, resources.Prefix) { |
|
|
|
|
// Do not strip prefix object output
|
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
return ListObjectsResult{}, probe.NewError(e) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
reqParams := listObjectsParams{} |
|
|
|
|
reqParams.Bucket = bucket |
|
|
|
|
reqParams.Prefix = filepath.FromSlash(prefix) |
|
|
|
|
reqParams.Marker = filepath.FromSlash(marker) |
|
|
|
|
reqParams.Delimiter = filepath.FromSlash(delimiter) |
|
|
|
|
reqParams.MaxKeys = maxKeys |
|
|
|
|
|
|
|
|
|
respCh := make(chan ListObjectsResult) |
|
|
|
|
fs.listServiceReqCh <- listServiceReq{reqParams, respCh} |
|
|
|
|
resp := <-respCh |
|
|
|
|
|
|
|
|
|
for i := range resp.Prefixes { |
|
|
|
|
resp.Prefixes[i] = filepath.ToSlash(resp.Prefixes[i]) |
|
|
|
|
} |
|
|
|
|
default: |
|
|
|
|
metadata, err = getMetadata(fs.path, bucket, name) |
|
|
|
|
if err != nil { |
|
|
|
|
return ObjectMetadata{}, resources, err.Trace() |
|
|
|
|
for i := range resp.Objects { |
|
|
|
|
resp.Objects[i].Object = filepath.ToSlash(resp.Objects[i].Object) |
|
|
|
|
} |
|
|
|
|
if reqParams.Delimiter == "" { |
|
|
|
|
// This element is set only if you have delimiter set.
|
|
|
|
|
// If response does not include the NextMaker and it is
|
|
|
|
|
// truncated, you can use the value of the last Key in the
|
|
|
|
|
// response as the marker in the subsequent request to get the
|
|
|
|
|
// next set of object keys.
|
|
|
|
|
resp.NextMarker = "" |
|
|
|
|
} |
|
|
|
|
sortUnique(sort.StringSlice(resources.CommonPrefixes)) |
|
|
|
|
return metadata, resources, nil |
|
|
|
|
return resp, nil |
|
|
|
|
} |
|
|
|
|
|
frankw
9 years ago