oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

listObjects: Channel based ftw - initial implementation.

Browse Source
master
Krishna Srinivas 9 years ago committed by Harshavardhana
parent 67a70eb6d6
commit 9e18bfa60e
5 changed files with 239 additions and 349 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 21
      api-response.go
  2. 12
      bucket-handlers.go
  3. 25
      pkg/fs/definitions.go
  4. 503
      pkg/fs/fs-bucket-listobjects.go
  5. 27
      pkg/fs/fs.go

21
api-response.go
Unescape View File

@ -92,7 +92,8 @@ func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyRe
} }
// generates an ListObjects response for the said bucket with other enumerated options. // generates an ListObjects response for the said bucket with other enumerated options.
func generateListObjectsResponse(bucket string, objects []fs.ObjectMetadata, bucketResources fs.BucketResourcesMetadata) ListObjectsResponse { // func generateListObjectsResponse(bucket string, objects []fs.ObjectMetadata, bucketResources fs.BucketResourcesMetadata) ListObjectsResponse {
func generateListObjectsResponse(bucket string, req fs.ListObjectsReq, resp fs.ListObjectsResp) ListObjectsResponse {
var contents []*Object var contents []*Object
var prefixes []*CommonPrefix var prefixes []*CommonPrefix
var owner = Owner{} var owner = Owner{}
@ -101,7 +102,7 @@ func generateListObjectsResponse(bucket string, objects []fs.ObjectMetadata, buc
owner.ID = "minio" owner.ID = "minio"
owner.DisplayName = "minio" owner.DisplayName = "minio"
for _, object := range objects { for _, object := range resp.Objects {
var content = &Object{} var content = &Object{}
if object.Object == "" { if object.Object == "" {
continue continue
@ -117,13 +118,15 @@ func generateListObjectsResponse(bucket string, objects []fs.ObjectMetadata, buc
// TODO - support EncodingType in xml decoding // TODO - support EncodingType in xml decoding
data.Name = bucket data.Name = bucket
data.Contents = contents data.Contents = contents
data.MaxKeys = bucketResources.Maxkeys
data.Prefix = bucketResources.Prefix data.MaxKeys = req.MaxKeys
data.Delimiter = bucketResources.Delimiter data.Prefix = req.Prefix
data.Marker = bucketResources.Marker data.Delimiter = req.Delimiter
data.NextMarker = bucketResources.NextMarker data.Marker = req.Marker
data.IsTruncated = bucketResources.IsTruncated
for _, prefix := range bucketResources.CommonPrefixes { data.NextMarker = resp.NextMarker
data.IsTruncated = resp.IsTruncated
for _, prefix := range resp.Prefixes {
var prefixItem = &CommonPrefix{} var prefixItem = &CommonPrefix{}
prefixItem.Prefix = prefix prefixItem.Prefix = prefix
prefixes = append(prefixes, prefixItem) prefixes = append(prefixes, prefixItem)

12
bucket-handlers.go
Unescape View File

@ -137,10 +137,16 @@ func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.R
resources.Maxkeys = maxObjectList resources.Maxkeys = maxObjectList
} }
objects, resources, err := api.Filesystem.ListObjects(bucket, resources) listReq := fs.ListObjectsReq{
Prefix: resources.Prefix,
Marker: resources.Marker,
Delimiter: resources.Delimiter,
MaxKeys: resources.Maxkeys,
}
listResp, err := api.Filesystem.ListObjects(bucket, listReq)
if err == nil { if err == nil {
// Generate response // generate response
response := generateListObjectsResponse(bucket, objects, resources) response := generateListObjectsResponse(bucket, listReq, listResp)
encodedSuccessResponse := encodeSuccessResponse(response) encodedSuccessResponse := encodeSuccessResponse(response)
// Write headers // Write headers
setCommonHeaders(w) setCommonHeaders(w)

25
pkg/fs/definitions.go
Unescape View File

@ -130,6 +130,31 @@ type BucketResourcesMetadata struct {
CommonPrefixes []string CommonPrefixes []string
} }
type ListObjectsReq struct {
Bucket string
Prefix string
Marker string
Delimiter string
MaxKeys int
}
type ListObjectsResp struct {
IsTruncated bool
NextMarker string
Objects []ObjectMetadata
Prefixes []string
}
type listServiceReq struct {
req ListObjectsReq
respCh chan ListObjectsResp
}
type listWorkerReq struct {
req ListObjectsReq
respCh chan ListObjectsResp
}
// CompletePart - completed part container // CompletePart - completed part container
type CompletePart struct { type CompletePart struct {
PartNumber int PartNumber int

503
pkg/fs/fs-bucket-listobjects.go
Unescape View File

@ -17,363 +17,214 @@
package fs package fs
import ( import (
"io/ioutil" "errors"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"sort"
"strings" "strings"
"time"
"github.com/minio/minio-xl/pkg/probe" "github.com/minio/minio-xl/pkg/probe"
) )
// ListObjects - GET bucket (list objects) func (fs Filesystem) listWorker(startReq ListObjectsReq) (chan<- listWorkerReq, *probe.Error) {
func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) { Separator := string(os.PathSeparator)
fs.lock.Lock() bucket := startReq.Bucket
defer fs.lock.Unlock() prefix := startReq.Prefix
if !IsValidBucketName(bucket) { marker := startReq.Marker
return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket}) delimiter := startReq.Delimiter
} quit := make(chan bool)
if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false { if marker != "" {
return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}) return nil, probe.NewError(errors.New("Not supported"))
}
bucket = fs.denormalizeBucket(bucket)
p := bucketDir{}
rootPrefix := filepath.Join(fs.path, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
}
p.root = rootPrefix
/// automatically treat incoming "/" as "\\" on windows due to its path constraints.
if runtime.GOOS == "windows" {
if resources.Prefix != "" {
resources.Prefix = strings.Replace(resources.Prefix, "/", string(os.PathSeparator), -1)
}
if resources.Delimiter != "" {
resources.Delimiter = strings.Replace(resources.Delimiter, "/", string(os.PathSeparator), -1)
}
if resources.Marker != "" {
resources.Marker = strings.Replace(resources.Marker, "/", string(os.PathSeparator), -1)
}
} }
if delimiter != "" && delimiter != Separator {
// if delimiter is supplied and not prefix then we are the very top level, list everything and move on. return nil, probe.NewError(errors.New("Not supported"))
if resources.Delimiter != "" && resources.Prefix == "" {
files, err := ioutil.ReadDir(rootPrefix)
if err != nil {
if os.IsNotExist(err) {
return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
}
return nil, resources, probe.NewError(err)
}
for _, fl := range files {
if strings.HasSuffix(fl.Name(), "$multiparts") {
continue
}
p.files = append(p.files, contentInfo{
Prefix: fl.Name(),
Size: fl.Size(),
Mode: fl.Mode(),
ModTime: fl.ModTime(),
FileInfo: fl,
})
}
} }
reqCh := make(chan listWorkerReq)
// If delimiter and prefix is supplied make sure that paging doesn't go deep, treat it as simple directory listing. walkerCh := make(chan ObjectMetadata)
if resources.Delimiter != "" && resources.Prefix != "" { go func() {
if !strings.HasSuffix(resources.Prefix, resources.Delimiter) { rootPath := filepath.Join(fs.path, bucket, prefix)
fl, err := os.Stat(filepath.Join(rootPrefix, resources.Prefix)) stripPath := filepath.Join(fs.path, bucket) + Separator
if err != nil { filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
if os.IsNotExist(err) { if path == rootPath {
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
}
return nil, resources, probe.NewError(err)
}
p.files = append(p.files, contentInfo{
Prefix: resources.Prefix,
Size: fl.Size(),
Mode: os.ModeDir,
ModTime: fl.ModTime(),
FileInfo: fl,
})
} else {
var prefixPath string
if runtime.GOOS == "windows" {
prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix
} else {
prefixPath = rootPrefix + string(os.PathSeparator) + resources.Prefix
}
files, err := ioutil.ReadDir(prefixPath)
if err != nil {
switch err := err.(type) {
case *os.PathError:
if err.Op == "open" {
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
}
}
return nil, resources, probe.NewError(err)
}
for _, fl := range files {
if strings.HasSuffix(fl.Name(), "$multiparts") {
continue
}
prefix := fl.Name()
if resources.Prefix != "" {
prefix = filepath.Join(resources.Prefix, fl.Name())
}
p.files = append(p.files, contentInfo{
Prefix: prefix,
Size: fl.Size(),
Mode: fl.Mode(),
ModTime: fl.ModTime(),
FileInfo: fl,
})
}
}
}
if resources.Delimiter == "" {
var files []contentInfo
getAllFiles := func(fp string, fl os.FileInfo, err error) error {
// If any error return back quickly
if err != nil {
return err
}
if strings.HasSuffix(fp, "$multiparts") {
return nil return nil
} }
// if file pointer equals to rootPrefix - discard it if info.IsDir() {
if fp == p.root { path = path + Separator
return nil
} }
if len(files) > resources.Maxkeys { objectName := strings.TrimPrefix(path, stripPath)
return ErrSkipFile object := ObjectMetadata{
Object: objectName,
Created: info.ModTime(),
Mode: info.Mode(),
Size: info.Size(),
} }
// Split the root prefix from the incoming file pointer select {
realFp := "" case walkerCh <- object:
if runtime.GOOS == "windows" { // do nothings
if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 { case <-quit:
realFp = splits[1] fmt.Println("walker got quit")
} // returning error ends the Walk()
} else { return errors.New("Ending")
if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 {
realFp = splits[1]
}
} }
// If path is a directory and has a prefix verify if the file pointer if delimiter == Separator && info.IsDir() {
// has the prefix if it does not skip the directory. return filepath.SkipDir
if fl.Mode().IsDir() {
if resources.Prefix != "" {
// Skip the directory on following situations
// - when prefix is part of file pointer along with the root path
// - when file pointer is part of the prefix along with root path
if !strings.HasPrefix(fp, filepath.Join(p.root, resources.Prefix)) &&
!strings.HasPrefix(filepath.Join(p.root, resources.Prefix), fp) {
return ErrSkipDir
}
}
}
// If path is a directory and has a marker verify if the file split file pointer
// is lesser than the Marker top level directory if yes skip it.
if fl.Mode().IsDir() {
if resources.Marker != "" {
if realFp != "" {
// For windows split with its own os.PathSeparator
if runtime.GOOS == "windows" {
if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
return ErrSkipDir
}
} else {
if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
return ErrSkipDir
}
}
}
}
} }
// If regular file verify return nil
if fl.Mode().IsRegular() { })
// If marker is present this will be used to check if filepointer is close(walkerCh)
// lexically higher than then Marker }()
if realFp != "" { go func() {
if resources.Marker != "" { resp := ListObjectsResp{}
if realFp > resources.Marker { for {
files = append(files, contentInfo{ select {
Prefix: realFp, case <-time.After(10 * time.Second):
Size: fl.Size(), fmt.Println("worker got timeout")
Mode: fl.Mode(), quit <- true
ModTime: fl.ModTime(), timeoutReq := ListObjectsReq{bucket, prefix, marker, delimiter, 0}
FileInfo: fl, fmt.Println("after timeout", fs)
}) fs.timeoutReqCh <- timeoutReq
// FIXME: can there be a race such that sender on reqCh panics?
return
case req := <-reqCh:
resp = ListObjectsResp{}
resp.Objects = make([]ObjectMetadata, 0)
resp.Prefixes = make([]string, 0)
count := 0
for object := range walkerCh {
if object.Mode.IsDir() {
if delimiter == "" {
// skip directories for recursive list
continue
} }
resp.Prefixes = append(resp.Prefixes, object.Object)
} else { } else {
files = append(files, contentInfo{ resp.Objects = append(resp.Objects, object)
Prefix: realFp, }
Size: fl.Size(), resp.NextMarker = object.Object
Mode: fl.Mode(), count++
ModTime: fl.ModTime(), if count == req.req.MaxKeys {
FileInfo: fl, resp.IsTruncated = true
}) break
} }
} }
fmt.Println("response objects: ", len(resp.Objects))
marker = resp.NextMarker
req.respCh <- resp
} }
// If file is a symlink follow it and populate values. }
if fl.Mode()&os.ModeSymlink == os.ModeSymlink { }()
st, err := os.Stat(fp) return reqCh, nil
if err != nil { }
return nil
func (fs *Filesystem) startListService() *probe.Error {
fmt.Println("startListService starting")
listServiceReqCh := make(chan listServiceReq)
timeoutReqCh := make(chan ListObjectsReq)
reqToListWorkerReqCh := make(map[string](chan<- listWorkerReq))
reqToStr := func(bucket string, prefix string, marker string, delimiter string) string {
return strings.Join([]string{bucket, prefix, marker, delimiter}, ":")
}
go func() {
for {
select {
case timeoutReq := <-timeoutReqCh:
fmt.Println("listservice got timeout on ", timeoutReq)
reqStr := reqToStr(timeoutReq.Bucket, timeoutReq.Prefix, timeoutReq.Marker, timeoutReq.Delimiter)
listWorkerReqCh, ok := reqToListWorkerReqCh[reqStr]
if ok {
close(listWorkerReqCh)
} }
// If marker is present this will be used to check if filepointer is delete(reqToListWorkerReqCh, reqStr)
// lexically higher than then Marker case serviceReq := <-listServiceReqCh:
if realFp != "" { fmt.Println("serviceReq received", serviceReq)
if resources.Marker != "" { fmt.Println("sending to listservicereqch", fs)
if realFp > resources.Marker {
files = append(files, contentInfo{ reqStr := reqToStr(serviceReq.req.Bucket, serviceReq.req.Prefix, serviceReq.req.Marker, serviceReq.req.Delimiter)
Prefix: realFp, listWorkerReqCh, ok := reqToListWorkerReqCh[reqStr]
Size: st.Size(), if !ok {
Mode: st.Mode(), var err *probe.Error
ModTime: st.ModTime(), listWorkerReqCh, err = fs.listWorker(serviceReq.req)
FileInfo: st, if err != nil {
}) fmt.Println("listWorker returned error", err)
} serviceReq.respCh <- ListObjectsResp{}
} else { return
files = append(files, contentInfo{
Prefix: realFp,
Size: st.Size(),
Mode: st.Mode(),
ModTime: st.ModTime(),
FileInfo: st,
})
} }
reqToListWorkerReqCh[reqStr] = listWorkerReqCh
} }
respCh := make(chan ListObjectsResp)
listWorkerReqCh <- listWorkerReq{serviceReq.req, respCh}
resp, ok := <-respCh
if !ok {
serviceReq.respCh <- ListObjectsResp{}
fmt.Println("listWorker resp was not ok")
return
}
delete(reqToListWorkerReqCh, reqStr)
if !resp.IsTruncated {
close(listWorkerReqCh)
} else {
reqStr = reqToStr(serviceReq.req.Bucket, serviceReq.req.Prefix, resp.NextMarker, serviceReq.req.Delimiter)
reqToListWorkerReqCh[reqStr] = listWorkerReqCh
}
serviceReq.respCh <- resp
} }
p.files = files
return nil
}
// If no delimiter is specified, crawl through everything.
err := Walk(rootPrefix, getAllFiles)
if err != nil {
if os.IsNotExist(err) {
return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
}
return nil, resources, probe.NewError(err)
} }
} }()
fs.timeoutReqCh = timeoutReqCh
fs.listServiceReqCh = listServiceReqCh
return nil
}
var metadataList []ObjectMetadata // ListObjects -
var metadata ObjectMetadata func (fs Filesystem) ListObjects(bucket string, req ListObjectsReq) (ListObjectsResp, *probe.Error) {
fs.lock.Lock()
defer fs.lock.Unlock()
// Filter objects Separator := string(os.PathSeparator)
for _, content := range p.files { if !IsValidBucketName(bucket) {
if len(metadataList) == resources.Maxkeys { return ListObjectsResp{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
resources.IsTruncated = true }
if resources.IsTruncated && resources.Delimiter != "" {
resources.NextMarker = metadataList[len(metadataList)-1].Object bucket = fs.denormalizeBucket(bucket)
} rootPrefix := filepath.Join(fs.path, bucket)
break // check bucket exists
} if _, e := os.Stat(rootPrefix); e != nil {
if content.Prefix > resources.Marker { if os.IsNotExist(e) {
var err *probe.Error return ListObjectsResp{}, probe.NewError(BucketNotFound{Bucket: bucket})
metadata, resources, err = fs.filterObjects(bucket, content, resources)
if err != nil {
return nil, resources, err.Trace()
}
// If windows replace all the incoming paths to API compatible paths
if runtime.GOOS == "windows" {
metadata.Object = sanitizeWindowsPath(metadata.Object)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
} }
return ListObjectsResp{}, probe.NewError(e)
} }
// Sanitize common prefixes back into API compatible paths
if runtime.GOOS == "windows" { canonicalize := func(str string) string {
resources.CommonPrefixes = sanitizeWindowsPaths(resources.CommonPrefixes...) return strings.Replace(str, "/", string(os.PathSeparator), -1)
}
decanonicalize := func(str string) string {
return strings.Replace(str, string(os.PathSeparator), "/", -1)
} }
return metadataList, resources, nil
}
func (fs Filesystem) filterObjects(bucket string, content contentInfo, resources BucketResourcesMetadata) (ObjectMetadata, BucketResourcesMetadata, *probe.Error) { req.Bucket = bucket
var err *probe.Error req.Prefix = canonicalize(req.Prefix)
var metadata ObjectMetadata req.Marker = canonicalize(req.Marker)
req.Delimiter = canonicalize(req.Delimiter)
name := content.Prefix if req.Delimiter != "" && req.Delimiter != Separator {
switch true { return ListObjectsResp{}, probe.NewError(errors.New("not supported"))
// Both delimiter and Prefix is present }
case resources.Delimiter != "" && resources.Prefix != "":
if strings.HasPrefix(name, resources.Prefix) { respCh := make(chan ListObjectsResp)
trimmedName := strings.TrimPrefix(name, resources.Prefix) fs.listServiceReqCh <- listServiceReq{req, respCh}
delimitedName := delimiter(trimmedName, resources.Delimiter) resp := <-respCh
switch true {
case name == resources.Prefix: for i := 0; i < len(resp.Prefixes); i++ {
// Use resources.Prefix to filter out delimited file resp.Prefixes[i] = decanonicalize(resp.Prefixes[i])
metadata, err = getMetadata(fs.path, bucket, name) }
if err != nil { for i := 0; i < len(resp.Objects); i++ {
return ObjectMetadata{}, resources, err.Trace() resp.Objects[i].Object = decanonicalize(resp.Objects[i].Object)
} }
if metadata.Mode.IsDir() { if req.Delimiter == "" {
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter) // unset NextMaker for recursive list
return ObjectMetadata{}, resources, nil resp.NextMarker = ""
}
case delimitedName == content.FileInfo.Name():
// Use resources.Prefix to filter out delimited files
metadata, err = getMetadata(fs.path, bucket, name)
if err != nil {
return ObjectMetadata{}, resources, err.Trace()
}
if metadata.Mode.IsDir() {
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter)
return ObjectMetadata{}, resources, nil
}
case delimitedName != "":
resources.CommonPrefixes = append(resources.CommonPrefixes, resources.Prefix+delimitedName)
}
}
// Delimiter present and Prefix is absent
case resources.Delimiter != "" && resources.Prefix == "":
delimitedName := delimiter(name, resources.Delimiter)
switch true {
case delimitedName == "":
metadata, err = getMetadata(fs.path, bucket, name)
if err != nil {
return ObjectMetadata{}, resources, err.Trace()
}
if metadata.Mode.IsDir() {
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter)
return ObjectMetadata{}, resources, nil
}
case delimitedName == content.FileInfo.Name():
metadata, err = getMetadata(fs.path, bucket, name)
if err != nil {
return ObjectMetadata{}, resources, err.Trace()
}
if metadata.Mode.IsDir() {
resources.CommonPrefixes = append(resources.CommonPrefixes, name+resources.Delimiter)
return ObjectMetadata{}, resources, nil
}
case delimitedName != "":
resources.CommonPrefixes = append(resources.CommonPrefixes, delimitedName)
}
// Delimiter is absent and only Prefix is present
case resources.Delimiter == "" && resources.Prefix != "":
if strings.HasPrefix(name, resources.Prefix) {
// Do not strip prefix object output
metadata, err = getMetadata(fs.path, bucket, name)
if err != nil {
return ObjectMetadata{}, resources, err.Trace()
}
}
default:
metadata, err = getMetadata(fs.path, bucket, name)
if err != nil {
return ObjectMetadata{}, resources, err.Trace()
}
} }
sortUnique(sort.StringSlice(resources.CommonPrefixes)) return resp, nil
return metadata, resources, nil
} }

27
pkg/fs/fs.go
Unescape View File

@ -27,12 +27,14 @@ import (
// Filesystem - local variables // Filesystem - local variables
type Filesystem struct { type Filesystem struct {
path string path string
minFreeDisk int64 minFreeDisk int64
maxBuckets int maxBuckets int
lock *sync.Mutex lock *sync.Mutex
multiparts *Multiparts multiparts *Multiparts
buckets *Buckets buckets *Buckets
listServiceReqCh chan<- listServiceReq
timeoutReqCh chan<- ListObjectsReq
} }
// Buckets holds acl information // Buckets holds acl information
@ -92,11 +94,10 @@ func New(rootPath string) (Filesystem, *probe.Error) {
return Filesystem{}, err.Trace() return Filesystem{}, err.Trace()
} }
} }
fs := Filesystem{lock: new(sync.Mutex)} a := Filesystem{lock: new(sync.Mutex)}
fs.path = rootPath a.path = rootPath
fs.multiparts = multiparts a.multiparts = multiparts
fs.buckets = buckets a.buckets = buckets
/// Defaults /// Defaults
// maximum buckets to be listed from list buckets. // maximum buckets to be listed from list buckets.
@ -104,6 +105,10 @@ func New(rootPath string) (Filesystem, *probe.Error) {
// minium free disk required for i/o operations to succeed. // minium free disk required for i/o operations to succeed.
fs.minFreeDisk = 10 fs.minFreeDisk = 10
err = fs.startListService()
if err != nil {
return Filesystem{}, err.Trace(rootPath)
}
// Return here. // Return here.
return fs, nil return fs, nil
} }

Write Preview
Loading…
Cancel
Save