tree-walk: unify FS and XL tree-walk with functional approach. (#2027)
Krishna Srinivas
9 years ago
committed by
Harshavardhana
parent
a8a3e95835
commit
7a8b8cd0a1
@ -1,147 +0,0 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"path" |
||||
"sort" |
||||
"strings" |
||||
) |
||||
|
||||
// treeWalk walks FS directory tree recursively pushing fileInfo into the channel as and when it encounters files.
|
||||
func (fs fsObjects) treeWalk(bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, isLeaf func(string, string) bool, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error { |
||||
// Example:
|
||||
// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
|
||||
// called with prefixDir="one/two/three/four/" and marker="five.txt"
|
||||
|
||||
var markerBase, markerDir string |
||||
if marker != "" { |
||||
// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
|
||||
markerSplit := strings.SplitN(marker, slashSeparator, 2) |
||||
markerDir = markerSplit[0] |
||||
if len(markerSplit) == 2 { |
||||
markerDir += slashSeparator |
||||
markerBase = markerSplit[1] |
||||
} |
||||
} |
||||
entries, err := fs.storage.ListDir(bucket, prefixDir) |
||||
if err != nil { |
||||
select { |
||||
case <-endWalkCh: |
||||
return errWalkAbort |
||||
case resultCh <- treeWalkResult{err: err}: |
||||
return err |
||||
} |
||||
} |
||||
|
||||
for i, entry := range entries { |
||||
if entryPrefixMatch != "" { |
||||
if !strings.HasPrefix(entry, entryPrefixMatch) { |
||||
entries[i] = "" |
||||
continue |
||||
} |
||||
} |
||||
if isLeaf(bucket, pathJoin(prefixDir, entry)) { |
||||
entries[i] = strings.TrimSuffix(entry, slashSeparator) |
||||
} |
||||
} |
||||
sort.Strings(entries) |
||||
// Skip the empty strings
|
||||
for len(entries) > 0 && entries[0] == "" { |
||||
entries = entries[1:] |
||||
} |
||||
if len(entries) == 0 { |
||||
return nil |
||||
} |
||||
// example:
|
||||
// If markerDir="four/" Search() returns the index of "four/" in the sorted
|
||||
// entries list so we skip all the entries till "four/"
|
||||
idx := sort.Search(len(entries), func(i int) bool { |
||||
return entries[i] >= markerDir |
||||
}) |
||||
entries = entries[idx:] |
||||
for i, entry := range entries { |
||||
if i == 0 && markerDir == entry { |
||||
if !recursive { |
||||
// Skip as the marker would already be listed in the previous listing.
|
||||
continue |
||||
} |
||||
if recursive && !strings.HasSuffix(entry, slashSeparator) { |
||||
// We should not skip for recursive listing and if markerDir is a directory
|
||||
// for ex. if marker is "four/five.txt" markerDir will be "four/" which
|
||||
// should not be skipped, instead it will need to be treeWalk()'ed into.
|
||||
|
||||
// Skip if it is a file though as it would be listed in previous listing.
|
||||
continue |
||||
} |
||||
} |
||||
|
||||
if recursive && strings.HasSuffix(entry, slashSeparator) { |
||||
// If the entry is a directory, we will need recurse into it.
|
||||
markerArg := "" |
||||
if entry == markerDir { |
||||
// We need to pass "five.txt" as marker only if we are
|
||||
// recursing into "four/"
|
||||
markerArg = markerBase |
||||
} |
||||
prefixMatch := "" // Valid only for first level treeWalk and empty for subdirectories.
|
||||
markIsEnd := i == len(entries)-1 && isEnd |
||||
if tErr := fs.treeWalk(bucket, path.Join(prefixDir, entry), prefixMatch, markerArg, recursive, isLeaf, resultCh, endWalkCh, markIsEnd); tErr != nil { |
||||
return tErr |
||||
} |
||||
continue |
||||
} |
||||
// EOF is set if we are at last entry and the caller indicated we at the end.
|
||||
isEOF := ((i == len(entries)-1) && isEnd) |
||||
select { |
||||
case <-endWalkCh: |
||||
return errWalkAbort |
||||
case resultCh <- treeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}: |
||||
} |
||||
} |
||||
// Everything is listed
|
||||
return nil |
||||
} |
||||
|
||||
// Initiate a new treeWalk in a goroutine.
|
||||
func (fs fsObjects) startTreeWalk(bucket, prefix, marker string, recursive bool, isLeaf func(string, string) bool, endWalkCh chan struct{}) chan treeWalkResult { |
||||
// Example 1
|
||||
// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
|
||||
// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
|
||||
// and entryPrefixMatch=""
|
||||
|
||||
// Example 2
|
||||
// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
|
||||
// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
|
||||
// and entryPrefixMatch="th"
|
||||
|
||||
resultCh := make(chan treeWalkResult, maxObjectList) |
||||
entryPrefixMatch := prefix |
||||
prefixDir := "" |
||||
lastIndex := strings.LastIndex(prefix, slashSeparator) |
||||
if lastIndex != -1 { |
||||
entryPrefixMatch = prefix[lastIndex+1:] |
||||
prefixDir = prefix[:lastIndex+1] |
||||
} |
||||
marker = strings.TrimPrefix(marker, prefixDir) |
||||
go func() { |
||||
isEnd := true // Indication to start walking the tree with end as true.
|
||||
fs.treeWalk(bucket, prefixDir, entryPrefixMatch, marker, recursive, isLeaf, resultCh, endWalkCh, isEnd) |
||||
close(resultCh) |
||||
}() |
||||
return resultCh |
||||
} |
||||
Loading…
Reference in new issue