tests: Add tests to treeWalk for sortedness, recursive listing and isEnd behaviour (#2209)

tree-walk_test.go

@@ -19,20 +19,14 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
+	"sort"
 	"strings"
 	"testing"
 	"time"
 )
 
-// Sample entries for the namespace.
-var volume = "testvolume"
-var files = []string{
-	"d/e",
-	"d/f",
-	"d/g/h",
-	"i/j/k",
-	"lmn",
-}
+// Fixed volume name that could be used across tests
+const volume = "testvolume"
 
 // Helper function that creates a volume and files in it.
 func createNamespace(disk StorageAPI, volume string, files []string) error {
@@ -97,6 +91,13 @@ func TestTreeWalk(t *testing.T) {
 		t.Errorf("Unable to create StorageAPI: %s", err)
 	}
 
+	var files = []string{
+		"d/e",
+		"d/f",
+		"d/g/h",
+		"i/j/k",
+		"lmn",
+	}
 	err = createNamespace(disk, volume, files)
 	if err != nil {
 		t.Fatal(err)
@@ -125,12 +126,12 @@ func TestTreeWalkTimeout(t *testing.T) {
 	if err != nil {
 		t.Errorf("Unable to create StorageAPI: %s", err)
 	}
-	var files []string
+	var myfiles []string
 	// Create maxObjectsList+1 number of entries.
 	for i := 0; i < maxObjectList+1; i++ {
-		files = append(files, fmt.Sprintf("file.%d", i))
+		myfiles = append(myfiles, fmt.Sprintf("file.%d", i))
 	}
-	err = createNamespace(disk, volume, files)
+	err = createNamespace(disk, volume, myfiles)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -259,3 +260,256 @@ func TestListDir(t *testing.T) {
 		t.Error("expected errDiskNotFound error.")
 	}
 }
+
+// TestRecursiveWalk - tests if treeWalk returns entries correctly with and
+// without recursively traversing prefixes.
+func TestRecursiveTreeWalk(t *testing.T) {
+	// Create a backend directories fsDir1.
+	fsDir1, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Errorf("Unable to create tmp directory: %s", err)
+	}
+
+	// Create two StorageAPIs disk1.
+	disk1, err := newStorageAPI(fsDir1)
+	if err != nil {
+		t.Errorf("Unable to create StorageAPI: %s", err)
+	}
+
+	// Create listDir function.
+	listDir := listDirFactory(func(volume, prefix string) bool {
+		return !strings.HasSuffix(prefix, slashSeparator)
+	}, disk1)
+
+	// Create the namespace.
+	var files = []string{
+		"d/e",
+		"d/f",
+		"d/g/h",
+		"i/j/k",
+		"lmn",
+	}
+	err = createNamespace(disk1, volume, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	endWalkCh := make(chan struct{})
+	testCases := []struct {
+		prefix    string
+		marker    string
+		recursive bool
+		expected  map[string]struct{}
+	}{
+		// with no prefix, no marker and no recursive traversal
+		{"", "", false, map[string]struct{}{
+			"d/":  {},
+			"i/":  {},
+			"lmn": {},
+		}},
+		// with no prefix, no marker and recursive traversal
+		{"", "", true, map[string]struct{}{
+			"d/f":   {},
+			"d/g/h": {},
+			"d/e":   {},
+			"i/j/k": {},
+			"lmn":   {},
+		}},
+		// with no prefix, marker and no recursive traversal
+		{"", "d/e", false, map[string]struct{}{
+			"d/f":  {},
+			"d/g/": {},
+			"i/":   {},
+			"lmn":  {},
+		}},
+		// with no prefix, marker and recursive traversal
+		{"", "d/e", true, map[string]struct{}{
+			"d/f":   {},
+			"d/g/h": {},
+			"i/j/k": {},
+			"lmn":   {},
+		}},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", false, map[string]struct{}{
+			"d/e":  {},
+			"d/f":  {},
+			"d/g/": {},
+		}},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", true, map[string]struct{}{
+			"d/e":   {},
+			"d/f":   {},
+			"d/g/h": {},
+		}},
+		// with prefix, marker and no recursive traversal
+		{"d/", "d/e", false, map[string]struct{}{
+			"d/f":  {},
+			"d/g/": {},
+		}},
+		// with prefix, marker and recursive traversal
+		{"d/", "d/e", true, map[string]struct{}{
+			"d/f":   {},
+			"d/g/h": {},
+		}},
+	}
+	for i, testCase := range testCases {
+		for entry := range startTreeWalk(volume,
+			testCase.prefix, testCase.marker, testCase.recursive,
+			listDir, endWalkCh) {
+			if _, found := testCase.expected[entry.entry]; !found {
+				t.Errorf("Test %d: Expected %s, but couldn't find", i+1, entry.entry)
+			}
+		}
+	}
+	err = removeAll(fsDir1)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestSortedness(t *testing.T) {
+	// Create a backend directories fsDir1.
+	fsDir1, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Errorf("Unable to create tmp directory: %s", err)
+	}
+
+	// Create two StorageAPIs disk1.
+	disk1, err := newStorageAPI(fsDir1)
+	if err != nil {
+		t.Errorf("Unable to create StorageAPI: %s", err)
+	}
+
+	// Create listDir function.
+	listDir := listDirFactory(func(volume, prefix string) bool {
+		return !strings.HasSuffix(prefix, slashSeparator)
+	}, disk1)
+
+	// Create the namespace.
+	var files = []string{
+		"d/e",
+		"d/f",
+		"d/g/h",
+		"i/j/k",
+		"lmn",
+	}
+	err = createNamespace(disk1, volume, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	endWalkCh := make(chan struct{})
+	testCases := []struct {
+		prefix    string
+		marker    string
+		recursive bool
+	}{
+		// with no prefix, no marker and no recursive traversal
+		{"", "", false},
+		// with no prefix, no marker and recursive traversal
+		{"", "", true},
+		// with no prefix, marker and no recursive traversal
+		{"", "d/e", false},
+		// with no prefix, marker and recursive traversal
+		{"", "d/e", true},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", false},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", true},
+		// with prefix, marker and no recursive traversal
+		{"d/", "d/e", false},
+		// with prefix, marker and recursive traversal
+		{"d/", "d/e", true},
+	}
+	for i, test := range testCases {
+		var actualEntries []string
+		for entry := range startTreeWalk(volume,
+			test.prefix, test.marker, test.recursive,
+			listDir, endWalkCh) {
+			actualEntries = append(actualEntries, entry.entry)
+		}
+		if !sort.IsSorted(sort.StringSlice(actualEntries)) {
+			t.Error(i+1, "Expected entries to be sort, but it wasn't")
+		}
+	}
+
+	// Remove directory created for testing
+	err = removeAll(fsDir1)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTreeWalkIsEnd(t *testing.T) {
+	// Create a backend directories fsDir1.
+	fsDir1, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Errorf("Unable to create tmp directory: %s", err)
+	}
+
+	// Create two StorageAPIs disk1.
+	disk1, err := newStorageAPI(fsDir1)
+	if err != nil {
+		t.Errorf("Unable to create StorageAPI: %s", err)
+	}
+
+	// Create listDir function.
+	listDir := listDirFactory(func(volume, prefix string) bool {
+		return !strings.HasSuffix(prefix, slashSeparator)
+	}, disk1)
+
+	// Create the namespace.
+	var files = []string{
+		"d/e",
+		"d/f",
+		"d/g/h",
+		"i/j/k",
+		"lmn",
+	}
+	err = createNamespace(disk1, volume, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	endWalkCh := make(chan struct{})
+	testCases := []struct {
+		prefix        string
+		marker        string
+		recursive     bool
+		expectedEntry string
+	}{
+		// with no prefix, no marker and no recursive traversal
+		{"", "", false, "lmn"},
+		// with no prefix, no marker and recursive traversal
+		{"", "", true, "lmn"},
+		// with no prefix, marker and no recursive traversal
+		{"", "d/e", false, "lmn"},
+		// with no prefix, marker and recursive traversal
+		{"", "d/e", true, "lmn"},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", false, "d/g/"},
+		// with prefix, no marker and no recursive traversal
+		{"d/", "", true, "d/g/h"},
+		// with prefix, marker and no recursive traversal
+		{"d/", "d/e", false, "d/g/"},
+		// with prefix, marker and recursive traversal
+		{"d/", "d/e", true, "d/g/h"},
+	}
+	for i, test := range testCases {
+		var entry treeWalkResult
+		for entry = range startTreeWalk(volume, test.prefix, test.marker, test.recursive, listDir, endWalkCh) {
+		}
+		if entry.entry != test.expectedEntry {
+			t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry)
+		}
+		if !entry.end {
+			t.Errorf("Test %d: Last entry %s, doesn't have EOF marker set", i, entry.entry)
+		}
+	}
+
+	// Remove directory created for testing
+	err = removeAll(fsDir1)
+	if err != nil {
+		t.Error(err)
+	}
+}