Improve ListObjects performance by listing in parallel (#7270)

The side affect of this change memory
increase, but this is a trade-off between
performance and actual memory usage.

For all practical scenarios this should be
an adequate change.

cmd/server_test.go

@@ -1693,7 +1693,7 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
 		{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
 	}
 
-	for i, testCase := range testCases {
+	for _, testCase := range testCases {
 		// create listObjectsV1 request with valid parameters
 		request, err = newTestSignedRequest("GET", testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer)
 		c.Assert(err, nil)
@@ -1706,7 +1706,6 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
 		getContent, err := ioutil.ReadAll(response.Body)
 		c.Assert(err, nil)
 
-		fmt.Printf("Test %d: %+v vs %+v\n", i+1, string(getContent), testCase.expectedStrings)
 		for _, expectedStr := range testCase.expectedStrings {
 			c.Assert(strings.Contains(string(getContent), expectedStr), true)
 		}

cmd/xl-sets.go

@@ -641,22 +641,27 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
-func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...[]StorageAPI) listDirFunc {
+func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...*xlObjects) listDirFunc {
 	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
-		for _, disk := range disks {
+		var diskEntries = make([][]string, len(disks))
+		var wg sync.WaitGroup
+		for index, disk := range disks {
 			if disk == nil {
 				continue
 			}
-
+			wg.Add(1)
-			var entries []string
+			go func(index int, disk StorageAPI) {
-			var newEntries []string
+				defer wg.Done()
-			var err error
+				diskEntries[index], _ = disk.ListDir(bucket, prefixDir, -1)
-			entries, err = disk.ListDir(bucket, prefixDir, -1)
+			}(index, disk)
-			if err != nil {
-				continue
 		}
 
+		wg.Wait()
+
 		// Find elements in entries which are not in mergedEntries
+		for _, entries := range diskEntries {
+			var newEntries []string
+
 			for _, entry := range entries {
 				idx := sort.SearchStrings(mergedEntries, entry)
 				// if entry is already present in mergedEntries don't add.
@@ -672,16 +677,16 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
+
 		return mergedEntries
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
 	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
-		for _, disks := range sets {
+		for _, set := range sets {
-			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
-			for _, entry := range entries {
+			for _, entry := range listDirInternal(bucket, prefixDir, prefixEntry, set.getLoadBalancedDisks()) {
 				idx := sort.SearchStrings(mergedEntries, entry)
 				// if entry is already present in mergedEntries don't add.
 				if idx < len(mergedEntries) && mergedEntries[idx] == entry {
@@ -696,8 +701,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
-		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
+		return filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf
 	}
 	return listDir
 }
@@ -743,12 +747,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 			return false
 		}
 
-		var setDisks = make([][]StorageAPI, len(s.sets))
+		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
-		for _, set := range s.sets {
-			setDisks = append(setDisks, set.getLoadBalancedDisks())
-		}
-
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
@@ -1353,12 +1352,7 @@ func (s *xlSets) listObjectsHeal(ctx context.Context, bucket, prefix, marker, de
 			return false
 		}
 
-		var setDisks = make([][]StorageAPI, len(s.sets))
+		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
-		for _, set := range s.sets {
-			setDisks = append(setDisks, set.getLoadBalancedDisks())
-		}
-
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}