From 4a36cd70358914e7d8dd0bee929ec95952c539c2 Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Thu, 17 Sep 2020 18:51:16 -0700
Subject: [PATCH] fix: improve performance ListObjectParts in FS mode (#10510)

from 20s for 10000 parts to less than 1sec

Without the patch
```
~ time aws --endpoint-url=http://localhost:9000 --profile minio s3api \
       list-parts --bucket testbucket --key test \
       --upload-id c1cd1f50-ea9a-4824-881c-63b5de95315a

real    0m20.394s
user    0m0.589s
sys     0m0.174s
```

With the patch
```
~ time aws --endpoint-url=http://localhost:9000 --profile minio s3api \
       list-parts --bucket testbucket --key test \
       --upload-id c1cd1f50-ea9a-4824-881c-63b5de95315a

real    0m0.891s
user    0m0.624s
sys     0m0.182s
```

fixes #10503
---
 cmd/fs-v1-multipart.go | 71 ++++++++++++++++++++----------------------
 1 file changed, 34 insertions(+), 37 deletions(-)

diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go
index 110ce4617..dbe5afb9e 100644
--- a/cmd/fs-v1-multipart.go
+++ b/cmd/fs-v1-multipart.go
@@ -446,52 +446,51 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		return result, toObjectErr(err, bucket)
 	}
 
-	partsMap := make(map[int]string)
+	partsMap := make(map[int]PartInfo)
 	for _, entry := range entries {
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag1, _, derr := fs.decodePartFile(entry)
+
+		partNumber, currentEtag, actualSize, derr := fs.decodePartFile(entry)
 		if derr != nil {
 			// Skip part files whose name don't match expected format. These could be backend filesystem specific files.
 			continue
 		}
-		etag2, ok := partsMap[partNumber]
-		if !ok {
-			partsMap[partNumber] = etag1
+
+		entryStat, err := fsStatFile(ctx, pathJoin(uploadIDDir, entry))
+		if err != nil {
 			continue
 		}
-		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag1)))
-		if serr != nil {
-			return result, toObjectErr(serr)
+
+		currentMeta := PartInfo{
+			PartNumber:   partNumber,
+			ETag:         currentEtag,
+			ActualSize:   actualSize,
+			Size:         entryStat.Size(),
+			LastModified: entryStat.ModTime(),
 		}
-		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag2)))
-		if serr != nil {
-			return result, toObjectErr(serr)
+
+		cachedMeta, ok := partsMap[partNumber]
+		if !ok {
+			partsMap[partNumber] = currentMeta
+			continue
 		}
-		if stat1.ModTime().After(stat2.ModTime()) {
-			partsMap[partNumber] = etag1
+
+		if currentMeta.LastModified.After(cachedMeta.LastModified) {
+			partsMap[partNumber] = currentMeta
 		}
 	}
 
 	var parts []PartInfo
-	var actualSize int64
-	for partNumber, etag := range partsMap {
-		partFile := getPartFile(entries, partNumber, etag)
-		if partFile == "" {
-			return result, InvalidPart{}
-		}
-		// Read the actualSize from the pathFileName.
-		subParts := strings.Split(partFile, ".")
-		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
-		if err != nil {
-			return result, InvalidPart{}
-		}
-		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
+	for _, partInfo := range partsMap {
+		parts = append(parts, partInfo)
 	}
+
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
 	})
+
 	i := 0
 	if partNumberMarker != 0 {
 		// If the marker was set, skip the entries till the marker.
@@ -515,21 +514,19 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 			result.NextPartNumberMarker = result.Parts[partsCount-1].PartNumber
 		}
 	}
-	for i, part := range result.Parts {
-		var stat os.FileInfo
-		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir,
-			fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
-		if err != nil {
-			return result, toObjectErr(err)
+
+	rc, _, err := fsOpenFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile), 0)
+	if err != nil {
+		if err == errFileNotFound || err == errFileAccessDenied {
+			return result, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
 		}
-		result.Parts[i].LastModified = stat.ModTime()
-		result.Parts[i].Size = part.ActualSize
+		return result, toObjectErr(err, bucket, object)
 	}
+	defer rc.Close()
 
-	fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
+	fsMetaBytes, err := ioutil.ReadAll(rc)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return result, err
+		return result, toObjectErr(err, bucket, object)
 	}
 
 	var fsMeta fsMetaV1