cleanup tmpDir any older entries automatically just like multipart (#10439)

also consider multipart uploads, temporary files in `.minio.sys/tmp`
as stale beyond 24hrs and clean them up automatically

cmd/erasure-multipart.go

@@ -71,7 +71,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 
@@ -91,13 +91,13 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean
 			if disk == nil {
 				continue
 			}
-			er.cleanupStaleMultipartUploadsOnDisk(ctx, disk, expiry)
+			er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
 		}
 	}
 }
 
 // Remove the old multipart uploads on the given disk.
-func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
 	now := time.Now()
 	shaDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, "", -1)
 	if err != nil {
@@ -119,6 +119,19 @@ func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context,
 			}
 		}
 	}
+	tmpDirs, err := disk.ListDir(ctx, minioMetaTmpBucket, "", -1)
+	if err != nil {
+		return
+	}
+	for _, tmpDir := range tmpDirs {
+		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "")
+		if err != nil {
+			continue
+		}
+		if now.Sub(fi.ModTime) > expiry {
+			er.deleteObject(ctx, minioMetaTmpBucket, tmpDir, fi.Erasure.DataBlocks+1)
+		}
+	}
 }
 
 // ListMultipartUploads - lists all the pending multipart

cmd/erasure-sets.go

@@ -369,8 +369,8 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			mrfOpCh:      make(chan partialOperation, 10000),
 		}
 
-		go s.sets[i].cleanupStaleMultipartUploads(ctx,
+		go s.sets[i].cleanupStaleUploads(ctx,
-			GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
+			GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	}
 
 	// Start the disk monitoring and connect routine.

cmd/fs-v1-multipart.go

@@ -824,7 +824,7 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/fs-v1-multipart_test.go

@@ -51,7 +51,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	cleanupWg.Add(1)
 	go func() {
 		defer cleanupWg.Done()
-		fs.cleanupStaleMultipartUploads(ctx, time.Millisecond, 0)
+		fs.cleanupStaleUploads(ctx, time.Millisecond, 0)
 	}()
 
 	// Wait for 100ms such that - we have given enough time for

cmd/fs-v1.go

@@ -178,7 +178,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
+	go fs.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	go intDataUpdateTracker.start(ctx, fsPath)
 
 	// Return successfully initialized object layer.

cmd/gateway/azure/gateway-azure.go

@@ -473,6 +473,11 @@ func azureCodesToObjectError(err error, serviceCode string, statusCode int, buck
 		err = minio.UnsupportedMetadata{}
 	case "BlobAccessTierNotSupportedForAccountType":
 		err = minio.NotImplemented{}
+	case "OutOfRangeInput":
+		err = minio.ObjectNameInvalid{
+			Bucket: bucket,
+			Object: object,
+		}
 	default:
 		switch statusCode {
 		case http.StatusNotFound:

cmd/gateway/s3/gateway-s3-sse.go

@@ -701,13 +701,13 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, clea
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			l.cleanupStaleEncMultipartUploadsOnGW(ctx, expiry)
+			l.cleanupStaleUploads(ctx, expiry)
 		}
 	}
 }
 
-// cleanupStaleMultipartUploads removes old custom encryption multipart uploads on backend
+// cleanupStaleUploads removes old custom encryption multipart uploads on backend
-func (l *s3EncObjects) cleanupStaleEncMultipartUploadsOnGW(ctx context.Context, expiry time.Duration) {
+func (l *s3EncObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
 	for {
 		buckets, err := l.s3Objects.ListBuckets(ctx)
 		if err != nil {

cmd/gateway/s3/gateway-s3.go

@@ -242,7 +242,7 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 
 		// Start stale enc multipart uploads cleanup routine.
 		go encS.cleanupStaleEncMultipartUploads(minio.GlobalContext,
-			minio.GlobalMultipartCleanupInterval, minio.GlobalMultipartExpiry)
+			minio.GlobalStaleUploadsCleanupInterval, minio.GlobalStaleUploadsExpiry)
 
 		return &encS, nil
 	}

cmd/globals.go

@@ -79,10 +79,10 @@ const (
 	// date and server date during signature verification.
 	globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
 
-	// GlobalMultipartExpiry - Expiry duration after which the multipart uploads are deemed stale.
+	// GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart, tmp directory are deemed stale.
-	GlobalMultipartExpiry = time.Hour * 24 * 3 // 3 days.
+	GlobalStaleUploadsExpiry = time.Hour * 24 // 24 hrs.
-	// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
+	// GlobalStaleUploadsCleanupInterval - Cleanup interval when the stale uploads cleanup is initiated.
-	GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
+	GlobalStaleUploadsCleanupInterval = time.Hour * 24 // 24 hrs.
 
 	// GlobalServiceExecutionInterval - Executes the Lifecycle events.
 	GlobalServiceExecutionInterval = time.Hour * 24 // 24 hrs.