Metacache add abs entry limit (#11483)

Add an absolute limit to the number of metacaches for a bucket.

Delete excess caches if they haven't been handed out in an hour.

cmd/metacache-bucket.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"path"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -344,6 +345,29 @@ func (b *bucketMetacache) cleanup() {
 		}
 	}
 
+	// If above limit, remove the caches with the oldest handout time.
+	if len(caches)-len(remove) > metacacheMaxEntries {
+		remainCaches := make([]metacache, 0, len(caches)-len(remove))
+		for id, cache := range caches {
+			if _, ok := remove[id]; ok {
+				continue
+			}
+			remainCaches = append(remainCaches, cache)
+		}
+		if len(remainCaches) > metacacheMaxEntries {
+			// Sort oldest last...
+			sort.Slice(remainCaches, func(i, j int) bool {
+				return remainCaches[i].lastHandout.Before(remainCaches[j].lastHandout)
+			})
+			// Keep first metacacheMaxEntries...
+			for _, cache := range remainCaches[metacacheMaxEntries:] {
+				if time.Since(cache.lastHandout) > time.Hour {
+					remove[cache.id] = struct{}{}
+				}
+			}
+		}
+	}
+
 	for id := range remove {
 		b.deleteCache(id)
 	}

cmd/metacache-manager.go

@@ -43,6 +43,7 @@ type metacacheManager struct {
 }
 
 const metacacheManagerTransientBucket = "**transient**"
+const metacacheMaxEntries = 5000
 
 // initManager will start async saving the cache.
 func (m *metacacheManager) initManager() {