metadata: Invalidate cache if unreadable and not updating (#10844)

If a scanning server shuts down unexpectedly we may have "successful" caches that are incomplete on a set.

In this case mark the cache with an error so it will no longer be handed out.

cmd/metacache-manager.go

@@ -233,9 +233,7 @@ func (o listPathOptions) checkMetacacheState(ctx context.Context, rpc *peerRESTC
 	// We operate on a copy...
 	o.Create = false
 	var cache metacache
-	if !o.Transient {
 	if rpc == nil || o.Transient {
-			// Local
 		cache = localMetacacheMgr.findCache(ctx, o)
 	} else {
 		c, err := rpc.GetMetacacheListing(ctx, o)
@@ -244,14 +242,22 @@ func (o listPathOptions) checkMetacacheState(ctx context.Context, rpc *peerRESTC
 		}
 		cache = *c
 	}
-	}
 
 	if cache.status == scanStateNone || cache.fileNotFound {
 		return errFileNotFound
 	}
 	if cache.status == scanStateSuccess || cache.status == scanStateStarted {
 		if time.Since(cache.lastUpdate) > metacacheMaxRunningAge {
-			return fmt.Errorf("timeout: list %s not updated for 1 minute", cache.id)
+			// We got a stale entry, mark error on handling server.
+			err := fmt.Errorf("timeout: list %s not updated", cache.id)
+			cache.error = err.Error()
+			cache.status = scanStateError
+			if rpc == nil || o.Transient {
+				localMetacacheMgr.updateCacheEntry(cache)
+			} else {
+				rpc.UpdateMetacacheListing(ctx, cache)
+			}
+			return err
 		}
 		return nil
 	}