From fdf65aa9b92c9cf719c19e8b55e0b9a1147bf3fb Mon Sep 17 00:00:00 2001
From: Anis Elleuch <vadmeste@users.noreply.github.com>
Date: Wed, 11 Mar 2020 23:00:31 -0700
Subject: [PATCH] heal: Add info about the next background healing round
 (#9122)

- avoid setting last heal activity when starting self-healing

This can be confusing to users thinking that the self healing
cycle was already performed.

- add info about the next background healing round
---
 cmd/admin-handlers.go       | 17 ++++++++++++++---
 cmd/admin-heal-ops.go       |  2 --
 cmd/global-heal.go          | 20 +++++++++++++++-----
 pkg/madmin/heal-commands.go |  1 +
 4 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go
index 7ec55ca8e..1012a632d 100644
--- a/cmd/admin-handlers.go
+++ b/cmd/admin-handlers.go
@@ -982,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go
index e33b4655a..97acbca27 100644
--- a/cmd/admin-heal-ops.go
+++ b/cmd/admin-heal-ops.go
@@ -575,8 +575,6 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
 	bucketsOnly := true // heal buckets only, not objects.
 	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
diff --git a/cmd/global-heal.go b/cmd/global-heal.go
index 52290abd1..7af88d897 100644
--- a/cmd/global-heal.go
+++ b/cmd/global-heal.go
@@ -29,7 +29,6 @@ import (
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
 	leaderTick    = time.Hour
-	healTick      = time.Hour
 	healInterval  = 30 * 24 * time.Hour
 )
 
@@ -75,6 +74,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.scannedItemsCount,
 		LastHealActivity:  bgSeq.lastHealActivity,
+		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
 	}
 }
 
@@ -112,6 +112,19 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	return nil
 }
 
+// Returns the duration to the next background healing round
+func durationToNextHealRound(lastHeal time.Time) time.Duration {
+	if lastHeal.IsZero() {
+		lastHeal = globalBootTime
+	}
+
+	d := lastHeal.Add(healInterval).Sub(UTCNow())
+	if d < 0 {
+		return time.Second
+	}
+	return d
+}
+
 // Healing leader will take the charge of healing all erasure sets
 func execLeaderTasks(z *xlZones) {
 	ctx := context.Background()
@@ -132,10 +145,7 @@ func execLeaderTasks(z *xlZones) {
 
 	lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
 	for {
-		if time.Since(lastScanTime) < healInterval {
-			time.Sleep(healTick)
-			continue
-		}
+		time.Sleep(durationToNextHealRound(lastScanTime))
 		for _, zone := range z.zones {
 			// Heal set by set
 			for i, set := range zone.sets {
diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go
index b3c2c59a8..0b9bdf554 100644
--- a/pkg/madmin/heal-commands.go
+++ b/pkg/madmin/heal-commands.go
@@ -274,6 +274,7 @@ func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts,
 type BgHealState struct {
 	ScannedItemsCount int64
 	LastHealActivity  time.Time
+	NextHealRound     time.Time
 }
 
 // BackgroundHealStatus returns the background heal status of the