fix: reduce the load on CPU when loading users/policies (#8984)

Trying to be conservative by slowing ourselves down
on a regular basis.
master
Harshavardhana 5 years ago committed by GitHub
parent 64ec17b463
commit 9ecd66007f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 24
      cmd/background-heal-ops.go
  2. 18
      cmd/iam-object-store.go
  3. 11
      cmd/xl-sets.go
  4. 12
      cmd/xl-zones.go

@ -52,6 +52,19 @@ func (h *healRoutine) queueHealTask(task healTask) {
h.tasks <- task
}
func waitForLowHTTPReq(tolerance int32) {
if httpServer := newHTTPServerFn(); httpServer != nil {
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (httpServer.GetRequestCount() >= tolerance) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)
}
}
}
// Wait for heal requests and process them
func (h *healRoutine) run() {
ctx := context.Background()
@ -62,16 +75,7 @@ func (h *healRoutine) run() {
break
}
if httpServer := newHTTPServerFn(); httpServer != nil {
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)
}
}
waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
var res madmin.HealResultItem
var err error

@ -579,6 +579,8 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
}
go func() {
defer close(ch)
marker := ""
for {
lo, err := objectAPI.ListObjects(context.Background(),
@ -588,9 +590,13 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
case ch <- itemOrErr{Err: err}:
case <-doneCh:
}
close(ch)
return
}
// Slow down listing and loading for config items to
// reduce load on the server
waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
marker = lo.NextMarker
lister := dirList(lo)
if !dirs {
@ -602,12 +608,10 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
select {
case ch <- itemOrErr{Item: item}:
case <-doneCh:
close(ch)
return
}
}
if !lo.IsTruncated {
close(ch)
return
}
}
@ -617,17 +621,17 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
func (iamOS *IAMObjectStore) watch(sys *IAMSys) {
watchDisk := func() {
ticker := time.NewTicker(globalRefreshIAMInterval)
defer ticker.Stop()
for {
select {
case <-GlobalServiceDoneCh:
return
case <-ticker.C:
iamOS.loadAll(sys, nil)
case <-time.NewTimer(globalRefreshIAMInterval).C:
err := iamOS.loadAll(sys, nil)
logger.LogIf(context.Background(), err)
}
}
}
// Refresh IAMSys in background.
go watchDisk()
}

@ -1664,16 +1664,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
continue
}
if httpServer := newHTTPServerFn(); httpServer != nil {
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (httpServer.GetRequestCount() >= int32(s.drivesPerSet)) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)
}
}
waitForLowHTTPReq(int32(s.drivesPerSet))
if err := healObject(bucket, entry.Name); err != nil {
return toObjectErr(err, bucket, entry.Name)

@ -24,7 +24,6 @@ import (
"net/http"
"strings"
"sync"
"time"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
@ -1364,16 +1363,7 @@ func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healOb
continue
}
if httpServer := newHTTPServerFn(); httpServer != nil {
// Wait at max 10 minute for an inprogress request before proceeding to heal
waitCount := 600
// Any requests in progress, delay the heal.
for (httpServer.GetRequestCount() >= int32(zoneDrivesPerSet[zoneIndex])) &&
waitCount > 0 {
waitCount--
time.Sleep(1 * time.Second)
}
}
waitForLowHTTPReq(int32(zoneDrivesPerSet[zoneIndex]))
if err := healObject(bucket, entry.Name); err != nil {
return toObjectErr(err, bucket, entry.Name)

Loading…
Cancel
Save