implement configurable timeout for OBD tests (#9324)

master
Sidhartha Mani 5 years ago committed by GitHub
parent 37d066b563
commit ec11e99667
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 58
      cmd/admin-handlers.go
  2. 14
      cmd/admin-router.go
  3. 6
      pkg/disk/obd.go
  4. 5
      pkg/madmin/obd.go

@ -1173,22 +1173,23 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
if objectAPI == nil {
return
}
deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
obdInfo := madmin.OBDInfo{}
setCommonHeaders(w)
w.Header().Set(xhttp.ContentType, string(mimeJSON))
w.WriteHeader(http.StatusOK)
vars := mux.Vars(r)
pulse := make(chan struct{})
obdDone := make(chan struct{})
obdInfo := madmin.OBDInfo{}
enc := json.NewEncoder(w)
partialWrite := func() {
doPartialWrite := func() {
logger.LogIf(ctx, enc.Encode(obdInfo))
}
partialWrite := func() {
pulse <- struct{}{}
}
finish := func() {
partialWrite()
w.(http.Flusher).Flush()
cancel()
obdDone <- struct{}{}
}
errResp := func(err error) {
@ -1199,15 +1200,48 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
finish()
}
deadline := 3600 * time.Second
deadlineStr := r.URL.Query().Get("deadline")
if deadlineStr != "" {
var err error
deadline, err = time.ParseDuration(deadlineStr)
if err != nil {
errResp(err)
return
}
}
deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline))
setCommonHeaders(w)
w.Header().Set(xhttp.ContentType, string(mimeJSON))
w.WriteHeader(http.StatusOK)
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
go func() {
loop:
for {
select {
case <-ticker.C:
doPartialWrite()
case <-pulse:
doPartialWrite()
case <-obdDone:
break loop
}
}
w.(http.Flusher).Flush()
cancel()
}()
nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
if err := nsLock.GetLock(newDynamicTimeout(10*time.Minute, 600*time.Second)); err != nil { // returns a locked lock
if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
errResp(err)
return
}
defer nsLock.Unlock()
vars := mux.Vars(r)
if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)

@ -178,9 +178,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
if !globalIsGateway {
// -- OBD API --
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).Queries("perfdrive", "{perfdrive:true|false}", "perfnet", "{perfnet:true|false}", "minioinfo", "{minioinfo:true|false}", "minioconfig", "{minioconfig:true|false}", "syscpu", "{syscpu:true|false}", "sysdiskhw", "{sysdiskhw:true|false}", "sysosinfo", "{sysosinfo:true|false}", "sysmem", "{sysmem:true|false}", "sysprocess", "{sysprocess:true|false}")
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
Queries("perfdrive", "{perfdrive:true|false}",
"perfnet", "{perfnet:true|false}",
"minioinfo", "{minioinfo:true|false}",
"minioconfig", "{minioconfig:true|false}",
"syscpu", "{syscpu:true|false}",
"sysdiskhw", "{sysdiskhw:true|false}",
"sysosinfo", "{sysosinfo:true|false}",
"sysmem", "{sysmem:true|false}",
"sysprocess", "{sysprocess:true|false}",
)
}
}

@ -21,7 +21,6 @@ import (
"context"
"fmt"
"os"
"runtime"
"time"
"github.com/dustin/go-humanize"
@ -52,8 +51,7 @@ type Throughput struct {
}
// GetOBDInfo about the drive
func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Throughput, error) {
runtime.LockOSThread()
func GetOBDInfo(ctx context.Context, drive, fsPath string) (Latency, Throughput, error) {
// Create a file with O_DIRECT flag, choose default umask and also make sure
// we are exclusively writing to a new file using O_EXCL.
@ -96,8 +94,6 @@ func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Thro
latencies[i] = float64(latencyInSecs)
}
runtime.UnlockOSThread()
for i := range latencies {
throughput := float64(blockSize) / latencies[i]
throughputs[i] = throughput

@ -221,11 +221,14 @@ var OBDDataTypesList = []OBDDataType{
// ServerOBDInfo - Connect to a minio server and call OBD Info Management API
// to fetch server's information represented by OBDInfo structure
func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType) <-chan OBDInfo {
func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType, deadline time.Duration) <-chan OBDInfo {
respChan := make(chan OBDInfo)
go func() {
v := url.Values{}
v.Set("deadline",
deadline.Truncate(1*time.Second).String())
// start with all set to false
for _, d := range OBDDataTypesList {
v.Set(string(d), "false")

Loading…
Cancel
Save