From ec11e99667e941875c89c6468fb9e37ce17816f6 Mon Sep 17 00:00:00 2001 From: Sidhartha Mani Date: Tue, 14 Apr 2020 11:48:32 -0700 Subject: [PATCH] implement configurable timeout for OBD tests (#9324) --- cmd/admin-handlers.go | 58 ++++++++++++++++++++++++++++++++++--------- cmd/admin-router.go | 14 +++++++++-- pkg/disk/obd.go | 6 +---- pkg/madmin/obd.go | 5 +++- 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 32a32ebf1..a297e1aff 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1173,22 +1173,23 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) if objectAPI == nil { return } - deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute)) - obdInfo := madmin.OBDInfo{} - setCommonHeaders(w) - w.Header().Set(xhttp.ContentType, string(mimeJSON)) - w.WriteHeader(http.StatusOK) + vars := mux.Vars(r) + pulse := make(chan struct{}) + obdDone := make(chan struct{}) + obdInfo := madmin.OBDInfo{} enc := json.NewEncoder(w) - partialWrite := func() { + doPartialWrite := func() { logger.LogIf(ctx, enc.Encode(obdInfo)) } + partialWrite := func() { + pulse <- struct{}{} + } + finish := func() { - partialWrite() - w.(http.Flusher).Flush() - cancel() + obdDone <- struct{}{} } errResp := func(err error) { @@ -1199,15 +1200,48 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) finish() } + deadline := 3600 * time.Second + deadlineStr := r.URL.Query().Get("deadline") + if deadlineStr != "" { + var err error + deadline, err = time.ParseDuration(deadlineStr) + if err != nil { + errResp(err) + return + } + } + deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline)) + + setCommonHeaders(w) + w.Header().Set(xhttp.ContentType, string(mimeJSON)) + w.WriteHeader(http.StatusOK) + + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + go func() { + loop: + for { + select { + case <-ticker.C: + doPartialWrite() + case <-pulse: + doPartialWrite() + case <-obdDone: + break loop + } + } + w.(http.Flusher).Flush() + cancel() + }() + nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress") - if err := nsLock.GetLock(newDynamicTimeout(10*time.Minute, 600*time.Second)); err != nil { // returns a locked lock + if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock errResp(err) return } defer nsLock.Unlock() - vars := mux.Vars(r) - if cpu, ok := vars["syscpu"]; ok && cpu == "true" { cpuInfo := getLocalCPUOBDInfo(deadlinedCtx) diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 7100c2ce6..e71ece2e4 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -178,9 +178,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler)) if !globalIsGateway { - // -- OBD API -- - adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).Queries("perfdrive", "{perfdrive:true|false}", "perfnet", "{perfnet:true|false}", "minioinfo", "{minioinfo:true|false}", "minioconfig", "{minioconfig:true|false}", "syscpu", "{syscpu:true|false}", "sysdiskhw", "{sysdiskhw:true|false}", "sysosinfo", "{sysosinfo:true|false}", "sysmem", "{sysmem:true|false}", "sysprocess", "{sysprocess:true|false}") + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo"). + HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)). + Queries("perfdrive", "{perfdrive:true|false}", + "perfnet", "{perfnet:true|false}", + "minioinfo", "{minioinfo:true|false}", + "minioconfig", "{minioconfig:true|false}", + "syscpu", "{syscpu:true|false}", + "sysdiskhw", "{sysdiskhw:true|false}", + "sysosinfo", "{sysosinfo:true|false}", + "sysmem", "{sysmem:true|false}", + "sysprocess", "{sysprocess:true|false}", + ) } } diff --git a/pkg/disk/obd.go b/pkg/disk/obd.go index 344af7a42..4580967b0 100644 --- a/pkg/disk/obd.go +++ b/pkg/disk/obd.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "os" - "runtime" "time" "github.com/dustin/go-humanize" @@ -52,8 +51,7 @@ type Throughput struct { } // GetOBDInfo about the drive -func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Throughput, error) { - runtime.LockOSThread() +func GetOBDInfo(ctx context.Context, drive, fsPath string) (Latency, Throughput, error) { // Create a file with O_DIRECT flag, choose default umask and also make sure // we are exclusively writing to a new file using O_EXCL. @@ -96,8 +94,6 @@ func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Thro latencies[i] = float64(latencyInSecs) } - runtime.UnlockOSThread() - for i := range latencies { throughput := float64(blockSize) / latencies[i] throughputs[i] = throughput diff --git a/pkg/madmin/obd.go b/pkg/madmin/obd.go index 9ebd70051..c6ebee716 100644 --- a/pkg/madmin/obd.go +++ b/pkg/madmin/obd.go @@ -221,11 +221,14 @@ var OBDDataTypesList = []OBDDataType{ // ServerOBDInfo - Connect to a minio server and call OBD Info Management API // to fetch server's information represented by OBDInfo structure -func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType) <-chan OBDInfo { +func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType, deadline time.Duration) <-chan OBDInfo { respChan := make(chan OBDInfo) go func() { v := url.Values{} + v.Set("deadline", + deadline.Truncate(1*time.Second).String()) + // start with all set to false for _, d := range OBDDataTypesList { v.Set(string(d), "false")