diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 307c3439d..dd8eedf1c 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -259,6 +259,7 @@ type ServerHTTPAPIStats struct { // ServerHTTPStats holds all type of http operations performed to/from the server // including their average execution time. type ServerHTTPStats struct { + S3RequestsInQueue int32 `json:"s3RequestsInQueue"` CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` diff --git a/cmd/handler-api.go b/cmd/handler-api.go index 38119b516..5806ae774 100644 --- a/cmd/handler-api.go +++ b/cmd/handler-api.go @@ -136,20 +136,25 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc { return } + globalHTTPStats.addRequestsInQueue(1) + deadlineTimer := time.NewTimer(deadline) defer deadlineTimer.Stop() select { case pool <- struct{}{}: defer func() { <-pool }() + globalHTTPStats.addRequestsInQueue(-1) f.ServeHTTP(w, r) case <-deadlineTimer.C: // Send a http timeout message writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrOperationMaxedOut), r.URL, guessIsBrowserReq(r)) + globalHTTPStats.addRequestsInQueue(-1) return case <-r.Context().Done(): + globalHTTPStats.addRequestsInQueue(-1) return } } diff --git a/cmd/http-stats.go b/cmd/http-stats.go index 4058acc7b..7065a95c1 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -137,23 +137,26 @@ func (stats *HTTPAPIStats) Load() map[string]int { // HTTPStats holds statistics information about // HTTP requests made by all clients type HTTPStats struct { + s3RequestsInQueue int32 currentS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats totalS3Errors HTTPAPIStats } +func (st *HTTPStats) addRequestsInQueue(i int32) { + atomic.AddInt32(&st.s3RequestsInQueue, i) +} + // Converts http stats into struct to be sent back to the client. func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { serverStats := ServerHTTPStats{} - + serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) serverStats.CurrentS3Requests = ServerHTTPAPIStats{ APIStats: st.currentS3Requests.Load(), } - serverStats.TotalS3Requests = ServerHTTPAPIStats{ APIStats: st.totalS3Requests.Load(), } - serverStats.TotalS3Errors = ServerHTTPAPIStats{ APIStats: st.totalS3Errors.Load(), } diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 083dae60a..a38c0eeca 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -78,6 +78,7 @@ const ( inflightTotal MetricName = "inflight_total" limitTotal MetricName = "limit_total" missedTotal MetricName = "missed_total" + waitingTotal MetricName = "waiting_total" objectTotal MetricName = "object_total" offlineTotal MetricName = "offline_total" onlineTotal MetricName = "online_total" @@ -386,10 +387,19 @@ func getS3RequestsInFlightMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: requestsSubsystem, Name: inflightTotal, - Help: "Total number of S3 requests currently in flight.", + Help: "Total number of S3 requests currently in flight", Type: gaugeMetric, } } +func getS3RequestsInQueueMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: waitingTotal, + Help: "Number of S3 requests in the waiting queue", + Type: counterMetric, + } +} func getS3RequestsTotalMD() MetricDescription { return MetricDescription{ Namespace: s3MetricNamespace, @@ -939,6 +949,10 @@ func getHTTPMetrics() MetricsGroup { Metrics: []Metric{}, initialize: func(ctx context.Context, metrics *MetricsGroup) { httpStats := globalHTTPStats.toServerHTTPStats() + metrics.Metrics = append(metrics.Metrics, Metric{ + Description: getS3RequestsInQueueMD(), + Value: float64(httpStats.S3RequestsInQueue), + }) for api, value := range httpStats.CurrentS3Requests.APIStats { metrics.Metrics = append(metrics.Metrics, Metric{ Description: getS3RequestsInFlightMD(),