diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index c66662ffd..3ad163db6 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -207,14 +207,37 @@ type ServerConnStats struct { Throughput uint64 `json:"throughput,omitempty"` } +// ServerHTTPMethodStats holds total number of HTTP operations from/to the server, +// including the average duration the call was spent. +type ServerHTTPMethodStats struct { + Count uint64 `json:"count"` + AvgDuration string `json:"avgDuration"` +} + +// ServerHTTPStats holds all type of http operations performed to/from the server +// including their average execution time. +type ServerHTTPStats struct { + TotalHEADStats ServerHTTPMethodStats `json:"totalHEADs"` + SuccessHEADStats ServerHTTPMethodStats `json:"successHEADs"` + TotalGETStats ServerHTTPMethodStats `json:"totalGETs"` + SuccessGETStats ServerHTTPMethodStats `json:"successGETs"` + TotalPUTStats ServerHTTPMethodStats `json:"totalPUTs"` + SuccessPUTStats ServerHTTPMethodStats `json:"successPUTs"` + TotalPOSTStats ServerHTTPMethodStats `json:"totalPOSTs"` + SuccessPOSTStats ServerHTTPMethodStats `json:"successPOSTs"` + TotalDELETEStats ServerHTTPMethodStats `json:"totalDELETEs"` + SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"` +} + // ServerInfo holds the information that will be returned by ServerInfo API type ServerInfo struct { StorageInfo StorageInfo `json:"storage"` ConnStats ServerConnStats `json:"network"` + HTTPStats ServerHTTPStats `json:"http"` Properties ServerProperties `json:"server"` } -// ServerInfoHandler - GET /?server-info +// ServerInfoHandler - GET /?info // ---------- // Get server information func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) { @@ -262,11 +285,13 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt TotalInputBytes: globalConnStats.getTotalInputBytes(), TotalOutputBytes: globalConnStats.getTotalOutputBytes(), } + httpStats := globalHTTPStats.toServerHTTPStats() // Build the whole returned information info := ServerInfo{ StorageInfo: storage, ConnStats: connStats, + HTTPStats: httpStats, Properties: properties, } @@ -277,6 +302,7 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt errorIf(err, "Failed to marshal storage info into json.") return } + // Reply with storage information (across nodes in a // distributed setup) as json. writeSuccessResponseJSON(w, jsonBytes) diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 6473cba63..2c9a1ab99 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -1313,6 +1313,52 @@ func TestSetConfigHandler(t *testing.T) { } } +func TestAdminServerInfo(t *testing.T) { + adminTestBed, err := prepareAdminXLTestBed() + if err != nil { + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") + } + defer adminTestBed.TearDown() + + // Initialize admin peers to make admin RPC calls. + eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"}) + if err != nil { + t.Fatalf("Failed to parse storage end point - %v", err) + } + + // Set globalMinioAddr to be able to distinguish local endpoints from remote. + globalMinioAddr = eps[0].Host + initGlobalAdminPeers(eps) + + // Prepare query params for set-config mgmt REST API. + queryVal := url.Values{} + queryVal.Set("info", "") + + req, err := buildAdminRequest(queryVal, "", http.MethodGet, 0, nil) + if err != nil { + t.Fatalf("Failed to construct get-config object request - %v", err) + } + + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Expected to succeed but failed with %d", rec.Code) + } + + result := ServerInfo{} + err = json.NewDecoder(rec.Body).Decode(&result) + if err != nil { + t.Fatalf("Failed to decode set config result json %v", err) + } + + if result.StorageInfo.Free == 0 { + t.Error("Expected StorageInfo.Free to be non empty") + } + if result.Properties.Region != globalMinioDefaultRegion { + t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, result.Properties.Region) + } +} + // TestToAdminAPIErr - test for toAdminAPIErr helper function. func TestToAdminAPIErr(t *testing.T) { testCases := []struct { diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 83cf0313c..facef3fc7 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -401,9 +401,21 @@ func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Wraps w to record http response information ww := &httpResponseRecorder{ResponseWriter: w} + // Time start before the call is about to start. + tBefore := UTCNow() + // Execute the request h.handler.ServeHTTP(ww, r) + // Time after call has completed. + tAfter := UTCNow() + + // Time duration in secs since the call started. + // + // We don't need to do nanosecond precision in this + // simply for the fact that it is not human readable. + durationSecs := tAfter.Sub(tBefore).Seconds() + // Update http statistics - globalHTTPStats.updateStats(r, ww) + globalHTTPStats.updateStats(r, ww, durationSecs) } diff --git a/cmd/http-stats.go b/cmd/http-stats.go new file mode 100644 index 000000000..ff917131d --- /dev/null +++ b/cmd/http-stats.go @@ -0,0 +1,188 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "net/http" + "time" + + "go.uber.org/atomic" +) + +// ConnStats - Network statistics +// Count total input/output transferred bytes during +// the server's life. +type ConnStats struct { + totalInputBytes atomic.Uint64 + totalOutputBytes atomic.Uint64 +} + +// Increase total input bytes +func (s *ConnStats) incInputBytes(n int) { + s.totalInputBytes.Add(uint64(n)) +} + +// Increase total output bytes +func (s *ConnStats) incOutputBytes(n int) { + s.totalOutputBytes.Add(uint64(n)) +} + +// Return total input bytes +func (s *ConnStats) getTotalInputBytes() uint64 { + return s.totalInputBytes.Load() +} + +// Return total output bytes +func (s *ConnStats) getTotalOutputBytes() uint64 { + return s.totalOutputBytes.Load() +} + +// Prepare new ConnStats structure +func newConnStats() *ConnStats { + return &ConnStats{} +} + +// HTTPMethodStats holds statistics information about +// a given HTTP method made by all clients +type HTTPMethodStats struct { + Counter atomic.Uint64 + Duration atomic.Float64 +} + +// HTTPStats holds statistics information about +// HTTP requests made by all clients +type HTTPStats struct { + // HEAD request stats. + totalHEADs HTTPMethodStats + successHEADs HTTPMethodStats + + // GET request stats. + totalGETs HTTPMethodStats + successGETs HTTPMethodStats + + // PUT request stats. + totalPUTs HTTPMethodStats + successPUTs HTTPMethodStats + + // POST request stats. + totalPOSTs HTTPMethodStats + successPOSTs HTTPMethodStats + + // DELETE request stats. + totalDELETEs HTTPMethodStats + successDELETEs HTTPMethodStats +} + +func durationStr(totalDuration, totalCount float64) string { + return fmt.Sprint(time.Duration(totalDuration/totalCount) * time.Second) +} + +// Converts http stats into struct to be sent back to the client. +func (st HTTPStats) toServerHTTPStats() ServerHTTPStats { + serverStats := ServerHTTPStats{} + serverStats.TotalHEADStats = ServerHTTPMethodStats{ + Count: st.totalHEADs.Counter.Load(), + AvgDuration: durationStr(st.totalHEADs.Duration.Load(), float64(st.totalHEADs.Counter.Load())), + } + serverStats.SuccessHEADStats = ServerHTTPMethodStats{ + Count: st.successHEADs.Counter.Load(), + AvgDuration: durationStr(st.successHEADs.Duration.Load(), float64(st.successHEADs.Counter.Load())), + } + serverStats.TotalGETStats = ServerHTTPMethodStats{ + Count: st.totalGETs.Counter.Load(), + AvgDuration: durationStr(st.totalGETs.Duration.Load(), float64(st.totalGETs.Counter.Load())), + } + serverStats.SuccessGETStats = ServerHTTPMethodStats{ + Count: st.successGETs.Counter.Load(), + AvgDuration: durationStr(st.successGETs.Duration.Load(), float64(st.successGETs.Counter.Load())), + } + serverStats.TotalPUTStats = ServerHTTPMethodStats{ + Count: st.totalPUTs.Counter.Load(), + AvgDuration: durationStr(st.totalPUTs.Duration.Load(), float64(st.totalPUTs.Counter.Load())), + } + serverStats.SuccessPUTStats = ServerHTTPMethodStats{ + Count: st.successPUTs.Counter.Load(), + AvgDuration: durationStr(st.successPUTs.Duration.Load(), float64(st.successPUTs.Counter.Load())), + } + serverStats.TotalPOSTStats = ServerHTTPMethodStats{ + Count: st.totalPOSTs.Counter.Load(), + AvgDuration: durationStr(st.totalPOSTs.Duration.Load(), float64(st.totalPOSTs.Counter.Load())), + } + serverStats.SuccessPOSTStats = ServerHTTPMethodStats{ + Count: st.successPOSTs.Counter.Load(), + AvgDuration: durationStr(st.successPOSTs.Duration.Load(), float64(st.successPOSTs.Counter.Load())), + } + serverStats.TotalDELETEStats = ServerHTTPMethodStats{ + Count: st.totalDELETEs.Counter.Load(), + AvgDuration: durationStr(st.totalDELETEs.Duration.Load(), float64(st.totalDELETEs.Counter.Load())), + } + serverStats.SuccessDELETEStats = ServerHTTPMethodStats{ + Count: st.successDELETEs.Counter.Load(), + AvgDuration: durationStr(st.successDELETEs.Duration.Load(), float64(st.successDELETEs.Counter.Load())), + } + return serverStats +} + +// Update statistics from http request and response data +func (st *HTTPStats) updateStats(r *http.Request, w *httpResponseRecorder, durationSecs float64) { + // A successful request has a 2xx response code + successReq := (w.respStatusCode >= 200 && w.respStatusCode < 300) + // Update stats according to method verb + switch r.Method { + case "HEAD": + st.totalHEADs.Counter.Inc() + st.totalHEADs.Duration.Add(durationSecs) + if successReq { + st.successHEADs.Counter.Inc() + st.successHEADs.Duration.Add(durationSecs) + } + case "GET": + st.totalGETs.Counter.Inc() + st.totalGETs.Duration.Add(durationSecs) + if successReq { + st.successGETs.Counter.Inc() + st.successGETs.Duration.Add(durationSecs) + } + case "PUT": + st.totalPUTs.Counter.Inc() + st.totalPUTs.Duration.Add(durationSecs) + if successReq { + st.successPUTs.Counter.Inc() + st.totalPUTs.Duration.Add(durationSecs) + } + case "POST": + st.totalPOSTs.Counter.Inc() + st.totalPOSTs.Duration.Add(durationSecs) + if successReq { + st.successPOSTs.Counter.Inc() + st.totalPOSTs.Duration.Add(durationSecs) + } + case "DELETE": + st.totalDELETEs.Counter.Inc() + st.totalDELETEs.Duration.Add(durationSecs) + if successReq { + st.successDELETEs.Counter.Inc() + st.successDELETEs.Duration.Add(durationSecs) + } + } +} + +// Prepare new HTTPStats structure +func newHTTPStats() *HTTPStats { + return &HTTPStats{} +} diff --git a/cmd/stats.go b/cmd/stats.go deleted file mode 100644 index 9dc691839..000000000 --- a/cmd/stats.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "net/http" - "sync/atomic" -) - -// counter - simplify atomic counting -type counter struct { - val uint64 -} - -// Inc increases counter atomically -func (c *counter) Inc(n uint64) { - atomic.AddUint64(&c.val, n) -} - -// Value fetches counter's value atomically -func (c *counter) Value() uint64 { - return atomic.LoadUint64(&c.val) -} - -// ConnStats - Network statistics -// Count total input/output transferred bytes during -// the server's life. -type ConnStats struct { - totalInputBytes counter - totalOutputBytes counter -} - -// Increase total input bytes -func (s *ConnStats) incInputBytes(n int) { - s.totalInputBytes.Inc(uint64(n)) -} - -// Increase total output bytes -func (s *ConnStats) incOutputBytes(n int) { - s.totalOutputBytes.Inc(uint64(n)) -} - -// Return total input bytes -func (s *ConnStats) getTotalInputBytes() uint64 { - return s.totalInputBytes.Value() -} - -// Return total output bytes -func (s *ConnStats) getTotalOutputBytes() uint64 { - return s.totalOutputBytes.Value() -} - -// Prepare new ConnStats structure -func newConnStats() *ConnStats { - return &ConnStats{} -} - -// httpStats holds statistics information about -// HTTP requests made by all clients -type httpStats struct { - // HEAD request stats - totalHEADs counter - successHEADs counter - // GET request stats - totalGETs counter - successGETs counter - // PUT request - totalPUTs counter - successPUTs counter - // POST request - totalPOSTs counter - successPOSTs counter - // DELETE request - totalDELETEs counter - successDELETEs counter -} - -// Update statistics from http request and response data -func (st *httpStats) updateStats(r *http.Request, w *httpResponseRecorder) { - // A successful request has a 2xx response code - successReq := (w.respStatusCode >= 200 && w.respStatusCode < 300) - // Update stats according to method verb - switch r.Method { - case "HEAD": - st.totalHEADs.Inc(1) - if successReq { - st.successHEADs.Inc(1) - } - case "GET": - st.totalGETs.Inc(1) - if successReq { - st.successGETs.Inc(1) - } - case "PUT": - st.totalPUTs.Inc(1) - if successReq { - st.successPUTs.Inc(1) - } - case "POST": - st.totalPOSTs.Inc(1) - if successReq { - st.successPOSTs.Inc(1) - } - case "DELETE": - st.totalDELETEs.Inc(1) - if successReq { - st.successDELETEs.Inc(1) - } - } -} - -// Prepare new HttpStats structure -func newHTTPStats() *httpStats { - return &httpStats{} -} diff --git a/vendor/go.uber.org/atomic/LICENSE.txt b/vendor/go.uber.org/atomic/LICENSE.txt new file mode 100644 index 000000000..8765c9fbc --- /dev/null +++ b/vendor/go.uber.org/atomic/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/go.uber.org/atomic/Makefile b/vendor/go.uber.org/atomic/Makefile new file mode 100644 index 000000000..4bc9a07f5 --- /dev/null +++ b/vendor/go.uber.org/atomic/Makefile @@ -0,0 +1,64 @@ +PACKAGES := $(shell glide nv) +# Many Go tools take file globs or directories as arguments instead of packages. +PACKAGE_FILES ?= *.go + + +# The linting tools evolve with each Go version, so run them only on the latest +# stable release. +GO_VERSION := $(shell go version | cut -d " " -f 3) +GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION))) +LINTABLE_MINOR_VERSIONS := 6 +ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),) +SHOULD_LINT := true +endif + + +export GO15VENDOREXPERIMENT=1 + + +.PHONY: build +build: + go build -i $(PACKAGES) + + +.PHONY: install +install: + glide --version || go get github.com/Masterminds/glide + glide install + + +.PHONY: test +test: + go test -cover -race $(PACKAGES) + + +.PHONY: install_ci +install_ci: install + go get github.com/wadey/gocovmerge + go get github.com/mattn/goveralls + go get golang.org/x/tools/cmd/cover +ifdef SHOULD_LINT + go get github.com/golang/lint/golint +endif + +.PHONY: lint +lint: +ifdef SHOULD_LINT + @rm -rf lint.log + @echo "Checking formatting..." + @gofmt -d -s $(PACKAGE_FILES) 2>&1 | tee lint.log + @echo "Checking vet..." + @$(foreach dir,$(PACKAGE_FILES),go tool vet $(dir) 2>&1 | tee -a lint.log;) + @echo "Checking lint..." + @$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;) + @echo "Checking for unresolved FIXMEs..." + @git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log + @[ ! -s lint.log ] +else + @echo "Skipping linters on" $(GO_VERSION) +endif + + +.PHONY: test_ci +test_ci: install_ci build + ./scripts/cover.sh $(shell go list $(PACKAGES)) diff --git a/vendor/go.uber.org/atomic/README.md b/vendor/go.uber.org/atomic/README.md new file mode 100644 index 000000000..bb4a12c18 --- /dev/null +++ b/vendor/go.uber.org/atomic/README.md @@ -0,0 +1,34 @@ +# atomic [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov] + +Simple wrappers for primitive types to enforce atomic access. + +## Installation +`go get -u go.uber.org/atomic` + +## Usage +The standard library's `sync/atomic` is powerful, but it's easy to forget which +variables must be accessed atomically. `go.uber.org/atomic` preserves all the +functionality of the standard library, but wraps the primitive types to +provide a safer, more convenient API. + +```go +var atom atomic.Uint32 +atom.Store(42) +atom.Sub(2) +atom.CAS(40, 11) +``` + +See the [documentation][doc] for a complete API specification. + +## Development Status +Stable. + +