From 63be4709b771faf42eb342e75e3f53f8d44979ca Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 11 Feb 2020 21:08:01 +0530 Subject: [PATCH] Add metrics support for Azure & GCS Gateway (#8954) We added support for caching and S3 related metrics in #8591. As a continuation, it would be helpful to add support for Azure & GCS gateway related metrics as well. --- cmd/gateway-common.go | 29 ++++++++++++++++++++++ cmd/gateway/azure/gateway-azure.go | 16 +++++++++++- cmd/gateway/gcs/gateway-gcs.go | 16 +++++++++++- cmd/gateway/s3/gateway-s3.go | 40 +++++------------------------- cmd/metrics.go | 10 ++++---- docs/metrics/prometheus/README.md | 12 +++++---- 6 files changed, 77 insertions(+), 46 deletions(-) diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index c4483209e..fd24b4046 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -417,3 +417,32 @@ func gatewayHandleEnvVars() { } } } + +// shouldMeterRequest checks whether incoming request should be added to prometheus gateway metrics +func shouldMeterRequest(req *http.Request) bool { + return !(guessIsBrowserReq(req) || guessIsHealthCheckReq(req) || guessIsMetricsReq(req)) +} + +// MetricsTransport is a custom wrapper around Transport to track metrics +type MetricsTransport struct { + Transport *http.Transport + Metrics *Metrics +} + +// RoundTrip implements the RoundTrip method for MetricsTransport +func (m MetricsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + metered := shouldMeterRequest(r) + if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + m.Metrics.IncRequests(r.Method) + m.Metrics.IncBytesSent(r.ContentLength) + } + // Make the request to the server. + resp, err := m.Transport.RoundTrip(r) + if err != nil { + return nil, err + } + if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + m.Metrics.IncBytesReceived(resp.ContentLength) + } + return resp, nil +} diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index d85fcda5f..78afdcfb6 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -146,7 +146,14 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro return &azureObjects{}, err } - httpClient := &http.Client{Transport: minio.NewCustomHTTPTransport()} + metrics := minio.NewMetrics() + + t := &minio.MetricsTransport{ + Transport: minio.NewCustomHTTPTransport(), + Metrics: metrics, + } + + httpClient := &http.Client{Transport: t} userAgent := fmt.Sprintf("APN/1.0 MinIO/1.0 MinIO/%s", minio.Version) pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{ @@ -168,6 +175,7 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro endpoint: endpointURL.String(), httpClient: httpClient, client: client, + metrics: metrics, }, nil } @@ -357,6 +365,7 @@ type azureObjects struct { minio.GatewayUnsupported endpoint string httpClient *http.Client + metrics *minio.Metrics client azblob.ServiceURL // Azure sdk client } @@ -460,6 +469,11 @@ func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) { return } +// GetMetrics returns this gateway's metrics +func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.Metrics, error) { + return a.metrics, nil +} + // Shutdown - save any gateway metadata to disk // if necessary and reload upon next restart. func (a *azureObjects) Shutdown(ctx context.Context) error { diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 5dc64f72c..632620a0e 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -179,6 +179,13 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) } } + metrics := minio.NewMetrics() + + t := &minio.MetricsTransport{ + Transport: minio.NewCustomHTTPTransport(), + Metrics: metrics, + } + // Initialize a GCS client. // Send user-agent in this format for Google to obtain usage insights while participating in the // Google Cloud Technology Partners (https://cloud.google.com/partners/) @@ -190,8 +197,9 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) gcs := &gcsGateway{ client: client, projectID: g.projectID, + metrics: metrics, httpClient: &http.Client{ - Transport: minio.NewCustomHTTPTransport(), + Transport: t, }, } @@ -333,6 +341,7 @@ type gcsGateway struct { minio.GatewayUnsupported client *storage.Client httpClient *http.Client + metrics *minio.Metrics projectID string } @@ -349,6 +358,11 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) { return googleCreds[gcsProjectIDKey], err } +// GetMetrics returns this gateway's metrics +func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.Metrics, error) { + return l.metrics, nil +} + // Cleanup old files in minio.sys.tmp of the given bucket. func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) { it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false}) diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 8de7fb36f..96ff312ff 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -106,11 +106,9 @@ func (g *S3) Name() string { const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<_requests`: Total number of GET & HEAD requests made to cloud backend. This metrics has a label `method` that identifies GET & HEAD Requests. +- `gateway__bytes_sent`: Total number of bytes sent to cloud backend (in GET & HEAD Requests). +- `gateway__bytes_received`: Total number of bytes received from cloud backend (in GET & HEAD Requests). + +Note that this is currently only support for Azure, S3 and GCS Gateway. ## Migration guide for the new set of metrics