From 876b79b8d8e10e4e4f213b495c6af0713e0f061c Mon Sep 17 00:00:00 2001 From: Krishna Srinivas <634494+krishnasrinivas@users.noreply.github.com> Date: Tue, 9 Feb 2021 01:00:44 -0800 Subject: [PATCH] read-health check endpoint returns success if cluster can serve read requests (#11310) --- cmd/erasure-server-pool.go | 34 ++++++++++++++++++++++++++++++ cmd/fs-v1.go | 6 ++++++ cmd/gateway-unsupported.go | 5 +++++ cmd/generic-handlers.go | 3 ++- cmd/healthcheck-handler.go | 23 ++++++++++++++++++++ cmd/healthcheck-router.go | 12 ++++++----- cmd/object-api-interface.go | 1 + docs/metrics/healthcheck/README.md | 21 +++++++++++++++++- 8 files changed, 98 insertions(+), 7 deletions(-) diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 23e865e29..cf851f0a6 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1461,6 +1461,40 @@ type HealthResult struct { WriteQuorum int } +// ReadHealth returns if the cluster can serve read requests +func (z *erasureServerPools) ReadHealth(ctx context.Context) bool { + erasureSetUpCount := make([][]int, len(z.serverPools)) + for i := range z.serverPools { + erasureSetUpCount[i] = make([]int, len(z.serverPools[i].sets)) + } + + diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx) + diskIDs = append(diskIDs, getLocalDiskIDs(z)) + + for _, localDiskIDs := range diskIDs { + for _, id := range localDiskIDs { + poolIdx, setIdx, err := z.getPoolAndSet(id) + if err != nil { + logger.LogIf(ctx, err) + continue + } + erasureSetUpCount[poolIdx][setIdx]++ + } + } + + b := z.BackendInfo() + readQuorum := b.StandardSCData[0] + + for poolIdx := range erasureSetUpCount { + for setIdx := range erasureSetUpCount[poolIdx] { + if erasureSetUpCount[poolIdx][setIdx] < readQuorum { + return false + } + } + } + return true +} + // Health - returns current status of the object layer health, // provides if write access exists across sets, additionally // can be used to query scenarios if health may be lost diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 4c9d4c26f..93427c717 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -1623,3 +1623,9 @@ func (fs *FSObjects) Health(ctx context.Context, opts HealthOptions) HealthResul Healthy: newObjectLayerFn() != nil, } } + +// ReadHealth returns "read" health of the object layer +func (fs *FSObjects) ReadHealth(ctx context.Context) bool { + _, err := os.Stat(fs.fsPath) + return err == nil +} diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index b255a869e..46e42cb4c 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -254,3 +254,8 @@ func (a GatewayUnsupported) IsCompressionSupported() bool { func (a GatewayUnsupported) Health(_ context.Context, _ HealthOptions) HealthResult { return HealthResult{} } + +// ReadHealth - No Op. +func (a GatewayUnsupported) ReadHealth(_ context.Context) bool { + return true +} diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 35b74e347..e86ec9436 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -216,7 +216,8 @@ func guessIsHealthCheckReq(req *http.Request) bool { return aType == authTypeAnonymous && (req.Method == http.MethodGet || req.Method == http.MethodHead) && (req.URL.Path == healthCheckPathPrefix+healthCheckLivenessPath || req.URL.Path == healthCheckPathPrefix+healthCheckReadinessPath || - req.URL.Path == healthCheckPathPrefix+healthCheckClusterPath) + req.URL.Path == healthCheckPathPrefix+healthCheckClusterPath || + req.URL.Path == healthCheckPathPrefix+healthCheckClusterReadPath) } // guessIsMetricsReq - returns true if incoming request looks diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index 785de682b..f34659725 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -64,6 +64,29 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, http.StatusOK, nil, mimeNone) } +// ClusterReadCheckHandler returns if the server is ready for requests. +func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ClusterReadCheckHandler") + + if shouldProxy() { + w.Header().Set(xhttp.MinIOServerStatus, unavailable) + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + return + } + + objLayer := newObjectLayerFn() + + ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline()) + defer cancel() + + result := objLayer.ReadHealth(ctx) + if !result { + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + return + } + writeResponse(w, http.StatusOK, nil, mimeNone) +} + // ReadinessCheckHandler Checks if the process is up. Always returns success. func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { if shouldProxy() { diff --git a/cmd/healthcheck-router.go b/cmd/healthcheck-router.go index cb4a6ad0f..27a903413 100644 --- a/cmd/healthcheck-router.go +++ b/cmd/healthcheck-router.go @@ -23,11 +23,12 @@ import ( ) const ( - healthCheckPath = "/health" - healthCheckLivenessPath = "/live" - healthCheckReadinessPath = "/ready" - healthCheckClusterPath = "/cluster" - healthCheckPathPrefix = minioReservedBucketPath + healthCheckPath + healthCheckPath = "/health" + healthCheckLivenessPath = "/live" + healthCheckReadinessPath = "/ready" + healthCheckClusterPath = "/cluster" + healthCheckClusterReadPath = "/cluster/read" + healthCheckPathPrefix = minioReservedBucketPath + healthCheckPath ) // registerHealthCheckRouter - add handler functions for liveness and readiness routes. @@ -38,6 +39,7 @@ func registerHealthCheckRouter(router *mux.Router) { // Cluster check handler to verify cluster is active healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler)) + healthRouter.Methods(http.MethodGet).Path(healthCheckClusterReadPath).HandlerFunc(httpTraceAll(ClusterReadCheckHandler)) // Liveness handler healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler)) diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index c973d25bd..d6ecd0ef6 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -155,6 +155,7 @@ type ObjectLayer interface { // Returns health of the backend Health(ctx context.Context, opts HealthOptions) HealthResult + ReadHealth(ctx context.Context) bool // ObjectTagging operations PutObjectTags(context.Context, string, string, string, ObjectOptions) (ObjectInfo, error) diff --git a/docs/metrics/healthcheck/README.md b/docs/metrics/healthcheck/README.md index 5b9c2dfbb..6f16250c4 100644 --- a/docs/metrics/healthcheck/README.md +++ b/docs/metrics/healthcheck/README.md @@ -20,7 +20,8 @@ livenessProbe: ``` ### Cluster probe -This probe is not useful in almost all cases, this is meant for administrators to see if quorum is available in any given cluster. The reply is '200 OK' if cluster has quorum if not it returns '503 Service Unavailable'. +#### Cluster-writeable probe +This probe is not useful in almost all cases, this is meant for administrators to see if write quorum is available in any given cluster. The reply is '200 OK' if cluster has write quorum if not it returns '503 Service Unavailable'. ``` curl http://minio1:9001/minio/health/cluster @@ -37,6 +38,24 @@ X-Xss-Protection: 1; mode=block Date: Tue, 21 Jul 2020 00:36:14 GMT ``` +#### Clustr-readable probe +This probe is not useful in almost all cases, this is meant for administrators to see if read quorum is available in any given cluster. The reply is '200 OK' if cluster has read quorum if not it returns '503 Service Unavailable'. + +``` +curl http://minio1:9001/minio/health/cluster/read +HTTP/1.1 503 Service Unavailable +Accept-Ranges: bytes +Content-Length: 0 +Content-Security-Policy: block-all-mixed-content +Server: MinIO/GOGET.GOGET +Vary: Origin +X-Amz-Bucket-Region: us-east-1 +X-Minio-Write-Quorum: 3 +X-Amz-Request-Id: 16239D6AB80EBECF +X-Xss-Protection: 1; mode=block +Date: Tue, 21 Jul 2020 00:36:14 GMT +``` + #### Checking cluster health for maintenance You may query the cluster probe endpoint to check if the node which received the request can be taken down for maintenance, if the server replies back '412 Precondition Failed' this means you will lose HA. '200 OK' means you are okay to proceed.