From c0ac25bfff9ebc8640d98412870b5fd4fcbc7c18 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 30 Jun 2020 11:28:27 -0700 Subject: [PATCH] fix: readiness needs to be like liveness (#9941) Readiness as no reasoning to be cluster scope because that is not how the k8s networking works for pods, all the pods to a deployment are not sharing the network in a singleton. Instead they are run as local scopes to themselves, with readiness failures the pod is potentially taken out of the network to be resolvable - this affects the distributed setup in myriad of different ways. Instead readiness should behave like liveness with local scope alone, and should be a dummy implementation. This PR all the startup times and overal k8s startup time dramatically improves. Added another handler called as `/minio/health/cluster` to understand the cluster scope health. --- cmd/config/api/help.go | 6 ---- cmd/endpoint.go | 4 +-- cmd/healthcheck-handler.go | 19 +++++++---- cmd/healthcheck-router.go | 4 +++ docs/config/README.md | 1 - docs/metrics/healthcheck/README.md | 51 +++++++++++++++++------------- 6 files changed, 46 insertions(+), 39 deletions(-) diff --git a/cmd/config/api/help.go b/cmd/config/api/help.go index 4f241e091..f9072a6df 100644 --- a/cmd/config/api/help.go +++ b/cmd/config/api/help.go @@ -33,12 +33,6 @@ var ( Optional: true, Type: "duration", }, - config.HelpKV{ - Key: apiReadyDeadline, - Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`, - Optional: true, - Type: "duration", - }, config.HelpKV{ Key: apiCorsAllowOrigin, Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`, diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 4ac6c357d..8dabc2022 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -44,8 +44,6 @@ const ( // URLEndpointType - URL style endpoint type enum. URLEndpointType - - retryInterval = 5 // In Seconds. ) // Endpoint - any type of endpoint. @@ -302,7 +300,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error { resolvedList := make([]bool, len(endpoints)) // Mark the starting time startTime := time.Now() - keepAliveTicker := time.NewTicker(retryInterval * time.Second) + keepAliveTicker := time.NewTicker(10 * time.Millisecond) defer keepAliveTicker.Stop() for { // Break if the local endpoint is found already Or all the endpoints are resolved. diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index dd224885c..859184653 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -21,13 +21,11 @@ import ( "net/http" ) -// ReadinessCheckHandler returns if the server is ready to receive requests. -// For FS - Checks if the backend disk is available -// For Erasure backend - Checks if all the erasure sets are writable -func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { - ctx := newContext(r, w, "ReadinessCheckHandler") +// ClusterCheckHandler returns if the server is ready for requests. +func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ClusterCheckCheckHandler") - objLayer := newObjectLayerWithoutSafeModeFn() + objLayer := newObjectLayerFn() // Service not initialized yet if objLayer == nil { writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) @@ -37,7 +35,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline()) defer cancel() - if !objLayer.IsReady(ctx) && newObjectLayerFn() == nil { + if !objLayer.IsReady(ctx) { writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) return } @@ -45,6 +43,13 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, http.StatusOK, nil, mimeNone) } +// ReadinessCheckHandler Checks if the process is up. Always returns success. +func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { + // TODO: only implement this function to notify that this pod is + // busy, at a local scope in future, for now '200 OK'. + writeResponse(w, http.StatusOK, nil, mimeNone) +} + // LivenessCheckHandler - Checks if the process is up. Always returns success. func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, http.StatusOK, nil, mimeNone) diff --git a/cmd/healthcheck-router.go b/cmd/healthcheck-router.go index 25a8622d2..cb4a6ad0f 100644 --- a/cmd/healthcheck-router.go +++ b/cmd/healthcheck-router.go @@ -26,6 +26,7 @@ const ( healthCheckPath = "/health" healthCheckLivenessPath = "/live" healthCheckReadinessPath = "/ready" + healthCheckClusterPath = "/cluster" healthCheckPathPrefix = minioReservedBucketPath + healthCheckPath ) @@ -35,6 +36,9 @@ func registerHealthCheckRouter(router *mux.Router) { // Healthcheck router healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter() + // Cluster check handler to verify cluster is active + healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler)) + // Liveness handler healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler)) healthRouter.Methods(http.MethodHead).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler)) diff --git a/docs/config/README.md b/docs/config/README.md index f1522fc9e..a6009ecef 100644 --- a/docs/config/README.md +++ b/docs/config/README.md @@ -190,7 +190,6 @@ or environment variables ``` MINIO_API_REQUESTS_MAX (number) set the maximum number of concurrent requests, e.g. "1600" MINIO_API_REQUESTS_DEADLINE (duration) set the deadline for API requests waiting to be processed e.g. "1m" -MINIO_API_READY_DEADLINE (duration) set the deadline for health check API /minio/health/ready e.g. "1m" MINIO_API_CORS_ALLOW_ORIGIN (csv) set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com" ``` diff --git a/docs/metrics/healthcheck/README.md b/docs/metrics/healthcheck/README.md index b6de19e6a..658d28ff8 100644 --- a/docs/metrics/healthcheck/README.md +++ b/docs/metrics/healthcheck/README.md @@ -1,35 +1,42 @@ ## MinIO Healthcheck -MinIO server exposes two un-authenticated, healthcheck endpoints - liveness probe and readiness probe at `/minio/health/live` and `/minio/health/ready` respectively. +MinIO server exposes three un-authenticated, healthcheck endpoints liveness probe, readiness probe and a cluster probe at `/minio/health/live`, `/minio/health/ready` and `/minio/health/cluster` respectively. ### Liveness probe -This probe is used to identify situations where the server is running but may not behave optimally, i.e. sluggish response or corrupt back-end. Such problems can be *only* fixed by a restart. +This probe always responds with '200 OK'. When liveness probe fails, Kubernetes like platforms restart the container. -Internally, MinIO liveness probe handler checks if backend is alive and in read quorum to take requests. - -When liveness probe fails, Kubernetes like platforms restart the container. +``` + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + initialDelaySeconds: 3 + periodSeconds: 1 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 +``` ### Readiness probe -This probe is used to identify situations where the server is not ready to accept requests yet. In most cases, such conditions recover in some time such as quorum not available on drives due to load. - -Internally, MinIO readiness probe handler checks for backend is alive and in read quorum then the server returns 200 OK, otherwise 503 Service Unavailable. - -Platforms like Kubernetes *do not* forward traffic to a pod until its readiness probe is successful. - -### Configuration example +This probe always responds with '200 OK'. When readiness probe fails, Kubernetes like platforms *do not* forward traffic to a pod. -Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-standalone-deployment.yaml). - -### Configure readiness deadline -Readiness checks need to respond faster in orchestrated environments, to facilitate this you can use the following environment variable before starting MinIO - -``` -MINIO_API_READY_DEADLINE (duration) set the deadline for health check API /minio/health/ready e.g. "1m" ``` + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + scheme: HTTP + initialDelaySeconds: 3 + periodSeconds: 1 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 -Set a *5s* deadline for MinIO to ensure readiness handler responds with-in 5seconds. -``` -export MINIO_API_READY_DEADLINE=5s ``` + +### Cluster probe + +This probe is not useful in almost all cases, this is meant for administrators to see if quorum is available in any given cluster. The reply is '200 OK' if cluster has quorum if not it returns '503 Service Unavailable'.