From 496fba3e9ab184fbe408942c0c377434825cb8a7 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Thu, 5 Sep 2019 14:50:56 +0530 Subject: [PATCH] Return 200 OK for liveness checks while distributed cluster starts (#8176) With this PR, liveness check responds with 200 OK with "server-not- initialized" header while objectLayer gets initialized. The header is removed as objectLayer is initialized. This is to allow MinIO distributed cluster to get started when running on an orchestration platforms like Docker Swarm. This PR also updates sample Swarm yaml files to use correct values for healthcheck fields. Fixes #8140 --- cmd/healthcheck-handler.go | 8 +++++- cmd/http/headers.go | 3 ++ .../docker-compose/docker-compose.yaml | 20 ++++++------- .../docker-swarm/docker-compose-secrets.yaml | 28 ++++++++----------- .../docker-swarm/docker-compose.yaml | 18 +++++------- 5 files changed, 37 insertions(+), 40 deletions(-) diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index 8617aceaf..b89d71769 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -22,6 +22,7 @@ import ( "os" "runtime" + xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" ) @@ -54,7 +55,12 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { objLayer := newObjectLayerFn() // Service not initialized yet if objLayer == nil { - writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + // Respond with 200 OK while server initializes to ensure a distributed cluster + // is able to start on orchestration platforms like Docker Swarm. + // Refer https://github.com/minio/minio/issues/8140 for more details. + // Make sure to add server not initialized status in header + w.Header().Set(xhttp.MinIOServerStatus, "Server-not-initialized") + writeSuccessResponseHeadersOnly(w) return } diff --git a/cmd/http/headers.go b/cmd/http/headers.go index c2db6ae2f..ae456947f 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -78,4 +78,7 @@ const ( // Deployment id. MinioDeploymentID = "x-minio-deployment-id" + + // Server-Status + MinIOServerStatus = "x-minio-server-status" ) diff --git a/docs/orchestration/docker-compose/docker-compose.yaml b/docs/orchestration/docker-compose/docker-compose.yaml index 1fb31f209..e3e50d0be 100644 --- a/docs/orchestration/docker-compose/docker-compose.yaml +++ b/docs/orchestration/docker-compose/docker-compose.yaml @@ -16,11 +16,10 @@ services: MINIO_SECRET_KEY: minio123 command: server http://minio{1...4}/data{1...2} healthcheck: - test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio2: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -34,11 +33,10 @@ services: MINIO_SECRET_KEY: minio123 command: server http://minio{1...4}/data{1...2} healthcheck: - test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio3: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -52,11 +50,10 @@ services: MINIO_SECRET_KEY: minio123 command: server http://minio{1...4}/data{1...2} healthcheck: - test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio4: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -70,11 +67,10 @@ services: MINIO_SECRET_KEY: minio123 command: server http://minio{1...4}/data{1...2} healthcheck: - test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m ## By default this config uses default local driver, ## For custom volumes replace with volume driver configuration. diff --git a/docs/orchestration/docker-swarm/docker-compose-secrets.yaml b/docs/orchestration/docker-swarm/docker-compose-secrets.yaml index e11b64ab6..75594066e 100644 --- a/docs/orchestration/docker-swarm/docker-compose-secrets.yaml +++ b/docs/orchestration/docker-swarm/docker-compose-secrets.yaml @@ -18,16 +18,15 @@ services: placement: constraints: - node.labels.minio1==true - command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export + command: server http://minio{1...4}/export secrets: - secret_key - access_key healthcheck: - test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio2: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -46,16 +45,15 @@ services: placement: constraints: - node.labels.minio2==true - command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export + command: server http://minio{1...4}/export secrets: - secret_key - access_key healthcheck: - test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio3: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -74,16 +72,15 @@ services: placement: constraints: - node.labels.minio3==true - command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export + command: server http://minio{1...4}/export secrets: - secret_key - access_key healthcheck: - test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio4: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -102,16 +99,15 @@ services: placement: constraints: - node.labels.minio4==true - command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export + command: server http://minio{1...4}/export secrets: - secret_key - access_key healthcheck: - test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m volumes: minio1-data: diff --git a/docs/orchestration/docker-swarm/docker-compose.yaml b/docs/orchestration/docker-swarm/docker-compose.yaml index adf3ec13c..d192dd2c8 100644 --- a/docs/orchestration/docker-swarm/docker-compose.yaml +++ b/docs/orchestration/docker-swarm/docker-compose.yaml @@ -27,11 +27,10 @@ services: - node.labels.minio1==true command: server http://minio{1...4}/export healthcheck: - test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio2: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -59,11 +58,10 @@ services: - node.labels.minio2==true command: server http://minio{1...4}/export healthcheck: - test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m minio3: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -91,11 +89,10 @@ services: - node.labels.minio3==true command: server http://minio{1...4}/export healthcheck: - test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"] + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 1m30s timeout: 20s retries: 3 - start_period: 3m minio4: image: minio/minio:RELEASE.2019-08-29T00-25-01Z @@ -123,11 +120,10 @@ services: - node.labels.minio4==true command: server http://minio{1...4}/export healthcheck: - test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"] - interval: 1m30s + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s timeout: 20s retries: 3 - start_period: 3m volumes: minio1-data: