Return 200 OK for liveness checks while distributed cluster starts (#8176)

With this PR, liveness check responds with 200 OK with "server-not-
initialized" header while objectLayer gets initialized. The header
is removed as objectLayer is initialized. This is to allow
MinIO distributed cluster to get started when running on an
orchestration platforms like Docker Swarm.

This PR also updates sample Swarm yaml files to use correct values
for healthcheck fields.

Fixes #8140

cmd/healthcheck-handler.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"runtime"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -54,7 +55,12 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
-		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		// Respond with 200 OK while server initializes to ensure a distributed cluster
+		// is able to start on orchestration platforms like Docker Swarm.
+		// Refer https://github.com/minio/minio/issues/8140 for more details.
+		// Make sure to add server not initialized status in header
+		w.Header().Set(xhttp.MinIOServerStatus, "Server-not-initialized")
+		writeSuccessResponseHeadersOnly(w)
 		return
 	}
 

cmd/http/headers.go

@@ -78,4 +78,7 @@ const (
 
 	// Deployment id.
 	MinioDeploymentID = "x-minio-deployment-id"
+
+	// Server-Status
+	MinIOServerStatus = "x-minio-server-status"
 )

docs/orchestration/docker-compose/docker-compose.yaml

@@ -16,11 +16,10 @@ services:
       MINIO_SECRET_KEY: minio123
     command: server http://minio{1...4}/data{1...2}
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio2:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -34,11 +33,10 @@ services:
       MINIO_SECRET_KEY: minio123
     command: server http://minio{1...4}/data{1...2}
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio3:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -52,11 +50,10 @@ services:
       MINIO_SECRET_KEY: minio123
     command: server http://minio{1...4}/data{1...2}
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio4:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -70,11 +67,10 @@ services:
       MINIO_SECRET_KEY: minio123
     command: server http://minio{1...4}/data{1...2}
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -18,16 +18,15 @@ services:
       placement:
         constraints:
           - node.labels.minio1==true
-    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
+    command: server http://minio{1...4}/export
     secrets:
       - secret_key
       - access_key
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio2:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -46,16 +45,15 @@ services:
       placement:
         constraints:
           - node.labels.minio2==true
-    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
+    command: server http://minio{1...4}/export
     secrets:
       - secret_key
       - access_key
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio3:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -74,16 +72,15 @@ services:
       placement:
         constraints:
           - node.labels.minio3==true
-    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
+    command: server http://minio{1...4}/export
     secrets:
       - secret_key
       - access_key
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio4:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -102,16 +99,15 @@ services:
       placement:
         constraints:
           - node.labels.minio4==true
-    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
+    command: server http://minio{1...4}/export
     secrets:
       - secret_key
       - access_key
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
 volumes:
   minio1-data:

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -27,11 +27,10 @@ services:
           - node.labels.minio1==true
     command: server http://minio{1...4}/export
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio1:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio2:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -59,11 +58,10 @@ services:
           - node.labels.minio2==true
     command: server http://minio{1...4}/export
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio2:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio3:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -91,11 +89,10 @@ services:
           - node.labels.minio3==true
     command: server http://minio{1...4}/export
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio3:9000/minio/health/live"]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 1m30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
   minio4:
     image: minio/minio:RELEASE.2019-08-29T00-25-01Z
@@ -123,11 +120,10 @@ services:
           - node.labels.minio4==true
     command: server http://minio{1...4}/export
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://minio4:9000/minio/health/live"]
-      interval: 1m30s
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
       timeout: 20s
       retries: 3
-      start_period: 3m
 
 volumes:
   minio1-data: