diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index 7fa80f2c4..73ce949ec 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -17,10 +17,12 @@ package cmd import ( - "context" "fmt" "net/http" + "os" "runtime" + + "github.com/minio/minio/cmd/logger" ) const ( @@ -40,19 +42,38 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, http.StatusOK, nil, mimeNone) } -// LivenessCheckHandler -- checks if server can ListBuckets internally. If not, server is -// considered to have failed and needs to be restarted. +// LivenessCheckHandler -- checks if server can reach its disks internally. +// If not, server is considered to have failed and needs to be restarted. // Liveness probes are used to detect situations where application (minio) // has gone into a state where it can not recover except by being restarted. func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "LivenessCheckHandler") + objLayer := newObjectLayerFn() // Service not initialized yet if objLayer == nil { writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) return } - // List buckets is unsuccessful, means server is having issues, send 503 service unavailable - if _, err := objLayer.ListBuckets(context.Background()); err != nil { + var totalLocalDisks int + var erroredDisks int + for _, endpoint := range globalEndpoints { + // Check only if local disks are accessible, we do not have + // to reach to rest of the other servers in a distributed setup. + if endpoint.IsLocal { + totalLocalDisks++ + // Attempt a stat to backend, any error resulting + // from this Stat() operation is considered as backend + // is not available, count them as errors. + if _, err := os.Stat(endpoint.Path); err != nil { + logger.LogIf(ctx, err) + erroredDisks++ + } + } + } + // If all exported local disks have errored, we simply let kubernetes + // take us down. + if totalLocalDisks == erroredDisks { writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) return }