From 1e82c4a7c4cae2771955f0e9b9dde5b5f1329da9 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 20 Feb 2019 21:42:03 +0530 Subject: [PATCH] Implement Docker healthcheck script in Go (#7105) Go script makes it easy to read/maintain. Also updated the timeout in Dockerfiles from 5s to default 30s and test interval to 1m Higher timeout makes sense as server may sometimes respond slowly if under high load as reported in #6974 Fixes #6974 --- .gitignore | 3 +- Dockerfile | 9 +- Dockerfile.dev | 7 +- Dockerfile.release | 21 ++++- Makefile | 1 + dockerscripts/healthcheck.go | 170 +++++++++++++++++++++++++++++++++++ dockerscripts/healthcheck.sh | 56 ------------ 7 files changed, 198 insertions(+), 69 deletions(-) create mode 100755 dockerscripts/healthcheck.go delete mode 100755 dockerscripts/healthcheck.sh diff --git a/.gitignore b/.gitignore index 60e920dc6..db1163546 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ parts/ prime/ stage/ .sia_temp/ -config.json \ No newline at end of file +config.json +healthcheck diff --git a/Dockerfile b/Dockerfile index b0736b934..2319c0c08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,8 @@ RUN \ apk add --no-cache git && \ go get -v -d github.com/minio/minio && \ cd /go/src/github.com/minio/minio && \ - go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" + go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \ + go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go FROM alpine:3.7 @@ -22,7 +23,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \ EXPOSE 9000 COPY --from=0 /go/bin/minio /usr/bin/minio -COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/ +COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck +COPY dockerscripts/docker-entrypoint.sh /usr/bin/ RUN \ apk add --no-cache ca-certificates 'curl>7.61.0' && \ @@ -32,7 +34,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] VOLUME ["/data"] -HEALTHCHECK --interval=30s --timeout=5s \ - CMD /usr/bin/healthcheck.sh +HEALTHCHECK --interval=1m CMD healthcheck CMD ["minio"] diff --git a/Dockerfile.dev b/Dockerfile.dev index 270210ff2..f42501688 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -2,7 +2,7 @@ FROM alpine:3.7 LABEL maintainer="Minio Inc " -COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/ +COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/ COPY minio /usr/bin/ ENV MINIO_UPDATE off @@ -14,7 +14,7 @@ RUN \ echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ chmod +x /usr/bin/minio && \ chmod +x /usr/bin/docker-entrypoint.sh && \ - chmod +x /usr/bin/healthcheck.sh + chmod +x /usr/bin/healthcheck EXPOSE 9000 @@ -22,7 +22,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] VOLUME ["/data"] -HEALTHCHECK --interval=30s --timeout=5s \ - CMD /usr/bin/healthcheck.sh +HEALTHCHECK --interval=1m CMD healthcheck CMD ["minio"] diff --git a/Dockerfile.release b/Dockerfile.release index 7dd0ff9c7..cff407336 100644 --- a/Dockerfile.release +++ b/Dockerfile.release @@ -1,8 +1,22 @@ +FROM golang:1.11.4-alpine3.7 + +ENV GOPATH /go +ENV CGO_ENABLED 0 + +WORKDIR /go/src/github.com/minio/ + +RUN \ + apk add --no-cache git && \ + go get -v -d github.com/minio/minio && \ + cd /go/src/github.com/minio/minio/dockerscripts && \ + go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go + FROM alpine:3.7 LABEL maintainer="Minio Inc " -COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/ +COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck +COPY dockerscripts/docker-entrypoint.sh /usr/bin/ ENV MINIO_UPDATE off ENV MINIO_ACCESS_KEY_FILE=access_key \ @@ -14,7 +28,7 @@ RUN \ curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \ chmod +x /usr/bin/minio && \ chmod +x /usr/bin/docker-entrypoint.sh && \ - chmod +x /usr/bin/healthcheck.sh + chmod +x /usr/bin/healthcheck EXPOSE 9000 @@ -22,7 +36,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] VOLUME ["/data"] -HEALTHCHECK --interval=30s --timeout=5s \ - CMD /usr/bin/healthcheck.sh +HEALTHCHECK --interval=1m CMD healthcheck CMD ["minio"] diff --git a/Makefile b/Makefile index f28240c8a..96170fba4 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ coverage: build build: checks @echo "Building minio binary to './minio'" @GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio + @GOFLAGS="" CGO_ENABLED=0 go build --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go docker: build @docker build -t $(TAG) . -f Dockerfile.dev diff --git a/dockerscripts/healthcheck.go b/dockerscripts/healthcheck.go new file mode 100755 index 000000000..376b2f26c --- /dev/null +++ b/dockerscripts/healthcheck.go @@ -0,0 +1,170 @@ +/* + * Minio Cloud Storage, (C) 2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bufio" + "crypto/tls" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "strings" + "time" + + xhttp "github.com/minio/minio/cmd/http" +) + +const ( + initGraceTime = 300 + healthPath = "/minio/health/live" + timeout = time.Duration(30 * time.Second) + minioProcess = "minio" +) + +// returns container boot time by finding +// modtime of /proc/1 directory +func getStartTime() time.Time { + di, err := os.Stat("/proc/1") + if err != nil { + // Cant stat proc dir successfully, exit with error + log.Fatal(err.Error()) + } + return di.ModTime() +} + +// Returns the ip:port of the Minio process +// running in the container +func findEndpoint() string { + cmd := exec.Command("netstat", "-ntlp") + stdout, err := cmd.StdoutPipe() + if err != nil { + // error getting stdout pipe + log.Fatal(err.Error()) + } + if err = cmd.Start(); err != nil { + // error starting the command + log.Fatal(err.Error()) + } + // split netstat output in rows + scanner := bufio.NewScanner(stdout) + scanner.Split(bufio.ScanLines) + // loop over the rows to find Minio process + for scanner.Scan() { + if strings.Contains(scanner.Text(), minioProcess) { + line := scanner.Text() + newLine := strings.Replace(line, ":::", "127.0.0.1:", 1) + fields := strings.Fields(newLine) + // index 3 in the row has the Local address + // find the last index of ":" - address will + // have port number after this index + i := strings.LastIndex(fields[3], ":") + // split address and port + addr := fields[3][:i] + port := fields[3][i+1:] + // add surrounding [] for ip6 address + if strings.Count(addr, ":") > 0 { + addr = strings.Join([]string{"[", addr, "]"}, "") + } + // return joint address and port + return strings.Join([]string{addr, port}, ":") + } + } + if err = cmd.Wait(); err != nil { + // command failed to run + log.Fatal(err.Error()) + } + // minio process not found, exit with error + os.Exit(1) + return "" +} + +func main() { + startTime := getStartTime() + + // In distributed environment like Swarm, traffic is routed + // to a container only when it reports a `healthy` status. So, we exit + // with 0 to ensure healthy status till distributed Minio starts (120s). + + // Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272 + + if (time.Now().Sub(startTime) / time.Second) < initGraceTime { + os.Exit(0) + } else { + endPoint := findEndpoint() + u, err := url.Parse(fmt.Sprintf("http://%s%s", endPoint, healthPath)) + if err != nil { + // Could not parse URL successfully + log.Fatal(err.Error()) + } + + // Minio server may be using self-signed or CA certificates. To avoid + // making Docker setup complicated, we skip verifying certificates here. + // This is because, following request tests for health status within + // containerized environment, i.e. requests are always made to the Minio + // server running on the same host. + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr, Timeout: timeout} + resp, err := client.Get(u.String()) + if err != nil { + // GET failed exit + log.Fatal(err.Error()) + } + if resp.StatusCode == http.StatusOK { + // Drain any response. + xhttp.DrainBody(resp.Body) + // exit with success + os.Exit(0) + } + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + // Drain any response. + xhttp.DrainBody(resp.Body) + // GET failed exit + log.Fatal(err.Error()) + } + bodyString := string(bodyBytes) + // Drain any response. + xhttp.DrainBody(resp.Body) + // This means sever is configured with https + if resp.StatusCode == http.StatusForbidden && bodyString == "SSL required" { + // Try with https + u.Scheme = "https" + resp, err = client.Get(u.String()) + if err != nil { + // GET failed exit + log.Fatal(err.Error()) + } + if resp.StatusCode == http.StatusOK { + // Drain any response. + xhttp.DrainBody(resp.Body) + // exit with success + os.Exit(0) + } + // Drain any response. + xhttp.DrainBody(resp.Body) + } + } + // Execution reaching here means none of + // the success cases were satisfied + os.Exit(1) +} diff --git a/dockerscripts/healthcheck.sh b/dockerscripts/healthcheck.sh deleted file mode 100755 index 82b65648b..000000000 --- a/dockerscripts/healthcheck.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -x - -_init () { - scheme="http://" - address="$(netstat -nplt 2>/dev/null | awk ' /(.*\/minio)/ { gsub(":::","127.0.0.1:",$4); print $4}')" - resource="/minio/health/live" - start=$(stat -c "%Y" /proc/1) -} - -healthcheck_main () { - # In distributed environment like Swarm, traffic is routed - # to a container only when it reports a `healthy` status. So, we exit - # with 0 to ensure healthy status till distributed Minio starts (120s). - # - # Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272 - if [ $(( $(date +%s) - start )) -lt 120 ]; then - exit 0 - else - # Get the http response code - http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource}) - - # Get the http response body - http_response_body=$(curl -k -s ${scheme}${address}${resource}) - - # server returns response 403 and body "SSL required" if non-TLS - # connection is attempted on a TLS-configured server. Change - # the scheme and try again - if [ "$http_response" = "403" ] && \ - [ "$http_response_body" = "SSL required" ]; then - scheme="https://" - http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource}) - fi - - # If http_response is 200 - server is up. - [ "$http_response" = "200" ] - fi -} - -_init && healthcheck_main