Implement Docker healthcheck script in Go (#7105)
Go script makes it easy to read/maintain. Also updated the timeout in Dockerfiles from 5s to default 30s and test interval to 1m Higher timeout makes sense as server may sometimes respond slowly if under high load as reported in #6974 Fixes #6974master
Nitish Tiwari
6 years ago
committed by
GitHub
parent
ce960565b1
commit
1e82c4a7c4
@ -0,0 +1,170 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"bufio" |
||||||
|
"crypto/tls" |
||||||
|
"fmt" |
||||||
|
"io/ioutil" |
||||||
|
"log" |
||||||
|
"net/http" |
||||||
|
"net/url" |
||||||
|
"os" |
||||||
|
"os/exec" |
||||||
|
"strings" |
||||||
|
"time" |
||||||
|
|
||||||
|
xhttp "github.com/minio/minio/cmd/http" |
||||||
|
) |
||||||
|
|
||||||
|
const ( |
||||||
|
initGraceTime = 300 |
||||||
|
healthPath = "/minio/health/live" |
||||||
|
timeout = time.Duration(30 * time.Second) |
||||||
|
minioProcess = "minio" |
||||||
|
) |
||||||
|
|
||||||
|
// returns container boot time by finding
|
||||||
|
// modtime of /proc/1 directory
|
||||||
|
func getStartTime() time.Time { |
||||||
|
di, err := os.Stat("/proc/1") |
||||||
|
if err != nil { |
||||||
|
// Cant stat proc dir successfully, exit with error
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
return di.ModTime() |
||||||
|
} |
||||||
|
|
||||||
|
// Returns the ip:port of the Minio process
|
||||||
|
// running in the container
|
||||||
|
func findEndpoint() string { |
||||||
|
cmd := exec.Command("netstat", "-ntlp") |
||||||
|
stdout, err := cmd.StdoutPipe() |
||||||
|
if err != nil { |
||||||
|
// error getting stdout pipe
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
if err = cmd.Start(); err != nil { |
||||||
|
// error starting the command
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
// split netstat output in rows
|
||||||
|
scanner := bufio.NewScanner(stdout) |
||||||
|
scanner.Split(bufio.ScanLines) |
||||||
|
// loop over the rows to find Minio process
|
||||||
|
for scanner.Scan() { |
||||||
|
if strings.Contains(scanner.Text(), minioProcess) { |
||||||
|
line := scanner.Text() |
||||||
|
newLine := strings.Replace(line, ":::", "127.0.0.1:", 1) |
||||||
|
fields := strings.Fields(newLine) |
||||||
|
// index 3 in the row has the Local address
|
||||||
|
// find the last index of ":" - address will
|
||||||
|
// have port number after this index
|
||||||
|
i := strings.LastIndex(fields[3], ":") |
||||||
|
// split address and port
|
||||||
|
addr := fields[3][:i] |
||||||
|
port := fields[3][i+1:] |
||||||
|
// add surrounding [] for ip6 address
|
||||||
|
if strings.Count(addr, ":") > 0 { |
||||||
|
addr = strings.Join([]string{"[", addr, "]"}, "") |
||||||
|
} |
||||||
|
// return joint address and port
|
||||||
|
return strings.Join([]string{addr, port}, ":") |
||||||
|
} |
||||||
|
} |
||||||
|
if err = cmd.Wait(); err != nil { |
||||||
|
// command failed to run
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
// minio process not found, exit with error
|
||||||
|
os.Exit(1) |
||||||
|
return "" |
||||||
|
} |
||||||
|
|
||||||
|
func main() { |
||||||
|
startTime := getStartTime() |
||||||
|
|
||||||
|
// In distributed environment like Swarm, traffic is routed
|
||||||
|
// to a container only when it reports a `healthy` status. So, we exit
|
||||||
|
// with 0 to ensure healthy status till distributed Minio starts (120s).
|
||||||
|
|
||||||
|
// Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272
|
||||||
|
|
||||||
|
if (time.Now().Sub(startTime) / time.Second) < initGraceTime { |
||||||
|
os.Exit(0) |
||||||
|
} else { |
||||||
|
endPoint := findEndpoint() |
||||||
|
u, err := url.Parse(fmt.Sprintf("http://%s%s", endPoint, healthPath)) |
||||||
|
if err != nil { |
||||||
|
// Could not parse URL successfully
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
|
||||||
|
// Minio server may be using self-signed or CA certificates. To avoid
|
||||||
|
// making Docker setup complicated, we skip verifying certificates here.
|
||||||
|
// This is because, following request tests for health status within
|
||||||
|
// containerized environment, i.e. requests are always made to the Minio
|
||||||
|
// server running on the same host.
|
||||||
|
tr := &http.Transport{ |
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
||||||
|
} |
||||||
|
client := &http.Client{Transport: tr, Timeout: timeout} |
||||||
|
resp, err := client.Get(u.String()) |
||||||
|
if err != nil { |
||||||
|
// GET failed exit
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
if resp.StatusCode == http.StatusOK { |
||||||
|
// Drain any response.
|
||||||
|
xhttp.DrainBody(resp.Body) |
||||||
|
// exit with success
|
||||||
|
os.Exit(0) |
||||||
|
} |
||||||
|
bodyBytes, err := ioutil.ReadAll(resp.Body) |
||||||
|
if err != nil { |
||||||
|
// Drain any response.
|
||||||
|
xhttp.DrainBody(resp.Body) |
||||||
|
// GET failed exit
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
bodyString := string(bodyBytes) |
||||||
|
// Drain any response.
|
||||||
|
xhttp.DrainBody(resp.Body) |
||||||
|
// This means sever is configured with https
|
||||||
|
if resp.StatusCode == http.StatusForbidden && bodyString == "SSL required" { |
||||||
|
// Try with https
|
||||||
|
u.Scheme = "https" |
||||||
|
resp, err = client.Get(u.String()) |
||||||
|
if err != nil { |
||||||
|
// GET failed exit
|
||||||
|
log.Fatal(err.Error()) |
||||||
|
} |
||||||
|
if resp.StatusCode == http.StatusOK { |
||||||
|
// Drain any response.
|
||||||
|
xhttp.DrainBody(resp.Body) |
||||||
|
// exit with success
|
||||||
|
os.Exit(0) |
||||||
|
} |
||||||
|
// Drain any response.
|
||||||
|
xhttp.DrainBody(resp.Body) |
||||||
|
} |
||||||
|
} |
||||||
|
// Execution reaching here means none of
|
||||||
|
// the success cases were satisfied
|
||||||
|
os.Exit(1) |
||||||
|
} |
||||||
@ -1,56 +0,0 @@ |
|||||||
#!/bin/sh |
|
||||||
# |
|
||||||
# Minio Cloud Storage, (C) 2017 Minio, Inc. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
set -x |
|
||||||
|
|
||||||
_init () { |
|
||||||
scheme="http://" |
|
||||||
address="$(netstat -nplt 2>/dev/null | awk ' /(.*\/minio)/ { gsub(":::","127.0.0.1:",$4); print $4}')" |
|
||||||
resource="/minio/health/live" |
|
||||||
start=$(stat -c "%Y" /proc/1) |
|
||||||
} |
|
||||||
|
|
||||||
healthcheck_main () { |
|
||||||
# In distributed environment like Swarm, traffic is routed |
|
||||||
# to a container only when it reports a `healthy` status. So, we exit |
|
||||||
# with 0 to ensure healthy status till distributed Minio starts (120s). |
|
||||||
# |
|
||||||
# Refer: https://github.com/moby/moby/pull/28938#issuecomment-301753272 |
|
||||||
if [ $(( $(date +%s) - start )) -lt 120 ]; then |
|
||||||
exit 0 |
|
||||||
else |
|
||||||
# Get the http response code |
|
||||||
http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource}) |
|
||||||
|
|
||||||
# Get the http response body |
|
||||||
http_response_body=$(curl -k -s ${scheme}${address}${resource}) |
|
||||||
|
|
||||||
# server returns response 403 and body "SSL required" if non-TLS |
|
||||||
# connection is attempted on a TLS-configured server. Change |
|
||||||
# the scheme and try again |
|
||||||
if [ "$http_response" = "403" ] && \ |
|
||||||
[ "$http_response_body" = "SSL required" ]; then |
|
||||||
scheme="https://" |
|
||||||
http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource}) |
|
||||||
fi |
|
||||||
|
|
||||||
# If http_response is 200 - server is up. |
|
||||||
[ "$http_response" = "200" ] |
|
||||||
fi |
|
||||||
} |
|
||||||
|
|
||||||
_init && healthcheck_main |
|
||||||
Loading…
Reference in new issue