From bebaff269ca3a58d11aced0add31fb7c6bc9d175 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 13 Dec 2018 23:37:46 -0800 Subject: [PATCH] Support IPv6 in minio command line (#6947) Fixes #6946 --- .travis.yml | 5 +++ buildscripts/verify-build.sh | 67 ++++++++++++++++++++++++++++++ cmd/admin-rpc-client.go | 5 +++ cmd/endpoint-ellipses_test.go | 57 ++++++++++++++++++++++++++ cmd/endpoint.go | 22 +++++----- cmd/endpoint_test.go | 14 +++---- cmd/gateway-main.go | 9 +--- cmd/http/listener.go | 4 +- cmd/net.go | 77 ++++++++++++++++++++++++++--------- cmd/net_test.go | 22 ++++++---- cmd/server-main.go | 3 +- cmd/server-startup-msg.go | 17 ++------ cmd/test-utils_test.go | 2 +- pkg/ellipses/ellipses.go | 29 ++++++++++--- pkg/ellipses/ellipses_test.go | 11 +++++ pkg/net/host.go | 6 ++- 16 files changed, 273 insertions(+), 77 deletions(-) diff --git a/.travis.yml b/.travis.yml index 423ff51fd..c21ceb2f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,11 @@ matrix: - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done - bash buildscripts/go-coverage.sh +before_script: + # Add an IPv6 config - see the corresponding Travis issue + # https://github.com/travis-ci/travis-ci/issues/8361 + - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi + before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi diff --git a/buildscripts/verify-build.sh b/buildscripts/verify-build.sh index 4547e222e..1ecf82874 100755 --- a/buildscripts/verify-build.sh +++ b/buildscripts/verify-build.sh @@ -68,6 +68,36 @@ function start_minio_erasure_sets() echo "$minio_pid" } +function start_minio_dist_erasure_sets_ipv6() +{ + declare -a minio_pids + export MINIO_ACCESS_KEY=$ACCESS_KEY + export MINIO_SECRET_KEY=$SECRET_KEY + "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 & + minio_pids[0]=$! + "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 & + minio_pids[1]=$! + "${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 & + minio_pids[2]=$! + "${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 & + minio_pids[3]=$! + "${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 & + minio_pids[4]=$! + "${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 & + minio_pids[5]=$! + "${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 & + minio_pids[6]=$! + "${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 & + minio_pids[7]=$! + "${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 & + minio_pids[8]=$! + "${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 & + minio_pids[9]=$! + + sleep 35 + echo "${minio_pids[@]}" +} + function start_minio_dist_erasure_sets() { declare -a minio_pids @@ -161,6 +191,34 @@ function run_test_erasure_sets() { return "$rv" } +function run_test_dist_erasure_sets_ipv6() +{ + minio_pids=( $(start_minio_dist_erasure_sets_ipv6) ) + + export SERVER_ENDPOINT="[::1]:9000" + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + for pid in "${minio_pids[@]}"; do + kill "$pid" + done + sleep 3 + + if [ "$rv" -ne 0 ]; then + for i in $(seq 0 9); do + echo "server$i log:" + cat "$WORK_DIR/dist-minio-v6-900$i.log" + done + fi + + for i in $(seq 0 9); do + rm -f "$WORK_DIR/dist-minio-v6-900$i.log" + done + + return "$rv" +} + function run_test_dist_erasure_sets() { minio_pids=( $(start_minio_dist_erasure_sets) ) @@ -237,6 +295,7 @@ function run_test_gateway_s3() { minio_pid="$(start_minio_gateway_s3)" + export SERVER_ENDPOINT="127.0.0.1:9000" export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") @@ -279,6 +338,7 @@ function __init__() exit 1 fi + sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS" chmod a+x "$FUNCTIONAL_TESTS" } @@ -319,6 +379,13 @@ function main() exit 1 fi + echo "Testing in Distributed Erasure setup as sets with ipv6" + if ! run_test_dist_erasure_sets_ipv6; then + echo "FAILED" + rm -fr "$WORK_DIR" + exit 1 + fi + echo "Testing in Gateway S3 setup" if ! run_test_gateway_s3; then echo "FAILED" diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go index c8bef0025..9f5a4b9f5 100644 --- a/cmd/admin-rpc-client.go +++ b/cmd/admin-rpc-client.go @@ -149,6 +149,11 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) { // Use first IPv4 instead of loopback address. localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort) } + if strings.HasPrefix(localAddr, "[::1]:") { + // Use first IPv4 instead of loopback address. + localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort) + } + adminPeerList = append(adminPeerList, adminPeer{ addr: localAddr, cmdRunner: localAdminClient{}, diff --git a/cmd/endpoint-ellipses_test.go b/cmd/endpoint-ellipses_test.go index b5dacc852..7849ed2eb 100644 --- a/cmd/endpoint-ellipses_test.go +++ b/cmd/endpoint-ellipses_test.go @@ -226,6 +226,17 @@ func TestGetSetIndexes(t *testing.T) { } } +func getHexSequences(start int, number int, paddinglen int) (seq []string) { + for i := start; i <= number; i++ { + if paddinglen == 0 { + seq = append(seq, fmt.Sprintf("%x", i)) + } else { + seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i)) + } + } + return seq +} + func getSequences(start int, number int, paddinglen int) (seq []string) { for i := start; i <= number; i++ { if paddinglen == 0 { @@ -472,6 +483,52 @@ func TestParseEndpointSet(t *testing.T) { }, true, }, + // IPv6 ellipses with hexadecimal expansion + { + "http://[2001:3984:3989::{1...a}]/disk{1...10}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 10, 0), + }, + { + Prefix: "http://[2001:3984:3989::", + Suffix: "]/disk", + Seq: getHexSequences(1, 10, 0), + }, + }, + }, + nil, + [][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}}, + }, + true, + }, + // IPv6 ellipses with hexadecimal expansion with 3 position numerics. + { + "http://[2001:3984:3989::{001...00a}]/disk{1...10}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 10, 0), + }, + { + Prefix: "http://[2001:3984:3989::", + Suffix: "]/disk", + Seq: getHexSequences(1, 10, 3), + }, + }, + }, + nil, + [][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}}, + }, + true, + }, } for i, testCase := range testCases { diff --git a/cmd/endpoint.go b/cmd/endpoint.go index d47c7cfd9..a25003478 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -112,6 +112,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535") } } + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } if host == "" { return ep, fmt.Errorf("invalid URL endpoint format: empty host name") @@ -152,7 +155,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { // Only check if the arg is an ip address and ask for scheme since its absent. // localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as // /mnt/export1. So we go ahead and start the minio server in FS modes in these cases. - if isHostIPv4(arg) { + if isHostIP(arg) { return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https") } u = &url.URL{Path: path.Clean(arg)} @@ -224,7 +227,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) { uniqueArgs.Add(arg) endpoints = append(endpoints, endpoint) } - return endpoints, nil } @@ -341,7 +343,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, if err != nil { host = endpoint.Host } - hostIPSet, _ := getHostIP4(host) + hostIPSet, _ := getHostIP(host) if IPSet, ok := pathIPMap[endpoint.Path]; ok { if !IPSet.Intersection(hostIPSet).IsEmpty() { return serverAddr, endpoints, setupType, @@ -411,12 +413,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, host = localServerAddr } - ipList, err := getHostIP4(host) + ipList, err := getHostIP(host) logger.FatalIf(err, "unexpected error when resolving host '%s'", host) - // Filter ipList by IPs those start with '127.'. + // Filter ipList by IPs those start with '127.' or '::1' loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool { - return strings.HasPrefix(ip, "127.") + return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") }, "") // If loop back IP is found and ipList contains only loop back IPs, then error out. @@ -485,10 +487,10 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) { // Local peer can be empty in FS or Erasure coded mode. // If so, return globalMinioHost + globalMinioPort value. if globalMinioHost != "" { - return globalMinioHost + ":" + globalMinioPort + return net.JoinHostPort(globalMinioHost, globalMinioPort) } - return "127.0.0.1:" + globalMinioPort + return net.JoinHostPort("127.0.0.1", globalMinioPort) } return peerSet.ToSlice()[0] } @@ -525,10 +527,10 @@ func updateDomainIPs(endPoints set.StringSet) { continue } } - IPs, _ := getHostIP4(host) + IPs, _ := getHostIP(host) ipList = ipList.Union(IPs) } globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool { - return !strings.HasPrefix(ip, "127.") + return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") }, "") } diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go index 8b8fc120e..e22766fc5 100644 --- a/cmd/endpoint_test.go +++ b/cmd/endpoint_test.go @@ -304,27 +304,27 @@ func TestCreateEndpoints(t *testing.T) { }, DistXLSetupType, nil}, } - for _, testCase := range testCases { + for i, testCase := range testCases { serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...) if err == nil { if testCase.expectedErr != nil { - t.Fatalf("error: expected = %v, got = ", testCase.expectedErr) + t.Fatalf("Test (%d) error: expected = %v, got = ", i+1, testCase.expectedErr) } else { if serverAddr != testCase.expectedServerAddr { - t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr) + t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr) } if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) { - t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints) + t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints) } if setupType != testCase.expectedSetupType { - t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType) + t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType) } } } else if testCase.expectedErr == nil { - t.Fatalf("error: expected = , got = %v", err) + t.Fatalf("Test (%d) error: expected = , got = %v", i+1, err) } else if err.Error() != testCase.expectedErr.Error() { - t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err) + t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err) } } } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index dbc407b3a..f294124b3 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -19,7 +19,6 @@ package cmd import ( "context" "fmt" - "net" "net/url" "os" "os/signal" @@ -135,11 +134,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { handleCommonCmdArgs(ctx) // Get port to listen on from gateway address - var pErr error - _, globalMinioPort, pErr = net.SplitHostPort(gatewayAddr) - if pErr != nil { - logger.FatalIf(pErr, "Unable to start gateway") - } + globalMinioHost, globalMinioPort = mustSplitHostPort(gatewayAddr) // On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back // to IPv6 address ie minio will start listening on IPv6 address whereas another @@ -301,7 +296,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { } // Print gateway startup message. - printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName) + printGatewayStartupMessage(getAPIEndpoints(), gatewayName) } handleSignals() diff --git a/cmd/http/listener.go b/cmd/http/listener.go index 462902c1f..9af8b3815 100644 --- a/cmd/http/listener.go +++ b/cmd/http/listener.go @@ -399,8 +399,8 @@ func newHTTPListener(serverAddrs []string, for _, serverAddr := range serverAddrs { var l net.Listener - if l, err = listen("tcp4", serverAddr); err != nil { - if l, err = fallbackListen("tcp4", serverAddr); err != nil { + if l, err = listen("tcp", serverAddr); err != nil { + if l, err = fallbackListen("tcp", serverAddr); err != nil { return nil, err } } diff --git a/cmd/net.go b/cmd/net.go index 47b79890b..d357449d1 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -37,14 +37,21 @@ import ( // IPv4 addresses of local host. var localIP4 = mustGetLocalIP4() +// IPv6 address of local host. +var localIP6 = mustGetLocalIP6() + // mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal. func mustSplitHostPort(hostPort string) (host, port string) { host, port, err := net.SplitHostPort(hostPort) + // Strip off IPv6 zone information. + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } logger.FatalIf(err, "Unable to split host port %s", hostPort) return host, port } -// mustGetLocalIP4 returns IPv4 addresses of local host. It panics on error. +// mustGetLocalIP4 returns IPv4 addresses of localhost. It panics on error. func mustGetLocalIP4() (ipList set.StringSet) { ipList = set.NewStringSet() addrs, err := net.InterfaceAddrs() @@ -67,8 +74,31 @@ func mustGetLocalIP4() (ipList set.StringSet) { return ipList } -// getHostIP4 returns IPv4 address of given host. -func getHostIP4(host string) (ipList set.StringSet, err error) { +// mustGetLocalIP6 returns IPv6 addresses of localhost. It panics on error. +func mustGetLocalIP6() (ipList set.StringSet) { + ipList = set.NewStringSet() + addrs, err := net.InterfaceAddrs() + logger.FatalIf(err, "Unable to get IP addresses of this host") + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip.To16() != nil { + ipList.Add(ip.String()) + } + } + + return ipList +} + +// getHostIP returns IP address of given host. +func getHostIP(host string) (ipList set.StringSet, err error) { var ips []net.IP if ips, err = net.LookupIP(host); err != nil { @@ -110,9 +140,7 @@ func getHostIP4(host string) (ipList set.StringSet, err error) { ipList = set.NewStringSet() for _, ip := range ips { - if ip.To4() != nil { - ipList.Add(ip.String()) - } + ipList.Add(ip.String()) } return ipList, err @@ -169,29 +197,32 @@ func sortIPs(ipList []string) []string { return append(nonIPs, ips...) } -func getAPIEndpoints(serverAddr string) (apiEndpoints []string) { - host, port := mustSplitHostPort(serverAddr) - +func getAPIEndpoints() (apiEndpoints []string) { var ipList []string - if host == "" { + if globalMinioHost == "" { ipList = sortIPs(localIP4.ToSlice()) + ipList = append(ipList, localIP6.ToSlice()...) } else { - ipList = []string{host} + ipList = []string{globalMinioHost} } for _, ip := range ipList { - apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), ip, port)) + apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(ip, globalMinioPort))) } return apiEndpoints } -// isHostIPv4 - helper for validating if the provided arg is an ip address. -func isHostIPv4(ipAddress string) bool { +// isHostIP - helper for validating if the provided arg is an ip address. +func isHostIP(ipAddress string) bool { host, _, err := net.SplitHostPort(ipAddress) if err != nil { host = ipAddress } + // Strip off IPv6 zone information. + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } return net.ParseIP(host) != nil } @@ -291,19 +322,20 @@ func extractHostPort(hostAddr string) (string, string, error) { // correspond to one of the local IP of the // current machine func isLocalHost(host string) (bool, error) { - hostIPs, err := getHostIP4(host) + hostIPs, err := getHostIP(host) if err != nil { return false, err } - // If intersection of two IP sets is not empty, then the host is local host. - isLocal := !localIP4.Intersection(hostIPs).IsEmpty() - return isLocal, nil + // If intersection of two IP sets is not empty, then the host is localhost. + isLocalv4 := !localIP4.Intersection(hostIPs).IsEmpty() + isLocalv6 := !localIP6.Intersection(hostIPs).IsEmpty() + return isLocalv4 || isLocalv6, nil } // sameLocalAddrs - returns true if two addresses, even with different // formats, point to the same machine, e.g: -// ':9000' and 'http://localhost:9000/' will return true +// ':9000' and 'http://localhost:9000/' will return true func sameLocalAddrs(addr1, addr2 string) (bool, error) { // Extract host & port from given parameters @@ -355,6 +387,11 @@ func CheckLocalServerAddr(serverAddr string) error { return uiErrInvalidAddressFlag(err) } + // Strip off IPv6 zone information. + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } + // Check whether port is a valid port number. p, err := strconv.Atoi(port) if err != nil { @@ -366,7 +403,7 @@ func CheckLocalServerAddr(serverAddr string) error { // 0.0.0.0 is a wildcard address and refers to local network // addresses. I.e, 0.0.0.0:9000 like ":9000" refers to port // 9000 on localhost. - if host != "" && host != net.IPv4zero.String() { + if host != "" && host != net.IPv4zero.String() && host != net.IPv6zero.String() { isLocalHost, err := isLocalHost(host) if err != nil { return err diff --git a/cmd/net_test.go b/cmd/net_test.go index e4fc38876..396a7dd66 100644 --- a/cmd/net_test.go +++ b/cmd/net_test.go @@ -138,7 +138,7 @@ func TestGetHostIP(t *testing.T) { } for _, testCase := range testCases { - ipList, err := getHostIP4(testCase.host) + ipList, err := getHostIP(testCase.host) if testCase.expectedErr == nil { if err != nil { t.Fatalf("error: expected = , got = %v", err) @@ -157,17 +157,22 @@ func TestGetHostIP(t *testing.T) { // Tests finalize api endpoints. func TestGetAPIEndpoints(t *testing.T) { + host, port := globalMinioHost, globalMinioAddr + defer func() { + globalMinioHost, globalMinioAddr = host, port + }() testCases := []struct { - serverAddr string + host, port string expectedResult string }{ - {":80", "http://127.0.0.1:80"}, - {"127.0.0.1:80", "http://127.0.0.1:80"}, - {"localhost:80", "http://localhost:80"}, + {"", "80", "http://127.0.0.1:80"}, + {"127.0.0.1", "80", "http://127.0.0.1:80"}, + {"localhost", "80", "http://localhost:80"}, } for i, testCase := range testCases { - apiEndpoints := getAPIEndpoints(testCase.serverAddr) + globalMinioHost, globalMinioPort = testCase.host, testCase.port + apiEndpoints := getAPIEndpoints() apiEndpointSet := set.CreateStringSet(apiEndpoints...) if !apiEndpointSet.Contains(testCase.expectedResult) { t.Fatalf("test %d: expected: Found, got: Not Found", i+1) @@ -330,7 +335,7 @@ func TestSameLocalAddrs(t *testing.T) { } } } -func TestIsHostIPv4(t *testing.T) { +func TestIsHostIP(t *testing.T) { testCases := []struct { args string expectedResult bool @@ -341,10 +346,11 @@ func TestIsHostIPv4(t *testing.T) { {"http://192.168.1.0", false}, {"http://192.168.1.0:9000", false}, {"192.168.1.0", true}, + {"[2001:3984:3989::20%eth0]:9000", true}, } for _, testCase := range testCases { - ret := isHostIPv4(testCase.args) + ret := isHostIP(testCase.args) if testCase.expectedResult != ret { t.Fatalf("expected: %v , got: %v", testCase.expectedResult, ret) } diff --git a/cmd/server-main.go b/cmd/server-main.go index 9f14faedf..fd5bc1c3f 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -389,8 +389,7 @@ func serverMain(ctx *cli.Context) { globalObjLayerMutex.Unlock() // Prints the formatted startup message once object layer is initialized. - apiEndpoints := getAPIEndpoints(globalMinioAddr) - printStartupMessage(apiEndpoints) + printStartupMessage(getAPIEndpoints()) // Set uptime time after object layer has initialized. globalBootTime = UTCNow() diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index c5a745b75..2b5744e7f 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -20,12 +20,12 @@ import ( "context" "crypto/x509" "fmt" - "net/url" "runtime" "strings" humanize "github.com/dustin/go-humanize" "github.com/minio/minio/cmd/logger" + xnet "github.com/minio/minio/pkg/net" ) // Documentation links, these are part of message printing code. @@ -83,23 +83,12 @@ func stripStandardPorts(apiEndpoints []string) (newAPIEndpoints []string) { newAPIEndpoints = make([]string, len(apiEndpoints)) // Check all API endpoints for standard ports and strip them. for i, apiEndpoint := range apiEndpoints { - url, err := url.Parse(apiEndpoint) + url, err := xnet.ParseURL(apiEndpoint) if err != nil { newAPIEndpoints[i] = apiEndpoint continue } - host, port := mustSplitHostPort(url.Host) - // For standard HTTP(s) ports such as "80" and "443" - // apiEndpoints should only be host without port. - switch { - case url.Scheme == "http" && port == "80": - fallthrough - case url.Scheme == "https" && port == "443": - url.Host = host - newAPIEndpoints[i] = url.String() - default: - newAPIEndpoints[i] = apiEndpoint - } + newAPIEndpoints[i] = url.String() } return newAPIEndpoints } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 5c0f8eb27..28290936b 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -2319,7 +2319,7 @@ func getEndpointsLocalAddr(endpoints EndpointList) string { } } - return globalMinioHost + ":" + globalMinioPort + return net.JoinHostPort(globalMinioHost, globalMinioPort) } // fetches a random number between range min-max. diff --git a/pkg/ellipses/ellipses.go b/pkg/ellipses/ellipses.go index 30bf09a2b..faab0834c 100644 --- a/pkg/ellipses/ellipses.go +++ b/pkg/ellipses/ellipses.go @@ -26,7 +26,7 @@ import ( var ( // Regex to extract ellipses syntax inputs. - regexpEllipses = regexp.MustCompile(`(.*)({[0-9]*\.\.\.[0-9]*})(.*)`) + regexpEllipses = regexp.MustCompile(`(.*)({[0-9a-z]*\.\.\.[0-9a-z]*})(.*)`) // Ellipses constants openBraces = "{" @@ -53,13 +53,24 @@ func parseEllipsesRange(pattern string) (seq []string, err error) { return nil, errors.New("Invalid argument") } + var hexadecimal bool var start, end uint64 if start, err = strconv.ParseUint(ellipsesRange[0], 10, 64); err != nil { - return nil, err + // Look for hexadecimal conversions if any. + start, err = strconv.ParseUint(ellipsesRange[0], 16, 64) + if err != nil { + return nil, err + } + hexadecimal = true } if end, err = strconv.ParseUint(ellipsesRange[1], 10, 64); err != nil { - return nil, err + // Look for hexadecimal conversions if any. + end, err = strconv.ParseUint(ellipsesRange[1], 16, 64) + if err != nil { + return nil, err + } + hexadecimal = true } if start > end { @@ -68,9 +79,17 @@ func parseEllipsesRange(pattern string) (seq []string, err error) { for i := start; i <= end; i++ { if strings.HasPrefix(ellipsesRange[0], "0") && len(ellipsesRange[0]) > 1 || strings.HasPrefix(ellipsesRange[1], "0") { - seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i)) + if hexadecimal { + seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", len(ellipsesRange[1])), i)) + } else { + seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i)) + } } else { - seq = append(seq, fmt.Sprintf("%d", i)) + if hexadecimal { + seq = append(seq, fmt.Sprintf("%x", i)) + } else { + seq = append(seq, fmt.Sprintf("%d", i)) + } } } diff --git a/pkg/ellipses/ellipses_test.go b/pkg/ellipses/ellipses_test.go index 08fac4a68..73e7de995 100644 --- a/pkg/ellipses/ellipses_test.go +++ b/pkg/ellipses/ellipses_test.go @@ -96,6 +96,7 @@ func TestHasEllipses(t *testing.T) { true, }, { + []string{ "mydisk-{1...4}{1..2.}", }, @@ -201,6 +202,11 @@ func TestFindEllipsesPatterns(t *testing.T) { false, 0, }, + { + "{f...z}", + false, + 0, + }, // Test for valid input. { "{1...64}", @@ -222,6 +228,11 @@ func TestFindEllipsesPatterns(t *testing.T) { true, 36, }, + { + "{1...a}", + true, + 10, + }, } for i, testCase := range testCases { diff --git a/pkg/net/host.go b/pkg/net/host.go index 59f57afd7..982672b1a 100644 --- a/pkg/net/host.go +++ b/pkg/net/host.go @@ -44,7 +44,7 @@ func (host Host) String() string { return host.Name } - return host.Name + ":" + host.Port.String() + return net.JoinHostPort(host.Name, host.Port.String()) } // Equal - checks whether given host is equal or not. @@ -129,6 +129,10 @@ func ParseHost(s string) (*Host, error) { isPortSet = true } + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } + if !isValidHost(host) { return nil, errors.New("invalid hostname") }