From d96584ef58815d7ee20470812983179d1fba26d3 Mon Sep 17 00:00:00 2001 From: Praveen raj Mani Date: Fri, 19 Apr 2019 22:56:44 +0530 Subject: [PATCH] Allow server to start if one of local nodes in docker/kubernetes setup is resolved (#7452) Allow server to start if one of the local nodes in docker/kubernetes setup is successfully resolved - The rule is that we need atleast one local node to work. We dont need to resolve the rest at that point. - In a non-orchestrational setup, we fail if we do not have atleast one local node up and running. - In an orchestrational setup (docker-swarm and kubernetes), We retry with a sleep of 5 seconds until any one local node shows up. Fixes #6995 --- cmd/endpoint-ellipses_test.go | 1 - cmd/endpoint.go | 107 +++++++++++++++++++++++++++++++--- cmd/endpoint_test.go | 72 ++++++++++++++--------- cmd/net.go | 38 +----------- cmd/server-main.go | 4 +- cmd/storage-rest_test.go | 4 ++ 6 files changed, 150 insertions(+), 76 deletions(-) diff --git a/cmd/endpoint-ellipses_test.go b/cmd/endpoint-ellipses_test.go index c333be20a..5c258d5be 100644 --- a/cmd/endpoint-ellipses_test.go +++ b/cmd/endpoint-ellipses_test.go @@ -44,7 +44,6 @@ func TestCreateServerEndpoints(t *testing.T) { {":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false}, // Same host cannot export same disk on two ports - special case localhost. {":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false}, - // Valid inputs. {":9000", []string{"/export1"}, true}, {":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true}, diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 1a155eb5f..959ed38a5 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "fmt" "net" "net/url" @@ -26,7 +27,9 @@ import ( "runtime" "strconv" "strings" + "time" + humanize "github.com/dustin/go-humanize" "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/cpu" @@ -44,6 +47,8 @@ const ( // URLEndpointType - URL style endpoint type enum. URLEndpointType + + retryInterval = 5 // In Seconds. ) // Endpoint - any type of endpoint. @@ -51,6 +56,7 @@ type Endpoint struct { *url.URL IsLocal bool SetIndex int + HostName string } func (endpoint Endpoint) String() string { @@ -75,6 +81,19 @@ func (endpoint Endpoint) IsHTTPS() bool { return endpoint.Scheme == "https" } +// UpdateIsLocal - resolves the host and updates if it is local or not. +func (endpoint *Endpoint) UpdateIsLocal() error { + if !endpoint.IsLocal { + isLocal, err := isLocalHost(endpoint.HostName) + if err != nil { + return err + } + endpoint.IsLocal = isLocal + + } + return nil +} + // NewEndpoint - returns new endpoint based on given arguments. func NewEndpoint(arg string) (ep Endpoint, e error) { // isEmptyPath - check whether given path is not empty. @@ -87,6 +106,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { } var isLocal bool + var host string u, err := url.Parse(arg) if err == nil && u.Host != "" { // URL style of endpoint. @@ -98,7 +118,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { return ep, fmt.Errorf("invalid URL endpoint format") } - var host, port string + var port string host, port, err = net.SplitHostPort(u.Host) if err != nil { if !strings.Contains(err.Error(), "missing port in address") { @@ -150,10 +170,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { } } - isLocal, err = isLocalHost(host) - if err != nil { - return ep, err - } } else { // Only check if the arg is an ip address and ask for scheme since its absent. // localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as @@ -166,8 +182,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) { } return Endpoint{ - URL: u, - IsLocal: isLocal, + URL: u, + IsLocal: isLocal, + HostName: host, }, nil } @@ -200,6 +217,74 @@ func (endpoints EndpointList) GetString(i int) string { return endpoints[i].String() } +// UpdateIsLocal - resolves the host and discovers the local host. +func (endpoints EndpointList) UpdateIsLocal() error { + var epsResolved int + var foundLocal bool + resolvedList := make([]bool, len(endpoints)) + // Mark the starting time + startTime := time.Now() + keepAliveTicker := time.NewTicker(retryInterval * time.Second) + defer keepAliveTicker.Stop() + for { + // Break if the local endpoint is found already. Or all the endpoints are resolved. + if foundLocal || (epsResolved == len(endpoints)) { + break + } + // Retry infinitely on Kubernetes and Docker swarm. + // This is needed as the remote hosts are sometime + // not available immediately. + select { + case <-globalOSSignalCh: + return fmt.Errorf("The endpoint resolution got interrupted") + default: + for i, resolved := range resolvedList { + if resolved { + continue + } + + // return err if not Docker or Kubernetes + // We use IsDocker() method to check for Docker Swarm environment + // as there is no reliable way to clearly identify Swarm from + // Docker environment. + isLocal, err := isLocalHost(endpoints[i].HostName) + if err != nil { + if !IsDocker() && !IsKubernetes() { + return err + } + // time elapsed + timeElapsed := time.Since(startTime) + // log error only if more than 1s elapsed + if timeElapsed > time.Second { + // log the message to console about the host not being + // resolveable. + reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName) + reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", "")) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) + } + } else { + resolvedList[i] = true + endpoints[i].IsLocal = isLocal + epsResolved++ + if !foundLocal { + foundLocal = isLocal + } + } + } + + // Wait for the tick, if the there exist a local endpoint in discovery. + // Non docker/kubernetes environment does not need to wait. + if !foundLocal && (IsDocker() && IsKubernetes()) { + <-keepAliveTicker.C + } + } + } + + return nil + +} + // localEndpointsMemUsage - returns ServerMemUsageInfo for only the // local endpoints from given list of endpoints func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo { @@ -302,6 +387,7 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) { uniqueArgs.Add(arg) endpoints = append(endpoints, endpoint) } + return endpoints, nil } @@ -341,6 +427,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, if err != nil { return serverAddr, endpoints, setupType, err } + if err := endpoint.UpdateIsLocal(); err != nil { + return serverAddr, endpoints, setupType, err + } if endpoint.Type() != PathEndpointType { return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup") } @@ -381,6 +470,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList, return serverAddr, endpoints, setupType, nil } + if err := endpoints.UpdateIsLocal(); err != nil { + return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error()) + } + // Here all endpoints are URL style. endpointPathSet := set.NewStringSet() localEndpointCount := 0 diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go index 49012794b..445e7526e 100644 --- a/cmd/endpoint_test.go +++ b/cmd/endpoint_test.go @@ -49,11 +49,11 @@ func TestNewEndpoint(t *testing.T) { {"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil}, {"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil}, {"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil}, - {"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil}, - {"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil}, - {"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil}, - {"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil}, - {"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil}, + {"http://localhost/path", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil}, + {"http://localhost/path//", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil}, + {"https://example.org/path", Endpoint{URL: u2, IsLocal: false, HostName: "example.org"}, URLEndpointType, nil}, + {"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil}, + {"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil}, {"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, {"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, {`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, @@ -71,6 +71,10 @@ func TestNewEndpoint(t *testing.T) { for _, testCase := range testCases { endpoint, err := NewEndpoint(testCase.arg) + if err == nil { + err = endpoint.UpdateIsLocal() + } + if testCase.expectedErr == nil { if err != nil { t.Fatalf("error: expected = , got = %v", err) @@ -261,46 +265,46 @@ func TestCreateEndpoints(t *testing.T) { // DistXL type {"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{ - Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]}, - Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]}, - Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]}, - Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]}, + Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0], HostName: nonLoopBackIP}, + Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1], HostName: nonLoopBackIP}, + Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2], HostName: "example.org"}, + Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3], HostName: "example.com"}, }, DistXLSetupType, nil}, {"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{ - Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]}, - Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]}, - Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]}, - Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]}, + Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0], HostName: nonLoopBackIP}, + Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1], HostName: nonLoopBackIP}, + Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2], HostName: "example.org"}, + Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3], HostName: "example.com"}, }, DistXLSetupType, nil}, {":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", EndpointList{ - Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]}, - Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]}, - Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]}, - Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]}, + Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0], HostName: nonLoopBackIP}, + Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1], HostName: "example.org"}, + Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2], HostName: "example.com"}, + Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3], HostName: "example.net"}, }, DistXLSetupType, nil}, {":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", EndpointList{ - Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]}, - Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]}, - Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]}, - Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]}, + Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0], HostName: nonLoopBackIP}, + Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1], HostName: "example.org"}, + Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2], HostName: "example.com"}, + Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3], HostName: "example.net"}, }, DistXLSetupType, nil}, {":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", EndpointList{ - Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]}, - Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]}, - Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]}, - Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]}, + Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0], HostName: nonLoopBackIP}, + Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1], HostName: nonLoopBackIP}, + Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2], HostName: nonLoopBackIP}, + Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3], HostName: nonLoopBackIP}, }, DistXLSetupType, nil}, // DistXL Setup using only local host. {":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", EndpointList{ - Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]}, - Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]}, - Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]}, - Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]}, + Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0], HostName: "localhost"}, + Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1], HostName: "localhost"}, + Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2], HostName: "127.0.0.1"}, + Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3], HostName: nonLoopBackIP}, }, DistXLSetupType, nil}, } @@ -357,6 +361,11 @@ func TestGetLocalPeer(t *testing.T) { for i, testCase := range testCases { endpoints, _ := NewEndpointList(testCase.endpointArgs...) + if !endpoints[0].IsLocal { + if err := endpoints.UpdateIsLocal(); err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } remotePeer := GetLocalPeer(endpoints) if remotePeer != testCase.expectedResult { t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer) @@ -384,6 +393,11 @@ func TestGetRemotePeers(t *testing.T) { for _, testCase := range testCases { endpoints, _ := NewEndpointList(testCase.endpointArgs...) + if !endpoints[0].IsLocal { + if err := endpoints.UpdateIsLocal(); err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } remotePeers := GetRemotePeers(endpoints) if !reflect.DeepEqual(remotePeers, testCase.expectedResult) { t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers) diff --git a/cmd/net.go b/cmd/net.go index 82d65bf4a..cfa2bbd7b 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -17,7 +17,6 @@ package cmd import ( - "context" "errors" "fmt" "net" @@ -27,9 +26,7 @@ import ( "strconv" "strings" "syscall" - "time" - humanize "github.com/dustin/go-humanize" "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/cmd/logger" ) @@ -102,40 +99,7 @@ func getHostIP(host string) (ipList set.StringSet, err error) { var ips []net.IP if ips, err = net.LookupIP(host); err != nil { - // return err if not Docker or Kubernetes - // We use IsDocker() method to check for Docker Swarm environment - // as there is no reliable way to clearly identify Swarm from - // Docker environment. - if !IsDocker() && !IsKubernetes() { - return ipList, err - } - - // channel to indicate completion of host resolution - doneCh := make(chan struct{}) - // Indicate retry routine to exit cleanly, upon this function return. - defer close(doneCh) - // Mark the starting time - startTime := time.Now() - // wait for hosts to resolve in exponentialbackoff manner - for range newRetryTimerSimple(doneCh) { - // Retry infinitely on Kubernetes and Docker swarm. - // This is needed as the remote hosts are sometime - // not available immediately. - if ips, err = net.LookupIP(host); err == nil { - break - } - // time elapsed - timeElapsed := time.Since(startTime) - // log error only if more than 1s elapsed - if timeElapsed > time.Second { - // log the message to console about the host not being - // resolveable. - reqInfo := (&logger.ReqInfo{}).AppendTags("host", host) - reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", "")) - ctx := logger.SetReqInfo(context.Background(), reqInfo) - logger.LogIf(ctx, err) - } - } + return ipList, err } ipList = set.NewStringSet() diff --git a/cmd/server-main.go b/cmd/server-main.go index 213311637..abc6a066f 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -200,6 +200,8 @@ func serverMain(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "server", 1) } + signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM) + // Disable logging until server initialization is complete, any // error during initialization will be shown as a fatal message logger.Disable = true @@ -305,8 +307,6 @@ func serverMain(ctx *cli.Context) { globalHTTPServerErrorCh <- globalHTTPServer.Start() }() - signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM) - newObject, err := newObjectLayer(globalEndpoints) if err != nil { // Stop watching for any certificate changes. diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go index b7abff5d1..588e457ed 100644 --- a/cmd/storage-rest_test.go +++ b/cmd/storage-rest_test.go @@ -508,6 +508,10 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES t.Fatalf("NewEndpoint failed %v", endpoint) } + if err := endpoint.UpdateIsLocal(); err != nil { + t.Fatalf("UpdateIsLocal failed %v", err) + } + registerStorageRESTHandlers(router, EndpointList{endpoint}) restClient, err := newStorageRESTClient(endpoint) if err != nil {