diff --git a/cmd/common-main.go b/cmd/common-main.go index 23f890193..2d2e9e248 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -195,7 +195,7 @@ func handleCommonEnvVars() { } else { // Add found interfaces IP address to global domain IPS, // loopback addresses will be naturally dropped. - updateDomainIPs(localIP4) + updateDomainIPs(mustGetLocalIP4()) } // In place update is true by default if the MINIO_UPDATE is not set diff --git a/cmd/endpoint-ellipses.go b/cmd/endpoint-ellipses.go index 9e5f816fe..bd07d04da 100644 --- a/cmd/endpoint-ellipses.go +++ b/cmd/endpoint-ellipses.go @@ -270,7 +270,7 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in if err != nil { return nil, -1, -1, err } - endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...) + endpointList, newSetupType, err := CreateEndpoints(serverAddr, false, setArgs...) if err != nil { return nil, -1, -1, err } @@ -283,6 +283,8 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in return endpointZones, len(setArgs[0]), setupType, nil } + var foundPrevLocal bool + // Verify the args setup-type appropriately. { setArgs, err := GetAllSets(args...) @@ -290,10 +292,13 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in return nil, -1, -1, err } - _, setupType, err = CreateEndpoints(serverAddr, setArgs...) + var endpoints Endpoints + endpoints, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...) if err != nil { return nil, -1, -1, err } + + foundPrevLocal = endpoints.atleastOneEndpointLocal() } for _, arg := range args { @@ -301,7 +306,7 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in if err != nil { return nil, -1, -1, err } - endpointList, _, err := CreateEndpoints(serverAddr, setArgs...) + endpointList, _, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...) if err != nil { return nil, -1, -1, err } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 1f419f05d..52f8a4dd7 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -249,9 +249,19 @@ func (endpoints Endpoints) doAnyHostsResolveToLocalhost() bool { return false } +func (endpoints Endpoints) atleastOneEndpointLocal() bool { + for _, endpoint := range endpoints { + if endpoint.IsLocal { + return true + } + } + return false +} + // UpdateIsLocal - resolves the host and discovers the local host. -func (endpoints Endpoints) UpdateIsLocal() error { +func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error { orchestrated := IsDocker() || IsKubernetes() + k8sReplicaSet := IsKubernetesReplicaSet() var epsResolved int var foundLocal bool @@ -284,8 +294,8 @@ func (endpoints Endpoints) UpdateIsLocal() error { endpoints[i].Hostname(), ) - if orchestrated && endpoints.doAnyHostsResolveToLocalhost() { - err := errors.New("hosts resolve 127.*, DNS not updated on k8s") + if k8sReplicaSet && endpoints.doAnyHostsResolveToLocalhost() { + err := errors.New("host found resolves to 127.*, DNS incorrectly configured retrying") // time elapsed timeElapsed := time.Since(startTime) // log error only if more than 1s elapsed @@ -329,6 +339,21 @@ func (endpoints Endpoints) UpdateIsLocal() error { } else { resolvedList[i] = true endpoints[i].IsLocal = isLocal + if k8sReplicaSet && !endpoints.atleastOneEndpointLocal() && !foundPrevLocal { + // In replicated set in k8s deployment, IPs might + // get resolved for older IPs, add this code + // to ensure that we wait for this server to + // participate atleast one disk and be local. + // + // In special cases for replica set with expanded + // zone setups we need to make sure to provide + // value of foundPrevLocal from zone1 if we already + // found a local setup. Only if we haven't found + // previous local we continue to wait to look for + // atleast one local. + resolvedList[i] = false + continue + } epsResolved++ if !foundLocal { foundLocal = isLocal @@ -439,7 +464,7 @@ func checkCrossDeviceMounts(endpoints Endpoints) (err error) { } // CreateEndpoints - validates and creates new endpoints for given args. -func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) { +func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endpoints, SetupType, error) { var endpoints Endpoints var setupType SetupType var err error @@ -505,7 +530,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, return endpoints, setupType, nil } - if err = endpoints.UpdateIsLocal(); err != nil { + if err = endpoints.UpdateIsLocal(foundLocal); err != nil { return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } @@ -695,6 +720,6 @@ func updateDomainIPs(endPoints set.StringSet) { if err != nil { host = ip } - return !net.ParseIP(host).IsLoopback() + return !net.ParseIP(host).IsLoopback() && host != "localhost" }, "") } diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go index a8b7c0acb..9a0fc780e 100644 --- a/cmd/endpoint_test.go +++ b/cmd/endpoint_test.go @@ -341,7 +341,7 @@ func TestCreateEndpoints(t *testing.T) { for _, testCase := range testCases { testCase := testCase t.Run("", func(t *testing.T) { - endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...) + endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, false, testCase.args...) if err == nil && testCase.expectedErr != nil { t.Errorf("error: expected = %v, got = ", testCase.expectedErr) } @@ -398,7 +398,7 @@ func TestGetLocalPeer(t *testing.T) { for i, testCase := range testCases { zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...) if !zendpoints[0].Endpoints[0].IsLocal { - if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil { t.Fatalf("error: expected = , got = %v", err) } } @@ -430,7 +430,7 @@ func TestGetRemotePeers(t *testing.T) { for _, testCase := range testCases { zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...) if !zendpoints[0].Endpoints[0].IsLocal { - if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil { t.Fatalf("error: expected = , got = %v", err) } } diff --git a/cmd/net.go b/cmd/net.go index 2287ed74d..a20ccad1f 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -162,8 +162,8 @@ func sortIPs(ipList []string) []string { func getAPIEndpoints() (apiEndpoints []string) { var ipList []string if globalMinioHost == "" { - ipList = sortIPs(localIP4.ToSlice()) - ipList = append(ipList, localIP6.ToSlice()...) + ipList = sortIPs(mustGetLocalIP4().ToSlice()) + ipList = append(ipList, mustGetLocalIP6().ToSlice()...) } else { ipList = []string{globalMinioHost} } @@ -278,8 +278,8 @@ func isLocalHost(host string, port string, localPort string) (bool, error) { } // If intersection of two IP sets is not empty, then the host is localhost. - isLocalv4 := !localIP4.Intersection(hostIPs).IsEmpty() - isLocalv6 := !localIP6.Intersection(hostIPs).IsEmpty() + isLocalv4 := !mustGetLocalIP4().Intersection(hostIPs).IsEmpty() + isLocalv6 := !mustGetLocalIP6().Intersection(hostIPs).IsEmpty() if port != "" { return (isLocalv4 || isLocalv6) && (port == localPort), nil } diff --git a/cmd/notification.go b/cmd/notification.go index e1cdd2381..ab6e002d2 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -798,6 +798,10 @@ func (sys *NotificationSys) RemoveRulesMap(bucketName string, rulesMap event.Rul // ConfiguredTargetIDs - returns list of configured target id's func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID { + if sys == nil { + return nil + } + sys.RLock() defer sys.RUnlock() diff --git a/cmd/update.go b/cmd/update.go index 4854f3c5d..af01f63b3 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -154,6 +154,11 @@ func IsDCOS() bool { return false } +// IsKubernetesReplicaSet returns true if minio is running in kubernetes replica set. +func IsKubernetesReplicaSet() bool { + return IsKubernetes() && (env.Get("KUBERNETES_REPLICA_SET", "") != "") +} + // IsKubernetes returns true if minio is running in kubernetes. func IsKubernetes() bool { if env.Get("SIMPLE_CI", "") == "" {