From d140074773a988a984264f720d48b2f869d97cd5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 19 Dec 2019 13:45:56 -0800 Subject: [PATCH] fix: replica set deployment for multi tenants (#8673) Changes in IP underneath are dynamic in replica sets with multiple tenants, so deploying in that fashion will not work until we wait for atleast one participatory server to be local. This PR also ensures that multi-tenant zone expansion also works in replica set k8s deployments. Introduces a new ENV `KUBERNETES_REPLICA_SET` check to call appropriate code paths. --- cmd/common-main.go | 2 +- cmd/endpoint-ellipses.go | 11 ++++++++--- cmd/endpoint.go | 37 +++++++++++++++++++++++++++++++------ cmd/endpoint_test.go | 6 +++--- cmd/net.go | 8 ++++---- cmd/notification.go | 4 ++++ cmd/update.go | 5 +++++ 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/cmd/common-main.go b/cmd/common-main.go index 23f890193..2d2e9e248 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -195,7 +195,7 @@ func handleCommonEnvVars() { } else { // Add found interfaces IP address to global domain IPS, // loopback addresses will be naturally dropped. - updateDomainIPs(localIP4) + updateDomainIPs(mustGetLocalIP4()) } // In place update is true by default if the MINIO_UPDATE is not set diff --git a/cmd/endpoint-ellipses.go b/cmd/endpoint-ellipses.go index 9e5f816fe..bd07d04da 100644 --- a/cmd/endpoint-ellipses.go +++ b/cmd/endpoint-ellipses.go @@ -270,7 +270,7 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in if err != nil { return nil, -1, -1, err } - endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...) + endpointList, newSetupType, err := CreateEndpoints(serverAddr, false, setArgs...) if err != nil { return nil, -1, -1, err } @@ -283,6 +283,8 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in return endpointZones, len(setArgs[0]), setupType, nil } + var foundPrevLocal bool + // Verify the args setup-type appropriately. { setArgs, err := GetAllSets(args...) @@ -290,10 +292,13 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in return nil, -1, -1, err } - _, setupType, err = CreateEndpoints(serverAddr, setArgs...) + var endpoints Endpoints + endpoints, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...) if err != nil { return nil, -1, -1, err } + + foundPrevLocal = endpoints.atleastOneEndpointLocal() } for _, arg := range args { @@ -301,7 +306,7 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in if err != nil { return nil, -1, -1, err } - endpointList, _, err := CreateEndpoints(serverAddr, setArgs...) + endpointList, _, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...) if err != nil { return nil, -1, -1, err } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 1f419f05d..52f8a4dd7 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -249,9 +249,19 @@ func (endpoints Endpoints) doAnyHostsResolveToLocalhost() bool { return false } +func (endpoints Endpoints) atleastOneEndpointLocal() bool { + for _, endpoint := range endpoints { + if endpoint.IsLocal { + return true + } + } + return false +} + // UpdateIsLocal - resolves the host and discovers the local host. -func (endpoints Endpoints) UpdateIsLocal() error { +func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error { orchestrated := IsDocker() || IsKubernetes() + k8sReplicaSet := IsKubernetesReplicaSet() var epsResolved int var foundLocal bool @@ -284,8 +294,8 @@ func (endpoints Endpoints) UpdateIsLocal() error { endpoints[i].Hostname(), ) - if orchestrated && endpoints.doAnyHostsResolveToLocalhost() { - err := errors.New("hosts resolve 127.*, DNS not updated on k8s") + if k8sReplicaSet && endpoints.doAnyHostsResolveToLocalhost() { + err := errors.New("host found resolves to 127.*, DNS incorrectly configured retrying") // time elapsed timeElapsed := time.Since(startTime) // log error only if more than 1s elapsed @@ -329,6 +339,21 @@ func (endpoints Endpoints) UpdateIsLocal() error { } else { resolvedList[i] = true endpoints[i].IsLocal = isLocal + if k8sReplicaSet && !endpoints.atleastOneEndpointLocal() && !foundPrevLocal { + // In replicated set in k8s deployment, IPs might + // get resolved for older IPs, add this code + // to ensure that we wait for this server to + // participate atleast one disk and be local. + // + // In special cases for replica set with expanded + // zone setups we need to make sure to provide + // value of foundPrevLocal from zone1 if we already + // found a local setup. Only if we haven't found + // previous local we continue to wait to look for + // atleast one local. + resolvedList[i] = false + continue + } epsResolved++ if !foundLocal { foundLocal = isLocal @@ -439,7 +464,7 @@ func checkCrossDeviceMounts(endpoints Endpoints) (err error) { } // CreateEndpoints - validates and creates new endpoints for given args. -func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) { +func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endpoints, SetupType, error) { var endpoints Endpoints var setupType SetupType var err error @@ -505,7 +530,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, return endpoints, setupType, nil } - if err = endpoints.UpdateIsLocal(); err != nil { + if err = endpoints.UpdateIsLocal(foundLocal); err != nil { return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } @@ -695,6 +720,6 @@ func updateDomainIPs(endPoints set.StringSet) { if err != nil { host = ip } - return !net.ParseIP(host).IsLoopback() + return !net.ParseIP(host).IsLoopback() && host != "localhost" }, "") } diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go index a8b7c0acb..9a0fc780e 100644 --- a/cmd/endpoint_test.go +++ b/cmd/endpoint_test.go @@ -341,7 +341,7 @@ func TestCreateEndpoints(t *testing.T) { for _, testCase := range testCases { testCase := testCase t.Run("", func(t *testing.T) { - endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...) + endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, false, testCase.args...) if err == nil && testCase.expectedErr != nil { t.Errorf("error: expected = %v, got = ", testCase.expectedErr) } @@ -398,7 +398,7 @@ func TestGetLocalPeer(t *testing.T) { for i, testCase := range testCases { zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...) if !zendpoints[0].Endpoints[0].IsLocal { - if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil { t.Fatalf("error: expected = , got = %v", err) } } @@ -430,7 +430,7 @@ func TestGetRemotePeers(t *testing.T) { for _, testCase := range testCases { zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...) if !zendpoints[0].Endpoints[0].IsLocal { - if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil { t.Fatalf("error: expected = , got = %v", err) } } diff --git a/cmd/net.go b/cmd/net.go index 2287ed74d..a20ccad1f 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -162,8 +162,8 @@ func sortIPs(ipList []string) []string { func getAPIEndpoints() (apiEndpoints []string) { var ipList []string if globalMinioHost == "" { - ipList = sortIPs(localIP4.ToSlice()) - ipList = append(ipList, localIP6.ToSlice()...) + ipList = sortIPs(mustGetLocalIP4().ToSlice()) + ipList = append(ipList, mustGetLocalIP6().ToSlice()...) } else { ipList = []string{globalMinioHost} } @@ -278,8 +278,8 @@ func isLocalHost(host string, port string, localPort string) (bool, error) { } // If intersection of two IP sets is not empty, then the host is localhost. - isLocalv4 := !localIP4.Intersection(hostIPs).IsEmpty() - isLocalv6 := !localIP6.Intersection(hostIPs).IsEmpty() + isLocalv4 := !mustGetLocalIP4().Intersection(hostIPs).IsEmpty() + isLocalv6 := !mustGetLocalIP6().Intersection(hostIPs).IsEmpty() if port != "" { return (isLocalv4 || isLocalv6) && (port == localPort), nil } diff --git a/cmd/notification.go b/cmd/notification.go index e1cdd2381..ab6e002d2 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -798,6 +798,10 @@ func (sys *NotificationSys) RemoveRulesMap(bucketName string, rulesMap event.Rul // ConfiguredTargetIDs - returns list of configured target id's func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID { + if sys == nil { + return nil + } + sys.RLock() defer sys.RUnlock() diff --git a/cmd/update.go b/cmd/update.go index 4854f3c5d..af01f63b3 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -154,6 +154,11 @@ func IsDCOS() bool { return false } +// IsKubernetesReplicaSet returns true if minio is running in kubernetes replica set. +func IsKubernetesReplicaSet() bool { + return IsKubernetes() && (env.Get("KUBERNETES_REPLICA_SET", "") != "") +} + // IsKubernetes returns true if minio is running in kubernetes. func IsKubernetes() bool { if env.Get("SIMPLE_CI", "") == "" {