From 5cd9f10a02820f5c87512881542e89041c19293d Mon Sep 17 00:00:00 2001 From: maihde Date: Fri, 2 Aug 2019 15:40:51 -0400 Subject: [PATCH] Support Federation on a single machine (#8009) When checking if federation is necessary, the code compares the SRV record stored in etcd against the list of endpoints that the MinIO server is exposing. If there is an intersection in this list the request is forwarded. The SRV record includes both the host and the port, but the intersection check previously only looked at the IP address. This would prevent federation from working in situations where the endpoint IP is the same for multiple MinIO servers. Some examples of where this can occur are: - running mulitiple copies of MinIO on the same host - using multiple MinIO servers behind a NAT with port-forwarding --- cmd/endpoint.go | 17 +++++++++++++---- cmd/endpoint_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ cmd/object-api-utils.go | 2 +- pkg/dns/etcd_dns.go | 14 +++++++++++++- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/cmd/endpoint.go b/cmd/endpoint.go index d7ec636fe..e5f0b3e02 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -720,18 +720,27 @@ func GetRemotePeers(endpoints EndpointList) []string { func updateDomainIPs(endPoints set.StringSet) { ipList := set.NewStringSet() for e := range endPoints { - host, _, err := net.SplitHostPort(e) + host, port, err := net.SplitHostPort(e) if err != nil { if strings.Contains(err.Error(), "missing port in address") { host = e + port = globalMinioPort } else { continue } } - IPs, _ := getHostIP(host) - ipList = ipList.Union(IPs) + IPs, err := getHostIP(host) + if err != nil { + continue + } + + IPsWithPort := IPs.ApplyFunc(func(ip string) string { + return net.JoinHostPort(ip, port) + }) + + ipList = ipList.Union(IPsWithPort) } globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool { - return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") + return !(strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") || strings.HasPrefix(ip, "[::1]")) }, "") } diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go index e684447aa..093896b8c 100644 --- a/cmd/endpoint_test.go +++ b/cmd/endpoint_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/minio/cli" + "github.com/minio/minio-go/v6/pkg/set" ) func TestSubOptimalEndpointInput(t *testing.T) { @@ -444,3 +445,43 @@ func TestGetRemotePeers(t *testing.T) { } } } + +func TestUpdateDomainIPs(t *testing.T) { + tempGlobalMinioPort := globalMinioPort + defer func() { + globalMinioPort = tempGlobalMinioPort + }() + globalMinioPort = "9000" + + tempGlobalDomainIPs := globalDomainIPs + defer func() { + globalDomainIPs = tempGlobalDomainIPs + }() + + ipv4TestCases := []struct { + endPoints set.StringSet + expectedResult set.StringSet + }{ + {set.NewStringSet(), set.NewStringSet()}, + {set.CreateStringSet("localhost"), set.NewStringSet()}, + {set.CreateStringSet("localhost", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")}, + {set.CreateStringSet("localhost:9001", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")}, + {set.CreateStringSet("localhost", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")}, + {set.CreateStringSet("localhost:9000", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")}, + + {set.CreateStringSet("10.0.0.1", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9000")}, + {set.CreateStringSet("10.0.0.1:9001", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9000")}, + {set.CreateStringSet("10.0.0.1", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9002")}, + {set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002")}, + } + + for _, testCase := range ipv4TestCases { + globalDomainIPs = nil + + updateDomainIPs(testCase.endPoints) + + if !testCase.expectedResult.Equals(globalDomainIPs) { + t.Fatalf("error: expected = %s, got = %s", testCase.expectedResult, globalDomainIPs) + } + } +} diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 7810b6721..1525241fb 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -319,7 +319,7 @@ func isMinioReservedBucket(bucketName string) bool { func getHostsSlice(records []dns.SrvRecord) []string { var hosts []string for _, r := range records { - hosts = append(hosts, r.Host) + hosts = append(hosts, net.JoinHostPort(r.Host, fmt.Sprintf("%d", r.Port))) } return hosts } diff --git a/pkg/dns/etcd_dns.go b/pkg/dns/etcd_dns.go index 44802c0a8..91e1ea90d 100644 --- a/pkg/dns/etcd_dns.go +++ b/pkg/dns/etcd_dns.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "sort" "strconv" "strings" @@ -215,9 +216,20 @@ func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string return nil, err } + // strip ports off of domainIPs + domainIPsWithoutPorts := domainIPs.ApplyFunc(func(ip string) string { + host, _, err := net.SplitHostPort(ip) + if err != nil { + if strings.Contains(err.Error(), "missing port in address") { + host = ip + } + } + return host + }) + return &coreDNS{ domainNames: domainNames, - domainIPs: domainIPs, + domainIPs: domainIPsWithoutPorts, domainPort: port, etcdClient: etcdClient, }, nil