From 70136fb55b3bee343d4914a86aba74124b363cd4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 25 Aug 2019 13:32:49 -0700 Subject: [PATCH] Look for network errors appropriately for RemoteStorageAPI (#8128) net.Error is very unreliable in providing better error handling, we need to ensure that we always have a fallback option in case of network failures. This fixes an important issue in our distributed server setups when one of the servers is down, all deployments out there are recommended to upgrade after this fix is merged to ensure that availability is not lost. Fixes #8127 Fixes #8016 Fixes #7964 --- cmd/utils.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/utils.go b/cmd/utils.go index 996013177..207e20822 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -29,6 +29,7 @@ import ( "io/ioutil" "net" "net/http" + "net/url" "os" "path/filepath" "reflect" @@ -446,10 +447,25 @@ func isNetworkOrHostDown(err error) bool { // or a non-temporary error. e, ok := err.(net.Error) if ok { - return e.Timeout() + urlErr, ok := e.(*url.Error) + if ok { + switch urlErr.Err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } + } + if e.Timeout() { + return true + } } + ok = false // Fallback to other mechanisms. - if strings.Contains(err.Error(), "i/o timeout") { + if strings.Contains(err.Error(), "Connection closed by foreign host") { + ok = true + } else if strings.Contains(err.Error(), "TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError. + ok = true + } else if strings.Contains(err.Error(), "i/o timeout") { // If error is - tcp timeoutError. ok = true } else if strings.Contains(err.Error(), "connection timed out") {