Handle connection failures on webhook/url pings (#8204)

Properly handle connection failures while replaying events

Fixes #8194

cmd/gateway-common.go

@@ -24,6 +24,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
+	xnet "github.com/minio/minio/pkg/net"
 
 	minio "github.com/minio/minio-go/v6"
 )
@@ -299,7 +300,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if isNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err) {
 		return BackendDown{}
 	}
 

cmd/storage-rest-client.go

@@ -45,7 +45,7 @@ func isNetworkError(err error) bool {
 		return true
 	}
 	if nerr, ok := err.(*rest.NetworkError); ok {
-		return isNetworkOrHostDown(nerr.Err)
+		return xnet.IsNetworkOrHostDown(nerr.Err)
 	}
 	return false
 }

cmd/update.go

@@ -35,6 +35,7 @@ import (
 	"github.com/inconshreveable/go-update"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
 	_ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier.
 )
 
@@ -283,7 +284,7 @@ func downloadReleaseURL(releaseChecksumURL string, timeout time.Duration, mode s
 	client := &http.Client{Transport: getUpdateTransport(timeout)}
 	resp, err := client.Do(req)
 	if err != nil {
-		if isNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err) {
 			return content, AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),
@@ -500,7 +501,7 @@ func doUpdate(updateURL, sha256Hex, mode string) (err error) {
 
 	resp, err := clnt.Do(req)
 	if err != nil {
-		if isNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err) {
 			return AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),

cmd/utils.go

@@ -29,7 +29,6 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -426,43 +425,6 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 	return logger.SetReqInfo(r.Context(), reqInfo)
 }
 
-// isNetworkOrHostDown - if there was a network error or if the host is down.
-func isNetworkOrHostDown(err error) bool {
-	if err == nil {
-		return false
-	}
-	// We need to figure if the error either a timeout
-	// or a non-temporary error.
-	e, ok := err.(net.Error)
-	if ok {
-		urlErr, ok := e.(*url.Error)
-		if ok {
-			switch urlErr.Err.(type) {
-			case *net.DNSError, *net.OpError, net.UnknownNetworkError:
-				return true
-			}
-		}
-		if e.Timeout() {
-			return true
-		}
-	}
-	ok = false
-	// Fallback to other mechanisms.
-	if strings.Contains(err.Error(), "Connection closed by foreign host") {
-		ok = true
-	} else if strings.Contains(err.Error(), "TLS handshake timeout") {
-		// If error is - tlsHandshakeTimeoutError.
-		ok = true
-	} else if strings.Contains(err.Error(), "i/o timeout") {
-		// If error is - tcp timeoutError.
-		ok = true
-	} else if strings.Contains(err.Error(), "connection timed out") {
-		// If err is a net.Dial timeout.
-		ok = true
-	}
-	return ok
-}
-
 // Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
 // If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
 func restQueries(keys ...string) []string {

pkg/event/target/elasticsearch.go

@@ -83,11 +83,8 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
 		return target.store.Put(eventData)
 	}
 	if dErr := target.args.URL.DialHTTP(); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
+		if xnet.IsNetworkOrHostDown(dErr) {
-			// To treat "connection refused" errors as errNotConnected.
+			return errNotConnected
-			if IsConnRefusedErr(urlErr.Err) {
-				return errNotConnected
-			}
 		}
 		return dErr
 	}
@@ -157,11 +154,8 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
 	}
 
 	if dErr := target.args.URL.DialHTTP(); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
+		if xnet.IsNetworkOrHostDown(dErr) {
-			// To treat "connection refused" errors as errNotConnected.
+			return errNotConnected
-			if IsConnRefusedErr(urlErr.Err) {
-				return errNotConnected
-			}
 		}
 		return dErr
 	}

pkg/event/target/webhook.go

@@ -87,11 +87,8 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
 		return pErr
 	}
 	if dErr := u.DialHTTP(); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
+		if xnet.IsNetworkOrHostDown(dErr) {
-			// To treat "connection refused" errors as errNotConnected.
+			return errNotConnected
-			if IsConnRefusedErr(urlErr.Err) {
-				return errNotConnected
-			}
 		}
 		return dErr
 	}
@@ -136,17 +133,13 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 
 // Send - reads an event from store and sends it to webhook.
 func (target *WebhookTarget) Send(eventKey string) error {
-
 	u, pErr := xnet.ParseURL(target.args.Endpoint.String())
 	if pErr != nil {
 		return pErr
 	}
 	if dErr := u.DialHTTP(); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
+		if xnet.IsNetworkOrHostDown(dErr) {
-			// To treat "connection refused" errors as errNotConnected.
+			return errNotConnected
-			if IsConnRefusedErr(urlErr.Err) {
-				return errNotConnected
-			}
 		}
 		return dErr
 	}

pkg/net/url.go

@@ -134,3 +134,43 @@ func ParseURL(s string) (u *URL, err error) {
 	u = &v
 	return u, nil
 }
+
+// IsNetworkOrHostDown - if there was a network error or if the host is down.
+func IsNetworkOrHostDown(err error) bool {
+	if err == nil {
+		return false
+	}
+	// We need to figure if the error either a timeout
+	// or a non-temporary error.
+	e, ok := err.(net.Error)
+	if ok {
+		urlErr, ok := e.(*url.Error)
+		if ok {
+			switch urlErr.Err.(type) {
+			case *net.DNSError, *net.OpError, net.UnknownNetworkError:
+				return true
+			}
+		}
+		if e.Timeout() {
+			return true
+		}
+	}
+	ok = false
+	// Fallback to other mechanisms.
+	if strings.Contains(err.Error(), "Connection closed by foreign host") {
+		ok = true
+	} else if strings.Contains(err.Error(), "TLS handshake timeout") {
+		// If error is - tlsHandshakeTimeoutError.
+		ok = true
+	} else if strings.Contains(err.Error(), "i/o timeout") {
+		// If error is - tcp timeoutError.
+		ok = true
+	} else if strings.Contains(err.Error(), "connection timed out") {
+		// If err is a net.Dial timeout.
+		ok = true
+	} else if strings.Contains(strings.ToLower(err.Error()), "503 service unavailable") {
+		// Denial errors
+		ok = true
+	}
+	return ok
+}