diff --git a/cmd/crypto/kes.go b/cmd/crypto/kes.go index b6add7e4d..536e6eee2 100644 --- a/cmd/crypto/kes.go +++ b/cmd/crypto/kes.go @@ -413,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io. } // If the error is not temp. / retryable => fail the request immediately. - if !xnet.IsNetworkOrHostDown(err) && + if !xnet.IsNetworkOrHostDown(err, false) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, context.DeadlineExceeded) { diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index b8fd38ec8..492a803d7 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool resp, err := clnt.Do(req) if err != nil { clnt.CloseIdleConnections() - return !xnet.IsNetworkOrHostDown(err) + return !xnet.IsNetworkOrHostDown(err, false) } xhttp.DrainBody(resp.Body) return true @@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error { object = params[1] } - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return BackendDown{} } diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go index 7211867fe..3c8a4bd6e 100644 --- a/cmd/lock-rest-client.go +++ b/cmd/lock-rest-client.go @@ -162,6 +162,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient { trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout) restClient := rest.NewClient(serverURL, trFn, newAuthToken) + restClient.ExpectTimeouts = true restClient.HealthCheckFn = func() bool { ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) // Instantiate a new rest client for healthcheck diff --git a/cmd/rest/client.go b/cmd/rest/client.go index 1fdb9960b..f74284145 100644 --- a/cmd/rest/client.go +++ b/cmd/rest/client.go @@ -77,6 +77,10 @@ type Client struct { // Should only be modified before any calls are made. MaxErrResponseSize int64 + // ExpectTimeouts indicates if context timeouts are expected. + // This will not mark the client offline in these cases. + ExpectTimeouts bool + httpClient *http.Client url *url.URL newAuthToken func(audience string) string @@ -114,7 +118,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod } resp, err := c.httpClient.Do(req) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) { logger.LogIf(ctx, err, "marking disk offline") c.MarkOffline() } @@ -144,7 +148,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod // Limit the ReadAll(), just in case, because of a bug, the server responds with large data. b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize)) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) { logger.LogIf(ctx, err, "marking disk offline") c.MarkOffline() } diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 61307bdc9..0237b722b 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -42,7 +42,7 @@ func isNetworkError(err error) bool { return false } if nerr, ok := err.(*rest.NetworkError); ok { - return xnet.IsNetworkOrHostDown(nerr.Err) + return xnet.IsNetworkOrHostDown(nerr.Err, false) } return false } diff --git a/cmd/update.go b/cmd/update.go index 2f768befa..0dc849e1c 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -309,7 +309,7 @@ func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content client := &http.Client{Transport: getUpdateTransport(timeout)} resp, err := client.Do(req) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return content, AdminError{ Code: AdminUpdateURLNotReachable, Message: err.Error(), @@ -501,7 +501,7 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string resp, err := clnt.Do(req) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return nil, AdminError{ Code: AdminUpdateURLNotReachable, Message: err.Error(), diff --git a/pkg/event/target/elasticsearch.go b/pkg/event/target/elasticsearch.go index b62d08671..05ec519af 100644 --- a/pkg/event/target/elasticsearch.go +++ b/pkg/event/target/elasticsearch.go @@ -124,7 +124,7 @@ func (target *ElasticsearchTarget) IsActive() (bool, error) { } _, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx) if err != nil { - if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { + if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) { return false, errNotConnected } return false, err @@ -138,7 +138,7 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error { return target.store.Put(eventData) } err := target.send(eventData) - if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { + if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) { return errNotConnected } return err @@ -214,7 +214,7 @@ func (target *ElasticsearchTarget) Send(eventKey string) error { } if err := target.send(eventData); err != nil { - if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { + if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) { return errNotConnected } return err @@ -267,7 +267,7 @@ func newClient(args ElasticsearchArgs) (*elastic.Client, error) { client, err := elastic.NewClient(options...) if err != nil { // https://github.com/olivere/elastic/wiki/Connection-Errors - if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { + if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) { return nil, errNotConnected } return nil, err diff --git a/pkg/event/target/webhook.go b/pkg/event/target/webhook.go index e79215a22..b35cbe57d 100644 --- a/pkg/event/target/webhook.go +++ b/pkg/event/target/webhook.go @@ -111,7 +111,7 @@ func (target *WebhookTarget) IsActive() (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return false, errNotConnected } return false, err @@ -119,7 +119,7 @@ func (target *WebhookTarget) IsActive() (bool, error) { resp, err := target.httpClient.Do(req) if err != nil { - if xnet.IsNetworkOrHostDown(err) || errors.Is(err, context.DeadlineExceeded) { + if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) { return false, errNotConnected } return false, err @@ -137,7 +137,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error { } err := target.send(eventData) if err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return errNotConnected } } @@ -197,7 +197,7 @@ func (target *WebhookTarget) Send(eventKey string) error { } if err := target.send(eventData); err != nil { - if xnet.IsNetworkOrHostDown(err) { + if xnet.IsNetworkOrHostDown(err, false) { return errNotConnected } return err diff --git a/pkg/net/url.go b/pkg/net/url.go index 283bf8acf..8aa3208e4 100644 --- a/pkg/net/url.go +++ b/pkg/net/url.go @@ -141,14 +141,18 @@ func ParseURL(s string) (u *URL, err error) { } // IsNetworkOrHostDown - if there was a network error or if the host is down. -func IsNetworkOrHostDown(err error) bool { +// expectTimeouts indicates that context timeouts are expected and does not +// indicate a downed host. Other timeouts still returns down. +func IsNetworkOrHostDown(err error, expectTimeouts bool) bool { if err == nil { return false } if errors.Is(err, context.Canceled) { return false } - + if expectTimeouts && errors.Is(err, context.DeadlineExceeded) { + return false + } // We need to figure if the error either a timeout // or a non-temporary error. e, ok := err.(net.Error)