rest client: Expect context timeouts for locks (#10782)

Add option for rest clients to not mark a remote offline for context timeouts.

This can be used if context timeouts are expected on the call.
master
Klaus Post 4 years ago committed by GitHub
parent 6b14c4ab1e
commit e63a44b734
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      cmd/crypto/kes.go
  2. 4
      cmd/gateway-common.go
  3. 1
      cmd/lock-rest-client.go
  4. 8
      cmd/rest/client.go
  5. 2
      cmd/storage-rest-client.go
  6. 4
      cmd/update.go
  7. 8
      pkg/event/target/elasticsearch.go
  8. 8
      pkg/event/target/webhook.go
  9. 8
      pkg/net/url.go

@ -413,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
} }
// If the error is not temp. / retryable => fail the request immediately. // If the error is not temp. / retryable => fail the request immediately.
if !xnet.IsNetworkOrHostDown(err) && if !xnet.IsNetworkOrHostDown(err, false) &&
!errors.Is(err, io.EOF) && !errors.Is(err, io.EOF) &&
!errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.ErrUnexpectedEOF) &&
!errors.Is(err, context.DeadlineExceeded) { !errors.Is(err, context.DeadlineExceeded) {

@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
resp, err := clnt.Do(req) resp, err := clnt.Do(req)
if err != nil { if err != nil {
clnt.CloseIdleConnections() clnt.CloseIdleConnections()
return !xnet.IsNetworkOrHostDown(err) return !xnet.IsNetworkOrHostDown(err, false)
} }
xhttp.DrainBody(resp.Body) xhttp.DrainBody(resp.Body)
return true return true
@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
object = params[1] object = params[1]
} }
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return BackendDown{} return BackendDown{}
} }

@ -162,6 +162,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout) trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
restClient := rest.NewClient(serverURL, trFn, newAuthToken) restClient := rest.NewClient(serverURL, trFn, newAuthToken)
restClient.ExpectTimeouts = true
restClient.HealthCheckFn = func() bool { restClient.HealthCheckFn = func() bool {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck

@ -77,6 +77,10 @@ type Client struct {
// Should only be modified before any calls are made. // Should only be modified before any calls are made.
MaxErrResponseSize int64 MaxErrResponseSize int64
// ExpectTimeouts indicates if context timeouts are expected.
// This will not mark the client offline in these cases.
ExpectTimeouts bool
httpClient *http.Client httpClient *http.Client
url *url.URL url *url.URL
newAuthToken func(audience string) string newAuthToken func(audience string) string
@ -114,7 +118,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
} }
resp, err := c.httpClient.Do(req) resp, err := c.httpClient.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline") logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline() c.MarkOffline()
} }
@ -144,7 +148,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
// Limit the ReadAll(), just in case, because of a bug, the server responds with large data. // Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize)) b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline") logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline() c.MarkOffline()
} }

@ -42,7 +42,7 @@ func isNetworkError(err error) bool {
return false return false
} }
if nerr, ok := err.(*rest.NetworkError); ok { if nerr, ok := err.(*rest.NetworkError); ok {
return xnet.IsNetworkOrHostDown(nerr.Err) return xnet.IsNetworkOrHostDown(nerr.Err, false)
} }
return false return false
} }

@ -309,7 +309,7 @@ func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content
client := &http.Client{Transport: getUpdateTransport(timeout)} client := &http.Client{Transport: getUpdateTransport(timeout)}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return content, AdminError{ return content, AdminError{
Code: AdminUpdateURLNotReachable, Code: AdminUpdateURLNotReachable,
Message: err.Error(), Message: err.Error(),
@ -501,7 +501,7 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string
resp, err := clnt.Do(req) resp, err := clnt.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return nil, AdminError{ return nil, AdminError{
Code: AdminUpdateURLNotReachable, Code: AdminUpdateURLNotReachable,
Message: err.Error(), Message: err.Error(),

@ -124,7 +124,7 @@ func (target *ElasticsearchTarget) IsActive() (bool, error) {
} }
_, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx) _, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx)
if err != nil { if err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -138,7 +138,7 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
return target.store.Put(eventData) return target.store.Put(eventData)
} }
err := target.send(eventData) err := target.send(eventData)
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err
@ -214,7 +214,7 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
} }
if err := target.send(eventData); err != nil { if err := target.send(eventData); err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err
@ -267,7 +267,7 @@ func newClient(args ElasticsearchArgs) (*elastic.Client, error) {
client, err := elastic.NewClient(options...) client, err := elastic.NewClient(options...)
if err != nil { if err != nil {
// https://github.com/olivere/elastic/wiki/Connection-Errors // https://github.com/olivere/elastic/wiki/Connection-Errors
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return nil, errNotConnected return nil, errNotConnected
} }
return nil, err return nil, err

@ -111,7 +111,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil) req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -119,7 +119,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
resp, err := target.httpClient.Do(req) resp, err := target.httpClient.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) || errors.Is(err, context.DeadlineExceeded) { if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -137,7 +137,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
} }
err := target.send(eventData) err := target.send(eventData)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
} }
@ -197,7 +197,7 @@ func (target *WebhookTarget) Send(eventKey string) error {
} }
if err := target.send(eventData); err != nil { if err := target.send(eventData); err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err

@ -141,14 +141,18 @@ func ParseURL(s string) (u *URL, err error) {
} }
// IsNetworkOrHostDown - if there was a network error or if the host is down. // IsNetworkOrHostDown - if there was a network error or if the host is down.
func IsNetworkOrHostDown(err error) bool { // expectTimeouts indicates that context timeouts are expected and does not
// indicate a downed host. Other timeouts still returns down.
func IsNetworkOrHostDown(err error, expectTimeouts bool) bool {
if err == nil { if err == nil {
return false return false
} }
if errors.Is(err, context.Canceled) { if errors.Is(err, context.Canceled) {
return false return false
} }
if expectTimeouts && errors.Is(err, context.DeadlineExceeded) {
return false
}
// We need to figure if the error either a timeout // We need to figure if the error either a timeout
// or a non-temporary error. // or a non-temporary error.
e, ok := err.(net.Error) e, ok := err.(net.Error)

Loading…
Cancel
Save