oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rest client: Expect context timeouts for locks (#10782)

Browse Source 
Add option for rest clients to not mark a remote offline for context timeouts.

This can be used if context timeouts are expected on the call.
master
Klaus Post 4 years ago committed by GitHub
parent 6b14c4ab1e
commit e63a44b734
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 27 additions and 18 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      cmd/crypto/kes.go
  2. 4
      cmd/gateway-common.go
  3. 1
      cmd/lock-rest-client.go
  4. 8
      cmd/rest/client.go
  5. 2
      cmd/storage-rest-client.go
  6. 4
      cmd/update.go
  7. 8
      pkg/event/target/elasticsearch.go
  8. 8
      pkg/event/target/webhook.go
  9. 8
      pkg/net/url.go

2
cmd/crypto/kes.go
Unescape View File

@ -413,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
} }
// If the error is not temp. / retryable => fail the request immediately. // If the error is not temp. / retryable => fail the request immediately.
if !xnet.IsNetworkOrHostDown(err) && if !xnet.IsNetworkOrHostDown(err, false) &&
!errors.Is(err, io.EOF) && !errors.Is(err, io.EOF) &&
!errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.ErrUnexpectedEOF) &&
!errors.Is(err, context.DeadlineExceeded) { !errors.Is(err, context.DeadlineExceeded) {

4
cmd/gateway-common.go
Unescape View File

@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
resp, err := clnt.Do(req) resp, err := clnt.Do(req)
if err != nil { if err != nil {
clnt.CloseIdleConnections() clnt.CloseIdleConnections()
return !xnet.IsNetworkOrHostDown(err) return !xnet.IsNetworkOrHostDown(err, false)
} }
xhttp.DrainBody(resp.Body) xhttp.DrainBody(resp.Body)
return true return true
@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
object = params[1] object = params[1]
} }
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return BackendDown{} return BackendDown{}
} }

1
cmd/lock-rest-client.go
Unescape View File

@ -162,6 +162,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout) trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
restClient := rest.NewClient(serverURL, trFn, newAuthToken) restClient := rest.NewClient(serverURL, trFn, newAuthToken)
restClient.ExpectTimeouts = true
restClient.HealthCheckFn = func() bool { restClient.HealthCheckFn = func() bool {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck

8
cmd/rest/client.go
Unescape View File

@ -77,6 +77,10 @@ type Client struct {
// Should only be modified before any calls are made. // Should only be modified before any calls are made.
MaxErrResponseSize int64 MaxErrResponseSize int64
// ExpectTimeouts indicates if context timeouts are expected.
// This will not mark the client offline in these cases.
ExpectTimeouts bool
httpClient *http.Client httpClient *http.Client
url *url.URL url *url.URL
newAuthToken func(audience string) string newAuthToken func(audience string) string
@ -114,7 +118,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
} }
resp, err := c.httpClient.Do(req) resp, err := c.httpClient.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline") logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline() c.MarkOffline()
} }
@ -144,7 +148,7 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
// Limit the ReadAll(), just in case, because of a bug, the server responds with large data. // Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize)) b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
logger.LogIf(ctx, err, "marking disk offline") logger.LogIf(ctx, err, "marking disk offline")
c.MarkOffline() c.MarkOffline()
} }

2
cmd/storage-rest-client.go
Unescape View File

@ -42,7 +42,7 @@ func isNetworkError(err error) bool {
return false return false
} }
if nerr, ok := err.(*rest.NetworkError); ok { if nerr, ok := err.(*rest.NetworkError); ok {
return xnet.IsNetworkOrHostDown(nerr.Err) return xnet.IsNetworkOrHostDown(nerr.Err, false)
} }
return false return false
} }

4
cmd/update.go
Unescape View File

@ -309,7 +309,7 @@ func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content
client := &http.Client{Transport: getUpdateTransport(timeout)} client := &http.Client{Transport: getUpdateTransport(timeout)}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return content, AdminError{ return content, AdminError{
Code: AdminUpdateURLNotReachable, Code: AdminUpdateURLNotReachable,
Message: err.Error(), Message: err.Error(),
@ -501,7 +501,7 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string
resp, err := clnt.Do(req) resp, err := clnt.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return nil, AdminError{ return nil, AdminError{
Code: AdminUpdateURLNotReachable, Code: AdminUpdateURLNotReachable,
Message: err.Error(), Message: err.Error(),

8
pkg/event/target/elasticsearch.go
Unescape View File

@ -124,7 +124,7 @@ func (target *ElasticsearchTarget) IsActive() (bool, error) {
} }
_, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx) _, code, err := target.client.Ping(target.args.URL.String()).HttpHeadOnly(true).Do(ctx)
if err != nil { if err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -138,7 +138,7 @@ func (target *ElasticsearchTarget) Save(eventData event.Event) error {
return target.store.Put(eventData) return target.store.Put(eventData)
} }
err := target.send(eventData) err := target.send(eventData)
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err
@ -214,7 +214,7 @@ func (target *ElasticsearchTarget) Send(eventKey string) error {
} }
if err := target.send(eventData); err != nil { if err := target.send(eventData); err != nil {
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err
@ -267,7 +267,7 @@ func newClient(args ElasticsearchArgs) (*elastic.Client, error) {
client, err := elastic.NewClient(options...) client, err := elastic.NewClient(options...)
if err != nil { if err != nil {
// https://github.com/olivere/elastic/wiki/Connection-Errors // https://github.com/olivere/elastic/wiki/Connection-Errors
if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err) { if elastic.IsConnErr(err) || elastic.IsContextErr(err) || xnet.IsNetworkOrHostDown(err, false) {
return nil, errNotConnected return nil, errNotConnected
} }
return nil, err return nil, err

8
pkg/event/target/webhook.go
Unescape View File

@ -111,7 +111,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil) req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -119,7 +119,7 @@ func (target *WebhookTarget) IsActive() (bool, error) {
resp, err := target.httpClient.Do(req) resp, err := target.httpClient.Do(req)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) || errors.Is(err, context.DeadlineExceeded) { if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
return false, errNotConnected return false, errNotConnected
} }
return false, err return false, err
@ -137,7 +137,7 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
} }
err := target.send(eventData) err := target.send(eventData)
if err != nil { if err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
} }
@ -197,7 +197,7 @@ func (target *WebhookTarget) Send(eventKey string) error {
} }
if err := target.send(eventData); err != nil { if err := target.send(eventData); err != nil {
if xnet.IsNetworkOrHostDown(err) { if xnet.IsNetworkOrHostDown(err, false) {
return errNotConnected return errNotConnected
} }
return err return err

8
pkg/net/url.go
Unescape View File

@ -141,14 +141,18 @@ func ParseURL(s string) (u *URL, err error) {
} }
// IsNetworkOrHostDown - if there was a network error or if the host is down. // IsNetworkOrHostDown - if there was a network error or if the host is down.
func IsNetworkOrHostDown(err error) bool { // expectTimeouts indicates that context timeouts are expected and does not
// indicate a downed host. Other timeouts still returns down.
func IsNetworkOrHostDown(err error, expectTimeouts bool) bool {
if err == nil { if err == nil {
return false return false
} }
if errors.Is(err, context.Canceled) { if errors.Is(err, context.Canceled) {
return false return false
} }
if expectTimeouts && errors.Is(err, context.DeadlineExceeded) {
return false
}
// We need to figure if the error either a timeout // We need to figure if the error either a timeout
// or a non-temporary error. // or a non-temporary error.
e, ok := err.(net.Error) e, ok := err.(net.Error)

Write Preview
Loading…
Cancel
Save