From e59ee14f4012e9aa3245dca139657f6d9b8b25f9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 3 Jul 2020 10:03:41 -0700 Subject: [PATCH] Tune tcp keep-alives with new kernel timeout options (#9963) For more deeper understanding https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/ --- cmd/http/dial_linux.go | 48 ++++++++++++++++++++++++++++----------- cmd/http/dial_others.go | 11 ++++++--- cmd/http/listener.go | 24 ++++++++------------ cmd/http/listener_test.go | 4 ---- cmd/http/server.go | 23 +++++++------------ cmd/http/server_test.go | 4 ---- cmd/update.go | 2 +- cmd/utils.go | 2 +- 8 files changed, 63 insertions(+), 55 deletions(-) diff --git a/cmd/http/dial_linux.go b/cmd/http/dial_linux.go index 3ae5b2e2c..d7aad4783 100644 --- a/cmd/http/dial_linux.go +++ b/cmd/http/dial_linux.go @@ -23,29 +23,51 @@ import ( "net" "syscall" "time" -) -const ( - // TCPFastOpenConnect sets the underlying socket to use - // the TCP fast open connect. This feature is supported - // since Linux 4.11. - TCPFastOpenConnect = 30 + "golang.org/x/sys/unix" ) +func setTCPParameters(c syscall.RawConn) error { + return c.Control(func(fdPtr uintptr) { + // got socket file descriptor to set parameters. + fd := int(fdPtr) + + // Enable TCP fast connect + // TCPFastOpenConnect sets the underlying socket to use + // the TCP fast open connect. This feature is supported + // since Linux 4.11. + _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1) + + // The time (in seconds) the connection needs to remain idle before + // TCP starts sending keepalive probes, set this to 5 secs + // system defaults to 7200 secs!!! + _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 5) + + // Number of probes. + // ~ cat /proc/sys/net/ipv4/tcp_keepalive_probes (defaults to 9, we reduce it to 5) + // 9 + _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5) + + // Wait time after successful probe in seconds. + // ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 2 secs) + // 75 + _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 2) + + // Set TCP_USER_TIMEOUT to TCP_KEEPIDLE + TCP_KEEPINTVL * TCP_KEEPCNT. + _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, 15) + }) +} + // DialContext is a function to make custom Dial for internode communications type DialContext func(ctx context.Context, network, address string) (net.Conn, error) // NewCustomDialContext setups a custom dialer for internode communications -func NewCustomDialContext(dialTimeout, dialKeepAlive time.Duration) DialContext { +func NewCustomDialContext(dialTimeout time.Duration) DialContext { return func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, + Timeout: dialTimeout, Control: func(network, address string, c syscall.RawConn) error { - return c.Control(func(fd uintptr) { - // Enable TCP fast connect - _ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, TCPFastOpenConnect, 1) - }) + return setTCPParameters(c) }, } return dialer.DialContext(ctx, network, addr) diff --git a/cmd/http/dial_others.go b/cmd/http/dial_others.go index 7ce7bc298..1925b0470 100644 --- a/cmd/http/dial_others.go +++ b/cmd/http/dial_others.go @@ -21,18 +21,23 @@ package http import ( "context" "net" + "syscall" "time" ) +// TODO: if possible implement for non-linux platforms, not a priority at the moment +func setTCPParameters(c syscall.RawConn) error { + return nil +} + // DialContext is a function to make custom Dial for internode communications type DialContext func(ctx context.Context, network, address string) (net.Conn, error) // NewCustomDialContext configures a custom dialer for internode communications -func NewCustomDialContext(dialTimeout, dialKeepAlive time.Duration) DialContext { +func NewCustomDialContext(dialTimeout time.Duration) DialContext { return func(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, + Timeout: dialTimeout, } return dialer.DialContext(ctx, network, addr) } diff --git a/cmd/http/listener.go b/cmd/http/listener.go index ed021c18a..1b0f5ff92 100644 --- a/cmd/http/listener.go +++ b/cmd/http/listener.go @@ -23,7 +23,6 @@ import ( "os" "sync" "syscall" - "time" ) type acceptResult struct { @@ -33,11 +32,10 @@ type acceptResult struct { // httpListener - HTTP listener capable of handling multiple server addresses. type httpListener struct { - mutex sync.Mutex // to guard Close() method. - tcpListeners []*net.TCPListener // underlaying TCP listeners. - acceptCh chan acceptResult // channel where all TCP listeners write accepted connection. - doneCh chan struct{} // done channel for TCP listener goroutines. - tcpKeepAliveTimeout time.Duration + mutex sync.Mutex // to guard Close() method. + tcpListeners []*net.TCPListener // underlaying TCP listeners. + acceptCh chan acceptResult // channel where all TCP listeners write accepted connection. + doneCh chan struct{} // done channel for TCP listener goroutines. } // isRoutineNetErr returns true if error is due to a network timeout, @@ -83,10 +81,10 @@ func (listener *httpListener) start() { // Closure to handle single connection. handleConn := func(tcpConn *net.TCPConn, doneCh <-chan struct{}) { - // Tune accepted TCP connection. - tcpConn.SetKeepAlive(true) - tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout) - + rawConn, err := tcpConn.SyscallConn() + if err == nil { + setTCPParameters(rawConn) + } send(acceptResult{tcpConn, nil}, doneCh) } @@ -167,8 +165,7 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) { // httpListener is capable to // * listen to multiple addresses // * controls incoming connections only doing HTTP protocol -func newHTTPListener(serverAddrs []string, - tcpKeepAliveTimeout time.Duration) (listener *httpListener, err error) { +func newHTTPListener(serverAddrs []string) (listener *httpListener, err error) { var tcpListeners []*net.TCPListener @@ -201,8 +198,7 @@ func newHTTPListener(serverAddrs []string, } listener = &httpListener{ - tcpListeners: tcpListeners, - tcpKeepAliveTimeout: tcpKeepAliveTimeout, + tcpListeners: tcpListeners, } listener.start() diff --git a/cmd/http/listener_test.go b/cmd/http/listener_test.go index 8a74c0683..437699fdc 100644 --- a/cmd/http/listener_test.go +++ b/cmd/http/listener_test.go @@ -151,7 +151,6 @@ func TestNewHTTPListener(t *testing.T) { for _, testCase := range testCases { listener, err := newHTTPListener( testCase.serverAddrs, - testCase.tcpKeepAliveTimeout, ) if !testCase.expectedErr { @@ -185,7 +184,6 @@ func TestHTTPListenerStartClose(t *testing.T) { for i, testCase := range testCases { listener, err := newHTTPListener( testCase.serverAddrs, - time.Duration(0), ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -225,7 +223,6 @@ func TestHTTPListenerAddr(t *testing.T) { for i, testCase := range testCases { listener, err := newHTTPListener( testCase.serverAddrs, - time.Duration(0), ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) @@ -262,7 +259,6 @@ func TestHTTPListenerAddrs(t *testing.T) { for i, testCase := range testCases { listener, err := newHTTPListener( testCase.serverAddrs, - time.Duration(0), ) if err != nil { t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) diff --git a/cmd/http/server.go b/cmd/http/server.go index 092d279f8..850f675bc 100644 --- a/cmd/http/server.go +++ b/cmd/http/server.go @@ -38,9 +38,6 @@ const ( // DefaultShutdownTimeout - default shutdown timeout used for graceful http server shutdown. DefaultShutdownTimeout = 5 * time.Second - // DefaultTCPKeepAliveTimeout - default TCP keep alive timeout for accepted connection. - DefaultTCPKeepAliveTimeout = 30 * time.Second - // DefaultMaxHeaderBytes - default maximum HTTP header size in bytes. DefaultMaxHeaderBytes = 1 * humanize.MiByte ) @@ -48,13 +45,12 @@ const ( // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling. type Server struct { http.Server - Addrs []string // addresses on which the server listens for new connection. - ShutdownTimeout time.Duration // timeout used for graceful server shutdown. - TCPKeepAliveTimeout time.Duration // timeout used for underneath TCP connection. - listenerMutex sync.Mutex // to guard 'listener' field. - listener *httpListener // HTTP listener for all 'Addrs' field. - inShutdown uint32 // indicates whether the server is in shutdown or not - requestCount int32 // counter holds no. of request in progress. + Addrs []string // addresses on which the server listens for new connection. + ShutdownTimeout time.Duration // timeout used for graceful server shutdown. + listenerMutex sync.Mutex // to guard 'listener' field. + listener *httpListener // HTTP listener for all 'Addrs' field. + inShutdown uint32 // indicates whether the server is in shutdown or not + requestCount int32 // counter holds no. of request in progress. } // GetRequestCount - returns number of request in progress. @@ -72,13 +68,11 @@ func (srv *Server) Start() (err error) { handler := srv.Handler // if srv.Handler holds non-synced state -> possible data race addrs := set.CreateStringSet(srv.Addrs...).ToSlice() // copy and remove duplicates - tcpKeepAliveTimeout := srv.TCPKeepAliveTimeout // Create new HTTP listener. var listener *httpListener listener, err = newHTTPListener( addrs, - tcpKeepAliveTimeout, ) if err != nil { return err @@ -204,9 +198,8 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat } httpServer := &Server{ - Addrs: addrs, - ShutdownTimeout: DefaultShutdownTimeout, - TCPKeepAliveTimeout: DefaultTCPKeepAliveTimeout, + Addrs: addrs, + ShutdownTimeout: DefaultShutdownTimeout, } httpServer.Handler = handler httpServer.TLSConfig = tlsConfig diff --git a/cmd/http/server_test.go b/cmd/http/server_test.go index 08f84c2eb..0a38dfbb6 100644 --- a/cmd/http/server_test.go +++ b/cmd/http/server_test.go @@ -73,10 +73,6 @@ func TestNewServer(t *testing.T) { t.Fatalf("Case %v: server.ShutdownTimeout: expected: %v, got: %v", (i + 1), DefaultShutdownTimeout, server.ShutdownTimeout) } - if server.TCPKeepAliveTimeout != DefaultTCPKeepAliveTimeout { - t.Fatalf("Case %v: server.TCPKeepAliveTimeout: expected: %v, got: %v", (i + 1), DefaultTCPKeepAliveTimeout, server.TCPKeepAliveTimeout) - } - if server.MaxHeaderBytes != DefaultMaxHeaderBytes { t.Fatalf("Case %v: server.MaxHeaderBytes: expected: %v, got: %v", (i + 1), DefaultMaxHeaderBytes, server.MaxHeaderBytes) } diff --git a/cmd/update.go b/cmd/update.go index dcf20c148..36069ec2d 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -409,7 +409,7 @@ const updateTimeout = 10 * time.Second func getUpdateTransport(timeout time.Duration) http.RoundTripper { var updateTransport http.RoundTripper = &http.Transport{ Proxy: http.ProxyFromEnvironment, - DialContext: xhttp.NewCustomDialContext(timeout, timeout), + DialContext: xhttp.NewCustomDialContext(timeout), IdleConnTimeout: timeout, TLSHandshakeTimeout: timeout, ExpectContinueTimeout: timeout, diff --git a/cmd/utils.go b/cmd/utils.go index 0a32e4b88..8f56a1242 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -454,7 +454,7 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu // https://golang.org/pkg/net/http/#Transport documentation tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, - DialContext: xhttp.NewCustomDialContext(dialTimeout, 15*time.Second), + DialContext: xhttp.NewCustomDialContext(dialTimeout), MaxIdleConnsPerHost: 16, MaxIdleConns: 16, IdleConnTimeout: 1 * time.Minute,