Remove "Connection" close instead reduce MaxConns per host (#7654)

This is necessary to avoid connection build up between servers
unexpectedly for example in a situation where 16 servers are
talking to each other and one server now allows a maximum of

15*4096 = 61440 idle connections

Will be kept in pool. Such a large pool is perhaps inefficient for
many reasons and also affects overall system resources.

This PR also reduces idleConnection timeout from 120 secs to 60 secs.

cmd/rest/client.go

@@ -100,9 +100,8 @@ func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAu
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           newCustomDialContext(timeout),
-		MaxIdleConnsPerHost:   4096,
-		MaxIdleConns:          4096,
-		IdleConnTimeout:       120 * time.Second,
+		MaxIdleConnsPerHost:   256,
+		IdleConnTimeout:       60 * time.Second,
 		TLSHandshakeTimeout:   30 * time.Second,
 		ExpectContinueTimeout: 10 * time.Second,
 		TLSClientConfig:       tlsConfig,

cmd/storage-rest-server.go

@@ -415,7 +415,6 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
 	}
 	defer w.(http.Flusher).Flush()
 
-	w.Header().Set("Connection", "close") // Pro-actively ask client to close this connection.
 	encoder := gob.NewEncoder(w)
 	for fi := range fch {
 		encoder.Encode(&fi)