Add request pool to improve delivery performance (#10353)
* Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reapingmaster
Eugen Rochko
6 years ago
committed by
GitHub
parent
2cfa427ea7
commit
0d9ffe56fb
@ -0,0 +1,63 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'connection_pool' |
||||
require_relative './shared_timed_stack' |
||||
|
||||
class ConnectionPool::SharedConnectionPool < ConnectionPool |
||||
def initialize(options = {}, &block) |
||||
super(options, &block) |
||||
|
||||
@available = ConnectionPool::SharedTimedStack.new(@size, &block) |
||||
end |
||||
|
||||
delegate :size, :flush, to: :@available |
||||
|
||||
def with(preferred_tag, options = {}) |
||||
Thread.handle_interrupt(Exception => :never) do |
||||
conn = checkout(preferred_tag, options) |
||||
|
||||
begin |
||||
Thread.handle_interrupt(Exception => :immediate) do |
||||
yield conn |
||||
end |
||||
ensure |
||||
checkin(preferred_tag) |
||||
end |
||||
end |
||||
end |
||||
|
||||
def checkout(preferred_tag, options = {}) |
||||
if ::Thread.current[key(preferred_tag)] |
||||
::Thread.current[key_count(preferred_tag)] += 1 |
||||
::Thread.current[key(preferred_tag)] |
||||
else |
||||
::Thread.current[key_count(preferred_tag)] = 1 |
||||
::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout) |
||||
end |
||||
end |
||||
|
||||
def checkin(preferred_tag) |
||||
if ::Thread.current[key(preferred_tag)] |
||||
if ::Thread.current[key_count(preferred_tag)] == 1 |
||||
@available.push(::Thread.current[key(preferred_tag)]) |
||||
::Thread.current[key(preferred_tag)] = nil |
||||
else |
||||
::Thread.current[key_count(preferred_tag)] -= 1 |
||||
end |
||||
else |
||||
raise ConnectionPool::Error, 'no connections are checked out' |
||||
end |
||||
|
||||
nil |
||||
end |
||||
|
||||
private |
||||
|
||||
def key(tag) |
||||
:"#{@key}-#{tag}" |
||||
end |
||||
|
||||
def key_count(tag) |
||||
:"#{@key_count}-#{tag}" |
||||
end |
||||
end |
||||
@ -0,0 +1,95 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class ConnectionPool::SharedTimedStack |
||||
def initialize(max = 0, &block) |
||||
@create_block = block |
||||
@max = max |
||||
@created = 0 |
||||
@queue = [] |
||||
@tagged_queue = Hash.new { |hash, key| hash[key] = [] } |
||||
@mutex = Mutex.new |
||||
@resource = ConditionVariable.new |
||||
end |
||||
|
||||
def push(connection) |
||||
@mutex.synchronize do |
||||
store_connection(connection) |
||||
@resource.broadcast |
||||
end |
||||
end |
||||
|
||||
alias << push |
||||
|
||||
def pop(preferred_tag, timeout = 5.0) |
||||
deadline = current_time + timeout |
||||
|
||||
@mutex.synchronize do |
||||
loop do |
||||
return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty? |
||||
|
||||
connection = try_create(preferred_tag) |
||||
return connection if connection |
||||
|
||||
to_wait = deadline - current_time |
||||
raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 |
||||
|
||||
@resource.wait(@mutex, to_wait) |
||||
end |
||||
end |
||||
end |
||||
|
||||
def empty? |
||||
size.zero? |
||||
end |
||||
|
||||
def size |
||||
@mutex.synchronize do |
||||
@queue.size |
||||
end |
||||
end |
||||
|
||||
def flush |
||||
@mutex.synchronize do |
||||
@queue.delete_if do |connection| |
||||
delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME) |
||||
|
||||
if delete |
||||
@tagged_queue[connection.site].delete(connection) |
||||
connection.close |
||||
@created -= 1 |
||||
end |
||||
|
||||
delete |
||||
end |
||||
end |
||||
end |
||||
|
||||
private |
||||
|
||||
def try_create(preferred_tag) |
||||
if @created == @max && !@queue.empty? |
||||
throw_away_connection = @queue.pop |
||||
@tagged_queue[throw_away_connection.site].delete(throw_away_connection) |
||||
@create_block.call(preferred_tag) |
||||
elsif @created != @max |
||||
connection = @create_block.call(preferred_tag) |
||||
@created += 1 |
||||
connection |
||||
end |
||||
end |
||||
|
||||
def fetch_preferred_connection(preferred_tag) |
||||
connection = @tagged_queue[preferred_tag].pop |
||||
@queue.delete(connection) |
||||
connection |
||||
end |
||||
|
||||
def current_time |
||||
Process.clock_gettime(Process::CLOCK_MONOTONIC) |
||||
end |
||||
|
||||
def store_connection(connection) |
||||
@tagged_queue[connection.site].push(connection) |
||||
@queue.push(connection) |
||||
end |
||||
end |
||||
@ -0,0 +1,114 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require_relative './connection_pool/shared_connection_pool' |
||||
|
||||
class RequestPool |
||||
def self.current |
||||
@current ||= RequestPool.new |
||||
end |
||||
|
||||
class Reaper |
||||
attr_reader :pool, :frequency |
||||
|
||||
def initialize(pool, frequency) |
||||
@pool = pool |
||||
@frequency = frequency |
||||
end |
||||
|
||||
def run |
||||
return unless frequency&.positive? |
||||
|
||||
Thread.new(frequency, pool) do |t, p| |
||||
loop do |
||||
sleep t |
||||
p.flush |
||||
end |
||||
end |
||||
end |
||||
end |
||||
|
||||
MAX_IDLE_TIME = 30 |
||||
WAIT_TIMEOUT = 5 |
||||
MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i |
||||
|
||||
class Connection |
||||
attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh |
||||
|
||||
def initialize(site) |
||||
@site = site |
||||
@http_client = http_client |
||||
@last_used_at = nil |
||||
@created_at = current_time |
||||
@dead = false |
||||
@fresh = true |
||||
end |
||||
|
||||
def use |
||||
@last_used_at = current_time |
||||
@in_use = true |
||||
|
||||
retries = 0 |
||||
|
||||
begin |
||||
yield @http_client |
||||
rescue HTTP::ConnectionError |
||||
# It's possible the connection was closed, so let's |
||||
# try re-opening it once |
||||
|
||||
close |
||||
|
||||
if @fresh || retries.positive? |
||||
raise |
||||
else |
||||
@http_client = http_client |
||||
retries += 1 |
||||
retry |
||||
end |
||||
rescue StandardError |
||||
# If this connection raises errors of any kind, it's |
||||
# better if it gets reaped as soon as possible |
||||
|
||||
close |
||||
@dead = true |
||||
raise |
||||
end |
||||
ensure |
||||
@fresh = false |
||||
@in_use = false |
||||
end |
||||
|
||||
def seconds_idle |
||||
current_time - (@last_used_at || @created_at) |
||||
end |
||||
|
||||
def close |
||||
@http_client.close |
||||
end |
||||
|
||||
private |
||||
|
||||
def http_client |
||||
Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME) |
||||
end |
||||
|
||||
def current_time |
||||
Process.clock_gettime(Process::CLOCK_MONOTONIC) |
||||
end |
||||
end |
||||
|
||||
def initialize |
||||
@pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) } |
||||
@reaper = Reaper.new(self, 30) |
||||
@reaper.run |
||||
end |
||||
|
||||
def with(site, &block) |
||||
@pool.with(site) do |connection| |
||||
ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do |
||||
connection.use(&block) |
||||
end |
||||
end |
||||
end |
||||
|
||||
delegate :size, :flush, to: :@pool |
||||
end |
||||
@ -0,0 +1,28 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'rails_helper' |
||||
|
||||
describe ConnectionPool::SharedConnectionPool do |
||||
class MiniConnection |
||||
attr_reader :site |
||||
|
||||
def initialize(site) |
||||
@site = site |
||||
end |
||||
end |
||||
|
||||
subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } } |
||||
|
||||
describe '#with' do |
||||
it 'runs a block with a connection' do |
||||
block_run = false |
||||
|
||||
subject.with('foo') do |connection| |
||||
expect(connection).to be_a MiniConnection |
||||
block_run = true |
||||
end |
||||
|
||||
expect(block_run).to be true |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,61 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'rails_helper' |
||||
|
||||
describe ConnectionPool::SharedTimedStack do |
||||
class MiniConnection |
||||
attr_reader :site |
||||
|
||||
def initialize(site) |
||||
@site = site |
||||
end |
||||
end |
||||
|
||||
subject { described_class.new(5) { |site| MiniConnection.new(site) } } |
||||
|
||||
describe '#push' do |
||||
it 'keeps the connection in the stack' do |
||||
subject.push(MiniConnection.new('foo')) |
||||
expect(subject.size).to eq 1 |
||||
end |
||||
end |
||||
|
||||
describe '#pop' do |
||||
it 'returns a connection' do |
||||
expect(subject.pop('foo')).to be_a MiniConnection |
||||
end |
||||
|
||||
it 'returns the same connection that was pushed in' do |
||||
connection = MiniConnection.new('foo') |
||||
subject.push(connection) |
||||
expect(subject.pop('foo')).to be connection |
||||
end |
||||
|
||||
it 'does not create more than maximum amount of connections' do |
||||
expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error |
||||
end |
||||
|
||||
it 'repurposes a connection for a different site when maximum amount is reached' do |
||||
5.times { subject.push(MiniConnection.new('foo')) } |
||||
expect(subject.pop('bar')).to be_a MiniConnection |
||||
end |
||||
end |
||||
|
||||
describe '#empty?' do |
||||
it 'returns true when no connections on the stack' do |
||||
expect(subject.empty?).to be true |
||||
end |
||||
|
||||
it 'returns false when there are connections on the stack' do |
||||
subject.push(MiniConnection.new('foo')) |
||||
expect(subject.empty?).to be false |
||||
end |
||||
end |
||||
|
||||
describe '#size' do |
||||
it 'returns the number of connections on the stack' do |
||||
2.times { subject.push(MiniConnection.new('foo')) } |
||||
expect(subject.size).to eq 2 |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,63 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'rails_helper' |
||||
|
||||
describe RequestPool do |
||||
subject { described_class.new } |
||||
|
||||
describe '#with' do |
||||
it 'returns a HTTP client for a host' do |
||||
subject.with('http://example.com') do |http_client| |
||||
expect(http_client).to be_a HTTP::Client |
||||
end |
||||
end |
||||
|
||||
it 'returns the same instance of HTTP client within the same thread for the same host' do |
||||
test_client = nil |
||||
|
||||
subject.with('http://example.com') { |http_client| test_client = http_client } |
||||
expect(test_client).to_not be_nil |
||||
subject.with('http://example.com') { |http_client| expect(http_client).to be test_client } |
||||
end |
||||
|
||||
it 'returns different HTTP clients for different hosts' do |
||||
test_client = nil |
||||
|
||||
subject.with('http://example.com') { |http_client| test_client = http_client } |
||||
expect(test_client).to_not be_nil |
||||
subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client } |
||||
end |
||||
|
||||
it 'grows to the number of threads accessing it' do |
||||
stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') |
||||
|
||||
subject |
||||
|
||||
threads = 20.times.map do |i| |
||||
Thread.new do |
||||
20.times do |
||||
subject.with('http://example.com') do |http_client| |
||||
http_client.get('/').flush |
||||
end |
||||
end |
||||
end |
||||
end |
||||
|
||||
threads.map(&:join) |
||||
|
||||
expect(subject.size).to be > 1 |
||||
end |
||||
|
||||
it 'closes idle connections' do |
||||
stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') |
||||
|
||||
subject.with('http://example.com') do |http_client| |
||||
http_client.get('/').flush |
||||
end |
||||
|
||||
expect(subject.size).to eq 1 |
||||
sleep RequestPool::MAX_IDLE_TIME + 30 + 1 |
||||
expect(subject.size).to eq 0 |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue