diff options
Diffstat (limited to 'test/with_dummyserver/test_connectionpool.py')
| -rw-r--r-- | test/with_dummyserver/test_connectionpool.py | 325 |
1 files changed, 172 insertions, 153 deletions
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 741ae7b..9294adf 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -29,22 +29,181 @@ from urllib3.exceptions import ( MaxRetryError, ReadTimeoutError, ProtocolError, + NewConnectionError, ) from urllib3.packages.six import b, u from urllib3.util.retry import Retry from urllib3.util.timeout import Timeout -import tornado -from dummyserver.testcase import HTTPDummyServerTestCase +from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS -from nose.tools import timed +from threading import Event log = logging.getLogger('urllib3.connectionpool') log.setLevel(logging.NOTSET) log.addHandler(logging.StreamHandler(sys.stdout)) +SHORT_TIMEOUT = 0.001 +LONG_TIMEOUT = 0.01 + + +class TestConnectionPoolTimeouts(SocketDummyServerTestCase): + + def test_timeout_float(self): + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=2) + + # Pool-global timeout + pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + block_event.set() # Release block + + # Shouldn't raise this time + ready_event.wait() + block_event.set() # Pre-release block + pool.request('GET', '/') + + def test_conn_closed(self): + block_event = Event() + self.start_basic_handler(block_send=block_event, num=1) + + pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) + conn = pool._get_conn() + pool._put_conn(conn) + try: + pool.urlopen('GET', '/') + self.fail("The request should fail with a timeout error.") + except ReadTimeoutError: + if conn.sock: + self.assertRaises(socket.error, conn.sock.recv, 1024) + finally: + pool._put_conn(conn) + + block_event.set() + + def test_timeout(self): + # Requests should time out when expected + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=6) + + # Pool-global timeout + timeout = Timeout(read=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + + conn = pool._get_conn() + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/') + pool._put_conn(conn) + block_event.set() # Release request + + ready_event.wait() + block_event.clear() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + block_event.set() # Release request + + # Request-specific timeouts should raise errors + pool = HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) + + conn = pool._get_conn() + ready_event.wait() + now = time.time() + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=timeout) + delta = time.time() - now + block_event.set() # Release request + + self.assertTrue(delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT") + pool._put_conn(conn) + + ready_event.wait() + now = time.time() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=timeout) + delta = time.time() - now + + self.assertTrue(delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT") + block_event.set() # Release request + + # Timeout int/float passed directly to request and _make_request should + # raise a request timeout + ready_event.wait() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=SHORT_TIMEOUT) + block_event.set() # Release request + + ready_event.wait() + conn = pool._new_conn() + # FIXME: This assert flakes sometimes. Not sure why. + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=SHORT_TIMEOUT) + block_event.set() # Release request + + def test_connect_timeout(self): + def noop_handler(listener): + return + + self._start_server(noop_handler) + + url = '/' + host, port = self.host, self.port + timeout = Timeout(connect=SHORT_TIMEOUT) + + # Pool-global timeout + pool = HTTPConnectionPool(host, port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) + + # Retries + retries = Retry(connect=0) + self.assertRaises(MaxRetryError, pool.request, 'GET', url, retries=retries) + + # Request-specific connection timeouts + big_timeout = Timeout(read=LONG_TIMEOUT, connect=LONG_TIMEOUT) + pool = HTTPConnectionPool(host, port, timeout=big_timeout, retries=False) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url, timeout=timeout) + + pool._put_conn(conn) + self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url, timeout=timeout) + + def test_total_applies_connect(self): + def noop_handler(listener): + return + + self._start_server(noop_handler) + + timeout = Timeout(total=None, connect=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') + + timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') + + def test_total_timeout(self): + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=2) + + # This will get the socket to raise an EAGAIN on the read + timeout = Timeout(connect=3, read=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + + block_event.set() + ready_event.wait() + block_event.clear() + + # The connect should succeed and this should hit the read timeout + timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + + def test_create_connection_timeout(self): + timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT) + pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout, retries=False) + conn = pool._new_conn() + self.assertRaises(ConnectTimeoutError, conn.connect) + + class TestConnectionPool(HTTPDummyServerTestCase): def setUp(self): @@ -124,26 +283,6 @@ class TestConnectionPool(HTTPDummyServerTestCase): r = self.pool.request('POST', '/upload', fields=fields) self.assertEqual(r.status, 200, r.data) - def test_timeout_float(self): - url = '/sleep?seconds=0.005' - # Pool-global timeout - pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False) - self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) - - def test_conn_closed(self): - pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False) - conn = pool._get_conn() - pool._put_conn(conn) - try: - url = '/sleep?seconds=0.005' - pool.urlopen('GET', url) - self.fail("The request should fail with a timeout error.") - except ReadTimeoutError: - if conn.sock: - self.assertRaises(socket.error, conn.sock.recv, 1024) - finally: - pool._put_conn(conn) - def test_nagle(self): """ Test that connections have TCP_NODELAY turned on """ # This test needs to be here in order to be run. socket.create_connection actually tries to @@ -152,10 +291,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): conn = pool._get_conn() pool._make_request(conn, 'GET', '/') tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) - assert tcp_nodelay_setting > 0, ("Expected TCP_NODELAY to be set on the " - "socket (with value greater than 0) " - "but instead was %s" % - tcp_nodelay_setting) + self.assertTrue(tcp_nodelay_setting) def test_socket_options(self): """Test that connections accept socket options.""" @@ -194,79 +330,6 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertTrue(nagle_disabled) self.assertTrue(using_keepalive) - @timed(0.5) - def test_timeout(self): - """ Requests should time out when expected """ - url = '/sleep?seconds=0.003' - timeout = Timeout(read=0.001) - - # Pool-global timeout - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) - - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, - conn, 'GET', url) - pool._put_conn(conn) - - time.sleep(0.02) # Wait for server to start receiving again. :( - - self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) - - # Request-specific timeouts should raise errors - pool = HTTPConnectionPool(self.host, self.port, timeout=0.1, retries=False) - - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, - conn, 'GET', url, timeout=timeout) - pool._put_conn(conn) - - time.sleep(0.02) # Wait for server to start receiving again. :( - - self.assertRaises(ReadTimeoutError, pool.request, - 'GET', url, timeout=timeout) - - # Timeout int/float passed directly to request and _make_request should - # raise a request timeout - self.assertRaises(ReadTimeoutError, pool.request, - 'GET', url, timeout=0.001) - conn = pool._new_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, - 'GET', url, timeout=0.001) - pool._put_conn(conn) - - # Timeout int/float passed directly to _make_request should not raise a - # request timeout if it's a high value - pool.request('GET', url, timeout=1) - - @requires_network - @timed(0.5) - def test_connect_timeout(self): - url = '/sleep?seconds=0.005' - timeout = Timeout(connect=0.001) - - # Pool-global timeout - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) - - # Retries - retries = Retry(connect=0) - self.assertRaises(MaxRetryError, pool.request, 'GET', url, - retries=retries) - - # Request-specific connection timeouts - big_timeout = Timeout(read=0.2, connect=0.2) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, - timeout=big_timeout, retries=False) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', - url, timeout=timeout) - - pool._put_conn(conn) - self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url, - timeout=timeout) - - def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() @@ -275,50 +338,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool.request('GET', '/', retries=Retry(connect=3)) self.fail("Should have failed with a connection error.") except MaxRetryError as e: - self.assertTrue(isinstance(e.reason, ProtocolError)) - self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED) - - def test_timeout_reset(self): - """ If the read timeout isn't set, socket timeout should reset """ - url = '/sleep?seconds=0.005' - timeout = Timeout(connect=0.001) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - try: - pool._make_request(conn, 'GET', url) - except ReadTimeoutError: - self.fail("This request shouldn't trigger a read timeout.") - - @requires_network - @timed(5.0) - def test_total_timeout(self): - url = '/sleep?seconds=0.005' - - timeout = Timeout(connect=3, read=5, total=0.001) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) - - # This will get the socket to raise an EAGAIN on the read - timeout = Timeout(connect=3, read=0) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) - - # The connect should succeed and this should hit the read timeout - timeout = Timeout(connect=3, read=5, total=0.002) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) - - @requires_network - def test_none_total_applies_connect(self): - url = '/sleep?seconds=0.005' - timeout = Timeout(total=None, connect=0.001) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', - url) + self.assertEqual(type(e.reason), NewConnectionError) def test_timeout_success(self): timeout = Timeout(connect=3, read=5, total=None) @@ -372,7 +392,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool.request('GET', '/', retries=5) self.fail("should raise timeout exception here") except MaxRetryError as e: - self.assertTrue(isinstance(e.reason, ProtocolError), e.reason) + self.assertEqual(type(e.reason), NewConnectionError) def test_keepalive(self): pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) @@ -607,16 +627,13 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) r = pool.request('GET', '/source_address') - assert r.data == b(addr[0]), ( - "expected the response to contain the source address {addr}, " - "but was {data}".format(data=r.data, addr=b(addr[0]))) + self.assertEqual(r.data, b(addr[0])) def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: - pool = HTTPConnectionPool(self.host, self.port, - source_address=addr, retries=False) - self.assertRaises(ProtocolError, - pool.request, 'GET', '/source_address') + pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) + # FIXME: This assert flakes sometimes. Not sure why. + self.assertRaises(NewConnectionError, pool.request, 'GET', '/source_address?{0}'.format(addr)) def test_stream_keepalive(self): x = 2 @@ -669,6 +686,8 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertEqual(http.pool.qsize(), http.pool.maxsize) + + class TestRetry(HTTPDummyServerTestCase): def setUp(self): self.pool = HTTPConnectionPool(self.host, self.port) @@ -695,7 +714,7 @@ class TestRetry(HTTPDummyServerTestCase): self.assertEqual(r.status, 303) pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001) - self.assertRaises(ProtocolError, pool.request, 'GET', '/test', retries=False) + self.assertRaises(NewConnectionError, pool.request, 'GET', '/test', retries=False) def test_read_retries(self): """ Should retry for status codes in the whitelist """ |