diff options
Diffstat (limited to 'test/with_dummyserver')
| -rw-r--r-- | test/with_dummyserver/test_connectionpool.py | 25 | ||||
| -rw-r--r-- | test/with_dummyserver/test_https.py | 5 | ||||
| -rw-r--r-- | test/with_dummyserver/test_poolmanager.py | 13 | ||||
| -rw-r--r-- | test/with_dummyserver/test_proxy_poolmanager.py | 42 | ||||
| -rw-r--r-- | test/with_dummyserver/test_socketlevel.py | 159 |
5 files changed, 234 insertions, 10 deletions
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index d6cb162..741ae7b 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -36,7 +36,7 @@ from urllib3.util.timeout import Timeout import tornado from dummyserver.testcase import HTTPDummyServerTestCase -from dummyserver.server import NoIPv6Warning +from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS from nose.tools import timed @@ -600,7 +600,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): def test_source_address(self): for addr, is_ipv6 in VALID_SOURCE_ADDRESSES: - if is_ipv6 and not socket.has_ipv6: + if is_ipv6 and not HAS_IPV6_AND_DNS: warnings.warn("No IPv6 support: skipping.", NoIPv6Warning) continue @@ -647,6 +647,27 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertEqual(b'123' * 4, response.read()) + def test_cleanup_on_connection_error(self): + ''' + Test that connections are recycled to the pool on + connection errors where no http response is received. + ''' + poolsize = 3 + with HTTPConnectionPool(self.host, self.port, maxsize=poolsize, block=True) as http: + self.assertEqual(http.pool.qsize(), poolsize) + + # force a connection error by supplying a non-existent + # url. We won't get a response for this and so the + # conn won't be implicitly returned to the pool. + self.assertRaises(MaxRetryError, + http.request, 'GET', '/redirect', fields={'target': '/'}, release_conn=False, retries=0) + + r = http.request('GET', '/redirect', fields={'target': '/'}, release_conn=False, retries=1) + r.release_conn() + + # the pool should still contain poolsize elements + self.assertEqual(http.pool.qsize(), http.pool.maxsize) + class TestRetry(HTTPDummyServerTestCase): def setUp(self): diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 992b8ef..63aea66 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -419,6 +419,11 @@ class TestHTTPS_TLSv1(HTTPSDummyServerTestCase): self._pool.ca_certs = DEFAULT_CA self._pool.request('GET', '/') + def test_set_cert_default_cert_required(self): + conn = VerifiedHTTPSConnection(self.host, self.port) + conn.set_cert(ca_certs='/etc/ssl/certs/custom.pem') + self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED') + class TestHTTPS_NoSAN(HTTPSDummyServerTestCase): certs = NO_SAN_CERTS diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 7e51c73..099ac52 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -1,6 +1,8 @@ import unittest import json +from nose.plugins.skip import SkipTest +from dummyserver.server import HAS_IPV6 from dummyserver.testcase import (HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase) from urllib3.poolmanager import PoolManager @@ -128,6 +130,14 @@ class TestPoolManager(HTTPDummyServerTestCase): def test_headers(self): http = PoolManager(headers={'Foo': 'bar'}) + r = http.request('GET', '%s/headers' % self.base_url) + returned_headers = json.loads(r.data.decode()) + self.assertEqual(returned_headers.get('Foo'), 'bar') + + r = http.request('POST', '%s/headers' % self.base_url) + returned_headers = json.loads(r.data.decode()) + self.assertEqual(returned_headers.get('Foo'), 'bar') + r = http.request_encode_url('GET', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') @@ -154,6 +164,9 @@ class TestPoolManager(HTTPDummyServerTestCase): class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase): + if not HAS_IPV6: + raise SkipTest("IPv6 is not supported on this system.") + def setUp(self): self.base_url = 'http://[%s]:%d' % (self.host, self.port) diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index df300fe..c593f2d 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -4,11 +4,12 @@ import unittest from nose.tools import timed -from dummyserver.testcase import HTTPDummyProxyTestCase +from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase from dummyserver.server import ( DEFAULT_CA, DEFAULT_CA_BAD, get_unreachable_address) from .. import TARPIT_HOST +from urllib3._collections import HTTPHeaderDict from urllib3.poolmanager import proxy_from_url, ProxyManager from urllib3.exceptions import ( MaxRetryError, SSLError, ProxyError, ConnectTimeoutError) @@ -48,7 +49,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): def test_proxy_conn_fail(self): host, port = get_unreachable_address() - http = proxy_from_url('http://%s:%s/' % (host, port), retries=1) + http = proxy_from_url('http://%s:%s/' % (host, port), retries=1, timeout=0.05) self.assertRaises(MaxRetryError, http.request, 'GET', '%s/' % self.https_url) self.assertRaises(MaxRetryError, http.request, 'GET', @@ -223,6 +224,22 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): self.assertEqual(returned_headers.get('Host'), '%s:%s'%(self.https_host,self.https_port)) + def test_headerdict(self): + default_headers = HTTPHeaderDict(a='b') + proxy_headers = HTTPHeaderDict() + proxy_headers.add('foo', 'bar') + + http = proxy_from_url( + self.proxy_url, + headers=default_headers, + proxy_headers=proxy_headers) + + request_headers = HTTPHeaderDict(baz='quux') + r = http.request('GET', '%s/headers' % self.http_url, headers=request_headers) + returned_headers = json.loads(r.data.decode()) + self.assertEqual(returned_headers.get('Foo'), 'bar') + self.assertEqual(returned_headers.get('Baz'), 'quux') + def test_proxy_pooling(self): http = proxy_from_url(self.proxy_url) @@ -283,5 +300,26 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): except MaxRetryError as e: assert isinstance(e.reason, ConnectTimeoutError) + +class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase): + + def setUp(self): + self.http_url = 'http://%s:%d' % (self.http_host, self.http_port) + self.http_url_alt = 'http://%s:%d' % (self.http_host_alt, + self.http_port) + self.https_url = 'https://%s:%d' % (self.https_host, self.https_port) + self.https_url_alt = 'https://%s:%d' % (self.https_host_alt, + self.https_port) + self.proxy_url = 'http://[%s]:%d' % (self.proxy_host, self.proxy_port) + + def test_basic_ipv6_proxy(self): + http = proxy_from_url(self.proxy_url) + + r = http.request('GET', '%s/' % self.http_url) + self.assertEqual(r.status, 200) + + r = http.request('GET', '%s/' % self.https_url) + self.assertEqual(r.status, 200) + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 6c99653..5af00e0 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -10,17 +10,24 @@ from urllib3.exceptions import ( SSLError, ProtocolError, ) +from urllib3.response import httplib from urllib3.util.ssl_ import HAS_SNI from urllib3.util.timeout import Timeout from urllib3.util.retry import Retry +from urllib3._collections import HTTPHeaderDict from dummyserver.testcase import SocketDummyServerTestCase from dummyserver.server import ( DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address) -from .. import onlyPy3 +from .. import onlyPy3, LogRecorder from nose.plugins.skip import SkipTest +try: + from mimetools import Message as MimeToolMessage +except ImportError: + class MimeToolMessage(object): + pass from threading import Event import socket import ssl @@ -119,8 +126,9 @@ class TestSocketClosing(SocketDummyServerTestCase): def test_connection_refused(self): # Does the pool retry if there is no listener on the port? host, port = get_unreachable_address() - pool = HTTPConnectionPool(host, port) - self.assertRaises(MaxRetryError, pool.request, 'GET', '/', retries=0) + http = HTTPConnectionPool(host, port, maxsize=3, block=True) + self.assertRaises(MaxRetryError, http.request, 'GET', '/', retries=0, release_conn=False) + self.assertEqual(http.pool.qsize(), http.pool.maxsize) def test_connection_read_timeout(self): timed_out = Event() @@ -133,13 +141,15 @@ class TestSocketClosing(SocketDummyServerTestCase): sock.close() self._start_server(socket_handler) - pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False) + http = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False, maxsize=3, block=True) try: - self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + self.assertRaises(ReadTimeoutError, http.request, 'GET', '/', release_conn=False) finally: timed_out.set() + self.assertEqual(http.pool.qsize(), http.pool.maxsize) + def test_https_connection_read_timeout(self): """ Handshake timeouts should fail with a Timeout""" timed_out = Event() @@ -297,6 +307,63 @@ class TestSocketClosing(SocketDummyServerTestCase): self.assertEqual(response.status, 200) self.assertEqual(response.data, b'foo') + def test_connection_cleanup_on_read_timeout(self): + timed_out = Event() + + def socket_handler(listener): + sock = listener.accept()[0] + buf = b'' + body = 'Hi' + while not buf.endswith(b'\r\n\r\n'): + buf = sock.recv(65536) + sock.send(('HTTP/1.1 200 OK\r\n' + 'Content-Type: text/plain\r\n' + 'Content-Length: %d\r\n' + '\r\n' % len(body)).encode('utf-8')) + + timed_out.wait() + sock.close() + + self._start_server(socket_handler) + with HTTPConnectionPool(self.host, self.port) as pool: + poolsize = pool.pool.qsize() + response = pool.urlopen('GET', '/', retries=0, preload_content=False, + timeout=Timeout(connect=1, read=0.001)) + try: + self.assertRaises(ReadTimeoutError, response.read) + self.assertEqual(poolsize, pool.pool.qsize()) + finally: + timed_out.set() + + def test_connection_cleanup_on_protocol_error_during_read(self): + body = 'Response' + partial_body = body[:2] + + def socket_handler(listener): + sock = listener.accept()[0] + + # Consume request + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf = sock.recv(65536) + + # Send partial response and close socket. + sock.send(( + 'HTTP/1.1 200 OK\r\n' + 'Content-Type: text/plain\r\n' + 'Content-Length: %d\r\n' + '\r\n' + '%s' % (len(body), partial_body)).encode('utf-8') + ) + sock.close() + + self._start_server(socket_handler) + with HTTPConnectionPool(self.host, self.port) as pool: + poolsize = pool.pool.qsize() + response = pool.request('GET', '/', retries=0, preload_content=False) + + self.assertRaises(ProtocolError, response.read) + self.assertEqual(poolsize, pool.pool.qsize()) class TestProxyManager(SocketDummyServerTestCase): @@ -355,7 +422,7 @@ class TestProxyManager(SocketDummyServerTestCase): base_url = 'http://%s:%d' % (self.host, self.port) # Define some proxy headers. - proxy_headers = {'For The Proxy': 'YEAH!'} + proxy_headers = HTTPHeaderDict({'For The Proxy': 'YEAH!'}) proxy = proxy_from_url(base_url, proxy_headers=proxy_headers) conn = proxy.connection_from_url('http://www.google.com/') @@ -617,6 +684,86 @@ class TestHeaders(SocketDummyServerTestCase): r = pool.request('GET', '/') self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity + def test_headers_are_sent_with_the_original_case(self): + headers = {'foo': 'bar', 'bAz': 'quux'} + parsed_headers = {} + + def socket_handler(listener): + sock = listener.accept()[0] + + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf += sock.recv(65536) + + headers_list = [header for header in buf.split(b'\r\n')[1:] if header] + + for header in headers_list: + (key, value) = header.split(b': ') + parsed_headers[key.decode()] = value.decode() + + # Send incomplete message (note Content-Length) + sock.send(( + 'HTTP/1.1 204 No Content\r\n' + 'Content-Length: 0\r\n' + '\r\n').encode('utf-8')) + + sock.close() + + self._start_server(socket_handler) + expected_headers = {'Accept-Encoding': 'identity', + 'Host': '{0}:{1}'.format(self.host, self.port)} + expected_headers.update(headers) + + pool = HTTPConnectionPool(self.host, self.port, retries=False) + pool.request('GET', '/', headers=HTTPHeaderDict(headers)) + self.assertEqual(expected_headers, parsed_headers) + + +class TestBrokenHeaders(SocketDummyServerTestCase): + def setUp(self): + if issubclass(httplib.HTTPMessage, MimeToolMessage): + raise SkipTest('Header parsing errors not available') + + super(TestBrokenHeaders, self).setUp() + + def _test_broken_header_parsing(self, headers): + handler = create_response_handler(( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'Content-type: text/plain\r\n' + ) + b'\r\n'.join(headers) + b'\r\n' + ) + + self._start_server(handler) + pool = HTTPConnectionPool(self.host, self.port, retries=False) + + with LogRecorder() as logs: + pool.request('GET', '/') + + for record in logs: + if 'Failed to parse headers' in record.msg and \ + pool._absolute_url('/') == record.args[0]: + return + self.fail('Missing log about unparsed headers') + + def test_header_without_name(self): + self._test_broken_header_parsing([ + b': Value\r\n', + b'Another: Header\r\n', + ]) + + def test_header_without_name_or_value(self): + self._test_broken_header_parsing([ + b':\r\n', + b'Another: Header\r\n', + ]) + + def test_header_without_colon_or_value(self): + self._test_broken_header_parsing([ + b'Broken Header', + b'Another: Header', + ]) + class TestHEAD(SocketDummyServerTestCase): def test_chunked_head_response_does_not_hang(self): |