From dff63335f212d32d7c1a4bb5276f2d31f5995ea1 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Tue, 22 Dec 2015 14:09:50 +0000 Subject: Import python-urllib3_1.13.1.orig.tar.gz --- test/test_util.py | 30 +++++++++++ test/with_dummyserver/test_connectionpool.py | 44 +++++++-------- test/with_dummyserver/test_https.py | 32 +++++++++-- test/with_dummyserver/test_poolmanager.py | 12 +++-- test/with_dummyserver/test_proxy_poolmanager.py | 8 +-- test/with_dummyserver/test_socketlevel.py | 71 +++++++++++++++++++++++++ 6 files changed, 164 insertions(+), 33 deletions(-) (limited to 'test') diff --git a/test/test_util.py b/test/test_util.py index fa59ada..ef4caab 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,3 +1,4 @@ +import hashlib import warnings import logging import unittest @@ -18,12 +19,14 @@ from urllib3.util.url import ( from urllib3.util.ssl_ import ( resolve_cert_reqs, ssl_wrap_socket, + _const_compare_digest_backport, ) from urllib3.exceptions import ( LocationParseError, TimeoutStateError, InsecureRequestWarning, SSLError, + SNIMissingWarning, ) from urllib3.util import is_fp_closed, ssl_ @@ -412,3 +415,30 @@ class TestUtil(unittest.TestCase): ssl_wrap_socket(ssl_context=mock_context, sock=socket) mock_context.wrap_socket.assert_called_once_with(socket) ssl_.HAS_SNI = HAS_SNI + + def test_ssl_wrap_socket_with_no_sni_warns(self): + socket = object() + mock_context = Mock() + # Ugly preservation of original value + HAS_SNI = ssl_.HAS_SNI + ssl_.HAS_SNI = False + with patch('warnings.warn') as warn: + ssl_wrap_socket(ssl_context=mock_context, sock=socket) + mock_context.wrap_socket.assert_called_once_with(socket) + ssl_.HAS_SNI = HAS_SNI + self.assertTrue(warn.call_count >= 1) + warnings = [call[0][1] for call in warn.call_args_list] + self.assertTrue(SNIMissingWarning in warnings) + + def test_const_compare_digest_fallback(self): + target = hashlib.sha256(b'abcdef').digest() + self.assertTrue(_const_compare_digest_backport(target, target)) + + prefix = target[:-1] + self.assertFalse(_const_compare_digest_backport(target, prefix)) + + suffix = target + b'0' + self.assertFalse(_const_compare_digest_backport(target, suffix)) + + incorrect = hashlib.sha256(b'xyz').digest() + self.assertFalse(_const_compare_digest_backport(target, incorrect)) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 9294adf..0f31fa0 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -49,6 +49,11 @@ SHORT_TIMEOUT = 0.001 LONG_TIMEOUT = 0.01 +def wait_for_socket(ready_event): + ready_event.wait() + ready_event.clear() + + class TestConnectionPoolTimeouts(SocketDummyServerTestCase): def test_timeout_float(self): @@ -57,11 +62,12 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) + wait_for_socket(ready_event) self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') block_event.set() # Release block # Shouldn't raise this time - ready_event.wait() + wait_for_socket(ready_event) block_event.set() # Pre-release block pool.request('GET', '/') @@ -92,12 +98,13 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): timeout = Timeout(read=SHORT_TIMEOUT) pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + wait_for_socket(ready_event) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/') pool._put_conn(conn) block_event.set() # Release request - ready_event.wait() + wait_for_socket(ready_event) block_event.clear() self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') block_event.set() # Release request @@ -106,7 +113,7 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): pool = HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) conn = pool._get_conn() - ready_event.wait() + wait_for_socket(ready_event) now = time.time() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=timeout) delta = time.time() - now @@ -115,7 +122,7 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): self.assertTrue(delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT") pool._put_conn(conn) - ready_event.wait() + wait_for_socket(ready_event) now = time.time() self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=timeout) delta = time.time() - now @@ -125,24 +132,19 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): # Timeout int/float passed directly to request and _make_request should # raise a request timeout - ready_event.wait() + wait_for_socket(ready_event) self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=SHORT_TIMEOUT) block_event.set() # Release request - ready_event.wait() + wait_for_socket(ready_event) conn = pool._new_conn() # FIXME: This assert flakes sometimes. Not sure why. self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=SHORT_TIMEOUT) block_event.set() # Release request def test_connect_timeout(self): - def noop_handler(listener): - return - - self._start_server(noop_handler) - url = '/' - host, port = self.host, self.port + host, port = TARPIT_HOST, 80 timeout = Timeout(connect=SHORT_TIMEOUT) # Pool-global timeout @@ -164,18 +166,15 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url, timeout=timeout) def test_total_applies_connect(self): - def noop_handler(listener): - return - - self._start_server(noop_handler) + host, port = TARPIT_HOST, 80 timeout = Timeout(total=None, connect=SHORT_TIMEOUT) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + pool = HTTPConnectionPool(host, port, timeout=timeout) conn = pool._get_conn() self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + pool = HTTPConnectionPool(host, port, timeout=timeout) conn = pool._get_conn() self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') @@ -183,13 +182,14 @@ class TestConnectionPoolTimeouts(SocketDummyServerTestCase): block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) + wait_for_socket(ready_event) # This will get the socket to raise an EAGAIN on the read timeout = Timeout(connect=3, read=SHORT_TIMEOUT) pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') block_event.set() - ready_event.wait() + wait_for_socket(ready_event) block_event.clear() # The connect should succeed and this should hit the read timeout @@ -666,15 +666,15 @@ class TestConnectionPool(HTTPDummyServerTestCase): def test_cleanup_on_connection_error(self): ''' - Test that connections are recycled to the pool on + Test that connections are recycled to the pool on connection errors where no http response is received. ''' poolsize = 3 with HTTPConnectionPool(self.host, self.port, maxsize=poolsize, block=True) as http: self.assertEqual(http.pool.qsize(), poolsize) - # force a connection error by supplying a non-existent - # url. We won't get a response for this and so the + # force a connection error by supplying a non-existent + # url. We won't get a response for this and so the # conn won't be implicitly returned to the pool. self.assertRaises(MaxRetryError, http.request, 'GET', '/redirect', fields={'target': '/'}, release_conn=False, retries=0) diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 862ebd9..7319d7e 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -8,9 +8,12 @@ import warnings import mock from nose.plugins.skip import SkipTest -from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.testcase import ( + HTTPSDummyServerTestCase, IPV6HTTPSDummyServerTestCase +) from dummyserver.server import (DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS, - NO_SAN_CERTS, NO_SAN_CA, DEFAULT_CA_DIR) + NO_SAN_CERTS, NO_SAN_CA, DEFAULT_CA_DIR, + IPV6_ADDR_CERTS, IPV6_ADDR_CA, HAS_IPV6) from test import ( onlyPy26OrOlder, @@ -35,6 +38,7 @@ from urllib3.exceptions import ( ) from urllib3.packages import six from urllib3.util.timeout import Timeout +from urllib3.util.ssl_ import HAS_SNI ResourceWarning = getattr( @@ -77,7 +81,10 @@ class TestHTTPS(HTTPSDummyServerTestCase): self.assertFalse(warn.called, warn.call_args_list) else: self.assertTrue(warn.called) - call, = warn.call_args_list + if HAS_SNI: + call = warn.call_args_list[0] + else: + call = warn.call_args_list[1] error = call[0][1] self.assertEqual(error, InsecurePlatformWarning) @@ -176,8 +183,10 @@ class TestHTTPS(HTTPSDummyServerTestCase): calls = warn.call_args_list if sys.version_info >= (2, 7, 9): category = calls[0][0][1] - else: + elif HAS_SNI: category = calls[1][0][1] + else: + category = calls[2][0][1] self.assertEqual(category, InsecureRequestWarning) @requires_network @@ -460,5 +469,20 @@ class TestHTTPS_NoSAN(HTTPSDummyServerTestCase): self.assertTrue(warn.called) + +class TestHTTPS_IPv6Addr(IPV6HTTPSDummyServerTestCase): + certs = IPV6_ADDR_CERTS + + def test_strip_square_brackets_before_validating(self): + """Test that the fix for #760 works.""" + if not HAS_IPV6: + raise SkipTest("Only runs on IPv6 systems") + https_pool = HTTPSConnectionPool('[::1]', self.port, + cert_reqs='CERT_REQUIRED', + ca_certs=IPV6_ADDR_CA) + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 099ac52..4065ff8 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -69,7 +69,7 @@ class TestPoolManager(HTTPDummyServerTestCase): try: http.request('GET', '%s/redirect' % self.base_url, fields={'target': cross_host_location}, - timeout=0.01, retries=0) + timeout=1, retries=0) self.fail("Request succeeded instead of raising an exception like it should.") except MaxRetryError: @@ -77,7 +77,7 @@ class TestPoolManager(HTTPDummyServerTestCase): r = http.request('GET', '%s/redirect' % self.base_url, fields={'target': '%s/echo?a=b' % self.base_url_alt}, - timeout=0.01, retries=1) + timeout=1, retries=1) self.assertEqual(r._pool.host, self.host_alt) @@ -137,7 +137,7 @@ class TestPoolManager(HTTPDummyServerTestCase): r = http.request('POST', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') - + r = http.request_encode_url('GET', '%s/headers' % self.base_url) returned_headers = json.loads(r.data.decode()) self.assertEqual(returned_headers.get('Foo'), 'bar') @@ -162,6 +162,12 @@ class TestPoolManager(HTTPDummyServerTestCase): r = http.request('GET', 'http://%s:%s/' % (self.host, self.port)) self.assertEqual(r.status, 200) + def test_http_with_ca_cert_dir(self): + http = PoolManager(ca_certs='REQUIRED', ca_cert_dir='/nosuchdir') + + r = http.request('GET', 'http://%s:%s/' % (self.host, self.port)) + self.assertEqual(r.status, 200) + class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase): if not HAS_IPV6: diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index b2894a8..b37d8bb 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -124,7 +124,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): try: http.request('GET', '%s/redirect' % self.http_url, fields={'target': cross_host_location}, - timeout=0.1, retries=0) + timeout=1, retries=0) self.fail("We don't want to follow redirects here.") except MaxRetryError: @@ -132,7 +132,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): r = http.request('GET', '%s/redirect' % self.http_url, fields={'target': '%s/echo?a=b' % self.http_url_alt}, - timeout=0.1, retries=1) + timeout=1, retries=1) self.assertNotEqual(r._pool.host, self.http_host_alt) def test_cross_protocol_redirect(self): @@ -142,7 +142,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): try: http.request('GET', '%s/redirect' % self.http_url, fields={'target': cross_protocol_location}, - timeout=0.1, retries=0) + timeout=1, retries=0) self.fail("We don't want to follow redirects here.") except MaxRetryError: @@ -150,7 +150,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): r = http.request('GET', '%s/redirect' % self.http_url, fields={'target': '%s/echo?a=b' % self.https_url}, - timeout=0.1, retries=1) + timeout=1, retries=1) self.assertEqual(r._pool.host, self.https_host) def test_headers(self): diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index d09002b..1e6113f 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -30,6 +30,7 @@ except ImportError: class MimeToolMessage(object): pass from threading import Event +import select import socket import ssl @@ -366,6 +367,72 @@ class TestSocketClosing(SocketDummyServerTestCase): self.assertRaises(ProtocolError, response.read) self.assertEqual(poolsize, pool.pool.qsize()) + def test_connection_closed_on_read_timeout_preload_false(self): + timed_out = Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # Consume request + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf = sock.recv(65535) + + # Send partial chunked response and then hang. + sock.send(( + 'HTTP/1.1 200 OK\r\n' + 'Content-Type: text/plain\r\n' + 'Transfer-Encoding: chunked\r\n' + '\r\n' + '8\r\n' + '12345678\r\n').encode('utf-8') + ) + timed_out.wait(5) + + # Expect a new request, but keep hold of the old socket to avoid + # leaking it. Because we don't want to hang this thread, we + # actually use select.select to confirm that a new request is + # coming in: this lets us time the thread out. + rlist, _, _ = select.select([listener], [], [], 1) + assert rlist + new_sock = listener.accept()[0] + + # Consume request + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf = new_sock.recv(65535) + + # Send complete chunked response. + new_sock.send(( + 'HTTP/1.1 200 OK\r\n' + 'Content-Type: text/plain\r\n' + 'Transfer-Encoding: chunked\r\n' + '\r\n' + '8\r\n' + '12345678\r\n' + '0\r\n\r\n').encode('utf-8') + ) + + new_sock.close() + sock.close() + + self._start_server(socket_handler) + with HTTPConnectionPool(self.host, self.port) as pool: + # First request should fail. + response = pool.urlopen('GET', '/', retries=0, + preload_content=False, + timeout=Timeout(connect=1, read=0.001)) + try: + self.assertRaises(ReadTimeoutError, response.read) + finally: + timed_out.set() + + # Second should succeed. + response = pool.urlopen('GET', '/', retries=0, + preload_content=False, + timeout=Timeout(connect=1, read=0.1)) + self.assertEqual(len(response.read()), 8) + class TestProxyManager(SocketDummyServerTestCase): @@ -437,6 +504,8 @@ class TestProxyManager(SocketDummyServerTestCase): self.assertTrue(b'For The Proxy: YEAH!\r\n' in r.data) def test_retries(self): + close_event = Event() + def echo_socket_handler(listener): sock = listener.accept()[0] # First request, which should fail @@ -455,6 +524,7 @@ class TestProxyManager(SocketDummyServerTestCase): '\r\n' '%s' % (len(buf), buf.decode('utf-8'))).encode('utf-8')) sock.close() + close_event.set() self._start_server(echo_socket_handler) base_url = 'http://%s:%d' % (self.host, self.port) @@ -466,6 +536,7 @@ class TestProxyManager(SocketDummyServerTestCase): assert_same_host=False, retries=1) self.assertEqual(r.status, 200) + close_event.wait(timeout=1) self.assertRaises(ProxyError, conn.urlopen, 'GET', 'http://www.google.com', assert_same_host=False, retries=False) -- cgit v1.2.3