From 2b3d330a120a16e97cecd5163b5d454dcfe38a2b Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Fri, 9 Oct 2015 23:07:31 +0200 Subject: Import python-urllib3_1.12.orig.tar.gz --- CHANGES.rst | 28 +- PKG-INFO | 30 ++- dev-requirements.txt | 12 +- dummyserver/certs/ca_path_test/98a2772e.0 | 23 ++ dummyserver/certs/ca_path_test/b6b9ccf9.0 | 23 ++ dummyserver/certs/ca_path_test/cacert.pem | 23 ++ dummyserver/handlers.py | 2 + dummyserver/server.py | 1 + dummyserver/testcase.py | 31 +++ test/__init__.py | 11 + test/test_util.py | 10 +- test/with_dummyserver/test_connectionpool.py | 325 +++++++++++++----------- test/with_dummyserver/test_https.py | 26 +- test/with_dummyserver/test_proxy_poolmanager.py | 4 +- test/with_dummyserver/test_socketlevel.py | 36 +-- urllib3.egg-info/PKG-INFO | 30 ++- urllib3.egg-info/SOURCES.txt | 4 + urllib3.egg-info/pbr.json | 1 + urllib3/__init__.py | 2 +- urllib3/connection.py | 19 +- urllib3/connectionpool.py | 20 +- urllib3/contrib/pyopenssl.py | 6 +- urllib3/exceptions.py | 3 + urllib3/response.py | 5 +- urllib3/util/connection.py | 8 +- urllib3/util/response.py | 5 +- urllib3/util/ssl_.py | 23 +- 27 files changed, 452 insertions(+), 259 deletions(-) create mode 100644 dummyserver/certs/ca_path_test/98a2772e.0 create mode 100644 dummyserver/certs/ca_path_test/b6b9ccf9.0 create mode 100644 dummyserver/certs/ca_path_test/cacert.pem create mode 100644 urllib3.egg-info/pbr.json diff --git a/CHANGES.rst b/CHANGES.rst index 7f5620f..ea10bb7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changes ======= +1.12 (2015-09-03) ++++++++++++++++++ + +* Rely on ``six`` for importing ``httplib`` to work around + conflicts with other Python 3 shims. (Issue #688) + +* Add support for directories of certificate authorities, as supported by + OpenSSL. (Issue #701) + +* New exception: ``NewConnectionError``, raised when we fail to establish + a new connection, usually ``ECONNREFUSED`` socket error. + + 1.11 (2015-07-21) +++++++++++++++++ @@ -9,10 +22,10 @@ Changes * ``pip install urllib3[secure]`` will install Certifi and PyOpenSSL as dependencies. (Issue #678) - + * Made ``HTTPHeaderDict`` usable as a ``headers`` input value (Issues #632, #679) - + * Added `urllib3.contrib.appengine `_ which has an ``AppEngineManager`` for using ``URLFetch`` in a Google AppEngine environment. (Issue #664) @@ -27,28 +40,25 @@ Changes * Fix pools not getting replenished when an error occurs during a request using ``release_conn=False``. (Issue #644) - + * Fix pool-default headers not applying for url-encoded requests like GET. (Issue #657) * log.warning in Python 3 when headers are skipped due to parsing errors. (Issue #642) - + * Close and discard connections if an error occurs during read. (Issue #660) - + * Fix host parsing for IPv6 proxies. (Issue #668) * Separate warning type SubjectAltNameWarning, now issued once per host. (Issue #671) - + * Fix ``httplib.IncompleteRead`` not getting converted to ``ProtocolError`` when using ``HTTPResponse.stream()`` (Issue #674) -* ... [Short description of non-trivial change.] (Issue #) - - 1.10.4 (2015-05-03) +++++++++++++++++++ diff --git a/PKG-INFO b/PKG-INFO index a19a535..cdd3ed3 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: urllib3 -Version: 1.11 +Version: 1.12 Summary: HTTP library with thread-safe connection pooling, file post, and more. Home-page: http://urllib3.readthedocs.org/ Author: Andrey Petrov @@ -156,6 +156,19 @@ Description: ======= Changes ======= + 1.12 (2015-09-03) + +++++++++++++++++ + + * Rely on ``six`` for importing ``httplib`` to work around + conflicts with other Python 3 shims. (Issue #688) + + * Add support for directories of certificate authorities, as supported by + OpenSSL. (Issue #701) + + * New exception: ``NewConnectionError``, raised when we fail to establish + a new connection, usually ``ECONNREFUSED`` socket error. + + 1.11 (2015-07-21) +++++++++++++++++ @@ -164,10 +177,10 @@ Description: ======= * ``pip install urllib3[secure]`` will install Certifi and PyOpenSSL as dependencies. (Issue #678) - + * Made ``HTTPHeaderDict`` usable as a ``headers`` input value (Issues #632, #679) - + * Added `urllib3.contrib.appengine `_ which has an ``AppEngineManager`` for using ``URLFetch`` in a Google AppEngine environment. (Issue #664) @@ -182,28 +195,25 @@ Description: ======= * Fix pools not getting replenished when an error occurs during a request using ``release_conn=False``. (Issue #644) - + * Fix pool-default headers not applying for url-encoded requests like GET. (Issue #657) * log.warning in Python 3 when headers are skipped due to parsing errors. (Issue #642) - + * Close and discard connections if an error occurs during read. (Issue #660) - + * Fix host parsing for IPv6 proxies. (Issue #668) * Separate warning type SubjectAltNameWarning, now issued once per host. (Issue #671) - + * Fix ``httplib.IncompleteRead`` not getting converted to ``ProtocolError`` when using ``HTTPResponse.stream()`` (Issue #674) - * ... [Short description of non-trivial change.] (Issue #) - - 1.10.4 (2015-05-03) +++++++++++++++++++ diff --git a/dev-requirements.txt b/dev-requirements.txt index a5e405d..b371cd6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ -nose==1.3.3 -nose-exclude==0.2.0 -mock==1.0.1 +nose==1.3.7 +nose-exclude==0.4.1 +mock==1.3.0 coverage==3.7.1 -tox==1.7.1 -twine==1.3.1 +tox==2.1.1 +twine==1.5.0 wheel==0.24.0 -tornado==4.1 +tornado==4.2.1 diff --git a/dummyserver/certs/ca_path_test/98a2772e.0 b/dummyserver/certs/ca_path_test/98a2772e.0 new file mode 100644 index 0000000..38d32dc --- /dev/null +++ b/dummyserver/certs/ca_path_test/98a2772e.0 @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDzDCCAzWgAwIBAgIJALPrscov4b/jMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD +VQQGEwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQK +EwVkdW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJ +KoZIhvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsMB4XDTExMTIyMjA3NTYxNVoXDTIx +MTIxOTA3NTYxNVowgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwG +A1UEBxMFZHVtbXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8G +A1UEAxMIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWww +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMrPxr1fZJ82az1N9/I1oU78rjZ8 +CNQjV0AzUbxNWiPRrzVrLtbPhHtXXN+NcVP9ahFbThjrF6TRt9/Q62xb4CuKihTL +v6k9ietyGkBbSnuE+MfUMgFVpvTUIkyFDbh6v3ZDV0XhYG/jIqoRpXUhjPVy+q8I +ImABuxafUjwKdrWXAgMBAAGjggFIMIIBRDAdBgNVHQ4EFgQUGXd/I2JiQllF+3Wd +x3NyBLszCi0wgbYGA1UdIwSBrjCBq4AUGXd/I2JiQllF+3Wdx3NyBLszCi2hgYek +gYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt +bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8GA1UEAxMIU25h +a2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWyCCQCz67HKL+G/ +4zAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBBjAJBgNVHRIEAjAA +MCsGCWCGSAGG+EIBDQQeFhxUaW55Q0EgR2VuZXJhdGVkIENlcnRpZmljYXRlMA4G +A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQUFAAOBgQBnnwtO8onsyhGOvS6cS8af +IRZyAXgouuPeP3Zrf5W80iZcV23u94969sPEIsD8Ujv5u0hUSrToGl4ahOMEOFNL +R5ndQOkh3VsepJnoE+RklZzbHWxU8onWlVzsNBFbclxidzaU3UHmdgXJAJL5nVSd +Zpn44QSS0UXsaC0mBimVNw== +-----END CERTIFICATE----- diff --git a/dummyserver/certs/ca_path_test/b6b9ccf9.0 b/dummyserver/certs/ca_path_test/b6b9ccf9.0 new file mode 100644 index 0000000..38d32dc --- /dev/null +++ b/dummyserver/certs/ca_path_test/b6b9ccf9.0 @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDzDCCAzWgAwIBAgIJALPrscov4b/jMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD +VQQGEwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQK +EwVkdW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJ +KoZIhvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsMB4XDTExMTIyMjA3NTYxNVoXDTIx +MTIxOTA3NTYxNVowgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwG +A1UEBxMFZHVtbXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8G +A1UEAxMIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWww +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMrPxr1fZJ82az1N9/I1oU78rjZ8 +CNQjV0AzUbxNWiPRrzVrLtbPhHtXXN+NcVP9ahFbThjrF6TRt9/Q62xb4CuKihTL +v6k9ietyGkBbSnuE+MfUMgFVpvTUIkyFDbh6v3ZDV0XhYG/jIqoRpXUhjPVy+q8I +ImABuxafUjwKdrWXAgMBAAGjggFIMIIBRDAdBgNVHQ4EFgQUGXd/I2JiQllF+3Wd +x3NyBLszCi0wgbYGA1UdIwSBrjCBq4AUGXd/I2JiQllF+3Wdx3NyBLszCi2hgYek +gYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt +bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8GA1UEAxMIU25h +a2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWyCCQCz67HKL+G/ +4zAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBBjAJBgNVHRIEAjAA +MCsGCWCGSAGG+EIBDQQeFhxUaW55Q0EgR2VuZXJhdGVkIENlcnRpZmljYXRlMA4G +A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQUFAAOBgQBnnwtO8onsyhGOvS6cS8af +IRZyAXgouuPeP3Zrf5W80iZcV23u94969sPEIsD8Ujv5u0hUSrToGl4ahOMEOFNL +R5ndQOkh3VsepJnoE+RklZzbHWxU8onWlVzsNBFbclxidzaU3UHmdgXJAJL5nVSd +Zpn44QSS0UXsaC0mBimVNw== +-----END CERTIFICATE----- diff --git a/dummyserver/certs/ca_path_test/cacert.pem b/dummyserver/certs/ca_path_test/cacert.pem new file mode 100644 index 0000000..38d32dc --- /dev/null +++ b/dummyserver/certs/ca_path_test/cacert.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDzDCCAzWgAwIBAgIJALPrscov4b/jMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD +VQQGEwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQK +EwVkdW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJ +KoZIhvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsMB4XDTExMTIyMjA3NTYxNVoXDTIx +MTIxOTA3NTYxNVowgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwG +A1UEBxMFZHVtbXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8G +A1UEAxMIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWww +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMrPxr1fZJ82az1N9/I1oU78rjZ8 +CNQjV0AzUbxNWiPRrzVrLtbPhHtXXN+NcVP9ahFbThjrF6TRt9/Q62xb4CuKihTL +v6k9ietyGkBbSnuE+MfUMgFVpvTUIkyFDbh6v3ZDV0XhYG/jIqoRpXUhjPVy+q8I +ImABuxafUjwKdrWXAgMBAAGjggFIMIIBRDAdBgNVHQ4EFgQUGXd/I2JiQllF+3Wd +x3NyBLszCi0wgbYGA1UdIwSBrjCBq4AUGXd/I2JiQllF+3Wdx3NyBLszCi2hgYek +gYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt +bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8GA1UEAxMIU25h +a2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWyCCQCz67HKL+G/ +4zAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBBjAJBgNVHRIEAjAA +MCsGCWCGSAGG+EIBDQQeFhxUaW55Q0EgR2VuZXJhdGVkIENlcnRpZmljYXRlMA4G +A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQUFAAOBgQBnnwtO8onsyhGOvS6cS8af +IRZyAXgouuPeP3Zrf5W80iZcV23u94969sPEIsD8Ujv5u0hUSrToGl4ahOMEOFNL +R5ndQOkh3VsepJnoE+RklZzbHWxU8onWlVzsNBFbclxidzaU3UHmdgXJAJL5nVSd +Zpn44QSS0UXsaC0mBimVNw== +-----END CERTIFICATE----- diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index ffa1dd3..43398cd 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -168,6 +168,8 @@ class TestingApp(RequestHandler): def sleep(self, request): "Sleep for a specified amount of ``seconds``" + # DO NOT USE THIS, IT'S DEPRECATED. + # FIXME: Delete this once appengine tests are fixed to not use this handler. seconds = float(request.params.get('seconds', '1')) time.sleep(seconds) return Response() diff --git a/dummyserver/server.py b/dummyserver/server.py index 1999474..e0b6345 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -37,6 +37,7 @@ NO_SAN_CERTS = { DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem') DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem') NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem') +DEFAULT_CA_DIR = os.path.join(CERTS_PATH, 'ca_path_test') def _has_ipv6(host): """ Returns True if the system can bind an IPv6 address. """ diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index de6aedd..e5ae51b 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -14,6 +14,11 @@ from dummyserver.handlers import TestingApp from dummyserver.proxy import ProxyHandler +def consume_socket(sock, chunks=65536): + while not sock.recv(chunks).endswith(b'\r\n\r\n'): + pass + + class SocketDummyServerTestCase(unittest.TestCase): """ A simple socket-based server is created for this class that is good for @@ -34,6 +39,32 @@ class SocketDummyServerTestCase(unittest.TestCase): raise Exception("most likely failed to start server") cls.port = cls.server_thread.port + @classmethod + def start_response_handler(cls, response, num=1, block_send=None): + ready_event = threading.Event() + def socket_handler(listener): + for _ in range(num): + ready_event.set() + ready_event.clear() + + sock = listener.accept()[0] + consume_socket(sock) + if block_send: + block_send.wait() + block_send.clear() + sock.send(response) + sock.close() + + cls._start_server(socket_handler) + return ready_event + + @classmethod + def start_basic_handler(cls, **kw): + return cls.start_response_handler( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n', **kw) + @classmethod def tearDownClass(cls): if hasattr(cls, 'server_thread'): diff --git a/test/__init__.py b/test/__init__.py index 172493c..f7c4a7a 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -56,6 +56,17 @@ def onlyPy27OrNewer(test): return test(*args, **kwargs) return wrapper +def onlyPy279OrNewer(test): + """Skips this test unless you are onl Python 2.7.9 or later.""" + + @functools.wraps(test) + def wrapper(*args, **kwargs): + msg = "{name} requires Python 2.7.9+ to run".format(name=test.__name__) + if sys.version_info < (2, 7, 9): + raise SkipTest(msg) + return test(*args, **kwargs) + return wrapper + def onlyPy3(test): """Skips this test unless you are on Python3.x""" diff --git a/test/test_util.py b/test/test_util.py index 19ba57e..fa59ada 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -393,7 +393,15 @@ class TestUtil(unittest.TestCase): ssl_wrap_socket(ssl_context=mock_context, ca_certs='/path/to/pem', sock=socket) mock_context.load_verify_locations.assert_called_once_with( - '/path/to/pem') + '/path/to/pem', None) + + def test_ssl_wrap_socket_loads_certificate_directories(self): + socket = object() + mock_context = Mock() + ssl_wrap_socket(ssl_context=mock_context, ca_cert_dir='/path/to/pems', + sock=socket) + mock_context.load_verify_locations.assert_called_once_with( + None, '/path/to/pems') def test_ssl_wrap_socket_with_no_sni(self): socket = object() diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 741ae7b..9294adf 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -29,22 +29,181 @@ from urllib3.exceptions import ( MaxRetryError, ReadTimeoutError, ProtocolError, + NewConnectionError, ) from urllib3.packages.six import b, u from urllib3.util.retry import Retry from urllib3.util.timeout import Timeout -import tornado -from dummyserver.testcase import HTTPDummyServerTestCase +from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS -from nose.tools import timed +from threading import Event log = logging.getLogger('urllib3.connectionpool') log.setLevel(logging.NOTSET) log.addHandler(logging.StreamHandler(sys.stdout)) +SHORT_TIMEOUT = 0.001 +LONG_TIMEOUT = 0.01 + + +class TestConnectionPoolTimeouts(SocketDummyServerTestCase): + + def test_timeout_float(self): + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=2) + + # Pool-global timeout + pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + block_event.set() # Release block + + # Shouldn't raise this time + ready_event.wait() + block_event.set() # Pre-release block + pool.request('GET', '/') + + def test_conn_closed(self): + block_event = Event() + self.start_basic_handler(block_send=block_event, num=1) + + pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) + conn = pool._get_conn() + pool._put_conn(conn) + try: + pool.urlopen('GET', '/') + self.fail("The request should fail with a timeout error.") + except ReadTimeoutError: + if conn.sock: + self.assertRaises(socket.error, conn.sock.recv, 1024) + finally: + pool._put_conn(conn) + + block_event.set() + + def test_timeout(self): + # Requests should time out when expected + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=6) + + # Pool-global timeout + timeout = Timeout(read=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + + conn = pool._get_conn() + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/') + pool._put_conn(conn) + block_event.set() # Release request + + ready_event.wait() + block_event.clear() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + block_event.set() # Release request + + # Request-specific timeouts should raise errors + pool = HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) + + conn = pool._get_conn() + ready_event.wait() + now = time.time() + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=timeout) + delta = time.time() - now + block_event.set() # Release request + + self.assertTrue(delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT") + pool._put_conn(conn) + + ready_event.wait() + now = time.time() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=timeout) + delta = time.time() - now + + self.assertTrue(delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT") + block_event.set() # Release request + + # Timeout int/float passed directly to request and _make_request should + # raise a request timeout + ready_event.wait() + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=SHORT_TIMEOUT) + block_event.set() # Release request + + ready_event.wait() + conn = pool._new_conn() + # FIXME: This assert flakes sometimes. Not sure why. + self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=SHORT_TIMEOUT) + block_event.set() # Release request + + def test_connect_timeout(self): + def noop_handler(listener): + return + + self._start_server(noop_handler) + + url = '/' + host, port = self.host, self.port + timeout = Timeout(connect=SHORT_TIMEOUT) + + # Pool-global timeout + pool = HTTPConnectionPool(host, port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) + + # Retries + retries = Retry(connect=0) + self.assertRaises(MaxRetryError, pool.request, 'GET', url, retries=retries) + + # Request-specific connection timeouts + big_timeout = Timeout(read=LONG_TIMEOUT, connect=LONG_TIMEOUT) + pool = HTTPConnectionPool(host, port, timeout=big_timeout, retries=False) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url, timeout=timeout) + + pool._put_conn(conn) + self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url, timeout=timeout) + + def test_total_applies_connect(self): + def noop_handler(listener): + return + + self._start_server(noop_handler) + + timeout = Timeout(total=None, connect=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') + + timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) + conn = pool._get_conn() + self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', '/') + + def test_total_timeout(self): + block_event = Event() + ready_event = self.start_basic_handler(block_send=block_event, num=2) + + # This will get the socket to raise an EAGAIN on the read + timeout = Timeout(connect=3, read=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + + block_event.set() + ready_event.wait() + block_event.clear() + + # The connect should succeed and this should hit the read timeout + timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT) + pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + + def test_create_connection_timeout(self): + timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT) + pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout, retries=False) + conn = pool._new_conn() + self.assertRaises(ConnectTimeoutError, conn.connect) + + class TestConnectionPool(HTTPDummyServerTestCase): def setUp(self): @@ -124,26 +283,6 @@ class TestConnectionPool(HTTPDummyServerTestCase): r = self.pool.request('POST', '/upload', fields=fields) self.assertEqual(r.status, 200, r.data) - def test_timeout_float(self): - url = '/sleep?seconds=0.005' - # Pool-global timeout - pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False) - self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) - - def test_conn_closed(self): - pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False) - conn = pool._get_conn() - pool._put_conn(conn) - try: - url = '/sleep?seconds=0.005' - pool.urlopen('GET', url) - self.fail("The request should fail with a timeout error.") - except ReadTimeoutError: - if conn.sock: - self.assertRaises(socket.error, conn.sock.recv, 1024) - finally: - pool._put_conn(conn) - def test_nagle(self): """ Test that connections have TCP_NODELAY turned on """ # This test needs to be here in order to be run. socket.create_connection actually tries to @@ -152,10 +291,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): conn = pool._get_conn() pool._make_request(conn, 'GET', '/') tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) - assert tcp_nodelay_setting > 0, ("Expected TCP_NODELAY to be set on the " - "socket (with value greater than 0) " - "but instead was %s" % - tcp_nodelay_setting) + self.assertTrue(tcp_nodelay_setting) def test_socket_options(self): """Test that connections accept socket options.""" @@ -194,79 +330,6 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertTrue(nagle_disabled) self.assertTrue(using_keepalive) - @timed(0.5) - def test_timeout(self): - """ Requests should time out when expected """ - url = '/sleep?seconds=0.003' - timeout = Timeout(read=0.001) - - # Pool-global timeout - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) - - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, - conn, 'GET', url) - pool._put_conn(conn) - - time.sleep(0.02) # Wait for server to start receiving again. :( - - self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) - - # Request-specific timeouts should raise errors - pool = HTTPConnectionPool(self.host, self.port, timeout=0.1, retries=False) - - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, - conn, 'GET', url, timeout=timeout) - pool._put_conn(conn) - - time.sleep(0.02) # Wait for server to start receiving again. :( - - self.assertRaises(ReadTimeoutError, pool.request, - 'GET', url, timeout=timeout) - - # Timeout int/float passed directly to request and _make_request should - # raise a request timeout - self.assertRaises(ReadTimeoutError, pool.request, - 'GET', url, timeout=0.001) - conn = pool._new_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, - 'GET', url, timeout=0.001) - pool._put_conn(conn) - - # Timeout int/float passed directly to _make_request should not raise a - # request timeout if it's a high value - pool.request('GET', url, timeout=1) - - @requires_network - @timed(0.5) - def test_connect_timeout(self): - url = '/sleep?seconds=0.005' - timeout = Timeout(connect=0.001) - - # Pool-global timeout - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) - - # Retries - retries = Retry(connect=0) - self.assertRaises(MaxRetryError, pool.request, 'GET', url, - retries=retries) - - # Request-specific connection timeouts - big_timeout = Timeout(read=0.2, connect=0.2) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, - timeout=big_timeout, retries=False) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', - url, timeout=timeout) - - pool._put_conn(conn) - self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url, - timeout=timeout) - - def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() @@ -275,50 +338,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool.request('GET', '/', retries=Retry(connect=3)) self.fail("Should have failed with a connection error.") except MaxRetryError as e: - self.assertTrue(isinstance(e.reason, ProtocolError)) - self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED) - - def test_timeout_reset(self): - """ If the read timeout isn't set, socket timeout should reset """ - url = '/sleep?seconds=0.005' - timeout = Timeout(connect=0.001) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - try: - pool._make_request(conn, 'GET', url) - except ReadTimeoutError: - self.fail("This request shouldn't trigger a read timeout.") - - @requires_network - @timed(5.0) - def test_total_timeout(self): - url = '/sleep?seconds=0.005' - - timeout = Timeout(connect=3, read=5, total=0.001) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url) - - # This will get the socket to raise an EAGAIN on the read - timeout = Timeout(connect=3, read=0) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) - - # The connect should succeed and this should hit the read timeout - timeout = Timeout(connect=3, read=5, total=0.002) - pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) - - @requires_network - def test_none_total_applies_connect(self): - url = '/sleep?seconds=0.005' - timeout = Timeout(total=None, connect=0.001) - pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout) - conn = pool._get_conn() - self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', - url) + self.assertEqual(type(e.reason), NewConnectionError) def test_timeout_success(self): timeout = Timeout(connect=3, read=5, total=None) @@ -372,7 +392,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool.request('GET', '/', retries=5) self.fail("should raise timeout exception here") except MaxRetryError as e: - self.assertTrue(isinstance(e.reason, ProtocolError), e.reason) + self.assertEqual(type(e.reason), NewConnectionError) def test_keepalive(self): pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) @@ -607,16 +627,13 @@ class TestConnectionPool(HTTPDummyServerTestCase): pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) r = pool.request('GET', '/source_address') - assert r.data == b(addr[0]), ( - "expected the response to contain the source address {addr}, " - "but was {data}".format(data=r.data, addr=b(addr[0]))) + self.assertEqual(r.data, b(addr[0])) def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: - pool = HTTPConnectionPool(self.host, self.port, - source_address=addr, retries=False) - self.assertRaises(ProtocolError, - pool.request, 'GET', '/source_address') + pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) + # FIXME: This assert flakes sometimes. Not sure why. + self.assertRaises(NewConnectionError, pool.request, 'GET', '/source_address?{0}'.format(addr)) def test_stream_keepalive(self): x = 2 @@ -669,6 +686,8 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertEqual(http.pool.qsize(), http.pool.maxsize) + + class TestRetry(HTTPDummyServerTestCase): def setUp(self): self.pool = HTTPConnectionPool(self.host, self.port) @@ -695,7 +714,7 @@ class TestRetry(HTTPDummyServerTestCase): self.assertEqual(r.status, 303) pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001) - self.assertRaises(ProtocolError, pool.request, 'GET', '/test', retries=False) + self.assertRaises(NewConnectionError, pool.request, 'GET', '/test', retries=False) def test_read_retries(self): """ Should retry for status codes in the whitelist """ diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 63aea66..862ebd9 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -10,10 +10,11 @@ from nose.plugins.skip import SkipTest from dummyserver.testcase import HTTPSDummyServerTestCase from dummyserver.server import (DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS, - NO_SAN_CERTS, NO_SAN_CA) + NO_SAN_CERTS, NO_SAN_CA, DEFAULT_CA_DIR) from test import ( onlyPy26OrOlder, + onlyPy279OrNewer, requires_network, TARPIT_HOST, clear_warnings, @@ -80,6 +81,27 @@ class TestHTTPS(HTTPSDummyServerTestCase): error = call[0][1] self.assertEqual(error, InsecurePlatformWarning) + @onlyPy279OrNewer + def test_ca_dir_verified(self): + https_pool = HTTPSConnectionPool(self.host, self.port, + cert_reqs='CERT_REQUIRED', + ca_cert_dir=DEFAULT_CA_DIR) + + conn = https_pool._new_conn() + self.assertEqual(conn.__class__, VerifiedHTTPSConnection) + + with mock.patch('warnings.warn') as warn: + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + + if sys.version_info >= (2, 7, 9): + self.assertFalse(warn.called, warn.call_args_list) + else: + self.assertTrue(warn.called) + call, = warn.call_args_list + error = call[0][1] + self.assertEqual(error, InsecurePlatformWarning) + def test_invalid_common_name(self): https_pool = HTTPSConnectionPool('127.0.0.1', self.port, cert_reqs='CERT_REQUIRED', @@ -296,8 +318,6 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool.ca_certs = DEFAULT_CA https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \ '7A:F2:8A:D7:1E:07:33:67:DE' - url = '/sleep?seconds=0.005' - self.assertRaises(ReadTimeoutError, https_pool.request, 'GET', url) timeout = Timeout(total=None) https_pool = HTTPSConnectionPool(self.host, self.port, timeout=timeout, diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index c593f2d..b2894a8 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -287,7 +287,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): https.request('GET', self.http_url, timeout=0.001) self.fail("Failed to raise retry error.") except MaxRetryError as e: - assert isinstance(e.reason, ConnectTimeoutError) + self.assertEqual(type(e.reason), ConnectTimeoutError) @timed(0.5) @@ -298,7 +298,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): https.request('GET', self.http_url) self.fail("Failed to raise retry error.") except MaxRetryError as e: - assert isinstance(e.reason, ConnectTimeoutError) + self.assertEqual(type(e.reason), ConnectTimeoutError) class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase): diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 5af00e0..d09002b 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -6,6 +6,7 @@ from urllib3.poolmanager import proxy_from_url from urllib3.exceptions import ( MaxRetryError, ProxyError, + ConnectTimeoutError, ReadTimeoutError, SSLError, ProtocolError, @@ -629,42 +630,23 @@ class TestSSL(SocketDummyServerTestCase): self.assertRaises(SSLError, request) -def consume_socket(sock, chunks=65536): - while not sock.recv(chunks).endswith(b'\r\n\r\n'): - pass - - -def create_response_handler(response, num=1): - def socket_handler(listener): - for _ in range(num): - sock = listener.accept()[0] - consume_socket(sock) - - sock.send(response) - sock.close() - - return socket_handler - - class TestErrorWrapping(SocketDummyServerTestCase): def test_bad_statusline(self): - handler = create_response_handler( + self.start_response_handler( b'HTTP/1.1 Omg What Is This?\r\n' b'Content-Length: 0\r\n' b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) self.assertRaises(ProtocolError, pool.request, 'GET', '/') def test_unknown_protocol(self): - handler = create_response_handler( + self.start_response_handler( b'HTTP/1000 200 OK\r\n' b'Content-Length: 0\r\n' b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) self.assertRaises(ProtocolError, pool.request, 'GET', '/') @@ -672,13 +654,12 @@ class TestHeaders(SocketDummyServerTestCase): @onlyPy3 def test_httplib_headers_case_insensitive(self): - handler = create_response_handler( + self.start_response_handler( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' b'Content-type: text/plain\r\n' b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain'} r = pool.request('GET', '/') @@ -727,14 +708,13 @@ class TestBrokenHeaders(SocketDummyServerTestCase): super(TestBrokenHeaders, self).setUp() def _test_broken_header_parsing(self, headers): - handler = create_response_handler(( + self.start_response_handler(( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' b'Content-type: text/plain\r\n' ) + b'\r\n'.join(headers) + b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) with LogRecorder() as logs: @@ -767,13 +747,12 @@ class TestBrokenHeaders(SocketDummyServerTestCase): class TestHEAD(SocketDummyServerTestCase): def test_chunked_head_response_does_not_hang(self): - handler = create_response_handler( + self.start_response_handler( b'HTTP/1.1 200 OK\r\n' b'Transfer-Encoding: chunked\r\n' b'Content-type: text/plain\r\n' b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) r = pool.request('HEAD', '/', timeout=1, preload_content=False) @@ -781,13 +760,12 @@ class TestHEAD(SocketDummyServerTestCase): self.assertEqual([], list(r.stream())) def test_empty_head_response_does_not_hang(self): - handler = create_response_handler( + self.start_response_handler( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 256\r\n' b'Content-type: text/plain\r\n' b'\r\n' ) - self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) r = pool.request('HEAD', '/', timeout=1, preload_content=False) diff --git a/urllib3.egg-info/PKG-INFO b/urllib3.egg-info/PKG-INFO index a19a535..cdd3ed3 100644 --- a/urllib3.egg-info/PKG-INFO +++ b/urllib3.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: urllib3 -Version: 1.11 +Version: 1.12 Summary: HTTP library with thread-safe connection pooling, file post, and more. Home-page: http://urllib3.readthedocs.org/ Author: Andrey Petrov @@ -156,6 +156,19 @@ Description: ======= Changes ======= + 1.12 (2015-09-03) + +++++++++++++++++ + + * Rely on ``six`` for importing ``httplib`` to work around + conflicts with other Python 3 shims. (Issue #688) + + * Add support for directories of certificate authorities, as supported by + OpenSSL. (Issue #701) + + * New exception: ``NewConnectionError``, raised when we fail to establish + a new connection, usually ``ECONNREFUSED`` socket error. + + 1.11 (2015-07-21) +++++++++++++++++ @@ -164,10 +177,10 @@ Description: ======= * ``pip install urllib3[secure]`` will install Certifi and PyOpenSSL as dependencies. (Issue #678) - + * Made ``HTTPHeaderDict`` usable as a ``headers`` input value (Issues #632, #679) - + * Added `urllib3.contrib.appengine `_ which has an ``AppEngineManager`` for using ``URLFetch`` in a Google AppEngine environment. (Issue #664) @@ -182,28 +195,25 @@ Description: ======= * Fix pools not getting replenished when an error occurs during a request using ``release_conn=False``. (Issue #644) - + * Fix pool-default headers not applying for url-encoded requests like GET. (Issue #657) * log.warning in Python 3 when headers are skipped due to parsing errors. (Issue #642) - + * Close and discard connections if an error occurs during read. (Issue #660) - + * Fix host parsing for IPv6 proxies. (Issue #668) * Separate warning type SubjectAltNameWarning, now issued once per host. (Issue #671) - + * Fix ``httplib.IncompleteRead`` not getting converted to ``ProtocolError`` when using ``HTTPResponse.stream()`` (Issue #674) - * ... [Short description of non-trivial change.] (Issue #) - - 1.10.4 (2015-05-03) +++++++++++++++++++ diff --git a/urllib3.egg-info/SOURCES.txt b/urllib3.egg-info/SOURCES.txt index 2f96e50..229925c 100644 --- a/urllib3.egg-info/SOURCES.txt +++ b/urllib3.egg-info/SOURCES.txt @@ -39,6 +39,9 @@ dummyserver/certs/server.key dummyserver/certs/server.key.org dummyserver/certs/server.no_san.crt dummyserver/certs/server.no_san.csr +dummyserver/certs/ca_path_test/98a2772e.0 +dummyserver/certs/ca_path_test/b6b9ccf9.0 +dummyserver/certs/ca_path_test/cacert.pem test/__init__.py test/benchmark.py test/port_helpers.py @@ -82,6 +85,7 @@ urllib3/response.py urllib3.egg-info/PKG-INFO urllib3.egg-info/SOURCES.txt urllib3.egg-info/dependency_links.txt +urllib3.egg-info/pbr.json urllib3.egg-info/requires.txt urllib3.egg-info/top_level.txt urllib3/contrib/__init__.py diff --git a/urllib3.egg-info/pbr.json b/urllib3.egg-info/pbr.json new file mode 100644 index 0000000..0af0534 --- /dev/null +++ b/urllib3.egg-info/pbr.json @@ -0,0 +1 @@ +{"is_release": false, "git_version": "d7d9caa"} \ No newline at end of file diff --git a/urllib3/__init__.py b/urllib3/__init__.py index 747d09a..86bb71d 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.11' +__version__ = '1.12' from .connectionpool import ( diff --git a/urllib3/connection.py b/urllib3/connection.py index f64dd1a..3eab1e2 100644 --- a/urllib3/connection.py +++ b/urllib3/connection.py @@ -1,7 +1,7 @@ import datetime import sys import socket -from socket import timeout as SocketTimeout +from socket import error as SocketError, timeout as SocketTimeout import warnings from .packages import six @@ -36,9 +36,10 @@ except NameError: # Python 2: from .exceptions import ( + NewConnectionError, ConnectTimeoutError, - SystemTimeWarning, SubjectAltNameWarning, + SystemTimeWarning, ) from .packages.ssl_match_hostname import match_hostname @@ -133,11 +134,15 @@ class HTTPConnection(_HTTPConnection, object): conn = connection.create_connection( (self.host, self.port), self.timeout, **extra_kw) - except SocketTimeout: + except SocketTimeout as e: raise ConnectTimeoutError( self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) + except SocketError as e: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + return conn def _prepare_conn(self, conn): @@ -185,20 +190,23 @@ class VerifiedHTTPSConnection(HTTPSConnection): """ cert_reqs = None ca_certs = None + ca_cert_dir = None ssl_version = None assert_fingerprint = None def set_cert(self, key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, - assert_hostname=None, assert_fingerprint=None): + assert_hostname=None, assert_fingerprint=None, + ca_cert_dir=None): - if ca_certs and cert_reqs is None: + if (ca_certs or ca_cert_dir) and cert_reqs is None: cert_reqs = 'CERT_REQUIRED' self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs self.ca_certs = ca_certs + self.ca_cert_dir = ca_cert_dir self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint @@ -237,6 +245,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, server_hostname=hostname, ssl_version=resolved_ssl_version) diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index c958725..b38ac68 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -22,10 +22,12 @@ from .exceptions import ( LocationValueError, MaxRetryError, ProxyError, + ConnectTimeoutError, ReadTimeoutError, SSLError, TimeoutError, InsecureRequestWarning, + NewConnectionError, ) from .packages.ssl_match_hostname import CertificateError from .packages import six @@ -422,7 +424,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - + # Use explicit default port for comparison when none is given if self.port and not port: port = port_by_scheme.get(scheme) @@ -592,13 +594,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn = True raise - except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: + except (TimeoutError, HTTPException, SocketError, ProtocolError) as e: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. conn = conn and conn.close() release_conn = True - if isinstance(e, SocketError) and self.proxy: + if isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) elif isinstance(e, (SocketError, HTTPException)): e = ProtocolError('Connection aborted.', e) @@ -675,10 +677,10 @@ class HTTPSConnectionPool(HTTPConnectionPool): ``assert_hostname`` and ``host`` in this order to verify connections. If ``assert_hostname`` is False, no verification is done. - The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and - ``ssl_version`` are only used if :mod:`ssl` is available and are fed into - :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket - into an SSL socket. + The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, + ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is + available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + the connection socket into an SSL socket. """ scheme = 'https' @@ -691,7 +693,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, ssl_version=None, assert_hostname=None, assert_fingerprint=None, - **conn_kw): + ca_cert_dir=None, **conn_kw): HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, block, headers, retries, _proxy, _proxy_headers, @@ -704,6 +706,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): self.cert_file = cert_file self.cert_reqs = cert_reqs self.ca_certs = ca_certs + self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint @@ -719,6 +722,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py index 19c5b4e..c20ae46 100644 --- a/urllib3/contrib/pyopenssl.py +++ b/urllib3/contrib/pyopenssl.py @@ -267,7 +267,7 @@ def _verify_callback(cnx, x509, err_no, err_depth, return_code): def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=None): + ssl_version=None, ca_cert_dir=None): ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: keyfile = keyfile or certfile # Match behaviour of the normal python ssl library @@ -276,9 +276,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ctx.use_privatekey_file(keyfile) if cert_reqs != ssl.CERT_NONE: ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback) - if ca_certs: + if ca_certs or ca_cert_dir: try: - ctx.load_verify_locations(ca_certs, None) + ctx.load_verify_locations(ca_certs, ca_cert_dir) except OpenSSL.SSL.Error as e: raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e) else: diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py index 36ce0d1..9607d65 100644 --- a/urllib3/exceptions.py +++ b/urllib3/exceptions.py @@ -112,6 +112,9 @@ class ConnectTimeoutError(TimeoutError): "Raised when a socket timeout occurs while connecting to a server" pass +class NewConnectionError(ConnectTimeoutError, PoolError): + "Raised when we fail to establish a new connection. Usually ECONNREFUSED." + pass class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." diff --git a/urllib3/response.py b/urllib3/response.py index 15d4aac..788eb6c 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -1,7 +1,3 @@ -try: - import http.client as httplib -except ImportError: - import httplib from contextlib import contextmanager import zlib import io @@ -12,6 +8,7 @@ from .exceptions import ( ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked ) from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head diff --git a/urllib3/util/connection.py b/urllib3/util/connection.py index 9ed5a64..4f2f0f1 100644 --- a/urllib3/util/connection.py +++ b/urllib3/util/connection.py @@ -80,16 +80,16 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, sock.connect(sa) return sock - except socket.error as _: - err = _ + except socket.error as e: + err = e if sock is not None: sock.close() sock = None if err is not None: raise err - else: - raise socket.error("getaddrinfo returns an empty list") + + raise socket.error("getaddrinfo returns an empty list") def _set_socket_options(sock, options): diff --git a/urllib3/util/response.py b/urllib3/util/response.py index 767ee15..2c1de15 100644 --- a/urllib3/util/response.py +++ b/urllib3/util/response.py @@ -1,7 +1,4 @@ -try: - import http.client as httplib -except ImportError: - import httplib +from ..packages.six.moves import http_client as httplib from ..exceptions import HeaderParsingError diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py index 311378b..47b817e 100644 --- a/urllib3/util/ssl_.py +++ b/urllib3/util/ssl_.py @@ -75,8 +75,11 @@ except ImportError: self.certfile = certfile self.keyfile = keyfile - def load_verify_locations(self, location): - self.ca_certs = location + def load_verify_locations(self, cafile=None, capath=None): + self.ca_certs = cafile + + if capath is not None: + raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): if not self.supports_set_ciphers: @@ -240,10 +243,11 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=None, ciphers=None, ssl_context=None): + ssl_version=None, ciphers=None, ssl_context=None, + ca_cert_dir=None): """ - All arguments except for server_hostname and ssl_context have the same - meaning as they do when using :func:`ssl.wrap_socket`. + All arguments except for server_hostname, ssl_context, and ca_cert_dir have + the same meaning as they do when using :func:`ssl.wrap_socket`. :param server_hostname: When SNI is supported, the expected hostname of the certificate @@ -253,15 +257,19 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, :param ciphers: A string of ciphers we wish the client to support. This is not supported on Python 2.6 as the ssl module does not support it. + :param ca_cert_dir: + A directory containing CA certificates in multiple separate files, as + supported by OpenSSL's -CApath flag or the capath argument to + SSLContext.load_verify_locations(). """ context = ssl_context if context is None: context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) - if ca_certs: + if ca_certs or ca_cert_dir: try: - context.load_verify_locations(ca_certs) + context.load_verify_locations(ca_certs, ca_cert_dir) except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError @@ -270,6 +278,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise + if certfile: context.load_cert_chain(certfile, keyfile) if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI -- cgit v1.2.3 From 3a7deaa269ec9e1dc5dfb860e29b053475647cf5 Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Thu, 8 Oct 2015 13:19:46 -0700 Subject: Do not use embedded copy of python-six. Forwarded: not-needed Last-Update: 2015-05-03 Patch-Name: 01_do-not-use-embedded-python-six.patch --- dummyserver/handlers.py | 2 +- test/__init__.py | 2 +- test/contrib/test_pyopenssl.py | 2 +- test/test_collections.py | 2 +- test/test_fields.py | 2 +- test/test_filepost.py | 2 +- test/test_retry.py | 2 +- urllib3/_collections.py | 2 +- urllib3/connection.py | 2 +- urllib3/connectionpool.py | 2 +- urllib3/fields.py | 2 +- urllib3/filepost.py | 4 ++-- urllib3/response.py | 4 ++-- urllib3/util/request.py | 2 +- urllib3/util/retry.py | 2 +- 15 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 43398cd..a7828a3 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -263,7 +263,7 @@ def _parse_header(line): """ import tornado.httputil import email.utils - from urllib3.packages import six + import six if not six.PY3: line = line.encode('utf-8') parts = tornado.httputil._parseparam(';' + line) diff --git a/test/__init__.py b/test/__init__.py index f7c4a7a..22d3616 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -8,7 +8,7 @@ import socket from nose.plugins.skip import SkipTest from urllib3.exceptions import MaxRetryError, HTTPWarning -from urllib3.packages import six +import six # We need a host that will not immediately close the connection with a TCP # Reset. SO suggests this hostname diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py index 5d57527..f23ff19 100644 --- a/test/contrib/test_pyopenssl.py +++ b/test/contrib/test_pyopenssl.py @@ -1,5 +1,5 @@ from nose.plugins.skip import SkipTest -from urllib3.packages import six +import six if six.PY3: raise SkipTest('Testing of PyOpenSSL disabled on PY3') diff --git a/test/test_collections.py b/test/test_collections.py index 9d72939..78ef634 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -4,7 +4,7 @@ from urllib3._collections import ( HTTPHeaderDict, RecentlyUsedContainer as Container ) -from urllib3.packages import six +import six xrange = six.moves.xrange from nose.plugins.skip import SkipTest diff --git a/test/test_fields.py b/test/test_fields.py index cdec68b..66da148 100644 --- a/test/test_fields.py +++ b/test/test_fields.py @@ -1,7 +1,7 @@ import unittest from urllib3.fields import guess_content_type, RequestField -from urllib3.packages.six import u +from six import u class TestRequestField(unittest.TestCase): diff --git a/test/test_filepost.py b/test/test_filepost.py index 390dbb3..ecc6710 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -2,7 +2,7 @@ import unittest from urllib3.filepost import encode_multipart_formdata, iter_fields from urllib3.fields import RequestField -from urllib3.packages.six import b, u +from six import b, u BOUNDARY = '!! test boundary !!' diff --git a/test/test_retry.py b/test/test_retry.py index 421e508..8fcc287 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -1,7 +1,7 @@ import unittest from urllib3.response import HTTPResponse -from urllib3.packages.six.moves import xrange +from six.moves import xrange from urllib3.util.retry import Retry from urllib3.exceptions import ( ConnectTimeoutError, diff --git a/urllib3/_collections.py b/urllib3/_collections.py index b68b9a5..af8074a 100644 --- a/urllib3/_collections.py +++ b/urllib3/_collections.py @@ -14,7 +14,7 @@ try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict -from .packages.six import iterkeys, itervalues, PY3 +from six import iterkeys, itervalues, PY3 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] diff --git a/urllib3/connection.py b/urllib3/connection.py index 3eab1e2..115eac9 100644 --- a/urllib3/connection.py +++ b/urllib3/connection.py @@ -3,7 +3,7 @@ import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings -from .packages import six +import six try: # Python 3 from http.client import HTTPConnection as _HTTPConnection, HTTPException diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index b38ac68..563f108 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -30,7 +30,7 @@ from .exceptions import ( NewConnectionError, ) from .packages.ssl_match_hostname import CertificateError -from .packages import six +import six from .connection import ( port_by_scheme, DummyConnection, diff --git a/urllib3/fields.py b/urllib3/fields.py index c853f8d..5fe3c24 100644 --- a/urllib3/fields.py +++ b/urllib3/fields.py @@ -1,7 +1,7 @@ import email.utils import mimetypes -from .packages import six +import six def guess_content_type(filename, default='application/octet-stream'): diff --git a/urllib3/filepost.py b/urllib3/filepost.py index 0fbf488..97ab970 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -3,8 +3,8 @@ import codecs from uuid import uuid4 from io import BytesIO -from .packages import six -from .packages.six import b +import six +from six import b from .fields import RequestField writer = codecs.lookup('utf-8')[3] diff --git a/urllib3/response.py b/urllib3/response.py index 788eb6c..c41bd2c 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -7,8 +7,8 @@ from ._collections import HTTPHeaderDict from .exceptions import ( ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked ) -from .packages.six import string_types as basestring, binary_type, PY3 -from .packages.six.moves import http_client as httplib +from six import string_types as basestring, binary_type, PY3 +from six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head diff --git a/urllib3/util/request.py b/urllib3/util/request.py index bc64f6b..5f4ccfd 100644 --- a/urllib3/util/request.py +++ b/urllib3/util/request.py @@ -1,6 +1,6 @@ from base64 import b64encode -from ..packages.six import b +from six import b ACCEPT_ENCODING = 'gzip,deflate' diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py index 1fb1f23..178b374 100644 --- a/urllib3/util/retry.py +++ b/urllib3/util/retry.py @@ -8,7 +8,7 @@ from ..exceptions import ( ReadTimeoutError, ResponseError, ) -from ..packages import six +import six log = logging.getLogger(__name__) -- cgit v1.2.3 From f27a60baa815150ce224d1be75464c46d987f749 Mon Sep 17 00:00:00 2001 From: Jamie Strandboge Date: Thu, 8 Oct 2015 13:19:47 -0700 Subject: require SSL certificate validation by default by using CERT_REQUIRED and using the system /etc/ssl/certs/ca-certificates.crt Bug-Ubuntu: https://launchpad.net/bugs/1047054 Bug-Debian: http://bugs.debian.org/686872 Last-Update: 2014-09-01 Patch-Name: 02_require-cert-verification.patch --- urllib3/connectionpool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index 563f108..a99d4ec 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -681,6 +681,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket. + + On Debian, SSL certificate validation is required by default """ scheme = 'https' @@ -690,8 +692,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, headers=None, retries=None, _proxy=None, _proxy_headers=None, - key_file=None, cert_file=None, cert_reqs=None, - ca_certs=None, ssl_version=None, + key_file=None, cert_file=None, cert_reqs='CERT_REQUIRED', + ca_certs='/etc/ssl/certs/ca-certificates.crt', ssl_version=None, assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None, **conn_kw): -- cgit v1.2.3 From a89ef67cc90bdfc8d4832cdd07baed149bdffd88 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 8 Oct 2015 13:19:49 -0700 Subject: Use setuptools.setup() so that the bdist_wheel command will work. Last-Update: 2014-05-15 Patch-Name: 03_force_setuptools.patch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6bdb7b9..1e1888d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup import os import re -- cgit v1.2.3 From 966d73488bc803f437395d8337bd322a0016084f Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Thu, 8 Oct 2015 13:19:50 -0700 Subject: Do not use logging-clear-handlers to see all logging output and disable cover-min-percentage since it require python-nose (>= 1.3): this way it will be easier to backport python-urllib3 to Wheezy. Forwarded: not-needed Last-Update: 2014-7-7 Patch-Name: 04_relax_nosetests_options.patch --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6a64ffa..25a6815 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,8 @@ [nosetests] -logging-clear-handlers = true +# logging-clear-handlers = true with-coverage = true cover-package = urllib3 -cover-min-percentage = 100 +# cover-min-percentage = 100 cover-erase = true [flake8] -- cgit v1.2.3 From 797a60975f0ff4dcf615fca6f8b51ce62e5e91a6 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 8 Oct 2015 13:19:51 -0700 Subject: Do not use embedded copy of ssl.match_hostname, when possible The system python has the necessary features backported, since 2.7.8-7 (and 221a1f9155e2, releasing in 2.7.9, upstream). However, alternative python implementations don't, yet, and urllib3 is used by pip in virtualenvs. Forwarded: not-needed Last-Update: 2014-11-18 Patch-Name: 05_avoid-embedded-ssl-match-hostname.patch --- urllib3/packages/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/urllib3/packages/__init__.py b/urllib3/packages/__init__.py index 37e8351..10a3aa8 100644 --- a/urllib3/packages/__init__.py +++ b/urllib3/packages/__init__.py @@ -1,4 +1,9 @@ from __future__ import absolute_import -from . import ssl_match_hostname - +try: + # cPython >= 2.7.9 has ssl features backported from Python3 + from ssl import CertificateError + del CertificateError + import ssl as ssl_match_hostname +except ImportError: + from . import ssl_match_hostname -- cgit v1.2.3 From 7db0aa9b18e70f43ef0bfcdec7f223e5b681aaf0 Mon Sep 17 00:00:00 2001 From: SVN-Git Migration Date: Thu, 8 Oct 2015 13:19:52 -0700 Subject: Rely on six to import httplib or http.client. Origin: https://github.com/shazow/urllib3/commit/f4eb94bc36277d5d584683a03fc9eb3950429a15 Patch-Name: 06_rely-on-six-to-import-httplib-or-http.client.patch --- urllib3/util/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/util/response.py b/urllib3/util/response.py index 2c1de15..6695809 100644 --- a/urllib3/util/response.py +++ b/urllib3/util/response.py @@ -1,4 +1,4 @@ -from ..packages.six.moves import http_client as httplib +from six.moves import http_client as httplib from ..exceptions import HeaderParsingError -- cgit v1.2.3