diff options
55 files changed, 774 insertions, 204 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index dd2cd2d..552d9b7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,38 @@ Changes ======= +1.10 (2014-12-14) ++++++++++++++++++ + +* Disabled SSLv3. (Issue #473) + +* Add ``Url.url`` property to return the composed url string. (Issue #394) + +* Fixed PyOpenSSL + gevent ``WantWriteError``. (Issue #412) + +* ``MaxRetryError.reason`` will always be an exception, not string. + (Issue #481) + +* Fixed SSL-related timeouts not being detected as timeouts. (Issue #492) + +* Py3: Use ``ssl.create_default_context()`` when available. (Issue #473) + +* Emit ``InsecureRequestWarning`` for *every* insecure HTTPS request. + (Issue #496) + +* Emit ``SecurityWarning`` when certificate has no ``subjectAltName``. + (Issue #499) + +* Close and discard sockets which experienced SSL-related errors. + (Issue #501) + +* Handle ``body`` param in ``.request(...)``. (Issue #513) + +* Respect timeout with HTTPS proxy. (Issue #505) + +* PyOpenSSL: Handle ZeroReturnError exception. (Issue #520) + + 1.9.1 (2014-09-13) ++++++++++++++++++ diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 97f3014..ecaf9bb 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -127,5 +127,11 @@ In chronological order: * Krishna Prasad <kprasad.iitd@gmail.com> * Google App Engine documentation +* Aaron Meurer <asmeurer@gmail.com> + * Added Url.url, which unparses a Url + +* Evgeny Kapun <abacabadabacaba@gmail.com> + * Bugfixes + * [Your name or handle] <[email or website]> * [Brief summary of your changes] @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: urllib3 -Version: 1.9.1 +Version: 1.10 Summary: HTTP library with thread-safe connection pooling, file post, and more. Home-page: http://urllib3.readthedocs.org/ Author: Andrey Petrov @@ -156,6 +156,38 @@ Description: ======= Changes ======= + 1.10 (2014-12-14) + +++++++++++++++++ + + * Disabled SSLv3. (Issue #473) + + * Add ``Url.url`` property to return the composed url string. (Issue #394) + + * Fixed PyOpenSSL + gevent ``WantWriteError``. (Issue #412) + + * ``MaxRetryError.reason`` will always be an exception, not string. + (Issue #481) + + * Fixed SSL-related timeouts not being detected as timeouts. (Issue #492) + + * Py3: Use ``ssl.create_default_context()`` when available. (Issue #473) + + * Emit ``InsecureRequestWarning`` for *every* insecure HTTPS request. + (Issue #496) + + * Emit ``SecurityWarning`` when certificate has no ``subjectAltName``. + (Issue #499) + + * Close and discard sockets which experienced SSL-related errors. + (Issue #501) + + * Handle ``body`` param in ``.request(...)``. (Issue #513) + + * Respect timeout with HTTPS proxy. (Issue #505) + + * PyOpenSSL: Handle ZeroReturnError exception. (Issue #520) + + 1.9.1 (2014-09-13) ++++++++++++++++++ diff --git a/dev-requirements.txt b/dev-requirements.txt index 8010704..2eb5875 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,8 @@ nose==1.3.3 mock==1.0.1 coverage==3.7.1 tox==1.7.1 +twine==1.3.1 +wheel==0.24.0 # Tornado 3.2.2 makes our tests flaky, so we stick with 3.1 tornado==3.1.1 diff --git a/docs/security.rst b/docs/security.rst index 5321e24..0566737 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -147,7 +147,6 @@ Unverified HTTPS requests will trigger a warning:: urllib3/connectionpool.py:736: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html - (This warning will only appear once by default.) This would be a great time to enable HTTPS verification: :ref:`certifi-with-urllib3`. diff --git a/dummyserver/__init__.pyc b/dummyserver/__init__.pyc Binary files differdeleted file mode 100644 index 24e9f56..0000000 --- a/dummyserver/__init__.pyc +++ /dev/null diff --git a/dummyserver/certs/README.rst b/dummyserver/certs/README.rst new file mode 100644 index 0000000..4fb6632 --- /dev/null +++ b/dummyserver/certs/README.rst @@ -0,0 +1,24 @@ +Creating a new SAN-less CRT +--------------------------- + +(Instructions lifted from Heroku_) + +1. Generate a new CSR:: + + openssl req -new -key server.key -out server.new.csr -nodes -days 10957 + +2. Generate a new CRT:: + + openssl x509 -req -in server.new.csr -signkey server.key -out server.new.crt -days 10957 + +Creating a new PEM file with your new CRT +----------------------------------------- + +1. Concatenate the ``crt`` and ``key`` files into one:: + + cat server.new.crt server.key > cacert.new.pem + + +:Last Modified: 1 Nov 2014 + +.. _Heroku: https://devcenter.heroku.com/articles/ssl-certificate-self diff --git a/dummyserver/certs/cacert.no_san.pem b/dummyserver/certs/cacert.no_san.pem new file mode 100644 index 0000000..6df351b --- /dev/null +++ b/dummyserver/certs/cacert.no_san.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIChzCCAfACCQCmk6is+6REjDANBgkqhkiG9w0BAQUFADCBhzELMAkGA1UEBhMC +Q0ExEDAOBgNVBAgMB09udGFyaW8xEDAOBgNVBAcMB09udGFyaW8xHzAdBgNVBAoM +FlNoYXpvdydzIFVzZWQgQ2FycyBJbmMxEjAQBgNVBAMMCWxvY2FsaG9zdDEfMB0G +CSqGSIb3DQEJARYQc2hhem93QGdtYWlsLmNvbTAeFw0xNDEyMDMyMjE3MjVaFw00 +NDEyMDIyMjE3MjVaMIGHMQswCQYDVQQGEwJDQTEQMA4GA1UECAwHT250YXJpbzEQ +MA4GA1UEBwwHT250YXJpbzEfMB0GA1UECgwWU2hhem93J3MgVXNlZCBDYXJzIElu +YzESMBAGA1UEAwwJbG9jYWxob3N0MR8wHQYJKoZIhvcNAQkBFhBzaGF6b3dAZ21h +aWwuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXe3FqmCWvP8XPxqtT ++0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB +0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN +3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAHI/m9/O +bVR3zBOJZUKlHzTRvfYbYhhfrrcQlbwhjKqNyZcQTL/bJdtQSL19g3ftC5wZPI+y +66R24MqGmRcv5kT32HcuIK1Xhx4nDqTqnTNvGkaIh5CqS4DEP+iqtwDoEbQt8DwL +ejKtvZlyQRKFPTMtmv4VsTIHeVOAj+pXn595 +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDXe3FqmCWvP8XPxqtT+0bfL1Tvzvebi46k0WIcUV8bP3vyYiSR +XG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB0y9ai/9doTNcaictdEBu8nfdXKoTtzrn ++VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQAB +AoGBANOGBM6bbhq7ImYU4qf8+RQrdVg2tc9Fzo+yTnn30sF/rx8/AiCDOV4qdGAh +HKjKKaGj2H/rotqoEFcxBy05LrgJXxydBP72e9PYhNgKOcSmCQu4yALIPEXfKuIM +zgAErHVJ2l79fif3D4hzNyz+u5E1A9n3FG9cgaJSiYP8IG2RAkEA82GZ8rBkSGQQ +ZQ3oFuzPAAL21lbj8D0p76fsCpvS7427DtZDOjhOIKZmaeykpv+qSzRraqEqjDRi +S4kjQvwh6QJBAOKniZ+NDo2lSpbOFk+XlmABK1DormVpj8KebHEZYok1lRI+WiX9 +Nnoe9YLgix7++6H5SBBCcTB4HvM+5A4BuwMCQQChcX/eZbXP81iQwB3Rfzp8xnqY +icDf7qKvz9Ma4myU7Y5E9EpaB1mD/P14jDpYcMW050vNyqTfpiwB8TFL0NZpAkEA +02jkFH9UyMgZV6qo4tqI98l/ZrtyF8OrxSNSEPhVkZf6EQc5vN9/lc8Uv1vESEgb +3AwRrKDcxRH2BHtv6qSwkwJAGjqnkIcEkA75r1e55/EF2chcZW1+tpwKupE8CtAH +VXGd5DVwt4cYWkLUj2gF2fJbV97uu2MAg5CFDb+vQ6p5eA== +-----END RSA PRIVATE KEY----- diff --git a/dummyserver/certs/server.no_san.crt b/dummyserver/certs/server.no_san.crt new file mode 100644 index 0000000..cb89a14 --- /dev/null +++ b/dummyserver/certs/server.no_san.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIChzCCAfACCQCmk6is+6REjDANBgkqhkiG9w0BAQUFADCBhzELMAkGA1UEBhMC +Q0ExEDAOBgNVBAgMB09udGFyaW8xEDAOBgNVBAcMB09udGFyaW8xHzAdBgNVBAoM +FlNoYXpvdydzIFVzZWQgQ2FycyBJbmMxEjAQBgNVBAMMCWxvY2FsaG9zdDEfMB0G +CSqGSIb3DQEJARYQc2hhem93QGdtYWlsLmNvbTAeFw0xNDEyMDMyMjE3MjVaFw00 +NDEyMDIyMjE3MjVaMIGHMQswCQYDVQQGEwJDQTEQMA4GA1UECAwHT250YXJpbzEQ +MA4GA1UEBwwHT250YXJpbzEfMB0GA1UECgwWU2hhem93J3MgVXNlZCBDYXJzIElu +YzESMBAGA1UEAwwJbG9jYWxob3N0MR8wHQYJKoZIhvcNAQkBFhBzaGF6b3dAZ21h +aWwuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXe3FqmCWvP8XPxqtT ++0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB +0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN +3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAHI/m9/O +bVR3zBOJZUKlHzTRvfYbYhhfrrcQlbwhjKqNyZcQTL/bJdtQSL19g3ftC5wZPI+y +66R24MqGmRcv5kT32HcuIK1Xhx4nDqTqnTNvGkaIh5CqS4DEP+iqtwDoEbQt8DwL +ejKtvZlyQRKFPTMtmv4VsTIHeVOAj+pXn595 +-----END CERTIFICATE----- diff --git a/dummyserver/certs/server.no_san.csr b/dummyserver/certs/server.no_san.csr new file mode 100644 index 0000000..d4bb7c3 --- /dev/null +++ b/dummyserver/certs/server.no_san.csr @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIByDCCATECAQAwgYcxCzAJBgNVBAYTAkNBMRAwDgYDVQQIDAdPbnRhcmlvMRAw +DgYDVQQHDAdPbnRhcmlvMR8wHQYDVQQKDBZTaGF6b3cncyBVc2VkIENhcnMgSW5j +MRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHNoYXpvd0BnbWFp +bC5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANd7cWqYJa8/xc/Gq1P7 +Rt8vVO/O95uLjqTRYhxRXxs/e/JiJJFcb0AubLNkfgYdj1RWzg4QOQIwM4FJ7MHT +L1qL/12hM1xqJy10QG7yd91cqhO3Ouf5VfhQ+uQfmGbs1DV9NC6PUxHvIEKZio3d +DZD4R+66PLHQXAHMC7LVopi7AgMBAAGgADANBgkqhkiG9w0BAQUFAAOBgQDGWkxr +mCa2h+/HnptucimU+T4QESBNc3fHhnnWaj4RXJaS0xwUDaG81INnxj6KNVgOtemK +VlwG7Ziqj1i+gZ1UpbmMp1YkSD/0+N8vb2BStuXlc5rP0+cG1DlzV1Dc+FaDHHsy +7MfyeHTa5FYdSeKsiAFHlQ84g08Pd7hW0c+SxA== +-----END CERTIFICATE REQUEST----- diff --git a/dummyserver/handlers.pyc b/dummyserver/handlers.pyc Binary files differdeleted file mode 100644 index 22aedc3..0000000 --- a/dummyserver/handlers.pyc +++ /dev/null diff --git a/dummyserver/proxy.pyc b/dummyserver/proxy.pyc Binary files differdeleted file mode 100644 index 23fa01d..0000000 --- a/dummyserver/proxy.pyc +++ /dev/null diff --git a/dummyserver/server.py b/dummyserver/server.py index 99f0835..6ee9a5d 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -28,8 +28,13 @@ DEFAULT_CERTS = { 'certfile': os.path.join(CERTS_PATH, 'server.crt'), 'keyfile': os.path.join(CERTS_PATH, 'server.key'), } +NO_SAN_CERTS = { + 'certfile': os.path.join(CERTS_PATH, 'server.no_san.crt'), + 'keyfile': DEFAULT_CERTS['keyfile'] +} DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem') DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem') +NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem') # Different types of servers we have: @@ -179,3 +184,17 @@ def get_unreachable_address(): return sockaddr else: s.close() + + +if __name__ == '__main__': + # For debugging dummyserver itself - python -m dummyserver.server + from .testcase import TestingApp + host = '127.0.0.1' + + io_loop = tornado.ioloop.IOLoop() + app = tornado.wsgi.WSGIContainer(TestingApp()) + server, port = run_tornado_app(app, io_loop, None, + 'http', host) + server_thread = run_loop_in_thread(io_loop) + + print("Listening on http://{host}:{port}".format(host=host, port=port)) diff --git a/dummyserver/server.pyc b/dummyserver/server.pyc Binary files differdeleted file mode 100644 index b997d0e..0000000 --- a/dummyserver/server.pyc +++ /dev/null diff --git a/dummyserver/testcase.pyc b/dummyserver/testcase.pyc Binary files differdeleted file mode 100644 index 29cc06a..0000000 --- a/dummyserver/testcase.pyc +++ /dev/null diff --git a/test/__init__.pyc b/test/__init__.pyc Binary files differdeleted file mode 100644 index 38b9317..0000000 --- a/test/__init__.pyc +++ /dev/null diff --git a/test/contrib/__init__.pyc b/test/contrib/__init__.pyc Binary files differdeleted file mode 100644 index 2d2fd5d..0000000 --- a/test/contrib/__init__.pyc +++ /dev/null diff --git a/test/contrib/test_pyopenssl.pyc b/test/contrib/test_pyopenssl.pyc Binary files differdeleted file mode 100644 index 6441273..0000000 --- a/test/contrib/test_pyopenssl.pyc +++ /dev/null diff --git a/test/port_helpers.pyc b/test/port_helpers.pyc Binary files differdeleted file mode 100644 index 7a1c425..0000000 --- a/test/port_helpers.pyc +++ /dev/null diff --git a/test/test_collections.pyc b/test/test_collections.pyc Binary files differdeleted file mode 100644 index d1ecd73..0000000 --- a/test/test_collections.pyc +++ /dev/null diff --git a/test/test_compatibility.pyc b/test/test_compatibility.pyc Binary files differdeleted file mode 100644 index 2dfdf75..0000000 --- a/test/test_compatibility.pyc +++ /dev/null diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 28fb89b..a6dbcf4 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -118,7 +118,7 @@ class TestConnectionPool(unittest.TestCase): str(MaxRetryError( HTTPConnectionPool(host='localhost'), "Test.", None)), "HTTPConnectionPool(host='localhost', port=None): " - "Max retries exceeded with url: Test. (Caused by redirect)") + "Max retries exceeded with url: Test. (Caused by None)") err = SocketError("Test") diff --git a/test/test_connectionpool.pyc b/test/test_connectionpool.pyc Binary files differdeleted file mode 100644 index e87a3b3..0000000 --- a/test/test_connectionpool.pyc +++ /dev/null diff --git a/test/test_exceptions.pyc b/test/test_exceptions.pyc Binary files differdeleted file mode 100644 index 3274e34..0000000 --- a/test/test_exceptions.pyc +++ /dev/null diff --git a/test/test_fields.pyc b/test/test_fields.pyc Binary files differdeleted file mode 100644 index 4622899..0000000 --- a/test/test_fields.pyc +++ /dev/null diff --git a/test/test_filepost.pyc b/test/test_filepost.pyc Binary files differdeleted file mode 100644 index ec54472..0000000 --- a/test/test_filepost.pyc +++ /dev/null diff --git a/test/test_poolmanager.pyc b/test/test_poolmanager.pyc Binary files differdeleted file mode 100644 index 077c2ac..0000000 --- a/test/test_poolmanager.pyc +++ /dev/null diff --git a/test/test_proxymanager.pyc b/test/test_proxymanager.pyc Binary files differdeleted file mode 100644 index 3696ee8..0000000 --- a/test/test_proxymanager.pyc +++ /dev/null diff --git a/test/test_response.pyc b/test/test_response.pyc Binary files differdeleted file mode 100644 index 99e5c0e..0000000 --- a/test/test_response.pyc +++ /dev/null diff --git a/test/test_retry.py b/test/test_retry.py index 7a3aa40..421e508 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -1,11 +1,13 @@ import unittest +from urllib3.response import HTTPResponse from urllib3.packages.six.moves import xrange from urllib3.util.retry import Retry from urllib3.exceptions import ( ConnectTimeoutError, + MaxRetryError, ReadTimeoutError, - MaxRetryError + ResponseError, ) @@ -154,3 +156,43 @@ class RetryTest(unittest.TestCase): def test_disabled(self): self.assertRaises(MaxRetryError, Retry(-1).increment) self.assertRaises(MaxRetryError, Retry(0).increment) + + def test_error_message(self): + retry = Retry(total=0) + try: + retry = retry.increment(error=ReadTimeoutError(None, "/", "read timed out")) + raise AssertionError("Should have raised a MaxRetryError") + except MaxRetryError as e: + assert 'Caused by redirect' not in str(e) + self.assertEqual(str(e.reason), 'None: read timed out') + + retry = Retry(total=1) + try: + retry = retry.increment('POST', '/') + retry = retry.increment('POST', '/') + raise AssertionError("Should have raised a MaxRetryError") + except MaxRetryError as e: + assert 'Caused by redirect' not in str(e) + self.assertTrue(isinstance(e.reason, ResponseError), + "%s should be a ResponseError" % e.reason) + self.assertEqual(str(e.reason), ResponseError.GENERIC_ERROR) + + retry = Retry(total=1) + try: + response = HTTPResponse(status=500) + retry = retry.increment('POST', '/', response=response) + retry = retry.increment('POST', '/', response=response) + raise AssertionError("Should have raised a MaxRetryError") + except MaxRetryError as e: + assert 'Caused by redirect' not in str(e) + msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) + self.assertEqual(str(e.reason), msg) + + retry = Retry(connect=1) + try: + retry = retry.increment(error=ConnectTimeoutError('conntimeout')) + retry = retry.increment(error=ConnectTimeoutError('conntimeout')) + raise AssertionError("Should have raised a MaxRetryError") + except MaxRetryError as e: + assert 'Caused by redirect' not in str(e) + self.assertEqual(str(e.reason), 'conntimeout') diff --git a/test/test_retry.pyc b/test/test_retry.pyc Binary files differdeleted file mode 100644 index 398c010..0000000 --- a/test/test_retry.pyc +++ /dev/null diff --git a/test/test_util.py b/test/test_util.py index 1811dbd..c850d91 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -2,8 +2,9 @@ import warnings import logging import unittest import ssl +from itertools import chain -from mock import patch +from mock import patch, Mock from urllib3 import add_stderr_logger, disable_warnings from urllib3.util.request import make_headers @@ -14,14 +15,15 @@ from urllib3.util.url import ( split_first, Url, ) -from urllib3.util.ssl_ import resolve_cert_reqs +from urllib3.util.ssl_ import resolve_cert_reqs, ssl_wrap_socket from urllib3.exceptions import ( LocationParseError, TimeoutStateError, InsecureRequestWarning, + SSLError, ) -from urllib3.util import is_fp_closed +from urllib3.util import is_fp_closed, ssl_ from . import clear_warnings @@ -89,45 +91,61 @@ class TestUtil(unittest.TestCase): self.assertRaises(LocationParseError, get_host, location) - def test_parse_url(self): - url_host_map = { - 'http://google.com/mail': Url('http', host='google.com', path='/mail'), - 'http://google.com/mail/': Url('http', host='google.com', path='/mail/'), - 'google.com/mail': Url(host='google.com', path='/mail'), - 'http://google.com/': Url('http', host='google.com', path='/'), - 'http://google.com': Url('http', host='google.com'), - 'http://google.com?foo': Url('http', host='google.com', path='', query='foo'), - - # Path/query/fragment - '': Url(), - '/': Url(path='/'), - '?': Url(path='', query=''), - '#': Url(path='', fragment=''), - '#?/!google.com/?foo#bar': Url(path='', fragment='?/!google.com/?foo#bar'), - '/foo': Url(path='/foo'), - '/foo?bar=baz': Url(path='/foo', query='bar=baz'), - '/foo?bar=baz#banana?apple/orange': Url(path='/foo', query='bar=baz', fragment='banana?apple/orange'), - - # Port - 'http://google.com/': Url('http', host='google.com', path='/'), - 'http://google.com:80/': Url('http', host='google.com', port=80, path='/'), - 'http://google.com:/': Url('http', host='google.com', path='/'), - 'http://google.com:80': Url('http', host='google.com', port=80), - 'http://google.com:': Url('http', host='google.com'), - - # Auth - 'http://foo:bar@localhost/': Url('http', auth='foo:bar', host='localhost', path='/'), - 'http://foo@localhost/': Url('http', auth='foo', host='localhost', path='/'), - 'http://foo:bar@baz@localhost/': Url('http', auth='foo:bar@baz', host='localhost', path='/'), - 'http://@': Url('http', host=None, auth='') + parse_url_host_map = { + 'http://google.com/mail': Url('http', host='google.com', path='/mail'), + 'http://google.com/mail/': Url('http', host='google.com', path='/mail/'), + 'google.com/mail': Url(host='google.com', path='/mail'), + 'http://google.com/': Url('http', host='google.com', path='/'), + 'http://google.com': Url('http', host='google.com'), + 'http://google.com?foo': Url('http', host='google.com', path='', query='foo'), + + # Path/query/fragment + '': Url(), + '/': Url(path='/'), + '#?/!google.com/?foo#bar': Url(path='', fragment='?/!google.com/?foo#bar'), + '/foo': Url(path='/foo'), + '/foo?bar=baz': Url(path='/foo', query='bar=baz'), + '/foo?bar=baz#banana?apple/orange': Url(path='/foo', query='bar=baz', fragment='banana?apple/orange'), + + # Port + 'http://google.com/': Url('http', host='google.com', path='/'), + 'http://google.com:80/': Url('http', host='google.com', port=80, path='/'), + 'http://google.com:80': Url('http', host='google.com', port=80), + + # Auth + 'http://foo:bar@localhost/': Url('http', auth='foo:bar', host='localhost', path='/'), + 'http://foo@localhost/': Url('http', auth='foo', host='localhost', path='/'), + 'http://foo:bar@baz@localhost/': Url('http', auth='foo:bar@baz', host='localhost', path='/'), + 'http://@': Url('http', host=None, auth='') + } + + non_round_tripping_parse_url_host_map = { + # Path/query/fragment + '?': Url(path='', query=''), + '#': Url(path='', fragment=''), + + # Empty Port + 'http://google.com:': Url('http', host='google.com'), + 'http://google.com:/': Url('http', host='google.com', path='/'), + } - for url, expected_url in url_host_map.items(): - returned_url = parse_url(url) - self.assertEqual(returned_url, expected_url) + + def test_parse_url(self): + for url, expected_Url in chain(self.parse_url_host_map.items(), self.non_round_tripping_parse_url_host_map.items()): + returned_Url = parse_url(url) + self.assertEqual(returned_Url, expected_Url) + + def test_unparse_url(self): + for url, expected_Url in self.parse_url_host_map.items(): + self.assertEqual(url, expected_Url.url) def test_parse_url_invalid_IPv6(self): self.assertRaises(ValueError, parse_url, '[::1') + def test_Url_str(self): + U = Url('http', host='google.com') + self.assertEqual(str(U), U.url) + def test_request_uri(self): url_host_map = { 'http://google.com/mail': '/mail', @@ -333,7 +351,7 @@ class TestUtil(unittest.TestCase): return True self.assertTrue(is_fp_closed(ClosedFile())) - + def test_is_fp_closed_object_has_none_fp(self): class NoneFpFile(object): @property @@ -355,3 +373,30 @@ class TestUtil(unittest.TestCase): pass self.assertRaises(ValueError, is_fp_closed, NotReallyAFile()) + + def test_ssl_wrap_socket_loads_the_cert_chain(self): + socket = object() + mock_context = Mock() + ssl_wrap_socket(ssl_context=mock_context, sock=socket, + certfile='/path/to/certfile') + + mock_context.load_cert_chain.assert_called_once_with( + '/path/to/certfile', None) + + def test_ssl_wrap_socket_loads_verify_locations(self): + socket = object() + mock_context = Mock() + ssl_wrap_socket(ssl_context=mock_context, ca_certs='/path/to/pem', + sock=socket) + mock_context.load_verify_locations.assert_called_once_with( + '/path/to/pem') + + def test_ssl_wrap_socket_with_no_sni(self): + socket = object() + mock_context = Mock() + # Ugly preservation of original value + HAS_SNI = ssl_.HAS_SNI + ssl_.HAS_SNI = False + ssl_wrap_socket(ssl_context=mock_context, sock=socket) + mock_context.wrap_socket.assert_called_once_with(socket) + ssl_.HAS_SNI = HAS_SNI diff --git a/test/test_util.pyc b/test/test_util.pyc Binary files differdeleted file mode 100644 index 0500c3b..0000000 --- a/test/test_util.pyc +++ /dev/null diff --git a/test/with_dummyserver/__init__.pyc b/test/with_dummyserver/__init__.pyc Binary files differdeleted file mode 100644 index 833be60..0000000 --- a/test/with_dummyserver/__init__.pyc +++ /dev/null diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 7d54fbf..cc0f011 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -13,8 +13,7 @@ except: from urllib import urlencode from .. import ( - requires_network, - onlyPy3, onlyPy27OrNewer, onlyPy26OrOlder, + requires_network, onlyPy3, onlyPy26OrOlder, TARPIT_HOST, VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES, ) from ..port_helpers import find_unused_port @@ -99,6 +98,13 @@ class TestConnectionPool(HTTPDummyServerTestCase): r = self.pool.request('POST', '/echo', fields=fields) self.assertEqual(r.data.count(b'name="foo"'), 2) + def test_request_method_body(self): + body = b'hi' + r = self.pool.request('POST', '/echo', body=body) + self.assertEqual(r.data, body) + + fields = [('hi', 'hello')] + self.assertRaises(TypeError, self.pool.request, 'POST', '/echo', body=body, fields=fields) def test_unicode_upload(self): fieldname = u('myfile') @@ -189,7 +195,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): @timed(0.5) def test_timeout(self): """ Requests should time out when expected """ - url = '/sleep?seconds=0.002' + url = '/sleep?seconds=0.003' timeout = Timeout(read=0.001) # Pool-global timeout diff --git a/test/with_dummyserver/test_connectionpool.pyc b/test/with_dummyserver/test_connectionpool.pyc Binary files differdeleted file mode 100644 index b8c38e9..0000000 --- a/test/with_dummyserver/test_connectionpool.pyc +++ /dev/null diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index cf3eee7..16ca589 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -9,7 +9,8 @@ import mock from nose.plugins.skip import SkipTest from dummyserver.testcase import HTTPSDummyServerTestCase -from dummyserver.server import DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS +from dummyserver.server import (DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS, + NO_SAN_CERTS, NO_SAN_CA) from test import ( onlyPy26OrOlder, @@ -168,7 +169,7 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool.request('HEAD', '/') def test_assert_hostname_false(self): - https_pool = HTTPSConnectionPool('127.0.0.1', self.port, + https_pool = HTTPSConnectionPool('localhost', self.port, cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA) @@ -176,7 +177,7 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool.request('GET', '/') def test_assert_specific_hostname(self): - https_pool = HTTPSConnectionPool('127.0.0.1', self.port, + https_pool = HTTPSConnectionPool('localhost', self.port, cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA) @@ -184,7 +185,7 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool.request('GET', '/') def test_assert_fingerprint_md5(self): - https_pool = HTTPSConnectionPool('127.0.0.1', self.port, + https_pool = HTTPSConnectionPool('localhost', self.port, cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA) @@ -193,7 +194,7 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool.request('GET', '/') def test_assert_fingerprint_sha1(self): - https_pool = HTTPSConnectionPool('127.0.0.1', self.port, + https_pool = HTTPSConnectionPool('localhost', self.port, cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA) @@ -329,6 +330,8 @@ class TestHTTPS(HTTPSDummyServerTestCase): https_pool._make_request(conn, 'GET', '/') def test_ssl_correct_system_time(self): + self._pool.cert_reqs = 'CERT_REQUIRED' + self._pool.ca_certs = DEFAULT_CA with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') self._pool.request('GET', '/') @@ -336,6 +339,8 @@ class TestHTTPS(HTTPSDummyServerTestCase): self.assertEqual([], w) def test_ssl_wrong_system_time(self): + self._pool.cert_reqs = 'CERT_REQUIRED' + self._pool.ca_certs = DEFAULT_CA with mock.patch('urllib3.connection.datetime') as mock_date: mock_date.date.today.return_value = datetime.date(1970, 1, 1) @@ -369,6 +374,27 @@ class TestHTTPS_TLSv1(HTTPSDummyServerTestCase): self._pool.ssl_version = 'SSLv3' self.assertRaises(SSLError, self._pool.request, 'GET', '/') + def test_discards_connection_on_sslerror(self): + self._pool.cert_reqs = 'CERT_REQUIRED' + self.assertRaises(SSLError, self._pool.request, 'GET', '/') + self._pool.ca_certs = DEFAULT_CA + self._pool.request('GET', '/') + + +class TestHTTPS_NoSAN(HTTPSDummyServerTestCase): + certs = NO_SAN_CERTS + + def test_warning_for_certs_without_a_san(self): + """Ensure that a warning is raised when the cert from the server has + no Subject Alternative Name.""" + with mock.patch('warnings.warn') as warn: + https_pool = HTTPSConnectionPool(self.host, self.port, + cert_reqs='CERT_REQUIRED', + ca_certs=NO_SAN_CA) + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + self.assertTrue(warn.called) + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_https.pyc b/test/with_dummyserver/test_https.pyc Binary files differdeleted file mode 100644 index 6d85316..0000000 --- a/test/with_dummyserver/test_https.pyc +++ /dev/null diff --git a/test/with_dummyserver/test_poolmanager.pyc b/test/with_dummyserver/test_poolmanager.pyc Binary files differdeleted file mode 100644 index 26c52e9..0000000 --- a/test/with_dummyserver/test_poolmanager.pyc +++ /dev/null diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 61eedf1..df300fe 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -1,13 +1,17 @@ -import unittest import json import socket +import unittest + +from nose.tools import timed from dummyserver.testcase import HTTPDummyProxyTestCase from dummyserver.server import ( DEFAULT_CA, DEFAULT_CA_BAD, get_unreachable_address) +from .. import TARPIT_HOST from urllib3.poolmanager import proxy_from_url, ProxyManager -from urllib3.exceptions import MaxRetryError, SSLError, ProxyError +from urllib3.exceptions import ( + MaxRetryError, SSLError, ProxyError, ConnectTimeoutError) from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection @@ -259,5 +263,25 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): self.assertEqual(sc3,sc4) + @timed(0.5) + def test_https_proxy_timeout(self): + https = proxy_from_url('https://{host}'.format(host=TARPIT_HOST)) + try: + https.request('GET', self.http_url, timeout=0.001) + self.fail("Failed to raise retry error.") + except MaxRetryError as e: + assert isinstance(e.reason, ConnectTimeoutError) + + + @timed(0.5) + def test_https_proxy_pool_timeout(self): + https = proxy_from_url('https://{host}'.format(host=TARPIT_HOST), + timeout=0.001) + try: + https.request('GET', self.http_url) + self.fail("Failed to raise retry error.") + except MaxRetryError as e: + assert isinstance(e.reason, ConnectTimeoutError) + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_proxy_poolmanager.pyc b/test/with_dummyserver/test_proxy_poolmanager.pyc Binary files differdeleted file mode 100644 index 12c320c..0000000 --- a/test/with_dummyserver/test_proxy_poolmanager.pyc +++ /dev/null diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index e1ac1c6..c1ef1be 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -137,6 +137,24 @@ class TestSocketClosing(SocketDummyServerTestCase): finally: timed_out.set() + def test_https_connection_read_timeout(self): + """ Handshake timeouts should fail with a Timeout""" + timed_out = Event() + def socket_handler(listener): + sock = listener.accept()[0] + while not sock.recv(65536): + pass + + timed_out.wait() + sock.close() + + self._start_server(socket_handler) + pool = HTTPSConnectionPool(self.host, self.port, timeout=0.001, retries=False) + try: + self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') + finally: + timed_out.set() + def test_timeout_errors_cause_retries(self): def socket_handler(listener): sock_timeout = listener.accept()[0] diff --git a/test/with_dummyserver/test_socketlevel.pyc b/test/with_dummyserver/test_socketlevel.pyc Binary files differdeleted file mode 100644 index ba3b19e..0000000 --- a/test/with_dummyserver/test_socketlevel.pyc +++ /dev/null diff --git a/urllib3.egg-info/PKG-INFO b/urllib3.egg-info/PKG-INFO index 964cd4b..7b5cf18 100644 --- a/urllib3.egg-info/PKG-INFO +++ b/urllib3.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: urllib3 -Version: 1.9.1 +Version: 1.10 Summary: HTTP library with thread-safe connection pooling, file post, and more. Home-page: http://urllib3.readthedocs.org/ Author: Andrey Petrov @@ -156,6 +156,38 @@ Description: ======= Changes ======= + 1.10 (2014-12-14) + +++++++++++++++++ + + * Disabled SSLv3. (Issue #473) + + * Add ``Url.url`` property to return the composed url string. (Issue #394) + + * Fixed PyOpenSSL + gevent ``WantWriteError``. (Issue #412) + + * ``MaxRetryError.reason`` will always be an exception, not string. + (Issue #481) + + * Fixed SSL-related timeouts not being detected as timeouts. (Issue #492) + + * Py3: Use ``ssl.create_default_context()`` when available. (Issue #473) + + * Emit ``InsecureRequestWarning`` for *every* insecure HTTPS request. + (Issue #496) + + * Emit ``SecurityWarning`` when certificate has no ``subjectAltName``. + (Issue #499) + + * Close and discard sockets which experienced SSL-related errors. + (Issue #501) + + * Handle ``body`` param in ``.request(...)``. (Issue #513) + + * Respect timeout with HTTPS proxy. (Issue #505) + + * PyOpenSSL: Handle ZeroReturnError exception. (Issue #520) + + 1.9.1 (2014-09-13) ++++++++++++++++++ diff --git a/urllib3.egg-info/SOURCES.txt b/urllib3.egg-info/SOURCES.txt index 2f0b5fc..6cb0fcf 100644 --- a/urllib3.egg-info/SOURCES.txt +++ b/urllib3.egg-info/SOURCES.txt @@ -21,16 +21,13 @@ docs/managers.rst docs/pools.rst docs/security.rst dummyserver/__init__.py -dummyserver/__init__.pyc dummyserver/handlers.py -dummyserver/handlers.pyc dummyserver/proxy.py -dummyserver/proxy.pyc dummyserver/server.py -dummyserver/server.pyc dummyserver/testcase.py -dummyserver/testcase.pyc +dummyserver/certs/README.rst dummyserver/certs/cacert.key +dummyserver/certs/cacert.no_san.pem dummyserver/certs/cacert.pem dummyserver/certs/client.csr dummyserver/certs/client.key @@ -40,49 +37,30 @@ dummyserver/certs/server.crt dummyserver/certs/server.csr dummyserver/certs/server.key dummyserver/certs/server.key.org +dummyserver/certs/server.no_san.crt +dummyserver/certs/server.no_san.csr test/__init__.py -test/__init__.pyc test/benchmark.py test/port_helpers.py -test/port_helpers.pyc test/test_collections.py -test/test_collections.pyc test/test_compatibility.py -test/test_compatibility.pyc test/test_connectionpool.py -test/test_connectionpool.pyc test/test_exceptions.py -test/test_exceptions.pyc test/test_fields.py -test/test_fields.pyc test/test_filepost.py -test/test_filepost.pyc test/test_poolmanager.py -test/test_poolmanager.pyc test/test_proxymanager.py -test/test_proxymanager.pyc test/test_response.py -test/test_response.pyc test/test_retry.py -test/test_retry.pyc test/test_util.py -test/test_util.pyc test/contrib/__init__.py -test/contrib/__init__.pyc test/contrib/test_pyopenssl.py -test/contrib/test_pyopenssl.pyc test/with_dummyserver/__init__.py -test/with_dummyserver/__init__.pyc test/with_dummyserver/test_connectionpool.py -test/with_dummyserver/test_connectionpool.pyc test/with_dummyserver/test_https.py -test/with_dummyserver/test_https.pyc test/with_dummyserver/test_poolmanager.py -test/with_dummyserver/test_poolmanager.pyc test/with_dummyserver/test_proxy_poolmanager.py -test/with_dummyserver/test_proxy_poolmanager.pyc test/with_dummyserver/test_socketlevel.py -test/with_dummyserver/test_socketlevel.pyc urllib3/__init__.py urllib3/_collections.py urllib3/connection.py diff --git a/urllib3/__init__.py b/urllib3/__init__.py index 3546d13..4f9d4a7 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.9.1' +__version__ = '1.10' from .connectionpool import ( @@ -55,9 +55,9 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -# Set security warning to only go off once by default. +# Set security warning to always go off by default. import warnings -warnings.simplefilter('module', exceptions.SecurityWarning) +warnings.simplefilter('always', exceptions.SecurityWarning) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/urllib3/_collections.py b/urllib3/_collections.py index d77ebb8..784342a 100644 --- a/urllib3/_collections.py +++ b/urllib3/_collections.py @@ -14,7 +14,7 @@ try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict -from .packages.six import itervalues +from .packages.six import iterkeys, itervalues __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] @@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping): def clear(self): with self.lock: # Copy pointers to all values, then wipe the mapping - # under Python 2, this copies the list of values twice :-| - values = list(self._container.values()) + values = list(itervalues(self._container)) self._container.clear() if self.dispose_func: @@ -95,7 +94,7 @@ class RecentlyUsedContainer(MutableMapping): def keys(self): with self.lock: - return self._container.keys() + return list(iterkeys(self._container)) class HTTPHeaderDict(MutableMapping): diff --git a/urllib3/connection.py b/urllib3/connection.py index cebdd86..e5de769 100644 --- a/urllib3/connection.py +++ b/urllib3/connection.py @@ -38,6 +38,7 @@ except NameError: # Python 2: from .exceptions import ( ConnectTimeoutError, SystemTimeWarning, + SecurityWarning, ) from .packages.ssl_match_hostname import match_hostname @@ -241,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.assert_fingerprint) elif resolved_cert_reqs != ssl.CERT_NONE \ and self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or hostname) + cert = self.sock.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn(( + 'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. ' + 'This feature is being removed by major browsers and deprecated by RFC 2818. ' + '(See https://github.com/shazow/urllib3/issues/497 for details.)'), + SecurityWarning + ) + match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index ac6e0ca..8bdf228 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -266,6 +266,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ pass + def _prepare_proxy(self, conn): + # Nothing to do for HTTP connections. + pass + def _get_timeout(self, timeout): """ Helper that always returns a :class:`urllib3.util.Timeout` """ if timeout is _Default: @@ -278,6 +282,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # can be removed later return Timeout.from_float(timeout) + def _raise_timeout(self, err, url, timeout_value): + """Is the error actually a timeout? Will raise a ReadTimeout or pass""" + + if isinstance(err, SocketTimeout): + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # See the above comment about EAGAIN in Python 3. In Python 2 we have + # to specifically catch it and throw the timeout error + if hasattr(err, 'errno') and err.errno in _blocking_errnos: + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # Catch possible read timeouts thrown as SSL errors. If not the + # case, rethrow the original. We need to do this because of: + # http://bugs.python.org/issue10272 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + def _make_request(self, conn, method, url, timeout=_Default, **httplib_request_kw): """ @@ -301,7 +322,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout # Trigger any extra validation we need to do. - self._validate_conn(conn) + try: + self._validate_conn(conn) + except (SocketTimeout, BaseSSLError) as e: + # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. @@ -331,28 +357,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): httplib_response = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older httplib_response = conn.getresponse() - except SocketTimeout: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - except BaseSSLError as e: - # Catch possible read timeouts thrown as SSL errors. If not the - # case, rethrow the original. We need to do this because of: - # http://bugs.python.org/issue10272 - if 'timed out' in str(e) or \ - 'did not complete (read)' in str(e): # Python 2.6 - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - raise - - except SocketError as e: # Platform-specific: Python 2 - # See the above comment about EAGAIN in Python 3. In Python 2 we - # have to specifically catch it and throw the timeout error - if e.errno in _blocking_errnos: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - + except (SocketTimeout, BaseSSLError, SocketError) as e: + self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. @@ -508,11 +514,18 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: # Request a connection from the queue. + timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) + conn.timeout = timeout_obj.connect_timeout + + is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) + if is_new_proxy_conn: + self._prepare_proxy(conn) + # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, - timeout=timeout, + timeout=timeout_obj, body=body, headers=headers) # If we're going to release the connection in ``finally:``, then @@ -537,9 +550,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise EmptyPoolError(self, "No pool connections are available.") except (BaseSSLError, CertificateError) as e: - # Release connection unconditionally because there is no way to - # close it externally in case of exception. - release_conn = True + # Close the connection. If a connection is reused on which there + # was a Certificate error, the next request will certainly raise + # another Certificate error. + if conn: + conn.close() + conn = None raise SSLError(e) except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: @@ -668,23 +684,25 @@ class HTTPSConnectionPool(HTTPConnectionPool): assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version - if self.proxy is not None: - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel + return conn - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self.host, self.port) - else: - set_tunnel(self.host, self.port, self.proxy_headers) + def _prepare_proxy(self, conn): + """ + Establish tunnel connection early, because otherwise httplib + would improperly set Host: header to proxy's IP:port. + """ + # Python 2.7+ + try: + set_tunnel = conn.set_tunnel + except AttributeError: # Platform-specific: Python 2.6 + set_tunnel = conn._set_tunnel - # Establish tunnel connection early, because otherwise httplib - # would improperly set Host: header to proxy's IP:port. - conn.connect() + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + set_tunnel(self.host, self.port) + else: + set_tunnel(self.host, self.port, self.proxy_headers) - return conn + conn.connect() def _new_conn(self): """ @@ -725,8 +743,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.org/en/latest/security.html ' - '(This warning will only appear once by default.)'), + 'https://urllib3.readthedocs.org/en/latest/security.html'), InsecureRequestWarning) diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py index 8475eeb..ee657fb 100644 --- a/urllib3/contrib/pyopenssl.py +++ b/urllib3/contrib/pyopenssl.py @@ -70,9 +70,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, - ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } + +try: + _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) +except AttributeError: + pass + _openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, @@ -186,6 +191,11 @@ class WrappedSocket(object): return b'' else: raise + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return b'' + else: + raise except OpenSSL.SSL.WantReadError: rd, wd, ed = select.select( [self.socket], [], [], self.socket.gettimeout()) @@ -199,8 +209,21 @@ class WrappedSocket(object): def settimeout(self, timeout): return self.socket.settimeout(timeout) + def _send_until_done(self, data): + while True: + try: + return self.connection.send(data) + except OpenSSL.SSL.WantWriteError: + _, wlist, _ = select.select([], [self.socket], [], + self.socket.gettimeout()) + if not wlist: + raise timeout() + continue + def sendall(self, data): - return self.connection.sendall(data) + while len(data): + sent = self._send_until_done(data) + data = data[sent:] def close(self): if self._makefile_refs < 1: @@ -248,6 +271,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ssl_version=None): ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: + keyfile = keyfile or certfile # Match behaviour of the normal python ssl library ctx.use_certificate_file(certfile) if keyfile: ctx.use_privatekey_file(keyfile) diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py index 7519ba9..0c6fd3c 100644 --- a/urllib3/exceptions.py +++ b/urllib3/exceptions.py @@ -72,11 +72,8 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s" % url - if reason: - message += " (Caused by %r)" % reason - else: - message += " (Caused by redirect)" + message = "Max retries exceeded with url: %s (Caused by %r)" % ( + url, reason) RequestError.__init__(self, pool, url, message) @@ -141,6 +138,12 @@ class LocationParseError(LocationValueError): self.location = location +class ResponseError(HTTPError): + "Used as a container for an error reason supplied in a MaxRetryError." + GENERIC_ERROR = 'too many error responses' + SPECIFIC_ERROR = 'too many {status_code} error responses' + + class SecurityWarning(HTTPWarning): "Warned when perfoming security reducing actions" pass diff --git a/urllib3/request.py b/urllib3/request.py index 51fe238..b08d6c9 100644 --- a/urllib3/request.py +++ b/urllib3/request.py @@ -118,18 +118,24 @@ class RequestMethods(object): which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. """ - if encode_multipart: - body, content_type = encode_multipart_formdata( - fields or {}, boundary=multipart_boundary) - else: - body, content_type = (urlencode(fields or {}), - 'application/x-www-form-urlencoded') - if headers is None: headers = self.headers - headers_ = {'Content-Type': content_type} - headers_.update(headers) + extra_kw = {'headers': {}} + + if fields: + if 'body' in urlopen_kw: + raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') + + if encode_multipart: + body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + else: + body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + + extra_kw['body'] = body + extra_kw['headers'] = {'Content-Type': content_type} + + extra_kw['headers'].update(headers) + extra_kw.update(urlopen_kw) - return self.urlopen(method, url, body=body, headers=headers_, - **urlopen_kw) + return self.urlopen(method, url, **extra_kw) diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py index eb560df..7e0959d 100644 --- a/urllib3/util/retry.py +++ b/urllib3/util/retry.py @@ -2,10 +2,11 @@ import time import logging from ..exceptions import ( - ProtocolError, ConnectTimeoutError, - ReadTimeoutError, MaxRetryError, + ProtocolError, + ReadTimeoutError, + ResponseError, ) from ..packages import six @@ -36,7 +37,6 @@ class Retry(object): Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless retries are disabled, in which case the causing exception will be raised. - :param int total: Total number of retries to allow. Takes precedence over other counts. @@ -184,13 +184,13 @@ class Retry(object): return isinstance(err, ConnectTimeoutError) def _is_read_error(self, err): - """ Errors that occur after the request has been started, so we can't - assume that the server did not process any of it. + """ Errors that occur after the request has been started, so we should + assume that the server began processing it. """ return isinstance(err, (ReadTimeoutError, ProtocolError)) def is_forced_retry(self, method, status_code): - """ Is this method/response retryable? (Based on method/codes whitelists) + """ Is this method/status code retryable? (Based on method/codes whitelists) """ if self.method_whitelist and method.upper() not in self.method_whitelist: return False @@ -198,8 +198,7 @@ class Retry(object): return self.status_forcelist and status_code in self.status_forcelist def is_exhausted(self): - """ Are we out of retries? - """ + """ Are we out of retries? """ retry_counts = (self.total, self.connect, self.read, self.redirect) retry_counts = list(filter(None, retry_counts)) if not retry_counts: @@ -230,6 +229,7 @@ class Retry(object): connect = self.connect read = self.read redirect = self.redirect + cause = 'unknown' if error and self._is_connection_error(error): # Connect retry? @@ -251,10 +251,16 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 + cause = 'too many redirects' else: - # FIXME: Nothing changed, scenario doesn't make sense. + # Incrementing because of a server error like a 500 in + # status_forcelist and a the given method is in the whitelist _observed_errors += 1 + cause = ResponseError.GENERIC_ERROR + if response and response.status: + cause = ResponseError.SPECIFIC_ERROR.format( + status_code=response.status) new_retry = self.new( total=total, @@ -262,7 +268,7 @@ class Retry(object): _observed_errors=_observed_errors) if new_retry.is_exhausted(): - raise MaxRetryError(_pool, url, error) + raise MaxRetryError(_pool, url, error or ResponseError(cause)) log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py index 9cfe2d2..a788b1b 100644 --- a/urllib3/util/ssl_.py +++ b/urllib3/util/ssl_.py @@ -4,18 +4,84 @@ from hashlib import md5, sha1 from ..exceptions import SSLError -try: # Test for SSL features - SSLContext = None - HAS_SNI = False +SSLContext = None +HAS_SNI = False +create_default_context = None + +import errno +import ssl - import ssl +try: # Test for SSL features from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 - from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: + from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION +except ImportError: + OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 + OP_NO_COMPRESSION = 0x20000 + +try: + from ssl import _DEFAULT_CIPHERS +except ImportError: + _DEFAULT_CIPHERS = ( + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' + 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:' + 'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5' + ) + +try: + from ssl import SSLContext # Modern SSL? +except ImportError: + import sys + + class SSLContext(object): # Platform-specific: Python 2 & 3.1 + supports_set_ciphers = sys.version_info >= (2, 7) + + def __init__(self, protocol_version): + self.protocol = protocol_version + # Use default values from a real SSLContext + self.check_hostname = False + self.verify_mode = ssl.CERT_NONE + self.ca_certs = None + self.options = 0 + self.certfile = None + self.keyfile = None + self.ciphers = None + + def load_cert_chain(self, certfile, keyfile): + self.certfile = certfile + self.keyfile = keyfile + + def load_verify_locations(self, location): + self.ca_certs = location + + def set_ciphers(self, cipher_suite): + if not self.supports_set_ciphers: + raise TypeError( + 'Your version of Python does not support setting ' + 'a custom cipher suite. Please upgrade to Python ' + '2.7, 3.2, or later if you need this functionality.' + ) + self.ciphers = cipher_suite + + def wrap_socket(self, socket, server_hostname=None): + kwargs = { + 'keyfile': self.keyfile, + 'certfile': self.certfile, + 'ca_certs': self.ca_certs, + 'cert_reqs': self.verify_mode, + 'ssl_version': self.protocol, + } + if self.supports_set_ciphers: # Platform-specific: Python 2.7+ + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) + else: # Platform-specific: Python 2.6 + return wrap_socket(socket, **kwargs) + + def assert_fingerprint(cert, fingerprint): """ Checks if given fingerprint matches the supplied certificate. @@ -91,42 +157,98 @@ def resolve_ssl_version(candidate): return candidate -if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - """ - All arguments except `server_hostname` have the same meaning as for - :func:`ssl.wrap_socket` - - :param server_hostname: - Hostname of the expected certificate - """ - context = SSLContext(ssl_version) - context.verify_mode = cert_reqs - - # Disable TLS compression to migitate CRIME attack (issue #309) - OP_NO_COMPRESSION = 0x20000 - context.options |= OP_NO_COMPRESSION - - if ca_certs: - try: - context.load_verify_locations(ca_certs) - # Py32 raises IOError - # Py33 raises FileNotFoundError - except Exception as e: # Reraise as SSLError +def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, + options=None, ciphers=None): + """All arguments have the same meaning as ``ssl_wrap_socket``. + + By default, this function does a lot of the same work that + ``ssl.create_default_context`` does on Python 3.4+. It: + + - Disables SSLv2, SSLv3, and compression + - Sets a restricted set of server ciphers + + If you wish to enable SSLv3, you can do:: + + from urllib3.util import ssl_ + context = ssl_.create_urllib3_context() + context.options &= ~ssl_.OP_NO_SSLv3 + + You can do the same to enable compression (substituting ``COMPRESSION`` + for ``SSLv3`` in the last line above). + + :param ssl_version: + The desired protocol version to use. This will default to + PROTOCOL_SSLv23 which will negotiate the highest protocol that both + the server and your installation of OpenSSL support. + :param cert_reqs: + Whether to require the certificate verification. This defaults to + ``ssl.CERT_REQUIRED``. + :param options: + Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, + ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``. + :param ciphers: + Which cipher suites to allow the server to select. + :returns: + Constructed SSLContext object with specified options + :rtype: SSLContext + """ + context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + + if options is None: + options = 0 + # SSLv2 is easily broken and is considered harmful and dangerous + options |= OP_NO_SSLv2 + # SSLv3 has several problems and is now dangerous + options |= OP_NO_SSLv3 + # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ + # (issue #309) + options |= OP_NO_COMPRESSION + + context.options |= options + + if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 + context.set_ciphers(ciphers or _DEFAULT_CIPHERS) + + context.verify_mode = cert_reqs + if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + context.check_hostname = (context.verify_mode == ssl.CERT_REQUIRED) + return context + + +def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, + ca_certs=None, server_hostname=None, + ssl_version=None, ciphers=None, ssl_context=None): + """ + All arguments except for server_hostname and ssl_context have the same + meaning as they do when using :func:`ssl.wrap_socket`. + + :param server_hostname: + When SNI is supported, the expected hostname of the certificate + :param ssl_context: + A pre-made :class:`SSLContext` object. If none is provided, one will + be created using :func:`create_urllib3_context`. + :param ciphers: + A string of ciphers we wish the client to support. This is not + supported on Python 2.6 as the ssl module does not support it. + """ + context = ssl_context + if context is None: + context = create_urllib3_context(ssl_version, cert_reqs, + ciphers=ciphers) + + if ca_certs: + try: + context.load_verify_locations(ca_certs) + except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + raise SSLError(e) + # Py33 raises FileNotFoundError which subclasses OSError + # These are not equivalent unless we check the errno attribute + except OSError as e: # Platform-specific: Python 3.3 and beyond + if e.errno == errno.ENOENT: raise SSLError(e) - if certfile: - # FIXME: This block needs a test. - context.load_cert_chain(certfile, keyfile) - if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI - return context.wrap_socket(sock, server_hostname=server_hostname) - return context.wrap_socket(sock) - -else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - return wrap_socket(sock, keyfile=keyfile, certfile=certfile, - ca_certs=ca_certs, cert_reqs=cert_reqs, - ssl_version=ssl_version) + raise + if certfile: + context.load_cert_chain(certfile, keyfile) + if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI + return context.wrap_socket(sock, server_hostname=server_hostname) + return context.wrap_socket(sock) diff --git a/urllib3/util/url.py b/urllib3/util/url.py index 487d456..b2ec834 100644 --- a/urllib3/util/url.py +++ b/urllib3/util/url.py @@ -40,6 +40,48 @@ class Url(namedtuple('Url', url_attrs)): return '%s:%d' % (self.host, self.port) return self.host + @property + def url(self): + """ + Convert self into a url + + This function should more or less round-trip with :func:`.parse_url`. The + returned url may not be exactly the same as the url inputted to + :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls + with a blank port will have : removed). + + Example: :: + + >>> U = parse_url('http://google.com/mail/') + >>> U.url + 'http://google.com/mail/' + >>> Url('http', 'username:password', 'host.com', 80, + ... '/path', 'query', 'fragment').url + 'http://username:password@host.com:80/path?query#fragment' + """ + scheme, auth, host, port, path, query, fragment = self + url = '' + + # We use "is not None" we want things to happen with empty strings (or 0 port) + if scheme is not None: + url += scheme + '://' + if auth is not None: + url += auth + '@' + if host is not None: + url += host + if port is not None: + url += ':' + str(port) + if path is not None: + url += path + if query is not None: + url += '?' + query + if fragment is not None: + url += '#' + fragment + + return url + + def __str__(self): + return self.url def split_first(s, delims): """ @@ -84,7 +126,7 @@ def parse_url(url): Example:: >>> parse_url('http://google.com/mail/') - Url(scheme='http', host='google.com', port=None, path='/', ...) + Url(scheme='http', host='google.com', port=None, path='/mail/', ...) >>> parse_url('google.com:80') Url(scheme=None, host='google.com', port=80, path=None, ...) >>> parse_url('/foo?bar') @@ -162,7 +204,6 @@ def parse_url(url): return Url(scheme, auth, host, port, path, query, fragment) - def get_host(url): """ Deprecated. Use :func:`.parse_url` instead. |