diff options
Diffstat (limited to 'requests/packages/urllib3')
| -rw-r--r-- | requests/packages/urllib3/connectionpool.py | 26 | ||||
| -rw-r--r-- | requests/packages/urllib3/poolmanager.py | 33 | ||||
| -rw-r--r-- | requests/packages/urllib3/response.py | 12 | ||||
| -rw-r--r-- | requests/packages/urllib3/util.py | 52 |
4 files changed, 94 insertions, 29 deletions
diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index af8760d..51c87f5 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -9,6 +9,7 @@ import socket import errno from socket import error as SocketError, timeout as SocketTimeout +from .util import resolve_cert_reqs, resolve_ssl_version try: # Python 3 from http.client import HTTPConnection, HTTPException @@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection): ssl_version = None def set_cert(self, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None): - ssl_req_scheme = { - 'CERT_NONE': ssl.CERT_NONE, - 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, - 'CERT_REQUIRED': ssl.CERT_REQUIRED - } + cert_reqs=None, ca_certs=None): self.key_file = key_file self.cert_file = cert_file - self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.cert_reqs = cert_reqs self.ca_certs = ca_certs def connect(self): # Add certificate verification sock = socket.create_connection((self.host, self.port), self.timeout) + resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) + resolved_ssl_version = resolve_ssl_version(self.ssl_version) + # Wrap socket using verification with the root certs in # trusted_root_certs self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, - cert_reqs=self.cert_reqs, + cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, server_hostname=self.host, - ssl_version=self.ssl_version) + ssl_version=resolved_ssl_version) - if self.ca_certs: + if resolved_cert_reqs != ssl.CERT_NONE: match_hostname(self.sock.getpeercert(), self.host) @@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None): + cert_reqs=None, ca_certs=None, ssl_version=None): HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, @@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) - if self.ssl_version is None: - connection.ssl_version = ssl.PROTOCOL_SSLv23 - else: - connection.ssl_version = self.ssl_version + connection.ssl_version = self.ssl_version return connection diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index a124202..6e7377c 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -58,6 +58,17 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + def _new_pool(self, scheme, host, port): + """ + Create a new :class:`ConnectionPool` based on host, port and scheme. + + This method is used to actually create the connection pools handed out + by :meth:`connection_from_url` and companion methods. It is intended + to be overridden for customization. + """ + pool_cls = pool_classes_by_scheme[scheme] + return pool_cls(host, port, **self.connection_pool_kw) + def clear(self): """ Empty our store of pools and direct them all to close. @@ -74,6 +85,7 @@ class PoolManager(RequestMethods): If ``port`` isn't given, it will be derived from the ``scheme`` using ``urllib3.connectionpool.port_by_scheme``. """ + scheme = scheme or 'http' port = port or port_by_scheme.get(scheme, 80) pool_key = (scheme, host, port) @@ -85,11 +97,8 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - pool_cls = pool_classes_by_scheme[scheme] - pool = pool_cls(host, port, **self.connection_pool_kw) - + pool = self._new_pool(scheme, host, port) self.pools[pool_key] = pool - return pool def connection_from_url(self, url): @@ -138,14 +147,24 @@ class PoolManager(RequestMethods): class ProxyManager(RequestMethods): """ Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method - will make requests to any url through the defined proxy. + will make requests to any url through the defined proxy. The ProxyManager + class will automatically set the 'Host' header if it is not provided. """ def __init__(self, proxy_pool): self.proxy_pool = proxy_pool - def _set_proxy_headers(self, headers=None): + def _set_proxy_headers(self, url, headers=None): + """ + Sets headers needed by proxies: specifically, the Accept and Host + headers. Only sets headers not provided by the user. + """ headers_ = {'Accept': '*/*'} + + host = parse_url(url).host + if host: + headers_['Host'] = host + if headers: headers_.update(headers) @@ -154,7 +173,7 @@ class ProxyManager(RequestMethods): def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." kw['assert_same_host'] = False - kw['headers'] = self._set_proxy_headers(kw.get('headers')) + kw['headers'] = self._set_proxy_headers(url, headers=kw.get('headers')) return self.proxy_pool.urlopen(method, url, **kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 833be62..0761dc0 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -145,7 +145,17 @@ class HTTPResponse(object): # cStringIO doesn't like amt=None data = self._fp.read() else: - return self._fp.read(amt) + data = self._fp.read(amt) + if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do not + # properly close the connection in all cases. There is no harm + # in redundantly calling close. + self._fp.close() + return data try: if decode_content and decoder: diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py index 8d8654f..b827bc4 100644 --- a/requests/packages/urllib3/util.py +++ b/requests/packages/urllib3/util.py @@ -22,6 +22,7 @@ try: # Test for SSL features SSLContext = None HAS_SNI = False + import ssl from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23 from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? @@ -263,10 +264,48 @@ def is_connection_dropped(conn): return True +def resolve_cert_reqs(candidate): + """ + Resolves the argument to a numeric constant, which can be passed to + the wrap_socket function/method from the ssl module. + Defaults to :data:`ssl.CERT_NONE`. + If given a string it is assumed to be the name of the constant in the + :mod:`ssl` module or its abbrevation. + (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. + If it's neither `None` nor a string we assume it is already the numeric + constant which can directly be passed to wrap_socket. + """ + if candidate is None: + return CERT_NONE + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'CERT_' + candidate) + return res + + return candidate + + +def resolve_ssl_version(candidate): + """ + like resolve_cert_reqs + """ + if candidate is None: + return PROTOCOL_SSLv23 + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'PROTOCOL_' + candidate) + return res + + return candidate + if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): """ All arguments except `server_hostname` have the same meaning as for :func:`ssl.wrap_socket` @@ -279,8 +318,9 @@ if SSLContext is not None: # Python 3.2+ if ca_certs: try: context.load_verify_locations(ca_certs) - except TypeError as e: # Reraise as SSLError - # FIXME: This block needs a test. + # Py32 raises IOError + # Py33 raises FileNotFoundError + except Exception as e: # Reraise as SSLError raise SSLError(e) if certfile: # FIXME: This block needs a test. @@ -290,9 +330,9 @@ if SSLContext is not None: # Python 3.2+ return context.wrap_socket(sock) else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): return wrap_socket(sock, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) |