diff options
Diffstat (limited to 'requests/packages')
22 files changed, 237 insertions, 95 deletions
diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 86bb71d..e43991a 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -2,10 +2,8 @@ urllib3 - Thread-safe connection pooling and re-using. """ -__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' -__license__ = 'MIT' -__version__ = '1.12' - +from __future__ import absolute_import +import warnings from .connectionpool import ( HTTPConnectionPool, @@ -32,8 +30,30 @@ except ImportError: def emit(self, record): pass +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.13.1' + +__all__ = ( + 'HTTPConnectionPool', + 'HTTPSConnectionPool', + 'PoolManager', + 'ProxyManager', + 'HTTPResponse', + 'Retry', + 'Timeout', + 'add_stderr_logger', + 'connection_from_url', + 'disable_warnings', + 'encode_multipart_formdata', + 'get_host', + 'make_headers', + 'proxy_from_url', +) + logging.getLogger(__name__).addHandler(NullHandler()) + def add_stderr_logger(level=logging.DEBUG): """ Helper for quickly adding a StreamHandler to the logger. Useful for @@ -55,7 +75,6 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -import warnings # SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host @@ -63,6 +82,9 @@ warnings.simplefilter('default', exceptions.SubjectAltNameWarning) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter('default', exceptions.InsecurePlatformWarning, append=True) +# SNIMissingWarnings should go off only once. +warnings.simplefilter('default', exceptions.SNIMissingWarning) + def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index b68b9a5..67f3ce9 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from collections import Mapping, MutableMapping try: from threading import RLock @@ -167,7 +168,7 @@ class HTTPHeaderDict(MutableMapping): def __ne__(self, other): return not self.__eq__(other) - if not PY3: # Python 2 + if not PY3: # Python 2 iterkeys = MutableMapping.iterkeys itervalues = MutableMapping.itervalues @@ -234,7 +235,7 @@ class HTTPHeaderDict(MutableMapping): """ if len(args) > 1: raise TypeError("extend() takes at most 1 positional " - "arguments ({} given)".format(len(args))) + "arguments ({0} given)".format(len(args))) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): @@ -304,7 +305,7 @@ class HTTPHeaderDict(MutableMapping): return list(self.iteritems()) @classmethod - def from_httplib(cls, message): # Python 2 + def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index 3eab1e2..1e4cd41 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -1,4 +1,6 @@ +from __future__ import absolute_import import datetime +import os import sys import socket from socket import error as SocketError, timeout as SocketTimeout @@ -6,18 +8,13 @@ import warnings from .packages import six try: # Python 3 - from http.client import HTTPConnection as _HTTPConnection, HTTPException + from http.client import HTTPConnection as _HTTPConnection + from http.client import HTTPException # noqa: unused in this module except ImportError: - from httplib import HTTPConnection as _HTTPConnection, HTTPException - - -class DummyConnection(object): - "Used to detect a failed ConnectionCls import." - pass - + from httplib import HTTPConnection as _HTTPConnection + from httplib import HTTPException # noqa: unused in this module try: # Compiled with SSL? - HTTPSConnection = DummyConnection import ssl BaseSSLError = ssl.SSLError except (ImportError, AttributeError): # Platform-specific: No SSL. @@ -61,6 +58,11 @@ port_by_scheme = { RECENT_DATE = datetime.date(2014, 1, 1) +class DummyConnection(object): + """Used to detect a failed ConnectionCls import.""" + pass + + class HTTPConnection(_HTTPConnection, object): """ Based on httplib.HTTPConnection but provides an extra constructor @@ -205,10 +207,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs - self.ca_certs = ca_certs - self.ca_cert_dir = ca_cert_dir self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) def connect(self): # Add certificate verification @@ -263,10 +265,19 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - match_hostname(cert, self.assert_hostname or hostname) - self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED - or self.assert_fingerprint is not None) + # In case the hostname is an IPv6 address, strip the square + # brackets from it before using it to validate. This is because + # a certificate with an IPv6 address in it won't have square + # brackets around that address. Sadly, match_hostname won't do this + # for us: it expects the plain host part without any extra work + # that might have been done to make it palatable to httplib. + asserted_hostname = self.assert_hostname or hostname + asserted_hostname = asserted_hostname.strip('[]') + match_hostname(cert, asserted_hostname) + + self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or + self.assert_fingerprint is not None) if ssl: diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index b38ac68..995b416 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import errno import logging import sys @@ -10,7 +11,8 @@ try: # Python 3 from queue import LifoQueue, Empty, Full except ImportError: from Queue import LifoQueue, Empty, Full - import Queue as _ # Platform-specific: Windows + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: unused from .exceptions import ( @@ -22,7 +24,6 @@ from .exceptions import ( LocationValueError, MaxRetryError, ProxyError, - ConnectTimeoutError, ReadTimeoutError, SSLError, TimeoutError, @@ -35,7 +36,7 @@ from .connection import ( port_by_scheme, DummyConnection, HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, ConnectionError + HTTPException, BaseSSLError, ) from .request import RequestMethods from .response import HTTPResponse @@ -54,7 +55,7 @@ log = logging.getLogger(__name__) _Default = object() -## Pool objects +# Pool objects class ConnectionPool(object): """ Base class for all connection pools, such as @@ -68,8 +69,7 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - # httplib doesn't like it when we include brackets in ipv6 addresses - self.host = host.strip('[]') + self.host = host self.port = port def __str__(self): @@ -645,22 +645,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return response log.info("Redirecting %s -> %s" % (url, redirect_location)) - return self.urlopen(method, redirect_location, body, headers, - retries=retries, redirect=redirect, - assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + return self.urlopen( + method, redirect_location, body, headers, + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) # Check if we should retry the HTTP response. if retries.is_forced_retry(method, status_code=response.status): retries = retries.increment(method, url, response=response, _pool=self) retries.sleep() log.info("Forced retry: %s" % url) - return self.urlopen(method, url, body, headers, - retries=retries, redirect=redirect, - assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + return self.urlopen( + method, url, body, headers, + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) return response diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index ed9d8b8..884cdb2 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging import os import warnings @@ -60,7 +61,7 @@ class AppEngineManager(RequestMethods): raise AppEnginePlatformError( "URLFetch is not available in this environment.") - if is_prod_appengine_v2(): + if is_prod_appengine_mvms(): raise AppEnginePlatformError( "Use normal urllib3.PoolManager instead of AppEngineManager" "on Managed VMs, as using URLFetch is not necessary in " @@ -108,14 +109,14 @@ class AppEngineManager(RequestMethods): raise TimeoutError(self, e) except urlfetch.InvalidURLError as e: - if 'too large' in e.message: + if 'too large' in str(e): raise AppEnginePlatformError( "URLFetch request too large, URLFetch only " "supports requests up to 10mb in size.", e) raise ProtocolError(e) except urlfetch.DownloadError as e: - if 'Too many redirects' in e.message: + if 'Too many redirects' in str(e): raise MaxRetryError(self, url, reason=e) raise ProtocolError(e) @@ -155,7 +156,7 @@ class AppEngineManager(RequestMethods): def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): - if is_prod_appengine_v1(): + if is_prod_appengine(): # Production GAE handles deflate encoding automatically, but does # not remove the encoding header. content_encoding = urlfetch_resp.headers.get('content-encoding') @@ -176,7 +177,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if not timeout.read is timeout.connect: + if timeout.read is not timeout.connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) @@ -199,12 +200,12 @@ class AppEngineManager(RequestMethods): def is_appengine(): return (is_local_appengine() or - is_prod_appengine_v1() or - is_prod_appengine_v2()) + is_prod_appengine() or + is_prod_appengine_mvms()) def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_v2() + return is_appengine() and not is_prod_appengine_mvms() def is_local_appengine(): @@ -212,11 +213,11 @@ def is_local_appengine(): 'Development/' in os.environ['SERVER_SOFTWARE']) -def is_prod_appengine_v1(): +def is_prod_appengine(): return ('APPENGINE_RUNTIME' in os.environ and 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_v2()) + not is_prod_appengine_mvms()) -def is_prod_appengine_v2(): +def is_prod_appengine_mvms(): return os.environ.get('GAE_VM', False) == 'true' diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index c6b266f..c136a23 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -3,6 +3,7 @@ NTLM authenticating pool, contributed by erikcederstran Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 """ +from __future__ import absolute_import try: from http.client import HTTPSConnection diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index c20ae46..5996153 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -43,6 +43,7 @@ Module Variables .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) ''' +from __future__ import absolute_import try: from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT @@ -53,7 +54,7 @@ except SyntaxError as e: import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint -from socket import _fileobject, timeout +from socket import _fileobject, timeout, error as SocketError import ssl import select @@ -71,6 +72,12 @@ _openssl_versions = { ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } +if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD + +if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD + try: _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) except AttributeError: @@ -79,8 +86,8 @@ except AttributeError: _openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, - ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER - + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ssl.CERT_REQUIRED: + OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS @@ -88,12 +95,6 @@ DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 -try: - _ = memoryview - has_memoryview = True -except NameError: - has_memoryview = False - orig_util_HAS_SNI = util.HAS_SNI orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket @@ -112,7 +113,7 @@ def extract_from_urllib3(): util.HAS_SNI = orig_util_HAS_SNI -### Note: This is a slightly bug-fixed version of same from ndg-httpsclient. +# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. class SubjectAltName(BaseSubjectAltName): '''ASN.1 implementation for subjectAltNames support''' @@ -123,7 +124,7 @@ class SubjectAltName(BaseSubjectAltName): constraint.ValueSizeConstraint(1, 1024) -### Note: This is a slightly bug-fixed version of same from ndg-httpsclient. +# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. def get_subj_alt_name(peer_cert): # Search through extensions dns_name = [] @@ -181,7 +182,7 @@ class WrappedSocket(object): if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: - raise + raise SocketError(e) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' @@ -212,12 +213,9 @@ class WrappedSocket(object): continue def sendall(self, data): - if has_memoryview and not isinstance(data, memoryview): - data = memoryview(data) - total_sent = 0 while total_sent < len(data): - sent = self._send_until_done(data[total_sent:total_sent+SSL_WRITE_BLOCKSIZE]) + sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) total_sent += sent def shutdown(self): @@ -226,7 +224,10 @@ class WrappedSocket(object): def close(self): if self._makefile_refs < 1: - return self.connection.close() + try: + return self.connection.close() + except OpenSSL.SSL.Error: + return else: self._makefile_refs -= 1 diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 9607d65..8e07eb6 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -1,16 +1,17 @@ +from __future__ import absolute_import +# Base Exceptions -## Base Exceptions class HTTPError(Exception): "Base exception used by this module." pass + class HTTPWarning(Warning): "Base warning used by this module." pass - class PoolError(HTTPError): "Base exception for errors caused within a pool." def __init__(self, pool, message): @@ -57,7 +58,7 @@ class ProtocolError(HTTPError): ConnectionError = ProtocolError -## Leaf Exceptions +# Leaf Exceptions class MaxRetryError(RequestError): """Raised when the maximum number of retries is exceeded. @@ -112,10 +113,12 @@ class ConnectTimeoutError(TimeoutError): "Raised when a socket timeout occurs while connecting to a server" pass + class NewConnectionError(ConnectTimeoutError, PoolError): "Raised when we fail to establish a new connection. Usually ECONNREFUSED." pass + class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." pass @@ -172,6 +175,11 @@ class InsecurePlatformWarning(SecurityWarning): pass +class SNIMissingWarning(HTTPWarning): + "Warned when making a HTTPS request without SNI available." + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/requests/packages/urllib3/fields.py b/requests/packages/urllib3/fields.py index c853f8d..c7d4811 100644 --- a/requests/packages/urllib3/fields.py +++ b/requests/packages/urllib3/fields.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import email.utils import mimetypes diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 0fbf488..97a2843 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import codecs from uuid import uuid4 diff --git a/requests/packages/urllib3/packages/__init__.py b/requests/packages/urllib3/packages/__init__.py index 37e8351..170e974 100644 --- a/requests/packages/urllib3/packages/__init__.py +++ b/requests/packages/urllib3/packages/__init__.py @@ -2,3 +2,4 @@ from __future__ import absolute_import from . import ssl_match_hostname +__all__ = ('ssl_match_hostname', ) diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 76b6a12..f13e673 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging try: # Python 3 @@ -25,7 +26,7 @@ pool_classes_by_scheme = { log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version') + 'ssl_version', 'ca_cert_dir') class PoolManager(RequestMethods): diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py index a1a12bc..d5aa62d 100644 --- a/requests/packages/urllib3/request.py +++ b/requests/packages/urllib3/request.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: from urllib.parse import urlencode except ImportError: @@ -133,7 +134,8 @@ class RequestMethods(object): if fields: if 'body' in urlopen_kw: - raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') + raise TypeError( + "request got values for both 'fields' and 'body', can only specify one.") if encode_multipart: body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 788eb6c..8f2a1b5 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,7 +1,9 @@ +from __future__ import absolute_import from contextlib import contextmanager import zlib import io from socket import timeout as SocketTimeout +from socket import error as SocketError from ._collections import HTTPHeaderDict from .exceptions import ( @@ -130,8 +132,8 @@ class HTTPResponse(io.IOBase): if "chunked" in encodings: self.chunked = True - # We certainly don't want to preload content when the response is chunked. - if not self.chunked and preload_content and not self._body: + # If requested, preload the body. + if preload_content and not self._body: self._body = self.read(decode_content=decode_content) def get_redirect_location(self): @@ -194,12 +196,22 @@ class HTTPResponse(io.IOBase): "Received response with content-encoding: %s, but " "failed to decode it." % content_encoding, e) - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + if flush_decoder and decode_content: + data += self._flush_decoder() return data + def _flush_decoder(self): + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + buf = self._decoder.decompress(b'') + return buf + self._decoder.flush() + + return b'' + @contextmanager def _error_catcher(self): """ @@ -227,15 +239,22 @@ class HTTPResponse(io.IOBase): raise ReadTimeoutError(self._pool, None, 'Read timed out.') - except HTTPException as e: + except (HTTPException, SocketError) as e: # This includes IncompleteRead. raise ProtocolError('Connection broken: %r' % e, e) + except Exception: # The response may not be closed but we're not going to use it anymore # so close it now to ensure that the connection is released back to the pool. if self._original_response and not self._original_response.isclosed(): self._original_response.close() + # Closing the response may not actually be sufficient to close + # everything, so if we have a hold of the connection close that + # too. + if self._connection is not None: + self._connection.close() + raise finally: if self._original_response and self._original_response.isclosed(): @@ -301,7 +320,6 @@ class HTTPResponse(io.IOBase): return data - def stream(self, amt=2**16, decode_content=None): """ A generator wrapper for the read() method. A call will block until @@ -340,9 +358,9 @@ class HTTPResponse(io.IOBase): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: # Python 3 + if PY3: # Python 3 headers = HTTPHeaderDict(headers.items()) - else: # Python 2 + else: # Python 2 headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute @@ -454,7 +472,8 @@ class HTTPResponse(io.IOBase): self._init_decoder() # FIXME: Rewrite this method and make it a class with a better structured logic. if not self.chunked: - raise ResponseNotChunked("Response is not chunked. " + raise ResponseNotChunked( + "Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") # Don't bother reading the body of a HEAD request. @@ -468,8 +487,18 @@ class HTTPResponse(io.IOBase): if self.chunk_left == 0: break chunk = self._handle_chunk(amt) - yield self._decode(chunk, decode_content=decode_content, - flush_decoder=True) + decoded = self._decode(chunk, decode_content=decode_content, + flush_decoder=False) + if decoded: + yield decoded + + if decode_content: + # On CPython and PyPy, we should never need to flush the + # decoder. However, on Jython we *might* need to, so + # lets defensively do it anyway. + decoded = self._flush_decoder() + if decoded: # Platform-specific: Jython. + yield decoded # Chunk content ends with \r\n: discard it. while True: diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py index 8becc81..c6c6243 100644 --- a/requests/packages/urllib3/util/__init__.py +++ b/requests/packages/urllib3/util/__init__.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers @@ -22,3 +23,22 @@ from .url import ( split_first, Url, ) + +__all__ = ( + 'HAS_SNI', + 'SSLContext', + 'Retry', + 'Timeout', + 'Url', + 'assert_fingerprint', + 'current_time', + 'is_connection_dropped', + 'is_fp_closed', + 'get_host', + 'parse_url', + 'make_headers', + 'resolve_cert_reqs', + 'resolve_ssl_version', + 'split_first', + 'ssl_wrap_socket', +) diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 4f2f0f1..01a4812 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import socket try: from select import poll, POLLIN diff --git a/requests/packages/urllib3/util/request.py b/requests/packages/urllib3/util/request.py index bc64f6b..7377931 100644 --- a/requests/packages/urllib3/util/request.py +++ b/requests/packages/urllib3/util/request.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from base64 import b64encode from ..packages.six import b diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index 2c1de15..bc72327 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from ..packages.six.moves import http_client as httplib from ..exceptions import HeaderParsingError @@ -44,7 +45,7 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError('expected httplib.Message, got {}.'.format( + raise TypeError('expected httplib.Message, got {0}.'.format( type(headers))) defects = getattr(headers, 'defects', None) diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 1fb1f23..03a0124 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import time import logging @@ -126,7 +127,7 @@ class Retry(object): self.method_whitelist = method_whitelist self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect - self._observed_errors = _observed_errors # TODO: use .history instead? + self._observed_errors = _observed_errors # TODO: use .history instead? def new(self, **kw): params = dict( @@ -206,7 +207,8 @@ class Retry(object): return min(retry_counts) < 0 - def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None): + def increment(self, method=None, url=None, response=None, error=None, + _pool=None, _stacktrace=None): """ Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not @@ -274,7 +276,6 @@ class Retry(object): return new_retry - def __repr__(self): return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' 'read={self.read}, redirect={self.redirect})').format( diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 47b817e..67f8344 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -1,7 +1,12 @@ +from __future__ import absolute_import +import errno +import warnings +import hmac + from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 -from ..exceptions import SSLError, InsecurePlatformWarning +from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None @@ -15,8 +20,23 @@ HASHFUNC_MAP = { 64: sha256, } -import errno -import warnings + +def _const_compare_digest_backport(a, b): + """ + Compare two digests of equal length in constant time. + + The digests must be of type str/bytes. + Returns True if the digests match, and False otherwise. + """ + result = abs(len(a) - len(b)) + for l, r in zip(bytearray(a), bytearray(b)): + result |= l ^ r + return result == 0 + + +_const_compare_digest = getattr(hmac, 'compare_digest', + _const_compare_digest_backport) + try: # Test for SSL features import ssl @@ -134,7 +154,7 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() - if cert_digest != fingerprint_bytes: + if not _const_compare_digest(cert_digest, fingerprint_bytes): raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' .format(fingerprint, hexlify(cert_digest))) @@ -283,4 +303,15 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, context.load_cert_chain(certfile, keyfile) if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI return context.wrap_socket(sock, server_hostname=server_hostname) + + warnings.warn( + 'An HTTPS request has been made, but the SNI (Subject Name ' + 'Indication) extension to TLS is not available on this platform. ' + 'This may cause the server to present an incorrect TLS ' + 'certificate, which can cause validation failures. For more ' + 'information, see ' + 'https://urllib3.readthedocs.org/en/latest/security.html' + '#snimissingwarning.', + SNIMissingWarning + ) return context.wrap_socket(sock) diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py index ea7027f..ff62f47 100644 --- a/requests/packages/urllib3/util/timeout.py +++ b/requests/packages/urllib3/util/timeout.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -9,6 +10,7 @@ from ..exceptions import TimeoutStateError # urllib3 _Default = object() + def current_time(): """ Retrieve the current time. This function is mocked out in unit testing. @@ -226,9 +228,9 @@ class Timeout(object): has not yet been called on this object. """ if (self.total is not None and - self.total is not self.DEFAULT_TIMEOUT and - self._read is not None and - self._read is not self.DEFAULT_TIMEOUT): + self.total is not self.DEFAULT_TIMEOUT and + self._read is not None and + self._read is not self.DEFAULT_TIMEOUT): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index e58050c..e996204 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from collections import namedtuple from ..exceptions import LocationParseError @@ -85,6 +86,7 @@ class Url(namedtuple('Url', url_attrs)): def __str__(self): return self.url + def split_first(s, delims): """ Given a string and an iterable of delimiters, split on the first found @@ -115,7 +117,7 @@ def split_first(s, delims): if min_idx is None or min_idx < 0: return s, '', None - return s[:min_idx], s[min_idx+1:], min_delim + return s[:min_idx], s[min_idx + 1:], min_delim def parse_url(url): @@ -206,6 +208,7 @@ def parse_url(url): return Url(scheme, auth, host, port, path, query, fragment) + def get_host(url): """ Deprecated. Use :func:`.parse_url` instead. |