diff options
Diffstat (limited to 'requests')
32 files changed, 346 insertions, 137 deletions
diff --git a/requests/__init__.py b/requests/__init__.py index 3d8188a..bd5b5b9 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at <http://python-requests.org>. """ __title__ = 'requests' -__version__ = '2.8.1' -__build__ = 0x020801 +__version__ = '2.9.1' +__build__ = 0x020901 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' @@ -62,7 +62,8 @@ from .sessions import session, Session from .status_codes import codes from .exceptions import ( RequestException, Timeout, URLRequired, - TooManyRedirects, HTTPError, ConnectionError + TooManyRedirects, HTTPError, ConnectionError, + FileModeWarning, ) # Set default logging handler to avoid "No handler found" warnings. @@ -75,3 +76,8 @@ except ImportError: pass logging.getLogger(__name__).addHandler(NullHandler()) + +import warnings + +# FileModeWarnings go off per the default. +warnings.simplefilter('default', FileModeWarning, append=True) diff --git a/requests/adapters.py b/requests/adapters.py index 7682db0..6266d5b 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -8,6 +8,7 @@ This module contains the transport adapters that Requests uses to define and maintain connections. """ +import os.path import socket from .models import Response @@ -107,7 +108,7 @@ class HTTPAdapter(BaseAdapter): def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because - # because self.poolmanager uses a lambda function, which isn't pickleable. + # self.poolmanager uses a lambda function, which isn't pickleable. self.proxy_manager = {} self.config = {} @@ -185,10 +186,15 @@ class HTTPAdapter(BaseAdapter): raise Exception("Could not find a suitable SSL CA certificate bundle.") conn.cert_reqs = 'CERT_REQUIRED' - conn.ca_certs = cert_loc + + if not os.path.isdir(cert_loc): + conn.ca_certs = cert_loc + else: + conn.ca_cert_dir = cert_loc else: conn.cert_reqs = 'CERT_NONE' conn.ca_certs = None + conn.ca_cert_dir = None if cert: if not isinstance(cert, basestring): @@ -394,7 +400,15 @@ class HTTPAdapter(BaseAdapter): low_conn.send(b'\r\n') low_conn.send(b'0\r\n\r\n') - r = low_conn.getresponse() + # Receive the response from the server + try: + # For Python 2.7+ versions, use buffering of HTTP + # responses + r = low_conn.getresponse(buffering=True) + except TypeError: + # For compatibility with Python 2.6 versions and back + r = low_conn.getresponse() + resp = HTTPResponse.from_httplib( r, pool=conn, diff --git a/requests/api.py b/requests/api.py index 72a777b..b21a1a4 100644 --- a/requests/api.py +++ b/requests/api.py @@ -33,7 +33,7 @@ def request(method, url, **kwargs): :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. - :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. + :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. :param stream: (optional) if ``False``, the response content will be immediately downloaded. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :return: :class:`Response <Response>` object @@ -46,13 +46,11 @@ def request(method, url, **kwargs): <Response [200]> """ - session = sessions.Session() - response = session.request(method=method, url=url, **kwargs) - # By explicitly closing the session, we avoid leaving sockets open which - # can trigger a ResourceWarning in some cases, and look like a memory leak - # in others. - session.close() - return response + # By using the 'with' statement we are sure the session is closed, thus we + # avoid leaving sockets open which can trigger a ResourceWarning in some + # cases, and look like a memory leak in others. + with sessions.Session() as session: + return session.request(method=method, url=url, **kwargs) def get(url, params=None, **kwargs): diff --git a/requests/auth.py b/requests/auth.py index 8c4e847..2af55fb 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -136,7 +136,7 @@ class HTTPDigestAuth(AuthBase): if _algorithm == 'MD5-SESS': HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce)) - if qop is None: + if not qop: respdig = KD(HA1, "%s:%s" % (nonce, HA2)) elif qop == 'auth' or 'auth' in qop.split(','): noncebit = "%s:%s:%s:%s:%s" % ( diff --git a/requests/cookies.py b/requests/cookies.py index 88b478c..b85fd2b 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -8,6 +8,7 @@ requests.utils imports from here, so be careful with imports. import copy import time +import calendar import collections from .compat import cookielib, urlparse, urlunparse, Morsel @@ -143,10 +144,13 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None): """ clearables = [] for cookie in cookiejar: - if cookie.name == name: - if domain is None or domain == cookie.domain: - if path is None or path == cookie.path: - clearables.append((cookie.domain, cookie.path, cookie.name)) + if cookie.name != name: + continue + if domain is not None and domain != cookie.domain: + continue + if path is not None and path != cookie.path: + continue + clearables.append((cookie.domain, cookie.path, cookie.name)) for domain, path, name in clearables: cookiejar.clear(domain, path, name) @@ -365,7 +369,7 @@ def _copy_cookie_jar(jar): return None if hasattr(jar, 'copy'): - # We're dealing with an instane of RequestsCookieJar + # We're dealing with an instance of RequestsCookieJar return jar.copy() # We're dealing with a generic CookieJar instance new_jar = copy.copy(jar) @@ -421,8 +425,9 @@ def morsel_to_cookie(morsel): raise TypeError('max-age: %s must be integer' % morsel['max-age']) elif morsel['expires']: time_template = '%a, %d-%b-%Y %H:%M:%S GMT' - expires = int(time.mktime( - time.strptime(morsel['expires'], time_template)) - time.timezone) + expires = calendar.timegm( + time.strptime(morsel['expires'], time_template) + ) return create_cookie( comment=morsel['comment'], comment_url=bool(morsel['comment']), diff --git a/requests/exceptions.py b/requests/exceptions.py index 89135a8..ba0b910 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -97,3 +97,18 @@ class StreamConsumedError(RequestException, TypeError): class RetryError(RequestException): """Custom retries logic failed""" + + +# Warnings + + +class RequestsWarning(Warning): + """Base warning for Requests.""" + pass + + +class FileModeWarning(RequestsWarning, DeprecationWarning): + """ + A file was opened in text mode, but Requests determined its binary length. + """ + pass diff --git a/requests/models.py b/requests/models.py index 2727bee..4bcbc54 100644 --- a/requests/models.py +++ b/requests/models.py @@ -324,7 +324,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_url(self, url, params): """Prepares the given HTTP URL.""" #: Accept objects that have string representations. - #: We're unable to blindy call unicode/str functions + #: We're unable to blindly call unicode/str functions #: as this will include the bytestring indicator (b'') #: on python 3.x. #: https://github.com/kennethreitz/requests/pull/2238 @@ -385,6 +385,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if isinstance(fragment, str): fragment = fragment.encode('utf-8') + if isinstance(params, (str, bytes)): + params = to_native_string(params) + enc_params = self._encode_params(params) if enc_params: if query: @@ -434,7 +437,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if files: raise NotImplementedError('Streamed bodies and files are mutually exclusive.') - if length is not None: + if length: self.headers['Content-Length'] = builtin_str(length) else: self.headers['Transfer-Encoding'] = 'chunked' @@ -631,7 +634,7 @@ class Response(object): @property def is_permanent_redirect(self): - """True if this Response one of the permanant versions of redirect""" + """True if this Response one of the permanent versions of redirect""" return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) @property diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 86bb71d..e43991a 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -2,10 +2,8 @@ urllib3 - Thread-safe connection pooling and re-using. """ -__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' -__license__ = 'MIT' -__version__ = '1.12' - +from __future__ import absolute_import +import warnings from .connectionpool import ( HTTPConnectionPool, @@ -32,8 +30,30 @@ except ImportError: def emit(self, record): pass +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.13.1' + +__all__ = ( + 'HTTPConnectionPool', + 'HTTPSConnectionPool', + 'PoolManager', + 'ProxyManager', + 'HTTPResponse', + 'Retry', + 'Timeout', + 'add_stderr_logger', + 'connection_from_url', + 'disable_warnings', + 'encode_multipart_formdata', + 'get_host', + 'make_headers', + 'proxy_from_url', +) + logging.getLogger(__name__).addHandler(NullHandler()) + def add_stderr_logger(level=logging.DEBUG): """ Helper for quickly adding a StreamHandler to the logger. Useful for @@ -55,7 +75,6 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -import warnings # SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host @@ -63,6 +82,9 @@ warnings.simplefilter('default', exceptions.SubjectAltNameWarning) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter('default', exceptions.InsecurePlatformWarning, append=True) +# SNIMissingWarnings should go off only once. +warnings.simplefilter('default', exceptions.SNIMissingWarning) + def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index b68b9a5..67f3ce9 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from collections import Mapping, MutableMapping try: from threading import RLock @@ -167,7 +168,7 @@ class HTTPHeaderDict(MutableMapping): def __ne__(self, other): return not self.__eq__(other) - if not PY3: # Python 2 + if not PY3: # Python 2 iterkeys = MutableMapping.iterkeys itervalues = MutableMapping.itervalues @@ -234,7 +235,7 @@ class HTTPHeaderDict(MutableMapping): """ if len(args) > 1: raise TypeError("extend() takes at most 1 positional " - "arguments ({} given)".format(len(args))) + "arguments ({0} given)".format(len(args))) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): @@ -304,7 +305,7 @@ class HTTPHeaderDict(MutableMapping): return list(self.iteritems()) @classmethod - def from_httplib(cls, message): # Python 2 + def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index 3eab1e2..1e4cd41 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -1,4 +1,6 @@ +from __future__ import absolute_import import datetime +import os import sys import socket from socket import error as SocketError, timeout as SocketTimeout @@ -6,18 +8,13 @@ import warnings from .packages import six try: # Python 3 - from http.client import HTTPConnection as _HTTPConnection, HTTPException + from http.client import HTTPConnection as _HTTPConnection + from http.client import HTTPException # noqa: unused in this module except ImportError: - from httplib import HTTPConnection as _HTTPConnection, HTTPException - - -class DummyConnection(object): - "Used to detect a failed ConnectionCls import." - pass - + from httplib import HTTPConnection as _HTTPConnection + from httplib import HTTPException # noqa: unused in this module try: # Compiled with SSL? - HTTPSConnection = DummyConnection import ssl BaseSSLError = ssl.SSLError except (ImportError, AttributeError): # Platform-specific: No SSL. @@ -61,6 +58,11 @@ port_by_scheme = { RECENT_DATE = datetime.date(2014, 1, 1) +class DummyConnection(object): + """Used to detect a failed ConnectionCls import.""" + pass + + class HTTPConnection(_HTTPConnection, object): """ Based on httplib.HTTPConnection but provides an extra constructor @@ -205,10 +207,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs - self.ca_certs = ca_certs - self.ca_cert_dir = ca_cert_dir self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) def connect(self): # Add certificate verification @@ -263,10 +265,19 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - match_hostname(cert, self.assert_hostname or hostname) - self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED - or self.assert_fingerprint is not None) + # In case the hostname is an IPv6 address, strip the square + # brackets from it before using it to validate. This is because + # a certificate with an IPv6 address in it won't have square + # brackets around that address. Sadly, match_hostname won't do this + # for us: it expects the plain host part without any extra work + # that might have been done to make it palatable to httplib. + asserted_hostname = self.assert_hostname or hostname + asserted_hostname = asserted_hostname.strip('[]') + match_hostname(cert, asserted_hostname) + + self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or + self.assert_fingerprint is not None) if ssl: diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index b38ac68..995b416 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import errno import logging import sys @@ -10,7 +11,8 @@ try: # Python 3 from queue import LifoQueue, Empty, Full except ImportError: from Queue import LifoQueue, Empty, Full - import Queue as _ # Platform-specific: Windows + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: unused from .exceptions import ( @@ -22,7 +24,6 @@ from .exceptions import ( LocationValueError, MaxRetryError, ProxyError, - ConnectTimeoutError, ReadTimeoutError, SSLError, TimeoutError, @@ -35,7 +36,7 @@ from .connection import ( port_by_scheme, DummyConnection, HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, ConnectionError + HTTPException, BaseSSLError, ) from .request import RequestMethods from .response import HTTPResponse @@ -54,7 +55,7 @@ log = logging.getLogger(__name__) _Default = object() -## Pool objects +# Pool objects class ConnectionPool(object): """ Base class for all connection pools, such as @@ -68,8 +69,7 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - # httplib doesn't like it when we include brackets in ipv6 addresses - self.host = host.strip('[]') + self.host = host self.port = port def __str__(self): @@ -645,22 +645,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return response log.info("Redirecting %s -> %s" % (url, redirect_location)) - return self.urlopen(method, redirect_location, body, headers, - retries=retries, redirect=redirect, - assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + return self.urlopen( + method, redirect_location, body, headers, + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) # Check if we should retry the HTTP response. if retries.is_forced_retry(method, status_code=response.status): retries = retries.increment(method, url, response=response, _pool=self) retries.sleep() log.info("Forced retry: %s" % url) - return self.urlopen(method, url, body, headers, - retries=retries, redirect=redirect, - assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + return self.urlopen( + method, url, body, headers, + retries=retries, redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, pool_timeout=pool_timeout, + release_conn=release_conn, **response_kw) return response diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index ed9d8b8..884cdb2 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging import os import warnings @@ -60,7 +61,7 @@ class AppEngineManager(RequestMethods): raise AppEnginePlatformError( "URLFetch is not available in this environment.") - if is_prod_appengine_v2(): + if is_prod_appengine_mvms(): raise AppEnginePlatformError( "Use normal urllib3.PoolManager instead of AppEngineManager" "on Managed VMs, as using URLFetch is not necessary in " @@ -108,14 +109,14 @@ class AppEngineManager(RequestMethods): raise TimeoutError(self, e) except urlfetch.InvalidURLError as e: - if 'too large' in e.message: + if 'too large' in str(e): raise AppEnginePlatformError( "URLFetch request too large, URLFetch only " "supports requests up to 10mb in size.", e) raise ProtocolError(e) except urlfetch.DownloadError as e: - if 'Too many redirects' in e.message: + if 'Too many redirects' in str(e): raise MaxRetryError(self, url, reason=e) raise ProtocolError(e) @@ -155,7 +156,7 @@ class AppEngineManager(RequestMethods): def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): - if is_prod_appengine_v1(): + if is_prod_appengine(): # Production GAE handles deflate encoding automatically, but does # not remove the encoding header. content_encoding = urlfetch_resp.headers.get('content-encoding') @@ -176,7 +177,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if not timeout.read is timeout.connect: + if timeout.read is not timeout.connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) @@ -199,12 +200,12 @@ class AppEngineManager(RequestMethods): def is_appengine(): return (is_local_appengine() or - is_prod_appengine_v1() or - is_prod_appengine_v2()) + is_prod_appengine() or + is_prod_appengine_mvms()) def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_v2() + return is_appengine() and not is_prod_appengine_mvms() def is_local_appengine(): @@ -212,11 +213,11 @@ def is_local_appengine(): 'Development/' in os.environ['SERVER_SOFTWARE']) -def is_prod_appengine_v1(): +def is_prod_appengine(): return ('APPENGINE_RUNTIME' in os.environ and 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_v2()) + not is_prod_appengine_mvms()) -def is_prod_appengine_v2(): +def is_prod_appengine_mvms(): return os.environ.get('GAE_VM', False) == 'true' diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index c6b266f..c136a23 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -3,6 +3,7 @@ NTLM authenticating pool, contributed by erikcederstran Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 """ +from __future__ import absolute_import try: from http.client import HTTPSConnection diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index c20ae46..5996153 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -43,6 +43,7 @@ Module Variables .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) ''' +from __future__ import absolute_import try: from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT @@ -53,7 +54,7 @@ except SyntaxError as e: import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint -from socket import _fileobject, timeout +from socket import _fileobject, timeout, error as SocketError import ssl import select @@ -71,6 +72,12 @@ _openssl_versions = { ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } +if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD + +if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD + try: _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) except AttributeError: @@ -79,8 +86,8 @@ except AttributeError: _openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, - ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER - + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ssl.CERT_REQUIRED: + OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS @@ -88,12 +95,6 @@ DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 -try: - _ = memoryview - has_memoryview = True -except NameError: - has_memoryview = False - orig_util_HAS_SNI = util.HAS_SNI orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket @@ -112,7 +113,7 @@ def extract_from_urllib3(): util.HAS_SNI = orig_util_HAS_SNI -### Note: This is a slightly bug-fixed version of same from ndg-httpsclient. +# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. class SubjectAltName(BaseSubjectAltName): '''ASN.1 implementation for subjectAltNames support''' @@ -123,7 +124,7 @@ class SubjectAltName(BaseSubjectAltName): constraint.ValueSizeConstraint(1, 1024) -### Note: This is a slightly bug-fixed version of same from ndg-httpsclient. +# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. def get_subj_alt_name(peer_cert): # Search through extensions dns_name = [] @@ -181,7 +182,7 @@ class WrappedSocket(object): if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: - raise + raise SocketError(e) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' @@ -212,12 +213,9 @@ class WrappedSocket(object): continue def sendall(self, data): - if has_memoryview and not isinstance(data, memoryview): - data = memoryview(data) - total_sent = 0 while total_sent < len(data): - sent = self._send_until_done(data[total_sent:total_sent+SSL_WRITE_BLOCKSIZE]) + sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) total_sent += sent def shutdown(self): @@ -226,7 +224,10 @@ class WrappedSocket(object): def close(self): if self._makefile_refs < 1: - return self.connection.close() + try: + return self.connection.close() + except OpenSSL.SSL.Error: + return else: self._makefile_refs -= 1 diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 9607d65..8e07eb6 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -1,16 +1,17 @@ +from __future__ import absolute_import +# Base Exceptions -## Base Exceptions class HTTPError(Exception): "Base exception used by this module." pass + class HTTPWarning(Warning): "Base warning used by this module." pass - class PoolError(HTTPError): "Base exception for errors caused within a pool." def __init__(self, pool, message): @@ -57,7 +58,7 @@ class ProtocolError(HTTPError): ConnectionError = ProtocolError -## Leaf Exceptions +# Leaf Exceptions class MaxRetryError(RequestError): """Raised when the maximum number of retries is exceeded. @@ -112,10 +113,12 @@ class ConnectTimeoutError(TimeoutError): "Raised when a socket timeout occurs while connecting to a server" pass + class NewConnectionError(ConnectTimeoutError, PoolError): "Raised when we fail to establish a new connection. Usually ECONNREFUSED." pass + class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." pass @@ -172,6 +175,11 @@ class InsecurePlatformWarning(SecurityWarning): pass +class SNIMissingWarning(HTTPWarning): + "Warned when making a HTTPS request without SNI available." + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/requests/packages/urllib3/fields.py b/requests/packages/urllib3/fields.py index c853f8d..c7d4811 100644 --- a/requests/packages/urllib3/fields.py +++ b/requests/packages/urllib3/fields.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import email.utils import mimetypes diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 0fbf488..97a2843 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import codecs from uuid import uuid4 diff --git a/requests/packages/urllib3/packages/__init__.py b/requests/packages/urllib3/packages/__init__.py index 37e8351..170e974 100644 --- a/requests/packages/urllib3/packages/__init__.py +++ b/requests/packages/urllib3/packages/__init__.py @@ -2,3 +2,4 @@ from __future__ import absolute_import from . import ssl_match_hostname +__all__ = ('ssl_match_hostname', ) diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 76b6a12..f13e673 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging try: # Python 3 @@ -25,7 +26,7 @@ pool_classes_by_scheme = { log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version') + 'ssl_version', 'ca_cert_dir') class PoolManager(RequestMethods): diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py index a1a12bc..d5aa62d 100644 --- a/requests/packages/urllib3/request.py +++ b/requests/packages/urllib3/request.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: from urllib.parse import urlencode except ImportError: @@ -133,7 +134,8 @@ class RequestMethods(object): if fields: if 'body' in urlopen_kw: - raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') + raise TypeError( + "request got values for both 'fields' and 'body', can only specify one.") if encode_multipart: body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 788eb6c..8f2a1b5 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,7 +1,9 @@ +from __future__ import absolute_import from contextlib import contextmanager import zlib import io from socket import timeout as SocketTimeout +from socket import error as SocketError from ._collections import HTTPHeaderDict from .exceptions import ( @@ -130,8 +132,8 @@ class HTTPResponse(io.IOBase): if "chunked" in encodings: self.chunked = True - # We certainly don't want to preload content when the response is chunked. - if not self.chunked and preload_content and not self._body: + # If requested, preload the body. + if preload_content and not self._body: self._body = self.read(decode_content=decode_content) def get_redirect_location(self): @@ -194,12 +196,22 @@ class HTTPResponse(io.IOBase): "Received response with content-encoding: %s, but " "failed to decode it." % content_encoding, e) - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + if flush_decoder and decode_content: + data += self._flush_decoder() return data + def _flush_decoder(self): + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + buf = self._decoder.decompress(b'') + return buf + self._decoder.flush() + + return b'' + @contextmanager def _error_catcher(self): """ @@ -227,15 +239,22 @@ class HTTPResponse(io.IOBase): raise ReadTimeoutError(self._pool, None, 'Read timed out.') - except HTTPException as e: + except (HTTPException, SocketError) as e: # This includes IncompleteRead. raise ProtocolError('Connection broken: %r' % e, e) + except Exception: # The response may not be closed but we're not going to use it anymore # so close it now to ensure that the connection is released back to the pool. if self._original_response and not self._original_response.isclosed(): self._original_response.close() + # Closing the response may not actually be sufficient to close + # everything, so if we have a hold of the connection close that + # too. + if self._connection is not None: + self._connection.close() + raise finally: if self._original_response and self._original_response.isclosed(): @@ -301,7 +320,6 @@ class HTTPResponse(io.IOBase): return data - def stream(self, amt=2**16, decode_content=None): """ A generator wrapper for the read() method. A call will block until @@ -340,9 +358,9 @@ class HTTPResponse(io.IOBase): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: # Python 3 + if PY3: # Python 3 headers = HTTPHeaderDict(headers.items()) - else: # Python 2 + else: # Python 2 headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute @@ -454,7 +472,8 @@ class HTTPResponse(io.IOBase): self._init_decoder() # FIXME: Rewrite this method and make it a class with a better structured logic. if not self.chunked: - raise ResponseNotChunked("Response is not chunked. " + raise ResponseNotChunked( + "Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") # Don't bother reading the body of a HEAD request. @@ -468,8 +487,18 @@ class HTTPResponse(io.IOBase): if self.chunk_left == 0: break chunk = self._handle_chunk(amt) - yield self._decode(chunk, decode_content=decode_content, - flush_decoder=True) + decoded = self._decode(chunk, decode_content=decode_content, + flush_decoder=False) + if decoded: + yield decoded + + if decode_content: + # On CPython and PyPy, we should never need to flush the + # decoder. However, on Jython we *might* need to, so + # lets defensively do it anyway. + decoded = self._flush_decoder() + if decoded: # Platform-specific: Jython. + yield decoded # Chunk content ends with \r\n: discard it. while True: diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py index 8becc81..c6c6243 100644 --- a/requests/packages/urllib3/util/__init__.py +++ b/requests/packages/urllib3/util/__init__.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers @@ -22,3 +23,22 @@ from .url import ( split_first, Url, ) + +__all__ = ( + 'HAS_SNI', + 'SSLContext', + 'Retry', + 'Timeout', + 'Url', + 'assert_fingerprint', + 'current_time', + 'is_connection_dropped', + 'is_fp_closed', + 'get_host', + 'parse_url', + 'make_headers', + 'resolve_cert_reqs', + 'resolve_ssl_version', + 'split_first', + 'ssl_wrap_socket', +) diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 4f2f0f1..01a4812 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import socket try: from select import poll, POLLIN diff --git a/requests/packages/urllib3/util/request.py b/requests/packages/urllib3/util/request.py index bc64f6b..7377931 100644 --- a/requests/packages/urllib3/util/request.py +++ b/requests/packages/urllib3/util/request.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from base64 import b64encode from ..packages.six import b diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index 2c1de15..bc72327 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from ..packages.six.moves import http_client as httplib from ..exceptions import HeaderParsingError @@ -44,7 +45,7 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError('expected httplib.Message, got {}.'.format( + raise TypeError('expected httplib.Message, got {0}.'.format( type(headers))) defects = getattr(headers, 'defects', None) diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 1fb1f23..03a0124 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import time import logging @@ -126,7 +127,7 @@ class Retry(object): self.method_whitelist = method_whitelist self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect - self._observed_errors = _observed_errors # TODO: use .history instead? + self._observed_errors = _observed_errors # TODO: use .history instead? def new(self, **kw): params = dict( @@ -206,7 +207,8 @@ class Retry(object): return min(retry_counts) < 0 - def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None): + def increment(self, method=None, url=None, response=None, error=None, + _pool=None, _stacktrace=None): """ Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not @@ -274,7 +276,6 @@ class Retry(object): return new_retry - def __repr__(self): return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' 'read={self.read}, redirect={self.redirect})').format( diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 47b817e..67f8344 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -1,7 +1,12 @@ +from __future__ import absolute_import +import errno +import warnings +import hmac + from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 -from ..exceptions import SSLError, InsecurePlatformWarning +from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None @@ -15,8 +20,23 @@ HASHFUNC_MAP = { 64: sha256, } -import errno -import warnings + +def _const_compare_digest_backport(a, b): + """ + Compare two digests of equal length in constant time. + + The digests must be of type str/bytes. + Returns True if the digests match, and False otherwise. + """ + result = abs(len(a) - len(b)) + for l, r in zip(bytearray(a), bytearray(b)): + result |= l ^ r + return result == 0 + + +_const_compare_digest = getattr(hmac, 'compare_digest', + _const_compare_digest_backport) + try: # Test for SSL features import ssl @@ -134,7 +154,7 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() - if cert_digest != fingerprint_bytes: + if not _const_compare_digest(cert_digest, fingerprint_bytes): raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' .format(fingerprint, hexlify(cert_digest))) @@ -283,4 +303,15 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, context.load_cert_chain(certfile, keyfile) if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI return context.wrap_socket(sock, server_hostname=server_hostname) + + warnings.warn( + 'An HTTPS request has been made, but the SNI (Subject Name ' + 'Indication) extension to TLS is not available on this platform. ' + 'This may cause the server to present an incorrect TLS ' + 'certificate, which can cause validation failures. For more ' + 'information, see ' + 'https://urllib3.readthedocs.org/en/latest/security.html' + '#snimissingwarning.', + SNIMissingWarning + ) return context.wrap_socket(sock) diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py index ea7027f..ff62f47 100644 --- a/requests/packages/urllib3/util/timeout.py +++ b/requests/packages/urllib3/util/timeout.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -9,6 +10,7 @@ from ..exceptions import TimeoutStateError # urllib3 _Default = object() + def current_time(): """ Retrieve the current time. This function is mocked out in unit testing. @@ -226,9 +228,9 @@ class Timeout(object): has not yet been called on this object. """ if (self.total is not None and - self.total is not self.DEFAULT_TIMEOUT and - self._read is not None and - self._read is not self.DEFAULT_TIMEOUT): + self.total is not self.DEFAULT_TIMEOUT and + self._read is not None and + self._read is not self.DEFAULT_TIMEOUT): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index e58050c..e996204 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from collections import namedtuple from ..exceptions import LocationParseError @@ -85,6 +86,7 @@ class Url(namedtuple('Url', url_attrs)): def __str__(self): return self.url + def split_first(s, delims): """ Given a string and an iterable of delimiters, split on the first found @@ -115,7 +117,7 @@ def split_first(s, delims): if min_idx is None or min_idx < 0: return s, '', None - return s[:min_idx], s[min_idx+1:], min_delim + return s[:min_idx], s[min_idx + 1:], min_delim def parse_url(url): @@ -206,6 +208,7 @@ def parse_url(url): return Url(scheme, auth, host, port, path, query, fragment) + def get_host(url): """ Deprecated. Use :func:`.parse_url` instead. diff --git a/requests/sessions.py b/requests/sessions.py index ad63902..9eaa36a 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -273,13 +273,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() >>> s.get('http://httpbin.org/get') - 200 + <Response [200]> Or as a context manager:: >>> with requests.Session() as s: >>> s.get('http://httpbin.org/get') - 200 + <Response [200]> """ __attrs__ = [ @@ -325,7 +325,7 @@ class Session(SessionRedirectMixin): #: limit, a :class:`TooManyRedirects` exception is raised. self.max_redirects = DEFAULT_REDIRECT_LIMIT - #: Trust environement settings for proxy configuration, default + #: Trust environment settings for proxy configuration, default #: authentication and similar. self.trust_env = True @@ -433,8 +433,8 @@ class Session(SessionRedirectMixin): hostname to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. - :param verify: (optional) if ``True``, the SSL cert will be verified. - A CA_BUNDLE path can also be provided. + :param verify: (optional) whether the SSL cert will be verified. + A CA_BUNDLE path can also be provided. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ @@ -634,7 +634,7 @@ class Session(SessionRedirectMixin): 'cert': cert} def get_adapter(self, url): - """Returns the appropriate connnection adapter for the given URL.""" + """Returns the appropriate connection adapter for the given URL.""" for (prefix, adapter) in self.adapters.items(): if url.lower().startswith(prefix): diff --git a/requests/status_codes.py b/requests/status_codes.py index 1db7fc0..a852574 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -78,6 +78,7 @@ _codes = { 507: ('insufficient_storage',), 509: ('bandwidth_limit_exceeded', 'bandwidth'), 510: ('not_extended',), + 511: ('network_authentication_required', 'network_auth', 'network_authentication'), } codes = LookupDict(name='status_codes') diff --git a/requests/utils.py b/requests/utils.py index 4a8c6d7..c5c3fd0 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -29,7 +29,7 @@ from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, basestring) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict -from .exceptions import InvalidURL +from .exceptions import InvalidURL, FileModeWarning _hush_pyflakes = (RequestsCookieJar,) @@ -48,23 +48,44 @@ def dict_to_sequence(d): def super_len(o): + total_length = 0 + current_position = 0 + if hasattr(o, '__len__'): - return len(o) + total_length = len(o) + + elif hasattr(o, 'len'): + total_length = o.len - if hasattr(o, 'len'): - return o.len + elif hasattr(o, 'getvalue'): + # e.g. BytesIO, cStringIO.StringIO + total_length = len(o.getvalue()) - if hasattr(o, 'fileno'): + elif hasattr(o, 'fileno'): try: fileno = o.fileno() except io.UnsupportedOperation: pass else: - return os.fstat(fileno).st_size + total_length = os.fstat(fileno).st_size - if hasattr(o, 'getvalue'): - # e.g. BytesIO, cStringIO.StringIO - return len(o.getvalue()) + # Having used fstat to determine the file length, we need to + # confirm that this file was opened up in binary mode. + if 'b' not in o.mode: + warnings.warn(( + "Requests has determined the content-length for this " + "request using the binary size of the file: however, the " + "file has been opened in text mode (i.e. without the 'b' " + "flag in the mode). This may lead to an incorrect " + "content-length. In Requests 3.0, support will be removed " + "for files in text mode."), + FileModeWarning + ) + + if hasattr(o, 'tell'): + current_position = o.tell() + + return max(0, total_length - current_position) def get_netrc_auth(url, raise_errors=False): @@ -94,8 +115,12 @@ def get_netrc_auth(url, raise_errors=False): ri = urlparse(url) - # Strip port numbers from netloc - host = ri.netloc.split(':')[0] + # Strip port numbers from netloc. This weird `if...encode`` dance is + # used for Python 3.2, which doesn't support unicode literals. + splitstr = b':' + if isinstance(url, str): + splitstr = splitstr.decode('ascii') + host = ri.netloc.split(splitstr)[0] try: _netrc = netrc(netrc_path).authenticators(host) |