aboutsummaryrefslogtreecommitdiff
path: root/requests
diff options
context:
space:
mode:
authorChristopher Baines <mail@cbaines.net>2015-12-22 13:46:21 +0000
committerChristopher Baines <mail@cbaines.net>2015-12-22 13:46:21 +0000
commit1f19c06843e6d266368e3b570352bdf7d789a0de (patch)
tree042bc99c162d671c2b2fb7cc5cb4400ef26c1bed /requests
parentcb40ec082506c0d9eb05978839bed2f12541af35 (diff)
downloadpython-requests-1f19c06843e6d266368e3b570352bdf7d789a0de.tar
python-requests-1f19c06843e6d266368e3b570352bdf7d789a0de.tar.gz
Import requests_2.9.1.orig.tar.gzupstream/2.9.1upstream
Diffstat (limited to 'requests')
-rw-r--r--requests/__init__.py12
-rw-r--r--requests/adapters.py20
-rw-r--r--requests/api.py14
-rw-r--r--requests/auth.py2
-rw-r--r--requests/cookies.py19
-rw-r--r--requests/exceptions.py15
-rw-r--r--requests/models.py9
-rw-r--r--requests/packages/urllib3/__init__.py32
-rw-r--r--requests/packages/urllib3/_collections.py7
-rw-r--r--requests/packages/urllib3/connection.py39
-rw-r--r--requests/packages/urllib3/connectionpool.py34
-rw-r--r--requests/packages/urllib3/contrib/appengine.py23
-rw-r--r--requests/packages/urllib3/contrib/ntlmpool.py1
-rw-r--r--requests/packages/urllib3/contrib/pyopenssl.py35
-rw-r--r--requests/packages/urllib3/exceptions.py14
-rw-r--r--requests/packages/urllib3/fields.py1
-rw-r--r--requests/packages/urllib3/filepost.py1
-rw-r--r--requests/packages/urllib3/packages/__init__.py1
-rw-r--r--requests/packages/urllib3/poolmanager.py3
-rw-r--r--requests/packages/urllib3/request.py4
-rw-r--r--requests/packages/urllib3/response.py53
-rw-r--r--requests/packages/urllib3/util/__init__.py20
-rw-r--r--requests/packages/urllib3/util/connection.py1
-rw-r--r--requests/packages/urllib3/util/request.py1
-rw-r--r--requests/packages/urllib3/util/response.py3
-rw-r--r--requests/packages/urllib3/util/retry.py7
-rw-r--r--requests/packages/urllib3/util/ssl_.py39
-rw-r--r--requests/packages/urllib3/util/timeout.py8
-rw-r--r--requests/packages/urllib3/util/url.py5
-rw-r--r--requests/sessions.py12
-rw-r--r--requests/status_codes.py1
-rw-r--r--requests/utils.py47
32 files changed, 346 insertions, 137 deletions
diff --git a/requests/__init__.py b/requests/__init__.py
index 3d8188a..bd5b5b9 100644
--- a/requests/__init__.py
+++ b/requests/__init__.py
@@ -42,8 +42,8 @@ is at <http://python-requests.org>.
"""
__title__ = 'requests'
-__version__ = '2.8.1'
-__build__ = 0x020801
+__version__ = '2.9.1'
+__build__ = 0x020901
__author__ = 'Kenneth Reitz'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2015 Kenneth Reitz'
@@ -62,7 +62,8 @@ from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
RequestException, Timeout, URLRequired,
- TooManyRedirects, HTTPError, ConnectionError
+ TooManyRedirects, HTTPError, ConnectionError,
+ FileModeWarning,
)
# Set default logging handler to avoid "No handler found" warnings.
@@ -75,3 +76,8 @@ except ImportError:
pass
logging.getLogger(__name__).addHandler(NullHandler())
+
+import warnings
+
+# FileModeWarnings go off per the default.
+warnings.simplefilter('default', FileModeWarning, append=True)
diff --git a/requests/adapters.py b/requests/adapters.py
index 7682db0..6266d5b 100644
--- a/requests/adapters.py
+++ b/requests/adapters.py
@@ -8,6 +8,7 @@ This module contains the transport adapters that Requests uses to define
and maintain connections.
"""
+import os.path
import socket
from .models import Response
@@ -107,7 +108,7 @@ class HTTPAdapter(BaseAdapter):
def __setstate__(self, state):
# Can't handle by adding 'proxy_manager' to self.__attrs__ because
- # because self.poolmanager uses a lambda function, which isn't pickleable.
+ # self.poolmanager uses a lambda function, which isn't pickleable.
self.proxy_manager = {}
self.config = {}
@@ -185,10 +186,15 @@ class HTTPAdapter(BaseAdapter):
raise Exception("Could not find a suitable SSL CA certificate bundle.")
conn.cert_reqs = 'CERT_REQUIRED'
- conn.ca_certs = cert_loc
+
+ if not os.path.isdir(cert_loc):
+ conn.ca_certs = cert_loc
+ else:
+ conn.ca_cert_dir = cert_loc
else:
conn.cert_reqs = 'CERT_NONE'
conn.ca_certs = None
+ conn.ca_cert_dir = None
if cert:
if not isinstance(cert, basestring):
@@ -394,7 +400,15 @@ class HTTPAdapter(BaseAdapter):
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')
- r = low_conn.getresponse()
+ # Receive the response from the server
+ try:
+ # For Python 2.7+ versions, use buffering of HTTP
+ # responses
+ r = low_conn.getresponse(buffering=True)
+ except TypeError:
+ # For compatibility with Python 2.6 versions and back
+ r = low_conn.getresponse()
+
resp = HTTPResponse.from_httplib(
r,
pool=conn,
diff --git a/requests/api.py b/requests/api.py
index 72a777b..b21a1a4 100644
--- a/requests/api.py
+++ b/requests/api.py
@@ -33,7 +33,7 @@ def request(method, url, **kwargs):
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
- :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
+ :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``.
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
:return: :class:`Response <Response>` object
@@ -46,13 +46,11 @@ def request(method, url, **kwargs):
<Response [200]>
"""
- session = sessions.Session()
- response = session.request(method=method, url=url, **kwargs)
- # By explicitly closing the session, we avoid leaving sockets open which
- # can trigger a ResourceWarning in some cases, and look like a memory leak
- # in others.
- session.close()
- return response
+ # By using the 'with' statement we are sure the session is closed, thus we
+ # avoid leaving sockets open which can trigger a ResourceWarning in some
+ # cases, and look like a memory leak in others.
+ with sessions.Session() as session:
+ return session.request(method=method, url=url, **kwargs)
def get(url, params=None, **kwargs):
diff --git a/requests/auth.py b/requests/auth.py
index 8c4e847..2af55fb 100644
--- a/requests/auth.py
+++ b/requests/auth.py
@@ -136,7 +136,7 @@ class HTTPDigestAuth(AuthBase):
if _algorithm == 'MD5-SESS':
HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce))
- if qop is None:
+ if not qop:
respdig = KD(HA1, "%s:%s" % (nonce, HA2))
elif qop == 'auth' or 'auth' in qop.split(','):
noncebit = "%s:%s:%s:%s:%s" % (
diff --git a/requests/cookies.py b/requests/cookies.py
index 88b478c..b85fd2b 100644
--- a/requests/cookies.py
+++ b/requests/cookies.py
@@ -8,6 +8,7 @@ requests.utils imports from here, so be careful with imports.
import copy
import time
+import calendar
import collections
from .compat import cookielib, urlparse, urlunparse, Morsel
@@ -143,10 +144,13 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
"""
clearables = []
for cookie in cookiejar:
- if cookie.name == name:
- if domain is None or domain == cookie.domain:
- if path is None or path == cookie.path:
- clearables.append((cookie.domain, cookie.path, cookie.name))
+ if cookie.name != name:
+ continue
+ if domain is not None and domain != cookie.domain:
+ continue
+ if path is not None and path != cookie.path:
+ continue
+ clearables.append((cookie.domain, cookie.path, cookie.name))
for domain, path, name in clearables:
cookiejar.clear(domain, path, name)
@@ -365,7 +369,7 @@ def _copy_cookie_jar(jar):
return None
if hasattr(jar, 'copy'):
- # We're dealing with an instane of RequestsCookieJar
+ # We're dealing with an instance of RequestsCookieJar
return jar.copy()
# We're dealing with a generic CookieJar instance
new_jar = copy.copy(jar)
@@ -421,8 +425,9 @@ def morsel_to_cookie(morsel):
raise TypeError('max-age: %s must be integer' % morsel['max-age'])
elif morsel['expires']:
time_template = '%a, %d-%b-%Y %H:%M:%S GMT'
- expires = int(time.mktime(
- time.strptime(morsel['expires'], time_template)) - time.timezone)
+ expires = calendar.timegm(
+ time.strptime(morsel['expires'], time_template)
+ )
return create_cookie(
comment=morsel['comment'],
comment_url=bool(morsel['comment']),
diff --git a/requests/exceptions.py b/requests/exceptions.py
index 89135a8..ba0b910 100644
--- a/requests/exceptions.py
+++ b/requests/exceptions.py
@@ -97,3 +97,18 @@ class StreamConsumedError(RequestException, TypeError):
class RetryError(RequestException):
"""Custom retries logic failed"""
+
+
+# Warnings
+
+
+class RequestsWarning(Warning):
+ """Base warning for Requests."""
+ pass
+
+
+class FileModeWarning(RequestsWarning, DeprecationWarning):
+ """
+ A file was opened in text mode, but Requests determined its binary length.
+ """
+ pass
diff --git a/requests/models.py b/requests/models.py
index 2727bee..4bcbc54 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -324,7 +324,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
def prepare_url(self, url, params):
"""Prepares the given HTTP URL."""
#: Accept objects that have string representations.
- #: We're unable to blindy call unicode/str functions
+ #: We're unable to blindly call unicode/str functions
#: as this will include the bytestring indicator (b'')
#: on python 3.x.
#: https://github.com/kennethreitz/requests/pull/2238
@@ -385,6 +385,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
if isinstance(fragment, str):
fragment = fragment.encode('utf-8')
+ if isinstance(params, (str, bytes)):
+ params = to_native_string(params)
+
enc_params = self._encode_params(params)
if enc_params:
if query:
@@ -434,7 +437,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
if files:
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
- if length is not None:
+ if length:
self.headers['Content-Length'] = builtin_str(length)
else:
self.headers['Transfer-Encoding'] = 'chunked'
@@ -631,7 +634,7 @@ class Response(object):
@property
def is_permanent_redirect(self):
- """True if this Response one of the permanant versions of redirect"""
+ """True if this Response one of the permanent versions of redirect"""
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
@property
diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py
index 86bb71d..e43991a 100644
--- a/requests/packages/urllib3/__init__.py
+++ b/requests/packages/urllib3/__init__.py
@@ -2,10 +2,8 @@
urllib3 - Thread-safe connection pooling and re-using.
"""
-__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
-__license__ = 'MIT'
-__version__ = '1.12'
-
+from __future__ import absolute_import
+import warnings
from .connectionpool import (
HTTPConnectionPool,
@@ -32,8 +30,30 @@ except ImportError:
def emit(self, record):
pass
+__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
+__license__ = 'MIT'
+__version__ = '1.13.1'
+
+__all__ = (
+ 'HTTPConnectionPool',
+ 'HTTPSConnectionPool',
+ 'PoolManager',
+ 'ProxyManager',
+ 'HTTPResponse',
+ 'Retry',
+ 'Timeout',
+ 'add_stderr_logger',
+ 'connection_from_url',
+ 'disable_warnings',
+ 'encode_multipart_formdata',
+ 'get_host',
+ 'make_headers',
+ 'proxy_from_url',
+)
+
logging.getLogger(__name__).addHandler(NullHandler())
+
def add_stderr_logger(level=logging.DEBUG):
"""
Helper for quickly adding a StreamHandler to the logger. Useful for
@@ -55,7 +75,6 @@ def add_stderr_logger(level=logging.DEBUG):
del NullHandler
-import warnings
# SecurityWarning's always go off by default.
warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host
@@ -63,6 +82,9 @@ warnings.simplefilter('default', exceptions.SubjectAltNameWarning)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
append=True)
+# SNIMissingWarnings should go off only once.
+warnings.simplefilter('default', exceptions.SNIMissingWarning)
+
def disable_warnings(category=exceptions.HTTPWarning):
"""
diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py
index b68b9a5..67f3ce9 100644
--- a/requests/packages/urllib3/_collections.py
+++ b/requests/packages/urllib3/_collections.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from collections import Mapping, MutableMapping
try:
from threading import RLock
@@ -167,7 +168,7 @@ class HTTPHeaderDict(MutableMapping):
def __ne__(self, other):
return not self.__eq__(other)
- if not PY3: # Python 2
+ if not PY3: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
@@ -234,7 +235,7 @@ class HTTPHeaderDict(MutableMapping):
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
- "arguments ({} given)".format(len(args)))
+ "arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
@@ -304,7 +305,7 @@ class HTTPHeaderDict(MutableMapping):
return list(self.iteritems())
@classmethod
- def from_httplib(cls, message): # Python 2
+ def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py
index 3eab1e2..1e4cd41 100644
--- a/requests/packages/urllib3/connection.py
+++ b/requests/packages/urllib3/connection.py
@@ -1,4 +1,6 @@
+from __future__ import absolute_import
import datetime
+import os
import sys
import socket
from socket import error as SocketError, timeout as SocketTimeout
@@ -6,18 +8,13 @@ import warnings
from .packages import six
try: # Python 3
- from http.client import HTTPConnection as _HTTPConnection, HTTPException
+ from http.client import HTTPConnection as _HTTPConnection
+ from http.client import HTTPException # noqa: unused in this module
except ImportError:
- from httplib import HTTPConnection as _HTTPConnection, HTTPException
-
-
-class DummyConnection(object):
- "Used to detect a failed ConnectionCls import."
- pass
-
+ from httplib import HTTPConnection as _HTTPConnection
+ from httplib import HTTPException # noqa: unused in this module
try: # Compiled with SSL?
- HTTPSConnection = DummyConnection
import ssl
BaseSSLError = ssl.SSLError
except (ImportError, AttributeError): # Platform-specific: No SSL.
@@ -61,6 +58,11 @@ port_by_scheme = {
RECENT_DATE = datetime.date(2014, 1, 1)
+class DummyConnection(object):
+ """Used to detect a failed ConnectionCls import."""
+ pass
+
+
class HTTPConnection(_HTTPConnection, object):
"""
Based on httplib.HTTPConnection but provides an extra constructor
@@ -205,10 +207,10 @@ class VerifiedHTTPSConnection(HTTPSConnection):
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = cert_reqs
- self.ca_certs = ca_certs
- self.ca_cert_dir = ca_cert_dir
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
def connect(self):
# Add certificate verification
@@ -263,10 +265,19 @@ class VerifiedHTTPSConnection(HTTPSConnection):
'for details.)'.format(hostname)),
SubjectAltNameWarning
)
- match_hostname(cert, self.assert_hostname or hostname)
- self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
- or self.assert_fingerprint is not None)
+ # In case the hostname is an IPv6 address, strip the square
+ # brackets from it before using it to validate. This is because
+ # a certificate with an IPv6 address in it won't have square
+ # brackets around that address. Sadly, match_hostname won't do this
+ # for us: it expects the plain host part without any extra work
+ # that might have been done to make it palatable to httplib.
+ asserted_hostname = self.assert_hostname or hostname
+ asserted_hostname = asserted_hostname.strip('[]')
+ match_hostname(cert, asserted_hostname)
+
+ self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or
+ self.assert_fingerprint is not None)
if ssl:
diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py
index b38ac68..995b416 100644
--- a/requests/packages/urllib3/connectionpool.py
+++ b/requests/packages/urllib3/connectionpool.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import errno
import logging
import sys
@@ -10,7 +11,8 @@ try: # Python 3
from queue import LifoQueue, Empty, Full
except ImportError:
from Queue import LifoQueue, Empty, Full
- import Queue as _ # Platform-specific: Windows
+ # Queue is imported for side effects on MS Windows
+ import Queue as _unused_module_Queue # noqa: unused
from .exceptions import (
@@ -22,7 +24,6 @@ from .exceptions import (
LocationValueError,
MaxRetryError,
ProxyError,
- ConnectTimeoutError,
ReadTimeoutError,
SSLError,
TimeoutError,
@@ -35,7 +36,7 @@ from .connection import (
port_by_scheme,
DummyConnection,
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
- HTTPException, BaseSSLError, ConnectionError
+ HTTPException, BaseSSLError,
)
from .request import RequestMethods
from .response import HTTPResponse
@@ -54,7 +55,7 @@ log = logging.getLogger(__name__)
_Default = object()
-## Pool objects
+# Pool objects
class ConnectionPool(object):
"""
Base class for all connection pools, such as
@@ -68,8 +69,7 @@ class ConnectionPool(object):
if not host:
raise LocationValueError("No host specified.")
- # httplib doesn't like it when we include brackets in ipv6 addresses
- self.host = host.strip('[]')
+ self.host = host
self.port = port
def __str__(self):
@@ -645,22 +645,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return response
log.info("Redirecting %s -> %s" % (url, redirect_location))
- return self.urlopen(method, redirect_location, body, headers,
- retries=retries, redirect=redirect,
- assert_same_host=assert_same_host,
- timeout=timeout, pool_timeout=pool_timeout,
- release_conn=release_conn, **response_kw)
+ return self.urlopen(
+ method, redirect_location, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
# Check if we should retry the HTTP response.
if retries.is_forced_retry(method, status_code=response.status):
retries = retries.increment(method, url, response=response, _pool=self)
retries.sleep()
log.info("Forced retry: %s" % url)
- return self.urlopen(method, url, body, headers,
- retries=retries, redirect=redirect,
- assert_same_host=assert_same_host,
- timeout=timeout, pool_timeout=pool_timeout,
- release_conn=release_conn, **response_kw)
+ return self.urlopen(
+ method, url, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
return response
diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py
index ed9d8b8..884cdb2 100644
--- a/requests/packages/urllib3/contrib/appengine.py
+++ b/requests/packages/urllib3/contrib/appengine.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import logging
import os
import warnings
@@ -60,7 +61,7 @@ class AppEngineManager(RequestMethods):
raise AppEnginePlatformError(
"URLFetch is not available in this environment.")
- if is_prod_appengine_v2():
+ if is_prod_appengine_mvms():
raise AppEnginePlatformError(
"Use normal urllib3.PoolManager instead of AppEngineManager"
"on Managed VMs, as using URLFetch is not necessary in "
@@ -108,14 +109,14 @@ class AppEngineManager(RequestMethods):
raise TimeoutError(self, e)
except urlfetch.InvalidURLError as e:
- if 'too large' in e.message:
+ if 'too large' in str(e):
raise AppEnginePlatformError(
"URLFetch request too large, URLFetch only "
"supports requests up to 10mb in size.", e)
raise ProtocolError(e)
except urlfetch.DownloadError as e:
- if 'Too many redirects' in e.message:
+ if 'Too many redirects' in str(e):
raise MaxRetryError(self, url, reason=e)
raise ProtocolError(e)
@@ -155,7 +156,7 @@ class AppEngineManager(RequestMethods):
def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw):
- if is_prod_appengine_v1():
+ if is_prod_appengine():
# Production GAE handles deflate encoding automatically, but does
# not remove the encoding header.
content_encoding = urlfetch_resp.headers.get('content-encoding')
@@ -176,7 +177,7 @@ class AppEngineManager(RequestMethods):
if timeout is Timeout.DEFAULT_TIMEOUT:
return 5 # 5s is the default timeout for URLFetch.
if isinstance(timeout, Timeout):
- if not timeout.read is timeout.connect:
+ if timeout.read is not timeout.connect:
warnings.warn(
"URLFetch does not support granular timeout settings, "
"reverting to total timeout.", AppEnginePlatformWarning)
@@ -199,12 +200,12 @@ class AppEngineManager(RequestMethods):
def is_appengine():
return (is_local_appengine() or
- is_prod_appengine_v1() or
- is_prod_appengine_v2())
+ is_prod_appengine() or
+ is_prod_appengine_mvms())
def is_appengine_sandbox():
- return is_appengine() and not is_prod_appengine_v2()
+ return is_appengine() and not is_prod_appengine_mvms()
def is_local_appengine():
@@ -212,11 +213,11 @@ def is_local_appengine():
'Development/' in os.environ['SERVER_SOFTWARE'])
-def is_prod_appengine_v1():
+def is_prod_appengine():
return ('APPENGINE_RUNTIME' in os.environ and
'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
- not is_prod_appengine_v2())
+ not is_prod_appengine_mvms())
-def is_prod_appengine_v2():
+def is_prod_appengine_mvms():
return os.environ.get('GAE_VM', False) == 'true'
diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py
index c6b266f..c136a23 100644
--- a/requests/packages/urllib3/contrib/ntlmpool.py
+++ b/requests/packages/urllib3/contrib/ntlmpool.py
@@ -3,6 +3,7 @@ NTLM authenticating pool, contributed by erikcederstran
Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
"""
+from __future__ import absolute_import
try:
from http.client import HTTPSConnection
diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py
index c20ae46..5996153 100644
--- a/requests/packages/urllib3/contrib/pyopenssl.py
+++ b/requests/packages/urllib3/contrib/pyopenssl.py
@@ -43,6 +43,7 @@ Module Variables
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
'''
+from __future__ import absolute_import
try:
from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
@@ -53,7 +54,7 @@ except SyntaxError as e:
import OpenSSL.SSL
from pyasn1.codec.der import decoder as der_decoder
from pyasn1.type import univ, constraint
-from socket import _fileobject, timeout
+from socket import _fileobject, timeout, error as SocketError
import ssl
import select
@@ -71,6 +72,12 @@ _openssl_versions = {
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}
+if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
+
+if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
+
try:
_openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
except AttributeError:
@@ -79,8 +86,8 @@ except AttributeError:
_openssl_verify = {
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
- ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
- + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+ ssl.CERT_REQUIRED:
+ OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS
@@ -88,12 +95,6 @@ DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS
# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384
-try:
- _ = memoryview
- has_memoryview = True
-except NameError:
- has_memoryview = False
-
orig_util_HAS_SNI = util.HAS_SNI
orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket
@@ -112,7 +113,7 @@ def extract_from_urllib3():
util.HAS_SNI = orig_util_HAS_SNI
-### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
+# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
class SubjectAltName(BaseSubjectAltName):
'''ASN.1 implementation for subjectAltNames support'''
@@ -123,7 +124,7 @@ class SubjectAltName(BaseSubjectAltName):
constraint.ValueSizeConstraint(1, 1024)
-### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
+# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
def get_subj_alt_name(peer_cert):
# Search through extensions
dns_name = []
@@ -181,7 +182,7 @@ class WrappedSocket(object):
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
return b''
else:
- raise
+ raise SocketError(e)
except OpenSSL.SSL.ZeroReturnError as e:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b''
@@ -212,12 +213,9 @@ class WrappedSocket(object):
continue
def sendall(self, data):
- if has_memoryview and not isinstance(data, memoryview):
- data = memoryview(data)
-
total_sent = 0
while total_sent < len(data):
- sent = self._send_until_done(data[total_sent:total_sent+SSL_WRITE_BLOCKSIZE])
+ sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
total_sent += sent
def shutdown(self):
@@ -226,7 +224,10 @@ class WrappedSocket(object):
def close(self):
if self._makefile_refs < 1:
- return self.connection.close()
+ try:
+ return self.connection.close()
+ except OpenSSL.SSL.Error:
+ return
else:
self._makefile_refs -= 1
diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py
index 9607d65..8e07eb6 100644
--- a/requests/packages/urllib3/exceptions.py
+++ b/requests/packages/urllib3/exceptions.py
@@ -1,16 +1,17 @@
+from __future__ import absolute_import
+# Base Exceptions
-## Base Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
+
class HTTPWarning(Warning):
"Base warning used by this module."
pass
-
class PoolError(HTTPError):
"Base exception for errors caused within a pool."
def __init__(self, pool, message):
@@ -57,7 +58,7 @@ class ProtocolError(HTTPError):
ConnectionError = ProtocolError
-## Leaf Exceptions
+# Leaf Exceptions
class MaxRetryError(RequestError):
"""Raised when the maximum number of retries is exceeded.
@@ -112,10 +113,12 @@ class ConnectTimeoutError(TimeoutError):
"Raised when a socket timeout occurs while connecting to a server"
pass
+
class NewConnectionError(ConnectTimeoutError, PoolError):
"Raised when we fail to establish a new connection. Usually ECONNREFUSED."
pass
+
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
@@ -172,6 +175,11 @@ class InsecurePlatformWarning(SecurityWarning):
pass
+class SNIMissingWarning(HTTPWarning):
+ "Warned when making a HTTPS request without SNI available."
+ pass
+
+
class ResponseNotChunked(ProtocolError, ValueError):
"Response needs to be chunked in order to read it as chunks."
pass
diff --git a/requests/packages/urllib3/fields.py b/requests/packages/urllib3/fields.py
index c853f8d..c7d4811 100644
--- a/requests/packages/urllib3/fields.py
+++ b/requests/packages/urllib3/fields.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import email.utils
import mimetypes
diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py
index 0fbf488..97a2843 100644
--- a/requests/packages/urllib3/filepost.py
+++ b/requests/packages/urllib3/filepost.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import codecs
from uuid import uuid4
diff --git a/requests/packages/urllib3/packages/__init__.py b/requests/packages/urllib3/packages/__init__.py
index 37e8351..170e974 100644
--- a/requests/packages/urllib3/packages/__init__.py
+++ b/requests/packages/urllib3/packages/__init__.py
@@ -2,3 +2,4 @@ from __future__ import absolute_import
from . import ssl_match_hostname
+__all__ = ('ssl_match_hostname', )
diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py
index 76b6a12..f13e673 100644
--- a/requests/packages/urllib3/poolmanager.py
+++ b/requests/packages/urllib3/poolmanager.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import logging
try: # Python 3
@@ -25,7 +26,7 @@ pool_classes_by_scheme = {
log = logging.getLogger(__name__)
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
- 'ssl_version')
+ 'ssl_version', 'ca_cert_dir')
class PoolManager(RequestMethods):
diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py
index a1a12bc..d5aa62d 100644
--- a/requests/packages/urllib3/request.py
+++ b/requests/packages/urllib3/request.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
try:
from urllib.parse import urlencode
except ImportError:
@@ -133,7 +134,8 @@ class RequestMethods(object):
if fields:
if 'body' in urlopen_kw:
- raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.')
+ raise TypeError(
+ "request got values for both 'fields' and 'body', can only specify one.")
if encode_multipart:
body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py
index 788eb6c..8f2a1b5 100644
--- a/requests/packages/urllib3/response.py
+++ b/requests/packages/urllib3/response.py
@@ -1,7 +1,9 @@
+from __future__ import absolute_import
from contextlib import contextmanager
import zlib
import io
from socket import timeout as SocketTimeout
+from socket import error as SocketError
from ._collections import HTTPHeaderDict
from .exceptions import (
@@ -130,8 +132,8 @@ class HTTPResponse(io.IOBase):
if "chunked" in encodings:
self.chunked = True
- # We certainly don't want to preload content when the response is chunked.
- if not self.chunked and preload_content and not self._body:
+ # If requested, preload the body.
+ if preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
@@ -194,12 +196,22 @@ class HTTPResponse(io.IOBase):
"Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding, e)
- if flush_decoder and decode_content and self._decoder:
- buf = self._decoder.decompress(binary_type())
- data += buf + self._decoder.flush()
+ if flush_decoder and decode_content:
+ data += self._flush_decoder()
return data
+ def _flush_decoder(self):
+ """
+ Flushes the decoder. Should only be called if the decoder is actually
+ being used.
+ """
+ if self._decoder:
+ buf = self._decoder.decompress(b'')
+ return buf + self._decoder.flush()
+
+ return b''
+
@contextmanager
def _error_catcher(self):
"""
@@ -227,15 +239,22 @@ class HTTPResponse(io.IOBase):
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
- except HTTPException as e:
+ except (HTTPException, SocketError) as e:
# This includes IncompleteRead.
raise ProtocolError('Connection broken: %r' % e, e)
+
except Exception:
# The response may not be closed but we're not going to use it anymore
# so close it now to ensure that the connection is released back to the pool.
if self._original_response and not self._original_response.isclosed():
self._original_response.close()
+ # Closing the response may not actually be sufficient to close
+ # everything, so if we have a hold of the connection close that
+ # too.
+ if self._connection is not None:
+ self._connection.close()
+
raise
finally:
if self._original_response and self._original_response.isclosed():
@@ -301,7 +320,6 @@ class HTTPResponse(io.IOBase):
return data
-
def stream(self, amt=2**16, decode_content=None):
"""
A generator wrapper for the read() method. A call will block until
@@ -340,9 +358,9 @@ class HTTPResponse(io.IOBase):
headers = r.msg
if not isinstance(headers, HTTPHeaderDict):
- if PY3: # Python 3
+ if PY3: # Python 3
headers = HTTPHeaderDict(headers.items())
- else: # Python 2
+ else: # Python 2
headers = HTTPHeaderDict.from_httplib(headers)
# HTTPResponse objects in Python 3 don't have a .strict attribute
@@ -454,7 +472,8 @@ class HTTPResponse(io.IOBase):
self._init_decoder()
# FIXME: Rewrite this method and make it a class with a better structured logic.
if not self.chunked:
- raise ResponseNotChunked("Response is not chunked. "
+ raise ResponseNotChunked(
+ "Response is not chunked. "
"Header 'transfer-encoding: chunked' is missing.")
# Don't bother reading the body of a HEAD request.
@@ -468,8 +487,18 @@ class HTTPResponse(io.IOBase):
if self.chunk_left == 0:
break
chunk = self._handle_chunk(amt)
- yield self._decode(chunk, decode_content=decode_content,
- flush_decoder=True)
+ decoded = self._decode(chunk, decode_content=decode_content,
+ flush_decoder=False)
+ if decoded:
+ yield decoded
+
+ if decode_content:
+ # On CPython and PyPy, we should never need to flush the
+ # decoder. However, on Jython we *might* need to, so
+ # lets defensively do it anyway.
+ decoded = self._flush_decoder()
+ if decoded: # Platform-specific: Jython.
+ yield decoded
# Chunk content ends with \r\n: discard it.
while True:
diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py
index 8becc81..c6c6243 100644
--- a/requests/packages/urllib3/util/__init__.py
+++ b/requests/packages/urllib3/util/__init__.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
from .request import make_headers
@@ -22,3 +23,22 @@ from .url import (
split_first,
Url,
)
+
+__all__ = (
+ 'HAS_SNI',
+ 'SSLContext',
+ 'Retry',
+ 'Timeout',
+ 'Url',
+ 'assert_fingerprint',
+ 'current_time',
+ 'is_connection_dropped',
+ 'is_fp_closed',
+ 'get_host',
+ 'parse_url',
+ 'make_headers',
+ 'resolve_cert_reqs',
+ 'resolve_ssl_version',
+ 'split_first',
+ 'ssl_wrap_socket',
+)
diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py
index 4f2f0f1..01a4812 100644
--- a/requests/packages/urllib3/util/connection.py
+++ b/requests/packages/urllib3/util/connection.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import socket
try:
from select import poll, POLLIN
diff --git a/requests/packages/urllib3/util/request.py b/requests/packages/urllib3/util/request.py
index bc64f6b..7377931 100644
--- a/requests/packages/urllib3/util/request.py
+++ b/requests/packages/urllib3/util/request.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from base64 import b64encode
from ..packages.six import b
diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py
index 2c1de15..bc72327 100644
--- a/requests/packages/urllib3/util/response.py
+++ b/requests/packages/urllib3/util/response.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from ..packages.six.moves import http_client as httplib
from ..exceptions import HeaderParsingError
@@ -44,7 +45,7 @@ def assert_header_parsing(headers):
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
- raise TypeError('expected httplib.Message, got {}.'.format(
+ raise TypeError('expected httplib.Message, got {0}.'.format(
type(headers)))
defects = getattr(headers, 'defects', None)
diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py
index 1fb1f23..03a0124 100644
--- a/requests/packages/urllib3/util/retry.py
+++ b/requests/packages/urllib3/util/retry.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import time
import logging
@@ -126,7 +127,7 @@ class Retry(object):
self.method_whitelist = method_whitelist
self.backoff_factor = backoff_factor
self.raise_on_redirect = raise_on_redirect
- self._observed_errors = _observed_errors # TODO: use .history instead?
+ self._observed_errors = _observed_errors # TODO: use .history instead?
def new(self, **kw):
params = dict(
@@ -206,7 +207,8 @@ class Retry(object):
return min(retry_counts) < 0
- def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None):
+ def increment(self, method=None, url=None, response=None, error=None,
+ _pool=None, _stacktrace=None):
""" Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
@@ -274,7 +276,6 @@ class Retry(object):
return new_retry
-
def __repr__(self):
return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
'read={self.read}, redirect={self.redirect})').format(
diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py
index 47b817e..67f8344 100644
--- a/requests/packages/urllib3/util/ssl_.py
+++ b/requests/packages/urllib3/util/ssl_.py
@@ -1,7 +1,12 @@
+from __future__ import absolute_import
+import errno
+import warnings
+import hmac
+
from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256
-from ..exceptions import SSLError, InsecurePlatformWarning
+from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
SSLContext = None
@@ -15,8 +20,23 @@ HASHFUNC_MAP = {
64: sha256,
}
-import errno
-import warnings
+
+def _const_compare_digest_backport(a, b):
+ """
+ Compare two digests of equal length in constant time.
+
+ The digests must be of type str/bytes.
+ Returns True if the digests match, and False otherwise.
+ """
+ result = abs(len(a) - len(b))
+ for l, r in zip(bytearray(a), bytearray(b)):
+ result |= l ^ r
+ return result == 0
+
+
+_const_compare_digest = getattr(hmac, 'compare_digest',
+ _const_compare_digest_backport)
+
try: # Test for SSL features
import ssl
@@ -134,7 +154,7 @@ def assert_fingerprint(cert, fingerprint):
cert_digest = hashfunc(cert).digest()
- if cert_digest != fingerprint_bytes:
+ if not _const_compare_digest(cert_digest, fingerprint_bytes):
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
.format(fingerprint, hexlify(cert_digest)))
@@ -283,4 +303,15 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
context.load_cert_chain(certfile, keyfile)
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
return context.wrap_socket(sock, server_hostname=server_hostname)
+
+ warnings.warn(
+ 'An HTTPS request has been made, but the SNI (Subject Name '
+ 'Indication) extension to TLS is not available on this platform. '
+ 'This may cause the server to present an incorrect TLS '
+ 'certificate, which can cause validation failures. For more '
+ 'information, see '
+ 'https://urllib3.readthedocs.org/en/latest/security.html'
+ '#snimissingwarning.',
+ SNIMissingWarning
+ )
return context.wrap_socket(sock)
diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py
index ea7027f..ff62f47 100644
--- a/requests/packages/urllib3/util/timeout.py
+++ b/requests/packages/urllib3/util/timeout.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT
@@ -9,6 +10,7 @@ from ..exceptions import TimeoutStateError
# urllib3
_Default = object()
+
def current_time():
"""
Retrieve the current time. This function is mocked out in unit testing.
@@ -226,9 +228,9 @@ class Timeout(object):
has not yet been called on this object.
"""
if (self.total is not None and
- self.total is not self.DEFAULT_TIMEOUT and
- self._read is not None and
- self._read is not self.DEFAULT_TIMEOUT):
+ self.total is not self.DEFAULT_TIMEOUT and
+ self._read is not None and
+ self._read is not self.DEFAULT_TIMEOUT):
# In case the connect timeout has not yet been established.
if self._start_connect is None:
return self._read
diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py
index e58050c..e996204 100644
--- a/requests/packages/urllib3/util/url.py
+++ b/requests/packages/urllib3/util/url.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from collections import namedtuple
from ..exceptions import LocationParseError
@@ -85,6 +86,7 @@ class Url(namedtuple('Url', url_attrs)):
def __str__(self):
return self.url
+
def split_first(s, delims):
"""
Given a string and an iterable of delimiters, split on the first found
@@ -115,7 +117,7 @@ def split_first(s, delims):
if min_idx is None or min_idx < 0:
return s, '', None
- return s[:min_idx], s[min_idx+1:], min_delim
+ return s[:min_idx], s[min_idx + 1:], min_delim
def parse_url(url):
@@ -206,6 +208,7 @@ def parse_url(url):
return Url(scheme, auth, host, port, path, query, fragment)
+
def get_host(url):
"""
Deprecated. Use :func:`.parse_url` instead.
diff --git a/requests/sessions.py b/requests/sessions.py
index ad63902..9eaa36a 100644
--- a/requests/sessions.py
+++ b/requests/sessions.py
@@ -273,13 +273,13 @@ class Session(SessionRedirectMixin):
>>> import requests
>>> s = requests.Session()
>>> s.get('http://httpbin.org/get')
- 200
+ <Response [200]>
Or as a context manager::
>>> with requests.Session() as s:
>>> s.get('http://httpbin.org/get')
- 200
+ <Response [200]>
"""
__attrs__ = [
@@ -325,7 +325,7 @@ class Session(SessionRedirectMixin):
#: limit, a :class:`TooManyRedirects` exception is raised.
self.max_redirects = DEFAULT_REDIRECT_LIMIT
- #: Trust environement settings for proxy configuration, default
+ #: Trust environment settings for proxy configuration, default
#: authentication and similar.
self.trust_env = True
@@ -433,8 +433,8 @@ class Session(SessionRedirectMixin):
hostname to the URL of the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
- :param verify: (optional) if ``True``, the SSL cert will be verified.
- A CA_BUNDLE path can also be provided.
+ :param verify: (optional) whether the SSL cert will be verified.
+ A CA_BUNDLE path can also be provided. Defaults to ``True``.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
"""
@@ -634,7 +634,7 @@ class Session(SessionRedirectMixin):
'cert': cert}
def get_adapter(self, url):
- """Returns the appropriate connnection adapter for the given URL."""
+ """Returns the appropriate connection adapter for the given URL."""
for (prefix, adapter) in self.adapters.items():
if url.lower().startswith(prefix):
diff --git a/requests/status_codes.py b/requests/status_codes.py
index 1db7fc0..a852574 100644
--- a/requests/status_codes.py
+++ b/requests/status_codes.py
@@ -78,6 +78,7 @@ _codes = {
507: ('insufficient_storage',),
509: ('bandwidth_limit_exceeded', 'bandwidth'),
510: ('not_extended',),
+ 511: ('network_authentication_required', 'network_auth', 'network_authentication'),
}
codes = LookupDict(name='status_codes')
diff --git a/requests/utils.py b/requests/utils.py
index 4a8c6d7..c5c3fd0 100644
--- a/requests/utils.py
+++ b/requests/utils.py
@@ -29,7 +29,7 @@ from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2,
basestring)
from .cookies import RequestsCookieJar, cookiejar_from_dict
from .structures import CaseInsensitiveDict
-from .exceptions import InvalidURL
+from .exceptions import InvalidURL, FileModeWarning
_hush_pyflakes = (RequestsCookieJar,)
@@ -48,23 +48,44 @@ def dict_to_sequence(d):
def super_len(o):
+ total_length = 0
+ current_position = 0
+
if hasattr(o, '__len__'):
- return len(o)
+ total_length = len(o)
+
+ elif hasattr(o, 'len'):
+ total_length = o.len
- if hasattr(o, 'len'):
- return o.len
+ elif hasattr(o, 'getvalue'):
+ # e.g. BytesIO, cStringIO.StringIO
+ total_length = len(o.getvalue())
- if hasattr(o, 'fileno'):
+ elif hasattr(o, 'fileno'):
try:
fileno = o.fileno()
except io.UnsupportedOperation:
pass
else:
- return os.fstat(fileno).st_size
+ total_length = os.fstat(fileno).st_size
- if hasattr(o, 'getvalue'):
- # e.g. BytesIO, cStringIO.StringIO
- return len(o.getvalue())
+ # Having used fstat to determine the file length, we need to
+ # confirm that this file was opened up in binary mode.
+ if 'b' not in o.mode:
+ warnings.warn((
+ "Requests has determined the content-length for this "
+ "request using the binary size of the file: however, the "
+ "file has been opened in text mode (i.e. without the 'b' "
+ "flag in the mode). This may lead to an incorrect "
+ "content-length. In Requests 3.0, support will be removed "
+ "for files in text mode."),
+ FileModeWarning
+ )
+
+ if hasattr(o, 'tell'):
+ current_position = o.tell()
+
+ return max(0, total_length - current_position)
def get_netrc_auth(url, raise_errors=False):
@@ -94,8 +115,12 @@ def get_netrc_auth(url, raise_errors=False):
ri = urlparse(url)
- # Strip port numbers from netloc
- host = ri.netloc.split(':')[0]
+ # Strip port numbers from netloc. This weird `if...encode`` dance is
+ # used for Python 3.2, which doesn't support unicode literals.
+ splitstr = b':'
+ if isinstance(url, str):
+ splitstr = splitstr.decode('ascii')
+ host = ri.netloc.split(splitstr)[0]
try:
_netrc = netrc(netrc_path).authenticators(host)