aboutsummaryrefslogtreecommitdiff
path: root/urllib3
diff options
context:
space:
mode:
authorChristopher Baines <mail@cbaines.net>2015-12-22 14:09:50 +0000
committerChristopher Baines <mail@cbaines.net>2015-12-22 14:09:50 +0000
commitdff63335f212d32d7c1a4bb5276f2d31f5995ea1 (patch)
tree4b32f426e18fec38cbfa9468d0dfc0cbf2279363 /urllib3
parent2b3d330a120a16e97cecd5163b5d454dcfe38a2b (diff)
downloadpython-urllib3-dff63335f212d32d7c1a4bb5276f2d31f5995ea1.tar
python-urllib3-dff63335f212d32d7c1a4bb5276f2d31f5995ea1.tar.gz
Import python-urllib3_1.13.1.orig.tar.gzupstream/1.13.1upstream
Diffstat (limited to 'urllib3')
-rw-r--r--urllib3/__init__.py32
-rw-r--r--urllib3/_collections.py7
-rw-r--r--urllib3/connection.py39
-rw-r--r--urllib3/connectionpool.py34
-rw-r--r--urllib3/contrib/appengine.py23
-rw-r--r--urllib3/contrib/ntlmpool.py1
-rw-r--r--urllib3/contrib/pyopenssl.py35
-rw-r--r--urllib3/exceptions.py14
-rw-r--r--urllib3/fields.py1
-rw-r--r--urllib3/filepost.py1
-rw-r--r--urllib3/packages/__init__.py1
-rw-r--r--urllib3/poolmanager.py3
-rw-r--r--urllib3/request.py4
-rw-r--r--urllib3/response.py53
-rw-r--r--urllib3/util/__init__.py20
-rw-r--r--urllib3/util/connection.py1
-rw-r--r--urllib3/util/request.py1
-rw-r--r--urllib3/util/response.py3
-rw-r--r--urllib3/util/retry.py7
-rw-r--r--urllib3/util/ssl_.py39
-rw-r--r--urllib3/util/timeout.py8
-rw-r--r--urllib3/util/url.py5
22 files changed, 237 insertions, 95 deletions
diff --git a/urllib3/__init__.py b/urllib3/__init__.py
index 86bb71d..e43991a 100644
--- a/urllib3/__init__.py
+++ b/urllib3/__init__.py
@@ -2,10 +2,8 @@
urllib3 - Thread-safe connection pooling and re-using.
"""
-__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
-__license__ = 'MIT'
-__version__ = '1.12'
-
+from __future__ import absolute_import
+import warnings
from .connectionpool import (
HTTPConnectionPool,
@@ -32,8 +30,30 @@ except ImportError:
def emit(self, record):
pass
+__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
+__license__ = 'MIT'
+__version__ = '1.13.1'
+
+__all__ = (
+ 'HTTPConnectionPool',
+ 'HTTPSConnectionPool',
+ 'PoolManager',
+ 'ProxyManager',
+ 'HTTPResponse',
+ 'Retry',
+ 'Timeout',
+ 'add_stderr_logger',
+ 'connection_from_url',
+ 'disable_warnings',
+ 'encode_multipart_formdata',
+ 'get_host',
+ 'make_headers',
+ 'proxy_from_url',
+)
+
logging.getLogger(__name__).addHandler(NullHandler())
+
def add_stderr_logger(level=logging.DEBUG):
"""
Helper for quickly adding a StreamHandler to the logger. Useful for
@@ -55,7 +75,6 @@ def add_stderr_logger(level=logging.DEBUG):
del NullHandler
-import warnings
# SecurityWarning's always go off by default.
warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host
@@ -63,6 +82,9 @@ warnings.simplefilter('default', exceptions.SubjectAltNameWarning)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
append=True)
+# SNIMissingWarnings should go off only once.
+warnings.simplefilter('default', exceptions.SNIMissingWarning)
+
def disable_warnings(category=exceptions.HTTPWarning):
"""
diff --git a/urllib3/_collections.py b/urllib3/_collections.py
index b68b9a5..67f3ce9 100644
--- a/urllib3/_collections.py
+++ b/urllib3/_collections.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from collections import Mapping, MutableMapping
try:
from threading import RLock
@@ -167,7 +168,7 @@ class HTTPHeaderDict(MutableMapping):
def __ne__(self, other):
return not self.__eq__(other)
- if not PY3: # Python 2
+ if not PY3: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
@@ -234,7 +235,7 @@ class HTTPHeaderDict(MutableMapping):
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
- "arguments ({} given)".format(len(args)))
+ "arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
@@ -304,7 +305,7 @@ class HTTPHeaderDict(MutableMapping):
return list(self.iteritems())
@classmethod
- def from_httplib(cls, message): # Python 2
+ def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
diff --git a/urllib3/connection.py b/urllib3/connection.py
index 3eab1e2..1e4cd41 100644
--- a/urllib3/connection.py
+++ b/urllib3/connection.py
@@ -1,4 +1,6 @@
+from __future__ import absolute_import
import datetime
+import os
import sys
import socket
from socket import error as SocketError, timeout as SocketTimeout
@@ -6,18 +8,13 @@ import warnings
from .packages import six
try: # Python 3
- from http.client import HTTPConnection as _HTTPConnection, HTTPException
+ from http.client import HTTPConnection as _HTTPConnection
+ from http.client import HTTPException # noqa: unused in this module
except ImportError:
- from httplib import HTTPConnection as _HTTPConnection, HTTPException
-
-
-class DummyConnection(object):
- "Used to detect a failed ConnectionCls import."
- pass
-
+ from httplib import HTTPConnection as _HTTPConnection
+ from httplib import HTTPException # noqa: unused in this module
try: # Compiled with SSL?
- HTTPSConnection = DummyConnection
import ssl
BaseSSLError = ssl.SSLError
except (ImportError, AttributeError): # Platform-specific: No SSL.
@@ -61,6 +58,11 @@ port_by_scheme = {
RECENT_DATE = datetime.date(2014, 1, 1)
+class DummyConnection(object):
+ """Used to detect a failed ConnectionCls import."""
+ pass
+
+
class HTTPConnection(_HTTPConnection, object):
"""
Based on httplib.HTTPConnection but provides an extra constructor
@@ -205,10 +207,10 @@ class VerifiedHTTPSConnection(HTTPSConnection):
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = cert_reqs
- self.ca_certs = ca_certs
- self.ca_cert_dir = ca_cert_dir
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
def connect(self):
# Add certificate verification
@@ -263,10 +265,19 @@ class VerifiedHTTPSConnection(HTTPSConnection):
'for details.)'.format(hostname)),
SubjectAltNameWarning
)
- match_hostname(cert, self.assert_hostname or hostname)
- self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
- or self.assert_fingerprint is not None)
+ # In case the hostname is an IPv6 address, strip the square
+ # brackets from it before using it to validate. This is because
+ # a certificate with an IPv6 address in it won't have square
+ # brackets around that address. Sadly, match_hostname won't do this
+ # for us: it expects the plain host part without any extra work
+ # that might have been done to make it palatable to httplib.
+ asserted_hostname = self.assert_hostname or hostname
+ asserted_hostname = asserted_hostname.strip('[]')
+ match_hostname(cert, asserted_hostname)
+
+ self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or
+ self.assert_fingerprint is not None)
if ssl:
diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py
index b38ac68..995b416 100644
--- a/urllib3/connectionpool.py
+++ b/urllib3/connectionpool.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import errno
import logging
import sys
@@ -10,7 +11,8 @@ try: # Python 3
from queue import LifoQueue, Empty, Full
except ImportError:
from Queue import LifoQueue, Empty, Full
- import Queue as _ # Platform-specific: Windows
+ # Queue is imported for side effects on MS Windows
+ import Queue as _unused_module_Queue # noqa: unused
from .exceptions import (
@@ -22,7 +24,6 @@ from .exceptions import (
LocationValueError,
MaxRetryError,
ProxyError,
- ConnectTimeoutError,
ReadTimeoutError,
SSLError,
TimeoutError,
@@ -35,7 +36,7 @@ from .connection import (
port_by_scheme,
DummyConnection,
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
- HTTPException, BaseSSLError, ConnectionError
+ HTTPException, BaseSSLError,
)
from .request import RequestMethods
from .response import HTTPResponse
@@ -54,7 +55,7 @@ log = logging.getLogger(__name__)
_Default = object()
-## Pool objects
+# Pool objects
class ConnectionPool(object):
"""
Base class for all connection pools, such as
@@ -68,8 +69,7 @@ class ConnectionPool(object):
if not host:
raise LocationValueError("No host specified.")
- # httplib doesn't like it when we include brackets in ipv6 addresses
- self.host = host.strip('[]')
+ self.host = host
self.port = port
def __str__(self):
@@ -645,22 +645,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return response
log.info("Redirecting %s -> %s" % (url, redirect_location))
- return self.urlopen(method, redirect_location, body, headers,
- retries=retries, redirect=redirect,
- assert_same_host=assert_same_host,
- timeout=timeout, pool_timeout=pool_timeout,
- release_conn=release_conn, **response_kw)
+ return self.urlopen(
+ method, redirect_location, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
# Check if we should retry the HTTP response.
if retries.is_forced_retry(method, status_code=response.status):
retries = retries.increment(method, url, response=response, _pool=self)
retries.sleep()
log.info("Forced retry: %s" % url)
- return self.urlopen(method, url, body, headers,
- retries=retries, redirect=redirect,
- assert_same_host=assert_same_host,
- timeout=timeout, pool_timeout=pool_timeout,
- release_conn=release_conn, **response_kw)
+ return self.urlopen(
+ method, url, body, headers,
+ retries=retries, redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
return response
diff --git a/urllib3/contrib/appengine.py b/urllib3/contrib/appengine.py
index ed9d8b8..884cdb2 100644
--- a/urllib3/contrib/appengine.py
+++ b/urllib3/contrib/appengine.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import logging
import os
import warnings
@@ -60,7 +61,7 @@ class AppEngineManager(RequestMethods):
raise AppEnginePlatformError(
"URLFetch is not available in this environment.")
- if is_prod_appengine_v2():
+ if is_prod_appengine_mvms():
raise AppEnginePlatformError(
"Use normal urllib3.PoolManager instead of AppEngineManager"
"on Managed VMs, as using URLFetch is not necessary in "
@@ -108,14 +109,14 @@ class AppEngineManager(RequestMethods):
raise TimeoutError(self, e)
except urlfetch.InvalidURLError as e:
- if 'too large' in e.message:
+ if 'too large' in str(e):
raise AppEnginePlatformError(
"URLFetch request too large, URLFetch only "
"supports requests up to 10mb in size.", e)
raise ProtocolError(e)
except urlfetch.DownloadError as e:
- if 'Too many redirects' in e.message:
+ if 'Too many redirects' in str(e):
raise MaxRetryError(self, url, reason=e)
raise ProtocolError(e)
@@ -155,7 +156,7 @@ class AppEngineManager(RequestMethods):
def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw):
- if is_prod_appengine_v1():
+ if is_prod_appengine():
# Production GAE handles deflate encoding automatically, but does
# not remove the encoding header.
content_encoding = urlfetch_resp.headers.get('content-encoding')
@@ -176,7 +177,7 @@ class AppEngineManager(RequestMethods):
if timeout is Timeout.DEFAULT_TIMEOUT:
return 5 # 5s is the default timeout for URLFetch.
if isinstance(timeout, Timeout):
- if not timeout.read is timeout.connect:
+ if timeout.read is not timeout.connect:
warnings.warn(
"URLFetch does not support granular timeout settings, "
"reverting to total timeout.", AppEnginePlatformWarning)
@@ -199,12 +200,12 @@ class AppEngineManager(RequestMethods):
def is_appengine():
return (is_local_appengine() or
- is_prod_appengine_v1() or
- is_prod_appengine_v2())
+ is_prod_appengine() or
+ is_prod_appengine_mvms())
def is_appengine_sandbox():
- return is_appengine() and not is_prod_appengine_v2()
+ return is_appengine() and not is_prod_appengine_mvms()
def is_local_appengine():
@@ -212,11 +213,11 @@ def is_local_appengine():
'Development/' in os.environ['SERVER_SOFTWARE'])
-def is_prod_appengine_v1():
+def is_prod_appengine():
return ('APPENGINE_RUNTIME' in os.environ and
'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
- not is_prod_appengine_v2())
+ not is_prod_appengine_mvms())
-def is_prod_appengine_v2():
+def is_prod_appengine_mvms():
return os.environ.get('GAE_VM', False) == 'true'
diff --git a/urllib3/contrib/ntlmpool.py b/urllib3/contrib/ntlmpool.py
index c6b266f..c136a23 100644
--- a/urllib3/contrib/ntlmpool.py
+++ b/urllib3/contrib/ntlmpool.py
@@ -3,6 +3,7 @@ NTLM authenticating pool, contributed by erikcederstran
Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
"""
+from __future__ import absolute_import
try:
from http.client import HTTPSConnection
diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py
index c20ae46..5996153 100644
--- a/urllib3/contrib/pyopenssl.py
+++ b/urllib3/contrib/pyopenssl.py
@@ -43,6 +43,7 @@ Module Variables
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
'''
+from __future__ import absolute_import
try:
from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
@@ -53,7 +54,7 @@ except SyntaxError as e:
import OpenSSL.SSL
from pyasn1.codec.der import decoder as der_decoder
from pyasn1.type import univ, constraint
-from socket import _fileobject, timeout
+from socket import _fileobject, timeout, error as SocketError
import ssl
import select
@@ -71,6 +72,12 @@ _openssl_versions = {
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}
+if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
+
+if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
+
try:
_openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
except AttributeError:
@@ -79,8 +86,8 @@ except AttributeError:
_openssl_verify = {
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
- ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
- + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+ ssl.CERT_REQUIRED:
+ OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS
@@ -88,12 +95,6 @@ DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS
# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384
-try:
- _ = memoryview
- has_memoryview = True
-except NameError:
- has_memoryview = False
-
orig_util_HAS_SNI = util.HAS_SNI
orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket
@@ -112,7 +113,7 @@ def extract_from_urllib3():
util.HAS_SNI = orig_util_HAS_SNI
-### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
+# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
class SubjectAltName(BaseSubjectAltName):
'''ASN.1 implementation for subjectAltNames support'''
@@ -123,7 +124,7 @@ class SubjectAltName(BaseSubjectAltName):
constraint.ValueSizeConstraint(1, 1024)
-### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
+# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
def get_subj_alt_name(peer_cert):
# Search through extensions
dns_name = []
@@ -181,7 +182,7 @@ class WrappedSocket(object):
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
return b''
else:
- raise
+ raise SocketError(e)
except OpenSSL.SSL.ZeroReturnError as e:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b''
@@ -212,12 +213,9 @@ class WrappedSocket(object):
continue
def sendall(self, data):
- if has_memoryview and not isinstance(data, memoryview):
- data = memoryview(data)
-
total_sent = 0
while total_sent < len(data):
- sent = self._send_until_done(data[total_sent:total_sent+SSL_WRITE_BLOCKSIZE])
+ sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
total_sent += sent
def shutdown(self):
@@ -226,7 +224,10 @@ class WrappedSocket(object):
def close(self):
if self._makefile_refs < 1:
- return self.connection.close()
+ try:
+ return self.connection.close()
+ except OpenSSL.SSL.Error:
+ return
else:
self._makefile_refs -= 1
diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py
index 9607d65..8e07eb6 100644
--- a/urllib3/exceptions.py
+++ b/urllib3/exceptions.py
@@ -1,16 +1,17 @@
+from __future__ import absolute_import
+# Base Exceptions
-## Base Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
+
class HTTPWarning(Warning):
"Base warning used by this module."
pass
-
class PoolError(HTTPError):
"Base exception for errors caused within a pool."
def __init__(self, pool, message):
@@ -57,7 +58,7 @@ class ProtocolError(HTTPError):
ConnectionError = ProtocolError
-## Leaf Exceptions
+# Leaf Exceptions
class MaxRetryError(RequestError):
"""Raised when the maximum number of retries is exceeded.
@@ -112,10 +113,12 @@ class ConnectTimeoutError(TimeoutError):
"Raised when a socket timeout occurs while connecting to a server"
pass
+
class NewConnectionError(ConnectTimeoutError, PoolError):
"Raised when we fail to establish a new connection. Usually ECONNREFUSED."
pass
+
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
@@ -172,6 +175,11 @@ class InsecurePlatformWarning(SecurityWarning):
pass
+class SNIMissingWarning(HTTPWarning):
+ "Warned when making a HTTPS request without SNI available."
+ pass
+
+
class ResponseNotChunked(ProtocolError, ValueError):
"Response needs to be chunked in order to read it as chunks."
pass
diff --git a/urllib3/fields.py b/urllib3/fields.py
index c853f8d..c7d4811 100644
--- a/urllib3/fields.py
+++ b/urllib3/fields.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import email.utils
import mimetypes
diff --git a/urllib3/filepost.py b/urllib3/filepost.py
index 0fbf488..97a2843 100644
--- a/urllib3/filepost.py
+++ b/urllib3/filepost.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import codecs
from uuid import uuid4
diff --git a/urllib3/packages/__init__.py b/urllib3/packages/__init__.py
index 37e8351..170e974 100644
--- a/urllib3/packages/__init__.py
+++ b/urllib3/packages/__init__.py
@@ -2,3 +2,4 @@ from __future__ import absolute_import
from . import ssl_match_hostname
+__all__ = ('ssl_match_hostname', )
diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py
index 76b6a12..f13e673 100644
--- a/urllib3/poolmanager.py
+++ b/urllib3/poolmanager.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import logging
try: # Python 3
@@ -25,7 +26,7 @@ pool_classes_by_scheme = {
log = logging.getLogger(__name__)
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
- 'ssl_version')
+ 'ssl_version', 'ca_cert_dir')
class PoolManager(RequestMethods):
diff --git a/urllib3/request.py b/urllib3/request.py
index a1a12bc..d5aa62d 100644
--- a/urllib3/request.py
+++ b/urllib3/request.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
try:
from urllib.parse import urlencode
except ImportError:
@@ -133,7 +134,8 @@ class RequestMethods(object):
if fields:
if 'body' in urlopen_kw:
- raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.')
+ raise TypeError(
+ "request got values for both 'fields' and 'body', can only specify one.")
if encode_multipart:
body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
diff --git a/urllib3/response.py b/urllib3/response.py
index 788eb6c..8f2a1b5 100644
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -1,7 +1,9 @@
+from __future__ import absolute_import
from contextlib import contextmanager
import zlib
import io
from socket import timeout as SocketTimeout
+from socket import error as SocketError
from ._collections import HTTPHeaderDict
from .exceptions import (
@@ -130,8 +132,8 @@ class HTTPResponse(io.IOBase):
if "chunked" in encodings:
self.chunked = True
- # We certainly don't want to preload content when the response is chunked.
- if not self.chunked and preload_content and not self._body:
+ # If requested, preload the body.
+ if preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
@@ -194,12 +196,22 @@ class HTTPResponse(io.IOBase):
"Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding, e)
- if flush_decoder and decode_content and self._decoder:
- buf = self._decoder.decompress(binary_type())
- data += buf + self._decoder.flush()
+ if flush_decoder and decode_content:
+ data += self._flush_decoder()
return data
+ def _flush_decoder(self):
+ """
+ Flushes the decoder. Should only be called if the decoder is actually
+ being used.
+ """
+ if self._decoder:
+ buf = self._decoder.decompress(b'')
+ return buf + self._decoder.flush()
+
+ return b''
+
@contextmanager
def _error_catcher(self):
"""
@@ -227,15 +239,22 @@ class HTTPResponse(io.IOBase):
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
- except HTTPException as e:
+ except (HTTPException, SocketError) as e:
# This includes IncompleteRead.
raise ProtocolError('Connection broken: %r' % e, e)
+
except Exception:
# The response may not be closed but we're not going to use it anymore
# so close it now to ensure that the connection is released back to the pool.
if self._original_response and not self._original_response.isclosed():
self._original_response.close()
+ # Closing the response may not actually be sufficient to close
+ # everything, so if we have a hold of the connection close that
+ # too.
+ if self._connection is not None:
+ self._connection.close()
+
raise
finally:
if self._original_response and self._original_response.isclosed():
@@ -301,7 +320,6 @@ class HTTPResponse(io.IOBase):
return data
-
def stream(self, amt=2**16, decode_content=None):
"""
A generator wrapper for the read() method. A call will block until
@@ -340,9 +358,9 @@ class HTTPResponse(io.IOBase):
headers = r.msg
if not isinstance(headers, HTTPHeaderDict):
- if PY3: # Python 3
+ if PY3: # Python 3
headers = HTTPHeaderDict(headers.items())
- else: # Python 2
+ else: # Python 2
headers = HTTPHeaderDict.from_httplib(headers)
# HTTPResponse objects in Python 3 don't have a .strict attribute
@@ -454,7 +472,8 @@ class HTTPResponse(io.IOBase):
self._init_decoder()
# FIXME: Rewrite this method and make it a class with a better structured logic.
if not self.chunked:
- raise ResponseNotChunked("Response is not chunked. "
+ raise ResponseNotChunked(
+ "Response is not chunked. "
"Header 'transfer-encoding: chunked' is missing.")
# Don't bother reading the body of a HEAD request.
@@ -468,8 +487,18 @@ class HTTPResponse(io.IOBase):
if self.chunk_left == 0:
break
chunk = self._handle_chunk(amt)
- yield self._decode(chunk, decode_content=decode_content,
- flush_decoder=True)
+ decoded = self._decode(chunk, decode_content=decode_content,
+ flush_decoder=False)
+ if decoded:
+ yield decoded
+
+ if decode_content:
+ # On CPython and PyPy, we should never need to flush the
+ # decoder. However, on Jython we *might* need to, so
+ # lets defensively do it anyway.
+ decoded = self._flush_decoder()
+ if decoded: # Platform-specific: Jython.
+ yield decoded
# Chunk content ends with \r\n: discard it.
while True:
diff --git a/urllib3/util/__init__.py b/urllib3/util/__init__.py
index 8becc81..c6c6243 100644
--- a/urllib3/util/__init__.py
+++ b/urllib3/util/__init__.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
from .request import make_headers
@@ -22,3 +23,22 @@ from .url import (
split_first,
Url,
)
+
+__all__ = (
+ 'HAS_SNI',
+ 'SSLContext',
+ 'Retry',
+ 'Timeout',
+ 'Url',
+ 'assert_fingerprint',
+ 'current_time',
+ 'is_connection_dropped',
+ 'is_fp_closed',
+ 'get_host',
+ 'parse_url',
+ 'make_headers',
+ 'resolve_cert_reqs',
+ 'resolve_ssl_version',
+ 'split_first',
+ 'ssl_wrap_socket',
+)
diff --git a/urllib3/util/connection.py b/urllib3/util/connection.py
index 4f2f0f1..01a4812 100644
--- a/urllib3/util/connection.py
+++ b/urllib3/util/connection.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import socket
try:
from select import poll, POLLIN
diff --git a/urllib3/util/request.py b/urllib3/util/request.py
index bc64f6b..7377931 100644
--- a/urllib3/util/request.py
+++ b/urllib3/util/request.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from base64 import b64encode
from ..packages.six import b
diff --git a/urllib3/util/response.py b/urllib3/util/response.py
index 2c1de15..bc72327 100644
--- a/urllib3/util/response.py
+++ b/urllib3/util/response.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from ..packages.six.moves import http_client as httplib
from ..exceptions import HeaderParsingError
@@ -44,7 +45,7 @@ def assert_header_parsing(headers):
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
- raise TypeError('expected httplib.Message, got {}.'.format(
+ raise TypeError('expected httplib.Message, got {0}.'.format(
type(headers)))
defects = getattr(headers, 'defects', None)
diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py
index 1fb1f23..03a0124 100644
--- a/urllib3/util/retry.py
+++ b/urllib3/util/retry.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import time
import logging
@@ -126,7 +127,7 @@ class Retry(object):
self.method_whitelist = method_whitelist
self.backoff_factor = backoff_factor
self.raise_on_redirect = raise_on_redirect
- self._observed_errors = _observed_errors # TODO: use .history instead?
+ self._observed_errors = _observed_errors # TODO: use .history instead?
def new(self, **kw):
params = dict(
@@ -206,7 +207,8 @@ class Retry(object):
return min(retry_counts) < 0
- def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None):
+ def increment(self, method=None, url=None, response=None, error=None,
+ _pool=None, _stacktrace=None):
""" Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
@@ -274,7 +276,6 @@ class Retry(object):
return new_retry
-
def __repr__(self):
return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
'read={self.read}, redirect={self.redirect})').format(
diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py
index 47b817e..67f8344 100644
--- a/urllib3/util/ssl_.py
+++ b/urllib3/util/ssl_.py
@@ -1,7 +1,12 @@
+from __future__ import absolute_import
+import errno
+import warnings
+import hmac
+
from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256
-from ..exceptions import SSLError, InsecurePlatformWarning
+from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
SSLContext = None
@@ -15,8 +20,23 @@ HASHFUNC_MAP = {
64: sha256,
}
-import errno
-import warnings
+
+def _const_compare_digest_backport(a, b):
+ """
+ Compare two digests of equal length in constant time.
+
+ The digests must be of type str/bytes.
+ Returns True if the digests match, and False otherwise.
+ """
+ result = abs(len(a) - len(b))
+ for l, r in zip(bytearray(a), bytearray(b)):
+ result |= l ^ r
+ return result == 0
+
+
+_const_compare_digest = getattr(hmac, 'compare_digest',
+ _const_compare_digest_backport)
+
try: # Test for SSL features
import ssl
@@ -134,7 +154,7 @@ def assert_fingerprint(cert, fingerprint):
cert_digest = hashfunc(cert).digest()
- if cert_digest != fingerprint_bytes:
+ if not _const_compare_digest(cert_digest, fingerprint_bytes):
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
.format(fingerprint, hexlify(cert_digest)))
@@ -283,4 +303,15 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
context.load_cert_chain(certfile, keyfile)
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
return context.wrap_socket(sock, server_hostname=server_hostname)
+
+ warnings.warn(
+ 'An HTTPS request has been made, but the SNI (Subject Name '
+ 'Indication) extension to TLS is not available on this platform. '
+ 'This may cause the server to present an incorrect TLS '
+ 'certificate, which can cause validation failures. For more '
+ 'information, see '
+ 'https://urllib3.readthedocs.org/en/latest/security.html'
+ '#snimissingwarning.',
+ SNIMissingWarning
+ )
return context.wrap_socket(sock)
diff --git a/urllib3/util/timeout.py b/urllib3/util/timeout.py
index ea7027f..ff62f47 100644
--- a/urllib3/util/timeout.py
+++ b/urllib3/util/timeout.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
# The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT
@@ -9,6 +10,7 @@ from ..exceptions import TimeoutStateError
# urllib3
_Default = object()
+
def current_time():
"""
Retrieve the current time. This function is mocked out in unit testing.
@@ -226,9 +228,9 @@ class Timeout(object):
has not yet been called on this object.
"""
if (self.total is not None and
- self.total is not self.DEFAULT_TIMEOUT and
- self._read is not None and
- self._read is not self.DEFAULT_TIMEOUT):
+ self.total is not self.DEFAULT_TIMEOUT and
+ self._read is not None and
+ self._read is not self.DEFAULT_TIMEOUT):
# In case the connect timeout has not yet been established.
if self._start_connect is None:
return self._read
diff --git a/urllib3/util/url.py b/urllib3/util/url.py
index e58050c..e996204 100644
--- a/urllib3/util/url.py
+++ b/urllib3/util/url.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from collections import namedtuple
from ..exceptions import LocationParseError
@@ -85,6 +86,7 @@ class Url(namedtuple('Url', url_attrs)):
def __str__(self):
return self.url
+
def split_first(s, delims):
"""
Given a string and an iterable of delimiters, split on the first found
@@ -115,7 +117,7 @@ def split_first(s, delims):
if min_idx is None or min_idx < 0:
return s, '', None
- return s[:min_idx], s[min_idx+1:], min_delim
+ return s[:min_idx], s[min_idx + 1:], min_delim
def parse_url(url):
@@ -206,6 +208,7 @@ def parse_url(url):
return Url(scheme, auth, host, port, path, query, fragment)
+
def get_host(url):
"""
Deprecated. Use :func:`.parse_url` instead.