aboutsummaryrefslogtreecommitdiff
path: root/requests/packages/urllib3
diff options
context:
space:
mode:
Diffstat (limited to 'requests/packages/urllib3')
-rw-r--r--requests/packages/urllib3/__init__.py2
-rw-r--r--requests/packages/urllib3/_collections.py2
-rw-r--r--requests/packages/urllib3/connectionpool.py46
-rw-r--r--requests/packages/urllib3/contrib/__init__.py0
-rw-r--r--requests/packages/urllib3/contrib/ntlmpool.py120
-rw-r--r--requests/packages/urllib3/contrib/pyopenssl.py167
-rw-r--r--requests/packages/urllib3/exceptions.py28
-rw-r--r--requests/packages/urllib3/filepost.py2
-rw-r--r--requests/packages/urllib3/poolmanager.py14
-rw-r--r--requests/packages/urllib3/request.py2
-rw-r--r--requests/packages/urllib3/response.py79
-rw-r--r--requests/packages/urllib3/util.py50
12 files changed, 451 insertions, 61 deletions
diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py
index 55de87e..bff80b8 100644
--- a/requests/packages/urllib3/__init__.py
+++ b/requests/packages/urllib3/__init__.py
@@ -1,5 +1,5 @@
# urllib3/__init__.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py
index a052b1d..b35a736 100644
--- a/requests/packages/urllib3/_collections.py
+++ b/requests/packages/urllib3/_collections.py
@@ -1,5 +1,5 @@
# urllib3/_collections.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py
index 51c87f5..f3e9260 100644
--- a/requests/packages/urllib3/connectionpool.py
+++ b/requests/packages/urllib3/connectionpool.py
@@ -1,5 +1,5 @@
# urllib3/connectionpool.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -9,7 +9,7 @@ import socket
import errno
from socket import error as SocketError, timeout as SocketTimeout
-from .util import resolve_cert_reqs, resolve_ssl_version
+from .util import resolve_cert_reqs, resolve_ssl_version, assert_fingerprint
try: # Python 3
from http.client import HTTPConnection, HTTPException
@@ -81,12 +81,15 @@ class VerifiedHTTPSConnection(HTTPSConnection):
ssl_version = None
def set_cert(self, key_file=None, cert_file=None,
- cert_reqs=None, ca_certs=None):
+ cert_reqs=None, ca_certs=None,
+ assert_hostname=None, assert_fingerprint=None):
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = cert_reqs
self.ca_certs = ca_certs
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
def connect(self):
# Add certificate verification
@@ -104,8 +107,12 @@ class VerifiedHTTPSConnection(HTTPSConnection):
ssl_version=resolved_ssl_version)
if resolved_cert_reqs != ssl.CERT_NONE:
- match_hostname(self.sock.getpeercert(), self.host)
-
+ if self.assert_fingerprint:
+ assert_fingerprint(self.sock.getpeercert(binary_form=True),
+ self.assert_fingerprint)
+ else:
+ match_hostname(self.sock.getpeercert(),
+ self.assert_hostname or self.host)
## Pool objects
@@ -439,12 +446,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
except Empty as e:
# Timed out by queue
- raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" %
+ raise TimeoutError(self, url,
+ "Request timed out. (pool_timeout=%s)" %
pool_timeout)
except SocketTimeout as e:
# Timed out by socket
- raise TimeoutError(self, "Request timed out. (timeout=%s)" %
+ raise TimeoutError(self, url,
+ "Request timed out. (timeout=%s)" %
timeout)
except BaseSSLError as e:
@@ -502,9 +511,13 @@ class HTTPSConnectionPool(HTTPConnectionPool):
:class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
instead of :class:`httplib.HTTPSConnection`.
- The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, and ``ssl_version``
- are only used if :mod:`ssl` is available and are fed into
- :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket.
+ :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
+ ``assert_hostname`` and ``host`` in this order to verify connections.
+
+ The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and
+ ``ssl_version`` are only used if :mod:`ssl` is available and are fed into
+ :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket
+ into an SSL socket.
"""
scheme = 'https'
@@ -512,8 +525,9 @@ class HTTPSConnectionPool(HTTPConnectionPool):
def __init__(self, host, port=None,
strict=False, timeout=None, maxsize=1,
block=False, headers=None,
- key_file=None, cert_file=None,
- cert_reqs=None, ca_certs=None, ssl_version=None):
+ key_file=None, cert_file=None, cert_reqs=None,
+ ca_certs=None, ssl_version=None,
+ assert_hostname=None, assert_fingerprint=None):
HTTPConnectionPool.__init__(self, host, port,
strict, timeout, maxsize,
@@ -523,6 +537,8 @@ class HTTPSConnectionPool(HTTPConnectionPool):
self.cert_reqs = cert_reqs
self.ca_certs = ca_certs
self.ssl_version = ssl_version
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
def _new_conn(self):
"""
@@ -532,7 +548,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
- if not ssl: # Platform-specific: Python compiled without +ssl
+ if not ssl: # Platform-specific: Python compiled without +ssl
if not HTTPSConnection or HTTPSConnection is object:
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
@@ -545,7 +561,9 @@ class HTTPSConnectionPool(HTTPConnectionPool):
port=self.port,
strict=self.strict)
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
- cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
+ cert_reqs=self.cert_reqs, ca_certs=self.ca_certs,
+ assert_hostname=self.assert_hostname,
+ assert_fingerprint=self.assert_fingerprint)
connection.ssl_version = self.ssl_version
diff --git a/requests/packages/urllib3/contrib/__init__.py b/requests/packages/urllib3/contrib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/requests/packages/urllib3/contrib/__init__.py
diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py
new file mode 100644
index 0000000..277ee0b
--- /dev/null
+++ b/requests/packages/urllib3/contrib/ntlmpool.py
@@ -0,0 +1,120 @@
+# urllib3/contrib/ntlmpool.py
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+#
+# This module is part of urllib3 and is released under
+# the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+"""
+NTLM authenticating pool, contributed by erikcederstran
+
+Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
+"""
+
+try:
+ from http.client import HTTPSConnection
+except ImportError:
+ from httplib import HTTPSConnection
+from logging import getLogger
+from ntlm import ntlm
+
+from urllib3 import HTTPSConnectionPool
+
+
+log = getLogger(__name__)
+
+
+class NTLMConnectionPool(HTTPSConnectionPool):
+ """
+ Implements an NTLM authentication version of an urllib3 connection pool
+ """
+
+ scheme = 'https'
+
+ def __init__(self, user, pw, authurl, *args, **kwargs):
+ """
+ authurl is a random URL on the server that is protected by NTLM.
+ user is the Windows user, probably in the DOMAIN\username format.
+ pw is the password for the user.
+ """
+ super(NTLMConnectionPool, self).__init__(*args, **kwargs)
+ self.authurl = authurl
+ self.rawuser = user
+ user_parts = user.split('\\', 1)
+ self.domain = user_parts[0].upper()
+ self.user = user_parts[1]
+ self.pw = pw
+
+ def _new_conn(self):
+ # Performs the NTLM handshake that secures the connection. The socket
+ # must be kept open while requests are performed.
+ self.num_connections += 1
+ log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' %
+ (self.num_connections, self.host, self.authurl))
+
+ headers = {}
+ headers['Connection'] = 'Keep-Alive'
+ req_header = 'Authorization'
+ resp_header = 'www-authenticate'
+
+ conn = HTTPSConnection(host=self.host, port=self.port)
+
+ # Send negotiation message
+ headers[req_header] = (
+ 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
+ log.debug('Request headers: %s' % headers)
+ conn.request('GET', self.authurl, None, headers)
+ res = conn.getresponse()
+ reshdr = dict(res.getheaders())
+ log.debug('Response status: %s %s' % (res.status, res.reason))
+ log.debug('Response headers: %s' % reshdr)
+ log.debug('Response data: %s [...]' % res.read(100))
+
+ # Remove the reference to the socket, so that it can not be closed by
+ # the response object (we want to keep the socket open)
+ res.fp = None
+
+ # Server should respond with a challenge message
+ auth_header_values = reshdr[resp_header].split(', ')
+ auth_header_value = None
+ for s in auth_header_values:
+ if s[:5] == 'NTLM ':
+ auth_header_value = s[5:]
+ if auth_header_value is None:
+ raise Exception('Unexpected %s response header: %s' %
+ (resp_header, reshdr[resp_header]))
+
+ # Send authentication message
+ ServerChallenge, NegotiateFlags = \
+ ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
+ auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge,
+ self.user,
+ self.domain,
+ self.pw,
+ NegotiateFlags)
+ headers[req_header] = 'NTLM %s' % auth_msg
+ log.debug('Request headers: %s' % headers)
+ conn.request('GET', self.authurl, None, headers)
+ res = conn.getresponse()
+ log.debug('Response status: %s %s' % (res.status, res.reason))
+ log.debug('Response headers: %s' % dict(res.getheaders()))
+ log.debug('Response data: %s [...]' % res.read()[:100])
+ if res.status != 200:
+ if res.status == 401:
+ raise Exception('Server rejected request: wrong '
+ 'username or password')
+ raise Exception('Wrong server response: %s %s' %
+ (res.status, res.reason))
+
+ res.fp = None
+ log.debug('Connection established')
+ return conn
+
+ def urlopen(self, method, url, body=None, headers=None, retries=3,
+ redirect=True, assert_same_host=True):
+ if headers is None:
+ headers = {}
+ headers['Connection'] = 'Keep-Alive'
+ return super(NTLMConnectionPool, self).urlopen(method, url, body,
+ headers, retries,
+ redirect,
+ assert_same_host)
diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py
new file mode 100644
index 0000000..5c4c6d8
--- /dev/null
+++ b/requests/packages/urllib3/contrib/pyopenssl.py
@@ -0,0 +1,167 @@
+'''SSL with SNI-support for Python 2.
+
+This needs the following packages installed:
+
+* pyOpenSSL (tested with 0.13)
+* ndg-httpsclient (tested with 0.3.2)
+* pyasn1 (tested with 0.1.6)
+
+To activate it call :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3`.
+This can be done in a ``sitecustomize`` module, or at any other time before
+your application begins using ``urllib3``, like this::
+
+ try:
+ import urllib3.contrib.pyopenssl
+ urllib3.contrib.pyopenssl.inject_into_urllib3()
+ except ImportError:
+ pass
+
+Now you can use :mod:`urllib3` as you normally would, and it will support SNI
+when the required modules are installed.
+'''
+
+from ndg.httpsclient.ssl_peer_verification import (ServerSSLCertVerification,
+ SUBJ_ALT_NAME_SUPPORT)
+from ndg.httpsclient.subj_alt_name import SubjectAltName
+import OpenSSL.SSL
+from pyasn1.codec.der import decoder as der_decoder
+from socket import _fileobject
+import ssl
+
+from .. import connectionpool
+from .. import util
+
+__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
+
+# SNI only *really* works if we can read the subjectAltName of certificates.
+HAS_SNI = SUBJ_ALT_NAME_SUPPORT
+
+# Map from urllib3 to PyOpenSSL compatible parameter-values.
+_openssl_versions = {
+ ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
+ ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD,
+ ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
+}
+_openssl_verify = {
+ ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
+ ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
+ ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+ + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+}
+
+
+orig_util_HAS_SNI = util.HAS_SNI
+orig_connectionpool_ssl_wrap_socket = connectionpool.ssl_wrap_socket
+
+
+def inject_into_urllib3():
+ 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
+
+ connectionpool.ssl_wrap_socket = ssl_wrap_socket
+ util.HAS_SNI = HAS_SNI
+
+
+def extract_from_urllib3():
+ 'Undo monkey-patching by :func:`inject_into_urllib3`.'
+
+ connectionpool.ssl_wrap_socket = orig_connectionpool_ssl_wrap_socket
+ util.HAS_SNI = orig_util_HAS_SNI
+
+
+### Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
+def get_subj_alt_name(peer_cert):
+ # Search through extensions
+ dns_name = []
+ if not SUBJ_ALT_NAME_SUPPORT:
+ return dns_name
+
+ general_names = SubjectAltName()
+ for i in range(peer_cert.get_extension_count()):
+ ext = peer_cert.get_extension(i)
+ ext_name = ext.get_short_name()
+ if ext_name != 'subjectAltName':
+ continue
+
+ # PyOpenSSL returns extension data in ASN.1 encoded form
+ ext_dat = ext.get_data()
+ decoded_dat = der_decoder.decode(ext_dat,
+ asn1Spec=general_names)
+
+ for name in decoded_dat:
+ if not isinstance(name, SubjectAltName):
+ continue
+ for entry in range(len(name)):
+ component = name.getComponentByPosition(entry)
+ if component.getName() != 'dNSName':
+ continue
+ dns_name.append(str(component.getComponent()))
+
+ return dns_name
+
+
+class WrappedSocket(object):
+ '''API-compatibility wrapper for Python OpenSSL's Connection-class.'''
+
+ def __init__(self, connection, socket):
+ self.connection = connection
+ self.socket = socket
+
+ def makefile(self, mode, bufsize=-1):
+ return _fileobject(self.connection, mode, bufsize)
+
+ def settimeout(self, timeout):
+ return self.socket.settimeout(timeout)
+
+ def sendall(self, data):
+ return self.connection.sendall(data)
+
+ def getpeercert(self, binary_form=False):
+ x509 = self.connection.get_peer_certificate()
+ if not x509:
+ raise ssl.SSLError('')
+
+ if binary_form:
+ return OpenSSL.crypto.dump_certificate(
+ OpenSSL.crypto.FILETYPE_ASN1,
+ x509)
+
+ return {
+ 'subject': (
+ (('commonName', x509.get_subject().CN),),
+ ),
+ 'subjectAltName': [
+ ('DNS', value)
+ for value in get_subj_alt_name(x509)
+ ]
+ }
+
+
+def _verify_callback(cnx, x509, err_no, err_depth, return_code):
+ return err_no == 0
+
+
+def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+ ca_certs=None, server_hostname=None,
+ ssl_version=None):
+ ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
+ if certfile:
+ ctx.use_certificate_file(certfile)
+ if keyfile:
+ ctx.use_privatekey_file(keyfile)
+ if cert_reqs != ssl.CERT_NONE:
+ ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback)
+ if ca_certs:
+ try:
+ ctx.load_verify_locations(ca_certs, None)
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e)
+
+ cnx = OpenSSL.SSL.Connection(ctx, sock)
+ cnx.set_tlsext_host_name(server_hostname)
+ cnx.set_connect_state()
+ try:
+ cnx.do_handshake()
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError('bad handshake', e)
+
+ return WrappedSocket(cnx, sock)
diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py
index c5eb962..2e2a259 100644
--- a/requests/packages/urllib3/exceptions.py
+++ b/requests/packages/urllib3/exceptions.py
@@ -1,5 +1,5 @@
# urllib3/exceptions.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -20,7 +20,18 @@ class PoolError(HTTPError):
def __reduce__(self):
# For pickling purposes.
- return self.__class__, (None, self.url)
+ return self.__class__, (None, None)
+
+
+class RequestError(PoolError):
+ "Base exception for PoolErrors that have associated URLs."
+ def __init__(self, pool, url, message):
+ self.url = url
+ PoolError.__init__(self, pool, message)
+
+ def __reduce__(self):
+ # For pickling purposes.
+ return self.__class__, (None, self.url, None)
class SSLError(HTTPError):
@@ -35,7 +46,7 @@ class DecodeError(HTTPError):
## Leaf Exceptions
-class MaxRetryError(PoolError):
+class MaxRetryError(RequestError):
"Raised when the maximum number of retries is exceeded."
def __init__(self, pool, url, reason=None):
@@ -47,22 +58,19 @@ class MaxRetryError(PoolError):
else:
message += " (Caused by redirect)"
- PoolError.__init__(self, pool, message)
- self.url = url
+ RequestError.__init__(self, pool, url, message)
-class HostChangedError(PoolError):
+class HostChangedError(RequestError):
"Raised when an existing pool gets a request for a foreign host."
def __init__(self, pool, url, retries=3):
message = "Tried to open a foreign host with url: %s" % url
- PoolError.__init__(self, pool, message)
-
- self.url = url
+ RequestError.__init__(self, pool, url, message)
self.retries = retries
-class TimeoutError(PoolError):
+class TimeoutError(RequestError):
"Raised when a socket timeout occurs."
pass
diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py
index 8d900bd..470309a 100644
--- a/requests/packages/urllib3/filepost.py
+++ b/requests/packages/urllib3/filepost.py
@@ -93,6 +93,6 @@ def encode_multipart_formdata(fields, boundary=None):
body.write(b('--%s--\r\n' % (boundary)))
- content_type = b('multipart/form-data; boundary=%s' % boundary)
+ content_type = str('multipart/form-data; boundary=%s' % boundary)
return body.getvalue(), content_type
diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py
index 6e7377c..ce0c248 100644
--- a/requests/packages/urllib3/poolmanager.py
+++ b/requests/packages/urllib3/poolmanager.py
@@ -1,5 +1,5 @@
# urllib3/poolmanager.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -23,6 +23,9 @@ pool_classes_by_scheme = {
log = logging.getLogger(__name__)
+SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
+ 'ssl_version')
+
class PoolManager(RequestMethods):
"""
@@ -67,7 +70,13 @@ class PoolManager(RequestMethods):
to be overridden for customization.
"""
pool_cls = pool_classes_by_scheme[scheme]
- return pool_cls(host, port, **self.connection_pool_kw)
+ kwargs = self.connection_pool_kw
+ if scheme == 'http':
+ kwargs = self.connection_pool_kw.copy()
+ for kw in SSL_KEYWORDS:
+ kwargs.pop(kw, None)
+
+ return pool_cls(host, port, **kwargs)
def clear(self):
"""
@@ -141,6 +150,7 @@ class PoolManager(RequestMethods):
log.info("Redirecting %s -> %s" % (url, redirect_location))
kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
+ kw['redirect'] = redirect
return self.urlopen(method, redirect_location, **kw)
diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py
index 2b4704e..bf0256e 100644
--- a/requests/packages/urllib3/request.py
+++ b/requests/packages/urllib3/request.py
@@ -1,5 +1,5 @@
# urllib3/request.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py
index 0761dc0..2fa4078 100644
--- a/requests/packages/urllib3/response.py
+++ b/requests/packages/urllib3/response.py
@@ -4,29 +4,48 @@
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
-import gzip
+
import logging
import zlib
-from io import BytesIO
-
from .exceptions import DecodeError
-from .packages.six import string_types as basestring
+from .packages.six import string_types as basestring, binary_type
log = logging.getLogger(__name__)
-def decode_gzip(data):
- gzipper = gzip.GzipFile(fileobj=BytesIO(data))
- return gzipper.read()
+class DeflateDecoder(object):
+
+ def __init__(self):
+ self._first_try = True
+ self._data = binary_type()
+ self._obj = zlib.decompressobj()
+ def __getattr__(self, name):
+ return getattr(self._obj, name)
-def decode_deflate(data):
- try:
- return zlib.decompress(data)
- except zlib.error:
- return zlib.decompress(data, -zlib.MAX_WBITS)
+ def decompress(self, data):
+ if not self._first_try:
+ return self._obj.decompress(data)
+
+ self._data += data
+ try:
+ return self._obj.decompress(data)
+ except zlib.error:
+ self._first_try = False
+ self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
+ try:
+ return self.decompress(self._data)
+ finally:
+ self._data = None
+
+
+def _get_decoder(mode):
+ if mode == 'gzip':
+ return zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+ return DeflateDecoder()
class HTTPResponse(object):
@@ -52,10 +71,7 @@ class HTTPResponse(object):
otherwise unused.
"""
- CONTENT_DECODERS = {
- 'gzip': decode_gzip,
- 'deflate': decode_deflate,
- }
+ CONTENT_DECODERS = ['gzip', 'deflate']
def __init__(self, body='', headers=None, status=0, version=0, reason=None,
strict=0, preload_content=True, decode_content=True,
@@ -65,8 +81,9 @@ class HTTPResponse(object):
self.version = version
self.reason = reason
self.strict = strict
+ self.decode_content = decode_content
- self._decode_content = decode_content
+ self._decoder = None
self._body = body if body and isinstance(body, basestring) else None
self._fp = None
self._original_response = original_response
@@ -115,13 +132,13 @@ class HTTPResponse(object):
parameters: ``decode_content`` and ``cache_content``.
:param amt:
- How much of the content to read. If specified, decoding and caching
- is skipped because we can't decode partial content nor does it make
- sense to cache partial content as the full response.
+ How much of the content to read. If specified, caching is skipped
+ because it doesn't make sense to cache partial content as the full
+ response.
:param decode_content:
If True, will attempt to decode the body based on the
- 'content-encoding' header. (Overridden if ``amt`` is set.)
+ 'content-encoding' header.
:param cache_content:
If True, will save the returned data such that the same result is
@@ -133,18 +150,24 @@ class HTTPResponse(object):
# Note: content-encoding value should be case-insensitive, per RFC 2616
# Section 3.5
content_encoding = self.headers.get('content-encoding', '').lower()
- decoder = self.CONTENT_DECODERS.get(content_encoding)
+ if self._decoder is None:
+ if content_encoding in self.CONTENT_DECODERS:
+ self._decoder = _get_decoder(content_encoding)
if decode_content is None:
- decode_content = self._decode_content
+ decode_content = self.decode_content
if self._fp is None:
return
+ flush_decoder = False
+
try:
if amt is None:
# cStringIO doesn't like amt=None
data = self._fp.read()
+ flush_decoder = True
else:
+ cache_content = False
data = self._fp.read(amt)
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
# Close the connection when no data is returned
@@ -155,15 +178,19 @@ class HTTPResponse(object):
# properly close the connection in all cases. There is no harm
# in redundantly calling close.
self._fp.close()
- return data
+ flush_decoder = True
try:
- if decode_content and decoder:
- data = decoder(data)
+ if decode_content and self._decoder:
+ data = self._decoder.decompress(data)
except (IOError, zlib.error):
raise DecodeError("Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
+ if flush_decoder and self._decoder:
+ buf = self._decoder.decompress(binary_type())
+ data += buf + self._decoder.flush()
+
if cache_content:
self._body = data
diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py
index b827bc4..544f9ed 100644
--- a/requests/packages/urllib3/util.py
+++ b/requests/packages/urllib3/util.py
@@ -1,5 +1,5 @@
# urllib3/util.py
-# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -8,6 +8,8 @@
from base64 import b64encode
from collections import namedtuple
from socket import error as SocketError
+from hashlib import md5, sha1
+from binascii import hexlify, unhexlify
try:
from select import poll, POLLIN
@@ -23,7 +25,7 @@ try: # Test for SSL features
HAS_SNI = False
import ssl
- from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23
+ from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
from ssl import SSLContext # Modern SSL?
from ssl import HAS_SNI # Has SNI?
except ImportError:
@@ -31,7 +33,7 @@ except ImportError:
from .packages import six
-from .exceptions import LocationParseError
+from .exceptions import LocationParseError, SSLError
class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])):
@@ -232,7 +234,7 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
return headers
-def is_connection_dropped(conn):
+def is_connection_dropped(conn): # Platform-specific
"""
Returns True if the connection is dropped and should be closed.
@@ -246,7 +248,7 @@ def is_connection_dropped(conn):
if not sock: # Platform-specific: AppEngine
return False
- if not poll: # Platform-specific
+ if not poll:
if not select: # Platform-specific: AppEngine
return False
@@ -302,6 +304,44 @@ def resolve_ssl_version(candidate):
return candidate
+
+def assert_fingerprint(cert, fingerprint):
+ """
+ Checks if given fingerprint matches the supplied certificate.
+
+ :param cert:
+ Certificate as bytes object.
+ :param fingerprint:
+ Fingerprint as string of hexdigits, can be interspersed by colons.
+ """
+
+ # Maps the length of a digest to a possible hash function producing
+ # this digest.
+ hashfunc_map = {
+ 16: md5,
+ 20: sha1
+ }
+
+ fingerprint = fingerprint.replace(':', '').lower()
+
+ digest_length, rest = divmod(len(fingerprint), 2)
+
+ if rest or digest_length not in hashfunc_map:
+ raise SSLError('Fingerprint is of invalid length.')
+
+ # We need encode() here for py32; works on py2 and p33.
+ fingerprint_bytes = unhexlify(fingerprint.encode())
+
+ hashfunc = hashfunc_map[digest_length]
+
+ cert_digest = hashfunc(cert).digest()
+
+ if not cert_digest == fingerprint_bytes:
+ raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
+ .format(hexlify(fingerprint_bytes),
+ hexlify(cert_digest)))
+
+
if SSLContext is not None: # Python 3.2+
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
ca_certs=None, server_hostname=None,