diff options
Diffstat (limited to 'urllib3')
| -rw-r--r-- | urllib3/__init__.py | 4 | ||||
| -rw-r--r-- | urllib3/connection.py | 43 | ||||
| -rw-r--r-- | urllib3/connectionpool.py | 6 | ||||
| -rw-r--r-- | urllib3/contrib/pyopenssl.py | 36 | ||||
| -rw-r--r-- | urllib3/exceptions.py | 21 | ||||
| -rw-r--r-- | urllib3/response.py | 15 | ||||
| -rw-r--r-- | urllib3/util/connection.py | 8 | ||||
| -rw-r--r-- | urllib3/util/response.py | 17 | ||||
| -rw-r--r-- | urllib3/util/retry.py | 2 |
9 files changed, 119 insertions, 33 deletions
diff --git a/urllib3/__init__.py b/urllib3/__init__.py index 56f5bf4..3546d13 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.9' +__version__ = '1.9.1' from .connectionpool import ( @@ -57,7 +57,7 @@ del NullHandler # Set security warning to only go off once by default. import warnings -warnings.simplefilter('module', exceptions.InsecureRequestWarning) +warnings.simplefilter('module', exceptions.SecurityWarning) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/urllib3/connection.py b/urllib3/connection.py index 0d578d7..cebdd86 100644 --- a/urllib3/connection.py +++ b/urllib3/connection.py @@ -1,6 +1,9 @@ +import datetime import sys import socket from socket import timeout as SocketTimeout +import warnings +from .packages import six try: # Python 3 from http.client import HTTPConnection as _HTTPConnection, HTTPException @@ -24,11 +27,19 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass +try: # Python 3: + # Not a no-op, we're adding this to the namespace so it can be imported. + ConnectionError = ConnectionError +except NameError: # Python 2: + class ConnectionError(Exception): + pass + + from .exceptions import ( ConnectTimeoutError, + SystemTimeWarning, ) from .packages.ssl_match_hostname import match_hostname -from .packages import six from .util.ssl_ import ( resolve_cert_reqs, @@ -37,14 +48,16 @@ from .util.ssl_ import ( assert_fingerprint, ) -from .util import connection +from .util import connection port_by_scheme = { 'http': 80, 'https': 443, } +RECENT_DATE = datetime.date(2014, 1, 1) + class HTTPConnection(_HTTPConnection, object): """ @@ -172,6 +185,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): cert_reqs = None ca_certs = None ssl_version = None + assert_fingerprint = None def set_cert(self, key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, @@ -206,6 +220,14 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: + warnings.warn(( + 'System time is way off (before {0}). This will probably ' + 'lead to SSL verification errors').format(RECENT_DATE), + SystemTimeWarning + ) + # Wrap socket using verification with the root certs in # trusted_root_certs self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, @@ -214,15 +236,16 @@ class VerifiedHTTPSConnection(HTTPSConnection): server_hostname=hostname, ssl_version=resolved_ssl_version) - if resolved_cert_reqs != ssl.CERT_NONE: - if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or hostname) + if self.assert_fingerprint: + assert_fingerprint(self.sock.getpeercert(binary_form=True), + self.assert_fingerprint) + elif resolved_cert_reqs != ssl.CERT_NONE \ + and self.assert_hostname is not False: + match_hostname(self.sock.getpeercert(), + self.assert_hostname or hostname) - self.is_verified = resolved_cert_reqs == ssl.CERT_REQUIRED + self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED + or self.assert_fingerprint is not None) if ssl: diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index 9317fdc..ac6e0ca 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -32,7 +32,7 @@ from .connection import ( port_by_scheme, DummyConnection, HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPException, BaseSSLError, ConnectionError ) from .request import RequestMethods from .response import HTTPResponse @@ -542,7 +542,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn = True raise SSLError(e) - except (TimeoutError, HTTPException, SocketError) as e: + except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: if conn: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. @@ -718,7 +718,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): super(HTTPSConnectionPool, self)._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not conn.sock: + if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` conn.connect() if not conn.is_verified: diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py index 7a9ea2e..8475eeb 100644 --- a/urllib3/contrib/pyopenssl.py +++ b/urllib3/contrib/pyopenssl.py @@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI when the required modules are installed. Activating this module also has the positive side effect of disabling SSL/TLS -encryption in Python 2 (see `CRIME attack`_). +compression in Python 2 (see `CRIME attack`_). If you want to configure the default list of supported cipher suites, you can set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. @@ -46,8 +46,12 @@ Module Variables ''' -from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT -from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName +try: + from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT + from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName +except SyntaxError as e: + raise ImportError(e) + import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint @@ -155,18 +159,24 @@ def get_subj_alt_name(peer_cert): class WrappedSocket(object): - '''API-compatibility wrapper for Python OpenSSL's Connection-class.''' + '''API-compatibility wrapper for Python OpenSSL's Connection-class. + + Note: _makefile_refs, _drop() and _reuse() are needed for the garbage + collector of pypy. + ''' def __init__(self, connection, socket, suppress_ragged_eofs=True): self.connection = connection self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs + self._makefile_refs = 0 def fileno(self): return self.socket.fileno() def makefile(self, mode, bufsize=-1): - return _fileobject(self, mode, bufsize) + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) def recv(self, *args, **kwargs): try: @@ -180,7 +190,7 @@ class WrappedSocket(object): rd, wd, ed = select.select( [self.socket], [], [], self.socket.gettimeout()) if not rd: - raise timeout() + raise timeout('The read operation timed out') else: return self.recv(*args, **kwargs) else: @@ -193,7 +203,10 @@ class WrappedSocket(object): return self.connection.sendall(data) def close(self): - return self.connection.shutdown() + if self._makefile_refs < 1: + return self.connection.shutdown() + else: + self._makefile_refs -= 1 def getpeercert(self, binary_form=False): x509 = self.connection.get_peer_certificate() @@ -216,6 +229,15 @@ class WrappedSocket(object): ] } + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py index fff8bfa..7519ba9 100644 --- a/urllib3/exceptions.py +++ b/urllib3/exceptions.py @@ -60,7 +60,14 @@ ConnectionError = ProtocolError ## Leaf Exceptions class MaxRetryError(RequestError): - "Raised when the maximum number of retries is exceeded." + """Raised when the maximum number of retries is exceeded. + + :param pool: The connection pool + :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` + :param string url: The requested Url + :param exceptions.Exception reason: The underlying error + + """ def __init__(self, pool, url, reason=None): self.reason = reason @@ -134,6 +141,16 @@ class LocationParseError(LocationValueError): self.location = location -class InsecureRequestWarning(HTTPWarning): +class SecurityWarning(HTTPWarning): + "Warned when perfoming security reducing actions" + pass + + +class InsecureRequestWarning(SecurityWarning): "Warned when making an unverified HTTPS request." pass + + +class SystemTimeWarning(SecurityWarning): + "Warned when system time is suspected to be wrong" + pass diff --git a/urllib3/response.py b/urllib3/response.py index 7e0d47f..e69de95 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -48,7 +48,10 @@ class HTTPResponse(io.IOBase): HTTP Response container. Backwards-compatible to httplib's HTTPResponse but the response ``body`` is - loaded and decoded on-demand when the ``data`` property is accessed. + loaded and decoded on-demand when the ``data`` property is accessed. This + class is also compatible with the Python standard library's :mod:`io` + module, and can hence be treated as a readable object in the context of that + framework. Extra parameters for behaviour not present in httplib.HTTPResponse: @@ -317,4 +320,14 @@ class HTTPResponse(io.IOBase): return self._fp.flush() def readable(self): + # This method is required for `io` module compatibility. return True + + def readinto(self, b): + # This method is required for `io` module compatibility. + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + else: + b[:len(temp)] = temp + return len(temp) diff --git a/urllib3/util/connection.py b/urllib3/util/connection.py index 062ee9d..2156993 100644 --- a/urllib3/util/connection.py +++ b/urllib3/util/connection.py @@ -66,13 +66,15 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, sock = None try: sock = socket.socket(af, socktype, proto) + + # If provided, set socket level options before connecting. + # This is the only addition urllib3 makes to this function. + _set_socket_options(sock, socket_options) + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(timeout) if source_address: sock.bind(source_address) - # If provided, set socket level options before connecting. - # This is the only addition urllib3 makes to this function. - _set_socket_options(sock, socket_options) sock.connect(sa) return sock diff --git a/urllib3/util/response.py b/urllib3/util/response.py index d0325bc..45fff55 100644 --- a/urllib3/util/response.py +++ b/urllib3/util/response.py @@ -5,9 +5,18 @@ def is_fp_closed(obj): :param obj: The file-like object to check. """ - if hasattr(obj, 'fp'): - # Object is a container for another file-like object that gets released - # on exhaustion (e.g. HTTPResponse) + + try: + # Check via the official file-like-object way. + return obj.closed + except AttributeError: + pass + + try: + # Check if the object is a container for another file-like object that + # gets released on exhaustion (e.g. HTTPResponse). return obj.fp is None + except AttributeError: + pass - return obj.closed + raise ValueError("Unable to determine whether fp is closed.") diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py index 9013197..eb560df 100644 --- a/urllib3/util/retry.py +++ b/urllib3/util/retry.py @@ -83,7 +83,7 @@ class Retry(object): same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. :param iterable status_forcelist: - A set of HTTP status codes that we should force a retry on. + A set of HTTP status codes that we should force a retry on. By default, this is disabled with ``None``. |