aboutsummaryrefslogtreecommitdiff
path: root/requests/packages/urllib3
diff options
context:
space:
mode:
Diffstat (limited to 'requests/packages/urllib3')
-rw-r--r--requests/packages/urllib3/__init__.py20
-rw-r--r--requests/packages/urllib3/_collections.py169
-rw-r--r--requests/packages/urllib3/connectionpool.py139
-rw-r--r--requests/packages/urllib3/exceptions.py26
-rw-r--r--requests/packages/urllib3/filepost.py36
-rw-r--r--requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py47
-rw-r--r--requests/packages/urllib3/packages/ordered_dict.py260
-rw-r--r--requests/packages/urllib3/packages/six.py53
-rw-r--r--requests/packages/urllib3/poolmanager.py93
-rw-r--r--requests/packages/urllib3/request.py26
-rw-r--r--requests/packages/urllib3/response.py12
-rw-r--r--requests/packages/urllib3/util.py244
12 files changed, 812 insertions, 313 deletions
diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py
index 81e76f5..55de87e 100644
--- a/requests/packages/urllib3/__init__.py
+++ b/requests/packages/urllib3/__init__.py
@@ -28,7 +28,7 @@ from .util import make_headers, get_host
# Set default logging handler to avoid "No handler found" warnings.
import logging
-try:
+try: # Python 2.7+
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
@@ -37,6 +37,22 @@ except ImportError:
logging.getLogger(__name__).addHandler(NullHandler())
+def add_stderr_logger(level=logging.DEBUG):
+ """
+ Helper for quickly adding a StreamHandler to the logger. Useful for
+ debugging.
+
+ Returns the handler after adding it.
+ """
+ # This method needs to be in this __init__.py to get the __name__ correct
+ # even if urllib3 is vendored within another package.
+ logger = logging.getLogger(__name__)
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+ logger.addHandler(handler)
+ logger.setLevel(level)
+ logger.debug('Added an stderr logging handler to logger: %s' % __name__)
+ return handler
+
# ... Clean up.
-del logging
del NullHandler
diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py
index 3cef081..a052b1d 100644
--- a/requests/packages/urllib3/_collections.py
+++ b/requests/packages/urllib3/_collections.py
@@ -4,128 +4,91 @@
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
-from collections import deque
+from collections import MutableMapping
+from threading import Lock
-from threading import RLock
+try: # Python 2.7+
+ from collections import OrderedDict
+except ImportError:
+ from .packages.ordered_dict import OrderedDict
-__all__ = ['RecentlyUsedContainer']
+__all__ = ['RecentlyUsedContainer']
-class AccessEntry(object):
- __slots__ = ('key', 'is_valid')
- def __init__(self, key, is_valid=True):
- self.key = key
- self.is_valid = is_valid
+_Null = object()
-class RecentlyUsedContainer(dict):
- """
- Provides a dict-like that maintains up to ``maxsize`` keys while throwing
- away the least-recently-used keys beyond ``maxsize``.
+class RecentlyUsedContainer(MutableMapping):
"""
+ Provides a thread-safe dict-like container which maintains up to
+ ``maxsize`` keys while throwing away the least-recently-used keys beyond
+ ``maxsize``.
- # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we
- # will attempt to cleanup the invalidated entries in the access_log
- # datastructure during the next 'get' operation.
- CLEANUP_FACTOR = 10
-
- def __init__(self, maxsize=10):
- self._maxsize = maxsize
-
- self._container = {}
-
- # We use a deque to to store our keys ordered by the last access.
- self.access_log = deque()
- self.access_log_lock = RLock()
-
- # We look up the access log entry by the key to invalidate it so we can
- # insert a new authorative entry at the head without having to dig and
- # find the old entry for removal immediately.
- self.access_lookup = {}
-
- # Trigger a heap cleanup when we get past this size
- self.access_log_limit = maxsize * self.CLEANUP_FACTOR
-
- def _invalidate_entry(self, key):
- "If exists: Invalidate old entry and return it."
- old_entry = self.access_lookup.get(key)
- if old_entry:
- old_entry.is_valid = False
+ :param maxsize:
+ Maximum number of recent elements to retain.
- return old_entry
-
- def _push_entry(self, key):
- "Push entry onto our access log, invalidate the old entry if exists."
- self._invalidate_entry(key)
-
- new_entry = AccessEntry(key)
- self.access_lookup[key] = new_entry
-
- self.access_log_lock.acquire()
- self.access_log.appendleft(new_entry)
- self.access_log_lock.release()
-
- def _prune_entries(self, num):
- "Pop entries from our access log until we popped ``num`` valid ones."
- while num > 0:
- self.access_log_lock.acquire()
- p = self.access_log.pop()
- self.access_log_lock.release()
-
- if not p.is_valid:
- continue # Invalidated entry, skip
-
- dict.pop(self, p.key, None)
- self.access_lookup.pop(p.key, None)
- num -= 1
+ :param dispose_func:
+ Every time an item is evicted from the container,
+ ``dispose_func(value)`` is called. Callback which will get called
+ """
- def _prune_invalidated_entries(self):
- "Rebuild our access_log without the invalidated entries."
- self.access_log_lock.acquire()
- self.access_log = deque(e for e in self.access_log if e.is_valid)
- self.access_log_lock.release()
+ ContainerCls = OrderedDict
- def _get_ordered_access_keys(self):
- "Return ordered access keys for inspection. Used for testing."
- self.access_log_lock.acquire()
- r = [e.key for e in self.access_log if e.is_valid]
- self.access_log_lock.release()
+ def __init__(self, maxsize=10, dispose_func=None):
+ self._maxsize = maxsize
+ self.dispose_func = dispose_func
- return r
+ self._container = self.ContainerCls()
+ self._lock = Lock()
def __getitem__(self, key):
- item = dict.get(self, key)
+ # Re-insert the item, moving it to the end of the eviction line.
+ with self._lock:
+ item = self._container.pop(key)
+ self._container[key] = item
+ return item
+
+ def __setitem__(self, key, value):
+ evicted_value = _Null
+ with self._lock:
+ # Possibly evict the existing value of 'key'
+ evicted_value = self._container.get(key, _Null)
+ self._container[key] = value
+
+ # If we didn't evict an existing value, we might have to evict the
+ # least recently used item from the beginning of the container.
+ if len(self._container) > self._maxsize:
+ _key, evicted_value = self._container.popitem(last=False)
+
+ if self.dispose_func and evicted_value is not _Null:
+ self.dispose_func(evicted_value)
- if not item:
- raise KeyError(key)
+ def __delitem__(self, key):
+ with self._lock:
+ value = self._container.pop(key)
- # Insert new entry with new high priority, also implicitly invalidates
- # the old entry.
- self._push_entry(key)
+ if self.dispose_func:
+ self.dispose_func(value)
- if len(self.access_log) > self.access_log_limit:
- # Heap is getting too big, try to clean up any tailing invalidated
- # entries.
- self._prune_invalidated_entries()
+ def __len__(self):
+ with self._lock:
+ return len(self._container)
- return item
+ def __iter__(self):
+ raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
- def __setitem__(self, key, item):
- # Add item to our container and access log
- dict.__setitem__(self, key, item)
- self._push_entry(key)
+ def clear(self):
+ with self._lock:
+ # Copy pointers to all values, then wipe the mapping
+ # under Python 2, this copies the list of values twice :-|
+ values = list(self._container.values())
+ self._container.clear()
- # Discard invalid and excess entries
- self._prune_entries(len(self) - self._maxsize)
+ if self.dispose_func:
+ for value in values:
+ self.dispose_func(value)
- def __delitem__(self, key):
- self._invalidate_entry(key)
- self.access_lookup.pop(key, None)
- dict.__delitem__(self, key)
-
- def get(self, key, default=None):
- try:
- return self[key]
- except KeyError:
- return default
+ def keys(self):
+ with self._lock:
+ return self._container.keys()
diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py
index 336aa77..af8760d 100644
--- a/requests/packages/urllib3/connectionpool.py
+++ b/requests/packages/urllib3/connectionpool.py
@@ -6,28 +6,29 @@
import logging
import socket
+import errno
from socket import error as SocketError, timeout as SocketTimeout
-try: # Python 3
+try: # Python 3
from http.client import HTTPConnection, HTTPException
from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
from httplib import HTTPConnection, HTTPException
from httplib import HTTP_PORT, HTTPS_PORT
-try: # Python 3
+try: # Python 3
from queue import LifoQueue, Empty, Full
except ImportError:
from Queue import LifoQueue, Empty, Full
-try: # Compiled with SSL?
+try: # Compiled with SSL?
HTTPSConnection = object
BaseSSLError = None
ssl = None
- try: # Python 3
+ try: # Python 3
from http.client import HTTPSConnection
except ImportError:
from httplib import HTTPSConnection
@@ -35,14 +36,15 @@ try: # Compiled with SSL?
import ssl
BaseSSLError = ssl.SSLError
-except (ImportError, AttributeError):
+except (ImportError, AttributeError): # Platform-specific: No SSL.
pass
from .request import RequestMethods
from .response import HTTPResponse
-from .util import get_host, is_connection_dropped
+from .util import get_host, is_connection_dropped, ssl_wrap_socket
from .exceptions import (
+ ClosedPoolError,
EmptyPoolError,
HostChangedError,
MaxRetryError,
@@ -75,6 +77,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
"""
cert_reqs = None
ca_certs = None
+ ssl_version = None
def set_cert(self, key_file=None, cert_file=None,
cert_reqs='CERT_NONE', ca_certs=None):
@@ -95,9 +98,12 @@ class VerifiedHTTPSConnection(HTTPSConnection):
# Wrap socket using verification with the root certs in
# trusted_root_certs
- self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
+ self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
cert_reqs=self.cert_reqs,
- ca_certs=self.ca_certs)
+ ca_certs=self.ca_certs,
+ server_hostname=self.host,
+ ssl_version=self.ssl_version)
+
if self.ca_certs:
match_hostname(self.sock.getpeercert(), self.host)
@@ -165,13 +171,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
block=False, headers=None):
- super(HTTPConnectionPool, self).__init__(host, port)
+ ConnectionPool.__init__(self, host, port)
+ RequestMethods.__init__(self, headers)
self.strict = strict
self.timeout = timeout
self.pool = self.QueueCls(maxsize)
self.block = block
- self.headers = headers or {}
# Fill the queue up so that doing get() on it will block properly
for _ in xrange(maxsize):
@@ -188,7 +194,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
self.num_connections += 1
log.info("Starting new HTTP connection (%d): %s" %
(self.num_connections, self.host))
- return HTTPConnection(host=self.host, port=self.port)
+ return HTTPConnection(host=self.host,
+ port=self.port,
+ strict=self.strict)
def _get_conn(self, timeout=None):
"""
@@ -206,10 +214,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
try:
conn = self.pool.get(block=self.block, timeout=timeout)
- # If this is a persistent connection, check if it got disconnected
- if conn and is_connection_dropped(conn):
- log.info("Resetting dropped connection: %s" % self.host)
- conn.close()
+ except AttributeError: # self.pool is None
+ raise ClosedPoolError(self, "Pool is closed.")
except Empty:
if self.block:
@@ -218,6 +224,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
"connections are allowed.")
pass # Oh well, we'll create a new connection then
+ # If this is a persistent connection, check if it got disconnected
+ if conn and is_connection_dropped(conn):
+ log.info("Resetting dropped connection: %s" % self.host)
+ conn.close()
+
return conn or self._new_conn()
def _put_conn(self, conn):
@@ -228,17 +239,26 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
Connection object for the current host and port as returned by
:meth:`._new_conn` or :meth:`._get_conn`.
- If the pool is already full, the connection is discarded because we
- exceeded maxsize. If connections are discarded frequently, then maxsize
- should be increased.
+ If the pool is already full, the connection is closed and discarded
+ because we exceeded maxsize. If connections are discarded frequently,
+ then maxsize should be increased.
+
+ If the pool is closed, then the connection will be closed and discarded.
"""
try:
self.pool.put(conn, block=False)
+ return # Everything is dandy, done.
+ except AttributeError:
+ # self.pool is None.
+ pass
except Full:
# This should never happen if self.block == True
log.warning("HttpConnectionPool is full, discarding connection: %s"
% self.host)
+ # Connection never got put back into the pool, close it.
+ conn.close()
+
def _make_request(self, conn, method, url, timeout=_Default,
**httplib_request_kw):
"""
@@ -258,22 +278,42 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if sock:
sock.settimeout(timeout)
- httplib_response = conn.getresponse()
+ try: # Python 2.7+, use buffering of HTTP responses
+ httplib_response = conn.getresponse(buffering=True)
+ except TypeError: # Python 2.6 and older
+ httplib_response = conn.getresponse()
# AppEngine doesn't have a version attr.
- http_version = getattr(conn, '_http_vsn_str', 'HTTP/?'),
+ http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
httplib_response.status,
httplib_response.length))
-
return httplib_response
+ def close(self):
+ """
+ Close all pooled connections and disable the pool.
+ """
+ # Disable access to the pool
+ old_pool, self.pool = self.pool, None
+
+ try:
+ while True:
+ conn = old_pool.get(block=False)
+ if conn:
+ conn.close()
+
+ except Empty:
+ pass # Done.
def is_same_host(self, url):
"""
Check if the given ``url`` is a member of the same host as this
connection pool.
"""
+ if url.startswith('/'):
+ return True
+
# TODO: Add optional support for socket.gethostbyname checking.
scheme, host, port = get_host(url)
@@ -281,8 +321,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Use explicit default port for comparison when none is given.
port = port_by_scheme.get(scheme)
- return (url.startswith('/') or
- (scheme, host, port) == (self.scheme, self.host, self.port))
+ return (scheme, host, port) == (self.scheme, self.host, self.port)
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True, timeout=_Default,
@@ -321,8 +360,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
Number of retries to allow before raising a MaxRetryError exception.
:param redirect:
- Automatically handle redirects (status codes 301, 302, 303, 307),
- each redirect counts as a retry.
+ If True, automatically handle redirects (status codes 301, 302,
+ 303, 307). Each redirect counts as a retry.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
@@ -375,7 +414,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
try:
# Request a connection from the queue
- # (Could raise SocketError: Bad file descriptor)
conn = self._get_conn(timeout=pool_timeout)
# Make the request on the httplib connection object
@@ -424,23 +462,35 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# This is necessary so we can access e below
err = e
+ if retries == 0:
+ raise MaxRetryError(self, url, e)
+
finally:
- if conn and release_conn:
- # Put the connection back to be reused
+ if release_conn:
+ # Put the connection back to be reused. If the connection is
+ # expired then it will be None, which will get replaced with a
+ # fresh connection during _get_conn.
self._put_conn(conn)
if not conn:
+ # Try again
log.warn("Retrying (%d attempts remain) after connection "
"broken by '%r': %s" % (retries, err, url))
return self.urlopen(method, url, body, headers, retries - 1,
- redirect, assert_same_host) # Try again
+ redirect, assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
# Handle redirect?
redirect_location = redirect and response.get_redirect_location()
if redirect_location:
+ if response.status == 303:
+ method = 'GET'
log.info("Redirecting %s -> %s" % (url, redirect_location))
return self.urlopen(method, redirect_location, body, headers,
- retries - 1, redirect, assert_same_host)
+ retries - 1, redirect, assert_same_host,
+ timeout=timeout, pool_timeout=pool_timeout,
+ release_conn=release_conn, **response_kw)
return response
@@ -451,11 +501,11 @@ class HTTPSConnectionPool(HTTPConnectionPool):
When Python is compiled with the :mod:`ssl` module, then
:class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
- instead of :class:httplib.HTTPSConnection`.
+ instead of :class:`httplib.HTTPSConnection`.
- The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters
+ The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, and ``ssl_version``
are only used if :mod:`ssl` is available and are fed into
- :meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket.
+ :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket.
"""
scheme = 'https'
@@ -464,15 +514,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
strict=False, timeout=None, maxsize=1,
block=False, headers=None,
key_file=None, cert_file=None,
- cert_reqs='CERT_NONE', ca_certs=None):
+ cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None):
- super(HTTPSConnectionPool, self).__init__(host, port,
- strict, timeout, maxsize,
- block, headers)
+ HTTPConnectionPool.__init__(self, host, port,
+ strict, timeout, maxsize,
+ block, headers)
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = cert_reqs
self.ca_certs = ca_certs
+ self.ssl_version = ssl_version
def _new_conn(self):
"""
@@ -487,11 +538,21 @@ class HTTPSConnectionPool(HTTPConnectionPool):
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
- return HTTPSConnection(host=self.host, port=self.port)
+ return HTTPSConnection(host=self.host,
+ port=self.port,
+ strict=self.strict)
- connection = VerifiedHTTPSConnection(host=self.host, port=self.port)
+ connection = VerifiedHTTPSConnection(host=self.host,
+ port=self.port,
+ strict=self.strict)
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
+
+ if self.ssl_version is None:
+ connection.ssl_version = ssl.PROTOCOL_SSLv23
+ else:
+ connection.ssl_version = self.ssl_version
+
return connection
diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py
index 15c9699..c5eb962 100644
--- a/requests/packages/urllib3/exceptions.py
+++ b/requests/packages/urllib3/exceptions.py
@@ -18,21 +18,36 @@ class PoolError(HTTPError):
self.pool = pool
HTTPError.__init__(self, "%s: %s" % (pool, message))
+ def __reduce__(self):
+ # For pickling purposes.
+ return self.__class__, (None, self.url)
+
class SSLError(HTTPError):
"Raised when SSL certificate fails in an HTTPS connection."
pass
+class DecodeError(HTTPError):
+ "Raised when automatic decoding based on Content-Type fails."
+ pass
+
+
## Leaf Exceptions
class MaxRetryError(PoolError):
"Raised when the maximum number of retries is exceeded."
- def __init__(self, pool, url):
+ def __init__(self, pool, url, reason=None):
+ self.reason = reason
+
message = "Max retries exceeded with url: %s" % url
- PoolError.__init__(self, pool, message)
+ if reason:
+ message += " (Caused by %s: %s)" % (type(reason), reason)
+ else:
+ message += " (Caused by redirect)"
+ PoolError.__init__(self, pool, message)
self.url = url
@@ -57,11 +72,16 @@ class EmptyPoolError(PoolError):
pass
+class ClosedPoolError(PoolError):
+ "Raised when a request enters a pool after the pool has been closed."
+ pass
+
+
class LocationParseError(ValueError, HTTPError):
"Raised when get_host or similar fails to parse the URL input."
def __init__(self, location):
message = "Failed to parse: %s" % location
- super(LocationParseError, self).__init__(self, message)
+ HTTPError.__init__(self, message)
self.location = location
diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py
index 344a103..8d900bd 100644
--- a/requests/packages/urllib3/filepost.py
+++ b/requests/packages/urllib3/filepost.py
@@ -7,11 +7,7 @@
import codecs
import mimetypes
-try:
- from mimetools import choose_boundary
-except ImportError:
- from .packages.mimetools_choose_boundary import choose_boundary
-
+from uuid import uuid4
from io import BytesIO
from .packages import six
@@ -20,6 +16,13 @@ from .packages.six import b
writer = codecs.lookup('utf-8')[3]
+def choose_boundary():
+ """
+ Our embarassingly-simple replacement for mimetools.choose_boundary.
+ """
+ return uuid4().hex
+
+
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
@@ -38,13 +41,16 @@ def iter_fields(fields):
def encode_multipart_formdata(fields, boundary=None):
"""
- Encode a dictionary of ``fields`` using the multipart/form-data mime format.
+ Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
:param fields:
- Dictionary of fields or list of (key, value) field tuples. The key is
- treated as the field name, and the value as the body of the form-data
- bytes. If the value is a tuple of two elements, then the first element
- is treated as the filename of the form-data section.
+ Dictionary of fields or list of (key, value) or (key, value, MIME type)
+ field tuples. The key is treated as the field name, and the value as
+ the body of the form-data bytes. If the value is a tuple of two
+ elements, then the first element is treated as the filename of the
+ form-data section and a suitable MIME type is guessed based on the
+ filename. If the value is a tuple of three elements, then the third
+ element is treated as an explicit MIME type of the form-data section.
Field names and filenames must be unicode.
@@ -60,16 +66,20 @@ def encode_multipart_formdata(fields, boundary=None):
body.write(b('--%s\r\n' % (boundary)))
if isinstance(value, tuple):
- filename, data = value
+ if len(value) == 3:
+ filename, data, content_type = value
+ else:
+ filename, data = value
+ content_type = get_content_type(filename)
writer(body).write('Content-Disposition: form-data; name="%s"; '
'filename="%s"\r\n' % (fieldname, filename))
body.write(b('Content-Type: %s\r\n\r\n' %
- (get_content_type(filename))))
+ (content_type,)))
else:
data = value
writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
% (fieldname))
- body.write(b'Content-Type: text/plain\r\n\r\n')
+ body.write(b'\r\n')
if isinstance(data, int):
data = str(data) # Backwards compatibility
diff --git a/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py b/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py
deleted file mode 100644
index a0109ab..0000000
--- a/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py
+++ /dev/null
@@ -1,47 +0,0 @@
-"""The function mimetools.choose_boundary() from Python 2.7, which seems to
-have disappeared in Python 3 (although email.generator._make_boundary() might
-work as a replacement?).
-
-Tweaked to use lock from threading rather than thread.
-"""
-import os
-from threading import Lock
-_counter_lock = Lock()
-
-_counter = 0
-def _get_next_counter():
- global _counter
- with _counter_lock:
- _counter += 1
- return _counter
-
-_prefix = None
-
-def choose_boundary():
- """Return a string usable as a multipart boundary.
-
- The string chosen is unique within a single program run, and
- incorporates the user id (if available), process id (if available),
- and current time. So it's very unlikely the returned string appears
- in message text, but there's no guarantee.
-
- The boundary contains dots so you have to quote it in the header."""
-
- global _prefix
- import time
- if _prefix is None:
- import socket
- try:
- hostid = socket.gethostbyname(socket.gethostname())
- except socket.gaierror:
- hostid = '127.0.0.1'
- try:
- uid = repr(os.getuid())
- except AttributeError:
- uid = '1'
- try:
- pid = repr(os.getpid())
- except AttributeError:
- pid = '1'
- _prefix = hostid + '.' + uid + '.' + pid
- return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter())
diff --git a/requests/packages/urllib3/packages/ordered_dict.py b/requests/packages/urllib3/packages/ordered_dict.py
new file mode 100644
index 0000000..7f8ee15
--- /dev/null
+++ b/requests/packages/urllib3/packages/ordered_dict.py
@@ -0,0 +1,260 @@
+# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
+# Passes Python2.7's test suite and incorporates all the latest updates.
+# Copyright 2009 Raymond Hettinger, released under the MIT License.
+# http://code.activestate.com/recipes/576693/
+
+try:
+ from thread import get_ident as _get_ident
+except ImportError:
+ from dummy_thread import get_ident as _get_ident
+
+try:
+ from _abcoll import KeysView, ValuesView, ItemsView
+except ImportError:
+ pass
+
+
+class OrderedDict(dict):
+ 'Dictionary that remembers insertion order'
+ # An inherited dict maps keys to values.
+ # The inherited dict provides __getitem__, __len__, __contains__, and get.
+ # The remaining methods are order-aware.
+ # Big-O running times for all methods are the same as for regular dictionaries.
+
+ # The internal self.__map dictionary maps keys to links in a doubly linked list.
+ # The circular doubly linked list starts and ends with a sentinel element.
+ # The sentinel element never gets deleted (this simplifies the algorithm).
+ # Each link is stored as a list of length three: [PREV, NEXT, KEY].
+
+ def __init__(self, *args, **kwds):
+ '''Initialize an ordered dictionary. Signature is the same as for
+ regular dictionaries, but keyword arguments are not recommended
+ because their insertion order is arbitrary.
+
+ '''
+ if len(args) > 1:
+ raise TypeError('expected at most 1 arguments, got %d' % len(args))
+ try:
+ self.__root
+ except AttributeError:
+ self.__root = root = [] # sentinel node
+ root[:] = [root, root, None]
+ self.__map = {}
+ self.__update(*args, **kwds)
+
+ def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
+ 'od.__setitem__(i, y) <==> od[i]=y'
+ # Setting a new item creates a new link which goes at the end of the linked
+ # list, and the inherited dictionary is updated with the new key/value pair.
+ if key not in self:
+ root = self.__root
+ last = root[0]
+ last[1] = root[0] = self.__map[key] = [last, root, key]
+ dict_setitem(self, key, value)
+
+ def __delitem__(self, key, dict_delitem=dict.__delitem__):
+ 'od.__delitem__(y) <==> del od[y]'
+ # Deleting an existing item uses self.__map to find the link which is
+ # then removed by updating the links in the predecessor and successor nodes.
+ dict_delitem(self, key)
+ link_prev, link_next, key = self.__map.pop(key)
+ link_prev[1] = link_next
+ link_next[0] = link_prev
+
+ def __iter__(self):
+ 'od.__iter__() <==> iter(od)'
+ root = self.__root
+ curr = root[1]
+ while curr is not root:
+ yield curr[2]
+ curr = curr[1]
+
+ def __reversed__(self):
+ 'od.__reversed__() <==> reversed(od)'
+ root = self.__root
+ curr = root[0]
+ while curr is not root:
+ yield curr[2]
+ curr = curr[0]
+
+ def clear(self):
+ 'od.clear() -> None. Remove all items from od.'
+ try:
+ for node in self.__map.itervalues():
+ del node[:]
+ root = self.__root
+ root[:] = [root, root, None]
+ self.__map.clear()
+ except AttributeError:
+ pass
+ dict.clear(self)
+
+ def popitem(self, last=True):
+ '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+ Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+ '''
+ if not self:
+ raise KeyError('dictionary is empty')
+ root = self.__root
+ if last:
+ link = root[0]
+ link_prev = link[0]
+ link_prev[1] = root
+ root[0] = link_prev
+ else:
+ link = root[1]
+ link_next = link[1]
+ root[1] = link_next
+ link_next[0] = root
+ key = link[2]
+ del self.__map[key]
+ value = dict.pop(self, key)
+ return key, value
+
+ # -- the following methods do not depend on the internal structure --
+
+ def keys(self):
+ 'od.keys() -> list of keys in od'
+ return list(self)
+
+ def values(self):
+ 'od.values() -> list of values in od'
+ return [self[key] for key in self]
+
+ def items(self):
+ 'od.items() -> list of (key, value) pairs in od'
+ return [(key, self[key]) for key in self]
+
+ def iterkeys(self):
+ 'od.iterkeys() -> an iterator over the keys in od'
+ return iter(self)
+
+ def itervalues(self):
+ 'od.itervalues -> an iterator over the values in od'
+ for k in self:
+ yield self[k]
+
+ def iteritems(self):
+ 'od.iteritems -> an iterator over the (key, value) items in od'
+ for k in self:
+ yield (k, self[k])
+
+ def update(*args, **kwds):
+ '''od.update(E, **F) -> None. Update od from dict/iterable E and F.
+
+ If E is a dict instance, does: for k in E: od[k] = E[k]
+ If E has a .keys() method, does: for k in E.keys(): od[k] = E[k]
+ Or if E is an iterable of items, does: for k, v in E: od[k] = v
+ In either case, this is followed by: for k, v in F.items(): od[k] = v
+
+ '''
+ if len(args) > 2:
+ raise TypeError('update() takes at most 2 positional '
+ 'arguments (%d given)' % (len(args),))
+ elif not args:
+ raise TypeError('update() takes at least 1 argument (0 given)')
+ self = args[0]
+ # Make progressively weaker assumptions about "other"
+ other = ()
+ if len(args) == 2:
+ other = args[1]
+ if isinstance(other, dict):
+ for key in other:
+ self[key] = other[key]
+ elif hasattr(other, 'keys'):
+ for key in other.keys():
+ self[key] = other[key]
+ else:
+ for key, value in other:
+ self[key] = value
+ for key, value in kwds.items():
+ self[key] = value
+
+ __update = update # let subclasses override update without breaking __init__
+
+ __marker = object()
+
+ def pop(self, key, default=__marker):
+ '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+ If key is not found, d is returned if given, otherwise KeyError is raised.
+
+ '''
+ if key in self:
+ result = self[key]
+ del self[key]
+ return result
+ if default is self.__marker:
+ raise KeyError(key)
+ return default
+
+ def setdefault(self, key, default=None):
+ 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+ if key in self:
+ return self[key]
+ self[key] = default
+ return default
+
+ def __repr__(self, _repr_running={}):
+ 'od.__repr__() <==> repr(od)'
+ call_key = id(self), _get_ident()
+ if call_key in _repr_running:
+ return '...'
+ _repr_running[call_key] = 1
+ try:
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, self.items())
+ finally:
+ del _repr_running[call_key]
+
+ def __reduce__(self):
+ 'Return state information for pickling'
+ items = [[k, self[k]] for k in self]
+ inst_dict = vars(self).copy()
+ for k in vars(OrderedDict()):
+ inst_dict.pop(k, None)
+ if inst_dict:
+ return (self.__class__, (items,), inst_dict)
+ return self.__class__, (items,)
+
+ def copy(self):
+ 'od.copy() -> a shallow copy of od'
+ return self.__class__(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
+ and values equal to v (which defaults to None).
+
+ '''
+ d = cls()
+ for key in iterable:
+ d[key] = value
+ return d
+
+ def __eq__(self, other):
+ '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
+ while comparison to a regular mapping is order-insensitive.
+
+ '''
+ if isinstance(other, OrderedDict):
+ return len(self)==len(other) and self.items() == other.items()
+ return dict.__eq__(self, other)
+
+ def __ne__(self, other):
+ return not self == other
+
+ # -- the following methods are only used in Python 2.7 --
+
+ def viewkeys(self):
+ "od.viewkeys() -> a set-like object providing a view on od's keys"
+ return KeysView(self)
+
+ def viewvalues(self):
+ "od.viewvalues() -> an object providing a view on od's values"
+ return ValuesView(self)
+
+ def viewitems(self):
+ "od.viewitems() -> a set-like object providing a view on od's items"
+ return ItemsView(self)
diff --git a/requests/packages/urllib3/packages/six.py b/requests/packages/urllib3/packages/six.py
index a64f6fb..27d8011 100644
--- a/requests/packages/urllib3/packages/six.py
+++ b/requests/packages/urllib3/packages/six.py
@@ -24,7 +24,7 @@ import sys
import types
__author__ = "Benjamin Peterson <benjamin@python.org>"
-__version__ = "1.1.0"
+__version__ = "1.2.0" # Revision 41c74fef2ded
# True if we are running on Python 3.
@@ -45,19 +45,23 @@ else:
text_type = unicode
binary_type = str
- # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
- class X(object):
- def __len__(self):
- return 1 << 31
- try:
- len(X())
- except OverflowError:
- # 32-bit
+ if sys.platform.startswith("java"):
+ # Jython always uses 32 bits.
MAXSIZE = int((1 << 31) - 1)
else:
- # 64-bit
- MAXSIZE = int((1 << 63) - 1)
- del X
+ # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+ class X(object):
+ def __len__(self):
+ return 1 << 31
+ try:
+ len(X())
+ except OverflowError:
+ # 32-bit
+ MAXSIZE = int((1 << 31) - 1)
+ else:
+ # 64-bit
+ MAXSIZE = int((1 << 63) - 1)
+ del X
def _add_doc(func, doc):
@@ -132,6 +136,7 @@ class _MovedItems(types.ModuleType):
_moved_attributes = [
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+ MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
MovedAttribute("reduce", "__builtin__", "functools"),
@@ -178,7 +183,7 @@ for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr)
del attr
-moves = sys.modules["six.moves"] = _MovedItems("moves")
+moves = sys.modules[__name__ + ".moves"] = _MovedItems("moves")
def add_move(move):
@@ -219,12 +224,19 @@ else:
_iteritems = "iteritems"
+try:
+ advance_iterator = next
+except NameError:
+ def advance_iterator(it):
+ return it.next()
+next = advance_iterator
+
+
if PY3:
def get_unbound_function(unbound):
return unbound
-
- advance_iterator = next
+ Iterator = object
def callable(obj):
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
@@ -232,9 +244,10 @@ else:
def get_unbound_function(unbound):
return unbound.im_func
+ class Iterator(object):
- def advance_iterator(it):
- return it.next()
+ def next(self):
+ return type(self).__next__(self)
callable = callable
_add_doc(get_unbound_function,
@@ -249,15 +262,15 @@ get_function_defaults = operator.attrgetter(_func_defaults)
def iterkeys(d):
"""Return an iterator over the keys of a dictionary."""
- return getattr(d, _iterkeys)()
+ return iter(getattr(d, _iterkeys)())
def itervalues(d):
"""Return an iterator over the values of a dictionary."""
- return getattr(d, _itervalues)()
+ return iter(getattr(d, _itervalues)())
def iteritems(d):
"""Return an iterator over the (key, value) pairs of a dictionary."""
- return getattr(d, _iteritems)()
+ return iter(getattr(d, _iteritems)())
if PY3:
diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py
index 310ea21..a124202 100644
--- a/requests/packages/urllib3/poolmanager.py
+++ b/requests/packages/urllib3/poolmanager.py
@@ -8,9 +8,9 @@ import logging
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import get_host, connection_from_url, port_by_scheme
-from .exceptions import HostChangedError
+from .connectionpool import connection_from_url, port_by_scheme
from .request import RequestMethods
+from .util import parse_url
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
@@ -30,8 +30,12 @@ class PoolManager(RequestMethods):
necessary connection pools for you.
:param num_pools:
- Number of connection pools to cache before discarding the least recently
- used pool.
+ Number of connection pools to cache before discarding the least
+ recently used pool.
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
:param \**connection_pool_kw:
Additional parameters are used to create fresh
@@ -40,27 +44,38 @@ class PoolManager(RequestMethods):
Example: ::
>>> manager = PoolManager(num_pools=2)
- >>> r = manager.urlopen("http://google.com/")
- >>> r = manager.urlopen("http://google.com/mail")
- >>> r = manager.urlopen("http://yahoo.com/")
+ >>> r = manager.request('GET', 'http://google.com/')
+ >>> r = manager.request('GET', 'http://google.com/mail')
+ >>> r = manager.request('GET', 'http://yahoo.com/')
>>> len(manager.pools)
2
"""
- # TODO: Make sure there are no memory leaks here.
-
- def __init__(self, num_pools=10, **connection_pool_kw):
+ def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
+ RequestMethods.__init__(self, headers)
self.connection_pool_kw = connection_pool_kw
- self.pools = RecentlyUsedContainer(num_pools)
+ self.pools = RecentlyUsedContainer(num_pools,
+ dispose_func=lambda p: p.close())
- def connection_from_host(self, host, port=80, scheme='http'):
+ def clear(self):
+ """
+ Empty our store of pools and direct them all to close.
+
+ This will not affect in-flight connections, but they will not be
+ re-used after completion.
+ """
+ self.pools.clear()
+
+ def connection_from_host(self, host, port=None, scheme='http'):
"""
Get a :class:`ConnectionPool` based on the host, port, and scheme.
- Note that an appropriate ``port`` value is required here to normalize
- connection pools in our container most effectively.
+ If ``port`` isn't given, it will be derived from the ``scheme`` using
+ ``urllib3.connectionpool.port_by_scheme``.
"""
+ port = port or port_by_scheme.get(scheme, 80)
+
pool_key = (scheme, host, port)
# If the scheme, host, or port doesn't match existing open connections,
@@ -86,26 +101,38 @@ class PoolManager(RequestMethods):
Additional parameters are taken from the :class:`.PoolManager`
constructor.
"""
- scheme, host, port = get_host(url)
-
- port = port or port_by_scheme.get(scheme, 80)
-
- return self.connection_from_host(host, port=port, scheme=scheme)
+ u = parse_url(url)
+ return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
- def urlopen(self, method, url, **kw):
+ def urlopen(self, method, url, redirect=True, **kw):
"""
- Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`.
+ Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
+ with custom cross-host redirect logic and only sends the request-uri
+ portion of the ``url``.
- ``url`` must be absolute, such that an appropriate
+ The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
- conn = self.connection_from_url(url)
- try:
- return conn.urlopen(method, url, **kw)
+ u = parse_url(url)
+ conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
+
+ kw['assert_same_host'] = False
+ kw['redirect'] = False
+ if 'headers' not in kw:
+ kw['headers'] = self.headers
- except HostChangedError as e:
- kw['retries'] = e.retries # Persist retries countdown
- return self.urlopen(method, e.url, **kw)
+ response = conn.urlopen(method, u.request_uri, **kw)
+
+ redirect_location = redirect and response.get_redirect_location()
+ if not redirect_location:
+ return response
+
+ if response.status == 303:
+ method = 'GET'
+
+ log.info("Redirecting %s -> %s" % (url, redirect_location))
+ kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
+ return self.urlopen(method, redirect_location, **kw)
class ProxyManager(RequestMethods):
@@ -118,13 +145,11 @@ class ProxyManager(RequestMethods):
self.proxy_pool = proxy_pool
def _set_proxy_headers(self, headers=None):
- headers = headers or {}
-
- # Same headers are curl passes for --proxy1.0
- headers['Accept'] = '*/*'
- headers['Proxy-Connection'] = 'Keep-Alive'
+ headers_ = {'Accept': '*/*'}
+ if headers:
+ headers_.update(headers)
- return headers
+ return headers_
def urlopen(self, method, url, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py
index 569ac96..2b4704e 100644
--- a/requests/packages/urllib3/request.py
+++ b/requests/packages/urllib3/request.py
@@ -36,12 +36,20 @@ class RequestMethods(object):
:meth:`.request` is for making any kind of request, it will look up the
appropriate encoding format and use one of the above two methods to make
the request.
+
+ Initializer parameters:
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
"""
_encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS'])
-
_encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE'])
+ def __init__(self, headers=None):
+ self.headers = headers or {}
+
def urlopen(self, method, url, body=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**kw): # Abstract
@@ -97,13 +105,16 @@ class RequestMethods(object):
such as with OAuth.
Supports an optional ``fields`` parameter of key/value strings AND
- key/filetuple. A filetuple is a (filename, data) tuple. For example: ::
+ key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
+ the MIME type is optional. For example: ::
fields = {
'foo': 'bar',
'fakefile': ('foofile.txt', 'contents of foofile'),
'realfile': ('barfile.txt', open('realfile').read()),
- 'nonamefile': ('contents of nonamefile field'),
+ 'typedfile': ('bazfile.bin', open('bazfile').read(),
+ 'image/jpeg'),
+ 'nonamefile': 'contents of nonamefile field',
}
When uploading a file, providing a filename (the first parameter of the
@@ -121,8 +132,11 @@ class RequestMethods(object):
body, content_type = (urlencode(fields or {}),
'application/x-www-form-urlencoded')
- headers = headers or {}
- headers.update({'Content-Type': content_type})
+ if headers is None:
+ headers = self.headers
+
+ headers_ = {'Content-Type': content_type}
+ headers_.update(headers)
- return self.urlopen(method, url, body=body, headers=headers,
+ return self.urlopen(method, url, body=body, headers=headers_,
**urlopen_kw)
diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py
index 5fab824..833be62 100644
--- a/requests/packages/urllib3/response.py
+++ b/requests/packages/urllib3/response.py
@@ -10,7 +10,7 @@ import zlib
from io import BytesIO
-from .exceptions import HTTPError
+from .exceptions import DecodeError
from .packages.six import string_types as basestring
@@ -130,7 +130,9 @@ class HTTPResponse(object):
after having ``.read()`` the file object. (Overridden if ``amt`` is
set.)
"""
- content_encoding = self.headers.get('content-encoding')
+ # Note: content-encoding value should be case-insensitive, per RFC 2616
+ # Section 3.5
+ content_encoding = self.headers.get('content-encoding', '').lower()
decoder = self.CONTENT_DECODERS.get(content_encoding)
if decode_content is None:
decode_content = self._decode_content
@@ -148,9 +150,9 @@ class HTTPResponse(object):
try:
if decode_content and decoder:
data = decoder(data)
- except IOError:
- raise HTTPError("Received response with content-encoding: %s, but "
- "failed to decode it." % content_encoding)
+ except (IOError, zlib.error):
+ raise DecodeError("Received response with content-encoding: %s, but "
+ "failed to decode it." % content_encoding)
if cache_content:
self._body = data
diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py
index 2684a2f..8d8654f 100644
--- a/requests/packages/urllib3/util.py
+++ b/requests/packages/urllib3/util.py
@@ -6,20 +6,179 @@
from base64 import b64encode
+from collections import namedtuple
+from socket import error as SocketError
try:
from select import poll, POLLIN
-except ImportError: # `poll` doesn't exist on OSX and other platforms
+except ImportError: # `poll` doesn't exist on OSX and other platforms
poll = False
try:
from select import select
- except ImportError: # `select` doesn't exist on AppEngine.
+ except ImportError: # `select` doesn't exist on AppEngine.
select = False
+try: # Test for SSL features
+ SSLContext = None
+ HAS_SNI = False
+
+ from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23
+ from ssl import SSLContext # Modern SSL?
+ from ssl import HAS_SNI # Has SNI?
+except ImportError:
+ pass
+
+
from .packages import six
from .exceptions import LocationParseError
+class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])):
+ """
+ Datastructure for representing an HTTP URL. Used as a return value for
+ :func:`parse_url`.
+ """
+ slots = ()
+
+ def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None):
+ return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment)
+
+ @property
+ def hostname(self):
+ """For backwards-compatibility with urlparse. We're nice like that."""
+ return self.host
+
+ @property
+ def request_uri(self):
+ """Absolute path including the query string."""
+ uri = self.path or '/'
+
+ if self.query is not None:
+ uri += '?' + self.query
+
+ return uri
+
+
+def split_first(s, delims):
+ """
+ Given a string and an iterable of delimiters, split on the first found
+ delimiter. Return two split parts and the matched delimiter.
+
+ If not found, then the first part is the full input string.
+
+ Example: ::
+
+ >>> split_first('foo/bar?baz', '?/=')
+ ('foo', 'bar?baz', '/')
+ >>> split_first('foo/bar?baz', '123')
+ ('foo/bar?baz', '', None)
+
+ Scales linearly with number of delims. Not ideal for large number of delims.
+ """
+ min_idx = None
+ min_delim = None
+ for d in delims:
+ idx = s.find(d)
+ if idx < 0:
+ continue
+
+ if min_idx is None or idx < min_idx:
+ min_idx = idx
+ min_delim = d
+
+ if min_idx is None or min_idx < 0:
+ return s, '', None
+
+ return s[:min_idx], s[min_idx+1:], min_delim
+
+
+def parse_url(url):
+ """
+ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
+ performed to parse incomplete urls. Fields not provided will be None.
+
+ Partly backwards-compatible with :mod:`urlparse`.
+
+ Example: ::
+
+ >>> parse_url('http://google.com/mail/')
+ Url(scheme='http', host='google.com', port=None, path='/', ...)
+ >>> parse_url('google.com:80')
+ Url(scheme=None, host='google.com', port=80, path=None, ...)
+ >>> parse_url('/foo?bar')
+ Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
+ """
+
+ # While this code has overlap with stdlib's urlparse, it is much
+ # simplified for our needs and less annoying.
+ # Additionally, this imeplementations does silly things to be optimal
+ # on CPython.
+
+ scheme = None
+ auth = None
+ host = None
+ port = None
+ path = None
+ fragment = None
+ query = None
+
+ # Scheme
+ if '://' in url:
+ scheme, url = url.split('://', 1)
+
+ # Find the earliest Authority Terminator
+ # (http://tools.ietf.org/html/rfc3986#section-3.2)
+ url, path_, delim = split_first(url, ['/', '?', '#'])
+
+ if delim:
+ # Reassemble the path
+ path = delim + path_
+
+ # Auth
+ if '@' in url:
+ auth, url = url.split('@', 1)
+
+ # IPv6
+ if url and url[0] == '[':
+ host, url = url[1:].split(']', 1)
+
+ # Port
+ if ':' in url:
+ _host, port = url.split(':', 1)
+
+ if not host:
+ host = _host
+
+ if not port.isdigit():
+ raise LocationParseError("Failed to parse: %s" % url)
+
+ port = int(port)
+
+ elif not host and url:
+ host = url
+
+ if not path:
+ return Url(scheme, auth, host, port, path, query, fragment)
+
+ # Fragment
+ if '#' in path:
+ path, fragment = path.split('#', 1)
+
+ # Query
+ if '?' in path:
+ path, query = path.split('?', 1)
+
+ return Url(scheme, auth, host, port, path, query, fragment)
+
+
+def get_host(url):
+ """
+ Deprecated. Use :func:`.parse_url` instead.
+ """
+ p = parse_url(url)
+ return p.scheme or 'http', p.hostname, p.port
+
+
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
basic_auth=None):
"""
@@ -72,60 +231,28 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
return headers
-def get_host(url):
- """
- Given a url, return its scheme, host and port (None if it's not there).
-
- For example: ::
-
- >>> get_host('http://google.com/mail/')
- ('http', 'google.com', None)
- >>> get_host('google.com:80')
- ('http', 'google.com', 80)
- """
-
- # This code is actually similar to urlparse.urlsplit, but much
- # simplified for our needs.
- port = None
- scheme = 'http'
-
- if '://' in url:
- scheme, url = url.split('://', 1)
- if '/' in url:
- url, _path = url.split('/', 1)
- if '@' in url:
- _auth, url = url.split('@', 1)
- if ':' in url:
- url, port = url.split(':', 1)
-
- if not port.isdigit():
- raise LocationParseError("Failed to parse: %s" % url)
-
- port = int(port)
-
- return scheme, url, port
-
-
-
def is_connection_dropped(conn):
"""
Returns True if the connection is dropped and should be closed.
:param conn:
- ``HTTPConnection`` object.
+ :class:`httplib.HTTPConnection` object.
Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us.
"""
sock = getattr(conn, 'sock', False)
- if not sock: #Platform-specific: AppEngine
+ if not sock: # Platform-specific: AppEngine
return False
if not poll: # Platform-specific
- if not select: #Platform-specific: AppEngine
+ if not select: # Platform-specific: AppEngine
return False
- return select([sock], [], [], 0.0)[0]
+ try:
+ return select([sock], [], [], 0.0)[0]
+ except SocketError:
+ return True
# This version is better on platforms that support it.
p = poll()
@@ -134,3 +261,38 @@ def is_connection_dropped(conn):
if fno == sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
+
+
+if SSLContext is not None: # Python 3.2+
+ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+ ca_certs=None, server_hostname=None,
+ ssl_version=PROTOCOL_SSLv23):
+ """
+ All arguments except `server_hostname` have the same meaning as for
+ :func:`ssl.wrap_socket`
+
+ :param server_hostname:
+ Hostname of the expected certificate
+ """
+ context = SSLContext(ssl_version)
+ context.verify_mode = cert_reqs
+ if ca_certs:
+ try:
+ context.load_verify_locations(ca_certs)
+ except TypeError as e: # Reraise as SSLError
+ # FIXME: This block needs a test.
+ raise SSLError(e)
+ if certfile:
+ # FIXME: This block needs a test.
+ context.load_cert_chain(certfile, keyfile)
+ if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI
+ return context.wrap_socket(sock, server_hostname=server_hostname)
+ return context.wrap_socket(sock)
+
+else: # Python 3.1 and earlier
+ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+ ca_certs=None, server_hostname=None,
+ ssl_version=PROTOCOL_SSLv23):
+ return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
+ ca_certs=ca_certs, cert_reqs=cert_reqs,
+ ssl_version=ssl_version)
generated by cgit v1.2.3 (git 2.25.4) at 2024-06-25 23:21:51 +0000