aboutsummaryrefslogtreecommitdiff
path: root/urllib3/poolmanager.py
diff options
context:
space:
mode:
Diffstat (limited to 'urllib3/poolmanager.py')
-rw-r--r--urllib3/poolmanager.py64
1 files changed, 42 insertions, 22 deletions
diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py
index 310ea21..8f5b54c 100644
--- a/urllib3/poolmanager.py
+++ b/urllib3/poolmanager.py
@@ -8,9 +8,9 @@ import logging
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import get_host, connection_from_url, port_by_scheme
-from .exceptions import HostChangedError
+from .connectionpool import connection_from_url, port_by_scheme
from .request import RequestMethods
+from .util import parse_url
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
@@ -48,19 +48,29 @@ class PoolManager(RequestMethods):
"""
- # TODO: Make sure there are no memory leaks here.
-
def __init__(self, num_pools=10, **connection_pool_kw):
self.connection_pool_kw = connection_pool_kw
- self.pools = RecentlyUsedContainer(num_pools)
+ self.pools = RecentlyUsedContainer(num_pools,
+ dispose_func=lambda p: p.close())
+
+ def clear(self):
+ """
+ Empty our store of pools and direct them all to close.
+
+ This will not affect in-flight connections, but they will not be
+ re-used after completion.
+ """
+ self.pools.clear()
- def connection_from_host(self, host, port=80, scheme='http'):
+ def connection_from_host(self, host, port=None, scheme='http'):
"""
Get a :class:`ConnectionPool` based on the host, port, and scheme.
- Note that an appropriate ``port`` value is required here to normalize
- connection pools in our container most effectively.
+ If ``port`` isn't given, it will be derived from the ``scheme`` using
+ ``urllib3.connectionpool.port_by_scheme``.
"""
+ port = port or port_by_scheme.get(scheme, 80)
+
pool_key = (scheme, host, port)
# If the scheme, host, or port doesn't match existing open connections,
@@ -86,26 +96,36 @@ class PoolManager(RequestMethods):
Additional parameters are taken from the :class:`.PoolManager`
constructor.
"""
- scheme, host, port = get_host(url)
-
- port = port or port_by_scheme.get(scheme, 80)
-
- return self.connection_from_host(host, port=port, scheme=scheme)
+ u = parse_url(url)
+ return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
- def urlopen(self, method, url, **kw):
+ def urlopen(self, method, url, redirect=True, **kw):
"""
- Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`.
+ Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
+ with custom cross-host redirect logic and only sends the request-uri
+ portion of the ``url``.
- ``url`` must be absolute, such that an appropriate
+ The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
- conn = self.connection_from_url(url)
- try:
- return conn.urlopen(method, url, **kw)
+ u = parse_url(url)
+ conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
+
+ kw['assert_same_host'] = False
+ kw['redirect'] = False
+
+ response = conn.urlopen(method, u.request_uri, **kw)
+
+ redirect_location = redirect and response.get_redirect_location()
+ if not redirect_location:
+ return response
+
+ if response.status == 303:
+ method = 'GET'
- except HostChangedError as e:
- kw['retries'] = e.retries # Persist retries countdown
- return self.urlopen(method, e.url, **kw)
+ log.info("Redirecting %s -> %s" % (url, redirect_location))
+ kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
+ return self.urlopen(method, redirect_location, **kw)
class ProxyManager(RequestMethods):