diff options
Diffstat (limited to 'requests/async.py')
-rw-r--r-- | requests/async.py | 47 |
1 files changed, 26 insertions, 21 deletions
diff --git a/requests/async.py b/requests/async.py index db25f6a..8bafb1e 100644 --- a/requests/async.py +++ b/requests/async.py @@ -12,6 +12,7 @@ by gevent. All API methods return a ``Request`` instance (as opposed to try: import gevent from gevent import monkey as curious_george + from gevent.pool import Pool except ImportError: raise RuntimeError('Gevent is required for requests.async.') @@ -19,25 +20,28 @@ except ImportError: curious_george.patch_all(thread=False) from . import api -from .hooks import dispatch_hook __all__ = ( 'map', - 'get', 'head', 'post', 'put', 'patch', 'delete', 'request' + 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' ) -def _patched(f): +def patched(f): """Patches a given API function to not send.""" def wrapped(*args, **kwargs): - return f(*args, return_response=False, **kwargs) + + kwargs['return_response'] = False + kwargs['prefetch'] = True + + return f(*args, **kwargs) return wrapped -def _send(r, pools=None): +def send(r, pools=None): """Sends a given Request object.""" if pools: @@ -45,34 +49,35 @@ def _send(r, pools=None): r.send() - # Post-request hook. - r = dispatch_hook('post_request', r.hooks, r) - - # Response manipulation hook. - r.response = dispatch_hook('response', r.hooks, r.response) - return r.response # Patched requests.api functions. -get = _patched(api.get) -head = _patched(api.head) -post = _patched(api.post) -put = _patched(api.put) -patch = _patched(api.patch) -delete = _patched(api.delete) -request = _patched(api.request) +get = patched(api.get) +options = patched(api.options) +head = patched(api.head) +post = patched(api.post) +put = patched(api.put) +patch = patched(api.patch) +delete = patched(api.delete) +request = patched(api.request) -def map(requests, prefetch=True): +def map(requests, prefetch=True, size=None): """Concurrently converts a list of Requests to Responses. :param requests: a collection of Request objects. :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. If None, no throttling occurs. """ - jobs = [gevent.spawn(_send, r) for r in requests] - gevent.joinall(jobs) + if size: + pool = Pool(size) + pool.map(send, requests) + pool.join() + else: + jobs = [gevent.spawn(send, r) for r in requests] + gevent.joinall(jobs) if prefetch: [r.response.content for r in requests] |