aboutsummaryrefslogtreecommitdiff
path: root/requests/async.py
diff options
context:
space:
mode:
Diffstat (limited to 'requests/async.py')
-rw-r--r--requests/async.py47
1 files changed, 26 insertions, 21 deletions
diff --git a/requests/async.py b/requests/async.py
index db25f6a..8bafb1e 100644
--- a/requests/async.py
+++ b/requests/async.py
@@ -12,6 +12,7 @@ by gevent. All API methods return a ``Request`` instance (as opposed to
try:
import gevent
from gevent import monkey as curious_george
+ from gevent.pool import Pool
except ImportError:
raise RuntimeError('Gevent is required for requests.async.')
@@ -19,25 +20,28 @@ except ImportError:
curious_george.patch_all(thread=False)
from . import api
-from .hooks import dispatch_hook
__all__ = (
'map',
- 'get', 'head', 'post', 'put', 'patch', 'delete', 'request'
+ 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
)
-def _patched(f):
+def patched(f):
"""Patches a given API function to not send."""
def wrapped(*args, **kwargs):
- return f(*args, return_response=False, **kwargs)
+
+ kwargs['return_response'] = False
+ kwargs['prefetch'] = True
+
+ return f(*args, **kwargs)
return wrapped
-def _send(r, pools=None):
+def send(r, pools=None):
"""Sends a given Request object."""
if pools:
@@ -45,34 +49,35 @@ def _send(r, pools=None):
r.send()
- # Post-request hook.
- r = dispatch_hook('post_request', r.hooks, r)
-
- # Response manipulation hook.
- r.response = dispatch_hook('response', r.hooks, r.response)
-
return r.response
# Patched requests.api functions.
-get = _patched(api.get)
-head = _patched(api.head)
-post = _patched(api.post)
-put = _patched(api.put)
-patch = _patched(api.patch)
-delete = _patched(api.delete)
-request = _patched(api.request)
+get = patched(api.get)
+options = patched(api.options)
+head = patched(api.head)
+post = patched(api.post)
+put = patched(api.put)
+patch = patched(api.patch)
+delete = patched(api.delete)
+request = patched(api.request)
-def map(requests, prefetch=True):
+def map(requests, prefetch=True, size=None):
"""Concurrently converts a list of Requests to Responses.
:param requests: a collection of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
+ :param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
"""
- jobs = [gevent.spawn(_send, r) for r in requests]
- gevent.joinall(jobs)
+ if size:
+ pool = Pool(size)
+ pool.map(send, requests)
+ pool.join()
+ else:
+ jobs = [gevent.spawn(send, r) for r in requests]
+ gevent.joinall(jobs)
if prefetch:
[r.response.content for r in requests]