diff options
Diffstat (limited to 'requests/async.py')
-rw-r--r-- | requests/async.py | 36 |
1 files changed, 27 insertions, 9 deletions
diff --git a/requests/async.py b/requests/async.py index 9488447..f2dad69 100644 --- a/requests/async.py +++ b/requests/async.py @@ -23,7 +23,7 @@ from . import api __all__ = ( - 'map', + 'map', 'imap', 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' ) @@ -46,15 +46,15 @@ def patched(f): return wrapped -def send(r, pool=None): - """Sends the request object using the specified pool. If a pool isn't +def send(r, pool=None, prefetch=False): + """Sends the request object using the specified pool. If a pool isn't specified this method blocks. Pools are useful because you can specify size and can hence limit concurrency.""" if pool != None: - return pool.spawn(r.send) + return pool.spawn(r.send, prefetch=prefetch) - return gevent.spawn(r.send) + return gevent.spawn(r.send, prefetch=prefetch) # Patched requests.api functions. @@ -79,10 +79,28 @@ def map(requests, prefetch=True, size=None): requests = list(requests) pool = Pool(size) if size else None - jobs = [send(r, pool) for r in requests] + jobs = [send(r, pool, prefetch=prefetch) for r in requests] gevent.joinall(jobs) - if prefetch: - [r.response.content for r in requests] + return [r.response for r in requests] - return [r.response for r in requests]
\ No newline at end of file + +def imap(requests, prefetch=True, size=2): + """Concurrently converts a generator object of Requests to + a generator of Responses. + + :param requests: a generator of Request objects. + :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. default is 2 + """ + + pool = Pool(size) + + def send(r): + r.send(prefetch) + return r.response + + for r in pool.imap_unordered(send, requests): + yield r + + pool.join()
\ No newline at end of file |