diff options
author | SVN-Git Migration <python-modules-team@lists.alioth.debian.org> | 2015-10-08 13:41:21 -0700 |
---|---|---|
committer | SVN-Git Migration <python-modules-team@lists.alioth.debian.org> | 2015-10-08 13:41:21 -0700 |
commit | e75853fc04102c7f72f2e955b63f9692c472f64a (patch) | |
tree | 9847c33530102dcf8f42bba6a562a7df34d651e6 /requests/async.py | |
parent | 365ef510aa3581a79709673e078401724601fc71 (diff) | |
download | python-requests-e75853fc04102c7f72f2e955b63f9692c472f64a.tar python-requests-e75853fc04102c7f72f2e955b63f9692c472f64a.tar.gz |
Imported Upstream version 0.10.8
Diffstat (limited to 'requests/async.py')
-rw-r--r-- | requests/async.py | 36 |
1 files changed, 27 insertions, 9 deletions
diff --git a/requests/async.py b/requests/async.py index 9488447..f2dad69 100644 --- a/requests/async.py +++ b/requests/async.py @@ -23,7 +23,7 @@ from . import api __all__ = ( - 'map', + 'map', 'imap', 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' ) @@ -46,15 +46,15 @@ def patched(f): return wrapped -def send(r, pool=None): - """Sends the request object using the specified pool. If a pool isn't +def send(r, pool=None, prefetch=False): + """Sends the request object using the specified pool. If a pool isn't specified this method blocks. Pools are useful because you can specify size and can hence limit concurrency.""" if pool != None: - return pool.spawn(r.send) + return pool.spawn(r.send, prefetch=prefetch) - return gevent.spawn(r.send) + return gevent.spawn(r.send, prefetch=prefetch) # Patched requests.api functions. @@ -79,10 +79,28 @@ def map(requests, prefetch=True, size=None): requests = list(requests) pool = Pool(size) if size else None - jobs = [send(r, pool) for r in requests] + jobs = [send(r, pool, prefetch=prefetch) for r in requests] gevent.joinall(jobs) - if prefetch: - [r.response.content for r in requests] + return [r.response for r in requests] - return [r.response for r in requests]
\ No newline at end of file + +def imap(requests, prefetch=True, size=2): + """Concurrently converts a generator object of Requests to + a generator of Responses. + + :param requests: a generator of Request objects. + :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. default is 2 + """ + + pool = Pool(size) + + def send(r): + r.send(prefetch) + return r.response + + for r in pool.imap_unordered(send, requests): + yield r + + pool.join()
\ No newline at end of file |