aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--HISTORY.rst20
-rw-r--r--MANIFEST.in2
-rw-r--r--PKG-INFO22
-rw-r--r--requests.egg-info/PKG-INFO22
-rw-r--r--requests.egg-info/SOURCES.txt2
-rw-r--r--requests/api.py85
-rw-r--r--requests/async.py93
-rw-r--r--requests/config.py1
-rw-r--r--requests/core.py10
-rw-r--r--requests/models.py160
-rw-r--r--requests/patches.py5
-rw-r--r--requests/utils.py192
-rwxr-xr-xtest_requests.py501
13 files changed, 965 insertions, 150 deletions
diff --git a/HISTORY.rst b/HISTORY.rst
index b09392b..053d37a 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -1,6 +1,25 @@
History
-------
+0.6.4 (2011-10-13)
+++++++++++++++++++
+
+* Automatic decoding of unicode, based on HTTP Headers.
+* New ``decode_unicode`` setting
+* Removal of ``r.read/close`` methods
+* New ``r.faw`` interface for advanced response usage.*
+* Automatic expansion of parameterized headers
+
+0.6.3 (2011-10-13)
+++++++++++++++++++
+
+* Beautiful ``requests.async`` module, for making async requests w/ gevent.
+
+0.6.2 (2011-10-09)
+++++++++++++++++++
+
+* GET/HEAD obeys allow_redirects=False
+
0.6.1 (2011-08-20)
++++++++++++++++++
@@ -28,6 +47,7 @@ History
* Improved https testing
* Bugfixes
+
0.5.1 (2011-07-23)
++++++++++++++++++
diff --git a/MANIFEST.in b/MANIFEST.in
index 94c50f7..39fbb99 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1 +1 @@
-include README.rst LICENSE HISTORY.rst \ No newline at end of file
+include README.rst LICENSE HISTORY.rst test_requests.py
diff --git a/PKG-INFO b/PKG-INFO
index 5240199..5cb0b72 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: requests
-Version: 0.6.1
+Version: 0.6.4
Summary: Python HTTP for Humans.
Home-page: http://python-requests.org
Author: Kenneth Reitz
@@ -115,6 +115,25 @@ Description: Requests: HTTP for Humans
History
-------
+ 0.6.4 (2011-10-13)
+ ++++++++++++++++++
+
+ * Automatic decoding of unicode, based on HTTP Headers.
+ * New ``decode_unicode`` setting
+ * Removal of ``r.read/close`` methods
+ * New ``r.faw`` interface for advanced response usage.*
+ * Automatic expansion of parameterized headers
+
+ 0.6.3 (2011-10-13)
+ ++++++++++++++++++
+
+ * Beautiful ``requests.async`` module, for making async requests w/ gevent.
+
+ 0.6.2 (2011-10-09)
+ ++++++++++++++++++
+
+ * GET/HEAD obeys allow_redirects=False
+
0.6.1 (2011-08-20)
++++++++++++++++++
@@ -142,6 +161,7 @@ Description: Requests: HTTP for Humans
* Improved https testing
* Bugfixes
+
0.5.1 (2011-07-23)
++++++++++++++++++
diff --git a/requests.egg-info/PKG-INFO b/requests.egg-info/PKG-INFO
index 5240199..5cb0b72 100644
--- a/requests.egg-info/PKG-INFO
+++ b/requests.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: requests
-Version: 0.6.1
+Version: 0.6.4
Summary: Python HTTP for Humans.
Home-page: http://python-requests.org
Author: Kenneth Reitz
@@ -115,6 +115,25 @@ Description: Requests: HTTP for Humans
History
-------
+ 0.6.4 (2011-10-13)
+ ++++++++++++++++++
+
+ * Automatic decoding of unicode, based on HTTP Headers.
+ * New ``decode_unicode`` setting
+ * Removal of ``r.read/close`` methods
+ * New ``r.faw`` interface for advanced response usage.*
+ * Automatic expansion of parameterized headers
+
+ 0.6.3 (2011-10-13)
+ ++++++++++++++++++
+
+ * Beautiful ``requests.async`` module, for making async requests w/ gevent.
+
+ 0.6.2 (2011-10-09)
+ ++++++++++++++++++
+
+ * GET/HEAD obeys allow_redirects=False
+
0.6.1 (2011-08-20)
++++++++++++++++++
@@ -142,6 +161,7 @@ Description: Requests: HTTP for Humans
* Improved https testing
* Bugfixes
+
0.5.1 (2011-07-23)
++++++++++++++++++
diff --git a/requests.egg-info/SOURCES.txt b/requests.egg-info/SOURCES.txt
index 65c3b1d..231a230 100644
--- a/requests.egg-info/SOURCES.txt
+++ b/requests.egg-info/SOURCES.txt
@@ -3,6 +3,7 @@ LICENSE
MANIFEST.in
README.rst
setup.py
+test_requests.py
requests/__init__.py
requests/api.py
requests/async.py
@@ -12,7 +13,6 @@ requests/exceptions.py
requests/hooks.py
requests/models.py
requests/monkeys.py
-requests/patches.py
requests/sessions.py
requests/status_codes.py
requests/structures.py
diff --git a/requests/api.py b/requests/api.py
index 0cea63d..1b847b7 100644
--- a/requests/api.py
+++ b/requests/api.py
@@ -15,18 +15,17 @@ import config
from .models import Request, Response, AuthObject
from .status_codes import codes
from .hooks import dispatch_hook
-from .utils import cookiejar_from_dict
+from .utils import cookiejar_from_dict, header_expand
-from urlparse import urlparse
__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete')
def request(method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
- timeout=None, allow_redirects=False, proxies=None, hooks=None):
+ timeout=None, allow_redirects=False, proxies=None, hooks=None, return_response=True):
- """Constructs and sends a :class:`Request <models.Request>`.
- Returns :class:`Response <models.Response>` object.
+ """Constructs and sends a :class:`Request <Request>`.
+ Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
@@ -39,13 +38,21 @@ def request(method, url,
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param return_response: (optional) If False, an un-sent Request object will returned.
"""
+ method = str(method).upper()
+
if cookies is None:
cookies = {}
cookies = cookiejar_from_dict(cookies)
+ # Expand header values
+ if headers:
+ for k, v in headers.items() or {}:
+ headers[k] = header_expand(v)
+
args = dict(
method = method,
url = url,
@@ -55,6 +62,7 @@ def request(method, url,
cookiejar = cookies,
files = files,
auth = auth,
+ hooks = hooks,
timeout = timeout or config.settings.timeout,
allow_redirects = allow_redirects,
proxies = proxies or config.settings.proxies,
@@ -68,6 +76,10 @@ def request(method, url,
# Pre-request hook.
r = dispatch_hook('pre_request', hooks, r)
+ # Don't send if asked nicely.
+ if not return_response:
+ return r
+
# Send the HTTP Request.
r.send()
@@ -85,50 +97,34 @@ def get(url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
+
+ kwargs.setdefault('allow_redirects', True)
return request('GET', url, **kwargs)
def head(url, **kwargs):
-
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
+ kwargs.setdefault('allow_redirects', True)
return request('HEAD', url, **kwargs)
def post(url, data='', **kwargs):
-
"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
- return request('POST', url, data=data, **kwargs)
+ return request('post', url, data=data, **kwargs)
def put(url, data='', **kwargs):
@@ -136,17 +132,10 @@ def put(url, data='', **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
- return request('PUT', url, data=data, **kwargs)
+ return request('put', url, data=data, **kwargs)
def patch(url, data='', **kwargs):
@@ -154,31 +143,17 @@ def patch(url, data='', **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
- return request('PATCH', url, **kwargs)
+ return request('patch', url, **kwargs)
def delete(url, **kwargs):
-
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
- :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
- :param auth: (optional) AuthObject to enable Basic HTTP Auth.
- :param timeout: (optional) Float describing the timeout of the request.
- :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
- :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+ :param **kwargs: Optional arguments that ``request`` takes.
"""
- return request('DELETE', url, **kwargs)
+ return request('delete', url, **kwargs)
diff --git a/requests/async.py b/requests/async.py
index ab04084..db25f6a 100644
--- a/requests/async.py
+++ b/requests/async.py
@@ -1,41 +1,84 @@
# -*- coding: utf-8 -*-
"""
- requests.async
- ~~~~~~~~~~~~~~
+requests.async
+~~~~~~~~~~~~~~
- This module implements the main Requests system, after monkey-patching
- the urllib2 module with eventlet or gevent..
-
- :copyright: (c) 2011 by Kenneth Reitz.
- :license: ISC, see LICENSE for more details.
+This module contains an asynchronous replica of ``requests.api``, powered
+by gevent. All API methods return a ``Request`` instance (as opposed to
+``Response``). A list of requests can be sent with ``map()``.
"""
+try:
+ import gevent
+ from gevent import monkey as curious_george
+except ImportError:
+ raise RuntimeError('Gevent is required for requests.async.')
+
+# Monkey-patch.
+curious_george.patch_all(thread=False)
-from __future__ import absolute_import
+from . import api
+from .hooks import dispatch_hook
-import urllib
-import urllib2
-from urllib2 import HTTPError
+__all__ = (
+ 'map',
+ 'get', 'head', 'post', 'put', 'patch', 'delete', 'request'
+)
-try:
- import eventlet
- eventlet.monkey_patch()
-except ImportError:
- pass
+def _patched(f):
+ """Patches a given API function to not send."""
+
+ def wrapped(*args, **kwargs):
+ return f(*args, return_response=False, **kwargs)
+
+ return wrapped
+
+
+def _send(r, pools=None):
+ """Sends a given Request object."""
+
+ if pools:
+ r._pools = pools
+
+ r.send()
+
+ # Post-request hook.
+ r = dispatch_hook('post_request', r.hooks, r)
+
+ # Response manipulation hook.
+ r.response = dispatch_hook('response', r.hooks, r.response)
+
+ return r.response
+
+
+# Patched requests.api functions.
+get = _patched(api.get)
+head = _patched(api.head)
+post = _patched(api.post)
+put = _patched(api.put)
+patch = _patched(api.patch)
+delete = _patched(api.delete)
+request = _patched(api.request)
+
+
+def map(requests, prefetch=True):
+ """Concurrently converts a list of Requests to Responses.
+
+ :param requests: a collection of Request objects.
+ :param prefetch: If False, the content will not be downloaded immediately.
+ """
+
+ jobs = [gevent.spawn(_send, r) for r in requests]
+ gevent.joinall(jobs)
+
+ if prefetch:
+ [r.response.content for r in requests]
-if not 'eventlet' in locals():
- try:
- from gevent import monkey
- monkey.patch_all()
- except ImportError:
- pass
+ return [r.response for r in requests]
-if not 'eventlet' in locals():
- raise ImportError('No Async adaptations of urllib2 found!')
-from .core import *
diff --git a/requests/config.py b/requests/config.py
index 39be2ed..794109c 100644
--- a/requests/config.py
+++ b/requests/config.py
@@ -62,6 +62,7 @@ settings.proxies = None
settings.verbose = None
settings.timeout = None
settings.max_redirects = 30
+settings.decode_unicode = True
#: Use socket.setdefaulttimeout() as fallback?
settings.timeout_fallback = True
diff --git a/requests/core.py b/requests/core.py
index 8ba34a2..de05cf9 100644
--- a/requests/core.py
+++ b/requests/core.py
@@ -12,16 +12,18 @@ This module implements the main Requests system.
"""
__title__ = 'requests'
-__version__ = '0.6.1'
-__build__ = 0x000601
+__version__ = '0.6.4'
+__build__ = 0x000604
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2011 Kenneth Reitz'
-from models import HTTPError
+from models import HTTPError, Request, Response
from api import *
from exceptions import *
from sessions import session
from status_codes import codes
-from config import settings \ No newline at end of file
+from config import settings
+
+import utils
diff --git a/requests/models.py b/requests/models.py
index 2d7fc8f..9a8f5f9 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -9,8 +9,10 @@ requests.models
import urllib
import urllib2
import socket
+import codecs
import zlib
+
from urllib2 import HTTPError
from urlparse import urlparse, urlunparse, urljoin
from datetime import datetime
@@ -20,9 +22,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA
from .structures import CaseInsensitiveDict
from .packages.poster.encode import multipart_encode
from .packages.poster.streaminghttp import register_openers, get_handlers
-from .utils import dict_from_cookiejar
-from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
+from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip
from .status_codes import codes
+from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
@@ -30,14 +32,14 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
class Request(object):
- """The :class:`Request <models.Request>` object. It carries out all functionality of
+ """The :class:`Request <Request>` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
"""
def __init__(self,
url=None, headers=dict(), files=None, method=None, data=dict(),
params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False,
- allow_redirects=False, proxies=None):
+ allow_redirects=False, proxies=None, hooks=None):
#: Float describ the timeout of the request.
# (Use socket.setdefaulttimeout() as fallback)
@@ -46,24 +48,24 @@ class Request(object):
#: Request URL.
self.url = url
- #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`.
+ #: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
self.headers = headers
#: Dictionary of files to multipart upload (``{filename: content}``).
self.files = files
- #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE.
+ #: HTTP Method to use.
self.method = method
#: Dictionary or byte of request body data to attach to the
- #: :class:`Request <models.Request>`.
+ #: :class:`Request <Request>`.
self.data = None
#: Dictionary or byte of querystring data to attach to the
- #: :class:`Request <models.Request>`.
+ #: :class:`Request <Request>`.
self.params = None
- #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history
+ #: True if :class:`Request <Request>` is part of a redirect chain (disables history
#: and HTTPError storage).
self.redirect = redirect
@@ -76,7 +78,7 @@ class Request(object):
self.data, self._enc_data = self._encode_params(data)
self.params, self._enc_params = self._encode_params(params)
- #: :class:`Response <models.Response>` instance, containing
+ #: :class:`Response <Response>` instance, containing
#: content and metadata of HTTP Response, once :attr:`sent <send>`.
self.response = Response()
@@ -85,15 +87,17 @@ class Request(object):
if not auth:
auth = auth_manager.get_auth(self.url)
- #: :class:`AuthObject` to attach to :class:`Request <models.Request>`.
+ #: :class:`AuthObject` to attach to :class:`Request <Request>`.
self.auth = auth
- #: CookieJar to attach to :class:`Request <models.Request>`.
+ #: CookieJar to attach to :class:`Request <Request>`.
self.cookiejar = cookiejar
#: True if Request has been sent.
self.sent = False
+ #: Event-handling hooks.
+ self.hooks = hooks
# Header manipulation and defaults.
@@ -132,9 +136,16 @@ class Request(object):
_handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
if self.auth:
- if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
+ if not isinstance(self.auth.handler,
+ (urllib2.AbstractBasicAuthHandler,
+ urllib2.AbstractDigestAuthHandler)):
+
# TODO: REMOVE THIS COMPLETELY
- auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
+ auth_manager.add_password(
+ self.auth.realm, self.url,
+ self.auth.username,
+ self.auth.password)
+
self.auth.handler = self.auth.handler(auth_manager)
auth_manager.add_auth(self.url, self.auth)
@@ -166,7 +177,10 @@ class Request(object):
def _build_response(self, resp, is_error=False):
- """Build internal :class:`Response <models.Response>` object from given response."""
+ """Build internal :class:`Response <Response>` object
+ from given response.
+ """
+
def build(resp):
@@ -175,12 +189,9 @@ class Request(object):
try:
response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None))
- response.read = resp.read
- response._resp = resp
- response._close = resp.close
+ response.raw = resp
if self.cookiejar:
-
response.cookies = dict_from_cookiejar(self.cookiejar)
@@ -203,12 +214,10 @@ class Request(object):
while (
('location' in r.headers) and
- ((self.method in ('GET', 'HEAD')) or
- (r.status_code is codes.see_other) or
- (self.allow_redirects))
+ ((r.status_code is codes.see_other) or (self.allow_redirects))
):
- r.close()
+ r.raw.close()
if not len(history) < settings.max_redirects:
raise TooManyRedirects()
@@ -257,8 +266,8 @@ class Request(object):
Otherwise, assumes the data is already encoded appropriately, and
returns it twice.
-
"""
+
if hasattr(data, 'items'):
result = []
for k, vs in data.items():
@@ -302,7 +311,6 @@ class Request(object):
"""
self._checks()
- success = False
# Logging
if settings.verbose:
@@ -363,10 +371,11 @@ class Request(object):
if hasattr(why, 'reason'):
if isinstance(why.reason, socket.timeout):
why = Timeout(why)
+ elif isinstance(why.reason, socket.error):
+ why = Timeout(why)
self._build_response(why, is_error=True)
-
else:
self._build_response(resp)
self.response.ok = True
@@ -377,37 +386,46 @@ class Request(object):
return self.sent
-
class Response(object):
- """The core :class:`Response <models.Response>` object. All
- :class:`Request <models.Request>` objects contain a
- :class:`response <models.Response>` attribute, which is an instance
+ """The core :class:`Response <Response>` object. All
+ :class:`Request <Request>` objects contain a
+ :class:`response <Response>` attribute, which is an instance
of this class.
"""
def __init__(self):
- #: Raw content of the response, in bytes.
- #: If ``content-encoding`` of response was set to ``gzip``, the
- #: response data will be automatically deflated.
+
self._content = None
+ self._content_consumed = False
+
#: Integer Code of responded HTTP Status.
self.status_code = None
+
#: Case-insensitive Dictionary of Response Headers.
#: For example, ``headers['content-encoding']`` will return the
#: value of a ``'Content-Encoding'`` response header.
self.headers = CaseInsensitiveDict()
+
+ #: File-like object representation of response (for advanced usage).
+ self.raw = None
+
#: Final URL location of Response.
self.url = None
+
#: True if no :attr:`error` occured.
self.ok = False
+
#: Resulting :class:`HTTPError` of request, if one occured.
self.error = None
- #: A list of :class:`Response <models.Response>` objects from
+
+ #: A list of :class:`Response <Response>` objects from
#: the history of the Request. Any redirect responses will end
#: up here.
self.history = []
- #: The Request that created the Response.
+
+ #: The :class:`Request <Request>` that created the Response.
self.request = None
+
#: A dictionary of Cookies the server sent back.
self.cookies = None
@@ -418,23 +436,65 @@ class Response(object):
def __nonzero__(self):
"""Returns true if :attr:`status_code` is 'OK'."""
+
return not self.error
+ def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
+ """Iterates over the response data. This avoids reading the content
+ at once into memory for large responses. The chunk size is the number
+ of bytes it should read into memory. This is not necessarily the
+ length of each item returned as decoding can take place.
+ """
+ if self._content_consumed:
+ raise RuntimeError('The content for this response was '
+ 'already consumed')
+
+ def generate():
+ while 1:
+ chunk = self.raw.read(chunk_size)
+ if not chunk:
+ break
+ yield chunk
+ self._content_consumed = True
+ gen = generate()
+ if 'gzip' in self.headers.get('content-encoding', ''):
+ gen = stream_decode_gzip(gen)
+ if decode_unicode is None:
+ decode_unicode = settings.decode_unicode
+ if decode_unicode:
+ gen = stream_decode_response_unicode(gen, self)
+ return gen
+
+ @property
+ def content(self):
+ """Content of the response, in bytes or unicode
+ (if available).
+ """
- def __getattr__(self, name):
- """Read and returns the full stream when accessing to :attr: `content`"""
- if name == 'content':
- if self._content is not None:
- return self._content
- self._content = self.read()
- if self.headers.get('content-encoding', '') == 'gzip':
- try:
- self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS)
- except zlib.error:
- pass
+ if self._content is not None:
return self._content
- else:
- raise AttributeError
+
+ if self._content_consumed:
+ raise RuntimeError('The content for this response was '
+ 'already consumed')
+
+ # Read the contents.
+ self._content = self.raw.read()
+
+ # Decode GZip'd content.
+ if 'gzip' in self.headers.get('content-encoding', ''):
+ try:
+ self._content = decode_gzip(self._content)
+ except zlib.error:
+ pass
+
+ # Decode unicode content.
+ if settings.decode_unicode:
+ self._content = get_unicode_from_response(self)
+
+ self._content_consumed = True
+ return self._content
+
def raise_for_status(self):
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occured."""
@@ -442,10 +502,6 @@ class Response(object):
raise self.error
- def close(self):
- if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'):
- self._resp.fp._sock.recv = None
- self._close()
class AuthManager(object):
"""Requests Authentication Manager."""
diff --git a/requests/patches.py b/requests/patches.py
deleted file mode 100644
index 43a3b4c..0000000
--- a/requests/patches.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-requests.monkeys
-"""
diff --git a/requests/utils.py b/requests/utils.py
index 8ac78b4..2e16163 100644
--- a/requests/utils.py
+++ b/requests/utils.py
@@ -9,15 +9,69 @@ that are also useful for external consumption.
"""
+import cgi
+import codecs
import cookielib
+import re
+import zlib
-def dict_from_cookiejar(cookiejar):
- """Returns a key/value dictionary from a CookieJar."""
+def header_expand(headers):
+ """Returns an HTTP Header value string from a dictionary.
+
+ Example expansion::
+
+ {'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
+ # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+
+ (('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
+ # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+ """
+
+ collector = []
+
+ if isinstance(headers, dict):
+ headers = headers.items()
+
+ elif isinstance(headers, basestring):
+ return headers
+
+ for i, (value, params) in enumerate(headers):
+
+ _params = []
+
+ for (p_k, p_v) in params.items():
+
+ _params.append('%s=%s' % (p_k, p_v))
+
+ collector.append(value)
+ collector.append('; ')
+
+ if len(params):
+
+ collector.append('; '.join(_params))
+
+ if not len(headers) == i+1:
+ collector.append(', ')
+
+
+ # Remove trailing seperators.
+ if collector[-1] in (', ', '; '):
+ del collector[-1]
+
+ return ''.join(collector)
+
+
+
+def dict_from_cookiejar(cj):
+ """Returns a key/value dictionary from a CookieJar.
+
+ :param cj: CookieJar object to extract cookies from.
+ """
cookie_dict = {}
- for _, cookies in cookiejar._cookies.items():
+ for _, cookies in cj._cookies.items():
for _, cookies in cookies.items():
for cookie in cookies.values():
# print cookie
@@ -27,7 +81,10 @@ def dict_from_cookiejar(cookiejar):
def cookiejar_from_dict(cookie_dict):
- """Returns a CookieJar from a key/value dictionary."""
+ """Returns a CookieJar from a key/value dictionary.
+
+ :param cookie_dict: Dict of key/values to insert into CookieJar.
+ """
# return cookiejar if one was passed in
if isinstance(cookie_dict, cookielib.CookieJar):
@@ -42,7 +99,11 @@ def cookiejar_from_dict(cookie_dict):
def add_dict_to_cookiejar(cj, cookie_dict):
- """Returns a CookieJar from a key/value dictionary."""
+ """Returns a CookieJar from a key/value dictionary.
+
+ :param cj: CookieJar to insert cookies into.
+ :param cookie_dict: Dict of key/values to insert into CookieJar.
+ """
for k, v in cookie_dict.items():
@@ -70,3 +131,124 @@ def add_dict_to_cookiejar(cj, cookie_dict):
cj.set_cookie(cookie)
return cj
+
+
+def get_encodings_from_content(content):
+ """Returns encodings from given content string.
+
+ :param content: bytestring to extract encodings from.
+ """
+
+ charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
+
+ return charset_re.findall(content)
+
+
+def get_encoding_from_headers(headers):
+ """Returns encodings from given HTTP Header Dict.
+
+ :param headers: dictionary to extract encoding from.
+ """
+
+ content_type = headers.get('content-type')
+
+ if not content_type:
+ return None
+
+ content_type, params = cgi.parse_header(content_type)
+
+ if 'charset' in params:
+ return params['charset'].strip("'\"")
+
+
+def unicode_from_html(content):
+ """Attempts to decode an HTML string into unicode.
+ If unsuccessful, the original content is returned.
+ """
+
+ encodings = get_encodings_from_content(content)
+
+ for encoding in encodings:
+
+ try:
+ return unicode(content, encoding)
+ except (UnicodeError, TypeError):
+ pass
+
+ return content
+
+
+def stream_decode_response_unicode(iterator, r):
+ """Stream decodes a iterator."""
+ encoding = get_encoding_from_headers(r.headers)
+ if encoding is None:
+ for item in iterator:
+ yield item
+ return
+
+ decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
+ for chunk in iterator:
+ rv = decoder.decode(chunk)
+ if rv:
+ yield rv
+ rv = decoder.decode('', final=True)
+ if rv:
+ yield rv
+
+
+def get_unicode_from_response(r):
+ """Returns the requested content back in unicode.
+
+ :param r: Reponse object to get unicode content from.
+
+ Tried:
+
+ 1. charset from content-type
+
+ 2. every encodings from ``<meta ... charset=XXX>``
+
+ 3. fall back and replace all unicode characters
+
+ """
+
+ tried_encodings = []
+
+ # Try charset from content-type
+ encoding = get_encoding_from_headers(r.headers)
+
+ if encoding:
+ try:
+ return unicode(r.content, encoding)
+ except UnicodeError:
+ tried_encodings.append(encoding)
+
+ # Fall back:
+ try:
+ return unicode(r.content, encoding, errors='replace')
+ except TypeError:
+ return r.content
+
+
+def decode_gzip(content):
+ """Return gzip-decoded string.
+
+ :param content: bytestring to gzip-decode.
+ """
+
+ return zlib.decompress(content, 16 + zlib.MAX_WBITS)
+
+
+def stream_decode_gzip(iterator):
+ """Stream decodes a gzip-encoded iterator"""
+ try:
+ dec = zlib.decompressobj(16 + zlib.MAX_WBITS)
+ for chunk in iterator:
+ rv = dec.decompress(chunk)
+ if rv:
+ yield rv
+ buf = dec.decompress('')
+ rv = buf + dec.flush()
+ if rv:
+ yield rv
+ except zlib.error:
+ pass
diff --git a/test_requests.py b/test_requests.py
new file mode 100755
index 0000000..83b827a
--- /dev/null
+++ b/test_requests.py
@@ -0,0 +1,501 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import with_statement
+
+import unittest
+import cookielib
+
+try:
+ import omnijson as json
+except ImportError:
+ import json
+
+import requests
+
+from requests.sessions import Session
+
+
+HTTPBIN_URL = 'http://httpbin.ep.io/'
+HTTPSBIN_URL = 'https://httpbin.ep.io/'
+
+# HTTPBIN_URL = 'http://staging.httpbin.org/'
+# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/'
+
+
+def httpbin(*suffix):
+ """Returns url for HTTPBIN resource."""
+
+ return HTTPBIN_URL + '/'.join(suffix)
+
+
+def httpsbin(*suffix):
+ """Returns url for HTTPSBIN resource."""
+
+ return HTTPSBIN_URL + '/'.join(suffix)
+
+
+SERVICES = (httpbin, httpsbin)
+
+
+
+class RequestsTestSuite(unittest.TestCase):
+ """Requests test cases."""
+
+ # It goes to eleven.
+ _multiprocess_can_split_ = True
+
+ def setUp(self):
+ pass
+
+
+ def tearDown(self):
+ """Teardown."""
+ pass
+
+
+ def test_invalid_url(self):
+ self.assertRaises(ValueError, requests.get, 'hiwpefhipowhefopw')
+
+
+ def test_HTTP_200_OK_GET(self):
+ r = requests.get(httpbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+ def test_HTTP_302_ALLOW_REDIRECT_GET(self):
+ r = requests.get(httpbin('redirect', '1'))
+ self.assertEqual(r.status_code, 200)
+
+ def test_HTTP_302_GET(self):
+ r = requests.get(httpbin('redirect', '1'), allow_redirects=False)
+ self.assertEqual(r.status_code, 302)
+
+ def test_HTTPS_200_OK_GET(self):
+ r = requests.get(httpsbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTP_200_OK_GET_WITH_PARAMS(self):
+ heads = {'User-agent': 'Mozilla/5.0'}
+
+ r = requests.get(httpbin('user-agent'), headers=heads)
+
+ assert heads['User-agent'] in r.content
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self):
+ heads = {'User-agent': 'Mozilla/5.0'}
+
+ r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads)
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_user_agent_transfers(self):
+ """Issue XX"""
+
+ heads = {
+ 'User-agent':
+ 'Mozilla/5.0 (github.com/kennethreitz/requests)'
+ }
+
+ r = requests.get(httpbin('user-agent'), headers=heads);
+ self.assertTrue(heads['User-agent'] in r.content)
+
+ heads = {
+ 'user-agent':
+ 'Mozilla/5.0 (github.com/kennethreitz/requests)'
+ }
+
+ r = requests.get(httpbin('user-agent'), headers=heads);
+ self.assertTrue(heads['user-agent'] in r.content)
+
+
+ def test_HTTP_200_OK_HEAD(self):
+ r = requests.head(httpbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTPS_200_OK_HEAD(self):
+ r = requests.head(httpsbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTP_200_OK_PUT(self):
+ r = requests.put(httpbin('put'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTPS_200_OK_PUT(self):
+ r = requests.put(httpsbin('put'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTP_200_OK_PATCH(self):
+ r = requests.patch(httpbin('patch'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_HTTPS_200_OK_PATCH(self):
+ r = requests.patch(httpsbin('patch'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_AUTH_HTTP_200_OK_GET(self):
+
+ for service in SERVICES:
+
+ auth = ('user', 'pass')
+ url = service('basic-auth', 'user', 'pass')
+
+ r = requests.get(url, auth=auth)
+ # print r.__dict__
+ self.assertEqual(r.status_code, 200)
+
+
+ r = requests.get(url)
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_POSTBIN_GET_POST_FILES(self):
+
+ for service in SERVICES:
+
+ url = service('post')
+ post = requests.post(url).raise_for_status()
+
+ post = requests.post(url, data={'some': 'data'})
+ self.assertEqual(post.status_code, 200)
+
+ post2 = requests.post(url, files={'some': open('test_requests.py')})
+ self.assertEqual(post2.status_code, 200)
+
+ post3 = requests.post(url, data='[{"some": "json"}]')
+ self.assertEqual(post3.status_code, 200)
+
+
+ def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self):
+
+ for service in SERVICES:
+
+ url = service('post')
+ post = requests.post(url,
+ files={'some': open('test_requests.py')},
+ data={'some': 'data'})
+
+ self.assertEqual(post.status_code, 200)
+
+
+ def test_POSTBIN_GET_POST_FILES_WITH_HEADERS(self):
+
+ for service in SERVICES:
+
+ url = service('post')
+
+ post2 = requests.post(url,
+ files={'some': open('test_requests.py')},
+ headers = {'User-Agent': 'requests-tests'})
+
+ self.assertEqual(post2.status_code, 200)
+
+
+ def test_nonzero_evaluation(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('status', '500'))
+ self.assertEqual(bool(r), False)
+
+ r = requests.get(service('/'))
+ self.assertEqual(bool(r), True)
+
+
+ def test_request_ok_set(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('status', '404'))
+ self.assertEqual(r.ok, False)
+
+
+ def test_status_raising(self):
+ r = requests.get(httpbin('status', '404'))
+ self.assertRaises(requests.HTTPError, r.raise_for_status)
+
+ r = requests.get(httpbin('status', '200'))
+ self.assertFalse(r.error)
+ r.raise_for_status()
+
+
+ def test_cookie_jar(self):
+
+ jar = cookielib.CookieJar()
+ self.assertFalse(jar)
+
+ url = httpbin('cookies', 'set', 'requests_cookie', 'awesome')
+ r = requests.get(url, cookies=jar)
+ self.assertTrue(jar)
+
+ cookie_found = False
+ for cookie in jar:
+ if cookie.name == 'requests_cookie':
+ self.assertEquals(cookie.value, 'awesome')
+ cookie_found = True
+ self.assertTrue(cookie_found)
+
+ r = requests.get(httpbin('cookies'), cookies=jar)
+ self.assertTrue('awesome' in r.content)
+
+
+ def test_decompress_gzip(self):
+
+ r = requests.get(httpbin('gzip'))
+ r.content.decode('ascii')
+
+
+ def test_unicode_get(self):
+
+ for service in SERVICES:
+
+ url = service('/')
+
+ requests.get(url, params={'foo': u'føø'})
+ requests.get(url, params={u'føø': u'føø'})
+ requests.get(url, params={'føø': 'føø'})
+ requests.get(url, params={'foo': u'foo'})
+ requests.get(service('ø'), params={'foo': u'foo'})
+
+
+ def test_httpauth_recursion(self):
+
+ http_auth = ('user', 'BADpass')
+
+ for service in SERVICES:
+ r = requests.get(service('basic-auth', 'user', 'pass'), auth=http_auth)
+ self.assertEquals(r.status_code, 401)
+
+
+ def test_settings(self):
+
+ def test():
+ r = requests.get(httpbin(''))
+ r.raise_for_status()
+
+ with requests.settings(timeout=0.0000001):
+ self.assertRaises(requests.Timeout, test)
+
+ with requests.settings(timeout=100):
+ requests.get(httpbin(''))
+
+
+ def test_urlencoded_post_data(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'), data=dict(test='fooaowpeuf'))
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post'))
+
+ rbody = json.loads(r.content)
+
+ self.assertEquals(rbody.get('form'), dict(test='fooaowpeuf'))
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_nonurlencoded_post_data(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'), data='fooaowpeuf')
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post'))
+
+ rbody = json.loads(r.content)
+ # Body wasn't valid url encoded data, so the server returns None as
+ # "form" and the raw body as "data".
+ self.assertEquals(rbody.get('form'), None)
+ self.assertEquals(rbody.get('data'), 'fooaowpeuf')
+
+
+ def test_urlencoded_post_querystring(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'), params=dict(test='fooaowpeuf'))
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?test=fooaowpeuf'))
+
+ rbody = json.loads(r.content)
+ self.assertEquals(rbody.get('form'), {}) # No form supplied
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_nonurlencoded_post_querystring(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'), params='fooaowpeuf')
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?fooaowpeuf'))
+
+ rbody = json.loads(r.content)
+ self.assertEquals(rbody.get('form'), {}) # No form supplied
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_urlencoded_post_query_and_data(self):
+
+ for service in SERVICES:
+
+ r = requests.post(
+ service('post'),
+ params=dict(test='fooaowpeuf'),
+ data=dict(test2="foobar"))
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?test=fooaowpeuf'))
+
+ rbody = json.loads(r.content)
+ self.assertEquals(rbody.get('form'), dict(test2='foobar'))
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_nonurlencoded_post_query_and_data(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'),
+ params='fooaowpeuf', data="foobar")
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?fooaowpeuf'))
+
+ rbody = json.loads(r.content)
+
+ self.assertEquals(rbody.get('form'), None)
+ self.assertEquals(rbody.get('data'), 'foobar')
+
+
+ def test_idna(self):
+ r = requests.get(u'http://➡.ws/httpbin')
+ assert 'httpbin' in r.url
+
+
+ def test_urlencoded_get_query_multivalued_param(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('get'), params=dict(test=['foo','baz']))
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.url, service('get?test=foo&test=baz'))
+
+
+ def test_urlencoded_post_querystring_multivalued(self):
+
+ for service in SERVICES:
+
+ r = requests.post(service('post'), params=dict(test=['foo','baz']))
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?test=foo&test=baz'))
+
+ rbody = json.loads(r.content)
+ self.assertEquals(rbody.get('form'), {}) # No form supplied
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_urlencoded_post_query_multivalued_and_data(self):
+
+ for service in SERVICES:
+
+ r = requests.post(
+ service('post'),
+ params=dict(test=['foo','baz']),
+ data=dict(test2="foobar",test3=['foo','baz']))
+
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(r.headers['content-type'], 'application/json')
+ self.assertEquals(r.url, service('post?test=foo&test=baz'))
+ rbody = json.loads(r.content)
+ self.assertEquals(rbody.get('form'), dict(test2='foobar',test3='foo'))
+ self.assertEquals(rbody.get('data'), '')
+
+
+ def test_GET_no_redirect(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('redirect', '3'), allow_redirects=False)
+ self.assertEquals(r.status_code, 302)
+ self.assertEquals(len(r.history), 0)
+
+
+ def test_HEAD_no_redirect(self):
+
+ for service in SERVICES:
+
+ r = requests.head(service('redirect', '3'), allow_redirects=False)
+ self.assertEquals(r.status_code, 302)
+ self.assertEquals(len(r.history), 0)
+
+
+ def test_redirect_history(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('redirect', '3'))
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(len(r.history), 3)
+
+
+ def test_relative_redirect_history(self):
+
+ for service in SERVICES:
+
+ r = requests.get(service('relative-redirect', '3'))
+ self.assertEquals(r.status_code, 200)
+ self.assertEquals(len(r.history), 3)
+
+
+ def test_session_HTTP_200_OK_GET(self):
+
+ s = Session()
+ r = s.get(httpbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_session_HTTPS_200_OK_GET(self):
+
+ s = Session()
+ r = s.get(httpsbin('/'))
+ self.assertEqual(r.status_code, 200)
+
+
+ def test_session_persistent_headers(self):
+
+ heads = {'User-agent': 'Mozilla/5.0'}
+
+ s = Session()
+ s.headers = heads
+ # Make 2 requests from Session object, should send header both times
+ r1 = s.get(httpbin('user-agent'))
+
+ assert heads['User-agent'] in r1.content
+ r2 = s.get(httpbin('user-agent'))
+
+ assert heads['User-agent'] in r2.content
+ self.assertEqual(r2.status_code, 200)
+
+
+if __name__ == '__main__':
+ unittest.main()