aboutsummaryrefslogtreecommitdiff
path: root/requests/models.py
diff options
context:
space:
mode:
Diffstat (limited to 'requests/models.py')
-rw-r--r--requests/models.py102
1 files changed, 67 insertions, 35 deletions
diff --git a/requests/models.py b/requests/models.py
index 5202e6f..6ed2b59 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -9,27 +9,26 @@ This module contains the primary objects that power Requests.
import collections
import logging
+import datetime
from io import BytesIO
from .hooks import default_hooks
from .structures import CaseInsensitiveDict
-from .status_codes import codes
from .auth import HTTPBasicAuth
from .cookies import cookiejar_from_dict, get_cookie_header
from .packages.urllib3.filepost import encode_multipart_formdata
from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL
from .utils import (
- stream_untransfer, guess_filename, requote_uri,
+ stream_untransfer, guess_filename, get_auth_from_url, requote_uri,
stream_decode_response_unicode, to_key_val_list, parse_header_links,
iter_slices, guess_json_utf, super_len)
from .compat import (
cookielib, urlparse, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
is_py2, chardet, json, builtin_str, basestring)
-REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
CONTENT_CHUNK_SIZE = 10 * 1024
-ITER_CHUNK_SIZE = 10 * 1024
+ITER_CHUNK_SIZE = 512
log = logging.getLogger(__name__)
@@ -121,7 +120,7 @@ class RequestEncodingMixin(object):
fp = StringIO(fp)
if isinstance(fp, bytes):
fp = BytesIO(fp)
-
+
if ft:
new_v = (fn, fp.read(), ft)
else:
@@ -188,7 +187,6 @@ class Request(RequestHooksMixin):
cookies=None,
hooks=None):
-
# Default empty dicts for dict params.
data = [] if data is None else data
files = [] if files is None else files
@@ -222,9 +220,12 @@ class Request(RequestHooksMixin):
p.prepare_headers(self.headers)
p.prepare_cookies(self.cookies)
p.prepare_body(self.data, self.files)
+ p.prepare_auth(self.auth, self.url)
# Note that prepare_auth must be last to enable authentication schemes
# such as OAuth to work on a fully prepared request.
- p.prepare_auth(self.auth)
+
+ # This MUST go after prepare_auth. Authenticators could add a hook
+ p.prepare_hooks(self.hooks)
return p
@@ -283,7 +284,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
# Support for unicode domain names and paths.
scheme, netloc, path, _params, query, fragment = urlparse(url)
- if not scheme:
+ if not (scheme and netloc):
raise MissingSchema("Invalid URL %r: No schema supplied" % url)
try:
@@ -323,6 +324,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
"""Prepares the given HTTP headers."""
if headers:
+ headers = dict((name.encode('ascii'), value) for name, value in headers.items())
self.headers = CaseInsensitiveDict(headers)
else:
self.headers = CaseInsensitiveDict()
@@ -342,6 +344,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
is_stream = all([
hasattr(data, '__iter__'),
not isinstance(data, basestring),
+ not isinstance(data, list),
not isinstance(data, dict)
])
@@ -357,7 +360,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
if length:
- self.headers['Content-Length'] = length
+ self.headers['Content-Length'] = str(length)
else:
self.headers['Transfer-Encoding'] = 'chunked'
# Check if file, fo, generator, iterator.
@@ -375,13 +378,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
else:
content_type = 'application/x-www-form-urlencoded'
- self.headers['Content-Length'] = '0'
- if hasattr(body, 'seek') and hasattr(body, 'tell'):
- body.seek(0, 2)
- self.headers['Content-Length'] = str(body.tell())
- body.seek(0, 0)
- elif body is not None:
- self.headers['Content-Length'] = str(len(body))
+ self.prepare_content_length(body)
# Add content-type if it wasn't explicitly provided.
if (content_type) and (not 'content-type' in self.headers):
@@ -389,8 +386,26 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
self.body = body
- def prepare_auth(self, auth):
+ def prepare_content_length(self, body):
+ if hasattr(body, 'seek') and hasattr(body, 'tell'):
+ body.seek(0, 2)
+ self.headers['Content-Length'] = str(body.tell())
+ body.seek(0, 0)
+ elif body is not None:
+ l = super_len(body)
+ if l:
+ self.headers['Content-Length'] = str(l)
+ elif self.method not in ('GET', 'HEAD'):
+ self.headers['Content-Length'] = '0'
+
+ def prepare_auth(self, auth, url=''):
"""Prepares the given HTTP auth data."""
+
+ # If no Auth is explicitly provided, extract it from the URL first.
+ if auth is None:
+ url_auth = get_auth_from_url(self.url)
+ auth = url_auth if any(url_auth) else None
+
if auth:
if isinstance(auth, tuple) and len(auth) == 2:
# special-case basic HTTP auth
@@ -402,6 +417,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
# Update self to reflect the auth changes.
self.__dict__.update(r.__dict__)
+ # Recompute Content-Length
+ self.prepare_content_length(self.body)
+
def prepare_cookies(self, cookies):
"""Prepares the given HTTP cookie data."""
@@ -415,6 +433,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
if cookie_header is not None:
self.headers['Cookie'] = cookie_header
+ def prepare_hooks(self, hooks):
+ """Prepares the given hooks."""
+ for event in hooks:
+ self.register_hook(event, hooks[event])
+
class Response(object):
"""The :class:`Response <Response>` object, which contains a
@@ -456,6 +479,10 @@ class Response(object):
#: A CookieJar of Cookies the server sent back.
self.cookies = cookiejar_from_dict({})
+ #: The amount of time elapsed between sending the request
+ #: and the arrival of the response (as a timedelta)
+ self.elapsed = datetime.timedelta(0)
+
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
@@ -467,6 +494,10 @@ class Response(object):
"""Returns true if :attr:`status_code` is 'OK'."""
return self.ok
+ def __iter__(self):
+ """Allows you to use a response as an iterator."""
+ return self.iter_content(128)
+
@property
def ok(self):
try:
@@ -482,10 +513,11 @@ class Response(object):
return chardet.detect(self.content)['encoding']
def iter_content(self, chunk_size=1, decode_unicode=False):
- """Iterates over the response data. This avoids reading the content
- at once into memory for large responses. The chunk size is the number
- of bytes it should read into memory. This is not necessarily the
- length of each item returned as decoding can take place.
+ """Iterates over the response data. When stream=True is set on the
+ request, this avoids reading the content at once into memory for
+ large responses. The chunk size is the number of bytes it should
+ read into memory. This is not necessarily the length of each item
+ returned as decoding can take place.
"""
if self._content_consumed:
# simulate reading small chunks of the content
@@ -507,16 +539,15 @@ class Response(object):
return gen
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
- """Iterates over the response data, one line at a time. This
- avoids reading the content at once into memory for large
- responses.
+ """Iterates over the response data, one line at a time. When
+ stream=True is set on the request, this avoids reading the
+ content at once into memory for large responses.
"""
pending = None
- for chunk in self.iter_content(
- chunk_size=chunk_size,
- decode_unicode=decode_unicode):
+ for chunk in self.iter_content(chunk_size=chunk_size,
+ decode_unicode=decode_unicode):
if pending is not None:
chunk = pending + chunk
@@ -590,8 +621,11 @@ class Response(object):
return content
- def json(self):
- """Returns the json-encoded content of a response, if any."""
+ def json(self, **kwargs):
+ """Returns the json-encoded content of a response, if any.
+
+ :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
+ """
if not self.encoding and len(self.content) > 3:
# No encoding set. JSON RFC 4627 section 3 states we should expect
@@ -600,8 +634,8 @@ class Response(object):
# a best guess).
encoding = guess_json_utf(self.content)
if encoding is not None:
- return json.loads(self.content.decode(encoding))
- return json.loads(self.text or self.content)
+ return json.loads(self.content.decode(encoding), **kwargs)
+ return json.loads(self.text or self.content, **kwargs)
@property
def links(self):
@@ -622,7 +656,7 @@ class Response(object):
return l
def raise_for_status(self):
- """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
+ """Raises stored :class:`HTTPError`, if one occurred."""
http_error_msg = ''
@@ -633,9 +667,7 @@ class Response(object):
http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
if http_error_msg:
- http_error = HTTPError(http_error_msg)
- http_error.response = self
- raise http_error
+ raise HTTPError(http_error_msg, response=self)
def close(self):
return self.raw.release_conn()