diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 102 |
1 files changed, 67 insertions, 35 deletions
diff --git a/requests/models.py b/requests/models.py index 5202e6f..6ed2b59 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,27 +9,26 @@ This module contains the primary objects that power Requests. import collections import logging +import datetime from io import BytesIO from .hooks import default_hooks from .structures import CaseInsensitiveDict -from .status_codes import codes from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header from .packages.urllib3.filepost import encode_multipart_formdata from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL from .utils import ( - stream_untransfer, guess_filename, requote_uri, + stream_untransfer, guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, iter_slices, guess_json_utf, super_len) from .compat import ( cookielib, urlparse, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, json, builtin_str, basestring) -REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) CONTENT_CHUNK_SIZE = 10 * 1024 -ITER_CHUNK_SIZE = 10 * 1024 +ITER_CHUNK_SIZE = 512 log = logging.getLogger(__name__) @@ -121,7 +120,7 @@ class RequestEncodingMixin(object): fp = StringIO(fp) if isinstance(fp, bytes): fp = BytesIO(fp) - + if ft: new_v = (fn, fp.read(), ft) else: @@ -188,7 +187,6 @@ class Request(RequestHooksMixin): cookies=None, hooks=None): - # Default empty dicts for dict params. data = [] if data is None else data files = [] if files is None else files @@ -222,9 +220,12 @@ class Request(RequestHooksMixin): p.prepare_headers(self.headers) p.prepare_cookies(self.cookies) p.prepare_body(self.data, self.files) + p.prepare_auth(self.auth, self.url) # Note that prepare_auth must be last to enable authentication schemes # such as OAuth to work on a fully prepared request. - p.prepare_auth(self.auth) + + # This MUST go after prepare_auth. Authenticators could add a hook + p.prepare_hooks(self.hooks) return p @@ -283,7 +284,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # Support for unicode domain names and paths. scheme, netloc, path, _params, query, fragment = urlparse(url) - if not scheme: + if not (scheme and netloc): raise MissingSchema("Invalid URL %r: No schema supplied" % url) try: @@ -323,6 +324,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Prepares the given HTTP headers.""" if headers: + headers = dict((name.encode('ascii'), value) for name, value in headers.items()) self.headers = CaseInsensitiveDict(headers) else: self.headers = CaseInsensitiveDict() @@ -342,6 +344,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): is_stream = all([ hasattr(data, '__iter__'), not isinstance(data, basestring), + not isinstance(data, list), not isinstance(data, dict) ]) @@ -357,7 +360,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): raise NotImplementedError('Streamed bodies and files are mutually exclusive.') if length: - self.headers['Content-Length'] = length + self.headers['Content-Length'] = str(length) else: self.headers['Transfer-Encoding'] = 'chunked' # Check if file, fo, generator, iterator. @@ -375,13 +378,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: content_type = 'application/x-www-form-urlencoded' - self.headers['Content-Length'] = '0' - if hasattr(body, 'seek') and hasattr(body, 'tell'): - body.seek(0, 2) - self.headers['Content-Length'] = str(body.tell()) - body.seek(0, 0) - elif body is not None: - self.headers['Content-Length'] = str(len(body)) + self.prepare_content_length(body) # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): @@ -389,8 +386,26 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.body = body - def prepare_auth(self, auth): + def prepare_content_length(self, body): + if hasattr(body, 'seek') and hasattr(body, 'tell'): + body.seek(0, 2) + self.headers['Content-Length'] = str(body.tell()) + body.seek(0, 0) + elif body is not None: + l = super_len(body) + if l: + self.headers['Content-Length'] = str(l) + elif self.method not in ('GET', 'HEAD'): + self.headers['Content-Length'] = '0' + + def prepare_auth(self, auth, url=''): """Prepares the given HTTP auth data.""" + + # If no Auth is explicitly provided, extract it from the URL first. + if auth is None: + url_auth = get_auth_from_url(self.url) + auth = url_auth if any(url_auth) else None + if auth: if isinstance(auth, tuple) and len(auth) == 2: # special-case basic HTTP auth @@ -402,6 +417,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) + # Recompute Content-Length + self.prepare_content_length(self.body) + def prepare_cookies(self, cookies): """Prepares the given HTTP cookie data.""" @@ -415,6 +433,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if cookie_header is not None: self.headers['Cookie'] = cookie_header + def prepare_hooks(self, hooks): + """Prepares the given hooks.""" + for event in hooks: + self.register_hook(event, hooks[event]) + class Response(object): """The :class:`Response <Response>` object, which contains a @@ -456,6 +479,10 @@ class Response(object): #: A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) + #: The amount of time elapsed between sending the request + #: and the arrival of the response (as a timedelta) + self.elapsed = datetime.timedelta(0) + def __repr__(self): return '<Response [%s]>' % (self.status_code) @@ -467,6 +494,10 @@ class Response(object): """Returns true if :attr:`status_code` is 'OK'.""" return self.ok + def __iter__(self): + """Allows you to use a response as an iterator.""" + return self.iter_content(128) + @property def ok(self): try: @@ -482,10 +513,11 @@ class Response(object): return chardet.detect(self.content)['encoding'] def iter_content(self, chunk_size=1, decode_unicode=False): - """Iterates over the response data. This avoids reading the content - at once into memory for large responses. The chunk size is the number - of bytes it should read into memory. This is not necessarily the - length of each item returned as decoding can take place. + """Iterates over the response data. When stream=True is set on the + request, this avoids reading the content at once into memory for + large responses. The chunk size is the number of bytes it should + read into memory. This is not necessarily the length of each item + returned as decoding can take place. """ if self._content_consumed: # simulate reading small chunks of the content @@ -507,16 +539,15 @@ class Response(object): return gen def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None): - """Iterates over the response data, one line at a time. This - avoids reading the content at once into memory for large - responses. + """Iterates over the response data, one line at a time. When + stream=True is set on the request, this avoids reading the + content at once into memory for large responses. """ pending = None - for chunk in self.iter_content( - chunk_size=chunk_size, - decode_unicode=decode_unicode): + for chunk in self.iter_content(chunk_size=chunk_size, + decode_unicode=decode_unicode): if pending is not None: chunk = pending + chunk @@ -590,8 +621,11 @@ class Response(object): return content - def json(self): - """Returns the json-encoded content of a response, if any.""" + def json(self, **kwargs): + """Returns the json-encoded content of a response, if any. + + :param \*\*kwargs: Optional arguments that ``json.loads`` takes. + """ if not self.encoding and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect @@ -600,8 +634,8 @@ class Response(object): # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: - return json.loads(self.content.decode(encoding)) - return json.loads(self.text or self.content) + return json.loads(self.content.decode(encoding), **kwargs) + return json.loads(self.text or self.content, **kwargs) @property def links(self): @@ -622,7 +656,7 @@ class Response(object): return l def raise_for_status(self): - """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" + """Raises stored :class:`HTTPError`, if one occurred.""" http_error_msg = '' @@ -633,9 +667,7 @@ class Response(object): http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason) if http_error_msg: - http_error = HTTPError(http_error_msg) - http_error.response = self - raise http_error + raise HTTPError(http_error_msg, response=self) def close(self): return self.raw.release_conn() |