diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 118 |
1 files changed, 81 insertions, 37 deletions
diff --git a/requests/models.py b/requests/models.py index 6cf2aaa..8fd9735 100644 --- a/requests/models.py +++ b/requests/models.py @@ -11,7 +11,7 @@ import collections import logging import datetime -from io import BytesIO +from io import BytesIO, UnsupportedOperation from .hooks import default_hooks from .structures import CaseInsensitiveDict @@ -19,14 +19,16 @@ from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header from .packages.urllib3.filepost import encode_multipart_formdata from .packages.urllib3.util import parse_url -from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL +from .exceptions import ( + HTTPError, RequestException, MissingSchema, InvalidURL, + ChunkedEncodingError) from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len) + iter_slices, guess_json_utf, super_len, to_native_string) from .compat import ( - cookielib, urlparse, urlunparse, urlsplit, urlencode, str, bytes, StringIO, - is_py2, chardet, json, builtin_str, basestring) + cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, + is_py2, chardet, json, builtin_str, basestring, IncompleteRead) CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 @@ -92,8 +94,10 @@ class RequestEncodingMixin(object): if parameters are supplied as a dict. """ - if (not files) or isinstance(data, str): - return None + if (not files): + raise ValueError("Files must be provided.") + elif isinstance(data, basestring): + raise ValueError("Data must not be a string.") new_fields = [] fields = to_key_val_list(data or {}) @@ -104,6 +108,10 @@ class RequestEncodingMixin(object): val = [val] for v in val: if v is not None: + # Don't call str() on bytestrings: in Py3 it all goes wrong. + if not isinstance(v, bytes): + v = str(v) + new_fields.append( (field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) @@ -139,6 +147,9 @@ class RequestHooksMixin(object): def register_hook(self, event, hook): """Properly register a hook.""" + if event not in self.hooks: + raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + if isinstance(hook, collections.Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): @@ -184,8 +195,8 @@ class Request(RequestHooksMixin): url=None, headers=None, files=None, - data=dict(), - params=dict(), + data=None, + params=None, auth=None, cookies=None, hooks=None): @@ -209,7 +220,6 @@ class Request(RequestHooksMixin): self.params = params self.auth = auth self.cookies = cookies - self.hooks = hooks def __repr__(self): return '<Request [%s]>' % (self.method) @@ -217,19 +227,17 @@ class Request(RequestHooksMixin): def prepare(self): """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it.""" p = PreparedRequest() - - p.prepare_method(self.method) - p.prepare_url(self.url, self.params) - p.prepare_headers(self.headers) - p.prepare_cookies(self.cookies) - p.prepare_body(self.data, self.files) - p.prepare_auth(self.auth, self.url) - # Note that prepare_auth must be last to enable authentication schemes - # such as OAuth to work on a fully prepared request. - - # This MUST go after prepare_auth. Authenticators could add a hook - p.prepare_hooks(self.hooks) - + p.prepare( + method=self.method, + url=self.url, + headers=self.headers, + files=self.files, + data=self.data, + params=self.params, + auth=self.auth, + cookies=self.cookies, + hooks=self.hooks, + ) return p @@ -264,9 +272,34 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): #: dictionary of callback hooks, for internal usage. self.hooks = default_hooks() + def prepare(self, method=None, url=None, headers=None, files=None, + data=None, params=None, auth=None, cookies=None, hooks=None): + """Prepares the the entire request with the given parameters.""" + + self.prepare_method(method) + self.prepare_url(url, params) + self.prepare_headers(headers) + self.prepare_cookies(cookies) + self.prepare_body(data, files) + self.prepare_auth(auth, url) + # Note that prepare_auth must be last to enable authentication schemes + # such as OAuth to work on a fully prepared request. + + # This MUST go after prepare_auth. Authenticators could add a hook + self.prepare_hooks(hooks) + def __repr__(self): return '<PreparedRequest [%s]>' % (self.method) + def copy(self): + p = PreparedRequest() + p.method = self.method + p.url = self.url + p.headers = self.headers + p.body = self.body + p.hooks = self.hooks + return p + def prepare_method(self, method): """Prepares the given HTTP method.""" self.method = method @@ -337,8 +370,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Prepares the given HTTP headers.""" if headers: - headers = dict((name.encode('ascii'), value) for name, value in headers.items()) - self.headers = CaseInsensitiveDict(headers) + self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items()) else: self.headers = CaseInsensitiveDict() @@ -352,7 +384,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): body = None content_type = None length = None - is_stream = False is_stream = all([ hasattr(data, '__iter__'), @@ -363,8 +394,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): try: length = super_len(data) - except (TypeError, AttributeError): - length = False + except (TypeError, AttributeError, UnsupportedOperation): + length = None if is_stream: body = data @@ -372,13 +403,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if files: raise NotImplementedError('Streamed bodies and files are mutually exclusive.') - if length: + if length is not None: self.headers['Content-Length'] = str(length) else: self.headers['Transfer-Encoding'] = 'chunked' - # Check if file, fo, generator, iterator. - # If not, run through normal process. - else: # Multi-part file uploads. if files: @@ -537,11 +565,22 @@ class Response(object): return iter_slices(self._content, chunk_size) def generate(): - while 1: - chunk = self.raw.read(chunk_size, decode_content=True) - if not chunk: - break - yield chunk + try: + # Special case for urllib3. + try: + for chunk in self.raw.stream(chunk_size, + decode_content=True): + yield chunk + except IncompleteRead as e: + raise ChunkedEncodingError(e) + except AttributeError: + # Standard file-like object. + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk + self._content_consumed = True gen = generate() @@ -683,4 +722,9 @@ class Response(object): raise HTTPError(http_error_msg, response=self) def close(self): + """Closes the underlying file descriptor and releases the connection + back to the pool. + + *Note: Should not normally need to be called explicitly.* + """ return self.raw.release_conn() |