diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 204 |
1 files changed, 103 insertions, 101 deletions
diff --git a/requests/models.py b/requests/models.py index 60f58d2..fbdfb56 100644 --- a/requests/models.py +++ b/requests/models.py @@ -7,6 +7,7 @@ requests.models This module contains the primary objects that power Requests. """ +import json import os from datetime import datetime @@ -15,6 +16,7 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth +from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError, LocationParseError from .packages.urllib3.exceptions import SSLError as _SSLError @@ -27,20 +29,22 @@ from .exceptions import ( URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL) from .utils import ( get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, - dict_from_string, stream_decode_response_unicode, get_netrc_auth, + stream_decode_response_unicode, get_netrc_auth, get_environ_proxies, DEFAULT_CA_BUNDLE_PATH) from .compat import ( - urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, - SimpleCookie, is_py2) + cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, + StringIO, is_py2) # Import chardet if it is available. try: import chardet + # hush pyflakes + chardet except ImportError: - pass + chardet = None REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - +CONTENT_CHUNK_SIZE = 10 * 1024 class Request(object): """The :class:`Request <Request>` object. It carries out all functionality of @@ -109,14 +113,11 @@ class Request(object): # If no proxies are given, allow configuration by environment variables # HTTP_PROXY and HTTPS_PROXY. if not self.proxies and self.config.get('trust_env'): - if 'HTTP_PROXY' in os.environ: - self.proxies['http'] = os.environ['HTTP_PROXY'] - if 'HTTPS_PROXY' in os.environ: - self.proxies['https'] = os.environ['HTTPS_PROXY'] + self.proxies = get_environ_proxies() - self.data, self._enc_data = self._encode_params(data) - self.params, self._enc_params = self._encode_params(params) - self.files, self._enc_files = self._encode_files(files) + self.data = data + self.params = params + self.files = files #: :class:`Response <Response>` instance, containing #: content and metadata of HTTP Response, once :attr:`sent <send>`. @@ -126,7 +127,10 @@ class Request(object): self.auth = auth #: CookieJar to attach to :class:`Request <Request>`. - self.cookies = dict(cookies or []) + if isinstance(cookies, cookielib.CookieJar): + self.cookies = cookies + else: + self.cookies = cookiejar_from_dict(cookies) #: True if Request has been sent. self.sent = False @@ -193,16 +197,11 @@ class Request(object): # Set encoding. response.encoding = get_encoding_from_headers(response.headers) - # Start off with our local cookies. - cookies = self.cookies or dict() - # Add new cookies from the server. - if 'set-cookie' in response.headers: - cookie_header = response.headers['set-cookie'] - cookies = dict_from_string(cookie_header) + extract_cookies_to_jar(self.cookies, self, resp) # Save cookies in Response. - response.cookies = cookies + response.cookies = self.cookies # No exceptions were harmed in the making of this request. response.error = getattr(resp, 'error', None) @@ -220,8 +219,6 @@ class Request(object): r = build(resp) - self.cookies.update(r.cookies) - if r.status_code in REDIRECT_STATI and not self.redirect: while (('location' in r.headers) and @@ -239,6 +236,7 @@ class Request(object): url = r.headers['location'] data = self.data + files = self.files # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -257,6 +255,7 @@ class Request(object): if r.status_code is codes.see_other: method = 'GET' data = None + files = None else: method = self.method @@ -266,10 +265,12 @@ class Request(object): if r.status_code in (codes.moved, codes.found) and self.method == 'POST': method = 'GET' data = None + files = None if (r.status_code == 303) and self.method != 'HEAD': method = 'GET' data = None + files = None # Remove the cookie headers that were sent. headers = self.headers @@ -281,7 +282,7 @@ class Request(object): request = Request( url=url, headers=headers, - files=self.files, + files=files, method=method, params=self.session.params, auth=self.auth, @@ -299,46 +300,46 @@ class Request(object): request.send() r = request.response - self.cookies.update(r.cookies) r.history = history self.response = r self.response.request = self - self.response.cookies.update(self.cookies) @staticmethod def _encode_params(data): """Encode parameters in a piece of data. - If the data supplied is a dictionary, encodes each parameter in it, and - returns a list of tuples containing the encoded parameters, and a urlencoded - version of that. - - Otherwise, assumes the data is already encoded appropriately, and - returns it twice. + Will successfully encode parameters when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. """ if isinstance(data, bytes): - return data, data - - if hasattr(data, '__iter__') and not isinstance(data, str): - data = dict(data) + return data + if isinstance(data, str): + return data + elif hasattr(data, '__iter__'): + try: + dict(data) + except ValueError: + raise ValueError('Unable to encode lists with elements that are not 2-tuples.') - if hasattr(data, 'items'): + params = list(data.items() if isinstance(data, dict) else data) result = [] - for k, vs in list(data.items()): + for k, vs in params: for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, str) else k, - v.encode('utf-8') if isinstance(v, str) else v)) - return result, urlencode(result, doseq=True) + result.append( + (k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v)) + return urlencode(result, doseq=True) else: - return data, data + return data - def _encode_files(self,files): + def _encode_files(self, files): if (not files) or isinstance(self.data, str): - return None, None + return None try: fields = self.data.copy() @@ -352,11 +353,13 @@ class Request(object): else: fn = guess_filename(v) or k fp = v + if isinstance(fp, (bytes, str)): + fp = StringIO(fp) fields.update({k: (fn, fp.read())}) (body, content_type) = encode_multipart_formdata(fields) - return files, (body, content_type) + return (body, content_type) @property def full_url(self): @@ -381,7 +384,6 @@ class Request(object): if not path: path = '/' - if is_py2: if isinstance(scheme, str): scheme = scheme.encode('utf-8') @@ -398,11 +400,12 @@ class Request(object): url = (urlunparse([scheme, netloc, path, params, query, fragment])) - if self._enc_params: + enc_params = self._encode_params(self.params) + if enc_params: if urlparse(url).query: - url = '%s&%s' % (url, self._enc_params) + url = '%s&%s' % (url, enc_params) else: - url = '%s?%s' % (url, self._enc_params) + url = '%s?%s' % (url, enc_params) if self.config.get('encode_uri', True): url = requote_uri(url) @@ -439,7 +442,7 @@ class Request(object): self.hooks[event].append(hook) - def deregister_hook(self,event,hook): + def deregister_hook(self, event, hook): """Deregister a previously registered hook. Returns True if the hook existed, False if not. """ @@ -464,6 +467,10 @@ class Request(object): # Build the URL url = self.full_url + # Pre-request hook. + r = dispatch_hook('pre_request', self.hooks, self) + self.__dict__.update(r.__dict__) + # Logging if self.config.get('verbose'): self.config.get('verbose').write('%s %s %s\n' % ( @@ -474,22 +481,6 @@ class Request(object): body = None content_type = None - # Multi-part file uploads. - if self.files: - (body, content_type) = self._enc_files - else: - if self.data: - - body = self._enc_data - if isinstance(self.data, str): - content_type = None - else: - content_type = 'application/x-www-form-urlencoded' - - # Add content-type if it wasn't explicitly provided. - if (content_type) and (not 'content-type' in self.headers): - self.headers['Content-Type'] = content_type - # Use .netrc auth if none was provided. if not self.auth and self.config.get('trust_env'): self.auth = get_netrc_auth(url) @@ -505,6 +496,22 @@ class Request(object): # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) + # Multi-part file uploads. + if self.files: + (body, content_type) = self._encode_files(self.files) + else: + if self.data: + + body = self._encode_params(self.data) + if isinstance(self.data, str): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' + + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type + _p = urlparse(url) proxy = self.proxies.get(_p.scheme) @@ -523,6 +530,7 @@ class Request(object): conn = self._poolmanager.connection_from_url(url) else: conn = connectionpool.connection_from_url(url) + self.headers['Connection'] = 'close' except LocationParseError as e: raise InvalidURL(e) @@ -563,24 +571,14 @@ class Request(object): if not self.sent or anyway: - if self.cookies: - - # Skip if 'cookie' header is explicitly set. - if 'cookie' not in self.headers: - - # Simple cookie with our dict. - c = SimpleCookie() - for (k, v) in list(self.cookies.items()): - c[k] = v - - # Turn it into a header. - cookie_header = c.output(header='', sep='; ').strip() - - # Attach Cookie header to request. + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: + cookie_header = get_cookie_header(self.cookies, self) + if cookie_header is not None: self.headers['Cookie'] = cookie_header - # Pre-request hook. - r = dispatch_hook('pre_request', self.hooks, self) + # Pre-send hook. + r = dispatch_hook('pre_send', self.hooks, self) self.__dict__.update(r.__dict__) try: @@ -621,7 +619,15 @@ class Request(object): else: raise - self._build_response(r) + # build_response can throw TooManyRedirects + try: + self._build_response(r) + except RequestException as e: + if self.config.get('safe_mode', False): + # In safe mode, catch the exception + self.response.error = e + else: + raise # Response manipulation hook. self.response = dispatch_hook('response', self.hooks, self.response) @@ -681,8 +687,8 @@ class Response(object): #: The :class:`Request <Request>` that created the Response. self.request = None - #: A dictionary of Cookies the server sent back. - self.cookies = {} + #: A CookieJar of Cookies the server sent back. + self.cookies = None #: Dictionary of configurations for this request. self.config = {} @@ -748,7 +754,7 @@ class Response(object): chunk = pending + chunk lines = chunk.splitlines() - if lines[-1][-1] == chunk[-1]: + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None @@ -773,7 +779,7 @@ class Response(object): if self.status_code is 0: self._content = None else: - self._content = bytes().join(self.iter_content()) or bytes() + self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() except AttributeError: self._content = None @@ -781,16 +787,6 @@ class Response(object): self._content_consumed = True return self._content - def _detected_encoding(self): - try: - detected = chardet.detect(self.content) or {} - return detected.get('encoding') - - # Trust that chardet isn't available or something went terribly wrong. - except Exception: - pass - - @property def text(self): """Content of the response, in unicode. @@ -803,9 +799,10 @@ class Response(object): content = None encoding = self.encoding - # Fallback to auto-detected encoding if chardet is available. + # Fallback to auto-detected encoding. if self.encoding is None: - encoding = self._detected_encoding() + if chardet is not None: + encoding = chardet.detect(self.content)['encoding'] # Decode unicode from given encoding. try: @@ -816,11 +813,17 @@ class Response(object): # # So we try blindly encoding. content = str(self.content, errors='replace') - except (UnicodeError, TypeError): - pass return content + @property + def json(self): + """Returns the json-encoded content of a request, if any.""" + try: + return json.loads(self.text or self.content) + except ValueError: + return None + def raise_for_status(self, allow_redirects=True): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" @@ -837,7 +840,6 @@ class Response(object): http_error.response = self raise http_error - elif (self.status_code >= 500) and (self.status_code < 600): http_error = HTTPError('%s Server Error' % self.status_code) http_error.response = self |