diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 201 |
1 files changed, 138 insertions, 63 deletions
diff --git a/requests/models.py b/requests/models.py index 099f1c6..2d7fc8f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -12,18 +12,21 @@ import socket import zlib from urllib2 import HTTPError -from urlparse import urlparse +from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings -from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler +from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +from .utils import dict_from_cookiejar +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .status_codes import codes -REDIRECT_STATI = (301, 302, 303, 307) +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + class Request(object): @@ -31,34 +34,42 @@ class Request(object): Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): - socket.setdefaulttimeout(timeout) + #: Float describ the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) + self.timeout = timeout #: Request URL. self.url = url + #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`. self.headers = headers + #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files + #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method + #: Dictionary or byte of request body data to attach to the #: :class:`Request <models.Request>`. self.data = None + #: Dictionary or byte of querystring data to attach to the #: :class:`Request <models.Request>`. self.params = None + #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = proxies @@ -73,24 +84,36 @@ class Request(object): auth = AuthObject(*auth) if not auth: auth = auth_manager.get_auth(self.url) + #: :class:`AuthObject` to attach to :class:`Request <models.Request>`. self.auth = auth + #: CookieJar to attach to :class:`Request <models.Request>`. self.cookiejar = cookiejar + #: True if Request has been sent. self.sent = False - def __repr__(self): - return '<Request [%s]>' % (self.method) + # Header manipulation and defaults. + if settings.accept_gzip: + settings.base_headers.update({'Accept-Encoding': 'gzip'}) - def __setattr__(self, name, value): - if (name == 'method') and (value): - if not value in self._METHODS: - raise InvalidMethod() + if headers: + headers = CaseInsensitiveDict(self.headers) + else: + headers = CaseInsensitiveDict() + + for (k, v) in settings.base_headers.items(): + if k not in headers: + headers[k] = v + + self.headers = headers - object.__setattr__(self, name, value) + + def __repr__(self): + return '<Request [%s]>' % (self.method) def _checks(self): @@ -110,6 +133,7 @@ class Request(object): if self.auth: if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) @@ -141,7 +165,7 @@ class Request(object): return opener.open - def _build_response(self, resp): + def _build_response(self, resp, is_error=False): """Build internal :class:`Response <models.Response>` object from given response.""" def build(resp): @@ -151,17 +175,20 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.content = resp.read() + response.read = resp.read + response._resp = resp + response._close = resp.close + + if self.cookiejar: + + response.cookies = dict_from_cookiejar(self.cookiejar) + + except AttributeError: pass - if response.headers['content-encoding'] == 'gzip': - try: - response.content = zlib.decompress(response.content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - - # TODO: Support deflate + if is_error: + response.error = resp response.url = getattr(resp, 'url', None) @@ -172,30 +199,36 @@ class Request(object): r = build(resp) - if r.status_code in REDIRECT_STATI: - self.redirect = True - - if self.redirect: + if r.status_code in REDIRECT_STATI and not self.redirect: while ( ('location' in r.headers) and ((self.method in ('GET', 'HEAD')) or - (r.status_code is 303) or + (r.status_code is codes.see_other) or (self.allow_redirects)) ): + r.close() + + if not len(history) < settings.max_redirects: + raise TooManyRedirects() + history.append(r) url = r.headers['location'] + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: - parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is 303: + if r.status_code is codes.see_other: method = 'GET' else: method = self.method @@ -211,6 +244,7 @@ class Request(object): r.history = history self.response = r + self.response.request = self @staticmethod @@ -218,25 +252,34 @@ class Request(object): """Encode parameters in a piece of data. If the data supplied is a dictionary, encodes each parameter in it, and - returns the dictionary of encoded parameters, and a urlencoded version - of that. + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. Otherwise, assumes the data is already encoded appropriately, and returns it twice. """ if hasattr(data, 'items'): - result = {} - for (k, v) in data.items(): - result[k.encode('utf-8') if isinstance(k, unicode) else k] \ - = v.encode('utf-8') if isinstance(v, unicode) else v - return result, urllib.urlencode(result) + result = [] + for k, vs in data.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v)) + return result, urllib.urlencode(result, doseq=True) else: return data, data def _build_url(self): - """Build the actual URL to use""" + """Build the actual URL to use.""" + + # Support for unicode domain names and paths. + scheme, netloc, path, params, query, fragment = urlparse(self.url) + netloc = netloc.encode('idna') + if isinstance(path, unicode): + path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) + self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: if urlparse(self.url).query: @@ -257,6 +300,7 @@ class Request(object): :param anyway: If True, request will be sent, even if it has already been sent. """ + self._checks() success = False @@ -285,13 +329,32 @@ class Request(object): req = _Request(url, data=self._enc_data, method=self.method) if self.headers: - req.headers.update(self.headers) + for k,v in self.headers.iteritems(): + req.add_header(k, v) if not self.sent or anyway: try: opener = self._get_opener() - resp = opener(req) + try: + + resp = opener(req, timeout=self.timeout) + + except TypeError, err: + # timeout argument is new since Python v2.6 + if not 'timeout' in str(err): + raise + + if settings.timeout_fallback: + # fall-back and use global socket timeout (This is not thread-safe!) + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.timeout) + + resp = opener(req) + + if settings.timeout_fallback: + # restore gobal timeout + socket.setdefaulttimeout(old_timeout) if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) @@ -301,28 +364,19 @@ class Request(object): if isinstance(why.reason, socket.timeout): why = Timeout(why) - self._build_response(why) - if not self.redirect: - self.response.error = why + self._build_response(why, is_error=True) + else: self._build_response(resp) self.response.ok = True - self.response.cached = False - else: - self.response.cached = True self.sent = self.response.ok - return self.sent - def read(self, *args): - return self.response.read() - - class Response(object): """The core :class:`Response <models.Response>` object. All @@ -335,7 +389,7 @@ class Response(object): #: Raw content of the response, in bytes. #: If ``content-encoding`` of response was set to ``gzip``, the #: response data will be automatically deflated. - self.content = None + self._content = None #: Integer Code of responded HTTP Status. self.status_code = None #: Case-insensitive Dictionary of Response Headers. @@ -348,12 +402,14 @@ class Response(object): self.ok = False #: Resulting :class:`HTTPError` of request, if one occured. self.error = None - #: True, if the response :attr:`content` is cached locally. - self.cached = False #: A list of :class:`Response <models.Response>` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] + #: The Request that created the Response. + self.request = None + #: A dictionary of Cookies the server sent back. + self.cookies = None def __repr__(self): @@ -365,17 +421,31 @@ class Response(object): return not self.error + def __getattr__(self, name): + """Read and returns the full stream when accessing to :attr: `content`""" + if name == 'content': + if self._content is not None: + return self._content + self._content = self.read() + if self.headers.get('content-encoding', '') == 'gzip': + try: + self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) + except zlib.error: + pass + return self._content + else: + raise AttributeError + def raise_for_status(self): - """Raises stored :class:`HTTPError`, if one occured.""" + """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: raise self.error - def read(self, *args): - """Returns :attr:`content`. Used for file-like object compatiblity.""" - - return self.content - + def close(self): + if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): + self._resp.fp._sock.recv = None + self._close() class AuthManager(object): """Requests Authentication Manager.""" @@ -450,8 +520,10 @@ class AuthManager(object): def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" + # note HTTP URLs do not have a userinfo component parts = urllib2.urlparse.urlsplit(uri) + if parts[1]: # URI scheme = parts[0] @@ -462,7 +534,9 @@ class AuthManager(object): scheme = None authority = uri path = '/' + host, port = urllib2.splitport(authority) + if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, @@ -532,17 +606,18 @@ class AuthObject(object): _handlers = { 'basic': HTTPBasicAuthHandler, + 'forced_basic': HTTPForcedBasicAuthHandler, 'digest': HTTPDigestAuthHandler, 'proxy_basic': urllib2.ProxyBasicAuthHandler, 'proxy_digest': urllib2.ProxyDigestAuthHandler } - def __init__(self, username, password, handler='basic', realm=None): + def __init__(self, username, password, handler='forced_basic', realm=None): self.username = username self.password = password self.realm = realm if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler) + self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) else: self.handler = handler |