diff options
Diffstat (limited to 'requests')
| -rw-r--r-- | requests/api.py | 100 | ||||
| -rw-r--r-- | requests/config.py | 15 | ||||
| -rw-r--r-- | requests/core.py | 8 | ||||
| -rw-r--r-- | requests/exceptions.py | 3 | ||||
| -rw-r--r-- | requests/hooks.py | 40 | ||||
| -rw-r--r-- | requests/models.py | 201 | ||||
| -rw-r--r-- | requests/monkeys.py | 62 | ||||
| -rw-r--r-- | requests/sessions.py | 84 | ||||
| -rw-r--r-- | requests/status_codes.py | 83 | ||||
| -rw-r--r-- | requests/structures.py | 49 | ||||
| -rw-r--r-- | requests/utils.py | 72 |
11 files changed, 591 insertions, 126 deletions
diff --git a/requests/api.py b/requests/api.py index 8e328d2..0cea63d 100644 --- a/requests/api.py +++ b/requests/api.py @@ -12,23 +12,28 @@ This module impliments the Requests API. """ import config -from .models import Request, Response, AuthManager, AuthObject, auth_manager +from .models import Request, Response, AuthObject +from .status_codes import codes +from .hooks import dispatch_hook +from .utils import cookiejar_from_dict +from urlparse import urlparse __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, - timeout=None, allow_redirects=False, proxies=None): + timeout=None, allow_redirects=False, proxies=None, hooks=None): - """Constructs and sends a :class:`Request <models.Request>`. Returns :class:`Response <models.Response>` object. + """Constructs and sends a :class:`Request <models.Request>`. + Returns :class:`Response <models.Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. @@ -36,7 +41,12 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - r = Request( + if cookies is None: + cookies = {} + + cookies = cookiejar_from_dict(cookies) + + args = dict( method = method, url = url, data = data, @@ -44,60 +54,65 @@ def request(method, url, headers = headers, cookiejar = cookies, files = files, - auth = auth or auth_manager.get_auth(url), + auth = auth, timeout = timeout or config.settings.timeout, allow_redirects = allow_redirects, - proxies = proxies + proxies = proxies or config.settings.proxies, ) + # Arguments manipulation hook. + args = dispatch_hook('args', hooks, args) + + r = Request(**args) + + # Pre-request hook. + r = dispatch_hook('pre_request', hooks, r) + + # Send the HTTP Request. r.send() + # Post-request hook. + r = dispatch_hook('post_request', hooks, r) + + # Response manipulation hook. + r.response = dispatch_hook('response', hooks, r.response) + return r.response -def get(url, - params=None, headers=None, cookies=None, auth=None, timeout=None, - proxies=None): +def get(url, **kwargs): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('GET', url, - params=params, headers=headers, cookies=cookies, auth=auth, - timeout=timeout, proxies=proxies) + return request('GET', url, **kwargs) -def head(url, - params=None, headers=None, cookies=None, auth=None, timeout=None, - proxies=None): +def head(url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('HEAD', url, - params=params, headers=headers, cookies=cookies, auth=auth, - timeout=timeout, proxies=proxies) + return request('HEAD', url, **kwargs) -def post(url, - data='', headers=None, files=None, cookies=None, auth=None, timeout=None, - allow_redirects=False, params=None, proxies=None): +def post(url, data='', **kwargs): """Sends a POST request. Returns :class:`Response` object. @@ -105,7 +120,7 @@ def post(url, :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -113,21 +128,17 @@ def post(url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('POST', url, - params=params, data=data, headers=headers, files=files, - cookies=cookies, auth=auth, timeout=timeout, - allow_redirects=allow_redirects, proxies=proxies) + return request('POST', url, data=data, **kwargs) -def put(url, data='', headers=None, files=None, cookies=None, auth=None, - timeout=None, allow_redirects=False, params=None, proxies=None): +def put(url, data='', **kwargs): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -135,21 +146,17 @@ def put(url, data='', headers=None, files=None, cookies=None, auth=None, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('PUT', url, - params=params, data=data, headers=headers, files=files, - cookies=cookies, auth=auth, timeout=timeout, - allow_redirects=allow_redirects, proxies=proxies) + return request('PUT', url, data=data, **kwargs) -def patch(url, data='', headers=None, files=None, cookies=None, auth=None, - timeout=None, allow_redirects=False, params=None, proxies=None): +def patch(url, data='', **kwargs): """Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. @@ -157,28 +164,21 @@ def patch(url, data='', headers=None, files=None, cookies=None, auth=None, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('PATCH', url, - params=params, data=data, headers=headers, files=files, - cookies=cookies, auth=auth, timeout=timeout, - allow_redirects=allow_redirects, proxies=proxies) + return request('PATCH', url, **kwargs) -def delete(url, - params=None, headers=None, cookies=None, auth=None, timeout=None, - allow_redirects=False, proxies=None): +def delete(url, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('DELETE', url, - params=params, headers=headers, cookies=cookies, auth=auth, - timeout=timeout, allow_redirects=allow_redirects, proxies=proxies) + return request('DELETE', url, **kwargs) diff --git a/requests/config.py b/requests/config.py index 0878da9..39be2ed 100644 --- a/requests/config.py +++ b/requests/config.py @@ -12,7 +12,7 @@ class Settings(object): _singleton = {} # attributes with defaults - __attrs__ = ('timeout', 'verbose') + __attrs__ = [] def __init__(self, **kwargs): super(Settings, self).__init__() @@ -53,4 +53,15 @@ class Settings(object): return None return object.__getattribute__(self, key) -settings = Settings()
\ No newline at end of file + +settings = Settings() + +settings.base_headers = {'User-Agent': 'python-requests.org'} +settings.accept_gzip = True +settings.proxies = None +settings.verbose = None +settings.timeout = None +settings.max_redirects = 30 + +#: Use socket.setdefaulttimeout() as fallback? +settings.timeout_fallback = True diff --git a/requests/core.py b/requests/core.py index 87f55e4..8ba34a2 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,14 +12,16 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.5.0' -__build__ = 0x000500 +__version__ = '0.6.1' +__build__ = 0x000601 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -from models import HTTPError, auth_manager +from models import HTTPError from api import * from exceptions import * +from sessions import session +from status_codes import codes from config import settings
\ No newline at end of file diff --git a/requests/exceptions.py b/requests/exceptions.py index eff7512..c08c614 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -21,3 +21,6 @@ class URLRequired(RequestException): class InvalidMethod(RequestException): """An inappropriate method was attempted.""" + +class TooManyRedirects(RequestException): + """Too many redirects.""" diff --git a/requests/hooks.py b/requests/hooks.py new file mode 100644 index 0000000..2938029 --- /dev/null +++ b/requests/hooks.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +""" +requests.hooks +~~~~~~~~~~~~~~ + +This module provides the capabilities for the Requests hooks system. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre_request``: + The Request object, directly before being sent. + +``post_request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + +""" + +import warnings + + +def dispatch_hook(key, hooks, hook_data): + """Dipatches a hook dictionary on a given peice of data.""" + + hooks = hooks or dict() + + if key in hooks: + try: + return hooks.get(key).__call__(hook_data) or hook_data + + except Exception, why: + warnings.warn(str(why)) + + return hook_data diff --git a/requests/models.py b/requests/models.py index 099f1c6..2d7fc8f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -12,18 +12,21 @@ import socket import zlib from urllib2 import HTTPError -from urlparse import urlparse +from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings -from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler +from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +from .utils import dict_from_cookiejar +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .status_codes import codes -REDIRECT_STATI = (301, 302, 303, 307) +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + class Request(object): @@ -31,34 +34,42 @@ class Request(object): Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): - socket.setdefaulttimeout(timeout) + #: Float describ the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) + self.timeout = timeout #: Request URL. self.url = url + #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`. self.headers = headers + #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files + #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method + #: Dictionary or byte of request body data to attach to the #: :class:`Request <models.Request>`. self.data = None + #: Dictionary or byte of querystring data to attach to the #: :class:`Request <models.Request>`. self.params = None + #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = proxies @@ -73,24 +84,36 @@ class Request(object): auth = AuthObject(*auth) if not auth: auth = auth_manager.get_auth(self.url) + #: :class:`AuthObject` to attach to :class:`Request <models.Request>`. self.auth = auth + #: CookieJar to attach to :class:`Request <models.Request>`. self.cookiejar = cookiejar + #: True if Request has been sent. self.sent = False - def __repr__(self): - return '<Request [%s]>' % (self.method) + # Header manipulation and defaults. + if settings.accept_gzip: + settings.base_headers.update({'Accept-Encoding': 'gzip'}) - def __setattr__(self, name, value): - if (name == 'method') and (value): - if not value in self._METHODS: - raise InvalidMethod() + if headers: + headers = CaseInsensitiveDict(self.headers) + else: + headers = CaseInsensitiveDict() + + for (k, v) in settings.base_headers.items(): + if k not in headers: + headers[k] = v + + self.headers = headers - object.__setattr__(self, name, value) + + def __repr__(self): + return '<Request [%s]>' % (self.method) def _checks(self): @@ -110,6 +133,7 @@ class Request(object): if self.auth: if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) @@ -141,7 +165,7 @@ class Request(object): return opener.open - def _build_response(self, resp): + def _build_response(self, resp, is_error=False): """Build internal :class:`Response <models.Response>` object from given response.""" def build(resp): @@ -151,17 +175,20 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.content = resp.read() + response.read = resp.read + response._resp = resp + response._close = resp.close + + if self.cookiejar: + + response.cookies = dict_from_cookiejar(self.cookiejar) + + except AttributeError: pass - if response.headers['content-encoding'] == 'gzip': - try: - response.content = zlib.decompress(response.content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - - # TODO: Support deflate + if is_error: + response.error = resp response.url = getattr(resp, 'url', None) @@ -172,30 +199,36 @@ class Request(object): r = build(resp) - if r.status_code in REDIRECT_STATI: - self.redirect = True - - if self.redirect: + if r.status_code in REDIRECT_STATI and not self.redirect: while ( ('location' in r.headers) and ((self.method in ('GET', 'HEAD')) or - (r.status_code is 303) or + (r.status_code is codes.see_other) or (self.allow_redirects)) ): + r.close() + + if not len(history) < settings.max_redirects: + raise TooManyRedirects() + history.append(r) url = r.headers['location'] + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: - parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is 303: + if r.status_code is codes.see_other: method = 'GET' else: method = self.method @@ -211,6 +244,7 @@ class Request(object): r.history = history self.response = r + self.response.request = self @staticmethod @@ -218,25 +252,34 @@ class Request(object): """Encode parameters in a piece of data. If the data supplied is a dictionary, encodes each parameter in it, and - returns the dictionary of encoded parameters, and a urlencoded version - of that. + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. Otherwise, assumes the data is already encoded appropriately, and returns it twice. """ if hasattr(data, 'items'): - result = {} - for (k, v) in data.items(): - result[k.encode('utf-8') if isinstance(k, unicode) else k] \ - = v.encode('utf-8') if isinstance(v, unicode) else v - return result, urllib.urlencode(result) + result = [] + for k, vs in data.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v)) + return result, urllib.urlencode(result, doseq=True) else: return data, data def _build_url(self): - """Build the actual URL to use""" + """Build the actual URL to use.""" + + # Support for unicode domain names and paths. + scheme, netloc, path, params, query, fragment = urlparse(self.url) + netloc = netloc.encode('idna') + if isinstance(path, unicode): + path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) + self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: if urlparse(self.url).query: @@ -257,6 +300,7 @@ class Request(object): :param anyway: If True, request will be sent, even if it has already been sent. """ + self._checks() success = False @@ -285,13 +329,32 @@ class Request(object): req = _Request(url, data=self._enc_data, method=self.method) if self.headers: - req.headers.update(self.headers) + for k,v in self.headers.iteritems(): + req.add_header(k, v) if not self.sent or anyway: try: opener = self._get_opener() - resp = opener(req) + try: + + resp = opener(req, timeout=self.timeout) + + except TypeError, err: + # timeout argument is new since Python v2.6 + if not 'timeout' in str(err): + raise + + if settings.timeout_fallback: + # fall-back and use global socket timeout (This is not thread-safe!) + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.timeout) + + resp = opener(req) + + if settings.timeout_fallback: + # restore gobal timeout + socket.setdefaulttimeout(old_timeout) if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) @@ -301,28 +364,19 @@ class Request(object): if isinstance(why.reason, socket.timeout): why = Timeout(why) - self._build_response(why) - if not self.redirect: - self.response.error = why + self._build_response(why, is_error=True) + else: self._build_response(resp) self.response.ok = True - self.response.cached = False - else: - self.response.cached = True self.sent = self.response.ok - return self.sent - def read(self, *args): - return self.response.read() - - class Response(object): """The core :class:`Response <models.Response>` object. All @@ -335,7 +389,7 @@ class Response(object): #: Raw content of the response, in bytes. #: If ``content-encoding`` of response was set to ``gzip``, the #: response data will be automatically deflated. - self.content = None + self._content = None #: Integer Code of responded HTTP Status. self.status_code = None #: Case-insensitive Dictionary of Response Headers. @@ -348,12 +402,14 @@ class Response(object): self.ok = False #: Resulting :class:`HTTPError` of request, if one occured. self.error = None - #: True, if the response :attr:`content` is cached locally. - self.cached = False #: A list of :class:`Response <models.Response>` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] + #: The Request that created the Response. + self.request = None + #: A dictionary of Cookies the server sent back. + self.cookies = None def __repr__(self): @@ -365,17 +421,31 @@ class Response(object): return not self.error + def __getattr__(self, name): + """Read and returns the full stream when accessing to :attr: `content`""" + if name == 'content': + if self._content is not None: + return self._content + self._content = self.read() + if self.headers.get('content-encoding', '') == 'gzip': + try: + self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) + except zlib.error: + pass + return self._content + else: + raise AttributeError + def raise_for_status(self): - """Raises stored :class:`HTTPError`, if one occured.""" + """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: raise self.error - def read(self, *args): - """Returns :attr:`content`. Used for file-like object compatiblity.""" - - return self.content - + def close(self): + if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): + self._resp.fp._sock.recv = None + self._close() class AuthManager(object): """Requests Authentication Manager.""" @@ -450,8 +520,10 @@ class AuthManager(object): def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" + # note HTTP URLs do not have a userinfo component parts = urllib2.urlparse.urlsplit(uri) + if parts[1]: # URI scheme = parts[0] @@ -462,7 +534,9 @@ class AuthManager(object): scheme = None authority = uri path = '/' + host, port = urllib2.splitport(authority) + if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, @@ -532,17 +606,18 @@ class AuthObject(object): _handlers = { 'basic': HTTPBasicAuthHandler, + 'forced_basic': HTTPForcedBasicAuthHandler, 'digest': HTTPDigestAuthHandler, 'proxy_basic': urllib2.ProxyBasicAuthHandler, 'proxy_digest': urllib2.ProxyDigestAuthHandler } - def __init__(self, username, password, handler='basic', realm=None): + def __init__(self, username, password, handler='forced_basic', realm=None): self.username = username self.password = password self.realm = realm if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler) + self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) else: self.handler = handler diff --git a/requests/monkeys.py b/requests/monkeys.py index 41cd370..c838071 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -9,7 +9,7 @@ Urllib2 Monkey patches. """ import urllib2 - +import re class Request(urllib2.Request): """Hidden wrapper around the urllib2.Request object. Allows for manual @@ -26,8 +26,9 @@ class Request(urllib2.Request): return urllib2.Request.get_method(self) -class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): +class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """HTTP Redirect handler.""" def http_error_301(self, req, fp, code, msg, headers): pass @@ -36,10 +37,13 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with authentication loop fixes.""" def __init__(self, *args, **kwargs): urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) self.retried_req = None + self.retried = 0 + def reset_retry_count(self): # Python 2.6.5 will call this on 401 or 407 errors and thus loop @@ -47,6 +51,7 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): # http_error_auth_reqed instead. pass + def http_error_auth_reqed(self, auth_header, host, req, headers): # Reset the retry counter once for each request. if req is not self.retried_req: @@ -59,6 +64,59 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): +class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with forced Authentication.""" + + auth_header = 'Authorization' + rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' + 'realm=(["\'])(.*?)\\2', re.I) + + def __init__(self, *args, **kwargs): + HTTPBasicAuthHandler.__init__(self, *args, **kwargs) + + + def http_error_401(self, req, fp, code, msg, headers): + url = req.get_full_url() + response = self._http_error_auth_reqed('www-authenticate', url, req, headers) + self.reset_retry_count() + return response + + http_error_404 = http_error_401 + + + def _http_error_auth_reqed(self, authreq, host, req, headers): + + authreq = headers.get(authreq, None) + + if self.retried > 5: + # retry sending the username:password 5 times before failing. + raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", + headers, None) + else: + self.retried += 1 + + if authreq: + + mo = self.rx.search(authreq) + + if mo: + scheme, quote, realm = mo.groups() + + if scheme.lower() == 'basic': + response = self.retry_http_basic_auth(host, req, realm) + + if response and response.code not in (401, 404): + self.retried = 0 + return response + else: + response = self.retry_http_basic_auth(host, req, 'Realm') + + if response and response.code not in (401, 404): + self.retried = 0 + return response + + + class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): def __init__(self, *args, **kwargs): diff --git a/requests/sessions.py b/requests/sessions.py new file mode 100644 index 0000000..50b09f6 --- /dev/null +++ b/requests/sessions.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +""" +requests.session +~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +requests (cookies, auth, proxies). + +""" + +import cookielib + +from . import api +from .utils import add_dict_to_cookiejar + + + +class Session(object): + """A Requests session.""" + + __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] + + + def __init__(self, **kwargs): + + # Set up a CookieJar to be used by default + self.cookies = cookielib.FileCookieJar() + + # Map args from kwargs to instance-local variables + map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), + kwargs.iterkeys(), kwargs.itervalues()) + + # Map and wrap requests.api methods + self._map_api_methods() + + + def __repr__(self): + return '<requests-client at 0x%x>' % (id(self)) + + def __enter__(self): + return self + + def __exit__(self, *args): + # print args + pass + + + def _map_api_methods(self): + """Reads each available method from requests.api and decorates + them with a wrapper, which inserts any instance-local attributes + (from __attrs__) that have been set, combining them with **kwargs. + """ + + def pass_args(func): + def wrapper_func(*args, **kwargs): + inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() + if k in self.__attrs__) + # Combine instance-local values with kwargs values, with + # priority to values in kwargs + kwargs = dict(inst_attrs.items() + kwargs.items()) + + # If a session request has a cookie_dict, inject the + # values into the existing CookieJar instead. + if isinstance(kwargs.get('cookies', None), dict): + kwargs['cookies'] = add_dict_to_cookiejar( + inst_attrs['cookies'], kwargs['cookies'] + ) + + if kwargs.get('headers', None) and inst_attrs.get('headers', None): + kwargs['headers'].update(inst_attrs['headers']) + + return func(*args, **kwargs) + return wrapper_func + + # Map and decorate each function available in requests.api + map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))), + api.__all__) + + +def session(**kwargs): + """Returns a :class:`Session` for context-managment.""" + + return Session(**kwargs)
\ No newline at end of file diff --git a/requests/status_codes.py b/requests/status_codes.py new file mode 100644 index 0000000..a809de6 --- /dev/null +++ b/requests/status_codes.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from .structures import LookupDict + +_codes = { + + # Informational. + 100: ('continue',), + 101: ('switching_protocols',), + 102: ('processing',), + 103: ('checkpoint',), + 122: ('uri_too_long', 'request_uri_too_long'), + 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'), + 201: ('created',), + 202: ('accepted',), + 203: ('non_authoritative_info', 'non_authoritative_information'), + 204: ('no_content',), + 205: ('reset_content', 'reset'), + 206: ('partial_content', 'partial'), + 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), + 208: ('im_used',), + + # Redirection. + 300: ('multiple_choices',), + 301: ('moved_permanently', 'moved', '\\o-'), + 302: ('found',), + 303: ('see_other', 'other'), + 304: ('not_modified',), + 305: ('use_proxy',), + 306: ('switch_proxy',), + 307: ('temporary_redirect', 'temporary_moved', 'temporary'), + 308: ('resume_incomplete', 'resume'), + + # Client Error. + 400: ('bad_request', 'bad'), + 401: ('unauthorized',), + 402: ('payment_required', 'payment'), + 403: ('forbidden',), + 404: ('not_found', '-o-'), + 405: ('method_not_allowed', 'not_allowed'), + 406: ('not_acceptable',), + 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), + 408: ('request_timeout', 'timeout'), + 409: ('conflict',), + 410: ('gone',), + 411: ('length_required',), + 412: ('precondition_failed', 'precondition'), + 413: ('request_entity_too_large',), + 414: ('request_uri_too_large',), + 415: ('unspported_media_type', 'unspported_media', 'media_type'), + 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), + 417: ('expectation_failed',), + 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 422: ('unprocessable_entity', 'unprocessable'), + 423: ('locked',), + 424: ('failed_depdendency', 'depdendency'), + 425: ('unordered_collection', 'unordered'), + 426: ('upgrade_required', 'upgrade'), + 444: ('no_response', 'none'), + 449: ('retry_with', 'retry'), + 450: ('blocked_by_windows_parental_controls', 'parental_controls'), + 499: ('client_closed_request',), + + # Server Error. + 500: ('internal_server_error', 'server_error', '/o\\'), + 501: ('not_implemented',), + 502: ('bad_gateway',), + 503: ('service_unavailable', 'unavailable'), + 504: ('gateway_timeout',), + 505: ('http_version_not_supported', 'http_version'), + 506: ('variant_also_negotiates',), + 507: ('insufficient_storage',), + 509: ('bandwidth_limit_exceeded', 'bandwidth'), + 510: ('not_extended',), +} + +codes = LookupDict(name='status_codes') + +for (code, titles) in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith('\\'): + setattr(codes, title.upper(), code)
\ No newline at end of file diff --git a/requests/structures.py b/requests/structures.py index bfee7b1..d068bf9 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -9,20 +9,57 @@ Datastructures that power Requests. """ class CaseInsensitiveDict(dict): - """Case-insensitive Dictionary for :class:`Response <models.Response>` Headers. + """Case-insensitive Dictionary For example, ``headers['content-encoding']`` will return the value of a ``'Content-Encoding'`` response header.""" - def _lower_keys(self): - return map(str.lower, self.keys()) + @property + def lower_keys(self): + if not hasattr(self, '_lower_keys') or not self._lower_keys: + self._lower_keys = dict((k.lower(), k) for k in self.iterkeys()) + return self._lower_keys + def _clear_lower_keys(self): + if hasattr(self, '_lower_keys'): + self._lower_keys.clear() - def __contains__(self, key): - return key.lower() in self._lower_keys() + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + self._clear_lower_keys() + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._lower_keys.clear() + def __contains__(self, key): + return key.lower() in self.lower_keys def __getitem__(self, key): # We allow fall-through here, so values default to None if key in self: - return self.items()[self._lower_keys().index(key.lower())][1] + return dict.__getitem__(self, self.lower_keys[key.lower()]) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + +class LookupDict(dict): + """Dictionary lookup object.""" + + def __init__(self, name=None): + self.name = name + super(LookupDict, self).__init__() + + def __repr__(self): + return '<lookup \'%s\'>' % (self.name) + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + + return self.__dict__.get(key, None) + + def get(self, key, default=None): + return self.__dict__.get(key, default)
\ No newline at end of file diff --git a/requests/utils.py b/requests/utils.py new file mode 100644 index 0000000..8ac78b4 --- /dev/null +++ b/requests/utils.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +""" +requests.utils +~~~~~~~~~~~~~~ + +This module provides utlity functions that are used within Requests +that are also useful for external consumption. + +""" + +import cookielib + + +def dict_from_cookiejar(cookiejar): + """Returns a key/value dictionary from a CookieJar.""" + + cookie_dict = {} + + for _, cookies in cookiejar._cookies.items(): + for _, cookies in cookies.items(): + for cookie in cookies.values(): + # print cookie + cookie_dict[cookie.name] = cookie.value + + return cookie_dict + + +def cookiejar_from_dict(cookie_dict): + """Returns a CookieJar from a key/value dictionary.""" + + # return cookiejar if one was passed in + if isinstance(cookie_dict, cookielib.CookieJar): + return cookie_dict + + # create cookiejar + cj = cookielib.CookieJar() + + cj = add_dict_to_cookiejar(cj, cookie_dict) + + return cj + + +def add_dict_to_cookiejar(cj, cookie_dict): + """Returns a CookieJar from a key/value dictionary.""" + + for k, v in cookie_dict.items(): + + cookie = cookielib.Cookie( + version=0, + name=k, + value=v, + port=None, + port_specified=False, + domain='', + domain_specified=False, + domain_initial_dot=False, + path='/', + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False + ) + + # add cookie to cookiejar + cj.set_cookie(cookie) + + return cj |