diff options
Diffstat (limited to 'requests')
| -rw-r--r-- | requests/api.py | 121 | ||||
| -rw-r--r-- | requests/config.py | 2 | ||||
| -rw-r--r-- | requests/core.py | 4 | ||||
| -rw-r--r-- | requests/models.py | 107 | ||||
| -rw-r--r-- | requests/monkeys.py | 1 | ||||
| -rw-r--r-- | requests/structures.py | 44 |
6 files changed, 175 insertions, 104 deletions
diff --git a/requests/api.py b/requests/api.py index 0e27410..8e328d2 100644 --- a/requests/api.py +++ b/requests/api.py @@ -11,37 +11,43 @@ This module impliments the Requests API. """ -import requests import config from .models import Request, Response, AuthManager, AuthObject, auth_manager -__all__ = ('request', 'get', 'head', 'post', 'put', 'delete') +__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') +def request(method, url, + params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=None, allow_redirects=False, proxies=None): - -def request(method, url, **kwargs): """Constructs and sends a :class:`Request <models.Request>`. Returns :class:`Response <models.Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET/HEAD/DELETE Parameters to send with the :class:`Request`. - :param data: (optional) Bytes/Dictionary of PUT/POST Data to send with the :class:`Request`. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - data = kwargs.pop('data', dict()) or kwargs.pop('params', dict()) - - r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', dict()), - cookiejar=kwargs.get('cookies', None), - files=kwargs.get('files', None), - auth=kwargs.get('auth', auth_manager.get_auth(url)), - timeout=kwargs.get('timeout', config.settings.timeout), - allow_redirects=kwargs.get('allow_redirects', None) + + r = Request( + method = method, + url = url, + data = data, + params = params, + headers = headers, + cookiejar = cookies, + files = files, + auth = auth or auth_manager.get_auth(url), + timeout = timeout or config.settings.timeout, + allow_redirects = allow_redirects, + proxies = proxies ) r.send() @@ -49,73 +55,130 @@ def request(method, url, **kwargs): return r.response -def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs): +def get(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + proxies=None): + """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('GET', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, proxies=proxies) -def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs): +def head(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + proxies=None): + """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('HEAD', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, proxies=proxies) + +def post(url, + data='', headers=None, files=None, cookies=None, auth=None, timeout=None, + allow_redirects=False, params=None, proxies=None): -def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary of POST data to send with the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + return request('POST', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) -def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs): +def put(url, data='', headers=None, files=None, cookies=None, auth=None, + timeout=None, allow_redirects=False, params=None, proxies=None): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Bytes of PUT Data to send with the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + return request('PUT', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) + + +def patch(url, data='', headers=None, files=None, cookies=None, auth=None, + timeout=None, allow_redirects=False, params=None, proxies=None): + """Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('PATCH', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) + +def delete(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + allow_redirects=False, proxies=None): -def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of DELETE Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('DELETE', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, allow_redirects=allow_redirects, proxies=proxies) diff --git a/requests/config.py b/requests/config.py index 63d3fa9..0878da9 100644 --- a/requests/config.py +++ b/requests/config.py @@ -12,7 +12,7 @@ class Settings(object): _singleton = {} # attributes with defaults - __attrs__ = ('timeout',) + __attrs__ = ('timeout', 'verbose') def __init__(self, **kwargs): super(Settings, self).__init__() diff --git a/requests/core.py b/requests/core.py index 7f3d723..87f55e4 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.4.1' -__build__ = 0x000401 +__version__ = '0.5.0' +__build__ = 0x000500 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' diff --git a/requests/models.py b/requests/models.py index 2c3241d..099f1c6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -6,7 +6,6 @@ requests.models """ -import requests import urllib import urllib2 import socket @@ -14,7 +13,9 @@ import zlib from urllib2 import HTTPError from urlparse import urlparse +from datetime import datetime +from .config import settings from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode @@ -22,17 +23,20 @@ from .packages.poster.streaminghttp import register_openers, get_handlers from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +REDIRECT_STATI = (301, 302, 303, 307) + class Request(object): """The :class:`Request <models.Request>` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE') + _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, - data=dict(), auth=None, cookiejar=None, timeout=None, - redirect=True, allow_redirects=False): + def __init__(self, + url=None, headers=dict(), files=None, method=None, data=dict(), + params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, + allow_redirects=False, proxies=None): socket.setdefaulttimeout(timeout) @@ -44,23 +48,22 @@ class Request(object): self.files = files #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method - #: Form or Byte data to attach to the :class:`Request <models.Request>`. - self.data = dict() + #: Dictionary or byte of request body data to attach to the + #: :class:`Request <models.Request>`. + self.data = None + #: Dictionary or byte of querystring data to attach to the + #: :class:`Request <models.Request>`. + self.params = None #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) + self.proxies = proxies - if hasattr(data, 'items'): - for (k, v) in data.items(): - self.data.update({ - k.encode('utf-8') if isinstance(k, unicode) else k: - v.encode('utf-8') if isinstance(v, unicode) else v - }) - self._enc_data = urllib.urlencode(self.data) - else: - self._enc_data = self.data = data + self.data, self._enc_data = self._encode_params(data) + self.params, self._enc_params = self._encode_params(params) #: :class:`Response <models.Response>` instance, containing #: content and metadata of HTTP Response, once :attr:`sent <send>`. @@ -113,6 +116,8 @@ class Request(object): _handlers.append(self.auth.handler) + if self.proxies: + _handlers.append(urllib2.ProxyHandler(self.proxies)) _handlers.append(HTTPRedirectHandler) @@ -135,6 +140,7 @@ class Request(object): return opener.open + def _build_response(self, resp): """Build internal :class:`Response <models.Response>` object from given response.""" @@ -155,6 +161,8 @@ class Request(object): except zlib.error: pass + # TODO: Support deflate + response.url = getattr(resp, 'url', None) return response @@ -164,6 +172,9 @@ class Request(object): r = build(resp) + if r.status_code in REDIRECT_STATI: + self.redirect = True + if self.redirect: while ( @@ -177,7 +188,7 @@ class Request(object): url = r.headers['location'] - # Facilitate for non-RFC2616-compliant 'location' headers + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: parent_url_components = urlparse(self.url) @@ -191,7 +202,8 @@ class Request(object): request = Request( url, self.headers, self.files, method, - self.data, self.auth, self.cookiejar, redirect=False + self.data, self.params, self.auth, self.cookiejar, + redirect=True ) request.send() r = request.response @@ -202,16 +214,37 @@ class Request(object): @staticmethod - def _build_url(url, data=None): - """Build URLs.""" + def _encode_params(data): + """Encode parameters in a piece of data. + + If the data supplied is a dictionary, encodes each parameter in it, and + returns the dictionary of encoded parameters, and a urlencoded version + of that. + + Otherwise, assumes the data is already encoded appropriately, and + returns it twice. - if urlparse(url).query: - return '%s&%s' % (url, data) + """ + if hasattr(data, 'items'): + result = {} + for (k, v) in data.items(): + result[k.encode('utf-8') if isinstance(k, unicode) else k] \ + = v.encode('utf-8') if isinstance(v, unicode) else v + return result, urllib.urlencode(result) else: - if data: - return '%s?%s' % (url, data) + return data, data + + + def _build_url(self): + """Build the actual URL to use""" + + if self._enc_params: + if urlparse(self.url).query: + return '%s&%s' % (self.url, self._enc_params) else: - return url + return '%s?%s' % (self.url, self._enc_params) + else: + return self.url def send(self, anyway=False): @@ -227,8 +260,16 @@ class Request(object): self._checks() success = False + # Logging + if settings.verbose: + settings.verbose.write('%s %s %s\n' % ( + datetime.now().isoformat(), self.method, self.url + )) + + + url = self._build_url() if self.method in ('GET', 'HEAD', 'DELETE'): - req = _Request(self._build_url(self.url, self._enc_data), method=self.method) + req = _Request(url, method=self.method) else: if self.files: @@ -238,10 +279,10 @@ class Request(object): self.files.update(self.data) datagen, headers = multipart_encode(self.files) - req = _Request(self.url, data=datagen, headers=headers, method=self.method) + req = _Request(url, data=datagen, headers=headers, method=self.method) else: - req = _Request(self.url, data=self._enc_data, method=self.method) + req = _Request(url, data=self._enc_data, method=self.method) if self.headers: req.headers.update(self.headers) @@ -255,12 +296,15 @@ class Request(object): if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) - except urllib2.HTTPError, why: + except (urllib2.HTTPError, urllib2.URLError), why: + if hasattr(why, 'reason'): + if isinstance(why.reason, socket.timeout): + why = Timeout(why) + self._build_response(why) if not self.redirect: self.response.error = why - except urllib2.URLError, error: - raise Timeout if isinstance(error.reason, socket.timeout) else error + else: self._build_response(resp) self.response.ok = True @@ -271,6 +315,7 @@ class Request(object): self.sent = self.response.ok + return self.sent diff --git a/requests/monkeys.py b/requests/monkeys.py index b8fe504..41cd370 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -26,7 +26,6 @@ class Request(urllib2.Request): return urllib2.Request.get_method(self) - class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): def http_error_301(self, req, fp, code, msg, headers): diff --git a/requests/structures.py b/requests/structures.py index 0c82c7b..bfee7b1 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,30 +8,14 @@ Datastructures that power Requests. """ -from UserDict import DictMixin - - -class CaseInsensitiveDict(DictMixin): +class CaseInsensitiveDict(dict): """Case-insensitive Dictionary for :class:`Response <models.Response>` Headers. For example, ``headers['content-encoding']`` will return the value of a ``'Content-Encoding'`` response header.""" - def __init__(self, *args, **kwargs): - # super(CaseInsensitiveDict, self).__init__() - self.data = dict(*args, **kwargs) - - def __repr__(self): - return self.data.__repr__() - - def __getstate__(self): - return self.data.copy() - - def __setstate__(self, d): - self.data = d - def _lower_keys(self): - return map(str.lower, self.data.keys()) + return map(str.lower, self.keys()) def __contains__(self, key): @@ -39,26 +23,6 @@ class CaseInsensitiveDict(DictMixin): def __getitem__(self, key): - - if key.lower() in self: + # We allow fall-through here, so values default to None + if key in self: return self.items()[self._lower_keys().index(key.lower())][1] - - - def __setitem__(self, key, value): - return self.data.__setitem__(key, value) - - - def __delitem__(self, key): - return self.data.__delitem__(key) - - - def __keys__(self): - return self.data.__keys__() - - - def __iter__(self): - return self.data.__iter__() - - - def iteritems(self): - return self.data.iteritems() |