aboutsummaryrefslogtreecommitdiff
path: root/requests
diff options
context:
space:
mode:
Diffstat (limited to 'requests')
-rw-r--r--requests/api.py100
-rw-r--r--requests/config.py15
-rw-r--r--requests/core.py8
-rw-r--r--requests/exceptions.py3
-rw-r--r--requests/hooks.py40
-rw-r--r--requests/models.py201
-rw-r--r--requests/monkeys.py62
-rw-r--r--requests/sessions.py84
-rw-r--r--requests/status_codes.py83
-rw-r--r--requests/structures.py49
-rw-r--r--requests/utils.py72
11 files changed, 591 insertions, 126 deletions
diff --git a/requests/api.py b/requests/api.py
index 8e328d2..0cea63d 100644
--- a/requests/api.py
+++ b/requests/api.py
@@ -12,23 +12,28 @@ This module impliments the Requests API.
"""
import config
-from .models import Request, Response, AuthManager, AuthObject, auth_manager
+from .models import Request, Response, AuthObject
+from .status_codes import codes
+from .hooks import dispatch_hook
+from .utils import cookiejar_from_dict
+from urlparse import urlparse
__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete')
def request(method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
- timeout=None, allow_redirects=False, proxies=None):
+ timeout=None, allow_redirects=False, proxies=None, hooks=None):
- """Constructs and sends a :class:`Request <models.Request>`. Returns :class:`Response <models.Response>` object.
+ """Constructs and sends a :class:`Request <models.Request>`.
+ Returns :class:`Response <models.Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
@@ -36,7 +41,12 @@ def request(method, url,
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- r = Request(
+ if cookies is None:
+ cookies = {}
+
+ cookies = cookiejar_from_dict(cookies)
+
+ args = dict(
method = method,
url = url,
data = data,
@@ -44,60 +54,65 @@ def request(method, url,
headers = headers,
cookiejar = cookies,
files = files,
- auth = auth or auth_manager.get_auth(url),
+ auth = auth,
timeout = timeout or config.settings.timeout,
allow_redirects = allow_redirects,
- proxies = proxies
+ proxies = proxies or config.settings.proxies,
)
+ # Arguments manipulation hook.
+ args = dispatch_hook('args', hooks, args)
+
+ r = Request(**args)
+
+ # Pre-request hook.
+ r = dispatch_hook('pre_request', hooks, r)
+
+ # Send the HTTP Request.
r.send()
+ # Post-request hook.
+ r = dispatch_hook('post_request', hooks, r)
+
+ # Response manipulation hook.
+ r.response = dispatch_hook('response', hooks, r.response)
+
return r.response
-def get(url,
- params=None, headers=None, cookies=None, auth=None, timeout=None,
- proxies=None):
+def get(url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('GET', url,
- params=params, headers=headers, cookies=cookies, auth=auth,
- timeout=timeout, proxies=proxies)
+ return request('GET', url, **kwargs)
-def head(url,
- params=None, headers=None, cookies=None, auth=None, timeout=None,
- proxies=None):
+def head(url, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('HEAD', url,
- params=params, headers=headers, cookies=cookies, auth=auth,
- timeout=timeout, proxies=proxies)
+ return request('HEAD', url, **kwargs)
-def post(url,
- data='', headers=None, files=None, cookies=None, auth=None, timeout=None,
- allow_redirects=False, params=None, proxies=None):
+def post(url, data='', **kwargs):
"""Sends a POST request. Returns :class:`Response` object.
@@ -105,7 +120,7 @@ def post(url,
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
@@ -113,21 +128,17 @@ def post(url,
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('POST', url,
- params=params, data=data, headers=headers, files=files,
- cookies=cookies, auth=auth, timeout=timeout,
- allow_redirects=allow_redirects, proxies=proxies)
+ return request('POST', url, data=data, **kwargs)
-def put(url, data='', headers=None, files=None, cookies=None, auth=None,
- timeout=None, allow_redirects=False, params=None, proxies=None):
+def put(url, data='', **kwargs):
"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
@@ -135,21 +146,17 @@ def put(url, data='', headers=None, files=None, cookies=None, auth=None,
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('PUT', url,
- params=params, data=data, headers=headers, files=files,
- cookies=cookies, auth=auth, timeout=timeout,
- allow_redirects=allow_redirects, proxies=proxies)
+ return request('PUT', url, data=data, **kwargs)
-def patch(url, data='', headers=None, files=None, cookies=None, auth=None,
- timeout=None, allow_redirects=False, params=None, proxies=None):
+def patch(url, data='', **kwargs):
"""Sends a PATCH request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
@@ -157,28 +164,21 @@ def patch(url, data='', headers=None, files=None, cookies=None, auth=None,
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('PATCH', url,
- params=params, data=data, headers=headers, files=files,
- cookies=cookies, auth=auth, timeout=timeout,
- allow_redirects=allow_redirects, proxies=proxies)
+ return request('PATCH', url, **kwargs)
-def delete(url,
- params=None, headers=None, cookies=None, auth=None, timeout=None,
- allow_redirects=False, proxies=None):
+def delete(url, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
- :param cookies: (optional) CookieJar object to send with the :class:`Request`.
+ :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
"""
- return request('DELETE', url,
- params=params, headers=headers, cookies=cookies, auth=auth,
- timeout=timeout, allow_redirects=allow_redirects, proxies=proxies)
+ return request('DELETE', url, **kwargs)
diff --git a/requests/config.py b/requests/config.py
index 0878da9..39be2ed 100644
--- a/requests/config.py
+++ b/requests/config.py
@@ -12,7 +12,7 @@ class Settings(object):
_singleton = {}
# attributes with defaults
- __attrs__ = ('timeout', 'verbose')
+ __attrs__ = []
def __init__(self, **kwargs):
super(Settings, self).__init__()
@@ -53,4 +53,15 @@ class Settings(object):
return None
return object.__getattribute__(self, key)
-settings = Settings() \ No newline at end of file
+
+settings = Settings()
+
+settings.base_headers = {'User-Agent': 'python-requests.org'}
+settings.accept_gzip = True
+settings.proxies = None
+settings.verbose = None
+settings.timeout = None
+settings.max_redirects = 30
+
+#: Use socket.setdefaulttimeout() as fallback?
+settings.timeout_fallback = True
diff --git a/requests/core.py b/requests/core.py
index 87f55e4..8ba34a2 100644
--- a/requests/core.py
+++ b/requests/core.py
@@ -12,14 +12,16 @@ This module implements the main Requests system.
"""
__title__ = 'requests'
-__version__ = '0.5.0'
-__build__ = 0x000500
+__version__ = '0.6.1'
+__build__ = 0x000601
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2011 Kenneth Reitz'
-from models import HTTPError, auth_manager
+from models import HTTPError
from api import *
from exceptions import *
+from sessions import session
+from status_codes import codes
from config import settings \ No newline at end of file
diff --git a/requests/exceptions.py b/requests/exceptions.py
index eff7512..c08c614 100644
--- a/requests/exceptions.py
+++ b/requests/exceptions.py
@@ -21,3 +21,6 @@ class URLRequired(RequestException):
class InvalidMethod(RequestException):
"""An inappropriate method was attempted."""
+
+class TooManyRedirects(RequestException):
+ """Too many redirects."""
diff --git a/requests/hooks.py b/requests/hooks.py
new file mode 100644
index 0000000..2938029
--- /dev/null
+++ b/requests/hooks.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests.hooks
+~~~~~~~~~~~~~~
+
+This module provides the capabilities for the Requests hooks system.
+
+Available hooks:
+
+``args``:
+ A dictionary of the arguments being sent to Request().
+
+``pre_request``:
+ The Request object, directly before being sent.
+
+``post_request``:
+ The Request object, directly after being sent.
+
+``response``:
+ The response generated from a Request.
+
+"""
+
+import warnings
+
+
+def dispatch_hook(key, hooks, hook_data):
+ """Dipatches a hook dictionary on a given peice of data."""
+
+ hooks = hooks or dict()
+
+ if key in hooks:
+ try:
+ return hooks.get(key).__call__(hook_data) or hook_data
+
+ except Exception, why:
+ warnings.warn(str(why))
+
+ return hook_data
diff --git a/requests/models.py b/requests/models.py
index 099f1c6..2d7fc8f 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -12,18 +12,21 @@ import socket
import zlib
from urllib2 import HTTPError
-from urlparse import urlparse
+from urlparse import urlparse, urlunparse, urljoin
from datetime import datetime
from .config import settings
-from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler
+from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler
from .structures import CaseInsensitiveDict
from .packages.poster.encode import multipart_encode
from .packages.poster.streaminghttp import register_openers, get_handlers
-from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod
+from .utils import dict_from_cookiejar
+from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
+from .status_codes import codes
-REDIRECT_STATI = (301, 302, 303, 307)
+REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
+
class Request(object):
@@ -31,34 +34,42 @@ class Request(object):
Requests. Recommended interface is with the Requests functions.
"""
- _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH')
-
def __init__(self,
url=None, headers=dict(), files=None, method=None, data=dict(),
params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False,
allow_redirects=False, proxies=None):
- socket.setdefaulttimeout(timeout)
+ #: Float describ the timeout of the request.
+ # (Use socket.setdefaulttimeout() as fallback)
+ self.timeout = timeout
#: Request URL.
self.url = url
+
#: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`.
self.headers = headers
+
#: Dictionary of files to multipart upload (``{filename: content}``).
self.files = files
+
#: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE.
self.method = method
+
#: Dictionary or byte of request body data to attach to the
#: :class:`Request <models.Request>`.
self.data = None
+
#: Dictionary or byte of querystring data to attach to the
#: :class:`Request <models.Request>`.
self.params = None
+
#: True if :class:`Request <models.Request>` is part of a redirect chain (disables history
#: and HTTPError storage).
self.redirect = redirect
+
#: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``)
self.allow_redirects = allow_redirects
+
# Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
self.proxies = proxies
@@ -73,24 +84,36 @@ class Request(object):
auth = AuthObject(*auth)
if not auth:
auth = auth_manager.get_auth(self.url)
+
#: :class:`AuthObject` to attach to :class:`Request <models.Request>`.
self.auth = auth
+
#: CookieJar to attach to :class:`Request <models.Request>`.
self.cookiejar = cookiejar
+
#: True if Request has been sent.
self.sent = False
- def __repr__(self):
- return '<Request [%s]>' % (self.method)
+ # Header manipulation and defaults.
+ if settings.accept_gzip:
+ settings.base_headers.update({'Accept-Encoding': 'gzip'})
- def __setattr__(self, name, value):
- if (name == 'method') and (value):
- if not value in self._METHODS:
- raise InvalidMethod()
+ if headers:
+ headers = CaseInsensitiveDict(self.headers)
+ else:
+ headers = CaseInsensitiveDict()
+
+ for (k, v) in settings.base_headers.items():
+ if k not in headers:
+ headers[k] = v
+
+ self.headers = headers
- object.__setattr__(self, name, value)
+
+ def __repr__(self):
+ return '<Request [%s]>' % (self.method)
def _checks(self):
@@ -110,6 +133,7 @@ class Request(object):
if self.auth:
if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
+ # TODO: REMOVE THIS COMPLETELY
auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
self.auth.handler = self.auth.handler(auth_manager)
auth_manager.add_auth(self.url, self.auth)
@@ -141,7 +165,7 @@ class Request(object):
return opener.open
- def _build_response(self, resp):
+ def _build_response(self, resp, is_error=False):
"""Build internal :class:`Response <models.Response>` object from given response."""
def build(resp):
@@ -151,17 +175,20 @@ class Request(object):
try:
response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None))
- response.content = resp.read()
+ response.read = resp.read
+ response._resp = resp
+ response._close = resp.close
+
+ if self.cookiejar:
+
+ response.cookies = dict_from_cookiejar(self.cookiejar)
+
+
except AttributeError:
pass
- if response.headers['content-encoding'] == 'gzip':
- try:
- response.content = zlib.decompress(response.content, 16+zlib.MAX_WBITS)
- except zlib.error:
- pass
-
- # TODO: Support deflate
+ if is_error:
+ response.error = resp
response.url = getattr(resp, 'url', None)
@@ -172,30 +199,36 @@ class Request(object):
r = build(resp)
- if r.status_code in REDIRECT_STATI:
- self.redirect = True
-
- if self.redirect:
+ if r.status_code in REDIRECT_STATI and not self.redirect:
while (
('location' in r.headers) and
((self.method in ('GET', 'HEAD')) or
- (r.status_code is 303) or
+ (r.status_code is codes.see_other) or
(self.allow_redirects))
):
+ r.close()
+
+ if not len(history) < settings.max_redirects:
+ raise TooManyRedirects()
+
history.append(r)
url = r.headers['location']
+ # Handle redirection without scheme (see: RFC 1808 Section 4)
+ if url.startswith('//'):
+ parsed_rurl = urlparse(r.url)
+ url = '%s:%s' % (parsed_rurl.scheme, url)
+
# Facilitate non-RFC2616-compliant 'location' headers
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
if not urlparse(url).netloc:
- parent_url_components = urlparse(self.url)
- url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url)
+ url = urljoin(r.url, urllib.quote(urllib.unquote(url)))
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
- if r.status_code is 303:
+ if r.status_code is codes.see_other:
method = 'GET'
else:
method = self.method
@@ -211,6 +244,7 @@ class Request(object):
r.history = history
self.response = r
+ self.response.request = self
@staticmethod
@@ -218,25 +252,34 @@ class Request(object):
"""Encode parameters in a piece of data.
If the data supplied is a dictionary, encodes each parameter in it, and
- returns the dictionary of encoded parameters, and a urlencoded version
- of that.
+ returns a list of tuples containing the encoded parameters, and a urlencoded
+ version of that.
Otherwise, assumes the data is already encoded appropriately, and
returns it twice.
"""
if hasattr(data, 'items'):
- result = {}
- for (k, v) in data.items():
- result[k.encode('utf-8') if isinstance(k, unicode) else k] \
- = v.encode('utf-8') if isinstance(v, unicode) else v
- return result, urllib.urlencode(result)
+ result = []
+ for k, vs in data.items():
+ for v in isinstance(vs, list) and vs or [vs]:
+ result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
+ v.encode('utf-8') if isinstance(v, unicode) else v))
+ return result, urllib.urlencode(result, doseq=True)
else:
return data, data
def _build_url(self):
- """Build the actual URL to use"""
+ """Build the actual URL to use."""
+
+ # Support for unicode domain names and paths.
+ scheme, netloc, path, params, query, fragment = urlparse(self.url)
+ netloc = netloc.encode('idna')
+ if isinstance(path, unicode):
+ path = path.encode('utf-8')
+ path = urllib.quote(urllib.unquote(path))
+ self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ]))
if self._enc_params:
if urlparse(self.url).query:
@@ -257,6 +300,7 @@ class Request(object):
:param anyway: If True, request will be sent, even if it has
already been sent.
"""
+
self._checks()
success = False
@@ -285,13 +329,32 @@ class Request(object):
req = _Request(url, data=self._enc_data, method=self.method)
if self.headers:
- req.headers.update(self.headers)
+ for k,v in self.headers.iteritems():
+ req.add_header(k, v)
if not self.sent or anyway:
try:
opener = self._get_opener()
- resp = opener(req)
+ try:
+
+ resp = opener(req, timeout=self.timeout)
+
+ except TypeError, err:
+ # timeout argument is new since Python v2.6
+ if not 'timeout' in str(err):
+ raise
+
+ if settings.timeout_fallback:
+ # fall-back and use global socket timeout (This is not thread-safe!)
+ old_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(self.timeout)
+
+ resp = opener(req)
+
+ if settings.timeout_fallback:
+ # restore gobal timeout
+ socket.setdefaulttimeout(old_timeout)
if self.cookiejar is not None:
self.cookiejar.extract_cookies(resp, req)
@@ -301,28 +364,19 @@ class Request(object):
if isinstance(why.reason, socket.timeout):
why = Timeout(why)
- self._build_response(why)
- if not self.redirect:
- self.response.error = why
+ self._build_response(why, is_error=True)
+
else:
self._build_response(resp)
self.response.ok = True
- self.response.cached = False
- else:
- self.response.cached = True
self.sent = self.response.ok
-
return self.sent
- def read(self, *args):
- return self.response.read()
-
-
class Response(object):
"""The core :class:`Response <models.Response>` object. All
@@ -335,7 +389,7 @@ class Response(object):
#: Raw content of the response, in bytes.
#: If ``content-encoding`` of response was set to ``gzip``, the
#: response data will be automatically deflated.
- self.content = None
+ self._content = None
#: Integer Code of responded HTTP Status.
self.status_code = None
#: Case-insensitive Dictionary of Response Headers.
@@ -348,12 +402,14 @@ class Response(object):
self.ok = False
#: Resulting :class:`HTTPError` of request, if one occured.
self.error = None
- #: True, if the response :attr:`content` is cached locally.
- self.cached = False
#: A list of :class:`Response <models.Response>` objects from
#: the history of the Request. Any redirect responses will end
#: up here.
self.history = []
+ #: The Request that created the Response.
+ self.request = None
+ #: A dictionary of Cookies the server sent back.
+ self.cookies = None
def __repr__(self):
@@ -365,17 +421,31 @@ class Response(object):
return not self.error
+ def __getattr__(self, name):
+ """Read and returns the full stream when accessing to :attr: `content`"""
+ if name == 'content':
+ if self._content is not None:
+ return self._content
+ self._content = self.read()
+ if self.headers.get('content-encoding', '') == 'gzip':
+ try:
+ self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS)
+ except zlib.error:
+ pass
+ return self._content
+ else:
+ raise AttributeError
+
def raise_for_status(self):
- """Raises stored :class:`HTTPError`, if one occured."""
+ """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured."""
if self.error:
raise self.error
- def read(self, *args):
- """Returns :attr:`content`. Used for file-like object compatiblity."""
-
- return self.content
-
+ def close(self):
+ if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'):
+ self._resp.fp._sock.recv = None
+ self._close()
class AuthManager(object):
"""Requests Authentication Manager."""
@@ -450,8 +520,10 @@ class AuthManager(object):
def reduce_uri(self, uri, default_port=True):
"""Accept authority or URI and extract only the authority and path."""
+
# note HTTP URLs do not have a userinfo component
parts = urllib2.urlparse.urlsplit(uri)
+
if parts[1]:
# URI
scheme = parts[0]
@@ -462,7 +534,9 @@ class AuthManager(object):
scheme = None
authority = uri
path = '/'
+
host, port = urllib2.splitport(authority)
+
if default_port and port is None and scheme is not None:
dport = {"http": 80,
"https": 443,
@@ -532,17 +606,18 @@ class AuthObject(object):
_handlers = {
'basic': HTTPBasicAuthHandler,
+ 'forced_basic': HTTPForcedBasicAuthHandler,
'digest': HTTPDigestAuthHandler,
'proxy_basic': urllib2.ProxyBasicAuthHandler,
'proxy_digest': urllib2.ProxyDigestAuthHandler
}
- def __init__(self, username, password, handler='basic', realm=None):
+ def __init__(self, username, password, handler='forced_basic', realm=None):
self.username = username
self.password = password
self.realm = realm
if isinstance(handler, basestring):
- self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler)
+ self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler)
else:
self.handler = handler
diff --git a/requests/monkeys.py b/requests/monkeys.py
index 41cd370..c838071 100644
--- a/requests/monkeys.py
+++ b/requests/monkeys.py
@@ -9,7 +9,7 @@ Urllib2 Monkey patches.
"""
import urllib2
-
+import re
class Request(urllib2.Request):
"""Hidden wrapper around the urllib2.Request object. Allows for manual
@@ -26,8 +26,9 @@ class Request(urllib2.Request):
return urllib2.Request.get_method(self)
-class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
+ """HTTP Redirect handler."""
def http_error_301(self, req, fp, code, msg, headers):
pass
@@ -36,10 +37,13 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+ """HTTP Basic Auth Handler with authentication loop fixes."""
def __init__(self, *args, **kwargs):
urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
self.retried_req = None
+ self.retried = 0
+
def reset_retry_count(self):
# Python 2.6.5 will call this on 401 or 407 errors and thus loop
@@ -47,6 +51,7 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
# http_error_auth_reqed instead.
pass
+
def http_error_auth_reqed(self, auth_header, host, req, headers):
# Reset the retry counter once for each request.
if req is not self.retried_req:
@@ -59,6 +64,59 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
+ """HTTP Basic Auth Handler with forced Authentication."""
+
+ auth_header = 'Authorization'
+ rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+ 'realm=(["\'])(.*?)\\2', re.I)
+
+ def __init__(self, *args, **kwargs):
+ HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
+
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ url = req.get_full_url()
+ response = self._http_error_auth_reqed('www-authenticate', url, req, headers)
+ self.reset_retry_count()
+ return response
+
+ http_error_404 = http_error_401
+
+
+ def _http_error_auth_reqed(self, authreq, host, req, headers):
+
+ authreq = headers.get(authreq, None)
+
+ if self.retried > 5:
+ # retry sending the username:password 5 times before failing.
+ raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+
+ if authreq:
+
+ mo = self.rx.search(authreq)
+
+ if mo:
+ scheme, quote, realm = mo.groups()
+
+ if scheme.lower() == 'basic':
+ response = self.retry_http_basic_auth(host, req, realm)
+
+ if response and response.code not in (401, 404):
+ self.retried = 0
+ return response
+ else:
+ response = self.retry_http_basic_auth(host, req, 'Realm')
+
+ if response and response.code not in (401, 404):
+ self.retried = 0
+ return response
+
+
+
class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
def __init__(self, *args, **kwargs):
diff --git a/requests/sessions.py b/requests/sessions.py
new file mode 100644
index 0000000..50b09f6
--- /dev/null
+++ b/requests/sessions.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests.session
+~~~~~~~~~~~~~~~
+
+This module provides a Session object to manage and persist settings across
+requests (cookies, auth, proxies).
+
+"""
+
+import cookielib
+
+from . import api
+from .utils import add_dict_to_cookiejar
+
+
+
+class Session(object):
+ """A Requests session."""
+
+ __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks']
+
+
+ def __init__(self, **kwargs):
+
+ # Set up a CookieJar to be used by default
+ self.cookies = cookielib.FileCookieJar()
+
+ # Map args from kwargs to instance-local variables
+ map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v),
+ kwargs.iterkeys(), kwargs.itervalues())
+
+ # Map and wrap requests.api methods
+ self._map_api_methods()
+
+
+ def __repr__(self):
+ return '<requests-client at 0x%x>' % (id(self))
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ # print args
+ pass
+
+
+ def _map_api_methods(self):
+ """Reads each available method from requests.api and decorates
+ them with a wrapper, which inserts any instance-local attributes
+ (from __attrs__) that have been set, combining them with **kwargs.
+ """
+
+ def pass_args(func):
+ def wrapper_func(*args, **kwargs):
+ inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems()
+ if k in self.__attrs__)
+ # Combine instance-local values with kwargs values, with
+ # priority to values in kwargs
+ kwargs = dict(inst_attrs.items() + kwargs.items())
+
+ # If a session request has a cookie_dict, inject the
+ # values into the existing CookieJar instead.
+ if isinstance(kwargs.get('cookies', None), dict):
+ kwargs['cookies'] = add_dict_to_cookiejar(
+ inst_attrs['cookies'], kwargs['cookies']
+ )
+
+ if kwargs.get('headers', None) and inst_attrs.get('headers', None):
+ kwargs['headers'].update(inst_attrs['headers'])
+
+ return func(*args, **kwargs)
+ return wrapper_func
+
+ # Map and decorate each function available in requests.api
+ map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))),
+ api.__all__)
+
+
+def session(**kwargs):
+ """Returns a :class:`Session` for context-managment."""
+
+ return Session(**kwargs) \ No newline at end of file
diff --git a/requests/status_codes.py b/requests/status_codes.py
new file mode 100644
index 0000000..a809de6
--- /dev/null
+++ b/requests/status_codes.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from .structures import LookupDict
+
+_codes = {
+
+ # Informational.
+ 100: ('continue',),
+ 101: ('switching_protocols',),
+ 102: ('processing',),
+ 103: ('checkpoint',),
+ 122: ('uri_too_long', 'request_uri_too_long'),
+ 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'),
+ 201: ('created',),
+ 202: ('accepted',),
+ 203: ('non_authoritative_info', 'non_authoritative_information'),
+ 204: ('no_content',),
+ 205: ('reset_content', 'reset'),
+ 206: ('partial_content', 'partial'),
+ 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
+ 208: ('im_used',),
+
+ # Redirection.
+ 300: ('multiple_choices',),
+ 301: ('moved_permanently', 'moved', '\\o-'),
+ 302: ('found',),
+ 303: ('see_other', 'other'),
+ 304: ('not_modified',),
+ 305: ('use_proxy',),
+ 306: ('switch_proxy',),
+ 307: ('temporary_redirect', 'temporary_moved', 'temporary'),
+ 308: ('resume_incomplete', 'resume'),
+
+ # Client Error.
+ 400: ('bad_request', 'bad'),
+ 401: ('unauthorized',),
+ 402: ('payment_required', 'payment'),
+ 403: ('forbidden',),
+ 404: ('not_found', '-o-'),
+ 405: ('method_not_allowed', 'not_allowed'),
+ 406: ('not_acceptable',),
+ 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'),
+ 408: ('request_timeout', 'timeout'),
+ 409: ('conflict',),
+ 410: ('gone',),
+ 411: ('length_required',),
+ 412: ('precondition_failed', 'precondition'),
+ 413: ('request_entity_too_large',),
+ 414: ('request_uri_too_large',),
+ 415: ('unspported_media_type', 'unspported_media', 'media_type'),
+ 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'),
+ 417: ('expectation_failed',),
+ 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'),
+ 422: ('unprocessable_entity', 'unprocessable'),
+ 423: ('locked',),
+ 424: ('failed_depdendency', 'depdendency'),
+ 425: ('unordered_collection', 'unordered'),
+ 426: ('upgrade_required', 'upgrade'),
+ 444: ('no_response', 'none'),
+ 449: ('retry_with', 'retry'),
+ 450: ('blocked_by_windows_parental_controls', 'parental_controls'),
+ 499: ('client_closed_request',),
+
+ # Server Error.
+ 500: ('internal_server_error', 'server_error', '/o\\'),
+ 501: ('not_implemented',),
+ 502: ('bad_gateway',),
+ 503: ('service_unavailable', 'unavailable'),
+ 504: ('gateway_timeout',),
+ 505: ('http_version_not_supported', 'http_version'),
+ 506: ('variant_also_negotiates',),
+ 507: ('insufficient_storage',),
+ 509: ('bandwidth_limit_exceeded', 'bandwidth'),
+ 510: ('not_extended',),
+}
+
+codes = LookupDict(name='status_codes')
+
+for (code, titles) in _codes.items():
+ for title in titles:
+ setattr(codes, title, code)
+ if not title.startswith('\\'):
+ setattr(codes, title.upper(), code) \ No newline at end of file
diff --git a/requests/structures.py b/requests/structures.py
index bfee7b1..d068bf9 100644
--- a/requests/structures.py
+++ b/requests/structures.py
@@ -9,20 +9,57 @@ Datastructures that power Requests.
"""
class CaseInsensitiveDict(dict):
- """Case-insensitive Dictionary for :class:`Response <models.Response>` Headers.
+ """Case-insensitive Dictionary
For example, ``headers['content-encoding']`` will return the
value of a ``'Content-Encoding'`` response header."""
- def _lower_keys(self):
- return map(str.lower, self.keys())
+ @property
+ def lower_keys(self):
+ if not hasattr(self, '_lower_keys') or not self._lower_keys:
+ self._lower_keys = dict((k.lower(), k) for k in self.iterkeys())
+ return self._lower_keys
+ def _clear_lower_keys(self):
+ if hasattr(self, '_lower_keys'):
+ self._lower_keys.clear()
- def __contains__(self, key):
- return key.lower() in self._lower_keys()
+ def __setitem__(self, key, value):
+ dict.__setitem__(self, key, value)
+ self._clear_lower_keys()
+
+ def __delitem__(self, key):
+ dict.__delitem__(self, key)
+ self._lower_keys.clear()
+ def __contains__(self, key):
+ return key.lower() in self.lower_keys
def __getitem__(self, key):
# We allow fall-through here, so values default to None
if key in self:
- return self.items()[self._lower_keys().index(key.lower())][1]
+ return dict.__getitem__(self, self.lower_keys[key.lower()])
+
+ def get(self, key, default=None):
+ if key in self:
+ return self[key]
+ else:
+ return default
+
+class LookupDict(dict):
+ """Dictionary lookup object."""
+
+ def __init__(self, name=None):
+ self.name = name
+ super(LookupDict, self).__init__()
+
+ def __repr__(self):
+ return '<lookup \'%s\'>' % (self.name)
+
+ def __getitem__(self, key):
+ # We allow fall-through here, so values default to None
+
+ return self.__dict__.get(key, None)
+
+ def get(self, key, default=None):
+ return self.__dict__.get(key, default) \ No newline at end of file
diff --git a/requests/utils.py b/requests/utils.py
new file mode 100644
index 0000000..8ac78b4
--- /dev/null
+++ b/requests/utils.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests.utils
+~~~~~~~~~~~~~~
+
+This module provides utlity functions that are used within Requests
+that are also useful for external consumption.
+
+"""
+
+import cookielib
+
+
+def dict_from_cookiejar(cookiejar):
+ """Returns a key/value dictionary from a CookieJar."""
+
+ cookie_dict = {}
+
+ for _, cookies in cookiejar._cookies.items():
+ for _, cookies in cookies.items():
+ for cookie in cookies.values():
+ # print cookie
+ cookie_dict[cookie.name] = cookie.value
+
+ return cookie_dict
+
+
+def cookiejar_from_dict(cookie_dict):
+ """Returns a CookieJar from a key/value dictionary."""
+
+ # return cookiejar if one was passed in
+ if isinstance(cookie_dict, cookielib.CookieJar):
+ return cookie_dict
+
+ # create cookiejar
+ cj = cookielib.CookieJar()
+
+ cj = add_dict_to_cookiejar(cj, cookie_dict)
+
+ return cj
+
+
+def add_dict_to_cookiejar(cj, cookie_dict):
+ """Returns a CookieJar from a key/value dictionary."""
+
+ for k, v in cookie_dict.items():
+
+ cookie = cookielib.Cookie(
+ version=0,
+ name=k,
+ value=v,
+ port=None,
+ port_specified=False,
+ domain='',
+ domain_specified=False,
+ domain_initial_dot=False,
+ path='/',
+ path_specified=True,
+ secure=False,
+ expires=None,
+ discard=True,
+ comment=None,
+ comment_url=None,
+ rest={'HttpOnly': None},
+ rfc2109=False
+ )
+
+ # add cookie to cookiejar
+ cj.set_cookie(cookie)
+
+ return cj