# -*- coding: utf-8 -*- """ requests.models ~~~~~~~~~~~~~~~ """ import requests import urllib import urllib2 import socket import zlib from urllib2 import HTTPError from urlparse import urlparse from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod class Request(object): """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. """ _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE') def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), auth=None, cookiejar=None, timeout=None, redirect=True, allow_redirects=False): socket.setdefaulttimeout(timeout) #: Request URL. self.url = url #: Dictonary of HTTP Headers to attach to the :class:`Request `. self.headers = headers #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method #: Form or Byte data to attach to the :class:`Request `. self.data = dict() #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects if hasattr(data, 'items'): for (k, v) in data.items(): self.data.update({ k.encode('utf-8') if isinstance(k, unicode) else k: v.encode('utf-8') if isinstance(v, unicode) else v }) self._enc_data = urllib.urlencode(self.data) else: self._enc_data = self.data = data #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() if isinstance(auth, (list, tuple)): auth = AuthObject(*auth) if not auth: auth = auth_manager.get_auth(self.url) #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth #: CookieJar to attach to :class:`Request `. self.cookiejar = cookiejar #: True if Request has been sent. self.sent = False def __repr__(self): return '' % (self.method) def __setattr__(self, name, value): if (name == 'method') and (value): if not value in self._METHODS: raise InvalidMethod() object.__setattr__(self, name, value) def _checks(self): """Deterministic checks for consistency.""" if not self.url: raise URLRequired def _get_opener(self): """Creates appropriate opener object for urllib2.""" _handlers = [] if self.cookiejar is not None: _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) if self.auth: if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) _handlers.append(self.auth.handler) _handlers.append(HTTPRedirectHandler) if not _handlers: return urllib2.urlopen if self.data or self.files: _handlers.extend(get_handlers()) opener = urllib2.build_opener(*_handlers) if self.headers: # Allow default headers in the opener to be overloaded normal_keys = [k.capitalize() for k in self.headers] for key, val in opener.addheaders[:]: if key not in normal_keys: continue # Remove it, we have a value to take its place opener.addheaders.remove((key, val)) return opener.open def _build_response(self, resp): """Build internal :class:`Response ` object from given response.""" def build(resp): response = Response() response.status_code = getattr(resp, 'code', None) try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.content = resp.read() except AttributeError: pass if response.headers['content-encoding'] == 'gzip': try: response.content = zlib.decompress(response.content, 16+zlib.MAX_WBITS) except zlib.error: pass response.url = getattr(resp, 'url', None) return response history = [] r = build(resp) if self.redirect: while ( ('location' in r.headers) and ((self.method in ('GET', 'HEAD')) or (r.status_code is 303) or (self.allow_redirects)) ): history.append(r) url = r.headers['location'] # Facilitate for non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: parent_url_components = urlparse(self.url) url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: method = 'GET' else: method = self.method request = Request( url, self.headers, self.files, method, self.data, self.auth, self.cookiejar, redirect=False ) request.send() r = request.response r.history = history self.response = r @staticmethod def _build_url(url, data=None): """Build URLs.""" if urlparse(url).query: return '%s&%s' % (url, data) else: if data: return '%s?%s' % (url, data) else: return url def send(self, anyway=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, self.response.status_code will contain the HTTPError code. Once a request is successfully sent, `sent` will equal True. :param anyway: If True, request will be sent, even if it has already been sent. """ self._checks() success = False if self.method in ('GET', 'HEAD', 'DELETE'): req = _Request(self._build_url(self.url, self._enc_data), method=self.method) else: if self.files: register_openers() if self.data: self.files.update(self.data) datagen, headers = multipart_encode(self.files) req = _Request(self.url, data=datagen, headers=headers, method=self.method) else: req = _Request(self.url, data=self._enc_data, method=self.method) if self.headers: req.headers.update(self.headers) if not self.sent or anyway: try: opener = self._get_opener() resp = opener(req) if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) except urllib2.HTTPError, why: self._build_response(why) if not self.redirect: self.response.error = why except urllib2.URLError, error: raise Timeout if isinstance(error.reason, socket.timeout) else error else: self._build_response(resp) self.response.ok = True self.response.cached = False else: self.response.cached = True self.sent = self.response.ok return self.sent def read(self, *args): return self.response.read() class Response(object): """The core :class:`Response ` object. All :class:`Request ` objects contain a :class:`response ` attribute, which is an instance of this class. """ def __init__(self): #: Raw content of the response, in bytes. #: If ``content-encoding`` of response was set to ``gzip``, the #: response data will be automatically deflated. self.content = None #: Integer Code of responded HTTP Status. self.status_code = None #: Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. self.headers = CaseInsensitiveDict() #: Final URL location of Response. self.url = None #: True if no :attr:`error` occured. self.ok = False #: Resulting :class:`HTTPError` of request, if one occured. self.error = None #: True, if the response :attr:`content` is cached locally. self.cached = False #: A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] def __repr__(self): return '' % (self.status_code) def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" return not self.error def raise_for_status(self): """Raises stored :class:`HTTPError`, if one occured.""" if self.error: raise self.error def read(self, *args): """Returns :attr:`content`. Used for file-like object compatiblity.""" return self.content class AuthManager(object): """Requests Authentication Manager.""" def __new__(cls): singleton = cls.__dict__.get('__singleton__') if singleton is not None: return singleton cls.__singleton__ = singleton = object.__new__(cls) return singleton def __init__(self): self.passwd = {} self._auth = {} def __repr__(self): return '' % (self.method) def add_auth(self, uri, auth): """Registers AuthObject to AuthManager.""" uri = self.reduce_uri(uri, False) # try to make it an AuthObject if not isinstance(auth, AuthObject): try: auth = AuthObject(*auth) except TypeError: pass self._auth[uri] = auth def add_password(self, realm, uri, user, passwd): """Adds password to AuthManager.""" # uri could be a single URI or a sequence if isinstance(uri, basestring): uri = [uri] reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) if reduced_uri not in self.passwd: self.passwd[reduced_uri] = {} self.passwd[reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): for uris, authinfo in self.passwd.iteritems(): reduced_authuri = self.reduce_uri(authuri, False) for uri in uris: if self.is_suburi(uri, reduced_authuri): return authinfo return (None, None) def get_auth(self, uri): (in_domain, in_path) = self.reduce_uri(uri, False) for domain, path, authority in ( (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() ): if in_domain == domain: if path in in_path: return authority def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" # note HTTP URLs do not have a userinfo component parts = urllib2.urlparse.urlsplit(uri) if parts[1]: # URI scheme = parts[0] authority = parts[1] path = parts[2] or '/' else: # host or host:port scheme = None authority = uri path = '/' host, port = urllib2.splitport(authority) if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, }.get(scheme) if dport is not None: authority = "%s:%d" % (host, dport) return authority, path def is_suburi(self, base, test): """Check if test is below base in a URI tree Both args must be URIs in reduced form. """ if base == test: return True if base[0] != test[0]: return False common = urllib2.posixpath.commonprefix((base[1], test[1])) if len(common) == len(base[1]): return True return False def empty(self): self.passwd = {} def remove(self, uri, realm=None): # uri could be a single URI or a sequence if isinstance(uri, basestring): uri = [uri] for default_port in True, False: reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) del self.passwd[reduced_uri][realm] def __contains__(self, uri): # uri could be a single URI or a sequence if isinstance(uri, basestring): uri = [uri] uri = tuple([self.reduce_uri(u, False) for u in uri]) if uri in self.passwd: return True return False auth_manager = AuthManager() class AuthObject(object): """The :class:`AuthObject` is a simple HTTP Authentication token. When given to a Requests function, it enables Basic HTTP Authentication for that Request. You can also enable Authorization for domain realms with AutoAuth. See AutoAuth for more details. :param username: Username to authenticate with. :param password: Password for given username. :param realm: (optional) the realm this auth applies to :param handler: (optional) basic || digest || proxy_basic || proxy_digest """ _handlers = { 'basic': HTTPBasicAuthHandler, 'digest': HTTPDigestAuthHandler, 'proxy_basic': urllib2.ProxyBasicAuthHandler, 'proxy_digest': urllib2.ProxyDigestAuthHandler } def __init__(self, username, password, handler='basic', realm=None): self.username = username self.password = password self.realm = realm if isinstance(handler, basestring): self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler) else: self.handler = handler