# -*- coding: utf-8 -*- """ requests.adapters ~~~~~~~~~~~~~~~~~ This module contains the transport adapters that Requests uses to define and maintain connections. """ import socket from .models import Response from .packages.urllib3.poolmanager import PoolManager, ProxyManager from .packages.urllib3.response import HTTPResponse from .compat import urlparse, basestring, urldefrag, unquote from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import TimeoutError from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .cookies import extract_cookies_to_jar from .exceptions import ConnectionError, Timeout, SSLError from .auth import _basic_auth_str DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 class BaseAdapter(object): """The Base Transport Adapter""" def __init__(self): super(BaseAdapter, self).__init__() def send(self): raise NotImplementedError def close(self): raise NotImplementedError class HTTPAdapter(BaseAdapter): """Built-In HTTP Adapter for Urllib3.""" __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize'] def __init__(self, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE): self.max_retries = DEFAULT_RETRIES self.config = {} super(HTTPAdapter, self).__init__() self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize self.init_poolmanager(pool_connections, pool_maxsize) def __getstate__(self): return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) def __setstate__(self, state): for attr, value in state.items(): setattr(self, attr, value) self.init_poolmanager(self._pool_connections, self._pool_maxsize) def init_poolmanager(self, connections, maxsize): # save these values for pickling self._pool_connections = connections self._pool_maxsize = maxsize self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize) def cert_verify(self, conn, url, verify, cert): if url.startswith('https') and verify: cert_loc = None # Allow self-specified cert location. if verify is not True: cert_loc = verify if not cert_loc: cert_loc = DEFAULT_CA_BUNDLE_PATH if not cert_loc: raise Exception("Could not find a suitable SSL CA certificate bundle.") conn.cert_reqs = 'CERT_REQUIRED' conn.ca_certs = cert_loc else: conn.cert_reqs = 'CERT_NONE' conn.ca_certs = None if cert: if not isinstance(cert, basestring): conn.cert_file = cert[0] conn.key_file = cert[1] else: conn.cert_file = cert def build_response(self, req, resp): response = Response() # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, 'status', None) # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = resp response.reason = response.raw.reason if isinstance(req.url, bytes): response.url = req.url.decode('utf-8') else: response.url = req.url # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, resp) # Give the Response some context. response.request = req response.connection = self return response def get_connection(self, url, proxies=None): """Returns a connection for the given URL.""" proxies = proxies or {} proxy = proxies.get(urlparse(url).scheme) if proxy: proxy = prepend_scheme_if_needed(proxy, urlparse(url).scheme) conn = ProxyManager(self.poolmanager.connection_from_url(proxy)) else: conn = self.poolmanager.connection_from_url(url) return conn def close(self): """Dispose of any internal state. Currently, this just closes the PoolManager, which closes pooled connections. """ self.poolmanager.clear() def request_url(self, request, proxies): """Obtain the url to use when making the final request. If the message is being sent through a proxy, the full URL has to be used. Otherwise, we should only use the path portion of the URL.""" proxies = proxies or {} proxy = proxies.get(urlparse(request.url).scheme) if proxy: url, _ = urldefrag(request.url) else: url = request.path_url return url def add_headers(self, request, **kwargs): """Add any headers needed by the connection. Currently this adds a Proxy-Authorization header.""" proxies = kwargs.get('proxies', {}) if proxies is None: proxies = {} proxy = proxies.get(urlparse(request.url).scheme) username, password = get_auth_from_url(proxy) if username and password: # Proxy auth usernames and passwords will be urlencoded, we need # to decode them. username = unquote(username) password = unquote(password) request.headers['Proxy-Authorization'] = _basic_auth_str(username, password) def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Sends PreparedRequest object. Returns Response object.""" conn = self.get_connection(request.url, proxies) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers(request, proxies=proxies) chunked = not (request.body is None or 'Content-Length' in request.headers) try: if not chunked: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout ) # Send the request. else: if hasattr(conn, 'proxy_pool'): conn = conn.proxy_pool low_conn = conn._get_conn(timeout=timeout) low_conn.putrequest(request.method, url, skip_accept_encoding=True) for header, value in request.headers.items(): low_conn.putheader(header, value) low_conn.endheaders() for i in request.body: low_conn.send(hex(len(i))[2:].encode('utf-8')) low_conn.send(b'\r\n') low_conn.send(i) low_conn.send(b'\r\n') low_conn.send(b'0\r\n\r\n') r = low_conn.getresponse() resp = HTTPResponse.from_httplib(r, pool=conn, connection=low_conn, preload_content=False, decode_content=False ) except socket.error as sockerr: raise ConnectionError(sockerr) except MaxRetryError as e: raise ConnectionError(e) except (_SSLError, _HTTPError) as e: if isinstance(e, _SSLError): raise SSLError(e) elif isinstance(e, TimeoutError): raise Timeout(e) else: raise r = self.build_response(request, resp) if not stream: r.content return r