diff options
Diffstat (limited to 'requests')
| -rw-r--r-- | requests/__init__.py | 4 | ||||
| -rw-r--r-- | requests/auth.py | 20 | ||||
| -rw-r--r-- | requests/exceptions.py | 5 | ||||
| -rw-r--r-- | requests/models.py | 101 | ||||
| -rw-r--r-- | requests/packages/urllib3/__init__.py | 2 | ||||
| -rw-r--r-- | requests/packages/urllib3/connectionpool.py | 9 | ||||
| -rw-r--r-- | requests/sessions.py | 3 | ||||
| -rw-r--r-- | requests/utils.py | 73 |
8 files changed, 142 insertions, 75 deletions
diff --git a/requests/__init__.py b/requests/__init__.py index 0ace12b..96ef774 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.11.1' -__build__ = 0x001101 +__version__ = '0.11.2' +__build__ = 0x001102 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2012 Kenneth Reitz' diff --git a/requests/auth.py b/requests/auth.py index 385dd27..353180a 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -56,6 +56,8 @@ class HTTPDigestAuth(AuthBase): def handle_401(self, r): """Takes the given response and tries digest-auth, if needed.""" + r.request.deregister_hook('response', self.handle_401) + s_auth = r.headers.get('www-authenticate', '') if 'digest' in s_auth.lower(): @@ -74,21 +76,21 @@ class HTTPDigestAuth(AuthBase): algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': - def h(x): + def md5_utf8(x): if isinstance(x, str): x = x.encode('utf-8') return hashlib.md5(x).hexdigest() - H = h + hash_utf8 = md5_utf8 elif algorithm == 'SHA': - def h(x): + def sha_utf8(x): if isinstance(x, str): x = x.encode('utf-8') return hashlib.sha1(x).hexdigest() - H = h + hash_utf8 = sha_utf8 # XXX MD5-sess - KD = lambda s, d: H("%s:%s" % (s, d)) + KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) - if H is None: + if hash_utf8 is None: return None # XXX not implemented yet @@ -115,10 +117,10 @@ class HTTPDigestAuth(AuthBase): s += randombytes(8) cnonce = (hashlib.sha1(s).hexdigest()[:16]) - noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) - respdig = KD(H(A1), noncebit) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2)) + respdig = KD(hash_utf8(A1), noncebit) elif qop is None: - respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2))) else: # XXX handle auth-int. return None diff --git a/requests/exceptions.py b/requests/exceptions.py index 3c262e3..1cffa80 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -35,4 +35,7 @@ class MissingSchema(RequestException, ValueError): """The URL schema (e.g. http or https) is missing.""" class InvalidSchema(RequestException, ValueError): - """See defaults.py for valid schemas."""
\ No newline at end of file + """See defaults.py for valid schemas.""" + +class InvalidURL(RequestException, ValueError): + """ The URL provided was somehow invalid. """
\ No newline at end of file diff --git a/requests/models.py b/requests/models.py index 70e3503..60f58d2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,7 +16,7 @@ from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth from .packages.urllib3.response import HTTPResponse -from .packages.urllib3.exceptions import MaxRetryError +from .packages.urllib3.exceptions import MaxRetryError, LocationParseError from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager @@ -24,10 +24,11 @@ from .packages.urllib3.filepost import encode_multipart_formdata from .defaults import SCHEMAS from .exceptions import ( ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, - URLRequired, SSLError, MissingSchema, InvalidSchema) + URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL) from .utils import ( get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, - dict_from_string, stream_decode_response_unicode, get_netrc_auth) + dict_from_string, stream_decode_response_unicode, get_netrc_auth, + DEFAULT_CA_BUNDLE_PATH) from .compat import ( urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, SimpleCookie, is_py2) @@ -61,6 +62,7 @@ class Request(object): proxies=None, hooks=None, config=None, + prefetch=False, _poolmanager=None, verify=None, session=None, @@ -80,7 +82,7 @@ class Request(object): self.headers = dict(headers or []) #: Dictionary of files to multipart upload (``{filename: content}``). - self.files = files + self.files = None #: HTTP Method to use. self.method = method @@ -90,7 +92,8 @@ class Request(object): self.data = None #: Dictionary or byte of querystring data to attach to the - #: :class:`Request <Request>`. + #: :class:`Request <Request>`. The dictionary values can be lists for representing + #: multivalued query parameters. self.params = None #: True if :class:`Request <Request>` is part of a redirect chain (disables history @@ -113,6 +116,7 @@ class Request(object): self.data, self._enc_data = self._encode_params(data) self.params, self._enc_params = self._encode_params(params) + self.files, self._enc_files = self._encode_files(files) #: :class:`Response <Response>` instance, containing #: content and metadata of HTTP Response, once :attr:`sent <send>`. @@ -147,6 +151,9 @@ class Request(object): #: SSL Certificate self.cert = cert + #: Prefetch response content + self.prefetch = prefetch + if headers: headers = CaseInsensitiveDict(self.headers) else: @@ -328,6 +335,29 @@ class Request(object): else: return data, data + def _encode_files(self,files): + + if (not files) or isinstance(self.data, str): + return None, None + + try: + fields = self.data.copy() + except AttributeError: + fields = dict(self.data) + + for (k, v) in list(files.items()): + # support for explicit filename + if isinstance(v, (tuple, list)): + fn, fp = v + else: + fn = guess_filename(v) or k + fp = v + fields.update({k: (fn, fp.read())}) + + (body, content_type) = encode_multipart_formdata(fields) + + return files, (body, content_type) + @property def full_url(self): """Build the actual URL to use.""" @@ -407,7 +437,18 @@ class Request(object): def register_hook(self, event, hook): """Properly register a hook.""" - return self.hooks[event].append(hook) + self.hooks[event].append(hook) + + def deregister_hook(self,event,hook): + """Deregister a previously registered hook. + Returns True if the hook existed, False if not. + """ + + try: + self.hooks[event].remove(hook) + return True + except ValueError: + return False def send(self, anyway=False, prefetch=False): """Sends the request. Returns True of successful, False if not. @@ -435,26 +476,7 @@ class Request(object): # Multi-part file uploads. if self.files: - if not isinstance(self.data, str): - - try: - fields = self.data.copy() - except AttributeError: - fields = dict(self.data) - - for (k, v) in list(self.files.items()): - # support for explicit filename - if isinstance(v, (tuple, list)): - fn, fp = v - else: - fn = guess_filename(v) or k - fp = v - fields.update({k: (fn, fp.read())}) - - (body, content_type) = encode_multipart_formdata(fields) - else: - pass - # TODO: Conflict? + (body, content_type) = self._enc_files else: if self.data: @@ -496,10 +518,13 @@ class Request(object): self.__dict__.update(r.__dict__) else: # Check to see if keep_alive is allowed. - if self.config.get('keep_alive'): - conn = self._poolmanager.connection_from_url(url) - else: - conn = connectionpool.connection_from_url(url) + try: + if self.config.get('keep_alive'): + conn = self._poolmanager.connection_from_url(url) + else: + conn = connectionpool.connection_from_url(url) + except LocationParseError as e: + raise InvalidURL(e) if url.startswith('https') and self.verify: @@ -513,13 +538,15 @@ class Request(object): if not cert_loc and self.config.get('trust_env'): cert_loc = os.environ.get('REQUESTS_CA_BUNDLE') - # Curl compatiblity. + # Curl compatibility. if not cert_loc and self.config.get('trust_env'): cert_loc = os.environ.get('CURL_CA_BUNDLE') - # Use the awesome certifi list. if not cert_loc: - cert_loc = __import__('certifi').where() + cert_loc = DEFAULT_CA_BUNDLE_PATH + + if not cert_loc: + raise Exception("Could not find a suitable SSL CA certificate bundle.") conn.cert_reqs = 'CERT_REQUIRED' conn.ca_certs = cert_loc @@ -604,7 +631,7 @@ class Request(object): self.__dict__.update(r.__dict__) # If prefetch is True, mark content as consumed. - if prefetch: + if prefetch or self.prefetch: # Save the response. self.response.content @@ -623,7 +650,7 @@ class Response(object): def __init__(self): - self._content = None + self._content = False self._content_consumed = False #: Integer Code of responded HTTP Status. @@ -679,7 +706,7 @@ class Response(object): return False return True - def iter_content(self, chunk_size=10 * 1024, decode_unicode=False): + def iter_content(self, chunk_size=1, decode_unicode=False): """Iterates over the response data. This avoids reading the content at once into memory for large responses. The chunk size is the number of bytes it should read into memory. This is not necessarily the @@ -736,7 +763,7 @@ class Response(object): def content(self): """Content of the response, in bytes.""" - if self._content is None: + if self._content is False: # Read the contents. try: if self._content_consumed: diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 2d6fece..81e76f5 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.3' +__version__ = 'dev' from .connectionpool import ( diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index c3cb3b1..336aa77 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -260,10 +260,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): httplib_response = conn.getresponse() - log.debug("\"%s %s %s\" %s %s" % - (method, url, - conn._http_vsn_str, # pylint: disable-msg=W0212 - httplib_response.status, httplib_response.length)) + # AppEngine doesn't have a version attr. + http_version = getattr(conn, '_http_vsn_str', 'HTTP/?'), + log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, + httplib_response.status, + httplib_response.length)) return httplib_response diff --git a/requests/sessions.py b/requests/sessions.py index 94c94bf..0e43030 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -52,7 +52,7 @@ class Session(object): __attrs__ = [ 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', - 'params', 'config', 'verify', 'cert'] + 'params', 'config', 'verify', 'cert', 'prefetch'] def __init__(self, @@ -179,6 +179,7 @@ class Session(object): allow_redirects=allow_redirects, proxies=proxies, config=config, + prefetch=prefetch, verify=verify, cert=cert, _poolmanager=self.poolmanager diff --git a/requests/utils.py b/requests/utils.py index b722d99..925547a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -21,9 +21,36 @@ from .compat import parse_http_list as _parse_list_header from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse from .compat import basestring, bytes, str +CERTIFI_BUNDLE_PATH = None +try: + # see if requests's own CA certificate bundle is installed + import certifi + CERTIFI_BUNDLE_PATH = certifi.where() +except ImportError: + pass NETRC_FILES = ('.netrc', '_netrc') +# common paths for the OS's CA certificate bundle +POSSIBLE_CA_BUNDLE_PATHS = [ + # Red Hat, CentOS, Fedora and friends (provided by the ca-certificates package): + '/etc/pki/tls/certs/ca-bundle.crt', + # Ubuntu, Debian, and friends (provided by the ca-certificates package): + '/etc/ssl/certs/ca-certificates.crt', + # FreeBSD (provided by the ca_root_nss package): + '/usr/local/share/certs/ca-root-nss.crt', +] + +def get_os_ca_bundle_path(): + """Try to pick an available CA certificate bundle provided by the OS.""" + for path in POSSIBLE_CA_BUNDLE_PATHS: + if os.path.exists(path): + return path + return None + +# if certifi is installed, use its CA bundle; +# otherwise, try and use the OS bundle +DEFAULT_CA_BUNDLE_PATH = CERTIFI_BUNDLE_PATH or get_os_ca_bundle_path() def dict_to_sequence(d): """Returns an internal sequence dictionary update.""" @@ -37,34 +64,40 @@ def dict_to_sequence(d): def get_netrc_auth(url): """Returns the Requests tuple auth for a given url from netrc.""" - locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES) - netrc_path = None + try: + locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES) + netrc_path = None - for loc in locations: - if os.path.exists(loc) and not netrc_path: - netrc_path = loc + for loc in locations: + if os.path.exists(loc) and not netrc_path: + netrc_path = loc - # Abort early if there isn't one. - if netrc_path is None: - return netrc_path + # Abort early if there isn't one. + if netrc_path is None: + return netrc_path - ri = urlparse(url) + ri = urlparse(url) - # Strip port numbers from netloc - host = ri.netloc.split(':')[0] + # Strip port numbers from netloc + host = ri.netloc.split(':')[0] - try: - _netrc = netrc(netrc_path).authenticators(host) - if _netrc: - # Return with login / password - login_i = (0 if _netrc[0] else 1) - return (_netrc[login_i], _netrc[2]) - except (NetrcParseError, IOError, AttributeError): - # If there was a parsing error or a permissions issue reading the file, - # we'll just skip netrc auth + try: + _netrc = netrc(netrc_path).authenticators(host) + if _netrc: + # Return with login / password + login_i = (0 if _netrc[0] else 1) + return (_netrc[login_i], _netrc[2]) + except (NetrcParseError, IOError): + # If there was a parsing error or a permissions issue reading the file, + # we'll just skip netrc auth + pass + + # AppEngine hackiness. + except AttributeError: pass + def dict_from_string(s): """Returns a MultiDict with Cookies.""" |