diff options
Diffstat (limited to 'requests/utils.py')
| -rw-r--r-- | requests/utils.py | 92 |
1 files changed, 59 insertions, 33 deletions
diff --git a/requests/utils.py b/requests/utils.py index f31cad8..0e0f69e 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -4,23 +4,35 @@ requests.utils ~~~~~~~~~~~~~~ -This module provides utlity functions that are used within Requests +This module provides utility functions that are used within Requests that are also useful for external consumption. """ import cgi import codecs -import cookielib import os import random import re import zlib -import urllib -from urllib2 import parse_http_list as _parse_list_header +from .compat import parse_http_list as _parse_list_header +from .compat import quote, unquote, cookielib, SimpleCookie, is_py2 +def dict_from_string(s): + """Returns a MultiDict with Cookies.""" + + cookies = dict() + + c = SimpleCookie() + c.load(s) + + for k,v in list(c.items()): + cookies.update({k: v.value}) + + return cookies + def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) @@ -132,16 +144,16 @@ def header_expand(headers): collector = [] if isinstance(headers, dict): - headers = headers.items() + headers = list(headers.items()) - elif isinstance(headers, basestring): + elif isinstance(headers, str): return headers for i, (value, params) in enumerate(headers): _params = [] - for (p_k, p_v) in params.items(): + for (p_k, p_v) in list(params.items()): _params.append('%s=%s' % (p_k, p_v)) @@ -166,17 +178,11 @@ def header_expand(headers): def randombytes(n): """Return n random bytes.""" - # Use /dev/urandom if it is available. Fall back to random module - # if not. It might be worthwhile to extend this function to use - # other platform-specific mechanisms for getting random bytes. - if os.path.exists("/dev/urandom"): - f = open("/dev/urandom") - s = f.read(n) - f.close() - return s - else: + if is_py2: L = [chr(random.randrange(0, 256)) for i in range(n)] - return "".join(L) + else: + L = [chr(random.randrange(0, 256)).encode('utf-8') for i in range(n)] + return b"".join(L) def dict_from_cookiejar(cj): @@ -187,9 +193,9 @@ def dict_from_cookiejar(cj): cookie_dict = {} - for _, cookies in cj._cookies.items(): - for _, cookies in cookies.items(): - for cookie in cookies.values(): + for _, cookies in list(cj._cookies.items()): + for _, cookies in list(cookies.items()): + for cookie in list(cookies.values()): # print cookie cookie_dict[cookie.name] = cookie.value @@ -221,7 +227,7 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cookie_dict: Dict of key/values to insert into CookieJar. """ - for k, v in cookie_dict.items(): + for k, v in list(cookie_dict.items()): cookie = cookielib.Cookie( version=0, @@ -276,6 +282,9 @@ def get_encoding_from_headers(headers): if 'charset' in params: return params['charset'].strip("'\"") + if 'text' in content_type: + return 'ISO-8859-1' + def unicode_from_html(content): """Attempts to decode an HTML string into unicode. @@ -287,7 +296,7 @@ def unicode_from_html(content): for encoding in encodings: try: - return unicode(content, encoding) + return str(content, encoding) except (UnicodeError, TypeError): pass @@ -334,13 +343,13 @@ def get_unicode_from_response(r): if encoding: try: - return unicode(r.content, encoding) + return str(r.content, encoding) except UnicodeError: tried_encodings.append(encoding) # Fall back: try: - return unicode(r.content, encoding, errors='replace') + return str(r.content, encoding, errors='replace') except TypeError: return r.content @@ -354,28 +363,45 @@ def decode_gzip(content): return zlib.decompress(content, 16 + zlib.MAX_WBITS) -def stream_decode_gzip(iterator): - """Stream decodes a gzip-encoded iterator""" +def stream_decompress(iterator, mode='gzip'): + """ + Stream decodes an iterator over compressed data + + :param iterator: An iterator over compressed data + :param mode: 'gzip' or 'deflate' + :return: An iterator over decompressed data + """ + + if mode not in ['gzip', 'deflate']: + raise ValueError('stream_decompress mode must be gzip or deflate') + + zlib_mode = 16 + zlib.MAX_WBITS if mode == 'gzip' else -zlib.MAX_WBITS + dec = zlib.decompressobj(zlib_mode) try: - dec = zlib.decompressobj(16 + zlib.MAX_WBITS) for chunk in iterator: rv = dec.decompress(chunk) if rv: yield rv + except zlib.error: + # If there was an error decompressing, just return the raw chunk + yield chunk + # Continue to return the rest of the raw data + for chunk in iterator: + yield chunk + else: + # Make sure everything has been returned from the decompression object buf = dec.decompress('') rv = buf + dec.flush() if rv: yield rv - except zlib.error: - pass def requote_path(path): """Re-quote the given URL path component. This function passes the given path through an unquote/quote cycle to - ensure that it is fully and consistenty quoted. + ensure that it is fully and consistently quoted. """ - parts = path.split("/") - parts = (urllib.quote(urllib.unquote(part), safe="") for part in parts) - return "/".join(parts) + parts = path.split(b"/") + parts = (quote(unquote(part), safe=b"") for part in parts) + return b"/".join(parts) |