diff options
Diffstat (limited to 'requests/utils.py')
| -rw-r--r-- | requests/utils.py | 134 |
1 files changed, 113 insertions, 21 deletions
diff --git a/requests/utils.py b/requests/utils.py index 3ec6131..7b7ff0a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -12,17 +12,19 @@ that are also useful for external consumption. import cgi import codecs import collections +import io import os import platform import re import sys -from netrc import netrc, NetrcParseError +import socket +import struct from . import __version__ from . import certs from .compat import parse_http_list as _parse_list_header -from .compat import (quote, urlparse, bytes, str, OrderedDict, urlunparse, - is_py2, is_py3, builtin_str, getproxies, proxy_bypass) +from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, + builtin_str, getproxies, proxy_bypass) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import MissingSchema, InvalidURL @@ -46,26 +48,47 @@ def dict_to_sequence(d): def super_len(o): if hasattr(o, '__len__'): return len(o) + if hasattr(o, 'len'): return o.len + if hasattr(o, 'fileno'): - return os.fstat(o.fileno()).st_size + try: + fileno = o.fileno() + except io.UnsupportedOperation: + pass + else: + return os.fstat(fileno).st_size + + if hasattr(o, 'getvalue'): + # e.g. BytesIO, cStringIO.StringI + return len(o.getvalue()) def get_netrc_auth(url): """Returns the Requests tuple auth for a given url from netrc.""" try: - locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES) + from netrc import netrc, NetrcParseError + netrc_path = None - for loc in locations: - if os.path.exists(loc) and not netrc_path: + for f in NETRC_FILES: + try: + loc = os.path.expanduser('~/{0}'.format(f)) + except KeyError: + # os.path.expanduser can fail when $HOME is undefined and + # getpwuid fails. See http://bugs.python.org/issue20164 & + # https://github.com/kennethreitz/requests/issues/1846 + return + + if os.path.exists(loc): netrc_path = loc + break # Abort early if there isn't one. if netrc_path is None: - return netrc_path + return ri = urlparse(url) @@ -393,6 +416,56 @@ def requote_uri(uri): return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") +def address_in_network(ip, net): + """ + This function allows you to check if on IP belongs to a network subnet + Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 + returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 + """ + ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] + netaddr, bits = net.split('/') + netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] + network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask + return (ipaddr & netmask) == (network & netmask) + + +def dotted_netmask(mask): + """ + Converts mask from /xx format to xxx.xxx.xxx.xxx + Example: if mask is 24 function returns 255.255.255.0 + """ + bits = 0xffffffff ^ (1 << 32 - mask) - 1 + return socket.inet_ntoa(struct.pack('>I', bits)) + + +def is_ipv4_address(string_ip): + try: + socket.inet_aton(string_ip) + except socket.error: + return False + return True + + +def is_valid_cidr(string_network): + """Very simple check of the cidr format in no_proxy variable""" + if string_network.count('/') == 1: + try: + mask = int(string_network.split('/')[1]) + except ValueError: + return False + + if mask < 1 or mask > 32: + return False + + try: + socket.inet_aton(string_network.split('/')[0]) + except socket.error: + return False + else: + return False + return True + + def get_environ_proxies(url): """Return a dict of environment proxies.""" @@ -408,15 +481,31 @@ def get_environ_proxies(url): # the end of the netloc, both with and without the port. no_proxy = no_proxy.replace(' ', '').split(',') - for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): - # The URL does match something in no_proxy, so we don't want - # to apply the proxies on this URL. - return {} + ip = netloc.split(':')[0] + if is_ipv4_address(ip): + for proxy_ip in no_proxy: + if is_valid_cidr(proxy_ip): + if address_in_network(ip, proxy_ip): + return {} + else: + for host in no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # The URL does match something in no_proxy, so we don't want + # to apply the proxies on this URL. + return {} # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. - if proxy_bypass(netloc): + # The proxy_bypass function is incredibly buggy on OS X in early versions + # of Python 2.6, so allow this call to fail. Only catch the specific + # exceptions we've seen, though: this call failing in other ways can reveal + # legitimate problems. + try: + bypass = proxy_bypass(netloc) + except (TypeError, socket.gaierror): + bypass = False + + if bypass: return {} # If we get here, we either didn't have no_proxy set or we're not going @@ -425,7 +514,7 @@ def get_environ_proxies(url): return getproxies() -def default_user_agent(): +def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" _implementation = platform.python_implementation() @@ -451,7 +540,7 @@ def default_user_agent(): p_system = 'Unknown' p_release = 'Unknown' - return " ".join(['python-requests/%s' % __version__, + return " ".join(['%s/%s' % (name, __version__), '%s/%s' % (_implementation, _implementation_version), '%s/%s' % (p_system, p_release)]) @@ -545,11 +634,14 @@ def except_on_missing_scheme(url): def get_auth_from_url(url): """Given a url with authentication components, extract them into a tuple of username,password.""" - if url: - parsed = urlparse(url) - return (parsed.username, parsed.password) - else: - return ('', '') + parsed = urlparse(url) + + try: + auth = (unquote(parsed.username), unquote(parsed.password)) + except (AttributeError, TypeError): + auth = ('', '') + + return auth def to_native_string(string, encoding='ascii'): |