diff options
| -rw-r--r-- | HISTORY.rst | 16 | ||||
| -rw-r--r-- | PKG-INFO | 20 | ||||
| -rw-r--r-- | README.rst | 2 | ||||
| -rw-r--r-- | requests.egg-info/PKG-INFO | 20 | ||||
| -rw-r--r-- | requests.egg-info/SOURCES.txt | 1 | ||||
| -rw-r--r-- | requests/__init__.py | 4 | ||||
| -rw-r--r-- | requests/api.py | 15 | ||||
| -rw-r--r-- | requests/async.py | 2 | ||||
| -rw-r--r-- | requests/auth.py | 2 | ||||
| -rw-r--r-- | requests/defaults.py | 3 | ||||
| -rw-r--r-- | requests/exceptions.py | 6 | ||||
| -rw-r--r-- | requests/models.py | 79 | ||||
| -rw-r--r-- | requests/packages/oreos/monkeys.py | 2 | ||||
| -rw-r--r-- | requests/packages/urllib3/__init__.py | 18 | ||||
| -rw-r--r-- | requests/packages/urllib3/connectionpool.py | 134 | ||||
| -rw-r--r-- | requests/packages/urllib3/filepost.py | 24 | ||||
| -rw-r--r-- | requests/packages/urllib3/poolmanager.py | 4 | ||||
| -rw-r--r-- | requests/packages/urllib3/request.py | 21 | ||||
| -rw-r--r-- | requests/packages/urllib3/response.py | 15 | ||||
| -rw-r--r-- | requests/packages/urllib3/util.py | 136 | ||||
| -rw-r--r-- | requests/sessions.py | 29 | ||||
| -rw-r--r-- | requests/utils.py | 12 |
22 files changed, 333 insertions, 232 deletions
diff --git a/HISTORY.rst b/HISTORY.rst index c05a71e..e73903d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,22 @@ History ------- + +0.11.1 (2012-03-30) ++++++++++++++++++++ + +* POST redirects now break RFC to do what browsers do: Follow up with a GET. +* New ``strict_mode`` configuration to disable new redirect behavior. + + +0.11.0 (2012-03-14) ++++++++++++++++++++ + +* Private SSL Certificate support +* Remove select.poll from Gevent monkeypatching +* Remove redundant generator for chunked transfer encoding +* Fix: Response.ok raises Timeout Exception in safe_mode + 0.10.8 (2012-03-09) +++++++++++++++++++ @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: requests -Version: 0.10.8 +Version: 0.11.1 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -81,12 +81,28 @@ Description: Requests: HTTP for Humans #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. .. _`the repository`: http://github.com/kennethreitz/requests - .. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS + .. _AUTHORS: https://github.com/kennethreitz/requests/blob/develop/AUTHORS.rst History ------- + + 0.11.1 (2012-03-30) + +++++++++++++++++++ + + * POST redirects now break RFC to do what browsers do: Follow up with a GET. + * New ``strict_mode`` configuration to disable new redirect behavior. + + + 0.11.0 (2012-03-14) + +++++++++++++++++++ + + * Private SSL Certificate support + * Remove select.poll from Gevent monkeypatching + * Remove redundant generator for chunked transfer encoding + * Fix: Response.ok raises Timeout Exception in safe_mode + 0.10.8 (2012-03-09) +++++++++++++++++++ @@ -73,4 +73,4 @@ Contribute #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. .. _`the repository`: http://github.com/kennethreitz/requests -.. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS +.. _AUTHORS: https://github.com/kennethreitz/requests/blob/develop/AUTHORS.rst diff --git a/requests.egg-info/PKG-INFO b/requests.egg-info/PKG-INFO index 677b300..68c9ccc 100644 --- a/requests.egg-info/PKG-INFO +++ b/requests.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: requests -Version: 0.10.8 +Version: 0.11.1 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -81,12 +81,28 @@ Description: Requests: HTTP for Humans #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. .. _`the repository`: http://github.com/kennethreitz/requests - .. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS + .. _AUTHORS: https://github.com/kennethreitz/requests/blob/develop/AUTHORS.rst History ------- + + 0.11.1 (2012-03-30) + +++++++++++++++++++ + + * POST redirects now break RFC to do what browsers do: Follow up with a GET. + * New ``strict_mode`` configuration to disable new redirect behavior. + + + 0.11.0 (2012-03-14) + +++++++++++++++++++ + + * Private SSL Certificate support + * Remove select.poll from Gevent monkeypatching + * Remove redundant generator for chunked transfer encoding + * Fix: Response.ok raises Timeout Exception in safe_mode + 0.10.8 (2012-03-09) +++++++++++++++++++ diff --git a/requests.egg-info/SOURCES.txt b/requests.egg-info/SOURCES.txt index 40e762b..71152be 100644 --- a/requests.egg-info/SOURCES.txt +++ b/requests.egg-info/SOURCES.txt @@ -35,6 +35,7 @@ requests/packages/urllib3/filepost.py requests/packages/urllib3/poolmanager.py requests/packages/urllib3/request.py requests/packages/urllib3/response.py +requests/packages/urllib3/util.py requests/packages/urllib3/packages/__init__.py requests/packages/urllib3/packages/six.py requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py diff --git a/requests/__init__.py b/requests/__init__.py index 73d81f6..0ace12b 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.10.8' -__build__ = 0x001008 +__version__ = '0.11.1' +__build__ = 0x001101 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2012 Kenneth Reitz' diff --git a/requests/api.py b/requests/api.py index b079eed..e0bf346 100644 --- a/requests/api.py +++ b/requests/api.py @@ -33,6 +33,7 @@ def request(method, url, **kwargs): :param config: (optional) A configuration dictionary. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. + :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ s = kwargs.pop('session') if 'session' in kwargs else sessions.session() @@ -44,7 +45,7 @@ def get(url, **kwargs): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -55,7 +56,7 @@ def options(url, **kwargs): """Sends a OPTIONS request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -66,7 +67,7 @@ def head(url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', False) @@ -78,7 +79,7 @@ def post(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('post', url, data=data, **kwargs) @@ -89,7 +90,7 @@ def put(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('put', url, data=data, **kwargs) @@ -100,7 +101,7 @@ def patch(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('patch', url, data=data, **kwargs) @@ -110,7 +111,7 @@ def delete(url, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('delete', url, **kwargs) diff --git a/requests/async.py b/requests/async.py index f2dad69..f12cf26 100644 --- a/requests/async.py +++ b/requests/async.py @@ -17,7 +17,7 @@ except ImportError: raise RuntimeError('Gevent is required for requests.async.') # Monkey-patch. -curious_george.patch_all(thread=False) +curious_george.patch_all(thread=False, select=False) from . import api diff --git a/requests/auth.py b/requests/auth.py index 2e2bebc..385dd27 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -11,7 +11,7 @@ import time import hashlib from base64 import b64encode -from .compat import urlparse, str, bytes +from .compat import urlparse, str from .utils import randombytes, parse_dict_header diff --git a/requests/defaults.py b/requests/defaults.py index 9af9773..6a7ea27 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -15,10 +15,12 @@ Configurations: :max_retries: The number of times a request should be retried in the event of a connection failure. :danger_mode: If true, Requests will raise errors immediately. :safe_mode: If true, Requests will catch all errors. +:strict_mode: If true, Requests will do its best to follow RFCs (e.g. POST redirects). :pool_maxsize: The maximium size of an HTTP connection pool. :pool_connections: The number of active HTTP connection pools to use. :encode_uri: If true, URIs will automatically be percent-encoded. :trust_env: If true, the surrouding environment will be trusted (environ, netrc). + """ SCHEMAS = ['http', 'https'] @@ -41,6 +43,7 @@ defaults['pool_maxsize'] = 10 defaults['max_retries'] = 0 defaults['danger_mode'] = False defaults['safe_mode'] = False +defaults['strict_mode'] = False defaults['keep_alive'] = True defaults['encode_uri'] = True defaults['trust_env'] = True diff --git a/requests/exceptions.py b/requests/exceptions.py index d5b2ab1..3c262e3 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -30,3 +30,9 @@ class URLRequired(RequestException): class TooManyRedirects(RequestException): """Too many redirects.""" + +class MissingSchema(RequestException, ValueError): + """The URL schema (e.g. http or https) is missing.""" + +class InvalidSchema(RequestException, ValueError): + """See defaults.py for valid schemas."""
\ No newline at end of file diff --git a/requests/models.py b/requests/models.py index 753e83a..70e3503 100644 --- a/requests/models.py +++ b/requests/models.py @@ -24,7 +24,7 @@ from .packages.urllib3.filepost import encode_multipart_formdata from .defaults import SCHEMAS from .exceptions import ( ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, - URLRequired, SSLError) + URLRequired, SSLError, MissingSchema, InvalidSchema) from .utils import ( get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, dict_from_string, stream_decode_response_unicode, get_netrc_auth) @@ -63,7 +63,8 @@ class Request(object): config=None, _poolmanager=None, verify=None, - session=None): + session=None, + cert=None): #: Dictionary of configurations for this request. self.config = dict(config or []) @@ -143,6 +144,9 @@ class Request(object): #: SSL Verification. self.verify = verify + #: SSL Certificate + self.cert = cert + if headers: headers = CaseInsensitiveDict(self.headers) else: @@ -212,6 +216,7 @@ class Request(object): self.cookies.update(r.cookies) if r.status_code in REDIRECT_STATI and not self.redirect: + while (('location' in r.headers) and ((r.status_code is codes.see_other) or (self.allow_redirects))): @@ -226,6 +231,7 @@ class Request(object): history.append(r) url = r.headers['location'] + data = self.data # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -243,9 +249,21 @@ class Request(object): # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is codes.see_other: method = 'GET' + data = None else: method = self.method + # Do what the browsers do if strict_mode is off... + if (not self.config.get('strict_mode')): + + if r.status_code in (codes.moved, codes.found) and self.method == 'POST': + method = 'GET' + data = None + + if (r.status_code == 303) and self.method != 'HEAD': + method = 'GET' + data = None + # Remove the cookie headers that were sent. headers = self.headers try: @@ -262,12 +280,14 @@ class Request(object): auth=self.auth, cookies=self.cookies, redirect=True, + data=data, config=self.config, timeout=self.timeout, _poolmanager=self._poolmanager, proxies=self.proxies, verify=self.verify, - session=self.session + session=self.session, + cert=self.cert ) request.send() @@ -321,10 +341,10 @@ class Request(object): scheme, netloc, path, params, query, fragment = urlparse(url) if not scheme: - raise ValueError("Invalid URL %r: No schema supplied" % url) + raise MissingSchema("Invalid URL %r: No schema supplied" % url) if not scheme in SCHEMAS: - raise ValueError("Invalid scheme %r" % scheme) + raise InvalidSchema("Invalid scheme %r" % scheme) netloc = netloc.encode('idna').decode('utf-8') @@ -507,6 +527,13 @@ class Request(object): conn.cert_reqs = 'CERT_NONE' conn.ca_certs = None + if self.cert and self.verify: + if len(self.cert) == 2: + conn.cert_file = self.cert[0] + conn.key_file = self.cert[1] + else: + conn.cert_file = self.cert + if not self.sent or anyway: if self.cookies: @@ -648,7 +675,7 @@ class Response(object): def ok(self): try: self.raise_for_status() - except HTTPError: + except RequestException: return False return True @@ -671,39 +698,7 @@ class Response(object): yield chunk self._content_consumed = True - def generate_chunked(): - resp = self.raw._original_response - fp = resp.fp - if resp.chunk_left is not None: - pending_bytes = resp.chunk_left - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes -= len(chunk) - yield chunk - fp.read(2) # throw away crlf - while 1: - #XXX correct line size? (httplib has 64kb, seems insane) - pending_bytes = fp.readline(40).strip() - if not len(pending_bytes): - # No content, like a HEAD request. Break out. - break - pending_bytes = int(pending_bytes, 16) - if pending_bytes == 0: - break - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes -= len(chunk) - yield chunk - fp.read(2) # throw away crlf - self._content_consumed = True - fp.close() - - if getattr(getattr(self.raw, '_original_response', None), 'chunked', False): - gen = generate_chunked() - else: - gen = generate() - - gen = stream_untransfer(gen, self) + gen = stream_untransfer(generate(), self) if decode_unicode: gen = stream_decode_response_unicode(gen, self) @@ -788,6 +783,12 @@ class Response(object): # Decode unicode from given encoding. try: content = str(self.content, encoding, errors='replace') + except LookupError: + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # So we try blindly encoding. + content = str(self.content, errors='replace') except (UnicodeError, TypeError): pass diff --git a/requests/packages/oreos/monkeys.py b/requests/packages/oreos/monkeys.py index 72ce68d..2cf9016 100644 --- a/requests/packages/oreos/monkeys.py +++ b/requests/packages/oreos/monkeys.py @@ -255,7 +255,7 @@ class CookieError(Exception): # _RFC2965Forbidden = "[]:{}=" _LegalChars = ( string.ascii_letters + string.digits + - "!#$%&'*+-.^_`|~_" + _RFC2965Forbidden ) + "!#$%&'*+-.^_`|~_@" + _RFC2965Forbidden ) _Translator = { '\000' : '\\000', '\001' : '\\001', '\002' : '\\002', '\003' : '\\003', '\004' : '\\004', '\005' : '\\005', diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 2e9c663..2d6fece 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -10,26 +10,20 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.2.2' +__version__ = '1.3' from .connectionpool import ( HTTPConnectionPool, HTTPSConnectionPool, - connection_from_url, - get_host, - make_headers) - - -from .exceptions import ( - HTTPError, - MaxRetryError, - SSLError, - TimeoutError) + connection_from_url +) +from . import exceptions +from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse -from .filepost import encode_multipart_formdata +from .util import make_headers, get_host # Set default logging handler to avoid "No handler found" warnings. diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 39e652e..c3cb3b1 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -7,15 +7,8 @@ import logging import socket -from base64 import b64encode from socket import error as SocketError, timeout as SocketTimeout -try: - from select import poll, POLLIN -except ImportError: # Doesn't exist on OSX and other platforms - from select import select - poll = False - try: # Python 3 from http.client import HTTPConnection, HTTPException from http.client import HTTP_PORT, HTTPS_PORT @@ -42,17 +35,16 @@ try: # Compiled with SSL? import ssl BaseSSLError = ssl.SSLError -except ImportError: +except (ImportError, AttributeError): pass -from .packages.ssl_match_hostname import match_hostname, CertificateError from .request import RequestMethods from .response import HTTPResponse +from .util import get_host, is_connection_dropped from .exceptions import ( EmptyPoolError, HostChangedError, - LocationParseError, MaxRetryError, SSLError, TimeoutError, @@ -61,6 +53,7 @@ from .exceptions import ( from .packages.ssl_match_hostname import match_hostname, CertificateError from .packages import six + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -72,6 +65,7 @@ port_by_scheme = { 'https': HTTPS_PORT, } + ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): @@ -107,6 +101,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): if self.ca_certs: match_hostname(self.sock.getpeercert(), self.host) + ## Pool objects class ConnectionPool(object): @@ -212,7 +207,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn = self.pool.get(block=self.block, timeout=timeout) # If this is a persistent connection, check if it got disconnected - if conn and conn.sock and is_connection_dropped(conn): + if conn and is_connection_dropped(conn): log.info("Resetting dropped connection: %s" % self.host) conn.close() @@ -256,9 +251,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): timeout = self.timeout conn.timeout = timeout # This only does anything in Py26+ - conn.request(method, url, **httplib_request_kw) - conn.sock.settimeout(timeout) + + # Set timeout + sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr. + if sock: + sock.settimeout(timeout) + httplib_response = conn.getresponse() log.debug("\"%s %s %s\" %s %s" % @@ -295,7 +294,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): .. note:: More commonly, it's appropriate to use a convenience method provided - by :class:`.RequestMethods`, such as :meth:`.request`. + by :class:`.RequestMethods`, such as :meth:`request`. .. note:: @@ -495,94 +494,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): return connection -## Helpers - -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None): - """ - Shortcuts for generating request headers. - - :param keep_alive: - If ``True``, adds 'connection: keep-alive' header. - - :param accept_encoding: - Can be a boolean, list, or string. - ``True`` translates to 'gzip,deflate'. - List will get joined by comma. - String will be used as provided. - - :param user_agent: - String representing the user-agent you want, such as - "python-urllib3/0.6" - - :param basic_auth: - Colon-separated username:password string for 'authorization: basic ...' - auth header. - - Example: :: - - >>> make_headers(keep_alive=True, user_agent="Batman/1.0") - {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} - >>> make_headers(accept_encoding=True) - {'accept-encoding': 'gzip,deflate'} - """ - headers = {} - if accept_encoding: - if isinstance(accept_encoding, str): - pass - elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) - else: - accept_encoding = 'gzip,deflate' - headers['accept-encoding'] = accept_encoding - - if user_agent: - headers['user-agent'] = user_agent - - if keep_alive: - headers['connection'] = 'keep-alive' - - if basic_auth: - headers['authorization'] = 'Basic ' + \ - b64encode(six.b(basic_auth)).decode('utf-8') - - return headers - - -def get_host(url): - """ - Given a url, return its scheme, host and port (None if it's not there). - - For example: :: - - >>> get_host('http://google.com/mail/') - ('http', 'google.com', None) - >>> get_host('google.com:80') - ('http', 'google.com', 80) - """ - - # This code is actually similar to urlparse.urlsplit, but much - # simplified for our needs. - port = None - scheme = 'http' - - if '://' in url: - scheme, url = url.split('://', 1) - if '/' in url: - url, _path = url.split('/', 1) - if '@' in url: - _auth, url = url.split('@', 1) - if ':' in url: - url, port = url.split(':', 1) - - if not port.isdigit(): - raise LocationParseError("Failed to parse: %s") - - port = int(port) - - return scheme, url, port - - def connection_from_url(url, **kw): """ Given a url, return an :class:`.ConnectionPool` instance of its host. @@ -608,22 +519,3 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) - - -def is_connection_dropped(conn): - """ - Returns True if the connection is dropped and should be closed. - - :param conn: - ``HTTPConnection`` object. - """ - if not poll: # Platform-specific - return select([conn.sock], [], [], 0.0)[0] - - # This version is better on platforms that support it. - p = poll() - p.register(conn.sock, POLLIN) - for (fno, ev) in p.poll(0.0): - if fno == conn.sock.fileno(): - # Either data is buffered (bad), or the connection is dropped. - return True diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index e1ec8af..344a103 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -24,15 +24,29 @@ def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' +def iter_fields(fields): + """ + Iterate over fields. + + Supports list of (k, v) tuples and dicts. + """ + if isinstance(fields, dict): + return ((k, v) for k, v in six.iteritems(fields)) + + return ((k, v) for k, v in fields) + + def encode_multipart_formdata(fields, boundary=None): """ Encode a dictionary of ``fields`` using the multipart/form-data mime format. :param fields: - Dictionary of fields. The key is treated as the field name, and the - value as the body of the form-data. If the value is a tuple of two - elements, then the first element is treated as the filename of the - form-data section. + Dictionary of fields or list of (key, value) field tuples. The key is + treated as the field name, and the value as the body of the form-data + bytes. If the value is a tuple of two elements, then the first element + is treated as the filename of the form-data section. + + Field names and filenames must be unicode. :param boundary: If not specified, then a random boundary will be generated using @@ -42,7 +56,7 @@ def encode_multipart_formdata(fields, boundary=None): if boundary is None: boundary = choose_boundary() - for fieldname, value in six.iteritems(fields): + for fieldname, value in iter_fields(fields): body.write(b('--%s\r\n' % (boundary))) if isinstance(value, tuple): diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index d42f35b..310ea21 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -39,11 +39,11 @@ class PoolManager(RequestMethods): Example: :: - >>> manager = PoolManager() + >>> manager = PoolManager(num_pools=2) >>> r = manager.urlopen("http://google.com/") >>> r = manager.urlopen("http://google.com/mail") >>> r = manager.urlopen("http://yahoo.com/") - >>> len(r.pools) + >>> len(manager.pools) 2 """ diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py index 5ea26a0..569ac96 100644 --- a/requests/packages/urllib3/request.py +++ b/requests/packages/urllib3/request.py @@ -44,7 +44,7 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, - **kw): + **kw): # Abstract raise NotImplemented("Classes extending RequestMethods must implement " "their own ``urlopen`` method.") @@ -126,22 +126,3 @@ class RequestMethods(object): return self.urlopen(method, url, body=body, headers=headers, **urlopen_kw) - - # Deprecated: - - def get_url(self, url, fields=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_url('GET', url, fields=fields, - **urlopen_kw) - - def post_url(self, url, fields=None, headers=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_body('POST', url, fields=fields, - headers=headers, - **urlopen_kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 4dd431e..5fab824 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -171,11 +171,22 @@ class HTTPResponse(object): with ``original_response=r``. """ + # Normalize headers between different versions of Python + headers = {} + for k, v in r.getheaders(): + # Python 3: Header keys are returned capitalised + k = k.lower() + + has_value = headers.get(k) + if has_value: # Python 3: Repeating header keys are unmerged. + v = ', '.join([has_value, v]) + + headers[k] = v + # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, 'strict', 0) return ResponseCls(body=r, - # In Python 3, the header keys are returned capitalised - headers=dict((k.lower(), v) for k,v in r.getheaders()), + headers=headers, status=r.status, version=r.version, reason=r.reason, diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py new file mode 100644 index 0000000..2684a2f --- /dev/null +++ b/requests/packages/urllib3/util.py @@ -0,0 +1,136 @@ +# urllib3/util.py +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from base64 import b64encode + +try: + from select import poll, POLLIN +except ImportError: # `poll` doesn't exist on OSX and other platforms + poll = False + try: + from select import select + except ImportError: # `select` doesn't exist on AppEngine. + select = False + +from .packages import six +from .exceptions import LocationParseError + + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + b64encode(six.b(basic_auth)).decode('utf-8') + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) + """ + + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + + if '://' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, _path = url.split('/', 1) + if '@' in url: + _auth, url = url.split('@', 1) + if ':' in url: + url, port = url.split(':', 1) + + if not port.isdigit(): + raise LocationParseError("Failed to parse: %s" % url) + + port = int(port) + + return scheme, url, port + + + +def is_connection_dropped(conn): + """ + Returns True if the connection is dropped and should be closed. + + :param conn: + ``HTTPConnection`` object. + + Note: For platforms like AppEngine, this will always return ``False`` to + let the platform handle connection recycling transparently for us. + """ + sock = getattr(conn, 'sock', False) + if not sock: #Platform-specific: AppEngine + return False + + if not poll: # Platform-specific + if not select: #Platform-specific: AppEngine + return False + + return select([sock], [], [], 0.0)[0] + + # This version is better on platforms that support it. + p = poll() + p.register(sock, POLLIN) + for (fno, ev) in p.poll(0.0): + if fno == sock.fileno(): + # Either data is buffered (bad), or the connection is dropped. + return True diff --git a/requests/sessions.py b/requests/sessions.py index 87320d6..94c94bf 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -52,7 +52,7 @@ class Session(object): __attrs__ = [ 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', - 'params', 'config', 'verify'] + 'params', 'config', 'verify', 'cert'] def __init__(self, @@ -65,7 +65,8 @@ class Session(object): params=None, config=None, prefetch=False, - verify=True): + verify=True, + cert=None): self.headers = headers or {} self.cookies = cookies or {} @@ -77,6 +78,7 @@ class Session(object): self.config = config or {} self.prefetch = prefetch self.verify = verify + self.cert = cert for (k, v) in list(defaults.items()): self.config.setdefault(k, v) @@ -113,13 +115,14 @@ class Session(object): files=None, auth=None, timeout=None, - allow_redirects=False, + allow_redirects=True, proxies=None, hooks=None, return_response=True, config=None, prefetch=False, - verify=None): + verify=None, + cert=None): """Constructs and sends a :class:`Request <Request>`. Returns :class:`Response <Response>` object. @@ -133,12 +136,13 @@ class Session(object): :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param return_response: (optional) If False, an un-sent Request object will returned. :param config: (optional) A configuration dictionary. :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. + :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ method = str(method).upper() @@ -176,6 +180,7 @@ class Session(object): proxies=proxies, config=config, verify=verify, + cert=cert, _poolmanager=self.poolmanager ) @@ -213,7 +218,7 @@ class Session(object): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -224,7 +229,7 @@ class Session(object): """Sends a OPTIONS request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -235,7 +240,7 @@ class Session(object): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', False) @@ -247,7 +252,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('post', url, data=data, **kwargs) @@ -258,7 +263,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('put', url, data=data, **kwargs) @@ -269,7 +274,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('patch', url, data=data, **kwargs) @@ -279,7 +284,7 @@ class Session(object): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('delete', url, **kwargs) diff --git a/requests/utils.py b/requests/utils.py index 6952a99..b722d99 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,7 +14,6 @@ import codecs import os import random import re -import traceback import zlib from netrc import netrc, NetrcParseError @@ -26,6 +25,15 @@ from .compat import basestring, bytes, str NETRC_FILES = ('.netrc', '_netrc') +def dict_to_sequence(d): + """Returns an internal sequence dictionary update.""" + + if hasattr(d, 'items'): + d = d.items() + + return d + + def get_netrc_auth(url): """Returns the Requests tuple auth for a given url from netrc.""" @@ -51,7 +59,7 @@ def get_netrc_auth(url): # Return with login / password login_i = (0 if _netrc[0] else 1) return (_netrc[login_i], _netrc[2]) - except (NetrcParseError, IOError): + except (NetrcParseError, IOError, AttributeError): # If there was a parsing error or a permissions issue reading the file, # we'll just skip netrc auth pass |