From d4aa2de2bb89ca384ad81db731bb99735e1db788 Mon Sep 17 00:00:00 2001 From: SVN-Git Migration Date: Thu, 8 Oct 2015 13:41:23 -0700 Subject: Imported Upstream version 0.12.1 --- HISTORY.rst | 21 ++++ PKG-INFO | 25 +++- requests.egg-info/PKG-INFO | 25 +++- requests.egg-info/SOURCES.txt | 2 + requests.egg-info/requires.txt | 6 +- requests/__init__.py | 4 +- requests/auth.py | 41 +++++++ requests/compat.py | 4 +- requests/cookies.py | 264 +++++++++++++++++++++++++++++++++++++++++ requests/hooks.py | 5 +- requests/models.py | 204 +++++++++++++++---------------- requests/sessions.py | 45 +++++-- requests/utils.py | 84 ++++--------- setup.py | 42 +++++-- tests/test_cookies.py | 207 ++++++++++++++++++++++++++++++++ tests/test_requests.py | 83 +++++++++++-- tests/test_requests_async.py | 1 - tests/test_requests_ext.py | 23 +++- 18 files changed, 875 insertions(+), 211 deletions(-) create mode 100644 requests/cookies.py create mode 100755 tests/test_cookies.py diff --git a/HISTORY.rst b/HISTORY.rst index 153cbb5..8b39235 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,27 @@ History ------- +0.12.1 (2012-05-08) ++++++++++++++++++++ + +- New ``Response.json`` property +- Ability to add string file uploads +- Fix out-of-range issue with iter_lines +- Fix iter_content default size +- Fix POST redirects containing files + +0.12.0 (2012-05-02) ++++++++++++++++++++ + +- EXPERIMENTAL OAUTH SUPPORT! +- Proper CookieJar-backed cookies interface with awesome dict-like interface. +- Speed fix for non-iterated content chunks. +- Move ``pre_request`` to a more usable place. +- New ``pre_send`` hook. +- Lazily encode data, params, files +- Load system Certificate Bundle if ``certify`` isn't available. +- Cleanups, fixes. + 0.11.2 (2012-04-22) +++++++++++++++++++ diff --git a/PKG-INFO b/PKG-INFO index dc8b700..55e0cb2 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ -Metadata-Version: 1.0 +Metadata-Version: 1.1 Name: requests -Version: 0.11.2 +Version: 0.12.1 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -101,6 +101,27 @@ Description: Requests: HTTP for Humans History ------- + 0.12.1 (2012-05-08) + +++++++++++++++++++ + + - New ``Response.json`` property + - Ability to add string file uploads + - Fix out-of-range issue with iter_lines + - Fix iter_content default size + - Fix POST redirects containing files + + 0.12.0 (2012-05-02) + +++++++++++++++++++ + + - EXPERIMENTAL OAUTH SUPPORT! + - Proper CookieJar-backed cookies interface with awesome dict-like interface. + - Speed fix for non-iterated content chunks. + - Move ``pre_request`` to a more usable place. + - New ``pre_send`` hook. + - Lazily encode data, params, files + - Load system Certificate Bundle if ``certify`` isn't available. + - Cleanups, fixes. + 0.11.2 (2012-04-22) +++++++++++++++++++ diff --git a/requests.egg-info/PKG-INFO b/requests.egg-info/PKG-INFO index dc8b700..55e0cb2 100644 --- a/requests.egg-info/PKG-INFO +++ b/requests.egg-info/PKG-INFO @@ -1,6 +1,6 @@ -Metadata-Version: 1.0 +Metadata-Version: 1.1 Name: requests -Version: 0.11.2 +Version: 0.12.1 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -101,6 +101,27 @@ Description: Requests: HTTP for Humans History ------- + 0.12.1 (2012-05-08) + +++++++++++++++++++ + + - New ``Response.json`` property + - Ability to add string file uploads + - Fix out-of-range issue with iter_lines + - Fix iter_content default size + - Fix POST redirects containing files + + 0.12.0 (2012-05-02) + +++++++++++++++++++ + + - EXPERIMENTAL OAUTH SUPPORT! + - Proper CookieJar-backed cookies interface with awesome dict-like interface. + - Speed fix for non-iterated content chunks. + - Move ``pre_request`` to a more usable place. + - New ``pre_send`` hook. + - Lazily encode data, params, files + - Load system Certificate Bundle if ``certify`` isn't available. + - Cleanups, fixes. + 0.11.2 (2012-04-22) +++++++++++++++++++ diff --git a/requests.egg-info/SOURCES.txt b/requests.egg-info/SOURCES.txt index f228744..ec1a22a 100644 --- a/requests.egg-info/SOURCES.txt +++ b/requests.egg-info/SOURCES.txt @@ -9,6 +9,7 @@ requests/api.py requests/async.py requests/auth.py requests/compat.py +requests/cookies.py requests/defaults.py requests/exceptions.py requests/hooks.py @@ -40,6 +41,7 @@ requests/packages/urllib3/packages/__init__.py requests/packages/urllib3/packages/six.py requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py requests/packages/urllib3/packages/ssl_match_hostname/__init__.py +tests/test_cookies.py tests/test_requests.py tests/test_requests_async.py tests/test_requests_ext.py diff --git a/requests.egg-info/requires.txt b/requests.egg-info/requires.txt index a8c0e5c..ccff72e 100644 --- a/requests.egg-info/requires.txt +++ b/requests.egg-info/requires.txt @@ -1,2 +1,6 @@ certifi>=0.0.7 -chardet>=1.0.0 \ No newline at end of file +oauthlib>=0.1.0,<0.2.0 +chardet>=1.0.0 + +[async] +gevent \ No newline at end of file diff --git a/requests/__init__.py b/requests/__init__.py index 96ef774..483ac87 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.11.2' -__build__ = 0x001102 +__version__ = '0.12.1' +__build__ = 0x001201 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2012 Kenneth Reitz' diff --git a/requests/auth.py b/requests/auth.py index 353180a..a20c545 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -11,10 +11,20 @@ import time import hashlib from base64 import b64encode + from .compat import urlparse, str from .utils import randombytes, parse_dict_header +try: + from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER) + from oauthlib.common import extract_params + # hush pyflakes: + SIGNATURE_HMAC; SIGNATURE_TYPE_AUTH_HEADER +except (ImportError, SyntaxError): + SIGNATURE_HMAC = None + SIGNATURE_TYPE_AUTH_HEADER = None +CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' def _basic_auth_str(username, password): """Returns a Basic Auth string.""" @@ -29,6 +39,37 @@ class AuthBase(object): raise NotImplementedError('Auth hooks must be callable.') +class OAuth1(AuthBase): + """Signs the request using OAuth 1 (RFC5849)""" + def __init__(self, client_key, + client_secret=None, + resource_owner_key=None, + resource_owner_secret=None, + callback_uri=None, + signature_method=SIGNATURE_HMAC, + signature_type=SIGNATURE_TYPE_AUTH_HEADER, + rsa_key=None, verifier=None): + + try: + signature_type = signature_type.upper() + except AttributeError: + pass + + self.client = Client(client_key, client_secret, resource_owner_key, + resource_owner_secret, callback_uri, signature_method, + signature_type, rsa_key, verifier) + + def __call__(self, r): + contenttype = r.headers.get('Content-Type', None) + decoded_body = extract_params(r.data) + if contenttype == None and decoded_body != None: + r.headers['Content-Type'] = 'application/x-www-form-urlencoded' + + r.url, r.headers, r.data = self.client.sign( + unicode(r.url), unicode(r.method), r.data, r.headers) + return r + + class HTTPBasicAuth(AuthBase): """Attaches HTTP Basic Authentication to the given Request object.""" def __init__(self, username, password): diff --git a/requests/compat.py b/requests/compat.py index fec7a01..37063f5 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -83,7 +83,7 @@ if is_py2: from urlparse import urlparse, urlunparse, urljoin, urlsplit from urllib2 import parse_http_list import cookielib - from .packages.oreos.monkeys import SimpleCookie + from Cookie import Morsel from StringIO import StringIO bytes = str @@ -96,7 +96,7 @@ elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote from urllib.request import parse_http_list from http import cookiejar as cookielib - from http.cookies import SimpleCookie + from http.cookies import Morsel from io import StringIO str = str diff --git a/requests/cookies.py b/requests/cookies.py new file mode 100644 index 0000000..0e0dd67 --- /dev/null +++ b/requests/cookies.py @@ -0,0 +1,264 @@ +""" +Compatibility code to be able to use `cookielib.CookieJar` with requests. + +requests.utils imports from here, so be careful with imports. +""" + +import collections +from .compat import cookielib, urlparse, Morsel + +try: + import threading + # grr, pyflakes: this fixes "redefinition of unused 'threading'" + threading +except ImportError: + import dummy_threading as threading + +class MockRequest(object): + """Wraps a `requests.Request` to mimic a `urllib2.Request`. + + The code in `cookielib.CookieJar` expects this interface in order to correctly + manage cookie policies, i.e., determine whether a cookie can be set, given the + domains of the request and the cookie. + + The original request object is read-only. The client is responsible for collecting + the new headers via `get_new_headers()` and interpreting them appropriately. You + probably want `get_cookie_header`, defined below. + """ + + def __init__(self, request): + self._r = request + self._new_headers = {} + + def get_type(self): + return urlparse(self._r.full_url).scheme + + def get_host(self): + return urlparse(self._r.full_url).netloc + + def get_origin_req_host(self): + if self._r.response.history: + r = self._r.response.history[0] + return urlparse(r).netloc + else: + return self.get_host() + + def get_full_url(self): + return self._r.full_url + + def is_unverifiable(self): + # unverifiable == redirected + return bool(self._r.response.history) + + def has_header(self, name): + return name in self._r.headers or name in self._new_headers + + def get_header(self, name, default=None): + return self._r.headers.get(name, self._new_headers.get(name, default)) + + def add_header(self, key, val): + """cookielib has no legitimate use for this method; add it back if you find one.""" + raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") + + def add_unredirected_header(self, name, value): + self._new_headers[name] = value + + def get_new_headers(self): + return self._new_headers + +class MockResponse(object): + """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. + + ...what? Basically, expose the parsed HTTP headers from the server response + the way `cookielib` expects to see them. + """ + + def __init__(self, headers): + """Make a MockResponse for `cookielib` to read. + + :param headers: a httplib.HTTPMessage or analogous carrying the headers + """ + self._headers = headers + + def info(self): + return self._headers + + def getheaders(self, name): + self._headers.getheaders(name) + +def extract_cookies_to_jar(jar, request, response): + """Extract the cookies from the response into a CookieJar. + + :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar) + :param request: our own requests.Request object + :param response: urllib3.HTTPResponse object + """ + # the _original_response field is the wrapped httplib.HTTPResponse object, + # and in safe mode, it may be None if the request didn't actually complete. + # in that case, just skip the cookie extraction. + if response._original_response is not None: + req = MockRequest(request) + # pull out the HTTPMessage with the headers and put it in the mock: + res = MockResponse(response._original_response.msg) + jar.extract_cookies(res, req) + +def get_cookie_header(jar, request): + """Produce an appropriate Cookie header string to be sent with `request`, or None.""" + r = MockRequest(request) + jar.add_cookie_header(r) + return r.get_new_headers().get('Cookie') + +def remove_cookie_by_name(cookiejar, name, domain=None, path=None): + """Unsets a cookie by name, by default over all domains and paths. + + Wraps CookieJar.clear(), is O(n). + """ + clearables = [] + for cookie in cookiejar: + if cookie.name == name: + if domain is None or domain == cookie.domain: + if path is None or path == cookie.path: + clearables.append((cookie.domain, cookie.path, cookie.name)) + + for domain, path, name in clearables: + cookiejar.clear(domain, path, name) + +class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): + """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. + + This is the CookieJar we create by default for requests and sessions that + don't specify one, since some clients may expect response.cookies and + session.cookies to support dict operations. + + Don't use the dict interface internally; it's just for compatibility with + with external client code. All `requests` code should work out of the box + with externally provided instances of CookieJar, e.g., LWPCookieJar and + FileCookieJar. + + Caution: dictionary operations that are normally O(1) may be O(n). + + Unlike a regular CookieJar, this class is pickleable. + """ + + def get(self, name, domain=None, path=None, default=None): + try: + return self._find(name, domain, path) + except KeyError: + return default + + def set(self, name, value, **kwargs): + # support client code that unsets cookies by assignment of a None value: + if value is None: + remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) + return + + if isinstance(value, Morsel): + c = morsel_to_cookie(value) + else: + c = create_cookie(name, value, **kwargs) + self.set_cookie(c) + return c + + def __getitem__(self, name): + return self._find(name) + + def __setitem__(self, name, value): + self.set(name, value) + + def __delitem__(self, name): + remove_cookie_by_name(self, name) + + def _find(self, name, domain=None, path=None): + for cookie in iter(self): + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + return cookie.value + + raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) + + def __getstate__(self): + state = self.__dict__.copy() + # remove the unpickleable RLock object + state.pop('_cookies_lock') + return state + + def __setstate__(self, state): + self.__dict__.update(state) + if '_cookies_lock' not in self.__dict__: + self._cookies_lock = threading.RLock() + + def copy(self): + """We're probably better off forbidding this.""" + raise NotImplementedError + +def create_cookie(name, value, **kwargs): + """Make a cookie from underspecified parameters. + + By default, the pair of `name` and `value` will be set for the domain '' + and sent on every request (this is sometimes called a "supercookie"). + """ + result = dict( + version=0, + name=name, + value=value, + port=None, + domain='', + path='/', + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False, + ) + + badargs = set(kwargs) - set(result) + if badargs: + err = 'create_cookie() got unexpected keyword arguments: %s' + raise TypeError(err % list(badargs)) + + result.update(kwargs) + result['port_specified'] = bool(result['port']) + result['domain_specified'] = bool(result['domain']) + result['domain_initial_dot'] = result['domain'].startswith('.') + result['path_specified'] = bool(result['path']) + + return cookielib.Cookie(**result) + +def morsel_to_cookie(morsel): + """Convert a Morsel object into a Cookie containing the one k/v pair.""" + c = create_cookie( + name=morsel.key, + value=morsel.value, + version=morsel['version'] or 0, + port=None, + port_specified=False, + domain=morsel['domain'], + domain_specified=bool(morsel['domain']), + domain_initial_dot=morsel['domain'].startswith('.'), + path=morsel['path'], + path_specified=bool(morsel['path']), + secure=bool(morsel['secure']), + expires=morsel['max-age'] or morsel['expires'], + discard=False, + comment=morsel['comment'], + comment_url=bool(morsel['comment']), + rest={'HttpOnly': morsel['httponly']}, + rfc2109=False, + ) + return c + +def cookiejar_from_dict(cookie_dict, cookiejar=None): + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ + if cookiejar is None: + cookiejar = RequestsCookieJar() + + if cookie_dict is not None: + for name in cookie_dict: + cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) + return cookiejar diff --git a/requests/hooks.py b/requests/hooks.py index 3560b89..13d0eb5 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -12,6 +12,9 @@ Available hooks: A dictionary of the arguments being sent to Request(). ``pre_request``: + The Request object, directly after being created. + +``pre_send``: The Request object, directly before being sent. ``post_request``: @@ -25,7 +28,7 @@ Available hooks: import traceback -HOOKS = ('args', 'pre_request', 'post_request', 'response') +HOOKS = ('args', 'pre_request', 'pre_send', 'post_request', 'response') def dispatch_hook(key, hooks, hook_data): diff --git a/requests/models.py b/requests/models.py index 60f58d2..fbdfb56 100644 --- a/requests/models.py +++ b/requests/models.py @@ -7,6 +7,7 @@ requests.models This module contains the primary objects that power Requests. """ +import json import os from datetime import datetime @@ -15,6 +16,7 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth +from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError, LocationParseError from .packages.urllib3.exceptions import SSLError as _SSLError @@ -27,20 +29,22 @@ from .exceptions import ( URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL) from .utils import ( get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, - dict_from_string, stream_decode_response_unicode, get_netrc_auth, + stream_decode_response_unicode, get_netrc_auth, get_environ_proxies, DEFAULT_CA_BUNDLE_PATH) from .compat import ( - urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, - SimpleCookie, is_py2) + cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, + StringIO, is_py2) # Import chardet if it is available. try: import chardet + # hush pyflakes + chardet except ImportError: - pass + chardet = None REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - +CONTENT_CHUNK_SIZE = 10 * 1024 class Request(object): """The :class:`Request ` object. It carries out all functionality of @@ -109,14 +113,11 @@ class Request(object): # If no proxies are given, allow configuration by environment variables # HTTP_PROXY and HTTPS_PROXY. if not self.proxies and self.config.get('trust_env'): - if 'HTTP_PROXY' in os.environ: - self.proxies['http'] = os.environ['HTTP_PROXY'] - if 'HTTPS_PROXY' in os.environ: - self.proxies['https'] = os.environ['HTTPS_PROXY'] + self.proxies = get_environ_proxies() - self.data, self._enc_data = self._encode_params(data) - self.params, self._enc_params = self._encode_params(params) - self.files, self._enc_files = self._encode_files(files) + self.data = data + self.params = params + self.files = files #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. @@ -126,7 +127,10 @@ class Request(object): self.auth = auth #: CookieJar to attach to :class:`Request `. - self.cookies = dict(cookies or []) + if isinstance(cookies, cookielib.CookieJar): + self.cookies = cookies + else: + self.cookies = cookiejar_from_dict(cookies) #: True if Request has been sent. self.sent = False @@ -193,16 +197,11 @@ class Request(object): # Set encoding. response.encoding = get_encoding_from_headers(response.headers) - # Start off with our local cookies. - cookies = self.cookies or dict() - # Add new cookies from the server. - if 'set-cookie' in response.headers: - cookie_header = response.headers['set-cookie'] - cookies = dict_from_string(cookie_header) + extract_cookies_to_jar(self.cookies, self, resp) # Save cookies in Response. - response.cookies = cookies + response.cookies = self.cookies # No exceptions were harmed in the making of this request. response.error = getattr(resp, 'error', None) @@ -220,8 +219,6 @@ class Request(object): r = build(resp) - self.cookies.update(r.cookies) - if r.status_code in REDIRECT_STATI and not self.redirect: while (('location' in r.headers) and @@ -239,6 +236,7 @@ class Request(object): url = r.headers['location'] data = self.data + files = self.files # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -257,6 +255,7 @@ class Request(object): if r.status_code is codes.see_other: method = 'GET' data = None + files = None else: method = self.method @@ -266,10 +265,12 @@ class Request(object): if r.status_code in (codes.moved, codes.found) and self.method == 'POST': method = 'GET' data = None + files = None if (r.status_code == 303) and self.method != 'HEAD': method = 'GET' data = None + files = None # Remove the cookie headers that were sent. headers = self.headers @@ -281,7 +282,7 @@ class Request(object): request = Request( url=url, headers=headers, - files=self.files, + files=files, method=method, params=self.session.params, auth=self.auth, @@ -299,46 +300,46 @@ class Request(object): request.send() r = request.response - self.cookies.update(r.cookies) r.history = history self.response = r self.response.request = self - self.response.cookies.update(self.cookies) @staticmethod def _encode_params(data): """Encode parameters in a piece of data. - If the data supplied is a dictionary, encodes each parameter in it, and - returns a list of tuples containing the encoded parameters, and a urlencoded - version of that. - - Otherwise, assumes the data is already encoded appropriately, and - returns it twice. + Will successfully encode parameters when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. """ if isinstance(data, bytes): - return data, data - - if hasattr(data, '__iter__') and not isinstance(data, str): - data = dict(data) + return data + if isinstance(data, str): + return data + elif hasattr(data, '__iter__'): + try: + dict(data) + except ValueError: + raise ValueError('Unable to encode lists with elements that are not 2-tuples.') - if hasattr(data, 'items'): + params = list(data.items() if isinstance(data, dict) else data) result = [] - for k, vs in list(data.items()): + for k, vs in params: for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, str) else k, - v.encode('utf-8') if isinstance(v, str) else v)) - return result, urlencode(result, doseq=True) + result.append( + (k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v)) + return urlencode(result, doseq=True) else: - return data, data + return data - def _encode_files(self,files): + def _encode_files(self, files): if (not files) or isinstance(self.data, str): - return None, None + return None try: fields = self.data.copy() @@ -352,11 +353,13 @@ class Request(object): else: fn = guess_filename(v) or k fp = v + if isinstance(fp, (bytes, str)): + fp = StringIO(fp) fields.update({k: (fn, fp.read())}) (body, content_type) = encode_multipart_formdata(fields) - return files, (body, content_type) + return (body, content_type) @property def full_url(self): @@ -381,7 +384,6 @@ class Request(object): if not path: path = '/' - if is_py2: if isinstance(scheme, str): scheme = scheme.encode('utf-8') @@ -398,11 +400,12 @@ class Request(object): url = (urlunparse([scheme, netloc, path, params, query, fragment])) - if self._enc_params: + enc_params = self._encode_params(self.params) + if enc_params: if urlparse(url).query: - url = '%s&%s' % (url, self._enc_params) + url = '%s&%s' % (url, enc_params) else: - url = '%s?%s' % (url, self._enc_params) + url = '%s?%s' % (url, enc_params) if self.config.get('encode_uri', True): url = requote_uri(url) @@ -439,7 +442,7 @@ class Request(object): self.hooks[event].append(hook) - def deregister_hook(self,event,hook): + def deregister_hook(self, event, hook): """Deregister a previously registered hook. Returns True if the hook existed, False if not. """ @@ -464,6 +467,10 @@ class Request(object): # Build the URL url = self.full_url + # Pre-request hook. + r = dispatch_hook('pre_request', self.hooks, self) + self.__dict__.update(r.__dict__) + # Logging if self.config.get('verbose'): self.config.get('verbose').write('%s %s %s\n' % ( @@ -474,22 +481,6 @@ class Request(object): body = None content_type = None - # Multi-part file uploads. - if self.files: - (body, content_type) = self._enc_files - else: - if self.data: - - body = self._enc_data - if isinstance(self.data, str): - content_type = None - else: - content_type = 'application/x-www-form-urlencoded' - - # Add content-type if it wasn't explicitly provided. - if (content_type) and (not 'content-type' in self.headers): - self.headers['Content-Type'] = content_type - # Use .netrc auth if none was provided. if not self.auth and self.config.get('trust_env'): self.auth = get_netrc_auth(url) @@ -505,6 +496,22 @@ class Request(object): # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) + # Multi-part file uploads. + if self.files: + (body, content_type) = self._encode_files(self.files) + else: + if self.data: + + body = self._encode_params(self.data) + if isinstance(self.data, str): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' + + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type + _p = urlparse(url) proxy = self.proxies.get(_p.scheme) @@ -523,6 +530,7 @@ class Request(object): conn = self._poolmanager.connection_from_url(url) else: conn = connectionpool.connection_from_url(url) + self.headers['Connection'] = 'close' except LocationParseError as e: raise InvalidURL(e) @@ -563,24 +571,14 @@ class Request(object): if not self.sent or anyway: - if self.cookies: - - # Skip if 'cookie' header is explicitly set. - if 'cookie' not in self.headers: - - # Simple cookie with our dict. - c = SimpleCookie() - for (k, v) in list(self.cookies.items()): - c[k] = v - - # Turn it into a header. - cookie_header = c.output(header='', sep='; ').strip() - - # Attach Cookie header to request. + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: + cookie_header = get_cookie_header(self.cookies, self) + if cookie_header is not None: self.headers['Cookie'] = cookie_header - # Pre-request hook. - r = dispatch_hook('pre_request', self.hooks, self) + # Pre-send hook. + r = dispatch_hook('pre_send', self.hooks, self) self.__dict__.update(r.__dict__) try: @@ -621,7 +619,15 @@ class Request(object): else: raise - self._build_response(r) + # build_response can throw TooManyRedirects + try: + self._build_response(r) + except RequestException as e: + if self.config.get('safe_mode', False): + # In safe mode, catch the exception + self.response.error = e + else: + raise # Response manipulation hook. self.response = dispatch_hook('response', self.hooks, self.response) @@ -681,8 +687,8 @@ class Response(object): #: The :class:`Request ` that created the Response. self.request = None - #: A dictionary of Cookies the server sent back. - self.cookies = {} + #: A CookieJar of Cookies the server sent back. + self.cookies = None #: Dictionary of configurations for this request. self.config = {} @@ -748,7 +754,7 @@ class Response(object): chunk = pending + chunk lines = chunk.splitlines() - if lines[-1][-1] == chunk[-1]: + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None @@ -773,7 +779,7 @@ class Response(object): if self.status_code is 0: self._content = None else: - self._content = bytes().join(self.iter_content()) or bytes() + self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() except AttributeError: self._content = None @@ -781,16 +787,6 @@ class Response(object): self._content_consumed = True return self._content - def _detected_encoding(self): - try: - detected = chardet.detect(self.content) or {} - return detected.get('encoding') - - # Trust that chardet isn't available or something went terribly wrong. - except Exception: - pass - - @property def text(self): """Content of the response, in unicode. @@ -803,9 +799,10 @@ class Response(object): content = None encoding = self.encoding - # Fallback to auto-detected encoding if chardet is available. + # Fallback to auto-detected encoding. if self.encoding is None: - encoding = self._detected_encoding() + if chardet is not None: + encoding = chardet.detect(self.content)['encoding'] # Decode unicode from given encoding. try: @@ -816,11 +813,17 @@ class Response(object): # # So we try blindly encoding. content = str(self.content, errors='replace') - except (UnicodeError, TypeError): - pass return content + @property + def json(self): + """Returns the json-encoded content of a request, if any.""" + try: + return json.loads(self.text or self.content) + except ValueError: + return None + def raise_for_status(self, allow_redirects=True): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" @@ -837,7 +840,6 @@ class Response(object): http_error.response = self raise http_error - elif (self.status_code >= 500) and (self.status_code < 600): http_error = HTTPError('%s Server Error' % self.status_code) http_error.response = self diff --git a/requests/sessions.py b/requests/sessions.py index 0e43030..8d517ab 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -9,13 +9,14 @@ requests (cookies, auth, proxies). """ +from .compat import cookielib +from .cookies import cookiejar_from_dict, remove_cookie_by_name from .defaults import defaults from .models import Request from .hooks import dispatch_hook from .utils import header_expand from .packages.urllib3.poolmanager import PoolManager - def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. @@ -69,7 +70,6 @@ class Session(object): cert=None): self.headers = headers or {} - self.cookies = cookies or {} self.auth = auth self.timeout = timeout self.proxies = proxies or {} @@ -86,11 +86,10 @@ class Session(object): self.init_poolmanager() # Set up a CookieJar to be used by default - self.cookies = {} - - # Add passed cookies in. - if cookies is not None: - self.cookies.update(cookies) + if isinstance(cookies, cookielib.CookieJar): + self.cookies = cookies + else: + self.cookies = cookiejar_from_dict(cookies) def init_poolmanager(self): self.poolmanager = PoolManager( @@ -134,7 +133,7 @@ class Session(object): :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. + :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. @@ -148,7 +147,6 @@ class Session(object): method = str(method).upper() # Default empty dicts for dict params. - cookies = {} if cookies is None else cookies data = {} if data is None else data files = {} if files is None else files headers = {} if headers is None else headers @@ -185,11 +183,33 @@ class Session(object): _poolmanager=self.poolmanager ) + # merge session cookies into passed-in ones + dead_cookies = None + # passed-in cookies must become a CookieJar: + if not isinstance(cookies, cookielib.CookieJar): + args['cookies'] = cookiejar_from_dict(cookies) + # support unsetting cookies that have been passed in with None values + # this is only meaningful when `cookies` is a dict --- + # for a real CookieJar, the client should use session.cookies.clear() + if cookies is not None: + dead_cookies = [name for name in cookies if cookies[name] is None] + # merge the session's cookies into the passed-in cookies: + for cookie in self.cookies: + args['cookies'].set_cookie(cookie) + # remove the unset cookies from the jar we'll be using with the current request + # (but not from the session's own store of cookies): + if dead_cookies is not None: + for name in dead_cookies: + remove_cookie_by_name(args['cookies'], name) + # Merge local kwargs with session kwargs. for attr in self.__attrs__: + # we already merged cookies: + if attr == 'cookies': + continue + session_val = getattr(self, attr, None) local_val = args.get(attr) - args[attr] = merge_kwargs(local_val, session_val) # Arguments manipulation hook. @@ -209,7 +229,10 @@ class Session(object): r.send(prefetch=prefetch) # Send any cookies back up the to the session. - self.cookies.update(r.response.cookies) + # (in safe mode, cookies may be None if the request didn't succeed) + if r.response.cookies is not None: + for cookie in r.response.cookies: + self.cookies.set_cookie(cookie) # Return the response. return r.response diff --git a/requests/utils.py b/requests/utils.py index 925547a..8365cc3 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -18,8 +18,11 @@ import zlib from netrc import netrc, NetrcParseError from .compat import parse_http_list as _parse_list_header -from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse +from .compat import quote, is_py2, urlparse from .compat import basestring, bytes, str +from .cookies import RequestsCookieJar, cookiejar_from_dict + +_hush_pyflakes = (RequestsCookieJar,) CERTIFI_BUNDLE_PATH = None try: @@ -97,25 +100,6 @@ def get_netrc_auth(url): pass - -def dict_from_string(s): - """Returns a MultiDict with Cookies.""" - - cookies = dict() - - try: - c = SimpleCookie() - c.load(s) - - for k, v in list(c.items()): - cookies.update({k: v.value}) - # This stuff is not to be trusted. - except Exception: - pass - - return cookies - - def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) @@ -290,24 +274,6 @@ def dict_from_cookiejar(cj): return cookie_dict -def cookiejar_from_dict(cookie_dict): - """Returns a CookieJar from a key/value dictionary. - - :param cookie_dict: Dict of key/values to insert into CookieJar. - """ - - # return cookiejar if one was passed in - if isinstance(cookie_dict, cookielib.CookieJar): - return cookie_dict - - # create cookiejar - cj = cookielib.CookieJar() - - cj = add_dict_to_cookiejar(cj, cookie_dict) - - return cj - - def add_dict_to_cookiejar(cj, cookie_dict): """Returns a CookieJar from a key/value dictionary. @@ -315,31 +281,9 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cookie_dict: Dict of key/values to insert into CookieJar. """ - for k, v in list(cookie_dict.items()): - - cookie = cookielib.Cookie( - version=0, - name=k, - value=v, - port=None, - port_specified=False, - domain='', - domain_specified=False, - domain_initial_dot=False, - path='/', - path_specified=True, - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False - ) - - # add cookie to cookiejar + cj2 = cookiejar_from_dict(cookie_dict) + for cookie in cj2: cj.set_cookie(cookie) - return cj @@ -502,3 +446,19 @@ def requote_uri(uri): # Then quote only illegal characters (do not quote reserved, unreserved, # or '%') return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") + +def get_environ_proxies(): + """Return a dict of environment proxies.""" + + proxy_keys = [ + 'all', + 'http', + 'https', + 'ftp', + 'socks', + 'no' + ] + + get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) + proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys] + return dict([(key, val) for (key, val) in proxies if val]) diff --git a/setup.py b/setup.py index 3133a97..1ab216b 100755 --- a/setup.py +++ b/setup.py @@ -1,27 +1,26 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- + +""" +distutils/setuptools install script. See inline comments for packaging documentation. +""" import os import sys + import requests -from requests.compat import is_py3, is_py2 +from requests.compat import is_py3 try: from setuptools import setup + # hush pyflakes + setup except ImportError: from distutils.core import setup - - if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') sys.exit() -if sys.argv[-1] == 'test': - os.system('python tests/test_requests.py') - sys.exit() - -required = ['certifi>=0.0.7',] packages = [ 'requests', 'requests.packages', @@ -31,12 +30,28 @@ packages = [ 'requests.packages.urllib3.packages.mimetools_choose_boundary', ] +# certifi is a Python package containing a CA certificate bundle for SSL verification. +# On certain supported platforms (e.g., Red Hat / Debian / FreeBSD), Requests can +# use the system CA bundle instead; see `requests.utils` for details. +# If your platform is supported, set `requires` to [] instead: +requires = ['certifi>=0.0.7'] + +# chardet is used to optimally guess the encodings of pages that don't declare one. +# At this time, chardet is not a required dependency. However, it's sufficiently +# important that pip/setuptools should install it when it's unavailable. if is_py3: - required.append('chardet2') + chardet_package = 'chardet2' else: - required.append('chardet>=1.0.0') - packages.append('requests.packages.oreos') + chardet_package = 'chardet>=1.0.0' + requires.append('oauthlib>=0.1.0,<0.2.0') + +requires.append(chardet_package) +# The async API in requests.async requires the gevent package. +# This is also not a required dependency. +extras_require = { + 'async': ['gevent'], +} setup( name='requests', @@ -50,7 +65,8 @@ setup( packages=packages, package_data={'': ['LICENSE', 'NOTICE']}, include_package_data=True, - install_requires=required, + install_requires=requires, + extras_require=extras_require, license=open("LICENSE").read(), classifiers=( 'Development Status :: 5 - Production/Stable', diff --git a/tests/test_cookies.py b/tests/test_cookies.py new file mode 100755 index 0000000..dca7d2c --- /dev/null +++ b/tests/test_cookies.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import json +import os +import tempfile +import unittest + +# Path hack. +sys.path.insert(0, os.path.abspath('..')) +import requests +from requests.compat import cookielib + +# More hacks +sys.path.append('.') +from test_requests import httpbin, TestBaseMixin + +class CookieTests(TestBaseMixin, unittest.TestCase): + + def test_cookies_from_response(self): + """Basic test that we correctly parse received cookies in the Response object.""" + r = requests.get(httpbin('cookies', 'set', 'myname', 'myvalue')) + + # test deprecated dictionary interface + self.assertEqual(r.cookies['myname'], 'myvalue') + # test CookieJar interface + jar = r.cookies + self.assertEqual(len(jar), 1) + cookie_from_jar = list(jar)[0] + self.assertCookieHas(cookie_from_jar, name='myname', value='myvalue') + + q = requests.get(httpbin('cookies'), cookies=jar) + self.assertEqual(json.loads(q.text)['cookies'], {'myname': 'myvalue'}) + + def test_crossdomain_cookies(self): + """Cookies should not be sent to domains they didn't originate from.""" + r = requests.get("http://github.com") + c = r.cookies + # github should send us cookies + self.assertTrue(len(c) >= 1) + + # github cookies should not be sent to httpbin.org: + r2 = requests.get(httpbin('cookies'), cookies=c) + self.assertEqual(json.loads(r2.text)['cookies'], {}) + + # let's do this again using the session object + s = requests.session() + s.get("http://github.com") + self.assertTrue(len(s.cookies) >= 1) + r = s.get(httpbin('cookies')) + self.assertEqual(json.loads(r.text)['cookies'], {}) + # we can set a cookie and get exactly that same-domain cookie back: + r = s.get(httpbin('cookies', 'set', 'myname', 'myvalue')) + self.assertEqual(json.loads(r.text)['cookies'], {'myname': 'myvalue'}) + + def test_overwrite(self): + """Cookies should get overwritten when appropriate.""" + r = requests.get(httpbin('cookies', 'set', 'shimon', 'yochai')) + cookies = r.cookies + requests.get(httpbin('cookies', 'set', 'elazar', 'shimon'), cookies=cookies) + r = requests.get(httpbin('cookies'), cookies=cookies) + self.assertEqual(json.loads(r.text)['cookies'], + {'shimon': 'yochai', 'elazar': 'shimon'}) + # overwrite the value of 'shimon' + r = requests.get(httpbin('cookies', 'set', 'shimon', 'gamaliel'), cookies=cookies) + self.assertEqual(len(cookies), 2) + r = requests.get(httpbin('cookies'), cookies=cookies) + self.assertEqual(json.loads(r.text)['cookies'], + {'shimon': 'gamaliel', 'elazar': 'shimon'}) + + def test_redirects(self): + """Test that cookies set by a 302 page are correctly processed.""" + r = requests.get(httpbin('cookies', 'set', 'redirects', 'work')) + self.assertEqual(r.history[0].status_code, 302) + expected_cookies = {'redirects': 'work'} + self.assertEqual(json.loads(r.text)['cookies'], expected_cookies) + + r2 = requests.get(httpbin('cookies', 'set', 'very', 'well'), cookies=r.cookies) + expected_cookies = {'redirects': 'work', 'very': 'well'} + self.assertEqual(json.loads(r2.text)['cookies'], expected_cookies) + self.assertTrue(r.cookies is r2.cookies) + + def test_none_cookie(self): + """Regression test: don't send a Cookie header with a string value of 'None'!""" + page = json.loads(requests.get(httpbin('headers')).text) + self.assertTrue('Cookie' not in page['headers']) + + def test_secure_cookies(self): + """Test that secure cookies can only be sent via https.""" + header = "Set-Cookie: ThisIsA=SecureCookie; Path=/; Secure; HttpOnly" + url = 'https://httpbin.org/response-headers?%s' % (requests.utils.quote(header),) + cookies = requests.get(url, verify=False).cookies + self.assertEqual(len(cookies), 1) + self.assertEqual(list(cookies)[0].secure, True) + + secure_resp = requests.get('https://httpbin.org/cookies', cookies=cookies, verify=False) + secure_cookies_sent = json.loads(secure_resp.text)['cookies'] + self.assertEqual(secure_cookies_sent, {'ThisIsA': 'SecureCookie'}) + + insecure_resp = requests.get('http://httpbin.org/cookies', cookies=cookies) + insecure_cookies_sent = json.loads(insecure_resp.text)['cookies'] + self.assertEqual(insecure_cookies_sent, {}) + +class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): + """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's.""" + + COOKIEJAR_CLASS = cookielib.LWPCookieJar + + def setUp(self): + # blank the file + self.cookiejar_file = tempfile.NamedTemporaryFile() + self.cookiejar_filename = self.cookiejar_file.name + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.save() + + def tearDown(self): + try: + self.cookiejar_file.close() + except OSError: + pass + + def test_cookiejar_persistence(self): + """Test that we can save cookies to a FileCookieJar.""" + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.load() + # initially should be blank + self.assertEqual(len(cookiejar), 0) + + response = requests.get(httpbin('cookies', 'set', 'key', 'value'), cookies=cookiejar) + self.assertEqual(len(cookiejar), 1) + cookie = list(cookiejar)[0] + self.assertEqual(json.loads(response.text)['cookies'], {'key': 'value'}) + self.assertCookieHas(cookie, name='key', value='value') + + # save and reload the cookies from the file: + cookiejar.save(ignore_discard=True) + cookiejar_2 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_2.load(ignore_discard=True) + self.assertEqual(len(cookiejar_2), 1) + cookie_2 = list(cookiejar_2)[0] + # this cookie should have been saved with the correct domain restriction: + self.assertCookieHas(cookie_2, name='key', value='value', + domain='httpbin.org', path='/') + + # httpbin sets session cookies, so if we don't ignore the discard attribute, + # there should be no cookie: + cookiejar_3 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_3.load() + self.assertEqual(len(cookiejar_3), 0) + + def test_crossdomain(self): + """Test persistence of the domains associated with the cookies.""" + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar.load() + self.assertEqual(len(cookiejar), 0) + + # github sets a cookie + requests.get("http://github.com", cookies=cookiejar) + num_github_cookies = len(cookiejar) + self.assertTrue(num_github_cookies >= 1) + # httpbin sets another + requests.get(httpbin('cookies', 'set', 'key', 'value'), cookies=cookiejar) + num_total_cookies = len(cookiejar) + self.assertTrue(num_total_cookies >= 2) + self.assertTrue(num_total_cookies > num_github_cookies) + + # save and load + cookiejar.save(ignore_discard=True) + cookiejar_2 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_2.load(ignore_discard=True) + self.assertEqual(len(cookiejar_2), num_total_cookies) + r = requests.get(httpbin('cookies'), cookies=cookiejar_2) + self.assertEqual(json.loads(r.text)['cookies'], {'key': 'value'}) + + def test_persistent_cookies(self): + """Test that we correctly interpret persistent cookies.""" + # httpbin's normal cookie methods don't send persistent cookies, + # so cook up the appropriate header and force it to send + header = "Set-Cookie: Persistent=CookiesAreScary; expires=Sun, 04-May-2032 04:56:50 GMT; path=/" + url = httpbin('response-headers?%s' % (requests.utils.quote(header),)) + cookiejar = self.COOKIEJAR_CLASS(self.cookiejar_filename) + + requests.get(url, cookies=cookiejar) + self.assertEqual(len(cookiejar), 1) + self.assertCookieHas(list(cookiejar)[0], name='Persistent', value='CookiesAreScary') + + requests.get(httpbin('cookies', 'set', 'ThisCookieIs', 'SessionOnly'), cookies=cookiejar) + self.assertEqual(len(cookiejar), 2) + self.assertEqual(len([c for c in cookiejar if c.name == 'Persistent']), 1) + self.assertEqual(len([c for c in cookiejar if c.name == 'ThisCookieIs']), 1) + + # save and load + cookiejar.save() + cookiejar_2 = self.COOKIEJAR_CLASS(self.cookiejar_filename) + cookiejar_2.load() + # we should only load the persistent cookie + self.assertEqual(len(cookiejar_2), 1) + self.assertCookieHas(list(cookiejar_2)[0], name='Persistent', value='CookiesAreScary') + +class MozCookieJarTest(LWPCookieJarTest): + """Same test, but substitute MozillaCookieJar.""" + + COOKIEJAR_CLASS = cookielib.MozillaCookieJar + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_requests.py b/tests/test_requests.py index ac8329e..530008a 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -10,7 +10,6 @@ sys.path.insert(0, os.path.abspath('..')) import json import os -import sys import unittest import pickle @@ -52,8 +51,16 @@ class TestSetup(object): # time.sleep(1) _httpbin = True +class TestBaseMixin(object): + + def assertCookieHas(self, cookie, **kwargs): + """Assert that a cookie has various specified properties.""" + for attr, expected_value in kwargs.items(): + cookie_attr = getattr(cookie, attr) + message = 'Failed comparison for %s: %s != %s' % (attr, cookie_attr, expected_value) + self.assertEqual(cookie_attr, expected_value, message) -class RequestsTestSuite(TestSetup, unittest.TestCase): +class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): """Requests test cases.""" def test_entry_points(self): @@ -322,6 +329,37 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): self.assertEqual(post2.status_code, 200) + def test_POSTBIN_GET_POST_FILES_STRINGS(self): + + for service in SERVICES: + + url = service('post') + + post1 = post(url, files={'fname.txt': 'fdata'}) + self.assertEqual(post1.status_code, 200) + + post2 = post(url, files={'fname.txt': 'fdata', 'fname2.txt':'more fdata'}) + self.assertEqual(post2.status_code, 200) + + post3 = post(url, files={'fname.txt': 'fdata', 'fname2.txt':open(__file__,'rb')}) + self.assertEqual(post3.status_code, 200) + + post4 = post(url, files={'fname.txt': 'fdata'}) + self.assertEqual(post4.status_code, 200) + + post5 = post(url, files={'file': ('file.txt', 'more fdata')}) + self.assertEqual(post5.status_code, 200) + + post6 = post(url, files={'fname.txt': '\xe9'}) + self.assertEqual(post6.status_code, 200) + + post7 = post(url, files={'fname.txt': 'fdata to verify'}) + rbody = json.loads(post7.text) + self.assertTrue(rbody.get('files', None)) + self.assertTrue(rbody['files'].get('fname.txt'), None) + self.assertEqual(rbody['files']['fname.txt'], 'fdata to verify') + + def test_nonzero_evaluation(self): for service in SERVICES: @@ -632,24 +670,24 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): # Those cookies persist transparently. c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Double check. r = get(httpbin('cookies'), cookies={}, session=s) c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Remove a cookie by setting it's value to None. r = get(httpbin('cookies'), cookies={'bessie': None}, session=s) c = json.loads(r.text).get('cookies') del _c['bessie'] - assert c == _c + self.assertEqual(c, _c) # Test session-level cookies. s = requests.session(cookies=_c) r = get(httpbin('cookies'), session=s) c = json.loads(r.text).get('cookies') - assert c == _c + self.assertEqual(c, _c) # Have the server set a cookie. r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s) @@ -698,9 +736,13 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): ds = pickle.loads(pickle.dumps(s)) self.assertEqual(s.headers, ds.headers) - self.assertEqual(s.cookies, ds.cookies) self.assertEqual(s.auth, ds.auth) + # Cookie doesn't have a good __eq__, so verify manually: + self.assertEqual(len(ds.cookies), 1) + for cookie in ds.cookies: + self.assertCookieHas(cookie, name='a-cookie', value='cookie-value') + def test_unpickled_session_requests(self): s = requests.session() r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s) @@ -837,10 +879,29 @@ class RequestsTestSuite(TestSetup, unittest.TestCase): r = requests.get(httpbin('status', '404')) r.text - def test_no_content(self): - r = requests.get(httpbin('status', "0"), config={"safe_mode":True}) - r.content - r.content + def test_max_redirects(self): + """Test the max_redirects config variable, normally and under safe_mode.""" + def unsafe_callable(): + requests.get(httpbin('redirect', '3'), config=dict(max_redirects=2)) + self.assertRaises(requests.exceptions.TooManyRedirects, unsafe_callable) + + # add safe mode + response = requests.get(httpbin('redirect', '3'), config=dict(safe_mode=True, max_redirects=2)) + self.assertTrue(response.content is None) + self.assertTrue(isinstance(response.error, requests.exceptions.TooManyRedirects)) + + def test_connection_keepalive_and_close(self): + """Test that we send 'Connection: close' when keep_alive is disabled.""" + # keep-alive should be on by default + r1 = requests.get(httpbin('get')) + # XXX due to proxying issues, test the header sent back by httpbin, rather than + # the header reported in its message body. See kennethreitz/httpbin#46 + self.assertEqual(r1.headers['Connection'].lower(), 'keep-alive') + + # but when we disable it, we should send a 'Connection: close' + # and get the same back: + r2 = requests.get(httpbin('get'), config=dict(keep_alive=False)) + self.assertEqual(r2.headers['Connection'].lower(), 'close') if __name__ == '__main__': unittest.main() diff --git a/tests/test_requests_async.py b/tests/test_requests_async.py index 1d28261..3a5a762 100755 --- a/tests/test_requests_async.py +++ b/tests/test_requests_async.py @@ -12,7 +12,6 @@ import select has_poll = hasattr(select, "poll") from requests import async -import envoy sys.path.append('.') from test_requests import httpbin, RequestsTestSuite, SERVICES diff --git a/tests/test_requests_ext.py b/tests/test_requests_ext.py index 1e4d89b..883bdce 100644 --- a/tests/test_requests_ext.py +++ b/tests/test_requests_ext.py @@ -104,8 +104,27 @@ class RequestsTestSuite(unittest.TestCase): 'php') assert r.ok - - + def test_cookies_on_redirects(self): + """Test interaction between cookie handling and redirection.""" + # get a cookie for tinyurl.com ONLY + s = requests.session() + s.get(url='http://tinyurl.com/preview.php?disable=1') + # we should have set a cookie for tinyurl: preview=0 + self.assertIn('preview', s.cookies) + self.assertEqual(s.cookies['preview'], '0') + self.assertEqual(list(s.cookies)[0].name, 'preview') + self.assertEqual(list(s.cookies)[0].domain, 'tinyurl.com') + + # get cookies on another domain + r2 = s.get(url='http://httpbin.org/cookies') + # the cookie is not there + self.assertNotIn('preview', json.loads(r2.text)['cookies']) + + # this redirects to another domain, httpbin.org + # cookies of the first domain should NOT be sent to the next one + r3 = s.get(url='http://tinyurl.com/7zp3jnr') + assert r3.url == 'http://httpbin.org/cookies' + self.assertNotIn('preview', json.loads(r2.text)['cookies']) if __name__ == '__main__': unittest.main() -- cgit v1.2.3