diff options
49 files changed, 2248 insertions, 518 deletions
diff --git a/HISTORY.rst b/HISTORY.rst index 0d0d216..09446b3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,180 @@ Release History --------------- +2.7.0 (2015-05-03) +++++++++++++++++++ + +This is the first release that follows our new release process. For more, see +[our documentation](http://docs.python-requests.org/en/latest/community/release-process/). + +**Bugfixes** + +- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + +2.6.2 (2015-04-23) +++++++++++++++++++ + +**Bugfixes** + +- Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + +2.6.1 (2015-04-22) +++++++++++++++++++ + +**Bugfixes** + +- Remove VendorAlias import machinery introduced in v2.5.2. + +- Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + +- Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + +- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + +- Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + +- Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + +2.6.0 (2015-03-14) +++++++++++++++++++ + +**Bugfixes** + +- CVE-2015-2296: Fix handling of cookies on redirect. Previously a cookie + without a host value set would use the hostname for the redirected URL + exposing requests users to session fixation attacks and potentially cookie + stealing. This was disclosed privately by Matthew Daley of + `BugFuzz <https://bugfuzz.com>`_. This affects all versions of requests from + v2.1.0 to v2.5.3 (inclusive on both ends). + +- Fix error when requests is an ``install_requires`` dependency and ``python + setup.py test`` is run. (#2462) + +- Fix error when urllib3 is unbundled and requests continues to use the + vendored import location. + +- Include fixes to ``urllib3``'s header handling. + +- Requests' handling of unvendored dependencies is now more restrictive. + +**Features and Improvements** + +- Support bytearrays when passed as parameters in the ``files`` argument. + (#2468) + +- Avoid data duplication when creating a request with ``str``, ``bytes``, or + ``bytearray`` input to the ``files`` argument. + +2.5.3 (2015-02-24) +++++++++++++++++++ + +**Bugfixes** + +- Revert changes to our vendored certificate bundle. For more context see + (#2455, #2456, and http://bugs.python.org/issue23476) + +2.5.2 (2015-02-23) +++++++++++++++++++ + +**Features and Improvements** + +- Add sha256 fingerprint support. (`shazow/urllib3#540`_) + +- Improve the performance of headers. (`shazow/urllib3#544`_) + +**Bugfixes** + +- Copy pip's import machinery. When downstream redistributors remove + requests.packages.urllib3 the import machinery will continue to let those + same symbols work. Example usage in requests' documentation and 3rd-party + libraries relying on the vendored copies of urllib3 will work without having + to fallback to the system urllib3. + +- Attempt to quote parts of the URL on redirect if unquoting and then quoting + fails. (#2356) + +- Fix filename type check for multipart form-data uploads. (#2411) + +- Properly handle the case where a server issuing digest authentication + challenges provides both auth and auth-int qop-values. (#2408) + +- Fix a socket leak. (`shazow/urllib3#549`_) + +- Fix multiple ``Set-Cookie`` headers properly. (`shazow/urllib3#534`_) + +- Disable the built-in hostname verification. (`shazow/urllib3#526`_) + +- Fix the behaviour of decoding an exhausted stream. (`shazow/urllib3#535`_) + +**Security** + +- Pulled in an updated ``cacert.pem``. + +- Drop RC4 from the default cipher list. (`shazow/urllib3#551`_) + +.. _shazow/urllib3#551: https://github.com/shazow/urllib3/pull/551 +.. _shazow/urllib3#549: https://github.com/shazow/urllib3/pull/549 +.. _shazow/urllib3#544: https://github.com/shazow/urllib3/pull/544 +.. _shazow/urllib3#540: https://github.com/shazow/urllib3/pull/540 +.. _shazow/urllib3#535: https://github.com/shazow/urllib3/pull/535 +.. _shazow/urllib3#534: https://github.com/shazow/urllib3/pull/534 +.. _shazow/urllib3#526: https://github.com/shazow/urllib3/pull/526 + +2.5.1 (2014-12-23) +++++++++++++++++++ + +**Behavioural Changes** + +- Only catch HTTPErrors in raise_for_status (#2382) + +**Bugfixes** + +- Handle LocationParseError from urllib3 (#2344) +- Handle file-like object filenames that are not strings (#2379) +- Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated (#2389) + +2.5.0 (2014-12-01) +++++++++++++++++++ + +**Improvements** + +- Allow usage of urllib3's Retry object with HTTPAdapters (#2216) +- The ``iter_lines`` method on a response now accepts a delimiter with which + to split the content (#2295) + +**Behavioural Changes** + +- Add deprecation warnings to functions in requests.utils that will be removed + in 3.0 (#2309) +- Sessions used by the functional API are always closed (#2326) +- Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) (#2323) + +**Bugfixes** + +- Only parse the URL once (#2353) +- Allow Content-Length header to always be overriden (#2332) +- Properly handle files in HTTPDigestAuth (#2333) +- Cap redirect_cache size to prevent memory abuse (#2299) +- Fix HTTPDigestAuth handling of redirects after authenticating successfully + (#2253) +- Fix crash with custom method parameter to Session.request (#2317) +- Fix how Link headers are parsed using the regular expression library (#2271) + +**Documentation** + +- Add more references for interlinking (#2348) +- Update CSS for theme (#2290) +- Update width of buttons and sidebar (#2289) +- Replace references of Gittip with Gratipay (#2282) +- Add link to changelog in sidebar (#2273) + 2.4.3 (2014-10-06) ++++++++++++++++++ @@ -55,7 +229,7 @@ Release History - Support for connect timeouts! Timeout now accepts a tuple (connect, read) which is used to set individual connect and read timeouts. - Allow copying of PreparedRequests without headers/cookies. - Updated bundled urllib3 version. -- Refactored settings loading from environment — new `Session.merge_environment_settings`. +- Refactored settings loading from environment -- new `Session.merge_environment_settings`. - Handle socket errors in iter_content. @@ -1,4 +1,4 @@ -Copyright 2014 Kenneth Reitz +Copyright 2015 Kenneth Reitz Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: requests -Version: 2.4.3 +Version: 2.7.0 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -9,11 +9,13 @@ License: Apache 2.0 Description: Requests: HTTP for Humans ========================= - .. image:: https://badge.fury.io/py/requests.png - :target: http://badge.fury.io/py/requests + .. image:: https://img.shields.io/pypi/v/requests.svg + :target: https://pypi.python.org/pypi/requests + + .. image:: https://img.shields.io/pypi/dm/requests.svg + :target: https://pypi.python.org/pypi/requests + - .. image:: https://pypip.in/d/requests/badge.png - :target: https://crate.io/packages/requests/ Requests is an Apache2 Licensed HTTP library, written in Python, for human @@ -27,7 +29,7 @@ Description: Requests: HTTP for Humans Things shouldn't be this way. Not in Python. - .. code-block:: pycon + .. code-block:: python >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) >>> r.status_code @@ -98,6 +100,180 @@ Description: Requests: HTTP for Humans Release History --------------- + 2.7.0 (2015-05-03) + ++++++++++++++++++ + + This is the first release that follows our new release process. For more, see + [our documentation](http://docs.python-requests.org/en/latest/community/release-process/). + + **Bugfixes** + + - Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + + 2.6.2 (2015-04-23) + ++++++++++++++++++ + + **Bugfixes** + + - Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + + 2.6.1 (2015-04-22) + ++++++++++++++++++ + + **Bugfixes** + + - Remove VendorAlias import machinery introduced in v2.5.2. + + - Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + + - Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + + - Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + + - Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + + - Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + + 2.6.0 (2015-03-14) + ++++++++++++++++++ + + **Bugfixes** + + - CVE-2015-2296: Fix handling of cookies on redirect. Previously a cookie + without a host value set would use the hostname for the redirected URL + exposing requests users to session fixation attacks and potentially cookie + stealing. This was disclosed privately by Matthew Daley of + `BugFuzz <https://bugfuzz.com>`_. This affects all versions of requests from + v2.1.0 to v2.5.3 (inclusive on both ends). + + - Fix error when requests is an ``install_requires`` dependency and ``python + setup.py test`` is run. (#2462) + + - Fix error when urllib3 is unbundled and requests continues to use the + vendored import location. + + - Include fixes to ``urllib3``'s header handling. + + - Requests' handling of unvendored dependencies is now more restrictive. + + **Features and Improvements** + + - Support bytearrays when passed as parameters in the ``files`` argument. + (#2468) + + - Avoid data duplication when creating a request with ``str``, ``bytes``, or + ``bytearray`` input to the ``files`` argument. + + 2.5.3 (2015-02-24) + ++++++++++++++++++ + + **Bugfixes** + + - Revert changes to our vendored certificate bundle. For more context see + (#2455, #2456, and http://bugs.python.org/issue23476) + + 2.5.2 (2015-02-23) + ++++++++++++++++++ + + **Features and Improvements** + + - Add sha256 fingerprint support. (`shazow/urllib3#540`_) + + - Improve the performance of headers. (`shazow/urllib3#544`_) + + **Bugfixes** + + - Copy pip's import machinery. When downstream redistributors remove + requests.packages.urllib3 the import machinery will continue to let those + same symbols work. Example usage in requests' documentation and 3rd-party + libraries relying on the vendored copies of urllib3 will work without having + to fallback to the system urllib3. + + - Attempt to quote parts of the URL on redirect if unquoting and then quoting + fails. (#2356) + + - Fix filename type check for multipart form-data uploads. (#2411) + + - Properly handle the case where a server issuing digest authentication + challenges provides both auth and auth-int qop-values. (#2408) + + - Fix a socket leak. (`shazow/urllib3#549`_) + + - Fix multiple ``Set-Cookie`` headers properly. (`shazow/urllib3#534`_) + + - Disable the built-in hostname verification. (`shazow/urllib3#526`_) + + - Fix the behaviour of decoding an exhausted stream. (`shazow/urllib3#535`_) + + **Security** + + - Pulled in an updated ``cacert.pem``. + + - Drop RC4 from the default cipher list. (`shazow/urllib3#551`_) + + .. _shazow/urllib3#551: https://github.com/shazow/urllib3/pull/551 + .. _shazow/urllib3#549: https://github.com/shazow/urllib3/pull/549 + .. _shazow/urllib3#544: https://github.com/shazow/urllib3/pull/544 + .. _shazow/urllib3#540: https://github.com/shazow/urllib3/pull/540 + .. _shazow/urllib3#535: https://github.com/shazow/urllib3/pull/535 + .. _shazow/urllib3#534: https://github.com/shazow/urllib3/pull/534 + .. _shazow/urllib3#526: https://github.com/shazow/urllib3/pull/526 + + 2.5.1 (2014-12-23) + ++++++++++++++++++ + + **Behavioural Changes** + + - Only catch HTTPErrors in raise_for_status (#2382) + + **Bugfixes** + + - Handle LocationParseError from urllib3 (#2344) + - Handle file-like object filenames that are not strings (#2379) + - Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated (#2389) + + 2.5.0 (2014-12-01) + ++++++++++++++++++ + + **Improvements** + + - Allow usage of urllib3's Retry object with HTTPAdapters (#2216) + - The ``iter_lines`` method on a response now accepts a delimiter with which + to split the content (#2295) + + **Behavioural Changes** + + - Add deprecation warnings to functions in requests.utils that will be removed + in 3.0 (#2309) + - Sessions used by the functional API are always closed (#2326) + - Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) (#2323) + + **Bugfixes** + + - Only parse the URL once (#2353) + - Allow Content-Length header to always be overriden (#2332) + - Properly handle files in HTTPDigestAuth (#2333) + - Cap redirect_cache size to prevent memory abuse (#2299) + - Fix HTTPDigestAuth handling of redirects after authenticating successfully + (#2253) + - Fix crash with custom method parameter to Session.request (#2317) + - Fix how Link headers are parsed using the regular expression library (#2271) + + **Documentation** + + - Add more references for interlinking (#2348) + - Update CSS for theme (#2290) + - Update width of buttons and sidebar (#2289) + - Replace references of Gittip with Gratipay (#2282) + - Add link to changelog in sidebar (#2273) + 2.4.3 (2014-10-06) ++++++++++++++++++ @@ -150,7 +326,7 @@ Description: Requests: HTTP for Humans - Support for connect timeouts! Timeout now accepts a tuple (connect, read) which is used to set individual connect and read timeouts. - Allow copying of PreparedRequests without headers/cookies. - Updated bundled urllib3 version. - - Refactored settings loading from environment — new `Session.merge_environment_settings`. + - Refactored settings loading from environment -- new `Session.merge_environment_settings`. - Handle socket errors in iter_content. @@ -1,11 +1,13 @@ Requests: HTTP for Humans ========================= -.. image:: https://badge.fury.io/py/requests.png - :target: http://badge.fury.io/py/requests +.. image:: https://img.shields.io/pypi/v/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://img.shields.io/pypi/dm/requests.svg + :target: https://pypi.python.org/pypi/requests + -.. image:: https://pypip.in/d/requests/badge.png - :target: https://crate.io/packages/requests/ Requests is an Apache2 Licensed HTTP library, written in Python, for human @@ -19,7 +21,7 @@ perform the simplest of tasks. Things shouldn't be this way. Not in Python. -.. code-block:: pycon +.. code-block:: python >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) >>> r.status_code diff --git a/debian/changelog b/debian/changelog index 24993e6..f5f80d5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,34 @@ +requests (2.7.0-1) experimental; urgency=medium + + * New upstream release. (Closes: #784095) + - Embedded copy (not used) of urllib3 does not require SSLv3 anymore. + (Closes: #770172) + * debian/control + - Move python-ndg-httpsclient, python-openssl and python-pyasn1 to Suggests + inside python-requests' stanza since Python 2.7.9 include SNI support + and PEP 476 made it as secure as Python 3. + - Bump python{,3}-urllib3 to 1.10.4. + * debian/copyright + - Update copyright years. + - Update to MPL-2.0 license stanza of requests/cacert.pem (not used but + shipped in orig tarball). + * debian/watch + - Use pypi.debian.net redirector. + * debian/patches/01_use-system-ca-certificates.patch + - Refresh and remove CA certificate bundle from MANIFEST.in. + (Closes: #781610) + * debian/patches/02_use-system-chardet-and-urllib3.patch + - Refresh. + * debian/patches/04_make-requests.packages.urllib3-same-as-urllib3.patch + - Refresh. + * debian/patches/05_do-not-ascribe-cookies-to-the-target-domain.patch + - Remove since fixed upstream. + * debian/python{,3}-requests.links + - Remove links thanks to the import machinery in + 04_make-requests.packages.urllib3-same-as-urllib3.patch + + -- Daniele Tricoli <eriol@mornie.org> Mon, 04 May 2015 21:43:40 +0200 + requests (2.4.3-6) unstable; urgency=medium * debian/patches/05_do-not-ascribe-cookies-to-the-target-domain.patch diff --git a/debian/control b/debian/control index 3cae43c..a6b2d11 100644 --- a/debian/control +++ b/debian/control @@ -9,11 +9,11 @@ Build-Depends: python-all (>= 2.6.6-3), python-chardet, python-setuptools, - python-urllib3 (>= 1.9.1), + python-urllib3 (>= 1.10.4), python3-all, python3-chardet, python3-setuptools, - python3-urllib3 (>= 1.7.1), + python3-urllib3 (>= 1.10.4), python3-wheel Standards-Version: 3.9.6 X-Python-Version: >= 2.7 @@ -29,8 +29,8 @@ Depends: ${python:Depends}, ca-certificates, python-chardet, - python-urllib3 (>= 1.9.1) -Recommends: + python-urllib3 (>= 1.10.4) +Suggests: python-ndg-httpsclient, python-openssl, python-pyasn1 @@ -60,7 +60,7 @@ Depends: ${python3:Depends}, ca-certificates, python3-chardet, - python3-urllib3 (>= 1.9.1) + python3-urllib3 (>= 1.10.4) Suggests: python3-ndg-httpsclient, python3-openssl, diff --git a/debian/copyright b/debian/copyright index 511dec5..4629601 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,7 +4,7 @@ Upstream-Contact: Kenneth Reitz <me@kennethreitz.com> Source: http://pypi.python.org/pypi/requests Files: * -Copyright: 2011-2014, Kenneth Reitz +Copyright: 2015, Kenneth Reitz License: Apache Files: requests/packages/urllib3/* @@ -30,10 +30,10 @@ License: LGPL-2.1+ Files: requests/cacert.pem Copyright: 2013, Mozilla -License: LGPL-2.1+ +License: MPL-2.0 Files: debian/* -Copyright: 2011-2014, Daniele Tricoli <eriol@mornie.org> +Copyright: 2011-2015, Daniele Tricoli <eriol@mornie.org> License: Apache License: Apache @@ -125,3 +125,378 @@ License: LGPL-2.1+ License, or (at your option) any later version. . See /usr/share/common-licenses/LGPL-2.1 for the full license text. + +License: MPL-2.0 + Mozilla Public License Version 2.0 + ================================== + . + 1. Definitions + -------------- + . + 1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + . + 1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + . + 1.3. "Contribution" + means Covered Software of a particular Contributor. + . + 1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + . + 1.5. "Incompatible With Secondary Licenses" + means + . + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + . + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + . + 1.6. "Executable Form" + means any form of the work other than Source Code Form. + . + 1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + . + 1.8. "License" + means this document. + . + 1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + . + 1.10. "Modifications" + means any of the following: + . + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + . + (b) any new file in Source Code Form that contains any Covered + Software. + . + 1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + . + 1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + . + 1.13. "Source Code Form" + means the form of the work preferred for making modifications. + . + 1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + . + 2. License Grants and Conditions + -------------------------------- + . + 2.1. Grants + . + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + . + (a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + . + (b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + . + 2.2. Effective Date + . + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + . + 2.3. Limitations on Grant Scope + . + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + . + (a) for any code that a Contributor has removed from Covered Software; + or + . + (b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + . + (c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + . + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + . + 2.4. Subsequent Licenses + . + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + . + 2.5. Representation + . + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights + to grant the rights to its Contributions conveyed by this License. + . + 2.6. Fair Use + . + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + . + 2.7. Conditions + . + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted + in Section 2.1. + . + 3. Responsibilities + ------------------- + . + 3.1. Distribution of Source Form + . + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + . + 3.2. Distribution of Executable Form + . + If You distribute Covered Software in Executable Form then: + . + (a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + . + (b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + . + 3.3. Distribution of a Larger Work + . + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + . + 3.4. Notices + . + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, + or limitations of liability) contained within the Source Code Form of + the Covered Software, except that You may alter any license notices to + the extent required to remedy known factual inaccuracies. + . + 3.5. Application of Additional Terms + . + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + . + 4. Inability to Comply Due to Statute or Regulation + --------------------------------------------------- + . + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Software due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description must + be placed in a text file included with all distributions of the Covered + Software under this License. Except to the extent prohibited by statute + or regulation, such description must be sufficiently detailed for a + recipient of ordinary skill to be able to understand it. + . + 5. Termination + -------------- + . + 5.1. The rights granted under this License will terminate automatically + if You fail to comply with any of its terms. However, if You become + compliant, then the rights granted under this License from a particular + Contributor are reinstated (a) provisionally, unless and until such + Contributor explicitly and finally terminates Your grants, and (b) on an + ongoing basis, if such Contributor fails to notify You of the + non-compliance by some reasonable means prior to 60 days after You have + come back into compliance. Moreover, Your grants from a particular + Contributor are reinstated on an ongoing basis if such Contributor + notifies You of the non-compliance by some reasonable means, this is the + first time You have received notice of non-compliance with this License + from such Contributor, and You become compliant prior to 30 days after + Your receipt of the notice. + . + 5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + . + 5.3. In the event of termination under Sections 5.1 or 5.2 above, all + end user license agreements (excluding distributors and resellers) which + have been validly granted by You or Your distributors under this License + prior to termination shall survive termination. + . + ************************************************************************ + * * + * 6. Disclaimer of Warranty * + * ------------------------- * + * * + * Covered Software is provided under this License on an "as is" * + * basis, without warranty of any kind, either expressed, implied, or * + * statutory, including, without limitation, warranties that the * + * Covered Software is free of defects, merchantable, fit for a * + * particular purpose or non-infringing. The entire risk as to the * + * quality and performance of the Covered Software is with You. * + * Should any Covered Software prove defective in any respect, You * + * (not any Contributor) assume the cost of any necessary servicing, * + * repair, or correction. This disclaimer of warranty constitutes an * + * essential part of this License. No use of any Covered Software is * + * authorized under this License except under this disclaimer. * + * * + ************************************************************************ + . + ************************************************************************ + * * + * 7. Limitation of Liability * + * -------------------------- * + * * + * Under no circumstances and under no legal theory, whether tort * + * (including negligence), contract, or otherwise, shall any * + * Contributor, or anyone who distributes Covered Software as * + * permitted above, be liable to You for any direct, indirect, * + * special, incidental, or consequential damages of any character * + * including, without limitation, damages for lost profits, loss of * + * goodwill, work stoppage, computer failure or malfunction, or any * + * and all other commercial damages or losses, even if such party * + * shall have been informed of the possibility of such damages. This * + * limitation of liability shall not apply to liability for death or * + * personal injury resulting from such party's negligence to the * + * extent applicable law prohibits such limitation. Some * + * jurisdictions do not allow the exclusion or limitation of * + * incidental or consequential damages, so this exclusion and * + * limitation may not apply to You. * + * * + ************************************************************************ + . + 8. Litigation + ------------- + . + Any litigation relating to this License may be brought only in the + courts of a jurisdiction where the defendant maintains its principal + place of business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. + Nothing in this Section shall prevent a party's ability to bring + cross-claims or counter-claims. + . + 9. Miscellaneous + ---------------- + . + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides + that the language of a contract shall be construed against the drafter + shall not be used to construe this License against a Contributor. + . + 10. Versions of the License + --------------------------- + . + 10.1. New Versions + . + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + . + 10.2. Effect of New Versions + . + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + . + 10.3. Modified Versions + . + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + . + 10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses + . + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + . + Exhibit A - Source Code Form License Notice + ------------------------------------------- + . + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + . + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to look + for such a notice. + . + You may add additional accurate notices of copyright ownership. + . + Exhibit B - "Incompatible With Secondary Licenses" Notice + --------------------------------------------------------- + . + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/debian/patches/01_use-system-ca-certificates.patch b/debian/patches/01_use-system-ca-certificates.patch index 258455e..b5cfeec 100644 --- a/debian/patches/01_use-system-ca-certificates.patch +++ b/debian/patches/01_use-system-ca-certificates.patch @@ -19,7 +19,7 @@ Last-Update: 2014-10-08 print(where()) --- a/setup.py +++ b/setup.py -@@ -43,7 +43,7 @@ +@@ -50,7 +50,7 @@ author_email='me@kennethreitz.com', url='http://python-requests.org', packages=packages, @@ -28,3 +28,8 @@ Last-Update: 2014-10-08 package_dir={'requests': 'requests'}, include_package_data=True, install_requires=requires, +--- a/MANIFEST.in ++++ b/MANIFEST.in +@@ -1 +1 @@ +-include README.rst LICENSE NOTICE HISTORY.rst test_requests.py requirements.txt requests/cacert.pem ++include README.rst LICENSE NOTICE HISTORY.rst test_requests.py requirements.txt diff --git a/debian/patches/02_use-system-chardet-and-urllib3.patch b/debian/patches/02_use-system-chardet-and-urllib3.patch index 8836c47..956ffb8 100644 --- a/debian/patches/02_use-system-chardet-and-urllib3.patch +++ b/debian/patches/02_use-system-chardet-and-urllib3.patch @@ -3,22 +3,22 @@ Description: Use the system python-chardet and python-urllib3 instead of the used to supply a stub for ``requests.packages.urllib3``. Author: Daniele Tricoli <eriol@mornie.org> Forwarded: not-needed -Last-Update: 2014-10-21 +Last-Update: 2015-05-04 --- a/requests/adapters.py +++ b/requests/adapters.py -@@ -11,21 +11,21 @@ +@@ -11,22 +11,22 @@ import socket from .models import Response --from .packages.urllib3 import Retry -from .packages.urllib3.poolmanager import PoolManager, proxy_from_url -from .packages.urllib3.response import HTTPResponse -from .packages.urllib3.util import Timeout as TimeoutSauce -+from urllib3 import Retry +-from .packages.urllib3.util.retry import Retry +from urllib3.poolmanager import PoolManager, proxy_from_url +from urllib3.response import HTTPResponse +from urllib3.util import Timeout as TimeoutSauce ++from urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth) @@ -30,6 +30,7 @@ Last-Update: 2014-10-21 -from .packages.urllib3.exceptions import ProtocolError -from .packages.urllib3.exceptions import ReadTimeoutError -from .packages.urllib3.exceptions import SSLError as _SSLError +-from .packages.urllib3.exceptions import ResponseError +from urllib3.exceptions import ConnectTimeoutError +from urllib3.exceptions import HTTPError as _HTTPError +from urllib3.exceptions import MaxRetryError @@ -37,9 +38,10 @@ Last-Update: 2014-10-21 +from urllib3.exceptions import ProtocolError +from urllib3.exceptions import ReadTimeoutError +from urllib3.exceptions import SSLError as _SSLError ++from urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError) + ProxyError, RetryError) --- a/requests/compat.py +++ b/requests/compat.py @@ -4,7 +4,7 @@ @@ -51,7 +53,7 @@ Last-Update: 2014-10-21 import sys -@@ -91,7 +91,7 @@ +@@ -39,7 +39,7 @@ import cookielib from Cookie import Morsel from StringIO import StringIO @@ -65,7 +67,7 @@ Last-Update: 2014-10-21 @@ -16,10 +16,10 @@ from .auth import HTTPBasicAuth - from .cookies import cookiejar_from_dict, get_cookie_header + from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar -from .packages.urllib3.fields import RequestField -from .packages.urllib3.filepost import encode_multipart_formdata -from .packages.urllib3.util import parse_url @@ -74,12 +76,12 @@ Last-Update: 2014-10-21 +from urllib3.filepost import encode_multipart_formdata +from urllib3.util import parse_url +from urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError) + DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) from .exceptions import ( - HTTPError, RequestException, MissingSchema, InvalidURL, + HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, --- a/setup.py +++ b/setup.py -@@ -19,12 +19,6 @@ +@@ -18,12 +18,6 @@ packages = [ 'requests', 'requests.packages', @@ -114,3 +116,14 @@ Last-Update: 2014-10-21 class RequestException(IOError): +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -21,7 +21,7 @@ + from .utils import to_key_val_list, default_headers, to_native_string + from .exceptions import ( + TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) +-from .packages.urllib3._collections import RecentlyUsedContainer ++from urllib3._collections import RecentlyUsedContainer + from .structures import CaseInsensitiveDict + + from .adapters import HTTPAdapter diff --git a/debian/patches/04_make-requests.packages.urllib3-same-as-urllib3.patch b/debian/patches/04_make-requests.packages.urllib3-same-as-urllib3.patch index 4e6f053..0e4db29 100644 --- a/debian/patches/04_make-requests.packages.urllib3-same-as-urllib3.patch +++ b/debian/patches/04_make-requests.packages.urllib3-same-as-urllib3.patch @@ -4,13 +4,13 @@ Author: Jakub Wilk <jwilk@debian.org> Forwarded: not-needed Bug-Debian: https://bugs.debian.org/769047 Bug-Debian: https://bugs.debian.org/769496 -Last-Update: 2014-11-14 +Last-Update: 2015-05-03 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -48,6 +48,28 @@ __license__ = 'Apache 2.0' - __copyright__ = 'Copyright 2014 Kenneth Reitz' + __copyright__ = 'Copyright 2015 Kenneth Reitz' +# On Debian make Python import system know that requests.packages.urllib3 +# and urllib3 are the same thing. diff --git a/debian/patches/05_do-not-ascribe-cookies-to-the-target-domain.patch b/debian/patches/05_do-not-ascribe-cookies-to-the-target-domain.patch deleted file mode 100644 index 3dd3bba..0000000 --- a/debian/patches/05_do-not-ascribe-cookies-to-the-target-domain.patch +++ /dev/null @@ -1,17 +0,0 @@ -Description: Session fixation and cookie stealing. - See http://www.openwall.com/lists/oss-security/2015/03/14/4 for a complete - description. -Origin: https://github.com/kennethreitz/requests/commit/3bd8afbff29e50b38f889b2f688785a669b9aafc -Bug-Debian: https://bugs.debian.org/780506 - ---- a/requests/sessions.py -+++ b/requests/sessions.py -@@ -168,7 +168,7 @@ - except KeyError: - pass - -- extract_cookies_to_jar(prepared_request._cookies, prepared_request, resp.raw) -+ extract_cookies_to_jar(prepared_request._cookies, req, resp.raw) - prepared_request._cookies.update(self.cookies) - prepared_request.prepare_cookies(prepared_request._cookies) - diff --git a/debian/patches/series b/debian/patches/series index bcd27f4..38fffac 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -2,4 +2,3 @@ 02_use-system-chardet-and-urllib3.patch 03_export-IncompleteRead.patch 04_make-requests.packages.urllib3-same-as-urllib3.patch -05_do-not-ascribe-cookies-to-the-target-domain.patch diff --git a/debian/python-requests.links b/debian/python-requests.links deleted file mode 100644 index aa25f83..0000000 --- a/debian/python-requests.links +++ /dev/null @@ -1,2 +0,0 @@ -# requests.packages.urllib3 is used as import location for urllib3 -usr/lib/python2.7/dist-packages/urllib3 usr/lib/python2.7/dist-packages/requests/packages/urllib3 diff --git a/debian/python3-requests.links b/debian/python3-requests.links deleted file mode 100644 index 7bbc4e3..0000000 --- a/debian/python3-requests.links +++ /dev/null @@ -1,2 +0,0 @@ -# requests.packages.urllib3 is used as import location for urllib3 -usr/lib/python3/dist-packages/urllib3 usr/lib/python3/dist-packages/requests/packages/urllib3 diff --git a/debian/watch b/debian/watch index 4c2ef1e..9234b78 100644 --- a/debian/watch +++ b/debian/watch @@ -1,2 +1,3 @@ version=3 -https://pypi.python.org/packages/source/r/requests/requests-(.*)\.tar\.gz +opts=uversionmangle=s/(rc|a|b|c)/~$1/ \ +http://pypi.debian.net/requests/requests-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff --git a/requests.egg-info/PKG-INFO b/requests.egg-info/PKG-INFO index 6220998..0deede0 100644 --- a/requests.egg-info/PKG-INFO +++ b/requests.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: requests -Version: 2.4.3 +Version: 2.7.0 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -9,11 +9,13 @@ License: Apache 2.0 Description: Requests: HTTP for Humans ========================= - .. image:: https://badge.fury.io/py/requests.png - :target: http://badge.fury.io/py/requests + .. image:: https://img.shields.io/pypi/v/requests.svg + :target: https://pypi.python.org/pypi/requests + + .. image:: https://img.shields.io/pypi/dm/requests.svg + :target: https://pypi.python.org/pypi/requests + - .. image:: https://pypip.in/d/requests/badge.png - :target: https://crate.io/packages/requests/ Requests is an Apache2 Licensed HTTP library, written in Python, for human @@ -27,7 +29,7 @@ Description: Requests: HTTP for Humans Things shouldn't be this way. Not in Python. - .. code-block:: pycon + .. code-block:: python >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) >>> r.status_code @@ -98,6 +100,180 @@ Description: Requests: HTTP for Humans Release History --------------- + 2.7.0 (2015-05-03) + ++++++++++++++++++ + + This is the first release that follows our new release process. For more, see + [our documentation](http://docs.python-requests.org/en/latest/community/release-process/). + + **Bugfixes** + + - Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + + 2.6.2 (2015-04-23) + ++++++++++++++++++ + + **Bugfixes** + + - Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + + 2.6.1 (2015-04-22) + ++++++++++++++++++ + + **Bugfixes** + + - Remove VendorAlias import machinery introduced in v2.5.2. + + - Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + + - Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + + - Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + + - Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + + - Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + + 2.6.0 (2015-03-14) + ++++++++++++++++++ + + **Bugfixes** + + - CVE-2015-2296: Fix handling of cookies on redirect. Previously a cookie + without a host value set would use the hostname for the redirected URL + exposing requests users to session fixation attacks and potentially cookie + stealing. This was disclosed privately by Matthew Daley of + `BugFuzz <https://bugfuzz.com>`_. This affects all versions of requests from + v2.1.0 to v2.5.3 (inclusive on both ends). + + - Fix error when requests is an ``install_requires`` dependency and ``python + setup.py test`` is run. (#2462) + + - Fix error when urllib3 is unbundled and requests continues to use the + vendored import location. + + - Include fixes to ``urllib3``'s header handling. + + - Requests' handling of unvendored dependencies is now more restrictive. + + **Features and Improvements** + + - Support bytearrays when passed as parameters in the ``files`` argument. + (#2468) + + - Avoid data duplication when creating a request with ``str``, ``bytes``, or + ``bytearray`` input to the ``files`` argument. + + 2.5.3 (2015-02-24) + ++++++++++++++++++ + + **Bugfixes** + + - Revert changes to our vendored certificate bundle. For more context see + (#2455, #2456, and http://bugs.python.org/issue23476) + + 2.5.2 (2015-02-23) + ++++++++++++++++++ + + **Features and Improvements** + + - Add sha256 fingerprint support. (`shazow/urllib3#540`_) + + - Improve the performance of headers. (`shazow/urllib3#544`_) + + **Bugfixes** + + - Copy pip's import machinery. When downstream redistributors remove + requests.packages.urllib3 the import machinery will continue to let those + same symbols work. Example usage in requests' documentation and 3rd-party + libraries relying on the vendored copies of urllib3 will work without having + to fallback to the system urllib3. + + - Attempt to quote parts of the URL on redirect if unquoting and then quoting + fails. (#2356) + + - Fix filename type check for multipart form-data uploads. (#2411) + + - Properly handle the case where a server issuing digest authentication + challenges provides both auth and auth-int qop-values. (#2408) + + - Fix a socket leak. (`shazow/urllib3#549`_) + + - Fix multiple ``Set-Cookie`` headers properly. (`shazow/urllib3#534`_) + + - Disable the built-in hostname verification. (`shazow/urllib3#526`_) + + - Fix the behaviour of decoding an exhausted stream. (`shazow/urllib3#535`_) + + **Security** + + - Pulled in an updated ``cacert.pem``. + + - Drop RC4 from the default cipher list. (`shazow/urllib3#551`_) + + .. _shazow/urllib3#551: https://github.com/shazow/urllib3/pull/551 + .. _shazow/urllib3#549: https://github.com/shazow/urllib3/pull/549 + .. _shazow/urllib3#544: https://github.com/shazow/urllib3/pull/544 + .. _shazow/urllib3#540: https://github.com/shazow/urllib3/pull/540 + .. _shazow/urllib3#535: https://github.com/shazow/urllib3/pull/535 + .. _shazow/urllib3#534: https://github.com/shazow/urllib3/pull/534 + .. _shazow/urllib3#526: https://github.com/shazow/urllib3/pull/526 + + 2.5.1 (2014-12-23) + ++++++++++++++++++ + + **Behavioural Changes** + + - Only catch HTTPErrors in raise_for_status (#2382) + + **Bugfixes** + + - Handle LocationParseError from urllib3 (#2344) + - Handle file-like object filenames that are not strings (#2379) + - Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated (#2389) + + 2.5.0 (2014-12-01) + ++++++++++++++++++ + + **Improvements** + + - Allow usage of urllib3's Retry object with HTTPAdapters (#2216) + - The ``iter_lines`` method on a response now accepts a delimiter with which + to split the content (#2295) + + **Behavioural Changes** + + - Add deprecation warnings to functions in requests.utils that will be removed + in 3.0 (#2309) + - Sessions used by the functional API are always closed (#2326) + - Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) (#2323) + + **Bugfixes** + + - Only parse the URL once (#2353) + - Allow Content-Length header to always be overriden (#2332) + - Properly handle files in HTTPDigestAuth (#2333) + - Cap redirect_cache size to prevent memory abuse (#2299) + - Fix HTTPDigestAuth handling of redirects after authenticating successfully + (#2253) + - Fix crash with custom method parameter to Session.request (#2317) + - Fix how Link headers are parsed using the regular expression library (#2271) + + **Documentation** + + - Add more references for interlinking (#2348) + - Update CSS for theme (#2290) + - Update width of buttons and sidebar (#2289) + - Replace references of Gittip with Gratipay (#2282) + - Add link to changelog in sidebar (#2273) + 2.4.3 (2014-10-06) ++++++++++++++++++ @@ -150,7 +326,7 @@ Description: Requests: HTTP for Humans - Support for connect timeouts! Timeout now accepts a tuple (connect, read) which is used to set individual connect and read timeouts. - Allow copying of PreparedRequests without headers/cookies. - Updated bundled urllib3 version. - - Refactored settings loading from environment — new `Session.merge_environment_settings`. + - Refactored settings loading from environment -- new `Session.merge_environment_settings`. - Handle socket errors in iter_content. diff --git a/requests.egg-info/requires.txt b/requests.egg-info/requires.txt index 99755a4..073431a 100644 --- a/requests.egg-info/requires.txt +++ b/requests.egg-info/requires.txt @@ -1,5 +1,6 @@ + [security] pyOpenSSL ndg-httpsclient -pyasn1 +pyasn1
\ No newline at end of file diff --git a/requests/__init__.py b/requests/__init__.py index d5e1956..4bca66e 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -36,17 +36,17 @@ usage: The other HTTP methods are supported - see `requests.api`. Full documentation is at <http://python-requests.org>. -:copyright: (c) 2014 by Kenneth Reitz. +:copyright: (c) 2015 by Kenneth Reitz. :license: Apache 2.0, see LICENSE for more details. """ __title__ = 'requests' -__version__ = '2.4.3' -__build__ = 0x020403 +__version__ = '2.7.0' +__build__ = 0x020700 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2014 Kenneth Reitz' +__copyright__ = 'Copyright 2015 Kenneth Reitz' # Attempt to enable urllib3's SNI support, if possible try: diff --git a/requests/adapters.py b/requests/adapters.py index abb25d1..02e0dd1 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -11,10 +11,10 @@ and maintain connections. import socket from .models import Response -from .packages.urllib3 import Retry from .packages.urllib3.poolmanager import PoolManager, proxy_from_url from .packages.urllib3.response import HTTPResponse from .packages.urllib3.util import Timeout as TimeoutSauce +from .packages.urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth) @@ -26,9 +26,10 @@ from .packages.urllib3.exceptions import ProxyError as _ProxyError from .packages.urllib3.exceptions import ProtocolError from .packages.urllib3.exceptions import ReadTimeoutError from .packages.urllib3.exceptions import SSLError as _SSLError +from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError) + ProxyError, RetryError) from .auth import _basic_auth_str DEFAULT_POOLBLOCK = False @@ -60,8 +61,12 @@ class HTTPAdapter(BaseAdapter): :param pool_connections: The number of urllib3 connection pools to cache. :param pool_maxsize: The maximum number of connections to save in the pool. :param int max_retries: The maximum number of retries each connection - should attempt. Note, this applies only to failed connections and - timeouts, never to requests where the server returns a response. + should attempt. Note, this applies only to failed DNS lookups, socket + connections and connection timeouts, never to requests where data has + made it to the server. By default, Requests does not retry failed + connections. If you need granular control over the conditions under + which we retry a request, import urllib3's ``Retry`` class and pass + that instead. :param pool_block: Whether the connection pool should block for connections. Usage:: @@ -77,7 +82,10 @@ class HTTPAdapter(BaseAdapter): def __init__(self, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, pool_block=DEFAULT_POOLBLOCK): - self.max_retries = max_retries + if max_retries == DEFAULT_RETRIES: + self.max_retries = Retry(0, read=False) + else: + self.max_retries = Retry.from_int(max_retries) self.config = {} self.proxy_manager = {} @@ -123,7 +131,7 @@ class HTTPAdapter(BaseAdapter): self._pool_block = block self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, **pool_kwargs) + block=block, strict=True, **pool_kwargs) def proxy_manager_for(self, proxy, **proxy_kwargs): """Return urllib3 ProxyManager for the given proxy. @@ -358,7 +366,7 @@ class HTTPAdapter(BaseAdapter): assert_same_host=False, preload_content=False, decode_content=False, - retries=Retry(self.max_retries, read=False), + retries=self.max_retries, timeout=timeout ) @@ -410,6 +418,9 @@ class HTTPAdapter(BaseAdapter): if isinstance(e.reason, ConnectTimeoutError): raise ConnectTimeout(e, request=request) + if isinstance(e.reason, ResponseError): + raise RetryError(e, request=request) + raise ConnectionError(e, request=request) except _ProxyError as e: diff --git a/requests/api.py b/requests/api.py index 4eaaf9e..d40fa38 100644 --- a/requests/api.py +++ b/requests/api.py @@ -16,7 +16,6 @@ from . import sessions def request(method, url, **kwargs): """Constructs and sends a :class:`Request <Request>`. - Returns :class:`Response <Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. @@ -37,6 +36,8 @@ def request(method, url, **kwargs): :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param stream: (optional) if ``False``, the response content will be immediately downloaded. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. + :return: :class:`Response <Response>` object + :rtype: requests.Response Usage:: @@ -46,25 +47,35 @@ def request(method, url, **kwargs): """ session = sessions.Session() - return session.request(method=method, url=url, **kwargs) + response = session.request(method=method, url=url, **kwargs) + # By explicitly closing the session, we avoid leaving sockets open which + # can trigger a ResourceWarning in some cases, and look like a memory leak + # in others. + session.close() + return response -def get(url, **kwargs): - """Sends a GET request. Returns :class:`Response` object. +def get(url, params=None, **kwargs): + """Sends a GET request. :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) - return request('get', url, **kwargs) + return request('get', url, params=params, **kwargs) def options(url, **kwargs): - """Sends a OPTIONS request. Returns :class:`Response` object. + """Sends a OPTIONS request. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) @@ -72,10 +83,12 @@ def options(url, **kwargs): def head(url, **kwargs): - """Sends a HEAD request. Returns :class:`Response` object. + """Sends a HEAD request. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', False) @@ -83,44 +96,52 @@ def head(url, **kwargs): def post(url, data=None, json=None, **kwargs): - """Sends a POST request. Returns :class:`Response` object. + """Sends a POST request. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ return request('post', url, data=data, json=json, **kwargs) def put(url, data=None, **kwargs): - """Sends a PUT request. Returns :class:`Response` object. + """Sends a PUT request. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ return request('put', url, data=data, **kwargs) def patch(url, data=None, **kwargs): - """Sends a PATCH request. Returns :class:`Response` object. + """Sends a PATCH request. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ return request('patch', url, data=data, **kwargs) def delete(url, **kwargs): - """Sends a DELETE request. Returns :class:`Response` object. + """Sends a DELETE request. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :return: :class:`Response <Response>` object + :rtype: requests.Response """ return request('delete', url, **kwargs) diff --git a/requests/auth.py b/requests/auth.py index 9b6426d..0ff9c29 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -17,6 +17,7 @@ from base64 import b64encode from .compat import urlparse, str from .cookies import extract_cookies_to_jar from .utils import parse_dict_header, to_native_string +from .status_codes import codes CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' CONTENT_TYPE_MULTI_PART = 'multipart/form-data' @@ -66,6 +67,7 @@ class HTTPDigestAuth(AuthBase): self.nonce_count = 0 self.chal = {} self.pos = None + self.num_401_calls = 1 def build_digest_header(self, method, url): @@ -101,7 +103,8 @@ class HTTPDigestAuth(AuthBase): # XXX not implemented yet entdig = None p_parsed = urlparse(url) - path = p_parsed.path + #: path is request-uri defined in RFC 2616 which should not be empty + path = p_parsed.path or "/" if p_parsed.query: path += '?' + p_parsed.query @@ -122,13 +125,15 @@ class HTTPDigestAuth(AuthBase): s += os.urandom(8) cnonce = (hashlib.sha1(s).hexdigest()[:16]) - noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, HA2) if _algorithm == 'MD5-SESS': HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce)) if qop is None: respdig = KD(HA1, "%s:%s" % (nonce, HA2)) elif qop == 'auth' or 'auth' in qop.split(','): + noncebit = "%s:%s:%s:%s:%s" % ( + nonce, ncvalue, cnonce, 'auth', HA2 + ) respdig = KD(HA1, noncebit) else: # XXX handle auth-int. @@ -150,6 +155,11 @@ class HTTPDigestAuth(AuthBase): return 'Digest %s' % (base) + def handle_redirect(self, r, **kwargs): + """Reset num_401_calls counter on redirects.""" + if r.is_redirect: + self.num_401_calls = 1 + def handle_401(self, r, **kwargs): """Takes the given response and tries digest-auth, if needed.""" @@ -162,7 +172,7 @@ class HTTPDigestAuth(AuthBase): if 'digest' in s_auth.lower() and num_401_calls < 2: - setattr(self, 'num_401_calls', num_401_calls + 1) + self.num_401_calls += 1 pat = re.compile(r'digest ', flags=re.IGNORECASE) self.chal = parse_dict_header(pat.sub('', s_auth, count=1)) @@ -182,7 +192,7 @@ class HTTPDigestAuth(AuthBase): return _r - setattr(self, 'num_401_calls', 1) + self.num_401_calls = 1 return r def __call__(self, r): @@ -192,6 +202,11 @@ class HTTPDigestAuth(AuthBase): try: self.pos = r.body.tell() except AttributeError: - pass + # In the case of HTTPDigestAuth being reused and the body of + # the previous request was a file-like object, pos has the + # file position of the previous body. Ensure it's set to + # None. + self.pos = None r.register_hook('response', self.handle_401) + r.register_hook('response', self.handle_redirect) return r diff --git a/requests/compat.py b/requests/compat.py index be5a1ed..70edff7 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -21,62 +21,10 @@ is_py2 = (_ver[0] == 2) #: Python 3.x? is_py3 = (_ver[0] == 3) -#: Python 3.0.x -is_py30 = (is_py3 and _ver[1] == 0) - -#: Python 3.1.x -is_py31 = (is_py3 and _ver[1] == 1) - -#: Python 3.2.x -is_py32 = (is_py3 and _ver[1] == 2) - -#: Python 3.3.x -is_py33 = (is_py3 and _ver[1] == 3) - -#: Python 3.4.x -is_py34 = (is_py3 and _ver[1] == 4) - -#: Python 2.7.x -is_py27 = (is_py2 and _ver[1] == 7) - -#: Python 2.6.x -is_py26 = (is_py2 and _ver[1] == 6) - -#: Python 2.5.x -is_py25 = (is_py2 and _ver[1] == 5) - -#: Python 2.4.x -is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice. - - -# --------- -# Platforms -# --------- - - -# Syntax sugar. -_ver = sys.version.lower() - -is_pypy = ('pypy' in _ver) -is_jython = ('jython' in _ver) -is_ironpython = ('iron' in _ver) - -# Assume CPython, if nothing else. -is_cpython = not any((is_pypy, is_jython, is_ironpython)) - -# Windows-based system. -is_windows = 'win32' in str(sys.platform).lower() - -# Standard Linux 2+ system. -is_linux = ('linux' in str(sys.platform).lower()) -is_osx = ('darwin' in str(sys.platform).lower()) -is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess. -is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. - try: import simplejson as json except (ImportError, SyntaxError): - # simplejson does not support Python 3.2, it thows a SyntaxError + # simplejson does not support Python 3.2, it throws a SyntaxError # because of u'...' Unicode literals. import json @@ -99,7 +47,6 @@ if is_py2: basestring = basestring numeric_types = (int, long, float) - elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag from urllib.request import parse_http_list, getproxies, proxy_bypass diff --git a/requests/cookies.py b/requests/cookies.py index 831c49c..1fbc934 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -6,6 +6,7 @@ Compatibility code to be able to use `cookielib.CookieJar` with requests. requests.utils imports from here, so be careful with imports. """ +import copy import time import collections from .compat import cookielib, urlparse, urlunparse, Morsel @@ -157,26 +158,28 @@ class CookieConflictError(RuntimeError): class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): - """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. + """Compatibility class; is a cookielib.CookieJar, but exposes a dict + interface. This is the CookieJar we create by default for requests and sessions that don't specify one, since some clients may expect response.cookies and session.cookies to support dict operations. - Don't use the dict interface internally; it's just for compatibility with - with external client code. All `requests` code should work out of the box - with externally provided instances of CookieJar, e.g., LWPCookieJar and - FileCookieJar. - - Caution: dictionary operations that are normally O(1) may be O(n). + Requests does not use the dict interface internally; it's just for + compatibility with external client code. All requests code should work + out of the box with externally provided instances of ``CookieJar``, e.g. + ``LWPCookieJar`` and ``FileCookieJar``. Unlike a regular CookieJar, this class is pickleable. - """ + .. warning:: dictionary operations that are normally O(1) may be O(n). + """ def get(self, name, default=None, domain=None, path=None): """Dict-like get() that also supports optional domain and path args in order to resolve naming collisions from using one cookie jar over - multiple domains. Caution: operation is O(n), not O(1).""" + multiple domains. + + .. warning:: operation is O(n), not O(1).""" try: return self._find_no_duplicates(name, domain, path) except KeyError: @@ -199,37 +202,38 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): return c def iterkeys(self): - """Dict-like iterkeys() that returns an iterator of names of cookies from the jar. - See itervalues() and iteritems().""" + """Dict-like iterkeys() that returns an iterator of names of cookies + from the jar. See itervalues() and iteritems().""" for cookie in iter(self): yield cookie.name def keys(self): - """Dict-like keys() that returns a list of names of cookies from the jar. - See values() and items().""" + """Dict-like keys() that returns a list of names of cookies from the + jar. See values() and items().""" return list(self.iterkeys()) def itervalues(self): - """Dict-like itervalues() that returns an iterator of values of cookies from the jar. - See iterkeys() and iteritems().""" + """Dict-like itervalues() that returns an iterator of values of cookies + from the jar. See iterkeys() and iteritems().""" for cookie in iter(self): yield cookie.value def values(self): - """Dict-like values() that returns a list of values of cookies from the jar. - See keys() and items().""" + """Dict-like values() that returns a list of values of cookies from the + jar. See keys() and items().""" return list(self.itervalues()) def iteritems(self): - """Dict-like iteritems() that returns an iterator of name-value tuples from the jar. - See iterkeys() and itervalues().""" + """Dict-like iteritems() that returns an iterator of name-value tuples + from the jar. See iterkeys() and itervalues().""" for cookie in iter(self): yield cookie.name, cookie.value def items(self): - """Dict-like items() that returns a list of name-value tuples from the jar. - See keys() and values(). Allows client-code to call "dict(RequestsCookieJar) - and get a vanilla python dict of key value pairs.""" + """Dict-like items() that returns a list of name-value tuples from the + jar. See keys() and values(). Allows client-code to call + ``dict(RequestsCookieJar)`` and get a vanilla python dict of key value + pairs.""" return list(self.iteritems()) def list_domains(self): @@ -259,8 +263,9 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): return False # there is only one domain in jar def get_dict(self, domain=None, path=None): - """Takes as an argument an optional domain and path and returns a plain old - Python dict of name-value pairs of cookies that meet the requirements.""" + """Takes as an argument an optional domain and path and returns a plain + old Python dict of name-value pairs of cookies that meet the + requirements.""" dictionary = {} for cookie in iter(self): if (domain is None or cookie.domain == domain) and (path is None @@ -269,21 +274,24 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): return dictionary def __getitem__(self, name): - """Dict-like __getitem__() for compatibility with client code. Throws exception - if there are more than one cookie with name. In that case, use the more - explicit get() method instead. Caution: operation is O(n), not O(1).""" + """Dict-like __getitem__() for compatibility with client code. Throws + exception if there are more than one cookie with name. In that case, + use the more explicit get() method instead. + + .. warning:: operation is O(n), not O(1).""" return self._find_no_duplicates(name) def __setitem__(self, name, value): - """Dict-like __setitem__ for compatibility with client code. Throws exception - if there is already a cookie of that name in the jar. In that case, use the more - explicit set() method instead.""" + """Dict-like __setitem__ for compatibility with client code. Throws + exception if there is already a cookie of that name in the jar. In that + case, use the more explicit set() method instead.""" self.set(name, value) def __delitem__(self, name): - """Deletes a cookie given a name. Wraps cookielib.CookieJar's remove_cookie_by_name().""" + """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s + ``remove_cookie_by_name()``.""" remove_cookie_by_name(self, name) def set_cookie(self, cookie, *args, **kwargs): @@ -295,15 +303,16 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """Updates this jar with cookies from another CookieJar or dict-like""" if isinstance(other, cookielib.CookieJar): for cookie in other: - self.set_cookie(cookie) + self.set_cookie(copy.copy(cookie)) else: super(RequestsCookieJar, self).update(other) def _find(self, name, domain=None, path=None): - """Requests uses this method internally to get cookie values. Takes as args name - and optional domain and path. Returns a cookie.value. If there are conflicting cookies, - _find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown - if there are conflicting cookies.""" + """Requests uses this method internally to get cookie values. Takes as + args name and optional domain and path. Returns a cookie.value. If + there are conflicting cookies, _find arbitrarily chooses one. See + _find_no_duplicates if you want an exception thrown if there are + conflicting cookies.""" for cookie in iter(self): if cookie.name == name: if domain is None or cookie.domain == domain: @@ -313,10 +322,11 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) def _find_no_duplicates(self, name, domain=None, path=None): - """__get_item__ and get call _find_no_duplicates -- never used in Requests internally. - Takes as args name and optional domain and path. Returns a cookie.value. - Throws KeyError if cookie is not found and CookieConflictError if there are - multiple cookies that match name and optionally domain and path.""" + """Both ``__get_item__`` and ``get`` call this function: it's never + used elsewhere in Requests. Takes as args name and optional domain and + path. Returns a cookie.value. Throws KeyError if cookie is not found + and CookieConflictError if there are multiple cookies that match name + and optionally domain and path.""" toReturn = None for cookie in iter(self): if cookie.name == name: @@ -350,6 +360,21 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): return new_cj +def _copy_cookie_jar(jar): + if jar is None: + return None + + if hasattr(jar, 'copy'): + # We're dealing with an instane of RequestsCookieJar + return jar.copy() + # We're dealing with a generic CookieJar instance + new_jar = copy.copy(jar) + new_jar.clear() + for cookie in jar: + new_jar.set_cookie(copy.copy(cookie)) + return new_jar + + def create_cookie(name, value, **kwargs): """Make a cookie from underspecified parameters. @@ -440,7 +465,7 @@ def merge_cookies(cookiejar, cookies): """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') - + if isinstance(cookies, dict): cookiejar = cookiejar_from_dict( cookies, cookiejar=cookiejar, overwrite=False) diff --git a/requests/exceptions.py b/requests/exceptions.py index 34c7a0d..89135a8 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -90,5 +90,10 @@ class ChunkedEncodingError(RequestException): class ContentDecodingError(RequestException, BaseHTTPError): """Failed to decode response content""" + class StreamConsumedError(RequestException, TypeError): """The content for this response was already consumed""" + + +class RetryError(RequestException): + """Custom retries logic failed""" diff --git a/requests/models.py b/requests/models.py index 17e5598..45b3ea9 100644 --- a/requests/models.py +++ b/requests/models.py @@ -15,16 +15,15 @@ from .hooks import default_hooks from .structures import CaseInsensitiveDict from .auth import HTTPBasicAuth -from .cookies import cookiejar_from_dict, get_cookie_header +from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .packages.urllib3.fields import RequestField from .packages.urllib3.filepost import encode_multipart_formdata from .packages.urllib3.util import parse_url from .packages.urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError) + DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) from .exceptions import ( - HTTPError, RequestException, MissingSchema, InvalidURL, - ChunkedEncodingError, ContentDecodingError, ConnectionError, - StreamConsumedError) + HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, + ContentDecodingError, ConnectionError, StreamConsumedError) from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, @@ -144,12 +143,13 @@ class RequestEncodingMixin(object): else: fn = guess_filename(v) or k fp = v - if isinstance(fp, str): - fp = StringIO(fp) - if isinstance(fp, bytes): - fp = BytesIO(fp) - rf = RequestField(name=k, data=fp.read(), + if isinstance(fp, (str, bytes, bytearray)): + fdata = fp + else: + fdata = fp.read() + + rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) new_fields.append(rf) @@ -320,7 +320,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): p.method = self.method p.url = self.url p.headers = self.headers.copy() if self.headers is not None else None - p._cookies = self._cookies.copy() if self._cookies is not None else None + p._cookies = _copy_cookie_jar(self._cookies) p.body = self.body p.hooks = self.hooks return p @@ -351,11 +351,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): return # Support for unicode domain names and paths. - scheme, auth, host, port, path, query, fragment = parse_url(url) + try: + scheme, auth, host, port, path, query, fragment = parse_url(url) + except LocationParseError as e: + raise InvalidURL(*e.args) if not scheme: raise MissingSchema("Invalid URL {0!r}: No schema supplied. " - "Perhaps you meant http://{0}?".format(url)) + "Perhaps you meant http://{0}?".format( + to_native_string(url, 'utf8'))) if not host: raise InvalidURL("Invalid URL %r: No host supplied" % url) @@ -472,7 +476,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): l = super_len(body) if l: self.headers['Content-Length'] = builtin_str(l) - elif self.method not in ('GET', 'HEAD'): + elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None): self.headers['Content-Length'] = '0' def prepare_auth(self, auth, url=''): @@ -498,7 +502,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.prepare_content_length(self.body) def prepare_cookies(self, cookies): - """Prepares the given HTTP cookie data.""" + """Prepares the given HTTP cookie data. + + This function eventually generates a ``Cookie`` header from the + given cookies using cookielib. Due to cookielib's design, the header + will not be regenerated if it already exists, meaning this function + can only be called once for the life of the + :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls + to ``prepare_cookies`` will have no actual effect, unless the "Cookie" + header is removed beforehand.""" if isinstance(cookies, cookielib.CookieJar): self._cookies = cookies @@ -511,6 +523,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_hooks(self, hooks): """Prepares the given hooks.""" + # hooks can be passed as None to the prepare method and to this + # method. To prevent iterating over None, simply use an empty list + # if hooks is False-y + hooks = hooks or [] for event in hooks: self.register_hook(event, hooks[event]) @@ -570,7 +586,11 @@ class Response(object): self.cookies = cookiejar_from_dict({}) #: The amount of time elapsed between sending the request - #: and the arrival of the response (as a timedelta) + #: and the arrival of the response (as a timedelta). + #: This property specifically measures the time taken between sending + #: the first byte of the request and finishing parsing the headers. It + #: is therefore unaffected by consuming the response content or the + #: value of the ``stream`` keyword argument. self.elapsed = datetime.timedelta(0) #: The :class:`PreparedRequest <PreparedRequest>` object to which this @@ -615,7 +635,7 @@ class Response(object): def ok(self): try: self.raise_for_status() - except RequestException: + except HTTPError: return False return True @@ -682,10 +702,12 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. + + .. note:: This method is not reentrant safe. """ pending = None @@ -694,7 +716,11 @@ class Response(object): if pending is not None: chunk = pending + chunk - lines = chunk.splitlines() + + if delimiter: + lines = chunk.split(delimiter) + else: + lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() diff --git a/requests/packages/chardet/__init__.py b/requests/packages/chardet/__init__.py index e4f0799..82c2a48 100644 --- a/requests/packages/chardet/__init__.py +++ b/requests/packages/chardet/__init__.py @@ -15,7 +15,7 @@ # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -__version__ = "2.2.1" +__version__ = "2.3.0" from sys import version_info diff --git a/requests/packages/chardet/chardetect.py b/requests/packages/chardet/chardetect.py index ecd0163..ffe892f 100755 --- a/requests/packages/chardet/chardetect.py +++ b/requests/packages/chardet/chardetect.py @@ -12,34 +12,68 @@ Example:: If no paths are provided, it takes its input from stdin. """ + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import sys from io import open -from sys import argv, stdin +from chardet import __version__ from chardet.universaldetector import UniversalDetector -def description_of(file, name='stdin'): - """Return a string describing the probable encoding of a file.""" +def description_of(lines, name='stdin'): + """ + Return a string describing the probable encoding of a file or + list of strings. + + :param lines: The lines to get the encoding of. + :type lines: Iterable of bytes + :param name: Name of file or collection of lines + :type name: str + """ u = UniversalDetector() - for line in file: + for line in lines: u.feed(line) u.close() result = u.result if result['encoding']: - return '%s: %s with confidence %s' % (name, - result['encoding'], - result['confidence']) + return '{0}: {1} with confidence {2}'.format(name, result['encoding'], + result['confidence']) else: - return '%s: no result' % name + return '{0}: no result'.format(name) -def main(): - if len(argv) <= 1: - print(description_of(stdin)) - else: - for path in argv[1:]: - with open(path, 'rb') as f: - print(description_of(f, path)) +def main(argv=None): + ''' + Handles command line arguments and gets things started. + + :param argv: List of arguments, as if specified on the command-line. + If None, ``sys.argv[1:]`` is used instead. + :type argv: list of str + ''' + # Get command line arguments + parser = argparse.ArgumentParser( + description="Takes one or more file paths and reports their detected \ + encodings", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + conflict_handler='resolve') + parser.add_argument('input', + help='File whose encoding we would like to determine.', + type=argparse.FileType('rb'), nargs='*', + default=[sys.stdin]) + parser.add_argument('--version', action='version', + version='%(prog)s {0}'.format(__version__)) + args = parser.parse_args(argv) + + for f in args.input: + if f.isatty(): + print("You are running chardetect interactively. Press " + + "CTRL-D twice at the start of a blank line to signal the " + + "end of your input. If you want help, run chardetect " + + "--help\n", file=sys.stderr) + print(description_of(f, f.name)) if __name__ == '__main__': diff --git a/requests/packages/chardet/jpcntx.py b/requests/packages/chardet/jpcntx.py index f7f69ba..59aeb6a 100644 --- a/requests/packages/chardet/jpcntx.py +++ b/requests/packages/chardet/jpcntx.py @@ -177,6 +177,12 @@ class JapaneseContextAnalysis: return -1, 1 class SJISContextAnalysis(JapaneseContextAnalysis): + def __init__(self): + self.charset_name = "SHIFT_JIS" + + def get_charset_name(self): + return self.charset_name + def get_order(self, aBuf): if not aBuf: return -1, 1 @@ -184,6 +190,8 @@ class SJISContextAnalysis(JapaneseContextAnalysis): first_char = wrap_ord(aBuf[0]) if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)): charLen = 2 + if (first_char == 0x87) or (0xFA <= first_char <= 0xFC): + self.charset_name = "CP932" else: charLen = 1 diff --git a/requests/packages/chardet/latin1prober.py b/requests/packages/chardet/latin1prober.py index ad695f5..eef3573 100644 --- a/requests/packages/chardet/latin1prober.py +++ b/requests/packages/chardet/latin1prober.py @@ -129,11 +129,11 @@ class Latin1Prober(CharSetProber): if total < 0.01: confidence = 0.0 else: - confidence = ((self._mFreqCounter[3] / total) - - (self._mFreqCounter[1] * 20.0 / total)) + confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0) + / total) if confidence < 0.0: confidence = 0.0 # lower the confidence of latin1 so that other more accurate # detector can take priority. - confidence = confidence * 0.5 + confidence = confidence * 0.73 return confidence diff --git a/requests/packages/chardet/mbcssm.py b/requests/packages/chardet/mbcssm.py index 3f93cfb..efe678c 100644 --- a/requests/packages/chardet/mbcssm.py +++ b/requests/packages/chardet/mbcssm.py @@ -353,7 +353,7 @@ SJIS_cls = ( 2,2,2,2,2,2,2,2, # 68 - 6f 2,2,2,2,2,2,2,2, # 70 - 77 2,2,2,2,2,2,2,1, # 78 - 7f - 3,3,3,3,3,3,3,3, # 80 - 87 + 3,3,3,3,3,2,2,3, # 80 - 87 3,3,3,3,3,3,3,3, # 88 - 8f 3,3,3,3,3,3,3,3, # 90 - 97 3,3,3,3,3,3,3,3, # 98 - 9f @@ -369,9 +369,8 @@ SJIS_cls = ( 2,2,2,2,2,2,2,2, # d8 - df 3,3,3,3,3,3,3,3, # e0 - e7 3,3,3,3,3,4,4,4, # e8 - ef - 4,4,4,4,4,4,4,4, # f0 - f7 - 4,4,4,4,4,0,0,0 # f8 - ff -) + 3,3,3,3,3,3,3,3, # f0 - f7 + 3,3,3,3,3,0,0,0) # f8 - ff SJIS_st = ( @@ -571,5 +570,3 @@ UTF8SMModel = {'classTable': UTF8_cls, 'stateTable': UTF8_st, 'charLenTable': UTF8CharLenTable, 'name': 'UTF-8'} - -# flake8: noqa diff --git a/requests/packages/chardet/sjisprober.py b/requests/packages/chardet/sjisprober.py index b173614..cd0e9e7 100644 --- a/requests/packages/chardet/sjisprober.py +++ b/requests/packages/chardet/sjisprober.py @@ -47,7 +47,7 @@ class SJISProber(MultiByteCharSetProber): self._mContextAnalyzer.reset() def get_charset_name(self): - return "SHIFT_JIS" + return self._mContextAnalyzer.get_charset_name() def feed(self, aBuf): aLen = len(aBuf) diff --git a/requests/packages/chardet/universaldetector.py b/requests/packages/chardet/universaldetector.py index 9a03ad3..476522b 100644 --- a/requests/packages/chardet/universaldetector.py +++ b/requests/packages/chardet/universaldetector.py @@ -71,9 +71,9 @@ class UniversalDetector: if not self._mGotData: # If the data starts with BOM, we know it is UTF - if aBuf[:3] == codecs.BOM: + if aBuf[:3] == codecs.BOM_UTF8: # EF BB BF UTF-8 with BOM - self.result = {'encoding': "UTF-8", 'confidence': 1.0} + self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0} elif aBuf[:4] == codecs.BOM_UTF32_LE: # FF FE 00 00 UTF-32, little-endian BOM self.result = {'encoding': "UTF-32LE", 'confidence': 1.0} diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 4b36b5a..f48ac4a 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = 'dev' +__version__ = '1.10.4' from .connectionpool import ( @@ -55,9 +55,12 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -# Set security warning to only go off once by default. import warnings -warnings.simplefilter('module', exceptions.SecurityWarning) +# SecurityWarning's always go off by default. +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +# InsecurePlatformWarning's don't vary between requests, so we keep it default. +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index d77ebb8..279416c 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -1,7 +1,7 @@ from collections import Mapping, MutableMapping try: from threading import RLock -except ImportError: # Platform-specific: No threads available +except ImportError: # Platform-specific: No threads available class RLock: def __enter__(self): pass @@ -10,11 +10,11 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ +try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict -from .packages.six import itervalues +from .packages.six import iterkeys, itervalues, PY3 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] @@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping): def clear(self): with self.lock: # Copy pointers to all values, then wipe the mapping - # under Python 2, this copies the list of values twice :-| - values = list(self._container.values()) + values = list(itervalues(self._container)) self._container.clear() if self.dispose_func: @@ -95,10 +94,17 @@ class RecentlyUsedContainer(MutableMapping): def keys(self): with self.lock: - return self._container.keys() + return list(iterkeys(self._container)) -class HTTPHeaderDict(MutableMapping): +_dict_setitem = dict.__setitem__ +_dict_getitem = dict.__getitem__ +_dict_delitem = dict.__delitem__ +_dict_contains = dict.__contains__ +_dict_setdefault = dict.setdefault + + +class HTTPHeaderDict(dict): """ :param headers: An iterable of field-value pairs. Must not contain multiple field names @@ -130,25 +136,75 @@ class HTTPHeaderDict(MutableMapping): 'foo=bar, baz=quxx' >>> headers['Content-Length'] '7' - - If you want to access the raw headers with their original casing - for debugging purposes you can access the private ``._data`` attribute - which is a normal python ``dict`` that maps the case-insensitive key to a - list of tuples stored as (case-sensitive-original-name, value). Using the - structure from above as our example: - - >>> headers._data - {'set-cookie': [('Set-Cookie', 'foo=bar'), ('set-cookie', 'baz=quxx')], - 'content-length': [('content-length', '7')]} """ def __init__(self, headers=None, **kwargs): - self._data = {} - if headers is None: - headers = {} - self.update(headers, **kwargs) + dict.__init__(self) + if headers is not None: + if isinstance(headers, HTTPHeaderDict): + self._copy_from(headers) + else: + self.extend(headers) + if kwargs: + self.extend(kwargs) + + def __setitem__(self, key, val): + return _dict_setitem(self, key.lower(), (key, val)) + + def __getitem__(self, key): + val = _dict_getitem(self, key.lower()) + return ', '.join(val[1:]) + + def __delitem__(self, key): + return _dict_delitem(self, key.lower()) - def add(self, key, value): + def __contains__(self, key): + return _dict_contains(self, key.lower()) + + def __eq__(self, other): + if not isinstance(other, Mapping) and not hasattr(other, 'keys'): + return False + if not isinstance(other, type(self)): + other = type(self)(other) + return dict((k1, self[k1]) for k1 in self) == dict((k2, other[k2]) for k2 in other) + + def __ne__(self, other): + return not self.__eq__(other) + + values = MutableMapping.values + get = MutableMapping.get + update = MutableMapping.update + + if not PY3: # Python 2 + iterkeys = MutableMapping.iterkeys + itervalues = MutableMapping.itervalues + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + # Using the MutableMapping function directly fails due to the private marker. + # Using ordinary dict.pop would expose the internal structures. + # So let's reinvent the wheel. + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def discard(self, key): + try: + del self[key] + except KeyError: + pass + + def add(self, key, val): """Adds a (name, value) pair, doesn't overwrite the value if it already exists. @@ -157,43 +213,111 @@ class HTTPHeaderDict(MutableMapping): >>> headers['foo'] 'bar, baz' """ - self._data.setdefault(key.lower(), []).append((key, value)) + key_lower = key.lower() + new_vals = key, val + # Keep the common case aka no item present as fast as possible + vals = _dict_setdefault(self, key_lower, new_vals) + if new_vals is not vals: + # new_vals was not inserted, as there was a previous one + if isinstance(vals, list): + # If already several items got inserted, we have a list + vals.append(val) + else: + # vals should be a tuple then, i.e. only one item so far + # Need to convert the tuple to list for further extension + _dict_setitem(self, key_lower, [vals[0], vals[1], val]) + + def extend(self, *args, **kwargs): + """Generic import function for any type of header-like object. + Adapted version of MutableMapping.update in order to insert items + with self.add instead of self.__setitem__ + """ + if len(args) > 1: + raise TypeError("extend() takes at most 1 positional " + "arguments ({} given)".format(len(args))) + other = args[0] if len(args) >= 1 else () + + if isinstance(other, HTTPHeaderDict): + for key, val in other.iteritems(): + self.add(key, val) + elif isinstance(other, Mapping): + for key in other: + self.add(key, other[key]) + elif hasattr(other, "keys"): + for key in other.keys(): + self.add(key, other[key]) + else: + for key, value in other: + self.add(key, value) + + for key, value in kwargs.items(): + self.add(key, value) def getlist(self, key): """Returns a list of all the values for the named field. Returns an empty list if the key doesn't exist.""" - return self[key].split(', ') if key in self else [] - - def copy(self): - h = HTTPHeaderDict() - for key in self._data: - for rawkey, value in self._data[key]: - h.add(rawkey, value) - return h - - def __eq__(self, other): - if not isinstance(other, Mapping): - return False - other = HTTPHeaderDict(other) - return dict((k1, self[k1]) for k1 in self._data) == \ - dict((k2, other[k2]) for k2 in other._data) - - def __getitem__(self, key): - values = self._data[key.lower()] - return ', '.join(value[1] for value in values) - - def __setitem__(self, key, value): - self._data[key.lower()] = [(key, value)] + try: + vals = _dict_getitem(self, key.lower()) + except KeyError: + return [] + else: + if isinstance(vals, tuple): + return [vals[1]] + else: + return vals[1:] + + # Backwards compatibility for httplib + getheaders = getlist + getallmatchingheaders = getlist + iget = getlist - def __delitem__(self, key): - del self._data[key.lower()] + def __repr__(self): + return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) - def __len__(self): - return len(self._data) + def _copy_from(self, other): + for key in other: + val = _dict_getitem(other, key) + if isinstance(val, list): + # Don't need to convert tuples + val = list(val) + _dict_setitem(self, key, val) - def __iter__(self): - for headers in itervalues(self._data): - yield headers[0][0] - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, dict(self.items())) + def copy(self): + clone = type(self)() + clone._copy_from(self) + return clone + + def iteritems(self): + """Iterate over all header lines, including duplicate ones.""" + for key in self: + vals = _dict_getitem(self, key) + for val in vals[1:]: + yield vals[0], val + + def itermerged(self): + """Iterate over all headers, merging duplicate ones together.""" + for key in self: + val = _dict_getitem(self, key) + yield val[0], ', '.join(val[1:]) + + def items(self): + return list(self.iteritems()) + + @classmethod + def from_httplib(cls, message): # Python 2 + """Read headers from a Python 2 httplib message object.""" + # python2.7 does not expose a proper API for exporting multiheaders + # efficiently. This function re-reads raw lines from the message + # object and extracts the multiheaders properly. + headers = [] + + for line in message.headers: + if line.startswith((' ', '\t')): + key, value = headers[-1] + headers[-1] = (key, value + '\r\n' + line.rstrip()) + continue + + key, value = line.split(':', 1) + headers.append((key, value.strip())) + + return cls(headers) diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index c6e1959..2a8c359 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -3,6 +3,7 @@ import sys import socket from socket import timeout as SocketTimeout import warnings +from .packages import six try: # Python 3 from http.client import HTTPConnection as _HTTPConnection, HTTPException @@ -26,12 +27,20 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass +try: # Python 3: + # Not a no-op, we're adding this to the namespace so it can be imported. + ConnectionError = ConnectionError +except NameError: # Python 2: + class ConnectionError(Exception): + pass + + from .exceptions import ( ConnectTimeoutError, SystemTimeWarning, + SecurityWarning, ) from .packages.ssl_match_hostname import match_hostname -from .packages import six from .util.ssl_ import ( resolve_cert_reqs, @@ -40,8 +49,8 @@ from .util.ssl_ import ( assert_fingerprint, ) -from .util import connection +from .util import connection port_by_scheme = { 'http': 80, @@ -233,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.assert_fingerprint) elif resolved_cert_reqs != ssl.CERT_NONE \ and self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or hostname) + cert = self.sock.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn(( + 'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. ' + 'This feature is being removed by major browsers and deprecated by RFC 2818. ' + '(See https://github.com/shazow/urllib3/issues/497 for details.)'), + SecurityWarning + ) + match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) @@ -244,3 +260,5 @@ if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection HTTPSConnection = VerifiedHTTPSConnection +else: + HTTPSConnection = DummyConnection diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 9cc2a95..117269a 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -32,7 +32,7 @@ from .connection import ( port_by_scheme, DummyConnection, HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPException, BaseSSLError, ConnectionError ) from .request import RequestMethods from .response import HTTPResponse @@ -72,6 +72,21 @@ class ConnectionPool(object): return '%s(host=%r, port=%r)' % (type(self).__name__, self.host, self.port) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + # Return False to re-raise any potential exceptions + return False + + def close(): + """ + Close all pooled connections and disable the pool. + """ + pass + + # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 _blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) @@ -266,6 +281,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ pass + def _prepare_proxy(self, conn): + # Nothing to do for HTTP connections. + pass + def _get_timeout(self, timeout): """ Helper that always returns a :class:`urllib3.util.Timeout` """ if timeout is _Default: @@ -278,6 +297,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # can be removed later return Timeout.from_float(timeout) + def _raise_timeout(self, err, url, timeout_value): + """Is the error actually a timeout? Will raise a ReadTimeout or pass""" + + if isinstance(err, SocketTimeout): + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # See the above comment about EAGAIN in Python 3. In Python 2 we have + # to specifically catch it and throw the timeout error + if hasattr(err, 'errno') and err.errno in _blocking_errnos: + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # Catch possible read timeouts thrown as SSL errors. If not the + # case, rethrow the original. We need to do this because of: + # http://bugs.python.org/issue10272 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + def _make_request(self, conn, method, url, timeout=_Default, **httplib_request_kw): """ @@ -301,7 +337,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout # Trigger any extra validation we need to do. - self._validate_conn(conn) + try: + self._validate_conn(conn) + except (SocketTimeout, BaseSSLError) as e: + # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. @@ -327,32 +368,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Receive the response from the server try: - try: # Python 2.7+, use buffering of HTTP responses + try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older httplib_response = conn.getresponse() - except SocketTimeout: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - except BaseSSLError as e: - # Catch possible read timeouts thrown as SSL errors. If not the - # case, rethrow the original. We need to do this because of: - # http://bugs.python.org/issue10272 - if 'timed out' in str(e) or \ - 'did not complete (read)' in str(e): # Python 2.6 - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - raise - - except SocketError as e: # Platform-specific: Python 2 - # See the above comment about EAGAIN in Python 3. In Python 2 we - # have to specifically catch it and throw the timeout error - if e.errno in _blocking_errnos: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - + except (SocketTimeout, BaseSSLError, SocketError) as e: + self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. @@ -508,11 +529,18 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: # Request a connection from the queue. + timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) + conn.timeout = timeout_obj.connect_timeout + + is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) + if is_new_proxy_conn: + self._prepare_proxy(conn) + # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, - timeout=timeout, + timeout=timeout_obj, body=body, headers=headers) # If we're going to release the connection in ``finally:``, then @@ -537,26 +565,36 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise EmptyPoolError(self, "No pool connections are available.") except (BaseSSLError, CertificateError) as e: - # Release connection unconditionally because there is no way to - # close it externally in case of exception. - release_conn = True + # Close the connection. If a connection is reused on which there + # was a Certificate error, the next request will certainly raise + # another Certificate error. + if conn: + conn.close() + conn = None raise SSLError(e) - except (TimeoutError, HTTPException, SocketError) as e: + except SSLError: + # Treat SSLError separately from BaseSSLError to preserve + # traceback. + if conn: + conn.close() + conn = None + raise + + except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: if conn: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. conn.close() conn = None - stacktrace = sys.exc_info()[2] if isinstance(e, SocketError) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) elif isinstance(e, (SocketError, HTTPException)): e = ProtocolError('Connection aborted.', e) - retries = retries.increment(method, url, error=e, - _pool=self, _stacktrace=stacktrace) + retries = retries.increment(method, url, error=e, _pool=self, + _stacktrace=sys.exc_info()[2]) retries.sleep() # Keep track of the error for the retry warning. @@ -668,23 +706,25 @@ class HTTPSConnectionPool(HTTPConnectionPool): assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version - if self.proxy is not None: - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel + return conn - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self.host, self.port) - else: - set_tunnel(self.host, self.port, self.proxy_headers) + def _prepare_proxy(self, conn): + """ + Establish tunnel connection early, because otherwise httplib + would improperly set Host: header to proxy's IP:port. + """ + # Python 2.7+ + try: + set_tunnel = conn.set_tunnel + except AttributeError: # Platform-specific: Python 2.6 + set_tunnel = conn._set_tunnel - # Establish tunnel connection early, because otherwise httplib - # would improperly set Host: header to proxy's IP:port. - conn.connect() + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + set_tunnel(self.host, self.port) + else: + set_tunnel(self.host, self.port, self.proxy_headers) - return conn + conn.connect() def _new_conn(self): """ @@ -695,7 +735,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): % (self.num_connections, self.host)) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - # Platform-specific: Python without ssl raise SSLError("Can't connect to HTTPS URL because the SSL " "module is not available.") @@ -725,8 +764,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.org/en/latest/security.html ' - '(This warning will only appear once by default.)'), + 'https://urllib3.readthedocs.org/en/latest/security.html'), InsecureRequestWarning) diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index 24de9e4..b2c34a8 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI when the required modules are installed. Activating this module also has the positive side effect of disabling SSL/TLS -encryption in Python 2 (see `CRIME attack`_). +compression in Python 2 (see `CRIME attack`_). If you want to configure the default list of supported cipher suites, you can set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. @@ -38,8 +38,6 @@ Module Variables ---------------- :var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites. - Default: ``ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES: - ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS`` .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) @@ -70,9 +68,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, - ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } + +try: + _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) +except AttributeError: + pass + _openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, @@ -80,22 +83,7 @@ _openssl_verify = { + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -# A secure default. -# Sources for more information on TLS ciphers: -# -# - https://wiki.mozilla.org/Security/Server_Side_TLS -# - https://www.ssllabs.com/projects/best-practices/index.html -# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ -# -# The general intent is: -# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), -# - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM over any AES-CBC for better performance and security, -# - use 3DES as fallback which is secure but slow, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_SSL_CIPHER_LIST = "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:" + \ - "ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:" + \ - "!aNULL:!MD5:!DSS" +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS orig_util_HAS_SNI = util.HAS_SNI @@ -186,6 +174,11 @@ class WrappedSocket(object): return b'' else: raise + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return b'' + else: + raise except OpenSSL.SSL.WantReadError: rd, wd, ed = select.select( [self.socket], [], [], self.socket.gettimeout()) @@ -199,8 +192,21 @@ class WrappedSocket(object): def settimeout(self, timeout): return self.socket.settimeout(timeout) + def _send_until_done(self, data): + while True: + try: + return self.connection.send(data) + except OpenSSL.SSL.WantWriteError: + _, wlist, _ = select.select([], [self.socket], [], + self.socket.gettimeout()) + if not wlist: + raise timeout() + continue + def sendall(self, data): - return self.connection.sendall(data) + while len(data): + sent = self._send_until_done(data) + data = data[sent:] def close(self): if self._makefile_refs < 1: @@ -248,6 +254,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ssl_version=None): ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: + keyfile = keyfile or certfile # Match behaviour of the normal python ssl library ctx.use_certificate_file(certfile) if keyfile: ctx.use_privatekey_file(keyfile) @@ -275,7 +282,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - select.select([sock], [], []) + rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + if not rd: + raise timeout('select timed out') continue except OpenSSL.SSL.Error as e: raise ssl.SSLError('bad handshake', e) diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 7519ba9..31bda1c 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -72,11 +72,8 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s" % url - if reason: - message += " (Caused by %r)" % reason - else: - message += " (Caused by redirect)" + message = "Max retries exceeded with url: %s (Caused by %r)" % ( + url, reason) RequestError.__init__(self, pool, url, message) @@ -141,6 +138,12 @@ class LocationParseError(LocationValueError): self.location = location +class ResponseError(HTTPError): + "Used as a container for an error reason supplied in a MaxRetryError." + GENERIC_ERROR = 'too many error responses' + SPECIFIC_ERROR = 'too many {status_code} error responses' + + class SecurityWarning(HTTPWarning): "Warned when perfoming security reducing actions" pass @@ -154,3 +157,13 @@ class InsecureRequestWarning(SecurityWarning): class SystemTimeWarning(SecurityWarning): "Warned when system time is suspected to be wrong" pass + + +class InsecurePlatformWarning(SecurityWarning): + "Warned when certain SSL configuration is not available on a platform." + pass + + +class ResponseNotChunked(ProtocolError, ValueError): + "Response needs to be chunked in order to read it as chunks." + pass diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 515dc96..b8d1e74 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -8,7 +8,7 @@ except ImportError: from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme -from .exceptions import LocationValueError +from .exceptions import LocationValueError, MaxRetryError from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry @@ -64,6 +64,14 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clear() + # Return False to re-raise any potential exceptions + return False + def _new_pool(self, scheme, host, port): """ Create a new :class:`ConnectionPool` based on host, port and scheme. @@ -167,7 +175,14 @@ class PoolManager(RequestMethods): if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) - kw['retries'] = retries.increment(method, redirect_location) + try: + retries = retries.increment(method, url, response=response, _pool=conn) + except MaxRetryError: + if retries.raise_on_redirect: + raise + return response + + kw['retries'] = retries kw['redirect'] = redirect log.info("Redirecting %s -> %s" % (url, redirect_location)) diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py index 51fe238..b08d6c9 100644 --- a/requests/packages/urllib3/request.py +++ b/requests/packages/urllib3/request.py @@ -118,18 +118,24 @@ class RequestMethods(object): which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. """ - if encode_multipart: - body, content_type = encode_multipart_formdata( - fields or {}, boundary=multipart_boundary) - else: - body, content_type = (urlencode(fields or {}), - 'application/x-www-form-urlencoded') - if headers is None: headers = self.headers - headers_ = {'Content-Type': content_type} - headers_.update(headers) + extra_kw = {'headers': {}} + + if fields: + if 'body' in urlopen_kw: + raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') + + if encode_multipart: + body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + else: + body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + + extra_kw['body'] = body + extra_kw['headers'] = {'Content-Type': content_type} + + extra_kw['headers'].update(headers) + extra_kw.update(urlopen_kw) - return self.urlopen(method, url, body=body, headers=headers_, - **urlopen_kw) + return self.urlopen(method, url, **extra_kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index e69de95..24140c4 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,15 +1,20 @@ +try: + import http.client as httplib +except ImportError: + import httplib import zlib import io from socket import timeout as SocketTimeout from ._collections import HTTPHeaderDict -from .exceptions import ProtocolError, DecodeError, ReadTimeoutError -from .packages.six import string_types as basestring, binary_type +from .exceptions import ( + ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked +) +from .packages.six import string_types as basestring, binary_type, PY3 from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed - class DeflateDecoder(object): def __init__(self): @@ -21,6 +26,9 @@ class DeflateDecoder(object): return getattr(self._obj, name) def decompress(self, data): + if not data: + return data + if not self._first_try: return self._obj.decompress(data) @@ -36,9 +44,23 @@ class DeflateDecoder(object): self._data = None +class GzipDecoder(object): + + def __init__(self): + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + + def __getattr__(self, name): + return getattr(self._obj, name) + + def decompress(self, data): + if not data: + return data + return self._obj.decompress(data) + + def _get_decoder(mode): if mode == 'gzip': - return zlib.decompressobj(16 + zlib.MAX_WBITS) + return GzipDecoder() return DeflateDecoder() @@ -76,9 +98,10 @@ class HTTPResponse(io.IOBase): strict=0, preload_content=True, decode_content=True, original_response=None, pool=None, connection=None): - self.headers = HTTPHeaderDict() - if headers: - self.headers.update(headers) + if isinstance(headers, HTTPHeaderDict): + self.headers = headers + else: + self.headers = HTTPHeaderDict(headers) self.status = status self.version = version self.reason = reason @@ -100,7 +123,17 @@ class HTTPResponse(io.IOBase): if hasattr(body, 'read'): self._fp = body - if preload_content and not self._body: + # Are we using the chunked-style of transfer encoding? + self.chunked = False + self.chunk_left = None + tr_enc = self.headers.get('transfer-encoding', '').lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: + self.chunked = True + + # We certainly don't want to preload content when the response is chunked. + if not self.chunked and preload_content and not self._body: self._body = self.read(decode_content=decode_content) def get_redirect_location(self): @@ -140,6 +173,35 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessar. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, e) + + if flush_decoder and decode_content and self._decoder: + buf = self._decoder.decompress(binary_type()) + data += buf + self._decoder.flush() + + return data + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional @@ -161,12 +223,7 @@ class HTTPResponse(io.IOBase): after having ``.read()`` the file object. (Overridden if ``amt`` is set.) """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + self._init_decoder() if decode_content is None: decode_content = self.decode_content @@ -202,7 +259,7 @@ class HTTPResponse(io.IOBase): except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? - if not 'read operation timed out' in str(e): # Defensive: + if 'read operation timed out' not in str(e): # Defensive: # This shouldn't happen but just in case we're missing an edge # case, let's avoid swallowing SSL errors. raise @@ -215,17 +272,7 @@ class HTTPResponse(io.IOBase): self._fp_bytes_read += len(data) - try: - if decode_content and self._decoder: - data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data @@ -252,11 +299,15 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - while not is_fp_closed(self._fp): - data = self.read(amt=amt, decode_content=decode_content) + if self.chunked: + for line in self.read_chunked(amt, decode_content=decode_content): + yield line + else: + while not is_fp_closed(self._fp): + data = self.read(amt=amt, decode_content=decode_content) - if data: - yield data + if data: + yield data @classmethod def from_httplib(ResponseCls, r, **response_kw): @@ -267,14 +318,16 @@ class HTTPResponse(io.IOBase): Remaining parameters are passed to the HTTPResponse constructor, along with ``original_response=r``. """ - - headers = HTTPHeaderDict() - for k, v in r.getheaders(): - headers.add(k, v) + headers = r.msg + if not isinstance(headers, HTTPHeaderDict): + if PY3: # Python 3 + headers = HTTPHeaderDict(headers.items()) + else: # Python 2 + headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, 'strict', 0) - return ResponseCls(body=r, + resp = ResponseCls(body=r, headers=headers, status=r.status, version=r.version, @@ -282,6 +335,7 @@ class HTTPResponse(io.IOBase): strict=strict, original_response=r, **response_kw) + return resp # Backwards-compatibility methods for httplib.HTTPResponse def getheaders(self): @@ -331,3 +385,82 @@ class HTTPResponse(io.IOBase): else: b[:len(temp)] = temp return len(temp) + + def _update_chunk_length(self): + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is not None: + return + line = self._fp.fp.readline() + line = line.split(b';', 1)[0] + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(line) + + def _handle_chunk(self, amt): + returned_chunk = None + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + returned_chunk = chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + returned_chunk = value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + returned_chunk = value + else: # amt > self.chunk_left + returned_chunk = self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + return returned_chunk + + def read_chunked(self, amt=None, decode_content=None): + """ + Similar to :meth:`HTTPResponse.read`, but with an additional + parameter: ``decode_content``. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + self._init_decoder() + # FIXME: Rewrite this method and make it a class with a better structured logic. + if not self.chunked: + raise ResponseNotChunked("Response is not chunked. " + "Header 'transfer-encoding: chunked' is missing.") + + if self._original_response and self._original_response._method.upper() == 'HEAD': + # Don't bother reading the body of a HEAD request. + # FIXME: Can we do this somehow without accessing private httplib _method? + self._original_response.close() + return + + while True: + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) + yield self._decode(chunk, decode_content=decode_content, + flush_decoder=True) + + # Chunk content ends with \r\n: discard it. + while True: + line = self._fp.fp.readline() + if not line: + # Some sites may not end with '\r\n'. + break + if line == b'\r\n': + break + + # We read everything; close the "file". + if self._original_response: + self._original_response.close() + self.release_conn() diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 2156993..859aec6 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -82,6 +82,7 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, err = _ if sock is not None: sock.close() + sock = None if err is not None: raise err diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index eb560df..7e0959d 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -2,10 +2,11 @@ import time import logging from ..exceptions import ( - ProtocolError, ConnectTimeoutError, - ReadTimeoutError, MaxRetryError, + ProtocolError, + ReadTimeoutError, + ResponseError, ) from ..packages import six @@ -36,7 +37,6 @@ class Retry(object): Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless retries are disabled, in which case the causing exception will be raised. - :param int total: Total number of retries to allow. Takes precedence over other counts. @@ -184,13 +184,13 @@ class Retry(object): return isinstance(err, ConnectTimeoutError) def _is_read_error(self, err): - """ Errors that occur after the request has been started, so we can't - assume that the server did not process any of it. + """ Errors that occur after the request has been started, so we should + assume that the server began processing it. """ return isinstance(err, (ReadTimeoutError, ProtocolError)) def is_forced_retry(self, method, status_code): - """ Is this method/response retryable? (Based on method/codes whitelists) + """ Is this method/status code retryable? (Based on method/codes whitelists) """ if self.method_whitelist and method.upper() not in self.method_whitelist: return False @@ -198,8 +198,7 @@ class Retry(object): return self.status_forcelist and status_code in self.status_forcelist def is_exhausted(self): - """ Are we out of retries? - """ + """ Are we out of retries? """ retry_counts = (self.total, self.connect, self.read, self.redirect) retry_counts = list(filter(None, retry_counts)) if not retry_counts: @@ -230,6 +229,7 @@ class Retry(object): connect = self.connect read = self.read redirect = self.redirect + cause = 'unknown' if error and self._is_connection_error(error): # Connect retry? @@ -251,10 +251,16 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 + cause = 'too many redirects' else: - # FIXME: Nothing changed, scenario doesn't make sense. + # Incrementing because of a server error like a 500 in + # status_forcelist and a the given method is in the whitelist _observed_errors += 1 + cause = ResponseError.GENERIC_ERROR + if response and response.status: + cause = ResponseError.SPECIFIC_ERROR.format( + status_code=response.status) new_retry = self.new( total=total, @@ -262,7 +268,7 @@ class Retry(object): _observed_errors=_observed_errors) if new_retry.is_exhausted(): - raise MaxRetryError(_pool, url, error) + raise MaxRetryError(_pool, url, error or ResponseError(cause)) log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 9cfe2d2..b846d42 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -1,21 +1,107 @@ from binascii import hexlify, unhexlify -from hashlib import md5, sha1 +from hashlib import md5, sha1, sha256 -from ..exceptions import SSLError +from ..exceptions import SSLError, InsecurePlatformWarning -try: # Test for SSL features - SSLContext = None - HAS_SNI = False +SSLContext = None +HAS_SNI = False +create_default_context = None + +import errno +import warnings +try: # Test for SSL features import ssl from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 - from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: + from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION +except ImportError: + OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 + OP_NO_COMPRESSION = 0x20000 + +# A secure default. +# Sources for more information on TLS ciphers: +# +# - https://wiki.mozilla.org/Security/Server_Side_TLS +# - https://www.ssllabs.com/projects/best-practices/index.html +# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ +# +# The general intent is: +# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), +# - prefer ECDHE over DHE for better performance, +# - prefer any AES-GCM over any AES-CBC for better performance and security, +# - use 3DES as fallback which is secure but slow, +# - disable NULL authentication, MD5 MACs and DSS for security reasons. +DEFAULT_CIPHERS = ( + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' + 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' + '!eNULL:!MD5' +) + +try: + from ssl import SSLContext # Modern SSL? +except ImportError: + import sys + + class SSLContext(object): # Platform-specific: Python 2 & 3.1 + supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or + (3, 2) <= sys.version_info) + + def __init__(self, protocol_version): + self.protocol = protocol_version + # Use default values from a real SSLContext + self.check_hostname = False + self.verify_mode = ssl.CERT_NONE + self.ca_certs = None + self.options = 0 + self.certfile = None + self.keyfile = None + self.ciphers = None + + def load_cert_chain(self, certfile, keyfile): + self.certfile = certfile + self.keyfile = keyfile + + def load_verify_locations(self, location): + self.ca_certs = location + + def set_ciphers(self, cipher_suite): + if not self.supports_set_ciphers: + raise TypeError( + 'Your version of Python does not support setting ' + 'a custom cipher suite. Please upgrade to Python ' + '2.7, 3.2, or later if you need this functionality.' + ) + self.ciphers = cipher_suite + + def wrap_socket(self, socket, server_hostname=None): + warnings.warn( + 'A true SSLContext object is not available. This prevents ' + 'urllib3 from configuring SSL appropriately and may cause ' + 'certain SSL connections to fail. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/security.html' + '#insecureplatformwarning.', + InsecurePlatformWarning + ) + kwargs = { + 'keyfile': self.keyfile, + 'certfile': self.certfile, + 'ca_certs': self.ca_certs, + 'cert_reqs': self.verify_mode, + 'ssl_version': self.protocol, + } + if self.supports_set_ciphers: # Platform-specific: Python 2.7+ + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) + else: # Platform-specific: Python 2.6 + return wrap_socket(socket, **kwargs) + + def assert_fingerprint(cert, fingerprint): """ Checks if given fingerprint matches the supplied certificate. @@ -30,7 +116,8 @@ def assert_fingerprint(cert, fingerprint): # this digest. hashfunc_map = { 16: md5, - 20: sha1 + 20: sha1, + 32: sha256, } fingerprint = fingerprint.replace(':', '').lower() @@ -91,42 +178,103 @@ def resolve_ssl_version(candidate): return candidate -if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - """ - All arguments except `server_hostname` have the same meaning as for - :func:`ssl.wrap_socket` - - :param server_hostname: - Hostname of the expected certificate - """ - context = SSLContext(ssl_version) - context.verify_mode = cert_reqs - - # Disable TLS compression to migitate CRIME attack (issue #309) - OP_NO_COMPRESSION = 0x20000 - context.options |= OP_NO_COMPRESSION - - if ca_certs: - try: - context.load_verify_locations(ca_certs) - # Py32 raises IOError - # Py33 raises FileNotFoundError - except Exception as e: # Reraise as SSLError +def create_urllib3_context(ssl_version=None, cert_reqs=None, + options=None, ciphers=None): + """All arguments have the same meaning as ``ssl_wrap_socket``. + + By default, this function does a lot of the same work that + ``ssl.create_default_context`` does on Python 3.4+. It: + + - Disables SSLv2, SSLv3, and compression + - Sets a restricted set of server ciphers + + If you wish to enable SSLv3, you can do:: + + from urllib3.util import ssl_ + context = ssl_.create_urllib3_context() + context.options &= ~ssl_.OP_NO_SSLv3 + + You can do the same to enable compression (substituting ``COMPRESSION`` + for ``SSLv3`` in the last line above). + + :param ssl_version: + The desired protocol version to use. This will default to + PROTOCOL_SSLv23 which will negotiate the highest protocol that both + the server and your installation of OpenSSL support. + :param cert_reqs: + Whether to require the certificate verification. This defaults to + ``ssl.CERT_REQUIRED``. + :param options: + Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, + ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``. + :param ciphers: + Which cipher suites to allow the server to select. + :returns: + Constructed SSLContext object with specified options + :rtype: SSLContext + """ + context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + + # Setting the default here, as we may have no ssl module on import + cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs + + if options is None: + options = 0 + # SSLv2 is easily broken and is considered harmful and dangerous + options |= OP_NO_SSLv2 + # SSLv3 has several problems and is now dangerous + options |= OP_NO_SSLv3 + # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ + # (issue #309) + options |= OP_NO_COMPRESSION + + context.options |= options + + if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + + context.verify_mode = cert_reqs + if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + # We do our own verification, including fingerprints and alternative + # hostnames. So disable it here + context.check_hostname = False + return context + + +def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, + ca_certs=None, server_hostname=None, + ssl_version=None, ciphers=None, ssl_context=None): + """ + All arguments except for server_hostname and ssl_context have the same + meaning as they do when using :func:`ssl.wrap_socket`. + + :param server_hostname: + When SNI is supported, the expected hostname of the certificate + :param ssl_context: + A pre-made :class:`SSLContext` object. If none is provided, one will + be created using :func:`create_urllib3_context`. + :param ciphers: + A string of ciphers we wish the client to support. This is not + supported on Python 2.6 as the ssl module does not support it. + """ + context = ssl_context + if context is None: + context = create_urllib3_context(ssl_version, cert_reqs, + ciphers=ciphers) + + if ca_certs: + try: + context.load_verify_locations(ca_certs) + except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + raise SSLError(e) + # Py33 raises FileNotFoundError which subclasses OSError + # These are not equivalent unless we check the errno attribute + except OSError as e: # Platform-specific: Python 3.3 and beyond + if e.errno == errno.ENOENT: raise SSLError(e) - if certfile: - # FIXME: This block needs a test. - context.load_cert_chain(certfile, keyfile) - if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI - return context.wrap_socket(sock, server_hostname=server_hostname) - return context.wrap_socket(sock) - -else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - return wrap_socket(sock, keyfile=keyfile, certfile=certfile, - ca_certs=ca_certs, cert_reqs=cert_reqs, - ssl_version=ssl_version) + raise + if certfile: + context.load_cert_chain(certfile, keyfile) + if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI + return context.wrap_socket(sock, server_hostname=server_hostname) + return context.wrap_socket(sock) diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index 487d456..e58050c 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)): def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): + if path and not path.startswith('/'): + path = '/' + path return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) @@ -40,6 +42,48 @@ class Url(namedtuple('Url', url_attrs)): return '%s:%d' % (self.host, self.port) return self.host + @property + def url(self): + """ + Convert self into a url + + This function should more or less round-trip with :func:`.parse_url`. The + returned url may not be exactly the same as the url inputted to + :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls + with a blank port will have : removed). + + Example: :: + + >>> U = parse_url('http://google.com/mail/') + >>> U.url + 'http://google.com/mail/' + >>> Url('http', 'username:password', 'host.com', 80, + ... '/path', 'query', 'fragment').url + 'http://username:password@host.com:80/path?query#fragment' + """ + scheme, auth, host, port, path, query, fragment = self + url = '' + + # We use "is not None" we want things to happen with empty strings (or 0 port) + if scheme is not None: + url += scheme + '://' + if auth is not None: + url += auth + '@' + if host is not None: + url += host + if port is not None: + url += ':' + str(port) + if path is not None: + url += path + if query is not None: + url += '?' + query + if fragment is not None: + url += '#' + fragment + + return url + + def __str__(self): + return self.url def split_first(s, delims): """ @@ -84,7 +128,7 @@ def parse_url(url): Example:: >>> parse_url('http://google.com/mail/') - Url(scheme='http', host='google.com', port=None, path='/', ...) + Url(scheme='http', host='google.com', port=None, path='/mail/', ...) >>> parse_url('google.com:80') Url(scheme=None, host='google.com', port=80, path=None, ...) >>> parse_url('/foo?bar') @@ -162,7 +206,6 @@ def parse_url(url): return Url(scheme, auth, host, port, path, query, fragment) - def get_host(url): """ Deprecated. Use :func:`.parse_url` instead. diff --git a/requests/sessions.py b/requests/sessions.py index d701ff2..820919e 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -13,7 +13,7 @@ from collections import Mapping from datetime import datetime from .auth import _basic_auth_str -from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str +from .compat import cookielib, OrderedDict, urljoin, urlparse from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT @@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook from .utils import to_key_val_list, default_headers, to_native_string from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) +from .packages.urllib3._collections import RecentlyUsedContainer from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter @@ -35,6 +36,8 @@ from .status_codes import codes # formerly defined here, reexposed here for backward compatibility from .models import REDIRECT_STATI +REDIRECT_CACHE_SIZE = 1000 + def merge_setting(request_setting, session_setting, dict_class=OrderedDict): """ @@ -87,7 +90,7 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): class SessionRedirectMixin(object): def resolve_redirects(self, resp, req, stream=False, timeout=None, - verify=True, cert=None, proxies=None): + verify=True, cert=None, proxies=None, **adapter_kwargs): """Receives a Response. Returns a generator of Responses.""" i = 0 @@ -128,7 +131,7 @@ class SessionRedirectMixin(object): # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. - if not urlparse(url).netloc: + if not parsed.netloc: url = urljoin(resp.url, requote_uri(url)) else: url = requote_uri(url) @@ -168,7 +171,10 @@ class SessionRedirectMixin(object): except KeyError: pass - extract_cookies_to_jar(prepared_request._cookies, prepared_request, resp.raw) + # Extract any cookies sent on the response to the cookiejar + # in the new request. Because we've mutated our copied prepared + # request, use the old one that we haven't yet touched. + extract_cookies_to_jar(prepared_request._cookies, req, resp.raw) prepared_request._cookies.update(self.cookies) prepared_request.prepare_cookies(prepared_request._cookies) @@ -187,6 +193,7 @@ class SessionRedirectMixin(object): cert=cert, proxies=proxies, allow_redirects=False, + **adapter_kwargs ) extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) @@ -273,7 +280,7 @@ class Session(SessionRedirectMixin): __attrs__ = [ 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', - 'max_redirects', 'redirect_cache' + 'max_redirects', ] def __init__(self): @@ -327,7 +334,8 @@ class Session(SessionRedirectMixin): self.mount('https://', HTTPAdapter()) self.mount('http://', HTTPAdapter()) - self.redirect_cache = {} + # Only store 1000 redirects to prevent using infinite memory + self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) def __enter__(self): return self @@ -425,7 +433,7 @@ class Session(SessionRedirectMixin): If Tuple, ('cert', 'key') pair. """ - method = builtin_str(method) + method = to_native_string(method) # Create the Request. req = Request( @@ -553,10 +561,6 @@ class Session(SessionRedirectMixin): # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') - timeout = kwargs.get('timeout') - verify = kwargs.get('verify') - cert = kwargs.get('cert') - proxies = kwargs.get('proxies') hooks = request.hooks # Get the appropriate adapter to use @@ -584,12 +588,7 @@ class Session(SessionRedirectMixin): extract_cookies_to_jar(self.cookies, request, r.raw) # Redirect resolving generator. - gen = self.resolve_redirects(r, request, - stream=stream, - timeout=timeout, - verify=verify, - cert=cert, - proxies=proxies) + gen = self.resolve_redirects(r, request, **kwargs) # Resolve redirects if allowed. history = [resp for resp in gen] if allow_redirects else [] @@ -658,12 +657,19 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state['redirect_cache'] = dict(self.redirect_cache) + return state def __setstate__(self, state): + redirect_cache = state.pop('redirect_cache', {}) for attr, value in state.items(): setattr(self, attr, value) + self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) + for redirect, to in redirect_cache.items(): + self.redirect_cache[redirect] = to + def session(): """Returns a :class:`Session` for context-management.""" diff --git a/requests/utils.py b/requests/utils.py index 1868f86..8fba62d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -19,12 +19,14 @@ import re import sys import socket import struct +import warnings from . import __version__ from . import certs from .compat import parse_http_list as _parse_list_header from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, - builtin_str, getproxies, proxy_bypass, urlunparse) + builtin_str, getproxies, proxy_bypass, urlunparse, + basestring) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import InvalidURL @@ -114,7 +116,8 @@ def get_netrc_auth(url): def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) - if name and name[0] != '<' and name[-1] != '>': + if (name and isinstance(name, basestring) and name[0] != '<' and + name[-1] != '>'): return os.path.basename(name) @@ -287,6 +290,11 @@ def get_encodings_from_content(content): :param content: bytestring to extract encodings from. """ + warnings.warn(( + 'In requests 3.0, get_encodings_from_content will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)'), + DeprecationWarning) charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I) pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I) @@ -354,6 +362,11 @@ def get_unicode_from_response(r): 2. fall back and replace all unicode characters """ + warnings.warn(( + 'In requests 3.0, get_unicode_from_response will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)'), + DeprecationWarning) tried_encodings = [] @@ -407,10 +420,18 @@ def requote_uri(uri): This function passes the given URI through an unquote/quote cycle to ensure that it is fully and consistently quoted. """ - # Unquote only the unreserved characters - # Then quote only illegal characters (do not quote reserved, unreserved, - # or '%') - return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") + safe_with_percent = "!#$%&'()*+,/:;=?@[]~" + safe_without_percent = "!#$&'()*+,/:;=?@[]~" + try: + # Unquote only the unreserved characters + # Then quote only illegal characters (do not quote reserved, + # unreserved, or '%') + return quote(unquote_unreserved(uri), safe=safe_with_percent) + except InvalidURL: + # We couldn't unquote the given URI, so let's try quoting it, but + # there may be unquoted '%'s in the URI. We need to make sure they're + # properly quoted so they do not cause issues elsewhere. + return quote(uri, safe=safe_without_percent) def address_in_network(ip, net): @@ -567,7 +588,7 @@ def parse_header_links(value): replace_chars = " '\"" - for val in value.split(","): + for val in re.split(", *<", value): try: url, params = val.split(";", 1) except ValueError: @@ -1,10 +1,9 @@ #!/usr/bin/env python import os +import re import sys -import requests - from codecs import open try: @@ -29,6 +28,14 @@ packages = [ requires = [] +version = '' +with open('requests/__init__.py', 'r') as fd: + version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError('Cannot find version information') + with open('README.rst', 'r', 'utf-8') as f: readme = f.read() with open('HISTORY.rst', 'r', 'utf-8') as f: @@ -36,7 +43,7 @@ with open('HISTORY.rst', 'r', 'utf-8') as f: setup( name='requests', - version=requests.__version__, + version=version, description='Python HTTP for Humans.', long_description=readme + '\n\n' + history, author='Kenneth Reitz', @@ -60,7 +67,6 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4' - ), extras_require={ 'security': ['pyOpenSSL', 'ndg-httpsclient', 'pyasn1'], diff --git a/test_requests.py b/test_requests.py index 0d93893..cad8c05 100755 --- a/test_requests.py +++ b/test_requests.py @@ -20,7 +20,7 @@ from requests.compat import ( from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import (ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, MissingSchema, - ReadTimeout, Timeout) + ReadTimeout, Timeout, RetryError) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -103,6 +103,14 @@ class RequestsTestCase(unittest.TestCase): head_req = requests.Request('HEAD', httpbin('head')).prepare() assert 'Content-Length' not in head_req.headers + def test_override_content_length(self): + headers = { + 'Content-Length': 'not zero' + } + r = requests.Request('POST', httpbin('post'), headers=headers).prepare() + assert 'Content-Length' in r.headers + assert r.headers['Content-Length'] == 'not zero' + def test_path_is_not_double_encoded(self): request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare() @@ -250,7 +258,7 @@ class RequestsTestCase(unittest.TestCase): """Do not send headers in Session.headers with None values.""" ses = requests.Session() ses.headers['Accept-Encoding'] = None - req = requests.Request('GET', 'http://httpbin.org/get') + req = requests.Request('GET', httpbin('get')) prep = ses.prepare_request(req) assert 'Accept-Encoding' not in prep.headers @@ -293,13 +301,20 @@ class RequestsTestCase(unittest.TestCase): r = s.get(url) assert r.status_code == 200 - def test_connection_error(self): + def test_connection_error_invalid_domain(self): """Connecting to an unknown domain should raise a ConnectionError""" with pytest.raises(ConnectionError): - requests.get("http://fooobarbangbazbing.httpbin.org") + requests.get("http://doesnotexist.google.com") + def test_connection_error_invalid_port(self): + """Connecting to an invalid port should raise a ConnectionError""" with pytest.raises(ConnectionError): - requests.get("http://httpbin.org:1") + requests.get("http://httpbin.org:1", timeout=1) + + def test_LocationParseError(self): + """Inputing a URL that cannot be parsed should raise an InvalidURL error""" + with pytest.raises(InvalidURL): + requests.get("http://fe80::5054:ff:fe5a:fc0") def test_basicauth_with_netrc(self): auth = ('user', 'pass') @@ -920,6 +935,27 @@ class RequestsTestCase(unittest.TestCase): assert 'multipart/form-data' in p.headers['Content-Type'] + def test_can_send_bytes_bytearray_objects_with_files(self): + # Test bytes: + data = {'a': 'this is a string'} + files = {'b': b'foo'} + r = requests.Request('POST', httpbin('post'), data=data, files=files) + p = r.prepare() + assert 'multipart/form-data' in p.headers['Content-Type'] + # Test bytearrays: + files = {'b': bytearray(b'foo')} + r = requests.Request('POST', httpbin('post'), data=data, files=files) + p = r.prepare() + assert 'multipart/form-data' in p.headers['Content-Type'] + + def test_can_send_file_object_with_non_string_filename(self): + f = io.BytesIO() + f.name = 2 + r = requests.Request('POST', httpbin('post'), files={'f': f}) + p = r.prepare() + + assert 'multipart/form-data' in p.headers['Content-Type'] + def test_autoset_header_values_are_native(self): data = 'this is a string' length = '16' @@ -1000,12 +1036,12 @@ class RequestsTestCase(unittest.TestCase): assert s == "Basic dGVzdDp0ZXN0" def test_requests_history_is_saved(self): - r = requests.get('https://httpbin.org/redirect/5') + r = requests.get(httpbin('redirect/5')) total = r.history[-1].history i = 0 for item in r.history: assert item.history == total[0:i] - i=i+1 + i = i + 1 def test_json_param_post_content_type_works(self): r = requests.post( @@ -1016,6 +1052,23 @@ class RequestsTestCase(unittest.TestCase): assert 'application/json' in r.request.headers['Content-Type'] assert {'life': 42} == r.json()['json'] + def test_response_iter_lines(self): + r = requests.get(httpbin('stream/4'), stream=True) + assert r.status_code == 200 + + it = r.iter_lines() + next(it) + assert len(list(it)) == 3 + + @pytest.mark.xfail + def test_response_iter_lines_reentrant(self): + """Response.iter_lines() is not reentrant safe""" + r = requests.get(httpbin('stream/4'), stream=True) + assert r.status_code == 200 + + next(r.iter_lines()) + assert len(list(r.iter_lines())) == 3 + class TestContentEncodingDetection(unittest.TestCase): @@ -1244,6 +1297,32 @@ class UtilsTestCase(unittest.TestCase): 'http://localhost.localdomain:5000/v1.0/') == {} assert get_environ_proxies('http://www.requests.com/') != {} + def test_guess_filename_when_int(self): + from requests.utils import guess_filename + assert None is guess_filename(1) + + def test_guess_filename_when_filename_is_an_int(self): + from requests.utils import guess_filename + fake = type('Fake', (object,), {'name': 1})() + assert None is guess_filename(fake) + + def test_guess_filename_with_file_like_obj(self): + from requests.utils import guess_filename + from requests import compat + fake = type('Fake', (object,), {'name': b'value'})() + guessed_name = guess_filename(fake) + assert b'value' == guessed_name + assert isinstance(guessed_name, compat.bytes) + + def test_guess_filename_with_unicode_name(self): + from requests.utils import guess_filename + from requests import compat + filename = b'value'.decode('utf-8') + fake = type('Fake', (object,), {'name': filename})() + guessed_name = guess_filename(fake) + assert filename == guessed_name + assert isinstance(guessed_name, compat.str) + def test_is_ipv4_address(self): from requests.utils import is_ipv4_address assert is_ipv4_address('8.8.8.8') @@ -1280,6 +1359,22 @@ class UtilsTestCase(unittest.TestCase): assert username == percent_encoding_test_chars assert password == percent_encoding_test_chars + def test_requote_uri_with_unquoted_percents(self): + """Ensure we handle unquoted percent signs in redirects. + + See: https://github.com/kennethreitz/requests/issues/2356 + """ + from requests.utils import requote_uri + bad_uri = 'http://example.com/fiz?buz=%ppicture' + quoted = 'http://example.com/fiz?buz=%25ppicture' + assert quoted == requote_uri(bad_uri) + + def test_requote_uri_properly_requotes(self): + """Ensure requoting doesn't break expectations.""" + from requests.utils import requote_uri + quoted = 'http://example.com/fiz?buz=%25ppicture' + assert quoted == requote_uri(quoted) + class TestMorselToCookieExpires(unittest.TestCase): @@ -1342,7 +1437,7 @@ class TestMorselToCookieMaxAge(unittest.TestCase): class TestTimeout: def test_stream_timeout(self): try: - requests.get('https://httpbin.org/delay/10', timeout=2.0) + requests.get(httpbin('delay/10'), timeout=2.0) except requests.exceptions.Timeout as e: assert 'Read timed out' in e.args[0].args[0] @@ -1389,6 +1484,11 @@ class TestTimeout: except ConnectTimeout: pass + def test_encoded_methods(self): + """See: https://github.com/kennethreitz/requests/issues/2316""" + r = requests.request(b'GET', httpbin('get')) + assert r.ok + SendCall = collections.namedtuple('SendCall', ('args', 'kwargs')) @@ -1437,7 +1537,7 @@ class TestRedirects: def test_requests_are_updated_each_time(self): session = RedirectSession([303, 307]) - prep = requests.Request('POST', 'http://httpbin.org/post').prepare() + prep = requests.Request('POST', httpbin('post')).prepare() r0 = session.send(prep) assert r0.request.method == 'POST' assert session.calls[-1] == SendCall((r0.request,), {}) @@ -1507,13 +1607,32 @@ def test_prepared_request_complete_copy(): ) assert_copy(p, p.copy()) + def test_prepare_unicode_url(): p = PreparedRequest() p.prepare( method='GET', - url=u('http://www.example.com/üniçø∂é') + url=u('http://www.example.com/üniçø∂é'), ) assert_copy(p, p.copy()) + +def test_urllib3_retries(): + from requests.packages.urllib3.util import Retry + s = requests.Session() + s.mount('http://', HTTPAdapter(max_retries=Retry( + total=2, status_forcelist=[500] + ))) + + with pytest.raises(RetryError): + s.get(httpbin('status/500')) + +def test_vendor_aliases(): + from requests.packages import urllib3 + from requests.packages import chardet + + with pytest.raises(ImportError): + from requests.packages import webbrowser + if __name__ == '__main__': unittest.main() |