aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSVN-Git Migration <python-modules-team@lists.alioth.debian.org>2015-10-08 13:19:39 -0700
committerSVN-Git Migration <python-modules-team@lists.alioth.debian.org>2015-10-08 13:19:39 -0700
commit54bdd56778a37ea9d56d451d4ae49b99cbbfceaa (patch)
tree3b69a7371b29898f08ced7514b42bad8afdc8759
parent0f393d00b51bc54c5075447e4a8b21f0bed6acd8 (diff)
downloadpython-urllib3-54bdd56778a37ea9d56d451d4ae49b99cbbfceaa.tar
python-urllib3-54bdd56778a37ea9d56d451d4ae49b99cbbfceaa.tar.gz
Imported Upstream version 1.9.1
-rw-r--r--CHANGES.rst21
-rw-r--r--CONTRIBUTORS.txt10
-rw-r--r--MANIFEST.in5
-rw-r--r--Makefile1
-rw-r--r--PKG-INFO26
-rw-r--r--README.rst3
-rw-r--r--dev-requirements.txt4
-rw-r--r--docs/Makefile130
-rw-r--r--docs/README14
-rw-r--r--docs/collections.rst13
-rw-r--r--docs/conf.py232
-rw-r--r--docs/contrib.rst14
-rw-r--r--docs/doc-requirements.txt12
-rw-r--r--docs/exceptions.rst7
-rw-r--r--docs/helpers.rst55
-rw-r--r--docs/index.rst340
-rw-r--r--docs/make.bat170
-rw-r--r--docs/managers.rst62
-rw-r--r--docs/pools.rst71
-rw-r--r--docs/security.rst161
-rw-r--r--dummyserver/__init__.pycbin0 -> 141 bytes
-rw-r--r--dummyserver/handlers.pycbin0 -> 9742 bytes
-rw-r--r--dummyserver/proxy.pycbin0 -> 4740 bytes
-rw-r--r--dummyserver/server.pycbin0 -> 5992 bytes
-rw-r--r--dummyserver/testcase.pycbin0 -> 5450 bytes
-rw-r--r--test/__init__.py92
-rw-r--r--test/__init__.pycbin0 -> 3946 bytes
-rw-r--r--test/benchmark.py77
-rw-r--r--test/contrib/__init__.py0
-rw-r--r--test/contrib/__init__.pycbin0 -> 142 bytes
-rw-r--r--test/contrib/test_pyopenssl.py23
-rw-r--r--test/contrib/test_pyopenssl.pycbin0 -> 1143 bytes
-rw-r--r--test/port_helpers.py100
-rw-r--r--test/port_helpers.pycbin0 -> 5719 bytes
-rw-r--r--test/test_collections.pycbin0 -> 6842 bytes
-rw-r--r--test/test_compatibility.pycbin0 -> 1372 bytes
-rw-r--r--test/test_connectionpool.pycbin0 -> 8862 bytes
-rw-r--r--test/test_exceptions.pycbin0 -> 1931 bytes
-rw-r--r--test/test_fields.pycbin0 -> 2739 bytes
-rw-r--r--test/test_filepost.pycbin0 -> 4916 bytes
-rw-r--r--test/test_poolmanager.pycbin0 -> 2499 bytes
-rw-r--r--test/test_proxymanager.pycbin0 -> 1670 bytes
-rw-r--r--test/test_response.py31
-rw-r--r--test/test_response.pycbin0 -> 14619 bytes
-rw-r--r--test/test_retry.pycbin0 -> 6491 bytes
-rw-r--r--test/test_util.py31
-rw-r--r--test/test_util.pycbin0 -> 15036 bytes
-rw-r--r--test/with_dummyserver/__init__.py0
-rw-r--r--test/with_dummyserver/__init__.pycbin0 -> 151 bytes
-rw-r--r--test/with_dummyserver/test_connectionpool.py706
-rw-r--r--test/with_dummyserver/test_connectionpool.pycbin0 -> 27640 bytes
-rw-r--r--test/with_dummyserver/test_https.py374
-rw-r--r--test/with_dummyserver/test_https.pycbin0 -> 15651 bytes
-rw-r--r--test/with_dummyserver/test_poolmanager.py136
-rw-r--r--test/with_dummyserver/test_poolmanager.pycbin0 -> 5591 bytes
-rw-r--r--test/with_dummyserver/test_proxy_poolmanager.py263
-rw-r--r--test/with_dummyserver/test_proxy_poolmanager.pycbin0 -> 9891 bytes
-rw-r--r--test/with_dummyserver/test_socketlevel.py544
-rw-r--r--test/with_dummyserver/test_socketlevel.pycbin0 -> 18715 bytes
-rw-r--r--urllib3.egg-info/PKG-INFO26
-rw-r--r--urllib3.egg-info/SOURCES.txt50
-rw-r--r--urllib3/__init__.py4
-rw-r--r--urllib3/connection.py43
-rw-r--r--urllib3/connectionpool.py6
-rw-r--r--urllib3/contrib/pyopenssl.py36
-rw-r--r--urllib3/exceptions.py21
-rw-r--r--urllib3/response.py15
-rw-r--r--urllib3/util/connection.py8
-rw-r--r--urllib3/util/response.py17
-rw-r--r--urllib3/util/retry.py2
70 files changed, 3919 insertions, 37 deletions
diff --git a/CHANGES.rst b/CHANGES.rst
index 9ada9c2..dd2cd2d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,27 @@
Changes
=======
+1.9.1 (2014-09-13)
+++++++++++++++++++
+
+* Apply socket arguments before binding. (Issue #427)
+
+* More careful checks if fp-like object is closed. (Issue #435)
+
+* Fixed packaging issues of some development-related files not
+ getting included. (Issue #440)
+
+* Allow performing *only* fingerprint verification. (Issue #444)
+
+* Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445)
+
+* Fixed PyOpenSSL compatibility with PyPy. (Issue #450)
+
+* Fixed ``BrokenPipeError`` and ``ConnectionError`` handling in Py3.
+ (Issue #443)
+
+
+
1.9 (2014-07-04)
++++++++++++++++
diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index e6178f1..97f3014 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -117,5 +117,15 @@ In chronological order:
* Arthur Grunseid <http://grunseid.com>
* source_address support and tests (with https://github.com/bui)
+* Ian Cordasco <graffatcolmingov@gmail.com>
+ * PEP8 Compliance and Linting
+ * Add ability to pass socket options to an HTTP Connection
+
+* Erik Tollerud <erik.tollerud@gmail.com>
+ * Support for standard library io module.
+
+* Krishna Prasad <kprasad.iitd@gmail.com>
+ * Google App Engine documentation
+
* [Your name or handle] <[email or website]>
* [Brief summary of your changes]
diff --git a/MANIFEST.in b/MANIFEST.in
index 6b37d64..4edfedd 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,5 @@
include README.rst CHANGES.rst LICENSE.txt CONTRIBUTORS.txt dev-requirements.txt Makefile
-recursive-include dummyserver *.*
+recursive-include dummyserver *
+recursive-include test *
+recursive-include docs *
+recursive-exclude docs/_build *
diff --git a/Makefile b/Makefile
index a6cdcfb..b692b12 100644
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,7 @@ clean:
find . -name "*.py[oc]" -delete
find . -name "__pycache__" -delete
rm -f $(REQUIREMENTS_OUT)
+ rm -rf docs/_build
test: requirements
nosetests
diff --git a/PKG-INFO b/PKG-INFO
index 168944c..964cd4b 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: urllib3
-Version: 1.9
+Version: 1.9.1
Summary: HTTP library with thread-safe connection pooling, file post, and more.
Home-page: http://urllib3.readthedocs.org/
Author: Andrey Petrov
@@ -13,6 +13,9 @@ Description: =======
.. image:: https://travis-ci.org/shazow/urllib3.png?branch=master
:target: https://travis-ci.org/shazow/urllib3
+ .. image:: https://www.bountysource.com/badge/tracker?tracker_id=192525
+ :target: https://www.bountysource.com/trackers/192525-urllib3?utm_source=192525&utm_medium=shield&utm_campaign=TRACKER_BADGE
+
Highlights
==========
@@ -153,6 +156,27 @@ Description: =======
Changes
=======
+ 1.9.1 (2014-09-13)
+ ++++++++++++++++++
+
+ * Apply socket arguments before binding. (Issue #427)
+
+ * More careful checks if fp-like object is closed. (Issue #435)
+
+ * Fixed packaging issues of some development-related files not
+ getting included. (Issue #440)
+
+ * Allow performing *only* fingerprint verification. (Issue #444)
+
+ * Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445)
+
+ * Fixed PyOpenSSL compatibility with PyPy. (Issue #450)
+
+ * Fixed ``BrokenPipeError`` and ``ConnectionError`` handling in Py3.
+ (Issue #443)
+
+
+
1.9 (2014-07-04)
++++++++++++++++
diff --git a/README.rst b/README.rst
index 6a81759..fc6bccf 100644
--- a/README.rst
+++ b/README.rst
@@ -5,6 +5,9 @@ urllib3
.. image:: https://travis-ci.org/shazow/urllib3.png?branch=master
:target: https://travis-ci.org/shazow/urllib3
+.. image:: https://www.bountysource.com/badge/tracker?tracker_id=192525
+ :target: https://www.bountysource.com/trackers/192525-urllib3?utm_source=192525&utm_medium=shield&utm_campaign=TRACKER_BADGE
+
Highlights
==========
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 6de0e09..8010704 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,7 @@
nose==1.3.3
mock==1.0.1
-tornado==3.2.2
coverage==3.7.1
tox==1.7.1
+
+# Tornado 3.2.2 makes our tests flaky, so we stick with 3.1
+tornado==3.1.1
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..135c543
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,130 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/urllib3.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/urllib3.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/urllib3"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/urllib3"
+ @echo "# devhelp"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
diff --git a/docs/README b/docs/README
new file mode 100644
index 0000000..9126c73
--- /dev/null
+++ b/docs/README
@@ -0,0 +1,14 @@
+# Building the Docs
+
+First install Sphinx:
+
+ pip install sphinx
+
+Install pyopenssl and certifi dependencies, to avoid some build errors. (Optional)
+
+ # This step is optional
+ pip install ndg-httpsclient pyasn1 certifi
+
+Then build:
+
+ cd docs && make html
diff --git a/docs/collections.rst b/docs/collections.rst
new file mode 100644
index 0000000..b348140
--- /dev/null
+++ b/docs/collections.rst
@@ -0,0 +1,13 @@
+Collections
+===========
+
+These datastructures are used to implement the behaviour of various urllib3
+components in a decoupled and application-agnostic design.
+
+.. automodule:: urllib3._collections
+
+ .. autoclass:: RecentlyUsedContainer
+ :members:
+
+ .. autoclass:: HTTPHeaderDict
+ :members:
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..7ac8393
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,232 @@
+# -*- coding: utf-8 -*-
+#
+# urllib3 documentation build configuration file, created by
+# sphinx-quickstart on Wed Oct 5 13:15:40 2011.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+from datetime import date
+import os
+import sys
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.insert(0, root_path)
+
+import urllib3
+
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.intersphinx',
+]
+
+# Test code blocks only when explicitly specified
+doctest_test_doctest_blocks = ''
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'urllib3'
+copyright = u'{year}, Andrey Petrov'.format(year=date.today().year)
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = urllib3.__version__
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'nature'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'urllib3doc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'urllib3.tex', u'urllib3 Documentation',
+ u'Andrey Petrov', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'urllib3', u'urllib3 Documentation',
+ [u'Andrey Petrov'], 1)
+]
+
+intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
diff --git a/docs/contrib.rst b/docs/contrib.rst
new file mode 100644
index 0000000..99c5492
--- /dev/null
+++ b/docs/contrib.rst
@@ -0,0 +1,14 @@
+.. _contrib-modules:
+
+Contrib Modules
+===============
+
+These modules implement various extra features, that may not be ready for
+prime time.
+
+.. _pyopenssl:
+
+SNI-support for Python 2
+------------------------
+
+.. automodule:: urllib3.contrib.pyopenssl
diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt
new file mode 100644
index 0000000..b7b6d66
--- /dev/null
+++ b/docs/doc-requirements.txt
@@ -0,0 +1,12 @@
+ndg-httpsclient==0.3.2
+pyasn1==0.1.7
+Sphinx==1.2.2
+Jinja2==2.7.3
+MarkupSafe==0.23
+Pygments==1.6
+cryptography==0.4
+six==1.7.2
+cffi==0.8.2
+docutils==0.11
+pycparser==2.10
+certifi==14.05.14 \ No newline at end of file
diff --git a/docs/exceptions.rst b/docs/exceptions.rst
new file mode 100644
index 0000000..f9e0553
--- /dev/null
+++ b/docs/exceptions.rst
@@ -0,0 +1,7 @@
+Exceptions
+==========
+
+Custom exceptions defined by urllib3
+
+.. automodule:: urllib3.exceptions
+ :members:
diff --git a/docs/helpers.rst b/docs/helpers.rst
new file mode 100644
index 0000000..79f268b
--- /dev/null
+++ b/docs/helpers.rst
@@ -0,0 +1,55 @@
+Helpers
+=======
+
+Useful methods for working with :mod:`httplib`, completely decoupled from
+code specific to **urllib3**.
+
+
+Timeouts
+--------
+
+.. automodule:: urllib3.util.timeout
+ :members:
+
+Retries
+-------
+
+.. automodule:: urllib3.util.retry
+ :members:
+
+URL Helpers
+-----------
+
+.. automodule:: urllib3.util.url
+ :members:
+
+Filepost
+--------
+
+.. automodule:: urllib3.filepost
+ :members:
+
+.. automodule:: urllib3.fields
+ :members:
+
+Request
+-------
+
+.. automodule:: urllib3.request
+ :members:
+
+.. automodule:: urllib3.util.request
+ :members:
+
+Response
+--------
+
+.. automodule:: urllib3.response
+ :members:
+ :undoc-members:
+
+SSL/TLS Helpers
+---------------
+
+.. automodule:: urllib3.util.ssl_
+ :members:
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..1fc8a9c
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,340 @@
+=====================
+urllib3 Documentation
+=====================
+
+.. toctree::
+ :hidden:
+
+ pools
+ managers
+ security
+ helpers
+ collections
+ contrib
+
+
+Highlights
+==========
+
+- Re-use the same socket connection for multiple requests, with optional
+ client-side certificate verification. See:
+ :class:`~urllib3.connectionpool.HTTPConnectionPool` and
+ :class:`~urllib3.connectionpool.HTTPSConnectionPool`
+
+- File posting. See:
+ :func:`~urllib3.filepost.encode_multipart_formdata`
+
+- Built-in redirection and retries (optional).
+
+- Supports gzip and deflate decoding. See:
+ :func:`~urllib3.response.decode_gzip` and
+ :func:`~urllib3.response.decode_deflate`
+
+- Thread-safe and sanity-safe.
+
+- Tested on Python 2.6+ and Python 3.2+, 100% unit test coverage.
+
+- Works with AppEngine, gevent, eventlib, and the standard library :mod:`io` module.
+
+- Small and easy to understand codebase perfect for extending and building upon.
+ For a more comprehensive solution, have a look at
+ `Requests <http://python-requests.org/>`_ which is also powered by urllib3.
+
+
+Getting Started
+===============
+
+Installing
+----------
+
+``pip install urllib3`` or fetch the latest source from
+`github.com/shazow/urllib3 <https://github.com/shazow/urllib3>`_.
+
+Usage
+-----
+
+.. doctest ::
+
+ >>> import urllib3
+ >>> http = urllib3.PoolManager()
+ >>> r = http.request('GET', 'http://example.com/')
+ >>> r.status
+ 200
+ >>> r.headers['server']
+ 'ECS (iad/182A)'
+ >>> 'data: ' + r.data
+ 'data: ...'
+
+
+**By default, urllib3 does not verify your HTTPS requests**.
+You'll need to supply a root certificate bundle, or use `certifi
+<https://certifi.io/>`_
+
+.. doctest ::
+
+ >>> import urllib3, certifi
+ >>> http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
+ >>> r = http.request('GET', 'https://insecure.com/')
+ Traceback (most recent call last):
+ ...
+ SSLError: hostname 'insecure.com' doesn't match 'svn.nmap.org'
+
+For more on making secure SSL/TLS HTTPS requests, read the :ref:`Security
+section <security>`.
+
+
+urllib3's responses respect the :mod:`io` framework from Python's
+standard library, allowing use of these standard objects for purposes
+like buffering:
+
+.. doctest ::
+
+ >>> http = urllib3.PoolManager()
+ >>> r = http.urlopen('GET','http://example.com/', preload_content=False)
+ >>> b = io.BufferedReader(r, 2048)
+ >>> firstpart = b.read(100)
+ >>> # ... your internet connection fails momentarily ...
+ >>> secondpart = b.read()
+
+
+Components
+==========
+
+:mod:`urllib3` tries to strike a fine balance between power, extendability, and
+sanity. To achieve this, the codebase is a collection of small reusable
+utilities and abstractions composed together in a few helpful layers.
+
+
+PoolManager
+-----------
+
+The highest level is the :doc:`PoolManager(...) <managers>`.
+
+The :class:`~urllib3.poolmanagers.PoolManager` will take care of reusing
+connections for you whenever you request the same host. This should cover most
+scenarios without significant loss of efficiency, but you can always drop down
+to a lower level component for more granular control.
+
+.. doctest ::
+
+ >>> import urllib3
+ >>> http = urllib3.PoolManager(10)
+ >>> r1 = http.request('GET', 'http://example.com/')
+ >>> r2 = http.request('GET', 'http://httpbin.org/')
+ >>> r3 = http.request('GET', 'http://httpbin.org/get')
+ >>> len(http.pools)
+ 2
+
+A :class:`~urllib3.poolmanagers.PoolManager` is a proxy for a collection of
+:class:`ConnectionPool` objects. They both inherit from
+:class:`~urllib3.request.RequestMethods` to make sure that their API is
+similar, so that instances of either can be passed around interchangeably.
+
+
+ProxyManager
+------------
+
+The :class:`~urllib3.poolmanagers.ProxyManager` is an HTTP proxy-aware
+subclass of :class:`~urllib3.poolmanagers.PoolManager`. It produces a single
+:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP
+connections and individual per-server:port
+:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled
+HTTPS connections:
+
+::
+
+ >>> proxy = urllib3.ProxyManager('http://localhost:3128/')
+ >>> r1 = proxy.request('GET', 'http://google.com/')
+ >>> r2 = proxy.request('GET', 'http://httpbin.org/')
+ >>> len(proxy.pools)
+ 1
+ >>> r3 = proxy.request('GET', 'https://httpbin.org/')
+ >>> r4 = proxy.request('GET', 'https://twitter.com/')
+ >>> len(proxy.pools)
+ 3
+
+
+ConnectionPool
+--------------
+
+The next layer is the :doc:`ConnectionPool(...) <pools>`.
+
+The :class:`~urllib3.connectionpool.HTTPConnectionPool` and
+:class:`~urllib3.connectionpool.HTTPSConnectionPool` classes allow you to
+define a pool of connections to a single host and make requests against this
+pool with automatic **connection reusing** and **thread safety**.
+
+When the :mod:`ssl` module is available, then
+:class:`~urllib3.connectionpool.HTTPSConnectionPool` objects can be configured
+to check SSL certificates against specific provided certificate authorities.
+
+.. doctest ::
+
+ >>> import urllib3
+ >>> conn = urllib3.connection_from_url('http://httpbin.org/')
+ >>> r1 = conn.request('GET', 'http://httpbin.org/')
+ >>> r2 = conn.request('GET', '/user-agent')
+ >>> r3 = conn.request('GET', 'http://example.com')
+ Traceback (most recent call last):
+ ...
+ urllib3.exceptions.HostChangedError: HTTPConnectionPool(host='httpbin.org', port=None): Tried to open a foreign host with url: http://example.com
+
+Again, a ConnectionPool is a pool of connections to a specific host. Trying to
+access a different host through the same pool will raise a ``HostChangedError``
+exception unless you specify ``assert_same_host=False``. Do this at your own
+risk as the outcome is completely dependent on the behaviour of the host server.
+
+If you need to access multiple hosts and don't want to manage your own
+collection of :class:`~urllib3.connectionpool.ConnectionPool` objects, then you
+should use a :class:`~urllib3.poolmanager.PoolManager`.
+
+A :class:`~urllib3.connectionpool.ConnectionPool` is composed of a collection
+of :class:`httplib.HTTPConnection` objects.
+
+
+Timeout
+-------
+
+A timeout can be set to abort socket operations on individual connections after
+the specified duration. The timeout can be defined as a float or an instance of
+:class:`~urllib3.util.timeout.Timeout` which gives more granular configuration
+over how much time is allowed for different stages of the request. This can be
+set for the entire pool or per-request.
+
+.. doctest ::
+
+ >>> from urllib3 import PoolManager, Timeout
+
+ >>> # Manager with 3 seconds combined timeout.
+ >>> http = PoolManager(timeout=3.0)
+ >>> r = http.request('GET', 'http://httpbin.org/delay/1')
+
+ >>> # Manager with 2 second timeout for the read phase, no limit for the rest.
+ >>> http = PoolManager(timeout=Timeout(read=2.0))
+ >>> r = http.request('GET', 'http://httpbin.org/delay/1')
+
+ >>> # Manager with no timeout but a request with a timeout of 1 seconds for
+ >>> # the connect phase and 2 seconds for the read phase.
+ >>> http = PoolManager()
+ >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(connect=1.0, read=2.0))
+
+ >>> # Same Manager but request with a 5 second total timeout.
+ >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(total=5.0))
+
+See the :class:`~urllib3.util.timeout.Timeout` definition for more details.
+
+
+Retry
+-----
+
+Retries can be configured by passing an instance of
+:class:`~urllib3.util.retry.Retry`, or disabled by passing ``False``, to the
+``retries`` parameter.
+
+Redirects are also considered to be a subset of retries but can be configured or
+disabled individually.
+
+::
+
+ >>> from urllib3 import PoolManager, Retry
+
+ >>> # Allow 3 retries total for all requests in this pool. These are the same:
+ >>> http = PoolManager(retries=3)
+ >>> http = PoolManager(retries=Retry(3))
+ >>> http = PoolManager(retries=Retry(total=3))
+
+ >>> r = http.request('GET', 'http://httpbin.org/redirect/2')
+ >>> # r.status -> 200
+
+ >>> # Disable redirects for this request.
+ >>> r = http.request('GET', 'http://httpbin.org/redirect/2', retries=Retry(3, redirect=False))
+ >>> # r.status -> 302
+
+ >>> # No total limit, but only do 5 connect retries, for this request.
+ >>> r = http.request('GET', 'http://httpbin.org/', retries=Retry(connect=5))
+
+
+See the :class:`~urllib3.util.retry.Retry` definition for more details.
+
+
+Foundation
+----------
+
+At the very core, just like its predecessors, :mod:`urllib3` is built on top of
+:mod:`httplib` -- the lowest level HTTP library included in the Python
+standard library.
+
+To aid the limited functionality of the :mod:`httplib` module, :mod:`urllib3`
+provides various helper methods which are used with the higher level components
+but can also be used independently.
+
+.. toctree::
+
+ helpers
+ exceptions
+
+
+Contrib Modules
+---------------
+
+These modules implement various extra features, that may not be ready for
+prime time.
+
+.. toctree::
+
+ contrib
+
+
+Contributing
+============
+
+#. `Check for open issues <https://github.com/shazow/urllib3/issues>`_ or open
+ a fresh issue to start a discussion around a feature idea or a bug. There is
+ a *Contributor Friendly* tag for issues that should be ideal for people who
+ are not very familiar with the codebase yet.
+#. Fork the `urllib3 repository on Github <https://github.com/shazow/urllib3>`_
+ to start making your changes.
+#. Write a test which shows that the bug was fixed or that the feature works
+ as expected.
+#. Send a pull request and bug the maintainer until it gets merged and published.
+ :) Make sure to add yourself to ``CONTRIBUTORS.txt``.
+
+
+Sponsorship
+===========
+
+Please consider sponsoring urllib3 development, especially if your company
+benefits from this library.
+
+* **Project Grant**: A grant for contiguous full-time development has the
+ biggest impact for progress. Periods of 3 to 10 days allow a contributor to
+ tackle substantial complex issues which are otherwise left to linger until
+ somebody can't afford to not fix them.
+
+ Contact `@shazow <https://github.com/shazow>`_ to arrange a grant for a core
+ contributor.
+
+* **One-off**: Development will continue regardless of funding, but donations help move
+ things further along quicker as the maintainer can allocate more time off to
+ work on urllib3 specifically.
+
+ .. raw:: html
+
+ <a href="https://donorbox.org/personal-sponsor-urllib3" style="background-color:#1275ff;color:#fff;text-decoration:none;font-family:Verdana,sans-serif;display:inline-block;font-size:14px;padding:7px 16px;border-radius:5px;margin-right:2em;vertical-align:top;border:1px solid rgba(160,160,160,0.5);background-image:linear-gradient(#7dc5ee,#008cdd 85%,#30a2e4);box-shadow:inset 0 1px 0 rgba(255,255,255,0.25);">Sponsor with Credit Card</a>
+
+ <a class="coinbase-button" data-code="137087702cf2e77ce400d53867b164e6" href="https://coinbase.com/checkouts/137087702cf2e77ce400d53867b164e6">Sponsor with Bitcoin</a><script src="https://coinbase.com/assets/button.js" type="text/javascript"></script>
+
+* **Recurring**: You're welcome to `support the maintainer on Gittip
+ <https://www.gittip.com/shazow/>`_.
+
+
+Recent Sponsors
+---------------
+
+Huge thanks to all the companies and individuals who financially contributed to
+the development of urllib3. Please send a PR if you've donated and would like
+to be listed.
+
+* `Stripe <https://stripe.com/>`_ (June 23, 2014)
+
+.. * [Company] ([optional tagline]), [optional description of grant] ([date])
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..41aa35b
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,170 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\urllib3.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\urllib3.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+:end
diff --git a/docs/managers.rst b/docs/managers.rst
new file mode 100644
index 0000000..f9cab03
--- /dev/null
+++ b/docs/managers.rst
@@ -0,0 +1,62 @@
+PoolManager
+===========
+
+.. automodule:: urllib3.poolmanager
+
+A pool manager is an abstraction for a collection of
+:doc:`ConnectionPools <pools>`.
+
+If you need to make requests to multiple hosts, then you can use a
+:class:`.PoolManager`, which takes care of maintaining your pools
+so you don't have to.
+
+.. doctest ::
+
+ >>> from urllib3 import PoolManager
+ >>> manager = PoolManager(10)
+ >>> r = manager.request('GET', 'http://example.com')
+ >>> r.headers['server']
+ 'ECS (iad/182A)'
+ >>> r = manager.request('GET', 'http://httpbin.org/')
+ >>> r.headers['server']
+ 'gunicorn/18.0'
+ >>> r = manager.request('POST', 'http://httpbin.org/headers')
+ >>> r = manager.request('HEAD', 'http://httpbin.org/cookies')
+ >>> len(manager.pools)
+ 2
+ >>> conn = manager.connection_from_host('httpbin.org')
+ >>> conn.num_requests
+ 3
+
+The API of a :class:`.PoolManager` object is similar to that of a
+:doc:`ConnectionPool <pools>`, so they can be passed around interchangeably.
+
+The PoolManager uses a Least Recently Used (LRU) policy for discarding old
+pools. That is, if you set the PoolManager ``num_pools`` to 10, then after
+making requests to 11 or more different hosts, the least recently used pools
+will be cleaned up eventually.
+
+Cleanup of stale pools does not happen immediately. You can read more about the
+implementation and the various adjustable variables within
+:class:`~urllib3._collections.RecentlyUsedContainer`.
+
+API
+---
+
+ .. autoclass:: PoolManager
+ :inherited-members:
+
+ProxyManager
+============
+
+:class:`.ProxyManager` is an HTTP proxy-aware subclass of :class:`.PoolManager`.
+It produces a single
+:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP
+connections and individual per-server:port
+:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled
+HTTPS connections.
+
+API
+---
+ .. autoclass:: ProxyManager
+
diff --git a/docs/pools.rst b/docs/pools.rst
new file mode 100644
index 0000000..63cb7d1
--- /dev/null
+++ b/docs/pools.rst
@@ -0,0 +1,71 @@
+ConnectionPools
+===============
+
+.. automodule:: urllib3.connectionpool
+
+A connection pool is a container for a collection of connections to a specific
+host.
+
+If you need to make requests to the same host repeatedly, then you should use a
+:class:`.HTTPConnectionPool`.
+
+.. doctest ::
+
+ >>> from urllib3 import HTTPConnectionPool
+ >>> pool = HTTPConnectionPool('ajax.googleapis.com', maxsize=1)
+ >>> r = pool.request('GET', '/ajax/services/search/web',
+ ... fields={'q': 'urllib3', 'v': '1.0'})
+ >>> r.status
+ 200
+ >>> r.headers['content-type']
+ 'text/javascript; charset=utf-8'
+ >>> 'data: ' + r.data # Content of the response
+ 'data: ...'
+ >>> r = pool.request('GET', '/ajax/services/search/web',
+ ... fields={'q': 'python', 'v': '1.0'})
+ >>> 'data: ' + r.data # Content of the response
+ 'data: ...'
+ >>> pool.num_connections
+ 1
+ >>> pool.num_requests
+ 2
+
+By default, the pool will cache just one connection. If you're planning on using
+such a pool in a multithreaded environment, you should set the ``maxsize`` of
+the pool to a higher number, such as the number of threads. You can also control
+many other variables like timeout, blocking, and default headers.
+
+Helpers
+-------
+
+There are various helper functions provided for instantiating these
+ConnectionPools more easily:
+
+ .. autofunction:: connection_from_url
+
+API
+---
+
+:mod:`urllib3.connectionpool` comes with two connection pools:
+
+ .. autoclass:: HTTPConnectionPool
+ :members:
+ :inherited-members:
+
+ .. autoclass:: HTTPSConnectionPool
+
+
+All of these pools inherit from a common base class:
+
+ .. autoclass:: ConnectionPool
+
+.. module:: urllib3.connection
+
+Related Classes
+---------------
+
+urllib3 implements its own :class:`HTTPConnection` object to allow for more
+flexibility than the standard library's implementation.
+
+.. autoclass:: HTTPConnection
+ :members:
diff --git a/docs/security.rst b/docs/security.rst
new file mode 100644
index 0000000..5321e24
--- /dev/null
+++ b/docs/security.rst
@@ -0,0 +1,161 @@
+.. _security:
+
+Security: Verified HTTPS with SSL/TLS
+=====================================
+
+Very important fact: **By default, urllib3 does not verify HTTPS requests.**
+
+The historic reason for this is that we rely on ``httplib`` for some of the
+HTTP protocol implementation, and ``httplib`` does not verify requests out of
+the box. This is not a good reason, but here we are.
+
+Luckily, it's not too hard to enable verified HTTPS requests and there are a
+few ways to do it.
+
+
+Python with SSL enabled
+-----------------------
+
+First we need to make sure your Python installation has SSL enabled. Easiest
+way to check is to simply open a Python shell and type `import ssl`::
+
+ >>> import ssl
+ Traceback (most recent call last):
+ ...
+ ImportError: No module named _ssl
+
+If you got an ``ImportError``, then your Python is not compiled with SSL support
+and you'll need to re-install it. Read
+`this StackOverflow thread <https://stackoverflow.com/questions/5128845/importerror-no-module-named-ssl>`_
+for details.
+
+Otherwise, if ``ssl`` imported cleanly, then we're ready to setup our certificates:
+:ref:`certifi-with-urllib3`.
+
+
+Enabling SSL on Google AppEngine
+++++++++++++++++++++++++++++++++
+
+If you're using Google App Engine, you'll need to add ``ssl`` as a library
+dependency to your yaml file, like this::
+
+ libraries:
+ - name: ssl
+ version: latest
+
+If it's still not working, you may need to enable billing on your account
+to `enable using sockets
+<https://developers.google.com/appengine/docs/python/sockets/>`_.
+
+
+.. _certifi-with-urllib3:
+
+Using Certifi with urllib3
+--------------------------
+
+`Certifi <http://certifi.io/>`_ is a package which ships with Mozilla's root
+certificates for easy programmatic access.
+
+1. Install the Python ``certifi`` package::
+
+ $ pip install certifi
+
+2. Setup your pool to require a certificate and provide the certifi bundle::
+
+ import urllib3
+ import certifi
+
+ http = urllib3.PoolManager(
+ cert_reqs='CERT_REQUIRED', # Force certificate check.
+ ca_certs=certifi.where(), # Path to the Certifi bundle.
+ )
+
+ # You're ready to make verified HTTPS requests.
+ try:
+ r = http.request('GET', 'https://example.com/')
+ except urllib3.exceptions.SSLError as e:
+ # Handle incorrect certificate error.
+ ...
+
+Make sure to update your ``certifi`` package regularly to get the latest root
+certificates.
+
+
+Using your system's root certificates
+-------------------------------------
+
+Your system's root certificates may be more up-to-date than maintaining your
+own, but the trick is finding where they live. Different operating systems have
+them in different places.
+
+For example, on most Linux distributions they're at
+``/etc/ssl/certs/ca-certificates.crt``. On Windows and OS X? `It's not so simple
+<https://stackoverflow.com/questions/10095676/openssl-reasonable-default-for-trusted-ca-certificates>`_.
+
+Once you find your root certificate file::
+
+ import urllib3
+
+ ca_certs = "/etc/ssl/certs/ca-certificates.crt" # Or wherever it lives.
+
+ http = urllib3.PoolManager(
+ cert_reqs='CERT_REQUIRED', # Force certificate check.
+ ca_certs=ca_certs, # Path to your certificate bundle.
+ )
+
+ # You're ready to make verified HTTPS requests.
+ try:
+ r = http.request('GET', 'https://example.com/')
+ except urllib3.exceptions.SSLError as e:
+ # Handle incorrect certificate error.
+ ...
+
+
+OpenSSL / PyOpenSSL
+-------------------
+
+By default, we use the standard library's ``ssl`` module. Unfortunately, there
+are several limitations which are addressed by PyOpenSSL:
+
+- (Python 2.x) SNI support.
+- (Python 2.x-3.2) Disabling compression to mitigate `CRIME attack
+ <https://en.wikipedia.org/wiki/CRIME_(security_exploit)>`_.
+
+To use the Python OpenSSL bindings instead, you'll need to install the required
+packages::
+
+ $ pip install pyopenssl ndg-httpsclient pyasn1
+
+Once the packages are installed, you can tell urllib3 to switch the ssl backend
+to PyOpenSSL with :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3`::
+
+ import urllib3.contrib.pyopenssl
+ urllib3.contrib.pyopenssl.inject_into_urllib3()
+
+Now you can continue using urllib3 as you normally would.
+
+For more details, check the :mod:`~urllib3.contrib.pyopenssl` module.
+
+
+InsecureRequestWarning
+----------------------
+
+.. versionadded:: 1.9
+
+Unverified HTTPS requests will trigger a warning::
+
+ urllib3/connectionpool.py:736: InsecureRequestWarning: Unverified HTTPS
+ request is being made. Adding certificate verification is strongly advised.
+ See: https://urllib3.readthedocs.org/en/latest/security.html
+ (This warning will only appear once by default.)
+
+This would be a great time to enable HTTPS verification:
+:ref:`certifi-with-urllib3`.
+
+If you know what you're doing and would like to disable this and other warnings,
+you can use :func:`~urllib3.disable_warnings`::
+
+ import urllib3
+ urllib3.disable_warnings()
+
+Making unverified HTTPS requests is strongly discouraged. ˙ ͜ʟ˙
diff --git a/dummyserver/__init__.pyc b/dummyserver/__init__.pyc
new file mode 100644
index 0000000..24e9f56
--- /dev/null
+++ b/dummyserver/__init__.pyc
Binary files differ
diff --git a/dummyserver/handlers.pyc b/dummyserver/handlers.pyc
new file mode 100644
index 0000000..22aedc3
--- /dev/null
+++ b/dummyserver/handlers.pyc
Binary files differ
diff --git a/dummyserver/proxy.pyc b/dummyserver/proxy.pyc
new file mode 100644
index 0000000..23fa01d
--- /dev/null
+++ b/dummyserver/proxy.pyc
Binary files differ
diff --git a/dummyserver/server.pyc b/dummyserver/server.pyc
new file mode 100644
index 0000000..b997d0e
--- /dev/null
+++ b/dummyserver/server.pyc
Binary files differ
diff --git a/dummyserver/testcase.pyc b/dummyserver/testcase.pyc
new file mode 100644
index 0000000..29cc06a
--- /dev/null
+++ b/dummyserver/testcase.pyc
Binary files differ
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..d56a4d3
--- /dev/null
+++ b/test/__init__.py
@@ -0,0 +1,92 @@
+import warnings
+import sys
+import errno
+import functools
+import socket
+
+from nose.plugins.skip import SkipTest
+
+from urllib3.exceptions import MaxRetryError, HTTPWarning
+from urllib3.packages import six
+
+# We need a host that will not immediately close the connection with a TCP
+# Reset. SO suggests this hostname
+TARPIT_HOST = '10.255.255.1'
+
+VALID_SOURCE_ADDRESSES = [('::1', 0), ('127.0.0.1', 0)]
+# RFC 5737: 192.0.2.0/24 is for testing only.
+# RFC 3849: 2001:db8::/32 is for documentation only.
+INVALID_SOURCE_ADDRESSES = [('192.0.2.255', 0), ('2001:db8::1', 0)]
+
+
+def clear_warnings(cls=HTTPWarning):
+ new_filters = []
+ for f in warnings.filters:
+ if issubclass(f[2], cls):
+ continue
+ new_filters.append(f)
+ warnings.filters[:] = new_filters
+
+def setUp():
+ clear_warnings()
+ warnings.simplefilter('ignore', HTTPWarning)
+
+
+def onlyPy26OrOlder(test):
+ """Skips this test unless you are on Python2.6.x or earlier."""
+
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "{name} only runs on Python2.6.x or older".format(name=test.__name__)
+ if sys.version_info >= (2, 7):
+ raise SkipTest(msg)
+ return test(*args, **kwargs)
+ return wrapper
+
+def onlyPy27OrNewer(test):
+ """Skips this test unless you are on Python 2.7.x or later."""
+
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "{name} requires Python 2.7.x+ to run".format(name=test.__name__)
+ if sys.version_info < (2, 7):
+ raise SkipTest(msg)
+ return test(*args, **kwargs)
+ return wrapper
+
+def onlyPy3(test):
+ """Skips this test unless you are on Python3.x"""
+
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "{name} requires Python3.x to run".format(name=test.__name__)
+ if not six.PY3:
+ raise SkipTest(msg)
+ return test(*args, **kwargs)
+ return wrapper
+
+def requires_network(test):
+ """Helps you skip tests that require the network"""
+
+ def _is_unreachable_err(err):
+ return getattr(err, 'errno', None) in (errno.ENETUNREACH,
+ errno.EHOSTUNREACH) # For OSX
+
+ @functools.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "Can't run {name} because the network is unreachable".format(
+ name=test.__name__)
+ try:
+ return test(*args, **kwargs)
+ except socket.error as e:
+ # This test needs an initial network connection to attempt the
+ # connection to the TARPIT_HOST. This fails if you are in a place
+ # without an Internet connection, so we skip the test in that case.
+ if _is_unreachable_err(e):
+ raise SkipTest(msg)
+ raise
+ except MaxRetryError as e:
+ if _is_unreachable_err(e.reason):
+ raise SkipTest(msg)
+ raise
+ return wrapper
diff --git a/test/__init__.pyc b/test/__init__.pyc
new file mode 100644
index 0000000..38b9317
--- /dev/null
+++ b/test/__init__.pyc
Binary files differ
diff --git a/test/benchmark.py b/test/benchmark.py
new file mode 100644
index 0000000..242e72f
--- /dev/null
+++ b/test/benchmark.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+"""
+Really simple rudimentary benchmark to compare ConnectionPool versus standard
+urllib to demonstrate the usefulness of connection re-using.
+"""
+from __future__ import print_function
+
+import sys
+import time
+import urllib
+
+sys.path.append('../')
+import urllib3
+
+
+# URLs to download. Doesn't matter as long as they're from the same host, so we
+# can take advantage of connection re-using.
+TO_DOWNLOAD = [
+ 'http://code.google.com/apis/apps/',
+ 'http://code.google.com/apis/base/',
+ 'http://code.google.com/apis/blogger/',
+ 'http://code.google.com/apis/calendar/',
+ 'http://code.google.com/apis/codesearch/',
+ 'http://code.google.com/apis/contact/',
+ 'http://code.google.com/apis/books/',
+ 'http://code.google.com/apis/documents/',
+ 'http://code.google.com/apis/finance/',
+ 'http://code.google.com/apis/health/',
+ 'http://code.google.com/apis/notebook/',
+ 'http://code.google.com/apis/picasaweb/',
+ 'http://code.google.com/apis/spreadsheets/',
+ 'http://code.google.com/apis/webmastertools/',
+ 'http://code.google.com/apis/youtube/',
+]
+
+
+def urllib_get(url_list):
+ assert url_list
+ for url in url_list:
+ now = time.time()
+ r = urllib.urlopen(url)
+ elapsed = time.time() - now
+ print("Got in %0.3f: %s" % (elapsed, url))
+
+
+def pool_get(url_list):
+ assert url_list
+ pool = urllib3.PoolManager()
+ for url in url_list:
+ now = time.time()
+ r = pool.request('GET', url, assert_same_host=False)
+ elapsed = time.time() - now
+ print("Got in %0.3fs: %s" % (elapsed, url))
+
+
+if __name__ == '__main__':
+ print("Running pool_get ...")
+ now = time.time()
+ pool_get(TO_DOWNLOAD)
+ pool_elapsed = time.time() - now
+
+ print("Running urllib_get ...")
+ now = time.time()
+ urllib_get(TO_DOWNLOAD)
+ urllib_elapsed = time.time() - now
+
+ print("Completed pool_get in %0.3fs" % pool_elapsed)
+ print("Completed urllib_get in %0.3fs" % urllib_elapsed)
+
+
+"""
+Example results:
+
+Completed pool_get in 1.163s
+Completed urllib_get in 2.318s
+"""
diff --git a/test/contrib/__init__.py b/test/contrib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/contrib/__init__.py
diff --git a/test/contrib/__init__.pyc b/test/contrib/__init__.pyc
new file mode 100644
index 0000000..2d2fd5d
--- /dev/null
+++ b/test/contrib/__init__.pyc
Binary files differ
diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py
new file mode 100644
index 0000000..5d57527
--- /dev/null
+++ b/test/contrib/test_pyopenssl.py
@@ -0,0 +1,23 @@
+from nose.plugins.skip import SkipTest
+from urllib3.packages import six
+
+if six.PY3:
+ raise SkipTest('Testing of PyOpenSSL disabled on PY3')
+
+try:
+ from urllib3.contrib.pyopenssl import (inject_into_urllib3,
+ extract_from_urllib3)
+except ImportError as e:
+ raise SkipTest('Could not import PyOpenSSL: %r' % e)
+
+
+from ..with_dummyserver.test_https import TestHTTPS, TestHTTPS_TLSv1
+from ..with_dummyserver.test_socketlevel import TestSNI, TestSocketClosing
+
+
+def setup_module():
+ inject_into_urllib3()
+
+
+def teardown_module():
+ extract_from_urllib3()
diff --git a/test/contrib/test_pyopenssl.pyc b/test/contrib/test_pyopenssl.pyc
new file mode 100644
index 0000000..6441273
--- /dev/null
+++ b/test/contrib/test_pyopenssl.pyc
Binary files differ
diff --git a/test/port_helpers.py b/test/port_helpers.py
new file mode 100644
index 0000000..e818a9b
--- /dev/null
+++ b/test/port_helpers.py
@@ -0,0 +1,100 @@
+# These helpers are copied from test_support.py in the Python 2.7 standard
+# library test suite.
+
+import socket
+
+
+# Don't use "localhost", since resolving it uses the DNS under recent
+# Windows versions (see issue #18792).
+HOST = "127.0.0.1"
+HOSTv6 = "::1"
+
+def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
+ """Returns an unused port that should be suitable for binding. This is
+ achieved by creating a temporary socket with the same family and type as
+ the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to
+ the specified host address (defaults to 0.0.0.0) with the port set to 0,
+ eliciting an unused ephemeral port from the OS. The temporary socket is
+ then closed and deleted, and the ephemeral port is returned.
+
+ Either this method or bind_port() should be used for any tests where a
+ server socket needs to be bound to a particular port for the duration of
+ the test. Which one to use depends on whether the calling code is creating
+ a python socket, or if an unused port needs to be provided in a constructor
+ or passed to an external program (i.e. the -accept argument to openssl's
+ s_server mode). Always prefer bind_port() over find_unused_port() where
+ possible. Hard coded ports should *NEVER* be used. As soon as a server
+ socket is bound to a hard coded port, the ability to run multiple instances
+ of the test simultaneously on the same host is compromised, which makes the
+ test a ticking time bomb in a buildbot environment. On Unix buildbots, this
+ may simply manifest as a failed test, which can be recovered from without
+ intervention in most cases, but on Windows, the entire python process can
+ completely and utterly wedge, requiring someone to log in to the buildbot
+ and manually kill the affected process.
+
+ (This is easy to reproduce on Windows, unfortunately, and can be traced to
+ the SO_REUSEADDR socket option having different semantics on Windows versus
+ Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind,
+ listen and then accept connections on identical host/ports. An EADDRINUSE
+ socket.error will be raised at some point (depending on the platform and
+ the order bind and listen were called on each socket).
+
+ However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE
+ will ever be raised when attempting to bind two identical host/ports. When
+ accept() is called on each socket, the second caller's process will steal
+ the port from the first caller, leaving them both in an awkwardly wedged
+ state where they'll no longer respond to any signals or graceful kills, and
+ must be forcibly killed via OpenProcess()/TerminateProcess().
+
+ The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option
+ instead of SO_REUSEADDR, which effectively affords the same semantics as
+ SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open
+ Source world compared to Windows ones, this is a common mistake. A quick
+ look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when
+ openssl.exe is called with the 's_server' option, for example. See
+ http://bugs.python.org/issue2550 for more info. The following site also
+ has a very thorough description about the implications of both REUSEADDR
+ and EXCLUSIVEADDRUSE on Windows:
+ http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx)
+
+ XXX: although this approach is a vast improvement on previous attempts to
+ elicit unused ports, it rests heavily on the assumption that the ephemeral
+ port returned to us by the OS won't immediately be dished back out to some
+ other process when we close and delete our temporary socket but before our
+ calling code has a chance to bind the returned port. We can deal with this
+ issue if/when we come across it."""
+ tempsock = socket.socket(family, socktype)
+ port = bind_port(tempsock)
+ tempsock.close()
+ del tempsock
+ return port
+
+def bind_port(sock, host=HOST):
+ """Bind the socket to a free port and return the port number. Relies on
+ ephemeral ports in order to ensure we are using an unbound port. This is
+ important as many tests may be running simultaneously, especially in a
+ buildbot environment. This method raises an exception if the sock.family
+ is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
+ or SO_REUSEPORT set on it. Tests should *never* set these socket options
+ for TCP/IP sockets. The only case for setting these options is testing
+ multicasting via multiple UDP sockets.
+
+ Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
+ on Windows), it will be set on the socket. This will prevent anyone else
+ from bind()'ing to our host/port for the duration of the test.
+ """
+ if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
+ if hasattr(socket, 'SO_REUSEADDR'):
+ if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
+ raise ValueError("tests should never set the SO_REUSEADDR " \
+ "socket option on TCP/IP sockets!")
+ if hasattr(socket, 'SO_REUSEPORT'):
+ if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
+ raise ValueError("tests should never set the SO_REUSEPORT " \
+ "socket option on TCP/IP sockets!")
+ if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
+
+ sock.bind((host, 0))
+ port = sock.getsockname()[1]
+ return port
diff --git a/test/port_helpers.pyc b/test/port_helpers.pyc
new file mode 100644
index 0000000..7a1c425
--- /dev/null
+++ b/test/port_helpers.pyc
Binary files differ
diff --git a/test/test_collections.pyc b/test/test_collections.pyc
new file mode 100644
index 0000000..d1ecd73
--- /dev/null
+++ b/test/test_collections.pyc
Binary files differ
diff --git a/test/test_compatibility.pyc b/test/test_compatibility.pyc
new file mode 100644
index 0000000..2dfdf75
--- /dev/null
+++ b/test/test_compatibility.pyc
Binary files differ
diff --git a/test/test_connectionpool.pyc b/test/test_connectionpool.pyc
new file mode 100644
index 0000000..e87a3b3
--- /dev/null
+++ b/test/test_connectionpool.pyc
Binary files differ
diff --git a/test/test_exceptions.pyc b/test/test_exceptions.pyc
new file mode 100644
index 0000000..3274e34
--- /dev/null
+++ b/test/test_exceptions.pyc
Binary files differ
diff --git a/test/test_fields.pyc b/test/test_fields.pyc
new file mode 100644
index 0000000..4622899
--- /dev/null
+++ b/test/test_fields.pyc
Binary files differ
diff --git a/test/test_filepost.pyc b/test/test_filepost.pyc
new file mode 100644
index 0000000..ec54472
--- /dev/null
+++ b/test/test_filepost.pyc
Binary files differ
diff --git a/test/test_poolmanager.pyc b/test/test_poolmanager.pyc
new file mode 100644
index 0000000..077c2ac
--- /dev/null
+++ b/test/test_poolmanager.pyc
Binary files differ
diff --git a/test/test_proxymanager.pyc b/test/test_proxymanager.pyc
new file mode 100644
index 0000000..3696ee8
--- /dev/null
+++ b/test/test_proxymanager.pyc
Binary files differ
diff --git a/test/test_response.py b/test/test_response.py
index ad134ee..7d67c93 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -182,6 +182,37 @@ class TestResponse(unittest.TestCase):
br.close()
self.assertEqual(resp.closed, True)
+ b = b'fooandahalf'
+ fp = BytesIO(b)
+ resp = HTTPResponse(fp, preload_content=False)
+ br = BufferedReader(resp, 5)
+
+ br.read(1) # sets up the buffer, reading 5
+ self.assertEqual(len(fp.read()), len(b) - 5)
+
+ # This is necessary to make sure the "no bytes left" part of `readinto`
+ # gets tested.
+ while not br.closed:
+ br.read(5)
+
+ def test_io_readinto(self):
+ # This test is necessary because in py2.6, `readinto` doesn't get called
+ # in `test_io_bufferedreader` like it does for all the other python
+ # versions. Probably this is because the `io` module in py2.6 is an
+ # old version that has a different underlying implementation.
+
+
+ fp = BytesIO(b'foo')
+ resp = HTTPResponse(fp, preload_content=False)
+
+ barr = bytearray(3)
+ assert resp.readinto(barr) == 3
+ assert b'foo' == barr
+
+ # The reader should already be empty, so this should read nothing.
+ assert resp.readinto(barr) == 0
+ assert b'foo' == barr
+
def test_streaming(self):
fp = BytesIO(b'foo')
resp = HTTPResponse(fp, preload_content=False)
diff --git a/test/test_response.pyc b/test/test_response.pyc
new file mode 100644
index 0000000..99e5c0e
--- /dev/null
+++ b/test/test_response.pyc
Binary files differ
diff --git a/test/test_retry.pyc b/test/test_retry.pyc
new file mode 100644
index 0000000..398c010
--- /dev/null
+++ b/test/test_retry.pyc
Binary files differ
diff --git a/test/test_util.py b/test/test_util.py
index 388d877..1811dbd 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -21,6 +21,8 @@ from urllib3.exceptions import (
InsecureRequestWarning,
)
+from urllib3.util import is_fp_closed
+
from . import clear_warnings
# This number represents a time in seconds, it doesn't mean anything in
@@ -324,3 +326,32 @@ class TestUtil(unittest.TestCase):
self.assertEqual(resolve_cert_reqs('REQUIRED'), ssl.CERT_REQUIRED)
self.assertEqual(resolve_cert_reqs('CERT_REQUIRED'), ssl.CERT_REQUIRED)
+ def test_is_fp_closed_object_supports_closed(self):
+ class ClosedFile(object):
+ @property
+ def closed(self):
+ return True
+
+ self.assertTrue(is_fp_closed(ClosedFile()))
+
+ def test_is_fp_closed_object_has_none_fp(self):
+ class NoneFpFile(object):
+ @property
+ def fp(self):
+ return None
+
+ self.assertTrue(is_fp_closed(NoneFpFile()))
+
+ def test_is_fp_closed_object_has_fp(self):
+ class FpFile(object):
+ @property
+ def fp(self):
+ return True
+
+ self.assertTrue(not is_fp_closed(FpFile()))
+
+ def test_is_fp_closed_object_has_neither_fp_nor_closed(self):
+ class NotReallyAFile(object):
+ pass
+
+ self.assertRaises(ValueError, is_fp_closed, NotReallyAFile())
diff --git a/test/test_util.pyc b/test/test_util.pyc
new file mode 100644
index 0000000..0500c3b
--- /dev/null
+++ b/test/test_util.pyc
Binary files differ
diff --git a/test/with_dummyserver/__init__.py b/test/with_dummyserver/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/with_dummyserver/__init__.py
diff --git a/test/with_dummyserver/__init__.pyc b/test/with_dummyserver/__init__.pyc
new file mode 100644
index 0000000..833be60
--- /dev/null
+++ b/test/with_dummyserver/__init__.pyc
Binary files differ
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py
new file mode 100644
index 0000000..7d54fbf
--- /dev/null
+++ b/test/with_dummyserver/test_connectionpool.py
@@ -0,0 +1,706 @@
+import errno
+import logging
+import socket
+import sys
+import unittest
+import time
+
+import mock
+
+try:
+ from urllib.parse import urlencode
+except:
+ from urllib import urlencode
+
+from .. import (
+ requires_network,
+ onlyPy3, onlyPy27OrNewer, onlyPy26OrOlder,
+ TARPIT_HOST, VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES,
+)
+from ..port_helpers import find_unused_port
+from urllib3 import (
+ encode_multipart_formdata,
+ HTTPConnectionPool,
+)
+from urllib3.exceptions import (
+ ConnectTimeoutError,
+ EmptyPoolError,
+ DecodeError,
+ MaxRetryError,
+ ReadTimeoutError,
+ ProtocolError,
+)
+from urllib3.packages.six import b, u
+from urllib3.util.retry import Retry
+from urllib3.util.timeout import Timeout
+
+import tornado
+from dummyserver.testcase import HTTPDummyServerTestCase
+
+from nose.tools import timed
+
+log = logging.getLogger('urllib3.connectionpool')
+log.setLevel(logging.NOTSET)
+log.addHandler(logging.StreamHandler(sys.stdout))
+
+
+class TestConnectionPool(HTTPDummyServerTestCase):
+
+ def setUp(self):
+ self.pool = HTTPConnectionPool(self.host, self.port)
+
+ def test_get(self):
+ r = self.pool.request('GET', '/specific_method',
+ fields={'method': 'GET'})
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_post_url(self):
+ r = self.pool.request('POST', '/specific_method',
+ fields={'method': 'POST'})
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_urlopen_put(self):
+ r = self.pool.urlopen('PUT', '/specific_method?method=PUT')
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_wrong_specific_method(self):
+ # To make sure the dummy server is actually returning failed responses
+ r = self.pool.request('GET', '/specific_method',
+ fields={'method': 'POST'})
+ self.assertEqual(r.status, 400, r.data)
+
+ r = self.pool.request('POST', '/specific_method',
+ fields={'method': 'GET'})
+ self.assertEqual(r.status, 400, r.data)
+
+ def test_upload(self):
+ data = "I'm in ur multipart form-data, hazing a cheezburgr"
+ fields = {
+ 'upload_param': 'filefield',
+ 'upload_filename': 'lolcat.txt',
+ 'upload_size': len(data),
+ 'filefield': ('lolcat.txt', data),
+ }
+
+ r = self.pool.request('POST', '/upload', fields=fields)
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_one_name_multiple_values(self):
+ fields = [
+ ('foo', 'a'),
+ ('foo', 'b'),
+ ]
+
+ # urlencode
+ r = self.pool.request('GET', '/echo', fields=fields)
+ self.assertEqual(r.data, b'foo=a&foo=b')
+
+ # multipart
+ r = self.pool.request('POST', '/echo', fields=fields)
+ self.assertEqual(r.data.count(b'name="foo"'), 2)
+
+
+ def test_unicode_upload(self):
+ fieldname = u('myfile')
+ filename = u('\xe2\x99\xa5.txt')
+ data = u('\xe2\x99\xa5').encode('utf8')
+ size = len(data)
+
+ fields = {
+ u('upload_param'): fieldname,
+ u('upload_filename'): filename,
+ u('upload_size'): size,
+ fieldname: (filename, data),
+ }
+
+ r = self.pool.request('POST', '/upload', fields=fields)
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_timeout_float(self):
+ url = '/sleep?seconds=0.005'
+ # Pool-global timeout
+ pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
+ self.assertRaises(ReadTimeoutError, pool.request, 'GET', url)
+
+ def test_conn_closed(self):
+ pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
+ conn = pool._get_conn()
+ pool._put_conn(conn)
+ try:
+ url = '/sleep?seconds=0.005'
+ pool.urlopen('GET', url)
+ self.fail("The request should fail with a timeout error.")
+ except ReadTimeoutError:
+ if conn.sock:
+ self.assertRaises(socket.error, conn.sock.recv, 1024)
+ finally:
+ pool._put_conn(conn)
+
+ def test_nagle(self):
+ """ Test that connections have TCP_NODELAY turned on """
+ # This test needs to be here in order to be run. socket.create_connection actually tries to
+ # connect to the host provided so we need a dummyserver to be running.
+ pool = HTTPConnectionPool(self.host, self.port)
+ conn = pool._get_conn()
+ pool._make_request(conn, 'GET', '/')
+ tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
+ assert tcp_nodelay_setting > 0, ("Expected TCP_NODELAY to be set on the "
+ "socket (with value greater than 0) "
+ "but instead was %s" %
+ tcp_nodelay_setting)
+
+ def test_socket_options(self):
+ """Test that connections accept socket options."""
+ # This test needs to be here in order to be run. socket.create_connection actually tries to
+ # connect to the host provided so we need a dummyserver to be running.
+ pool = HTTPConnectionPool(self.host, self.port, socket_options=[
+ (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+ ])
+ s = pool._new_conn()._new_conn() # Get the socket
+ using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
+ self.assertTrue(using_keepalive)
+ s.close()
+
+ def test_disable_default_socket_options(self):
+ """Test that passing None disables all socket options."""
+ # This test needs to be here in order to be run. socket.create_connection actually tries to
+ # connect to the host provided so we need a dummyserver to be running.
+ pool = HTTPConnectionPool(self.host, self.port, socket_options=None)
+ s = pool._new_conn()._new_conn()
+ using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0
+ self.assertTrue(using_nagle)
+ s.close()
+
+ def test_defaults_are_applied(self):
+ """Test that modifying the default socket options works."""
+ # This test needs to be here in order to be run. socket.create_connection actually tries to
+ # connect to the host provided so we need a dummyserver to be running.
+ pool = HTTPConnectionPool(self.host, self.port)
+ # Get the HTTPConnection instance
+ conn = pool._new_conn()
+ # Update the default socket options
+ conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
+ s = conn._new_conn()
+ nagle_disabled = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0
+ using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
+ self.assertTrue(nagle_disabled)
+ self.assertTrue(using_keepalive)
+
+ @timed(0.5)
+ def test_timeout(self):
+ """ Requests should time out when expected """
+ url = '/sleep?seconds=0.002'
+ timeout = Timeout(read=0.001)
+
+ # Pool-global timeout
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False)
+
+ conn = pool._get_conn()
+ self.assertRaises(ReadTimeoutError, pool._make_request,
+ conn, 'GET', url)
+ pool._put_conn(conn)
+
+ time.sleep(0.02) # Wait for server to start receiving again. :(
+
+ self.assertRaises(ReadTimeoutError, pool.request, 'GET', url)
+
+ # Request-specific timeouts should raise errors
+ pool = HTTPConnectionPool(self.host, self.port, timeout=0.1, retries=False)
+
+ conn = pool._get_conn()
+ self.assertRaises(ReadTimeoutError, pool._make_request,
+ conn, 'GET', url, timeout=timeout)
+ pool._put_conn(conn)
+
+ time.sleep(0.02) # Wait for server to start receiving again. :(
+
+ self.assertRaises(ReadTimeoutError, pool.request,
+ 'GET', url, timeout=timeout)
+
+ # Timeout int/float passed directly to request and _make_request should
+ # raise a request timeout
+ self.assertRaises(ReadTimeoutError, pool.request,
+ 'GET', url, timeout=0.001)
+ conn = pool._new_conn()
+ self.assertRaises(ReadTimeoutError, pool._make_request, conn,
+ 'GET', url, timeout=0.001)
+ pool._put_conn(conn)
+
+ # Timeout int/float passed directly to _make_request should not raise a
+ # request timeout if it's a high value
+ pool.request('GET', url, timeout=1)
+
+ @requires_network
+ @timed(0.5)
+ def test_connect_timeout(self):
+ url = '/sleep?seconds=0.005'
+ timeout = Timeout(connect=0.001)
+
+ # Pool-global timeout
+ pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url)
+
+ # Retries
+ retries = Retry(connect=0)
+ self.assertRaises(MaxRetryError, pool.request, 'GET', url,
+ retries=retries)
+
+ # Request-specific connection timeouts
+ big_timeout = Timeout(read=0.2, connect=0.2)
+ pool = HTTPConnectionPool(TARPIT_HOST, self.port,
+ timeout=big_timeout, retries=False)
+ conn = pool._get_conn()
+ self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET',
+ url, timeout=timeout)
+
+ pool._put_conn(conn)
+ self.assertRaises(ConnectTimeoutError, pool.request, 'GET', url,
+ timeout=timeout)
+
+
+ def test_connection_error_retries(self):
+ """ ECONNREFUSED error should raise a connection error, with retries """
+ port = find_unused_port()
+ pool = HTTPConnectionPool(self.host, port)
+ try:
+ pool.request('GET', '/', retries=Retry(connect=3))
+ self.fail("Should have failed with a connection error.")
+ except MaxRetryError as e:
+ self.assertTrue(isinstance(e.reason, ProtocolError))
+ self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
+
+ def test_timeout_reset(self):
+ """ If the read timeout isn't set, socket timeout should reset """
+ url = '/sleep?seconds=0.005'
+ timeout = Timeout(connect=0.001)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ try:
+ pool._make_request(conn, 'GET', url)
+ except ReadTimeoutError:
+ self.fail("This request shouldn't trigger a read timeout.")
+
+ @requires_network
+ @timed(5.0)
+ def test_total_timeout(self):
+ url = '/sleep?seconds=0.005'
+
+ timeout = Timeout(connect=3, read=5, total=0.001)
+ pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url)
+
+ # This will get the socket to raise an EAGAIN on the read
+ timeout = Timeout(connect=3, read=0)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url)
+
+ # The connect should succeed and this should hit the read timeout
+ timeout = Timeout(connect=3, read=5, total=0.002)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url)
+
+ @requires_network
+ def test_none_total_applies_connect(self):
+ url = '/sleep?seconds=0.005'
+ timeout = Timeout(total=None, connect=0.001)
+ pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ self.assertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET',
+ url)
+
+ def test_timeout_success(self):
+ timeout = Timeout(connect=3, read=5, total=None)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ pool.request('GET', '/')
+ # This should not raise a "Timeout already started" error
+ pool.request('GET', '/')
+
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ # This should also not raise a "Timeout already started" error
+ pool.request('GET', '/')
+
+ timeout = Timeout(total=None)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ pool.request('GET', '/')
+
+ def test_tunnel(self):
+ # note the actual httplib.py has no tests for this functionality
+ timeout = Timeout(total=None)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ conn = pool._get_conn()
+ try:
+ conn.set_tunnel(self.host, self.port)
+ except AttributeError: # python 2.6
+ conn._set_tunnel(self.host, self.port)
+
+ conn._tunnel = mock.Mock(return_value=None)
+ pool._make_request(conn, 'GET', '/')
+ conn._tunnel.assert_called_once_with()
+
+ # test that it's not called when tunnel is not set
+ timeout = Timeout(total=None)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
+ conn = pool._get_conn()
+
+ conn._tunnel = mock.Mock(return_value=None)
+ pool._make_request(conn, 'GET', '/')
+ self.assertEqual(conn._tunnel.called, False)
+
+ def test_redirect(self):
+ r = self.pool.request('GET', '/redirect', fields={'target': '/'}, redirect=False)
+ self.assertEqual(r.status, 303)
+
+ r = self.pool.request('GET', '/redirect', fields={'target': '/'})
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_bad_connect(self):
+ pool = HTTPConnectionPool('badhost.invalid', self.port)
+ try:
+ pool.request('GET', '/', retries=5)
+ self.fail("should raise timeout exception here")
+ except MaxRetryError as e:
+ self.assertTrue(isinstance(e.reason, ProtocolError), e.reason)
+
+ def test_keepalive(self):
+ pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1)
+
+ r = pool.request('GET', '/keepalive?close=0')
+ r = pool.request('GET', '/keepalive?close=0')
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(pool.num_connections, 1)
+ self.assertEqual(pool.num_requests, 2)
+
+ def test_keepalive_close(self):
+ pool = HTTPConnectionPool(self.host, self.port,
+ block=True, maxsize=1, timeout=2)
+
+ r = pool.request('GET', '/keepalive?close=1', retries=0,
+ headers={
+ "Connection": "close",
+ })
+
+ self.assertEqual(pool.num_connections, 1)
+
+ # The dummyserver will have responded with Connection:close,
+ # and httplib will properly cleanup the socket.
+
+ # We grab the HTTPConnection object straight from the Queue,
+ # because _get_conn() is where the check & reset occurs
+ # pylint: disable-msg=W0212
+ conn = pool.pool.get()
+ self.assertEqual(conn.sock, None)
+ pool._put_conn(conn)
+
+ # Now with keep-alive
+ r = pool.request('GET', '/keepalive?close=0', retries=0,
+ headers={
+ "Connection": "keep-alive",
+ })
+
+ # The dummyserver responded with Connection:keep-alive, the connection
+ # persists.
+ conn = pool.pool.get()
+ self.assertNotEqual(conn.sock, None)
+ pool._put_conn(conn)
+
+ # Another request asking the server to close the connection. This one
+ # should get cleaned up for the next request.
+ r = pool.request('GET', '/keepalive?close=1', retries=0,
+ headers={
+ "Connection": "close",
+ })
+
+ self.assertEqual(r.status, 200)
+
+ conn = pool.pool.get()
+ self.assertEqual(conn.sock, None)
+ pool._put_conn(conn)
+
+ # Next request
+ r = pool.request('GET', '/keepalive?close=0')
+
+ def test_post_with_urlencode(self):
+ data = {'banana': 'hammock', 'lol': 'cat'}
+ r = self.pool.request('POST', '/echo', fields=data, encode_multipart=False)
+ self.assertEqual(r.data.decode('utf-8'), urlencode(data))
+
+ def test_post_with_multipart(self):
+ data = {'banana': 'hammock', 'lol': 'cat'}
+ r = self.pool.request('POST', '/echo',
+ fields=data,
+ encode_multipart=True)
+ body = r.data.split(b'\r\n')
+
+ encoded_data = encode_multipart_formdata(data)[0]
+ expected_body = encoded_data.split(b'\r\n')
+
+ # TODO: Get rid of extra parsing stuff when you can specify
+ # a custom boundary to encode_multipart_formdata
+ """
+ We need to loop the return lines because a timestamp is attached
+ from within encode_multipart_formdata. When the server echos back
+ the data, it has the timestamp from when the data was encoded, which
+ is not equivalent to when we run encode_multipart_formdata on
+ the data again.
+ """
+ for i, line in enumerate(body):
+ if line.startswith(b'--'):
+ continue
+
+ self.assertEqual(body[i], expected_body[i])
+
+ def test_check_gzip(self):
+ r = self.pool.request('GET', '/encodingrequest',
+ headers={'accept-encoding': 'gzip'})
+ self.assertEqual(r.headers.get('content-encoding'), 'gzip')
+ self.assertEqual(r.data, b'hello, world!')
+
+ def test_check_deflate(self):
+ r = self.pool.request('GET', '/encodingrequest',
+ headers={'accept-encoding': 'deflate'})
+ self.assertEqual(r.headers.get('content-encoding'), 'deflate')
+ self.assertEqual(r.data, b'hello, world!')
+
+ def test_bad_decode(self):
+ self.assertRaises(DecodeError, self.pool.request,
+ 'GET', '/encodingrequest',
+ headers={'accept-encoding': 'garbage-deflate'})
+
+ self.assertRaises(DecodeError, self.pool.request,
+ 'GET', '/encodingrequest',
+ headers={'accept-encoding': 'garbage-gzip'})
+
+ def test_connection_count(self):
+ pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
+
+ pool.request('GET', '/')
+ pool.request('GET', '/')
+ pool.request('GET', '/')
+
+ self.assertEqual(pool.num_connections, 1)
+ self.assertEqual(pool.num_requests, 3)
+
+ def test_connection_count_bigpool(self):
+ http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16)
+
+ http_pool.request('GET', '/')
+ http_pool.request('GET', '/')
+ http_pool.request('GET', '/')
+
+ self.assertEqual(http_pool.num_connections, 1)
+ self.assertEqual(http_pool.num_requests, 3)
+
+ def test_partial_response(self):
+ pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
+
+ req_data = {'lol': 'cat'}
+ resp_data = urlencode(req_data).encode('utf-8')
+
+ r = pool.request('GET', '/echo', fields=req_data, preload_content=False)
+
+ self.assertEqual(r.read(5), resp_data[:5])
+ self.assertEqual(r.read(), resp_data[5:])
+
+ def test_lazy_load_twice(self):
+ # This test is sad and confusing. Need to figure out what's
+ # going on with partial reads and socket reuse.
+
+ pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
+
+ payload_size = 1024 * 2
+ first_chunk = 512
+
+ boundary = 'foo'
+
+ req_data = {'count': 'a' * payload_size}
+ resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0]
+
+ req2_data = {'count': 'b' * payload_size}
+ resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0]
+
+ r1 = pool.request('POST', '/echo', fields=req_data, multipart_boundary=boundary, preload_content=False)
+
+ self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk])
+
+ try:
+ r2 = pool.request('POST', '/echo', fields=req2_data, multipart_boundary=boundary,
+ preload_content=False, pool_timeout=0.001)
+
+ # This branch should generally bail here, but maybe someday it will
+ # work? Perhaps by some sort of magic. Consider it a TODO.
+
+ self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk])
+
+ self.assertEqual(r1.read(), resp_data[first_chunk:])
+ self.assertEqual(r2.read(), resp2_data[first_chunk:])
+ self.assertEqual(pool.num_requests, 2)
+
+ except EmptyPoolError:
+ self.assertEqual(r1.read(), resp_data[first_chunk:])
+ self.assertEqual(pool.num_requests, 1)
+
+ self.assertEqual(pool.num_connections, 1)
+
+ def test_for_double_release(self):
+ MAXSIZE=5
+
+ # Check default state
+ pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
+ self.assertEqual(pool.num_connections, 0)
+ self.assertEqual(pool.pool.qsize(), MAXSIZE)
+
+ # Make an empty slot for testing
+ pool.pool.get()
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
+
+ # Check state after simple request
+ pool.urlopen('GET', '/')
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
+
+ # Check state without release
+ pool.urlopen('GET', '/', preload_content=False)
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
+
+ pool.urlopen('GET', '/')
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
+
+ # Check state after read
+ pool.urlopen('GET', '/').data
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
+
+ pool.urlopen('GET', '/')
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
+
+ def test_release_conn_parameter(self):
+ MAXSIZE=5
+ pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
+ self.assertEqual(pool.pool.qsize(), MAXSIZE)
+
+ # Make request without releasing connection
+ pool.request('GET', '/', release_conn=False, preload_content=False)
+ self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
+
+ def test_dns_error(self):
+ pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001)
+ self.assertRaises(MaxRetryError, pool.request, 'GET', '/test', retries=2)
+
+ def test_source_address(self):
+ for addr in VALID_SOURCE_ADDRESSES:
+ pool = HTTPConnectionPool(self.host, self.port,
+ source_address=addr, retries=False)
+ r = pool.request('GET', '/source_address')
+ assert r.data == b(addr[0]), (
+ "expected the response to contain the source address {addr}, "
+ "but was {data}".format(data=r.data, addr=b(addr[0])))
+
+ def test_source_address_error(self):
+ for addr in INVALID_SOURCE_ADDRESSES:
+ pool = HTTPConnectionPool(self.host, self.port,
+ source_address=addr, retries=False)
+ self.assertRaises(ProtocolError,
+ pool.request, 'GET', '/source_address')
+
+ @onlyPy3
+ def test_httplib_headers_case_insensitive(self):
+ HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain',
+ 'Server': 'TornadoServer/%s' % tornado.version}
+ r = self.pool.request('GET', '/specific_method',
+ fields={'method': 'GET'})
+ self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
+
+
+class TestRetry(HTTPDummyServerTestCase):
+ def setUp(self):
+ self.pool = HTTPConnectionPool(self.host, self.port)
+
+ def test_max_retry(self):
+ try:
+ r = self.pool.request('GET', '/redirect',
+ fields={'target': '/'},
+ retries=0)
+ self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
+ except MaxRetryError:
+ pass
+
+ def test_disabled_retry(self):
+ """ Disabled retries should disable redirect handling. """
+ r = self.pool.request('GET', '/redirect',
+ fields={'target': '/'},
+ retries=False)
+ self.assertEqual(r.status, 303)
+
+ r = self.pool.request('GET', '/redirect',
+ fields={'target': '/'},
+ retries=Retry(redirect=False))
+ self.assertEqual(r.status, 303)
+
+ pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001)
+ self.assertRaises(ProtocolError, pool.request, 'GET', '/test', retries=False)
+
+ def test_read_retries(self):
+ """ Should retry for status codes in the whitelist """
+ retry = Retry(read=1, status_forcelist=[418])
+ resp = self.pool.request('GET', '/successful_retry',
+ headers={'test-name': 'test_read_retries'},
+ retries=retry)
+ self.assertEqual(resp.status, 200)
+
+ def test_read_total_retries(self):
+ """ HTTP response w/ status code in the whitelist should be retried """
+ headers = {'test-name': 'test_read_total_retries'}
+ retry = Retry(total=1, status_forcelist=[418])
+ resp = self.pool.request('GET', '/successful_retry',
+ headers=headers, retries=retry)
+ self.assertEqual(resp.status, 200)
+
+ def test_retries_wrong_whitelist(self):
+ """HTTP response w/ status code not in whitelist shouldn't be retried"""
+ retry = Retry(total=1, status_forcelist=[202])
+ resp = self.pool.request('GET', '/successful_retry',
+ headers={'test-name': 'test_wrong_whitelist'},
+ retries=retry)
+ self.assertEqual(resp.status, 418)
+
+ def test_default_method_whitelist_retried(self):
+ """ urllib3 should retry methods in the default method whitelist """
+ retry = Retry(total=1, status_forcelist=[418])
+ resp = self.pool.request('OPTIONS', '/successful_retry',
+ headers={'test-name': 'test_default_whitelist'},
+ retries=retry)
+ self.assertEqual(resp.status, 200)
+
+ def test_retries_wrong_method_list(self):
+ """Method not in our whitelist should not be retried, even if code matches"""
+ headers = {'test-name': 'test_wrong_method_whitelist'}
+ retry = Retry(total=1, status_forcelist=[418],
+ method_whitelist=['POST'])
+ resp = self.pool.request('GET', '/successful_retry',
+ headers=headers, retries=retry)
+ self.assertEqual(resp.status, 418)
+
+ def test_read_retries_unsuccessful(self):
+ headers = {'test-name': 'test_read_retries_unsuccessful'}
+ resp = self.pool.request('GET', '/successful_retry',
+ headers=headers, retries=1)
+ self.assertEqual(resp.status, 418)
+
+ def test_retry_reuse_safe(self):
+ """ It should be possible to reuse a Retry object across requests """
+ headers = {'test-name': 'test_retry_safe'}
+ retry = Retry(total=1, status_forcelist=[418])
+ resp = self.pool.request('GET', '/successful_retry',
+ headers=headers, retries=retry)
+ self.assertEqual(resp.status, 200)
+ resp = self.pool.request('GET', '/successful_retry',
+ headers=headers, retries=retry)
+ self.assertEqual(resp.status, 200)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/with_dummyserver/test_connectionpool.pyc b/test/with_dummyserver/test_connectionpool.pyc
new file mode 100644
index 0000000..b8c38e9
--- /dev/null
+++ b/test/with_dummyserver/test_connectionpool.pyc
Binary files differ
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
new file mode 100644
index 0000000..cf3eee7
--- /dev/null
+++ b/test/with_dummyserver/test_https.py
@@ -0,0 +1,374 @@
+import datetime
+import logging
+import ssl
+import sys
+import unittest
+import warnings
+
+import mock
+from nose.plugins.skip import SkipTest
+
+from dummyserver.testcase import HTTPSDummyServerTestCase
+from dummyserver.server import DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS
+
+from test import (
+ onlyPy26OrOlder,
+ requires_network,
+ TARPIT_HOST,
+ clear_warnings,
+)
+from urllib3 import HTTPSConnectionPool
+from urllib3.connection import (
+ VerifiedHTTPSConnection,
+ UnverifiedHTTPSConnection,
+ RECENT_DATE,
+)
+from urllib3.exceptions import (
+ SSLError,
+ ReadTimeoutError,
+ ConnectTimeoutError,
+ InsecureRequestWarning,
+ SystemTimeWarning,
+)
+from urllib3.util.timeout import Timeout
+
+
+log = logging.getLogger('urllib3.connectionpool')
+log.setLevel(logging.NOTSET)
+log.addHandler(logging.StreamHandler(sys.stdout))
+
+
+
+class TestHTTPS(HTTPSDummyServerTestCase):
+ def setUp(self):
+ self._pool = HTTPSConnectionPool(self.host, self.port)
+
+ def test_simple(self):
+ r = self._pool.request('GET', '/')
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_set_ssl_version_to_tlsv1(self):
+ self._pool.ssl_version = ssl.PROTOCOL_TLSv1
+ r = self._pool.request('GET', '/')
+ self.assertEqual(r.status, 200, r.data)
+
+ def test_verified(self):
+ https_pool = HTTPSConnectionPool(self.host, self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ conn = https_pool._new_conn()
+ self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
+
+ with mock.patch('warnings.warn') as warn:
+ r = https_pool.request('GET', '/')
+ self.assertEqual(r.status, 200)
+ self.assertFalse(warn.called, warn.call_args_list)
+
+ def test_invalid_common_name(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+ try:
+ https_pool.request('GET', '/')
+ self.fail("Didn't raise SSL invalid common name")
+ except SSLError as e:
+ self.assertTrue("doesn't match" in str(e))
+
+ def test_verified_with_bad_ca_certs(self):
+ https_pool = HTTPSConnectionPool(self.host, self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA_BAD)
+
+ try:
+ https_pool.request('GET', '/')
+ self.fail("Didn't raise SSL error with bad CA certs")
+ except SSLError as e:
+ self.assertTrue('certificate verify failed' in str(e),
+ "Expected 'certificate verify failed',"
+ "instead got: %r" % e)
+
+ def test_verified_without_ca_certs(self):
+ # default is cert_reqs=None which is ssl.CERT_NONE
+ https_pool = HTTPSConnectionPool(self.host, self.port,
+ cert_reqs='CERT_REQUIRED')
+
+ try:
+ https_pool.request('GET', '/')
+ self.fail("Didn't raise SSL error with no CA certs when"
+ "CERT_REQUIRED is set")
+ except SSLError as e:
+ # there is a different error message depending on whether or
+ # not pyopenssl is injected
+ self.assertTrue('No root certificates specified' in str(e) or
+ 'certificate verify failed' in str(e),
+ "Expected 'No root certificates specified' or "
+ "'certificate verify failed', "
+ "instead got: %r" % e)
+
+ def test_no_ssl(self):
+ pool = HTTPSConnectionPool(self.host, self.port)
+ pool.ConnectionCls = None
+ self.assertRaises(SSLError, pool._new_conn)
+ self.assertRaises(SSLError, pool.request, 'GET', '/')
+
+ def test_unverified_ssl(self):
+ """ Test that bare HTTPSConnection can connect, make requests """
+ pool = HTTPSConnectionPool(self.host, self.port)
+ pool.ConnectionCls = UnverifiedHTTPSConnection
+
+ with mock.patch('warnings.warn') as warn:
+ r = pool.request('GET', '/')
+ self.assertEqual(r.status, 200)
+ self.assertTrue(warn.called)
+
+ call, = warn.call_args_list
+ category = call[0][1]
+ self.assertEqual(category, InsecureRequestWarning)
+
+ def test_ssl_unverified_with_ca_certs(self):
+ pool = HTTPSConnectionPool(self.host, self.port,
+ cert_reqs='CERT_NONE',
+ ca_certs=DEFAULT_CA_BAD)
+
+ with mock.patch('warnings.warn') as warn:
+ r = pool.request('GET', '/')
+ self.assertEqual(r.status, 200)
+ self.assertTrue(warn.called)
+
+ call, = warn.call_args_list
+ category = call[0][1]
+ self.assertEqual(category, InsecureRequestWarning)
+
+ @requires_network
+ def test_ssl_verified_with_platform_ca_certs(self):
+ """
+ We should rely on the platform CA file to validate authenticity of SSL
+ certificates. Since this file is used by many components of the OS,
+ such as curl, apt-get, etc., we decided to not touch it, in order to
+ not compromise the security of the OS running the test suite (typically
+ urllib3 developer's OS).
+
+ This test assumes that httpbin.org uses a certificate signed by a well
+ known Certificate Authority.
+ """
+ try:
+ import urllib3.contrib.pyopenssl
+ except ImportError:
+ raise SkipTest('Test requires PyOpenSSL')
+ if (urllib3.connection.ssl_wrap_socket is
+ urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
+ # Not patched
+ raise SkipTest('Test should only be run after PyOpenSSL '
+ 'monkey patching')
+
+ https_pool = HTTPSConnectionPool('httpbin.org', 443,
+ cert_reqs=ssl.CERT_REQUIRED)
+
+ https_pool.request('HEAD', '/')
+
+ def test_assert_hostname_false(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ https_pool.assert_hostname = False
+ https_pool.request('GET', '/')
+
+ def test_assert_specific_hostname(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ https_pool.assert_hostname = 'localhost'
+ https_pool.request('GET', '/')
+
+ def test_assert_fingerprint_md5(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
+ ':E7:30:F8:CD:C8:5B'
+ https_pool.request('GET', '/')
+
+ def test_assert_fingerprint_sha1(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
+ '7A:F2:8A:D7:1E:07:33:67:DE'
+ https_pool.request('GET', '/')
+
+ def test_assert_invalid_fingerprint(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=DEFAULT_CA)
+
+ https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
+ 'AA:AA:AA:AA:AA:AA:AA:AA:AA'
+
+ self.assertRaises(SSLError, https_pool.request, 'GET', '/')
+ https_pool._get_conn()
+
+ # Uneven length
+ https_pool.assert_fingerprint = 'AA:A'
+ self.assertRaises(SSLError, https_pool.request, 'GET', '/')
+ https_pool._get_conn()
+
+ # Invalid length
+ https_pool.assert_fingerprint = 'AA'
+ self.assertRaises(SSLError, https_pool.request, 'GET', '/')
+
+ def test_verify_none_and_bad_fingerprint(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_NONE',
+ ca_certs=DEFAULT_CA_BAD)
+
+ https_pool.assert_fingerprint = 'AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:' \
+ 'AA:AA:AA:AA:AA:AA:AA:AA:AA'
+ self.assertRaises(SSLError, https_pool.request, 'GET', '/')
+
+ def test_verify_none_and_good_fingerprint(self):
+ https_pool = HTTPSConnectionPool('127.0.0.1', self.port,
+ cert_reqs='CERT_NONE',
+ ca_certs=DEFAULT_CA_BAD)
+
+ https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
+ '7A:F2:8A:D7:1E:07:33:67:DE'
+ https_pool.request('GET', '/')
+
+ @requires_network
+ def test_https_timeout(self):
+ timeout = Timeout(connect=0.001)
+ https_pool = HTTPSConnectionPool(TARPIT_HOST, self.port,
+ timeout=timeout, retries=False,
+ cert_reqs='CERT_REQUIRED')
+
+ timeout = Timeout(total=None, connect=0.001)
+ https_pool = HTTPSConnectionPool(TARPIT_HOST, self.port,
+ timeout=timeout, retries=False,
+ cert_reqs='CERT_REQUIRED')
+ self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
+
+ timeout = Timeout(read=0.001)
+ https_pool = HTTPSConnectionPool(self.host, self.port,
+ timeout=timeout, retries=False,
+ cert_reqs='CERT_REQUIRED')
+ https_pool.ca_certs = DEFAULT_CA
+ https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
+ '7A:F2:8A:D7:1E:07:33:67:DE'
+ url = '/sleep?seconds=0.005'
+ self.assertRaises(ReadTimeoutError, https_pool.request, 'GET', url)
+
+ timeout = Timeout(total=None)
+ https_pool = HTTPSConnectionPool(self.host, self.port, timeout=timeout,
+ cert_reqs='CERT_NONE')
+ https_pool.request('GET', '/')
+
+ def test_tunnel(self):
+ """ test the _tunnel behavior """
+ timeout = Timeout(total=None)
+ https_pool = HTTPSConnectionPool(self.host, self.port, timeout=timeout,
+ cert_reqs='CERT_NONE')
+ conn = https_pool._new_conn()
+ try:
+ conn.set_tunnel(self.host, self.port)
+ except AttributeError: # python 2.6
+ conn._set_tunnel(self.host, self.port)
+ conn._tunnel = mock.Mock()
+ https_pool._make_request(conn, 'GET', '/')
+ conn._tunnel.assert_called_once_with()
+
+ @onlyPy26OrOlder
+ def test_tunnel_old_python(self):
+ """HTTPSConnection can still make connections if _tunnel_host isn't set
+
+ The _tunnel_host attribute was added in 2.6.3 - because our test runners
+ generally use the latest Python 2.6, we simulate the old version by
+ deleting the attribute from the HTTPSConnection.
+ """
+ conn = self._pool._new_conn()
+ del conn._tunnel_host
+ self._pool._make_request(conn, 'GET', '/')
+
+ @requires_network
+ def test_enhanced_timeout(self):
+ def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
+ https_pool = HTTPSConnectionPool(TARPIT_HOST, self.port,
+ timeout=timeout,
+ retries=False,
+ cert_reqs=cert_reqs)
+ return https_pool
+
+ https_pool = new_pool(Timeout(connect=0.001))
+ conn = https_pool._new_conn()
+ self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/')
+ self.assertRaises(ConnectTimeoutError, https_pool._make_request, conn,
+ 'GET', '/')
+
+ https_pool = new_pool(Timeout(connect=5))
+ self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/',
+ timeout=Timeout(connect=0.001))
+
+ t = Timeout(total=None)
+ https_pool = new_pool(t)
+ conn = https_pool._new_conn()
+ self.assertRaises(ConnectTimeoutError, https_pool.request, 'GET', '/',
+ timeout=Timeout(total=None, connect=0.001))
+
+ def test_enhanced_ssl_connection(self):
+ fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:7A:F2:8A:D7:1E:07:33:67:DE'
+
+ conn = VerifiedHTTPSConnection(self.host, self.port)
+ https_pool = HTTPSConnectionPool(self.host, self.port,
+ cert_reqs='CERT_REQUIRED', ca_certs=DEFAULT_CA,
+ assert_fingerprint=fingerprint)
+
+ https_pool._make_request(conn, 'GET', '/')
+
+ def test_ssl_correct_system_time(self):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self._pool.request('GET', '/')
+
+ self.assertEqual([], w)
+
+ def test_ssl_wrong_system_time(self):
+ with mock.patch('urllib3.connection.datetime') as mock_date:
+ mock_date.date.today.return_value = datetime.date(1970, 1, 1)
+
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self._pool.request('GET', '/')
+
+ self.assertEqual(len(w), 1)
+ warning = w[0]
+
+ self.assertEqual(SystemTimeWarning, warning.category)
+ self.assertTrue(str(RECENT_DATE) in warning.message.args[0])
+
+
+class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
+ certs = DEFAULT_CERTS.copy()
+ certs['ssl_version'] = ssl.PROTOCOL_TLSv1
+
+ def setUp(self):
+ self._pool = HTTPSConnectionPool(self.host, self.port)
+
+ def test_set_ssl_version_to_sslv3(self):
+ self._pool.ssl_version = ssl.PROTOCOL_SSLv3
+ self.assertRaises(SSLError, self._pool.request, 'GET', '/')
+
+ def test_ssl_version_as_string(self):
+ self._pool.ssl_version = 'PROTOCOL_SSLv3'
+ self.assertRaises(SSLError, self._pool.request, 'GET', '/')
+
+ def test_ssl_version_as_short_string(self):
+ self._pool.ssl_version = 'SSLv3'
+ self.assertRaises(SSLError, self._pool.request, 'GET', '/')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/with_dummyserver/test_https.pyc b/test/with_dummyserver/test_https.pyc
new file mode 100644
index 0000000..6d85316
--- /dev/null
+++ b/test/with_dummyserver/test_https.pyc
Binary files differ
diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py
new file mode 100644
index 0000000..52ff974
--- /dev/null
+++ b/test/with_dummyserver/test_poolmanager.py
@@ -0,0 +1,136 @@
+import unittest
+import json
+
+from dummyserver.testcase import (HTTPDummyServerTestCase,
+ IPv6HTTPDummyServerTestCase)
+from urllib3.poolmanager import PoolManager
+from urllib3.connectionpool import port_by_scheme
+from urllib3.exceptions import MaxRetryError, SSLError
+
+
+class TestPoolManager(HTTPDummyServerTestCase):
+
+ def setUp(self):
+ self.base_url = 'http://%s:%d' % (self.host, self.port)
+ self.base_url_alt = 'http://%s:%d' % (self.host_alt, self.port)
+
+ def test_redirect(self):
+ http = PoolManager()
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': '%s/' % self.base_url},
+ redirect=False)
+
+ self.assertEqual(r.status, 303)
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': '%s/' % self.base_url})
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_redirect_twice(self):
+ http = PoolManager()
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': '%s/redirect' % self.base_url},
+ redirect=False)
+
+ self.assertEqual(r.status, 303)
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': '%s/redirect?target=%s/' % (self.base_url, self.base_url)})
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_redirect_to_relative_url(self):
+ http = PoolManager()
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields = {'target': '/redirect'},
+ redirect = False)
+
+ self.assertEqual(r.status, 303)
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields = {'target': '/redirect'})
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_cross_host_redirect(self):
+ http = PoolManager()
+
+ cross_host_location = '%s/echo?a=b' % self.base_url_alt
+ try:
+ http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': cross_host_location},
+ timeout=0.01, retries=0)
+ self.fail("Request succeeded instead of raising an exception like it should.")
+
+ except MaxRetryError:
+ pass
+
+ r = http.request('GET', '%s/redirect' % self.base_url,
+ fields={'target': '%s/echo?a=b' % self.base_url_alt},
+ timeout=0.01, retries=1)
+
+ self.assertEqual(r._pool.host, self.host_alt)
+
+ def test_missing_port(self):
+ # Can a URL that lacks an explicit port like ':80' succeed, or
+ # will all such URLs fail with an error?
+
+ http = PoolManager()
+
+ # By globally adjusting `port_by_scheme` we pretend for a moment
+ # that HTTP's default port is not 80, but is the port at which
+ # our test server happens to be listening.
+ port_by_scheme['http'] = self.port
+ try:
+ r = http.request('GET', 'http://%s/' % self.host, retries=0)
+ finally:
+ port_by_scheme['http'] = 80
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_headers(self):
+ http = PoolManager(headers={'Foo': 'bar'})
+
+ r = http.request_encode_url('GET', '%s/headers' % self.base_url)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+
+ r = http.request_encode_body('POST', '%s/headers' % self.base_url)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+
+ r = http.request_encode_url('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+
+ r = http.request_encode_body('GET', '%s/headers' % self.base_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+
+ def test_http_with_ssl_keywords(self):
+ http = PoolManager(ca_certs='REQUIRED')
+
+ r = http.request('GET', 'http://%s:%s/' % (self.host, self.port))
+ self.assertEqual(r.status, 200)
+
+
+class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase):
+ def setUp(self):
+ self.base_url = 'http://[%s]:%d' % (self.host, self.port)
+
+ def test_ipv6(self):
+ http = PoolManager()
+ http.request('GET', self.base_url)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/with_dummyserver/test_poolmanager.pyc b/test/with_dummyserver/test_poolmanager.pyc
new file mode 100644
index 0000000..26c52e9
--- /dev/null
+++ b/test/with_dummyserver/test_poolmanager.pyc
Binary files differ
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
new file mode 100644
index 0000000..61eedf1
--- /dev/null
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -0,0 +1,263 @@
+import unittest
+import json
+import socket
+
+from dummyserver.testcase import HTTPDummyProxyTestCase
+from dummyserver.server import (
+ DEFAULT_CA, DEFAULT_CA_BAD, get_unreachable_address)
+
+from urllib3.poolmanager import proxy_from_url, ProxyManager
+from urllib3.exceptions import MaxRetryError, SSLError, ProxyError
+from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection
+
+
+class TestHTTPProxyManager(HTTPDummyProxyTestCase):
+
+ def setUp(self):
+ self.http_url = 'http://%s:%d' % (self.http_host, self.http_port)
+ self.http_url_alt = 'http://%s:%d' % (self.http_host_alt,
+ self.http_port)
+ self.https_url = 'https://%s:%d' % (self.https_host, self.https_port)
+ self.https_url_alt = 'https://%s:%d' % (self.https_host_alt,
+ self.https_port)
+ self.proxy_url = 'http://%s:%d' % (self.proxy_host, self.proxy_port)
+
+ def test_basic_proxy(self):
+ http = proxy_from_url(self.proxy_url)
+
+ r = http.request('GET', '%s/' % self.http_url)
+ self.assertEqual(r.status, 200)
+
+ r = http.request('GET', '%s/' % self.https_url)
+ self.assertEqual(r.status, 200)
+
+ def test_nagle_proxy(self):
+ """ Test that proxy connections do not have TCP_NODELAY turned on """
+ http = proxy_from_url(self.proxy_url)
+ hc2 = http.connection_from_host(self.http_host, self.http_port)
+ conn = hc2._get_conn()
+ hc2._make_request(conn, 'GET', '/')
+ tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
+ self.assertEqual(tcp_nodelay_setting, 0,
+ ("Expected TCP_NODELAY for proxies to be set "
+ "to zero, instead was %s" % tcp_nodelay_setting))
+
+ def test_proxy_conn_fail(self):
+ host, port = get_unreachable_address()
+ http = proxy_from_url('http://%s:%s/' % (host, port), retries=1)
+ self.assertRaises(MaxRetryError, http.request, 'GET',
+ '%s/' % self.https_url)
+ self.assertRaises(MaxRetryError, http.request, 'GET',
+ '%s/' % self.http_url)
+
+ try:
+ http.request('GET', '%s/' % self.http_url)
+ self.fail("Failed to raise retry error.")
+ except MaxRetryError as e:
+ self.assertEqual(type(e.reason), ProxyError)
+
+ def test_oldapi(self):
+ http = ProxyManager(connection_from_url(self.proxy_url))
+
+ r = http.request('GET', '%s/' % self.http_url)
+ self.assertEqual(r.status, 200)
+
+ r = http.request('GET', '%s/' % self.https_url)
+ self.assertEqual(r.status, 200)
+
+ def test_proxy_verified(self):
+ http = proxy_from_url(self.proxy_url, cert_reqs='REQUIRED',
+ ca_certs=DEFAULT_CA_BAD)
+ https_pool = http._new_pool('https', self.https_host,
+ self.https_port)
+ try:
+ https_pool.request('GET', '/')
+ self.fail("Didn't raise SSL error with wrong CA")
+ except SSLError as e:
+ self.assertTrue('certificate verify failed' in str(e),
+ "Expected 'certificate verify failed',"
+ "instead got: %r" % e)
+
+ http = proxy_from_url(self.proxy_url, cert_reqs='REQUIRED',
+ ca_certs=DEFAULT_CA)
+ https_pool = http._new_pool('https', self.https_host,
+ self.https_port)
+
+ conn = https_pool._new_conn()
+ self.assertEqual(conn.__class__, VerifiedHTTPSConnection)
+ https_pool.request('GET', '/') # Should succeed without exceptions.
+
+ http = proxy_from_url(self.proxy_url, cert_reqs='REQUIRED',
+ ca_certs=DEFAULT_CA)
+ https_fail_pool = http._new_pool('https', '127.0.0.1', self.https_port)
+
+ try:
+ https_fail_pool.request('GET', '/')
+ self.fail("Didn't raise SSL invalid common name")
+ except SSLError as e:
+ self.assertTrue("doesn't match" in str(e))
+
+ def test_redirect(self):
+ http = proxy_from_url(self.proxy_url)
+
+ r = http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': '%s/' % self.http_url},
+ redirect=False)
+
+ self.assertEqual(r.status, 303)
+
+ r = http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': '%s/' % self.http_url})
+
+ self.assertEqual(r.status, 200)
+ self.assertEqual(r.data, b'Dummy server!')
+
+ def test_cross_host_redirect(self):
+ http = proxy_from_url(self.proxy_url)
+
+ cross_host_location = '%s/echo?a=b' % self.http_url_alt
+ try:
+ http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': cross_host_location},
+ timeout=0.1, retries=0)
+ self.fail("We don't want to follow redirects here.")
+
+ except MaxRetryError:
+ pass
+
+ r = http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': '%s/echo?a=b' % self.http_url_alt},
+ timeout=0.1, retries=1)
+ self.assertNotEqual(r._pool.host, self.http_host_alt)
+
+ def test_cross_protocol_redirect(self):
+ http = proxy_from_url(self.proxy_url)
+
+ cross_protocol_location = '%s/echo?a=b' % self.https_url
+ try:
+ http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': cross_protocol_location},
+ timeout=0.1, retries=0)
+ self.fail("We don't want to follow redirects here.")
+
+ except MaxRetryError:
+ pass
+
+ r = http.request('GET', '%s/redirect' % self.http_url,
+ fields={'target': '%s/echo?a=b' % self.https_url},
+ timeout=0.1, retries=1)
+ self.assertEqual(r._pool.host, self.https_host)
+
+ def test_headers(self):
+ http = proxy_from_url(self.proxy_url,headers={'Foo': 'bar'},
+ proxy_headers={'Hickory': 'dickory'})
+
+ r = http.request_encode_url('GET', '%s/headers' % self.http_url)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+ self.assertEqual(returned_headers.get('Hickory'), 'dickory')
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.http_host,self.http_port))
+
+ r = http.request_encode_url('GET', '%s/headers' % self.http_url_alt)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+ self.assertEqual(returned_headers.get('Hickory'), 'dickory')
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.http_host_alt,self.http_port))
+
+ r = http.request_encode_url('GET', '%s/headers' % self.https_url)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+ self.assertEqual(returned_headers.get('Hickory'), None)
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.https_host,self.https_port))
+
+ r = http.request_encode_url('GET', '%s/headers' % self.https_url_alt)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+ self.assertEqual(returned_headers.get('Hickory'), None)
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.https_host_alt,self.https_port))
+
+ r = http.request_encode_body('POST', '%s/headers' % self.http_url)
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), 'bar')
+ self.assertEqual(returned_headers.get('Hickory'), 'dickory')
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.http_host,self.http_port))
+
+ r = http.request_encode_url('GET', '%s/headers' % self.http_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+ self.assertEqual(returned_headers.get('Hickory'), 'dickory')
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.http_host,self.http_port))
+
+ r = http.request_encode_url('GET', '%s/headers' % self.https_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+ self.assertEqual(returned_headers.get('Hickory'), None)
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.https_host,self.https_port))
+
+ r = http.request_encode_body('GET', '%s/headers' % self.http_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+ self.assertEqual(returned_headers.get('Hickory'), 'dickory')
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.http_host,self.http_port))
+
+ r = http.request_encode_body('GET', '%s/headers' % self.https_url, headers={'Baz': 'quux'})
+ returned_headers = json.loads(r.data.decode())
+ self.assertEqual(returned_headers.get('Foo'), None)
+ self.assertEqual(returned_headers.get('Baz'), 'quux')
+ self.assertEqual(returned_headers.get('Hickory'), None)
+ self.assertEqual(returned_headers.get('Host'),
+ '%s:%s'%(self.https_host,self.https_port))
+
+ def test_proxy_pooling(self):
+ http = proxy_from_url(self.proxy_url)
+
+ for x in range(2):
+ r = http.urlopen('GET', self.http_url)
+ self.assertEqual(len(http.pools), 1)
+
+ for x in range(2):
+ r = http.urlopen('GET', self.http_url_alt)
+ self.assertEqual(len(http.pools), 1)
+
+ for x in range(2):
+ r = http.urlopen('GET', self.https_url)
+ self.assertEqual(len(http.pools), 2)
+
+ for x in range(2):
+ r = http.urlopen('GET', self.https_url_alt)
+ self.assertEqual(len(http.pools), 3)
+
+ def test_proxy_pooling_ext(self):
+ http = proxy_from_url(self.proxy_url)
+ hc1 = http.connection_from_url(self.http_url)
+ hc2 = http.connection_from_host(self.http_host, self.http_port)
+ hc3 = http.connection_from_url(self.http_url_alt)
+ hc4 = http.connection_from_host(self.http_host_alt, self.http_port)
+ self.assertEqual(hc1,hc2)
+ self.assertEqual(hc2,hc3)
+ self.assertEqual(hc3,hc4)
+
+ sc1 = http.connection_from_url(self.https_url)
+ sc2 = http.connection_from_host(self.https_host,
+ self.https_port,scheme='https')
+ sc3 = http.connection_from_url(self.https_url_alt)
+ sc4 = http.connection_from_host(self.https_host_alt,
+ self.https_port,scheme='https')
+ self.assertEqual(sc1,sc2)
+ self.assertNotEqual(sc2,sc3)
+ self.assertEqual(sc3,sc4)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/with_dummyserver/test_proxy_poolmanager.pyc b/test/with_dummyserver/test_proxy_poolmanager.pyc
new file mode 100644
index 0000000..12c320c
--- /dev/null
+++ b/test/with_dummyserver/test_proxy_poolmanager.pyc
Binary files differ
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
new file mode 100644
index 0000000..e1ac1c6
--- /dev/null
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -0,0 +1,544 @@
+# TODO: Break this module up into pieces. Maybe group by functionality tested
+# rather than the socket level-ness of it.
+
+from urllib3 import HTTPConnectionPool, HTTPSConnectionPool
+from urllib3.poolmanager import proxy_from_url
+from urllib3.exceptions import (
+ MaxRetryError,
+ ProxyError,
+ ReadTimeoutError,
+ SSLError,
+ ProtocolError,
+)
+from urllib3.util.ssl_ import HAS_SNI
+from urllib3.util.timeout import Timeout
+from urllib3.util.retry import Retry
+
+from dummyserver.testcase import SocketDummyServerTestCase
+from dummyserver.server import (
+ DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address)
+
+from nose.plugins.skip import SkipTest
+from threading import Event
+import socket
+import ssl
+
+
+class TestCookies(SocketDummyServerTestCase):
+
+ def test_multi_setcookie(self):
+ def multicookie_response_handler(listener):
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ sock.send(b'HTTP/1.1 200 OK\r\n'
+ b'Set-Cookie: foo=1\r\n'
+ b'Set-Cookie: bar=1\r\n'
+ b'\r\n')
+ sock.close()
+
+ self._start_server(multicookie_response_handler)
+ pool = HTTPConnectionPool(self.host, self.port)
+ r = pool.request('GET', '/', retries=0)
+ self.assertEqual(r.headers, {'set-cookie': 'foo=1, bar=1'})
+
+
+class TestSNI(SocketDummyServerTestCase):
+
+ def test_hostname_in_first_request_packet(self):
+ if not HAS_SNI:
+ raise SkipTest('SNI-support not available')
+
+ done_receiving = Event()
+ self.buf = b''
+
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+
+ self.buf = sock.recv(65536) # We only accept one packet
+ done_receiving.set() # let the test know it can proceed
+ sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPSConnectionPool(self.host, self.port)
+ try:
+ pool.request('GET', '/', retries=0)
+ except SSLError: # We are violating the protocol
+ pass
+ done_receiving.wait()
+ self.assertTrue(self.host.encode() in self.buf,
+ "missing hostname in SSL handshake")
+
+
+class TestSocketClosing(SocketDummyServerTestCase):
+
+ def test_recovery_when_server_closes_connection(self):
+ # Does the pool work seamlessly if an open connection in the
+ # connection pool gets hung up on by the server, then reaches
+ # the front of the queue again?
+
+ done_closing = Event()
+
+ def socket_handler(listener):
+ for i in 0, 1:
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf = sock.recv(65536)
+
+ body = 'Response %d' % i
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(body), body)).encode('utf-8'))
+
+ sock.close() # simulate a server timing out, closing socket
+ done_closing.set() # let the test know it can proceed
+
+ self._start_server(socket_handler)
+ pool = HTTPConnectionPool(self.host, self.port)
+
+ response = pool.request('GET', '/', retries=0)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.data, b'Response 0')
+
+ done_closing.wait() # wait until the socket in our pool gets closed
+
+ response = pool.request('GET', '/', retries=0)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.data, b'Response 1')
+
+ def test_connection_refused(self):
+ # Does the pool retry if there is no listener on the port?
+ host, port = get_unreachable_address()
+ pool = HTTPConnectionPool(host, port)
+ self.assertRaises(MaxRetryError, pool.request, 'GET', '/', retries=0)
+
+ def test_connection_read_timeout(self):
+ timed_out = Event()
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+ while not sock.recv(65536).endswith(b'\r\n\r\n'):
+ pass
+
+ timed_out.wait()
+ sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
+
+ try:
+ self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/')
+ finally:
+ timed_out.set()
+
+ def test_timeout_errors_cause_retries(self):
+ def socket_handler(listener):
+ sock_timeout = listener.accept()[0]
+
+ # Wait for a second request before closing the first socket.
+ sock = listener.accept()[0]
+ sock_timeout.close()
+
+ # Second request.
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ # Now respond immediately.
+ body = 'Response 2'
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(body), body)).encode('utf-8'))
+
+ sock.close()
+
+ # In situations where the main thread throws an exception, the server
+ # thread can hang on an accept() call. This ensures everything times
+ # out within 1 second. This should be long enough for any socket
+ # operations in the test suite to complete
+ default_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(1)
+
+ try:
+ self._start_server(socket_handler)
+ t = Timeout(connect=0.001, read=0.001)
+ pool = HTTPConnectionPool(self.host, self.port, timeout=t)
+
+ response = pool.request('GET', '/', retries=1)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.data, b'Response 2')
+ finally:
+ socket.setdefaulttimeout(default_timeout)
+
+ def test_delayed_body_read_timeout(self):
+ timed_out = Event()
+
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+ buf = b''
+ body = 'Hi'
+ while not buf.endswith(b'\r\n\r\n'):
+ buf = sock.recv(65536)
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n' % len(body)).encode('utf-8'))
+
+ timed_out.wait()
+ sock.send(body.encode('utf-8'))
+ sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPConnectionPool(self.host, self.port)
+
+ response = pool.urlopen('GET', '/', retries=0, preload_content=False,
+ timeout=Timeout(connect=1, read=0.001))
+ try:
+ self.assertRaises(ReadTimeoutError, response.read)
+ finally:
+ timed_out.set()
+
+ def test_incomplete_response(self):
+ body = 'Response'
+ partial_body = body[:2]
+
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+
+ # Consume request
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf = sock.recv(65536)
+
+ # Send partial response and close socket.
+ sock.send((
+ 'HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(body), partial_body)).encode('utf-8')
+ )
+ sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPConnectionPool(self.host, self.port)
+
+ response = pool.request('GET', '/', retries=0, preload_content=False)
+ self.assertRaises(ProtocolError, response.read)
+
+ def test_retry_weird_http_version(self):
+ """ Retry class should handle httplib.BadStatusLine errors properly """
+
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+ # First request.
+ # Pause before responding so the first request times out.
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ # send unknown http protocol
+ body = "bad http 0.5 response"
+ sock.send(('HTTP/0.5 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(body), body)).encode('utf-8'))
+ sock.close()
+
+ # Second request.
+ sock = listener.accept()[0]
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ # Now respond immediately.
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ 'foo' % (len('foo'))).encode('utf-8'))
+
+ sock.close() # Close the socket.
+
+ self._start_server(socket_handler)
+ pool = HTTPConnectionPool(self.host, self.port)
+ retry = Retry(read=1)
+ response = pool.request('GET', '/', retries=retry)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.data, b'foo')
+
+
+
+class TestProxyManager(SocketDummyServerTestCase):
+
+ def test_simple(self):
+ def echo_socket_handler(listener):
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(buf), buf.decode('utf-8'))).encode('utf-8'))
+ sock.close()
+
+ self._start_server(echo_socket_handler)
+ base_url = 'http://%s:%d' % (self.host, self.port)
+ proxy = proxy_from_url(base_url)
+
+ r = proxy.request('GET', 'http://google.com/')
+
+ self.assertEqual(r.status, 200)
+ # FIXME: The order of the headers is not predictable right now. We
+ # should fix that someday (maybe when we migrate to
+ # OrderedDict/MultiDict).
+ self.assertEqual(sorted(r.data.split(b'\r\n')),
+ sorted([
+ b'GET http://google.com/ HTTP/1.1',
+ b'Host: google.com',
+ b'Accept-Encoding: identity',
+ b'Accept: */*',
+ b'',
+ b'',
+ ]))
+
+ def test_headers(self):
+ def echo_socket_handler(listener):
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(buf), buf.decode('utf-8'))).encode('utf-8'))
+ sock.close()
+
+ self._start_server(echo_socket_handler)
+ base_url = 'http://%s:%d' % (self.host, self.port)
+
+ # Define some proxy headers.
+ proxy_headers = {'For The Proxy': 'YEAH!'}
+ proxy = proxy_from_url(base_url, proxy_headers=proxy_headers)
+
+ conn = proxy.connection_from_url('http://www.google.com/')
+
+ r = conn.urlopen('GET', 'http://www.google.com/', assert_same_host=False)
+
+ self.assertEqual(r.status, 200)
+ # FIXME: The order of the headers is not predictable right now. We
+ # should fix that someday (maybe when we migrate to
+ # OrderedDict/MultiDict).
+ self.assertTrue(b'For The Proxy: YEAH!\r\n' in r.data)
+
+ def test_retries(self):
+ def echo_socket_handler(listener):
+ sock = listener.accept()[0]
+ # First request, which should fail
+ sock.close()
+
+ # Second request
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+
+ sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: %d\r\n'
+ '\r\n'
+ '%s' % (len(buf), buf.decode('utf-8'))).encode('utf-8'))
+ sock.close()
+
+ self._start_server(echo_socket_handler)
+ base_url = 'http://%s:%d' % (self.host, self.port)
+
+ proxy = proxy_from_url(base_url)
+ conn = proxy.connection_from_url('http://www.google.com')
+
+ r = conn.urlopen('GET', 'http://www.google.com',
+ assert_same_host=False, retries=1)
+ self.assertEqual(r.status, 200)
+
+ self.assertRaises(ProxyError, conn.urlopen, 'GET',
+ 'http://www.google.com',
+ assert_same_host=False, retries=False)
+
+ def test_connect_reconn(self):
+ def proxy_ssl_one(listener):
+ sock = listener.accept()[0]
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += sock.recv(65536)
+ s = buf.decode('utf-8')
+ if not s.startswith('CONNECT '):
+ sock.send(('HTTP/1.1 405 Method not allowed\r\n'
+ 'Allow: CONNECT\r\n\r\n').encode('utf-8'))
+ sock.close()
+ return
+
+ if not s.startswith('CONNECT %s:443' % (self.host,)):
+ sock.send(('HTTP/1.1 403 Forbidden\r\n\r\n').encode('utf-8'))
+ sock.close()
+ return
+
+ sock.send(('HTTP/1.1 200 Connection Established\r\n\r\n').encode('utf-8'))
+ ssl_sock = ssl.wrap_socket(sock,
+ server_side=True,
+ keyfile=DEFAULT_CERTS['keyfile'],
+ certfile=DEFAULT_CERTS['certfile'],
+ ca_certs=DEFAULT_CA)
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += ssl_sock.recv(65536)
+
+ ssl_sock.send(('HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: 2\r\n'
+ 'Connection: close\r\n'
+ '\r\n'
+ 'Hi').encode('utf-8'))
+ ssl_sock.close()
+ def echo_socket_handler(listener):
+ proxy_ssl_one(listener)
+ proxy_ssl_one(listener)
+
+ self._start_server(echo_socket_handler)
+ base_url = 'http://%s:%d' % (self.host, self.port)
+
+ proxy = proxy_from_url(base_url)
+
+ url = 'https://{0}'.format(self.host)
+ conn = proxy.connection_from_url(url)
+ r = conn.urlopen('GET', url, retries=0)
+ self.assertEqual(r.status, 200)
+ r = conn.urlopen('GET', url, retries=0)
+ self.assertEqual(r.status, 200)
+
+
+class TestSSL(SocketDummyServerTestCase):
+
+ def test_ssl_failure_midway_through_conn(self):
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+ sock2 = sock.dup()
+ ssl_sock = ssl.wrap_socket(sock,
+ server_side=True,
+ keyfile=DEFAULT_CERTS['keyfile'],
+ certfile=DEFAULT_CERTS['certfile'],
+ ca_certs=DEFAULT_CA)
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += ssl_sock.recv(65536)
+
+ # Deliberately send from the non-SSL socket.
+ sock2.send((
+ 'HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: 2\r\n'
+ '\r\n'
+ 'Hi').encode('utf-8'))
+ sock2.close()
+ ssl_sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPSConnectionPool(self.host, self.port)
+
+ self.assertRaises(SSLError, pool.request, 'GET', '/', retries=0)
+
+ def test_ssl_read_timeout(self):
+ timed_out = Event()
+
+ def socket_handler(listener):
+ sock = listener.accept()[0]
+ ssl_sock = ssl.wrap_socket(sock,
+ server_side=True,
+ keyfile=DEFAULT_CERTS['keyfile'],
+ certfile=DEFAULT_CERTS['certfile'],
+ ca_certs=DEFAULT_CA)
+
+ buf = b''
+ while not buf.endswith(b'\r\n\r\n'):
+ buf += ssl_sock.recv(65536)
+
+ # Send incomplete message (note Content-Length)
+ ssl_sock.send((
+ 'HTTP/1.1 200 OK\r\n'
+ 'Content-Type: text/plain\r\n'
+ 'Content-Length: 10\r\n'
+ '\r\n'
+ 'Hi-').encode('utf-8'))
+ timed_out.wait()
+
+ sock.close()
+ ssl_sock.close()
+
+ self._start_server(socket_handler)
+ pool = HTTPSConnectionPool(self.host, self.port)
+
+ response = pool.urlopen('GET', '/', retries=0, preload_content=False,
+ timeout=Timeout(connect=1, read=0.001))
+ try:
+ self.assertRaises(ReadTimeoutError, response.read)
+ finally:
+ timed_out.set()
+
+
+def consume_socket(sock, chunks=65536):
+ while not sock.recv(chunks).endswith(b'\r\n\r\n'):
+ pass
+
+
+def create_response_handler(response, num=1):
+ def socket_handler(listener):
+ for _ in range(num):
+ sock = listener.accept()[0]
+ consume_socket(sock)
+
+ sock.send(response)
+ sock.close()
+
+ return socket_handler
+
+
+class TestErrorWrapping(SocketDummyServerTestCase):
+
+ def test_bad_statusline(self):
+ handler = create_response_handler(
+ b'HTTP/1.1 Omg What Is This?\r\n'
+ b'Content-Length: 0\r\n'
+ b'\r\n'
+ )
+ self._start_server(handler)
+ pool = HTTPConnectionPool(self.host, self.port, retries=False)
+ self.assertRaises(ProtocolError, pool.request, 'GET', '/')
+
+ def test_unknown_protocol(self):
+ handler = create_response_handler(
+ b'HTTP/1000 200 OK\r\n'
+ b'Content-Length: 0\r\n'
+ b'\r\n'
+ )
+ self._start_server(handler)
+ pool = HTTPConnectionPool(self.host, self.port, retries=False)
+ self.assertRaises(ProtocolError, pool.request, 'GET', '/')
diff --git a/test/with_dummyserver/test_socketlevel.pyc b/test/with_dummyserver/test_socketlevel.pyc
new file mode 100644
index 0000000..ba3b19e
--- /dev/null
+++ b/test/with_dummyserver/test_socketlevel.pyc
Binary files differ
diff --git a/urllib3.egg-info/PKG-INFO b/urllib3.egg-info/PKG-INFO
index 168944c..964cd4b 100644
--- a/urllib3.egg-info/PKG-INFO
+++ b/urllib3.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: urllib3
-Version: 1.9
+Version: 1.9.1
Summary: HTTP library with thread-safe connection pooling, file post, and more.
Home-page: http://urllib3.readthedocs.org/
Author: Andrey Petrov
@@ -13,6 +13,9 @@ Description: =======
.. image:: https://travis-ci.org/shazow/urllib3.png?branch=master
:target: https://travis-ci.org/shazow/urllib3
+ .. image:: https://www.bountysource.com/badge/tracker?tracker_id=192525
+ :target: https://www.bountysource.com/trackers/192525-urllib3?utm_source=192525&utm_medium=shield&utm_campaign=TRACKER_BADGE
+
Highlights
==========
@@ -153,6 +156,27 @@ Description: =======
Changes
=======
+ 1.9.1 (2014-09-13)
+ ++++++++++++++++++
+
+ * Apply socket arguments before binding. (Issue #427)
+
+ * More careful checks if fp-like object is closed. (Issue #435)
+
+ * Fixed packaging issues of some development-related files not
+ getting included. (Issue #440)
+
+ * Allow performing *only* fingerprint verification. (Issue #444)
+
+ * Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445)
+
+ * Fixed PyOpenSSL compatibility with PyPy. (Issue #450)
+
+ * Fixed ``BrokenPipeError`` and ``ConnectionError`` handling in Py3.
+ (Issue #443)
+
+
+
1.9 (2014-07-04)
++++++++++++++++
diff --git a/urllib3.egg-info/SOURCES.txt b/urllib3.egg-info/SOURCES.txt
index e0b9ddd..2f0b5fc 100644
--- a/urllib3.egg-info/SOURCES.txt
+++ b/urllib3.egg-info/SOURCES.txt
@@ -7,11 +7,29 @@ README.rst
dev-requirements.txt
setup.cfg
setup.py
+docs/Makefile
+docs/README
+docs/collections.rst
+docs/conf.py
+docs/contrib.rst
+docs/doc-requirements.txt
+docs/exceptions.rst
+docs/helpers.rst
+docs/index.rst
+docs/make.bat
+docs/managers.rst
+docs/pools.rst
+docs/security.rst
dummyserver/__init__.py
+dummyserver/__init__.pyc
dummyserver/handlers.py
+dummyserver/handlers.pyc
dummyserver/proxy.py
+dummyserver/proxy.pyc
dummyserver/server.py
+dummyserver/server.pyc
dummyserver/testcase.py
+dummyserver/testcase.pyc
dummyserver/certs/cacert.key
dummyserver/certs/cacert.pem
dummyserver/certs/client.csr
@@ -22,17 +40,49 @@ dummyserver/certs/server.crt
dummyserver/certs/server.csr
dummyserver/certs/server.key
dummyserver/certs/server.key.org
+test/__init__.py
+test/__init__.pyc
+test/benchmark.py
+test/port_helpers.py
+test/port_helpers.pyc
test/test_collections.py
+test/test_collections.pyc
test/test_compatibility.py
+test/test_compatibility.pyc
test/test_connectionpool.py
+test/test_connectionpool.pyc
test/test_exceptions.py
+test/test_exceptions.pyc
test/test_fields.py
+test/test_fields.pyc
test/test_filepost.py
+test/test_filepost.pyc
test/test_poolmanager.py
+test/test_poolmanager.pyc
test/test_proxymanager.py
+test/test_proxymanager.pyc
test/test_response.py
+test/test_response.pyc
test/test_retry.py
+test/test_retry.pyc
test/test_util.py
+test/test_util.pyc
+test/contrib/__init__.py
+test/contrib/__init__.pyc
+test/contrib/test_pyopenssl.py
+test/contrib/test_pyopenssl.pyc
+test/with_dummyserver/__init__.py
+test/with_dummyserver/__init__.pyc
+test/with_dummyserver/test_connectionpool.py
+test/with_dummyserver/test_connectionpool.pyc
+test/with_dummyserver/test_https.py
+test/with_dummyserver/test_https.pyc
+test/with_dummyserver/test_poolmanager.py
+test/with_dummyserver/test_poolmanager.pyc
+test/with_dummyserver/test_proxy_poolmanager.py
+test/with_dummyserver/test_proxy_poolmanager.pyc
+test/with_dummyserver/test_socketlevel.py
+test/with_dummyserver/test_socketlevel.pyc
urllib3/__init__.py
urllib3/_collections.py
urllib3/connection.py
diff --git a/urllib3/__init__.py b/urllib3/__init__.py
index 56f5bf4..3546d13 100644
--- a/urllib3/__init__.py
+++ b/urllib3/__init__.py
@@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
-__version__ = '1.9'
+__version__ = '1.9.1'
from .connectionpool import (
@@ -57,7 +57,7 @@ del NullHandler
# Set security warning to only go off once by default.
import warnings
-warnings.simplefilter('module', exceptions.InsecureRequestWarning)
+warnings.simplefilter('module', exceptions.SecurityWarning)
def disable_warnings(category=exceptions.HTTPWarning):
"""
diff --git a/urllib3/connection.py b/urllib3/connection.py
index 0d578d7..cebdd86 100644
--- a/urllib3/connection.py
+++ b/urllib3/connection.py
@@ -1,6 +1,9 @@
+import datetime
import sys
import socket
from socket import timeout as SocketTimeout
+import warnings
+from .packages import six
try: # Python 3
from http.client import HTTPConnection as _HTTPConnection, HTTPException
@@ -24,11 +27,19 @@ except (ImportError, AttributeError): # Platform-specific: No SSL.
pass
+try: # Python 3:
+ # Not a no-op, we're adding this to the namespace so it can be imported.
+ ConnectionError = ConnectionError
+except NameError: # Python 2:
+ class ConnectionError(Exception):
+ pass
+
+
from .exceptions import (
ConnectTimeoutError,
+ SystemTimeWarning,
)
from .packages.ssl_match_hostname import match_hostname
-from .packages import six
from .util.ssl_ import (
resolve_cert_reqs,
@@ -37,14 +48,16 @@ from .util.ssl_ import (
assert_fingerprint,
)
-from .util import connection
+from .util import connection
port_by_scheme = {
'http': 80,
'https': 443,
}
+RECENT_DATE = datetime.date(2014, 1, 1)
+
class HTTPConnection(_HTTPConnection, object):
"""
@@ -172,6 +185,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
cert_reqs = None
ca_certs = None
ssl_version = None
+ assert_fingerprint = None
def set_cert(self, key_file=None, cert_file=None,
cert_reqs=None, ca_certs=None,
@@ -206,6 +220,14 @@ class VerifiedHTTPSConnection(HTTPSConnection):
# Override the host with the one we're requesting data from.
hostname = self._tunnel_host
+ is_time_off = datetime.date.today() < RECENT_DATE
+ if is_time_off:
+ warnings.warn((
+ 'System time is way off (before {0}). This will probably '
+ 'lead to SSL verification errors').format(RECENT_DATE),
+ SystemTimeWarning
+ )
+
# Wrap socket using verification with the root certs in
# trusted_root_certs
self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
@@ -214,15 +236,16 @@ class VerifiedHTTPSConnection(HTTPSConnection):
server_hostname=hostname,
ssl_version=resolved_ssl_version)
- if resolved_cert_reqs != ssl.CERT_NONE:
- if self.assert_fingerprint:
- assert_fingerprint(self.sock.getpeercert(binary_form=True),
- self.assert_fingerprint)
- elif self.assert_hostname is not False:
- match_hostname(self.sock.getpeercert(),
- self.assert_hostname or hostname)
+ if self.assert_fingerprint:
+ assert_fingerprint(self.sock.getpeercert(binary_form=True),
+ self.assert_fingerprint)
+ elif resolved_cert_reqs != ssl.CERT_NONE \
+ and self.assert_hostname is not False:
+ match_hostname(self.sock.getpeercert(),
+ self.assert_hostname or hostname)
- self.is_verified = resolved_cert_reqs == ssl.CERT_REQUIRED
+ self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
+ or self.assert_fingerprint is not None)
if ssl:
diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py
index 9317fdc..ac6e0ca 100644
--- a/urllib3/connectionpool.py
+++ b/urllib3/connectionpool.py
@@ -32,7 +32,7 @@ from .connection import (
port_by_scheme,
DummyConnection,
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
- HTTPException, BaseSSLError,
+ HTTPException, BaseSSLError, ConnectionError
)
from .request import RequestMethods
from .response import HTTPResponse
@@ -542,7 +542,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
release_conn = True
raise SSLError(e)
- except (TimeoutError, HTTPException, SocketError) as e:
+ except (TimeoutError, HTTPException, SocketError, ConnectionError) as e:
if conn:
# Discard the connection for these exceptions. It will be
# be replaced during the next _get_conn() call.
@@ -718,7 +718,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
super(HTTPSConnectionPool, self)._validate_conn(conn)
# Force connect early to allow us to validate the connection.
- if not conn.sock:
+ if not getattr(conn, 'sock', None): # AppEngine might not have `.sock`
conn.connect()
if not conn.is_verified:
diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py
index 7a9ea2e..8475eeb 100644
--- a/urllib3/contrib/pyopenssl.py
+++ b/urllib3/contrib/pyopenssl.py
@@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI
when the required modules are installed.
Activating this module also has the positive side effect of disabling SSL/TLS
-encryption in Python 2 (see `CRIME attack`_).
+compression in Python 2 (see `CRIME attack`_).
If you want to configure the default list of supported cipher suites, you can
set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
@@ -46,8 +46,12 @@ Module Variables
'''
-from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
-from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
+try:
+ from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
+ from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
+except SyntaxError as e:
+ raise ImportError(e)
+
import OpenSSL.SSL
from pyasn1.codec.der import decoder as der_decoder
from pyasn1.type import univ, constraint
@@ -155,18 +159,24 @@ def get_subj_alt_name(peer_cert):
class WrappedSocket(object):
- '''API-compatibility wrapper for Python OpenSSL's Connection-class.'''
+ '''API-compatibility wrapper for Python OpenSSL's Connection-class.
+
+ Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
+ collector of pypy.
+ '''
def __init__(self, connection, socket, suppress_ragged_eofs=True):
self.connection = connection
self.socket = socket
self.suppress_ragged_eofs = suppress_ragged_eofs
+ self._makefile_refs = 0
def fileno(self):
return self.socket.fileno()
def makefile(self, mode, bufsize=-1):
- return _fileobject(self, mode, bufsize)
+ self._makefile_refs += 1
+ return _fileobject(self, mode, bufsize, close=True)
def recv(self, *args, **kwargs):
try:
@@ -180,7 +190,7 @@ class WrappedSocket(object):
rd, wd, ed = select.select(
[self.socket], [], [], self.socket.gettimeout())
if not rd:
- raise timeout()
+ raise timeout('The read operation timed out')
else:
return self.recv(*args, **kwargs)
else:
@@ -193,7 +203,10 @@ class WrappedSocket(object):
return self.connection.sendall(data)
def close(self):
- return self.connection.shutdown()
+ if self._makefile_refs < 1:
+ return self.connection.shutdown()
+ else:
+ self._makefile_refs -= 1
def getpeercert(self, binary_form=False):
x509 = self.connection.get_peer_certificate()
@@ -216,6 +229,15 @@ class WrappedSocket(object):
]
}
+ def _reuse(self):
+ self._makefile_refs += 1
+
+ def _drop(self):
+ if self._makefile_refs < 1:
+ self.close()
+ else:
+ self._makefile_refs -= 1
+
def _verify_callback(cnx, x509, err_no, err_depth, return_code):
return err_no == 0
diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py
index fff8bfa..7519ba9 100644
--- a/urllib3/exceptions.py
+++ b/urllib3/exceptions.py
@@ -60,7 +60,14 @@ ConnectionError = ProtocolError
## Leaf Exceptions
class MaxRetryError(RequestError):
- "Raised when the maximum number of retries is exceeded."
+ """Raised when the maximum number of retries is exceeded.
+
+ :param pool: The connection pool
+ :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
+ :param string url: The requested Url
+ :param exceptions.Exception reason: The underlying error
+
+ """
def __init__(self, pool, url, reason=None):
self.reason = reason
@@ -134,6 +141,16 @@ class LocationParseError(LocationValueError):
self.location = location
-class InsecureRequestWarning(HTTPWarning):
+class SecurityWarning(HTTPWarning):
+ "Warned when perfoming security reducing actions"
+ pass
+
+
+class InsecureRequestWarning(SecurityWarning):
"Warned when making an unverified HTTPS request."
pass
+
+
+class SystemTimeWarning(SecurityWarning):
+ "Warned when system time is suspected to be wrong"
+ pass
diff --git a/urllib3/response.py b/urllib3/response.py
index 7e0d47f..e69de95 100644
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -48,7 +48,10 @@ class HTTPResponse(io.IOBase):
HTTP Response container.
Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
- loaded and decoded on-demand when the ``data`` property is accessed.
+ loaded and decoded on-demand when the ``data`` property is accessed. This
+ class is also compatible with the Python standard library's :mod:`io`
+ module, and can hence be treated as a readable object in the context of that
+ framework.
Extra parameters for behaviour not present in httplib.HTTPResponse:
@@ -317,4 +320,14 @@ class HTTPResponse(io.IOBase):
return self._fp.flush()
def readable(self):
+ # This method is required for `io` module compatibility.
return True
+
+ def readinto(self, b):
+ # This method is required for `io` module compatibility.
+ temp = self.read(len(b))
+ if len(temp) == 0:
+ return 0
+ else:
+ b[:len(temp)] = temp
+ return len(temp)
diff --git a/urllib3/util/connection.py b/urllib3/util/connection.py
index 062ee9d..2156993 100644
--- a/urllib3/util/connection.py
+++ b/urllib3/util/connection.py
@@ -66,13 +66,15 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
sock = None
try:
sock = socket.socket(af, socktype, proto)
+
+ # If provided, set socket level options before connecting.
+ # This is the only addition urllib3 makes to this function.
+ _set_socket_options(sock, socket_options)
+
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
- # If provided, set socket level options before connecting.
- # This is the only addition urllib3 makes to this function.
- _set_socket_options(sock, socket_options)
sock.connect(sa)
return sock
diff --git a/urllib3/util/response.py b/urllib3/util/response.py
index d0325bc..45fff55 100644
--- a/urllib3/util/response.py
+++ b/urllib3/util/response.py
@@ -5,9 +5,18 @@ def is_fp_closed(obj):
:param obj:
The file-like object to check.
"""
- if hasattr(obj, 'fp'):
- # Object is a container for another file-like object that gets released
- # on exhaustion (e.g. HTTPResponse)
+
+ try:
+ # Check via the official file-like-object way.
+ return obj.closed
+ except AttributeError:
+ pass
+
+ try:
+ # Check if the object is a container for another file-like object that
+ # gets released on exhaustion (e.g. HTTPResponse).
return obj.fp is None
+ except AttributeError:
+ pass
- return obj.closed
+ raise ValueError("Unable to determine whether fp is closed.")
diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py
index 9013197..eb560df 100644
--- a/urllib3/util/retry.py
+++ b/urllib3/util/retry.py
@@ -83,7 +83,7 @@ class Retry(object):
same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
:param iterable status_forcelist:
- A set of HTTP status codes that we should force a retry on.
+ A set of HTTP status codes that we should force a retry on.
By default, this is disabled with ``None``.