diff options
Diffstat (limited to 'docs')
| -rw-r--r-- | docs/Makefile | 130 | ||||
| -rw-r--r-- | docs/README | 14 | ||||
| -rw-r--r-- | docs/collections.rst | 13 | ||||
| -rw-r--r-- | docs/conf.py | 232 | ||||
| -rw-r--r-- | docs/contrib.rst | 14 | ||||
| -rw-r--r-- | docs/doc-requirements.txt | 12 | ||||
| -rw-r--r-- | docs/exceptions.rst | 7 | ||||
| -rw-r--r-- | docs/helpers.rst | 55 | ||||
| -rw-r--r-- | docs/index.rst | 340 | ||||
| -rw-r--r-- | docs/make.bat | 170 | ||||
| -rw-r--r-- | docs/managers.rst | 62 | ||||
| -rw-r--r-- | docs/pools.rst | 71 | ||||
| -rw-r--r-- | docs/security.rst | 161 |
13 files changed, 1281 insertions, 0 deletions
diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..135c543 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,130 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/urllib3.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/urllib3.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/urllib3" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/urllib3" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/README b/docs/README new file mode 100644 index 0000000..9126c73 --- /dev/null +++ b/docs/README @@ -0,0 +1,14 @@ +# Building the Docs + +First install Sphinx: + + pip install sphinx + +Install pyopenssl and certifi dependencies, to avoid some build errors. (Optional) + + # This step is optional + pip install ndg-httpsclient pyasn1 certifi + +Then build: + + cd docs && make html diff --git a/docs/collections.rst b/docs/collections.rst new file mode 100644 index 0000000..b348140 --- /dev/null +++ b/docs/collections.rst @@ -0,0 +1,13 @@ +Collections +=========== + +These datastructures are used to implement the behaviour of various urllib3 +components in a decoupled and application-agnostic design. + +.. automodule:: urllib3._collections + + .. autoclass:: RecentlyUsedContainer + :members: + + .. autoclass:: HTTPHeaderDict + :members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..7ac8393 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +# +# urllib3 documentation build configuration file, created by +# sphinx-quickstart on Wed Oct 5 13:15:40 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +from datetime import date +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.insert(0, root_path) + +import urllib3 + + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', +] + +# Test code blocks only when explicitly specified +doctest_test_doctest_blocks = '' + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'urllib3' +copyright = u'{year}, Andrey Petrov'.format(year=date.today().year) + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = urllib3.__version__ +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'urllib3doc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'urllib3.tex', u'urllib3 Documentation', + u'Andrey Petrov', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'urllib3', u'urllib3 Documentation', + [u'Andrey Petrov'], 1) +] + +intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} diff --git a/docs/contrib.rst b/docs/contrib.rst new file mode 100644 index 0000000..99c5492 --- /dev/null +++ b/docs/contrib.rst @@ -0,0 +1,14 @@ +.. _contrib-modules: + +Contrib Modules +=============== + +These modules implement various extra features, that may not be ready for +prime time. + +.. _pyopenssl: + +SNI-support for Python 2 +------------------------ + +.. automodule:: urllib3.contrib.pyopenssl diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt new file mode 100644 index 0000000..b7b6d66 --- /dev/null +++ b/docs/doc-requirements.txt @@ -0,0 +1,12 @@ +ndg-httpsclient==0.3.2 +pyasn1==0.1.7 +Sphinx==1.2.2 +Jinja2==2.7.3 +MarkupSafe==0.23 +Pygments==1.6 +cryptography==0.4 +six==1.7.2 +cffi==0.8.2 +docutils==0.11 +pycparser==2.10 +certifi==14.05.14
\ No newline at end of file diff --git a/docs/exceptions.rst b/docs/exceptions.rst new file mode 100644 index 0000000..f9e0553 --- /dev/null +++ b/docs/exceptions.rst @@ -0,0 +1,7 @@ +Exceptions +========== + +Custom exceptions defined by urllib3 + +.. automodule:: urllib3.exceptions + :members: diff --git a/docs/helpers.rst b/docs/helpers.rst new file mode 100644 index 0000000..79f268b --- /dev/null +++ b/docs/helpers.rst @@ -0,0 +1,55 @@ +Helpers +======= + +Useful methods for working with :mod:`httplib`, completely decoupled from +code specific to **urllib3**. + + +Timeouts +-------- + +.. automodule:: urllib3.util.timeout + :members: + +Retries +------- + +.. automodule:: urllib3.util.retry + :members: + +URL Helpers +----------- + +.. automodule:: urllib3.util.url + :members: + +Filepost +-------- + +.. automodule:: urllib3.filepost + :members: + +.. automodule:: urllib3.fields + :members: + +Request +------- + +.. automodule:: urllib3.request + :members: + +.. automodule:: urllib3.util.request + :members: + +Response +-------- + +.. automodule:: urllib3.response + :members: + :undoc-members: + +SSL/TLS Helpers +--------------- + +.. automodule:: urllib3.util.ssl_ + :members: diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..1fc8a9c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,340 @@ +===================== +urllib3 Documentation +===================== + +.. toctree:: + :hidden: + + pools + managers + security + helpers + collections + contrib + + +Highlights +========== + +- Re-use the same socket connection for multiple requests, with optional + client-side certificate verification. See: + :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.connectionpool.HTTPSConnectionPool` + +- File posting. See: + :func:`~urllib3.filepost.encode_multipart_formdata` + +- Built-in redirection and retries (optional). + +- Supports gzip and deflate decoding. See: + :func:`~urllib3.response.decode_gzip` and + :func:`~urllib3.response.decode_deflate` + +- Thread-safe and sanity-safe. + +- Tested on Python 2.6+ and Python 3.2+, 100% unit test coverage. + +- Works with AppEngine, gevent, eventlib, and the standard library :mod:`io` module. + +- Small and easy to understand codebase perfect for extending and building upon. + For a more comprehensive solution, have a look at + `Requests <http://python-requests.org/>`_ which is also powered by urllib3. + + +Getting Started +=============== + +Installing +---------- + +``pip install urllib3`` or fetch the latest source from +`github.com/shazow/urllib3 <https://github.com/shazow/urllib3>`_. + +Usage +----- + +.. doctest :: + + >>> import urllib3 + >>> http = urllib3.PoolManager() + >>> r = http.request('GET', 'http://example.com/') + >>> r.status + 200 + >>> r.headers['server'] + 'ECS (iad/182A)' + >>> 'data: ' + r.data + 'data: ...' + + +**By default, urllib3 does not verify your HTTPS requests**. +You'll need to supply a root certificate bundle, or use `certifi +<https://certifi.io/>`_ + +.. doctest :: + + >>> import urllib3, certifi + >>> http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) + >>> r = http.request('GET', 'https://insecure.com/') + Traceback (most recent call last): + ... + SSLError: hostname 'insecure.com' doesn't match 'svn.nmap.org' + +For more on making secure SSL/TLS HTTPS requests, read the :ref:`Security +section <security>`. + + +urllib3's responses respect the :mod:`io` framework from Python's +standard library, allowing use of these standard objects for purposes +like buffering: + +.. doctest :: + + >>> http = urllib3.PoolManager() + >>> r = http.urlopen('GET','http://example.com/', preload_content=False) + >>> b = io.BufferedReader(r, 2048) + >>> firstpart = b.read(100) + >>> # ... your internet connection fails momentarily ... + >>> secondpart = b.read() + + +Components +========== + +:mod:`urllib3` tries to strike a fine balance between power, extendability, and +sanity. To achieve this, the codebase is a collection of small reusable +utilities and abstractions composed together in a few helpful layers. + + +PoolManager +----------- + +The highest level is the :doc:`PoolManager(...) <managers>`. + +The :class:`~urllib3.poolmanagers.PoolManager` will take care of reusing +connections for you whenever you request the same host. This should cover most +scenarios without significant loss of efficiency, but you can always drop down +to a lower level component for more granular control. + +.. doctest :: + + >>> import urllib3 + >>> http = urllib3.PoolManager(10) + >>> r1 = http.request('GET', 'http://example.com/') + >>> r2 = http.request('GET', 'http://httpbin.org/') + >>> r3 = http.request('GET', 'http://httpbin.org/get') + >>> len(http.pools) + 2 + +A :class:`~urllib3.poolmanagers.PoolManager` is a proxy for a collection of +:class:`ConnectionPool` objects. They both inherit from +:class:`~urllib3.request.RequestMethods` to make sure that their API is +similar, so that instances of either can be passed around interchangeably. + + +ProxyManager +------------ + +The :class:`~urllib3.poolmanagers.ProxyManager` is an HTTP proxy-aware +subclass of :class:`~urllib3.poolmanagers.PoolManager`. It produces a single +:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP +connections and individual per-server:port +:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled +HTTPS connections: + +:: + + >>> proxy = urllib3.ProxyManager('http://localhost:3128/') + >>> r1 = proxy.request('GET', 'http://google.com/') + >>> r2 = proxy.request('GET', 'http://httpbin.org/') + >>> len(proxy.pools) + 1 + >>> r3 = proxy.request('GET', 'https://httpbin.org/') + >>> r4 = proxy.request('GET', 'https://twitter.com/') + >>> len(proxy.pools) + 3 + + +ConnectionPool +-------------- + +The next layer is the :doc:`ConnectionPool(...) <pools>`. + +The :class:`~urllib3.connectionpool.HTTPConnectionPool` and +:class:`~urllib3.connectionpool.HTTPSConnectionPool` classes allow you to +define a pool of connections to a single host and make requests against this +pool with automatic **connection reusing** and **thread safety**. + +When the :mod:`ssl` module is available, then +:class:`~urllib3.connectionpool.HTTPSConnectionPool` objects can be configured +to check SSL certificates against specific provided certificate authorities. + +.. doctest :: + + >>> import urllib3 + >>> conn = urllib3.connection_from_url('http://httpbin.org/') + >>> r1 = conn.request('GET', 'http://httpbin.org/') + >>> r2 = conn.request('GET', '/user-agent') + >>> r3 = conn.request('GET', 'http://example.com') + Traceback (most recent call last): + ... + urllib3.exceptions.HostChangedError: HTTPConnectionPool(host='httpbin.org', port=None): Tried to open a foreign host with url: http://example.com + +Again, a ConnectionPool is a pool of connections to a specific host. Trying to +access a different host through the same pool will raise a ``HostChangedError`` +exception unless you specify ``assert_same_host=False``. Do this at your own +risk as the outcome is completely dependent on the behaviour of the host server. + +If you need to access multiple hosts and don't want to manage your own +collection of :class:`~urllib3.connectionpool.ConnectionPool` objects, then you +should use a :class:`~urllib3.poolmanager.PoolManager`. + +A :class:`~urllib3.connectionpool.ConnectionPool` is composed of a collection +of :class:`httplib.HTTPConnection` objects. + + +Timeout +------- + +A timeout can be set to abort socket operations on individual connections after +the specified duration. The timeout can be defined as a float or an instance of +:class:`~urllib3.util.timeout.Timeout` which gives more granular configuration +over how much time is allowed for different stages of the request. This can be +set for the entire pool or per-request. + +.. doctest :: + + >>> from urllib3 import PoolManager, Timeout + + >>> # Manager with 3 seconds combined timeout. + >>> http = PoolManager(timeout=3.0) + >>> r = http.request('GET', 'http://httpbin.org/delay/1') + + >>> # Manager with 2 second timeout for the read phase, no limit for the rest. + >>> http = PoolManager(timeout=Timeout(read=2.0)) + >>> r = http.request('GET', 'http://httpbin.org/delay/1') + + >>> # Manager with no timeout but a request with a timeout of 1 seconds for + >>> # the connect phase and 2 seconds for the read phase. + >>> http = PoolManager() + >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(connect=1.0, read=2.0)) + + >>> # Same Manager but request with a 5 second total timeout. + >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(total=5.0)) + +See the :class:`~urllib3.util.timeout.Timeout` definition for more details. + + +Retry +----- + +Retries can be configured by passing an instance of +:class:`~urllib3.util.retry.Retry`, or disabled by passing ``False``, to the +``retries`` parameter. + +Redirects are also considered to be a subset of retries but can be configured or +disabled individually. + +:: + + >>> from urllib3 import PoolManager, Retry + + >>> # Allow 3 retries total for all requests in this pool. These are the same: + >>> http = PoolManager(retries=3) + >>> http = PoolManager(retries=Retry(3)) + >>> http = PoolManager(retries=Retry(total=3)) + + >>> r = http.request('GET', 'http://httpbin.org/redirect/2') + >>> # r.status -> 200 + + >>> # Disable redirects for this request. + >>> r = http.request('GET', 'http://httpbin.org/redirect/2', retries=Retry(3, redirect=False)) + >>> # r.status -> 302 + + >>> # No total limit, but only do 5 connect retries, for this request. + >>> r = http.request('GET', 'http://httpbin.org/', retries=Retry(connect=5)) + + +See the :class:`~urllib3.util.retry.Retry` definition for more details. + + +Foundation +---------- + +At the very core, just like its predecessors, :mod:`urllib3` is built on top of +:mod:`httplib` -- the lowest level HTTP library included in the Python +standard library. + +To aid the limited functionality of the :mod:`httplib` module, :mod:`urllib3` +provides various helper methods which are used with the higher level components +but can also be used independently. + +.. toctree:: + + helpers + exceptions + + +Contrib Modules +--------------- + +These modules implement various extra features, that may not be ready for +prime time. + +.. toctree:: + + contrib + + +Contributing +============ + +#. `Check for open issues <https://github.com/shazow/urllib3/issues>`_ or open + a fresh issue to start a discussion around a feature idea or a bug. There is + a *Contributor Friendly* tag for issues that should be ideal for people who + are not very familiar with the codebase yet. +#. Fork the `urllib3 repository on Github <https://github.com/shazow/urllib3>`_ + to start making your changes. +#. Write a test which shows that the bug was fixed or that the feature works + as expected. +#. Send a pull request and bug the maintainer until it gets merged and published. + :) Make sure to add yourself to ``CONTRIBUTORS.txt``. + + +Sponsorship +=========== + +Please consider sponsoring urllib3 development, especially if your company +benefits from this library. + +* **Project Grant**: A grant for contiguous full-time development has the + biggest impact for progress. Periods of 3 to 10 days allow a contributor to + tackle substantial complex issues which are otherwise left to linger until + somebody can't afford to not fix them. + + Contact `@shazow <https://github.com/shazow>`_ to arrange a grant for a core + contributor. + +* **One-off**: Development will continue regardless of funding, but donations help move + things further along quicker as the maintainer can allocate more time off to + work on urllib3 specifically. + + .. raw:: html + + <a href="https://donorbox.org/personal-sponsor-urllib3" style="background-color:#1275ff;color:#fff;text-decoration:none;font-family:Verdana,sans-serif;display:inline-block;font-size:14px;padding:7px 16px;border-radius:5px;margin-right:2em;vertical-align:top;border:1px solid rgba(160,160,160,0.5);background-image:linear-gradient(#7dc5ee,#008cdd 85%,#30a2e4);box-shadow:inset 0 1px 0 rgba(255,255,255,0.25);">Sponsor with Credit Card</a> + + <a class="coinbase-button" data-code="137087702cf2e77ce400d53867b164e6" href="https://coinbase.com/checkouts/137087702cf2e77ce400d53867b164e6">Sponsor with Bitcoin</a><script src="https://coinbase.com/assets/button.js" type="text/javascript"></script> + +* **Recurring**: You're welcome to `support the maintainer on Gittip + <https://www.gittip.com/shazow/>`_. + + +Recent Sponsors +--------------- + +Huge thanks to all the companies and individuals who financially contributed to +the development of urllib3. Please send a PR if you've donated and would like +to be listed. + +* `Stripe <https://stripe.com/>`_ (June 23, 2014) + +.. * [Company] ([optional tagline]), [optional description of grant] ([date]) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..41aa35b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,170 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^<target^>` where ^<target^> is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\urllib3.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\urllib3.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/docs/managers.rst b/docs/managers.rst new file mode 100644 index 0000000..f9cab03 --- /dev/null +++ b/docs/managers.rst @@ -0,0 +1,62 @@ +PoolManager +=========== + +.. automodule:: urllib3.poolmanager + +A pool manager is an abstraction for a collection of +:doc:`ConnectionPools <pools>`. + +If you need to make requests to multiple hosts, then you can use a +:class:`.PoolManager`, which takes care of maintaining your pools +so you don't have to. + +.. doctest :: + + >>> from urllib3 import PoolManager + >>> manager = PoolManager(10) + >>> r = manager.request('GET', 'http://example.com') + >>> r.headers['server'] + 'ECS (iad/182A)' + >>> r = manager.request('GET', 'http://httpbin.org/') + >>> r.headers['server'] + 'gunicorn/18.0' + >>> r = manager.request('POST', 'http://httpbin.org/headers') + >>> r = manager.request('HEAD', 'http://httpbin.org/cookies') + >>> len(manager.pools) + 2 + >>> conn = manager.connection_from_host('httpbin.org') + >>> conn.num_requests + 3 + +The API of a :class:`.PoolManager` object is similar to that of a +:doc:`ConnectionPool <pools>`, so they can be passed around interchangeably. + +The PoolManager uses a Least Recently Used (LRU) policy for discarding old +pools. That is, if you set the PoolManager ``num_pools`` to 10, then after +making requests to 11 or more different hosts, the least recently used pools +will be cleaned up eventually. + +Cleanup of stale pools does not happen immediately. You can read more about the +implementation and the various adjustable variables within +:class:`~urllib3._collections.RecentlyUsedContainer`. + +API +--- + + .. autoclass:: PoolManager + :inherited-members: + +ProxyManager +============ + +:class:`.ProxyManager` is an HTTP proxy-aware subclass of :class:`.PoolManager`. +It produces a single +:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP +connections and individual per-server:port +:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled +HTTPS connections. + +API +--- + .. autoclass:: ProxyManager + diff --git a/docs/pools.rst b/docs/pools.rst new file mode 100644 index 0000000..63cb7d1 --- /dev/null +++ b/docs/pools.rst @@ -0,0 +1,71 @@ +ConnectionPools +=============== + +.. automodule:: urllib3.connectionpool + +A connection pool is a container for a collection of connections to a specific +host. + +If you need to make requests to the same host repeatedly, then you should use a +:class:`.HTTPConnectionPool`. + +.. doctest :: + + >>> from urllib3 import HTTPConnectionPool + >>> pool = HTTPConnectionPool('ajax.googleapis.com', maxsize=1) + >>> r = pool.request('GET', '/ajax/services/search/web', + ... fields={'q': 'urllib3', 'v': '1.0'}) + >>> r.status + 200 + >>> r.headers['content-type'] + 'text/javascript; charset=utf-8' + >>> 'data: ' + r.data # Content of the response + 'data: ...' + >>> r = pool.request('GET', '/ajax/services/search/web', + ... fields={'q': 'python', 'v': '1.0'}) + >>> 'data: ' + r.data # Content of the response + 'data: ...' + >>> pool.num_connections + 1 + >>> pool.num_requests + 2 + +By default, the pool will cache just one connection. If you're planning on using +such a pool in a multithreaded environment, you should set the ``maxsize`` of +the pool to a higher number, such as the number of threads. You can also control +many other variables like timeout, blocking, and default headers. + +Helpers +------- + +There are various helper functions provided for instantiating these +ConnectionPools more easily: + + .. autofunction:: connection_from_url + +API +--- + +:mod:`urllib3.connectionpool` comes with two connection pools: + + .. autoclass:: HTTPConnectionPool + :members: + :inherited-members: + + .. autoclass:: HTTPSConnectionPool + + +All of these pools inherit from a common base class: + + .. autoclass:: ConnectionPool + +.. module:: urllib3.connection + +Related Classes +--------------- + +urllib3 implements its own :class:`HTTPConnection` object to allow for more +flexibility than the standard library's implementation. + +.. autoclass:: HTTPConnection + :members: diff --git a/docs/security.rst b/docs/security.rst new file mode 100644 index 0000000..5321e24 --- /dev/null +++ b/docs/security.rst @@ -0,0 +1,161 @@ +.. _security: + +Security: Verified HTTPS with SSL/TLS +===================================== + +Very important fact: **By default, urllib3 does not verify HTTPS requests.** + +The historic reason for this is that we rely on ``httplib`` for some of the +HTTP protocol implementation, and ``httplib`` does not verify requests out of +the box. This is not a good reason, but here we are. + +Luckily, it's not too hard to enable verified HTTPS requests and there are a +few ways to do it. + + +Python with SSL enabled +----------------------- + +First we need to make sure your Python installation has SSL enabled. Easiest +way to check is to simply open a Python shell and type `import ssl`:: + + >>> import ssl + Traceback (most recent call last): + ... + ImportError: No module named _ssl + +If you got an ``ImportError``, then your Python is not compiled with SSL support +and you'll need to re-install it. Read +`this StackOverflow thread <https://stackoverflow.com/questions/5128845/importerror-no-module-named-ssl>`_ +for details. + +Otherwise, if ``ssl`` imported cleanly, then we're ready to setup our certificates: +:ref:`certifi-with-urllib3`. + + +Enabling SSL on Google AppEngine +++++++++++++++++++++++++++++++++ + +If you're using Google App Engine, you'll need to add ``ssl`` as a library +dependency to your yaml file, like this:: + + libraries: + - name: ssl + version: latest + +If it's still not working, you may need to enable billing on your account +to `enable using sockets +<https://developers.google.com/appengine/docs/python/sockets/>`_. + + +.. _certifi-with-urllib3: + +Using Certifi with urllib3 +-------------------------- + +`Certifi <http://certifi.io/>`_ is a package which ships with Mozilla's root +certificates for easy programmatic access. + +1. Install the Python ``certifi`` package:: + + $ pip install certifi + +2. Setup your pool to require a certificate and provide the certifi bundle:: + + import urllib3 + import certifi + + http = urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', # Force certificate check. + ca_certs=certifi.where(), # Path to the Certifi bundle. + ) + + # You're ready to make verified HTTPS requests. + try: + r = http.request('GET', 'https://example.com/') + except urllib3.exceptions.SSLError as e: + # Handle incorrect certificate error. + ... + +Make sure to update your ``certifi`` package regularly to get the latest root +certificates. + + +Using your system's root certificates +------------------------------------- + +Your system's root certificates may be more up-to-date than maintaining your +own, but the trick is finding where they live. Different operating systems have +them in different places. + +For example, on most Linux distributions they're at +``/etc/ssl/certs/ca-certificates.crt``. On Windows and OS X? `It's not so simple +<https://stackoverflow.com/questions/10095676/openssl-reasonable-default-for-trusted-ca-certificates>`_. + +Once you find your root certificate file:: + + import urllib3 + + ca_certs = "/etc/ssl/certs/ca-certificates.crt" # Or wherever it lives. + + http = urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', # Force certificate check. + ca_certs=ca_certs, # Path to your certificate bundle. + ) + + # You're ready to make verified HTTPS requests. + try: + r = http.request('GET', 'https://example.com/') + except urllib3.exceptions.SSLError as e: + # Handle incorrect certificate error. + ... + + +OpenSSL / PyOpenSSL +------------------- + +By default, we use the standard library's ``ssl`` module. Unfortunately, there +are several limitations which are addressed by PyOpenSSL: + +- (Python 2.x) SNI support. +- (Python 2.x-3.2) Disabling compression to mitigate `CRIME attack + <https://en.wikipedia.org/wiki/CRIME_(security_exploit)>`_. + +To use the Python OpenSSL bindings instead, you'll need to install the required +packages:: + + $ pip install pyopenssl ndg-httpsclient pyasn1 + +Once the packages are installed, you can tell urllib3 to switch the ssl backend +to PyOpenSSL with :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3`:: + + import urllib3.contrib.pyopenssl + urllib3.contrib.pyopenssl.inject_into_urllib3() + +Now you can continue using urllib3 as you normally would. + +For more details, check the :mod:`~urllib3.contrib.pyopenssl` module. + + +InsecureRequestWarning +---------------------- + +.. versionadded:: 1.9 + +Unverified HTTPS requests will trigger a warning:: + + urllib3/connectionpool.py:736: InsecureRequestWarning: Unverified HTTPS + request is being made. Adding certificate verification is strongly advised. + See: https://urllib3.readthedocs.org/en/latest/security.html + (This warning will only appear once by default.) + +This would be a great time to enable HTTPS verification: +:ref:`certifi-with-urllib3`. + +If you know what you're doing and would like to disable this and other warnings, +you can use :func:`~urllib3.disable_warnings`:: + + import urllib3 + urllib3.disable_warnings() + +Making unverified HTTPS requests is strongly discouraged. ˙ ͜ʟ˙ |