Imported Upstream version 1.5

7 files changed, 291 insertions, 159 deletions

diff --git a/test/__init__.py b/test/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/test/__init__.py
+++ /dev/null
diff --git a/test/benchmark.py b/test/benchmark.py
deleted file mode 100644
index e7049c4..0000000
--- a/test/benchmark.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Really simple rudimentary benchmark to compare ConnectionPool versus standard
-urllib to demonstrate the usefulness of connection re-using.
-"""
-from __future__ import print_function
-
-import sys
-import time
-import urllib
-
-sys.path.append('../')
-import urllib3
-
-
-# URLs to download. Doesn't matter as long as they're from the same host, so we
-# can take advantage of connection re-using.
-TO_DOWNLOAD = [
-    'http://code.google.com/apis/apps/',
-    'http://code.google.com/apis/base/',
-    'http://code.google.com/apis/blogger/',
-    'http://code.google.com/apis/calendar/',
-    'http://code.google.com/apis/codesearch/',
-    'http://code.google.com/apis/contact/',
-    'http://code.google.com/apis/books/',
-    'http://code.google.com/apis/documents/',
-    'http://code.google.com/apis/finance/',
-    'http://code.google.com/apis/health/',
-    'http://code.google.com/apis/notebook/',
-    'http://code.google.com/apis/picasaweb/',
-    'http://code.google.com/apis/spreadsheets/',
-    'http://code.google.com/apis/webmastertools/',
-    'http://code.google.com/apis/youtube/',
-]
-
-
-def urllib_get(url_list):
-    assert url_list
-    for url in url_list:
-        now = time.time()
-        r = urllib.urlopen(url)
-        elapsed = time.time() - now
-        print("Got in %0.3f: %s" % (elapsed, url))
-
-
-def pool_get(url_list):
-    assert url_list
-    pool = urllib3.connection_from_url(url_list[0])
-    for url in url_list:
-        now = time.time()
-        r = pool.get_url(url)
-        elapsed = time.time() - now
-        print("Got in %0.3fs: %s" % (elapsed, url))
-
-
-if __name__ == '__main__':
-    print("Running pool_get ...")
-    now = time.time()
-    pool_get(TO_DOWNLOAD)
-    pool_elapsed = time.time() - now
-
-    print("Running urllib_get ...")
-    now = time.time()
-    urllib_get(TO_DOWNLOAD)
-    urllib_elapsed = time.time() - now
-
-    print("Completed pool_get in %0.3fs" % pool_elapsed)
-    print("Completed urllib_get in %0.3fs" % urllib_elapsed)
-
-
-"""
-Example results:
-
-Completed pool_get in 1.163s
-Completed urllib_get in 2.318s
-"""
diff --git a/test/test_collections.py b/test/test_collections.py
index 6cb5aca..098b31a 100644
--- a/test/test_collections.py
+++ b/test/test_collections.py
@@ -36,19 +36,7 @@ class TestLRUContainer(unittest.TestCase):
         d[5] = '5'
 
         # Check state
-        self.assertEqual(list(d.keys()), [0, 2, 3, 4, 5])
-
-    def test_pruning(self):
-        d = Container(5)
-
-        for i in xrange(5):
-            d[i] = str(i)
-
-        # Contend 2 entries for the most-used slot to balloon the heap
-        for i in xrange(100):
-            d.get(i % 2)
-
-        self.assertTrue(len(d.access_log) <= d.CLEANUP_FACTOR * d._maxsize)
+        self.assertEqual(list(d.keys()), [2, 3, 4, 0, 5])
 
     def test_same_key(self):
         d = Container(5)
@@ -57,10 +45,7 @@ class TestLRUContainer(unittest.TestCase):
             d['foo'] = i
 
         self.assertEqual(list(d.keys()), ['foo'])
-
-        d._prune_invalidated_entries()
-
-        self.assertEqual(len(d.access_log), 1)
+        self.assertEqual(len(d), 1)
 
     def test_access_ordering(self):
         d = Container(5)
@@ -68,13 +53,14 @@ class TestLRUContainer(unittest.TestCase):
         for i in xrange(10):
             d[i] = True
 
-        self.assertEqual(d._get_ordered_access_keys(), [9,8,7,6,5])
+        # Keys should be ordered by access time
+        self.assertEqual(list(d.keys()), [5, 6, 7, 8, 9])
 
         new_order = [7,8,6,9,5]
-        for k in reversed(new_order):
+        for k in new_order:
             d[k]
 
-        self.assertEqual(d._get_ordered_access_keys(), new_order)
+        self.assertEqual(list(d.keys()), new_order)
 
     def test_delete(self):
         d = Container(5)
@@ -107,6 +93,35 @@ class TestLRUContainer(unittest.TestCase):
 
         self.assertRaises(KeyError, lambda: d[5])
 
+    def test_disposal(self):
+        evicted_items = []
+
+        def dispose_func(arg):
+            # Save the evicted datum for inspection
+            evicted_items.append(arg)
+
+        d = Container(5, dispose_func=dispose_func)
+        for i in xrange(5):
+            d[i] = i
+        self.assertEqual(list(d.keys()), list(xrange(5)))
+        self.assertEqual(evicted_items, []) # Nothing disposed
+
+        d[5] = 5
+        self.assertEqual(list(d.keys()), list(xrange(1, 6)))
+        self.assertEqual(evicted_items, [0])
+
+        del d[1]
+        self.assertEqual(evicted_items, [0, 1])
+
+        d.clear()
+        self.assertEqual(evicted_items, [0, 1, 2, 3, 4, 5])
+
+    def test_iter(self):
+        d = Container()
+
+        with self.assertRaises(NotImplementedError):
+            for i in d:
+                self.fail("Iteration shouldn't be implemented.")
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
index c32c6dc..afc3098 100644
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -1,30 +1,28 @@
 import unittest
 
 from urllib3.connectionpool import connection_from_url, HTTPConnectionPool
-from urllib3.util import get_host, make_headers
-from urllib3.exceptions import EmptyPoolError, LocationParseError
+from urllib3.packages.ssl_match_hostname import CertificateError
+from urllib3.exceptions import (
+    ClosedPoolError,
+    EmptyPoolError,
+    HostChangedError,
+    MaxRetryError,
+    SSLError,
+    TimeoutError,
+)
+
+from socket import timeout as SocketTimeout
+from ssl import SSLError as BaseSSLError
+
+try:   # Python 3
+    from queue import Empty
+    from http.client import HTTPException
+except ImportError:
+    from Queue import Empty
+    from httplib import HTTPException
 
 
 class TestConnectionPool(unittest.TestCase):
-    def test_get_host(self):
-        url_host_map = {
-            'http://google.com/mail': ('http', 'google.com', None),
-            'http://google.com/mail/': ('http', 'google.com', None),
-            'google.com/mail': ('http', 'google.com', None),
-            'http://google.com/': ('http', 'google.com', None),
-            'http://google.com': ('http', 'google.com', None),
-            'http://www.google.com': ('http', 'www.google.com', None),
-            'http://mail.google.com': ('http', 'mail.google.com', None),
-            'http://google.com:8000/mail/': ('http', 'google.com', 8000),
-            'http://google.com:8000': ('http', 'google.com', 8000),
-            'https://google.com': ('https', 'google.com', None),
-            'https://google.com:8000': ('https', 'google.com', 8000),
-            'http://user:password@127.0.0.1:1234': ('http', '127.0.0.1', 1234),
-        }
-        for url, expected_host in url_host_map.items():
-            returned_host = get_host(url)
-            self.assertEquals(returned_host, expected_host)
-
     def test_same_host(self):
         same_host = [
             ('http://google.com/', '/'),
@@ -50,45 +48,6 @@ class TestConnectionPool(unittest.TestCase):
             c = connection_from_url(a)
             self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b))
 
-    def test_invalid_host(self):
-        # TODO: Add more tests
-        invalid_host = [
-            'http://google.com:foo',
-        ]
-
-        for location in invalid_host:
-            self.assertRaises(LocationParseError, get_host, location)
-
-
-    def test_make_headers(self):
-        self.assertEqual(
-            make_headers(accept_encoding=True),
-            {'accept-encoding': 'gzip,deflate'})
-
-        self.assertEqual(
-            make_headers(accept_encoding='foo,bar'),
-            {'accept-encoding': 'foo,bar'})
-
-        self.assertEqual(
-            make_headers(accept_encoding=['foo', 'bar']),
-            {'accept-encoding': 'foo,bar'})
-
-        self.assertEqual(
-            make_headers(accept_encoding=True, user_agent='banana'),
-            {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'})
-
-        self.assertEqual(
-            make_headers(user_agent='banana'),
-            {'user-agent': 'banana'})
-
-        self.assertEqual(
-            make_headers(keep_alive=True),
-            {'connection': 'keep-alive'})
-
-        self.assertEqual(
-            make_headers(basic_auth='foo:bar'),
-            {'authorization': 'Basic Zm9vOmJhcg=='})
-
     def test_max_connections(self):
         pool = HTTPConnectionPool(host='localhost', maxsize=1, block=True)
 
@@ -127,6 +86,67 @@ class TestConnectionPool(unittest.TestCase):
             str(EmptyPoolError(HTTPConnectionPool(host='localhost'), "Test.")),
             "HTTPConnectionPool(host='localhost', port=None): Test.")
 
+    def test_pool_size(self):
+        POOL_SIZE = 1
+        pool = HTTPConnectionPool(host='localhost', maxsize=POOL_SIZE, block=True)
+
+        def _raise(ex):
+            raise ex()
+
+        def _test(exception, expect):
+            pool._make_request = lambda *args, **kwargs: _raise(exception)
+            with self.assertRaises(expect):
+                pool.request('GET', '/')
+
+            self.assertEqual(pool.pool.qsize(), POOL_SIZE)
+
+        #make sure that all of the exceptions return the connection to the pool
+        _test(Empty, TimeoutError)
+        _test(SocketTimeout, TimeoutError)
+        _test(BaseSSLError, SSLError)
+        _test(CertificateError, SSLError)
+
+        # The pool should never be empty, and with these two exceptions being raised,
+        # a retry will be triggered, but that retry will fail, eventually raising
+        # MaxRetryError, not EmptyPoolError
+        # See: https://github.com/shazow/urllib3/issues/76
+        pool._make_request = lambda *args, **kwargs: _raise(HTTPException)
+        with self.assertRaises(MaxRetryError):
+            pool.request('GET', '/', retries=1, pool_timeout=0.01)
+        self.assertEqual(pool.pool.qsize(), POOL_SIZE)
+
+    def test_assert_same_host(self):
+        c = connection_from_url('http://google.com:80')
+
+        with self.assertRaises(HostChangedError):
+            c.request('GET', 'http://yahoo.com:80', assert_same_host=True)
+
+    def test_pool_close(self):
+        pool = connection_from_url('http://google.com:80')
+
+        # Populate with some connections
+        conn1 = pool._get_conn()
+        conn2 = pool._get_conn()
+        conn3 = pool._get_conn()
+        pool._put_conn(conn1)
+        pool._put_conn(conn2)
+
+        old_pool_queue = pool.pool
+
+        pool.close()
+        self.assertEqual(pool.pool, None)
+
+        with self.assertRaises(ClosedPoolError):
+            pool._get_conn()
+
+        pool._put_conn(conn3)
+
+        with self.assertRaises(ClosedPoolError):
+            pool._get_conn()
+
+        with self.assertRaises(Empty):
+            old_pool_queue.get(block=False)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py
index 12722f7..273abf9 100644
--- a/test/test_poolmanager.py
+++ b/test/test_poolmanager.py
@@ -2,6 +2,7 @@ import unittest
 
 from urllib3.poolmanager import PoolManager
 from urllib3 import connection_from_url
+from urllib3.exceptions import ClosedPoolError
 
 
 class TestPoolManager(unittest.TestCase):
@@ -42,6 +43,29 @@ class TestPoolManager(unittest.TestCase):
 
         self.assertEqual(len(connections), 5)
 
+    def test_manager_clear(self):
+        p = PoolManager(5)
+
+        conn_pool = p.connection_from_url('http://google.com')
+        self.assertEqual(len(p.pools), 1)
+
+        conn = conn_pool._get_conn()
+
+        p.clear()
+        self.assertEqual(len(p.pools), 0)
+
+        with self.assertRaises(ClosedPoolError):
+            conn_pool._get_conn()
+
+        conn_pool._put_conn(conn)
+
+        with self.assertRaises(ClosedPoolError):
+            conn_pool._get_conn()
+
+        self.assertEqual(len(p.pools), 0)
+
+
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_response.py b/test/test_response.py
index 0ef379c..964f677 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -1,9 +1,9 @@
 import unittest
-import zlib
 
 from io import BytesIO
 
 from urllib3.response import HTTPResponse
+from urllib3.exceptions import DecodeError
 
 class TestLegacyResponse(unittest.TestCase):
     def test_getheaders(self):
@@ -50,7 +50,7 @@ class TestResponse(unittest.TestCase):
 
     def test_decode_bad_data(self):
         fp = BytesIO(b'\x00' * 10)
-        self.assertRaises(zlib.error, HTTPResponse, fp, headers={
+        self.assertRaises(DecodeError, HTTPResponse, fp, headers={
             'content-encoding': 'deflate'
         })
 
diff --git a/test/test_util.py b/test/test_util.py
new file mode 100644
index 0000000..a989da6
--- /dev/null
+++ b/test/test_util.py
@@ -0,0 +1,150 @@
+import unittest
+import logging
+
+from urllib3 import add_stderr_logger
+from urllib3.util import get_host, make_headers, split_first, parse_url, Url
+from urllib3.exceptions import LocationParseError
+
+
+class TestUtil(unittest.TestCase):
+    def test_get_host(self):
+        url_host_map = {
+            # Hosts
+            'http://google.com/mail': ('http', 'google.com', None),
+            'http://google.com/mail/': ('http', 'google.com', None),
+            'google.com/mail': ('http', 'google.com', None),
+            'http://google.com/': ('http', 'google.com', None),
+            'http://google.com': ('http', 'google.com', None),
+            'http://www.google.com': ('http', 'www.google.com', None),
+            'http://mail.google.com': ('http', 'mail.google.com', None),
+            'http://google.com:8000/mail/': ('http', 'google.com', 8000),
+            'http://google.com:8000': ('http', 'google.com', 8000),
+            'https://google.com': ('https', 'google.com', None),
+            'https://google.com:8000': ('https', 'google.com', 8000),
+            'http://user:password@127.0.0.1:1234': ('http', '127.0.0.1', 1234),
+            'http://google.com/foo=http://bar:42/baz': ('http', 'google.com', None),
+            'http://google.com?foo=http://bar:42/baz': ('http', 'google.com', None),
+            'http://google.com#foo=http://bar:42/baz': ('http', 'google.com', None),
+
+            # IPv4
+            '173.194.35.7': ('http', '173.194.35.7', None),
+            'http://173.194.35.7': ('http', '173.194.35.7', None),
+            'http://173.194.35.7/test': ('http', '173.194.35.7', None),
+            'http://173.194.35.7:80': ('http', '173.194.35.7', 80),
+            'http://173.194.35.7:80/test': ('http', '173.194.35.7', 80),
+
+            # IPv6
+            '[2a00:1450:4001:c01::67]': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]/test': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]:80': ('http', '2a00:1450:4001:c01::67', 80),
+            'http://[2a00:1450:4001:c01::67]:80/test': ('http', '2a00:1450:4001:c01::67', 80),
+
+            # More IPv6 from http://www.ietf.org/rfc/rfc2732.txt
+            'http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8000/index.html': ('http', 'FEDC:BA98:7654:3210:FEDC:BA98:7654:3210', 8000),
+            'http://[1080:0:0:0:8:800:200C:417A]/index.html': ('http', '1080:0:0:0:8:800:200C:417A', None),
+            'http://[3ffe:2a00:100:7031::1]': ('http', '3ffe:2a00:100:7031::1', None),
+            'http://[1080::8:800:200C:417A]/foo': ('http', '1080::8:800:200C:417A', None),
+            'http://[::192.9.5.5]/ipng': ('http', '::192.9.5.5', None),
+            'http://[::FFFF:129.144.52.38]:42/index.html': ('http', '::FFFF:129.144.52.38', 42),
+            'http://[2010:836B:4179::836B:4179]': ('http', '2010:836B:4179::836B:4179', None),
+        }
+        for url, expected_host in url_host_map.items():
+            returned_host = get_host(url)
+            self.assertEquals(returned_host, expected_host)
+
+    def test_invalid_host(self):
+        # TODO: Add more tests
+        invalid_host = [
+            'http://google.com:foo',
+        ]
+
+        for location in invalid_host:
+            self.assertRaises(LocationParseError, get_host, location)
+
+    def test_parse_url(self):
+        url_host_map = {
+            'http://google.com/mail': Url('http', host='google.com', path='/mail'),
+            'http://google.com/mail/': Url('http', host='google.com', path='/mail/'),
+            'google.com/mail': Url(host='google.com', path='/mail'),
+            'http://google.com/': Url('http', host='google.com', path='/'),
+            'http://google.com': Url('http', host='google.com'),
+            'http://google.com?foo': Url('http', host='google.com', path='', query='foo'),
+            '': Url(),
+            '/': Url(path='/'),
+            '?': Url(path='', query=''),
+            '#': Url(path='', fragment=''),
+            '#?/!google.com/?foo#bar': Url(path='', fragment='?/!google.com/?foo#bar'),
+            '/foo': Url(path='/foo'),
+            '/foo?bar=baz': Url(path='/foo', query='bar=baz'),
+            '/foo?bar=baz#banana?apple/orange': Url(path='/foo', query='bar=baz', fragment='banana?apple/orange'),
+        }
+        for url, expected_url in url_host_map.items():
+            returned_url = parse_url(url)
+            self.assertEquals(returned_url, expected_url)
+
+    def test_request_uri(self):
+        url_host_map = {
+            'http://google.com/mail': '/mail',
+            'http://google.com/mail/': '/mail/',
+            'http://google.com/': '/',
+            'http://google.com': '/',
+            '': '/',
+            '/': '/',
+            '?': '/?',
+            '#': '/',
+            '/foo?bar=baz': '/foo?bar=baz',
+        }
+        for url, expected_request_uri in url_host_map.items():
+            returned_url = parse_url(url)
+            self.assertEquals(returned_url.request_uri, expected_request_uri)
+
+    def test_make_headers(self):
+        self.assertEqual(
+            make_headers(accept_encoding=True),
+            {'accept-encoding': 'gzip,deflate'})
+
+        self.assertEqual(
+            make_headers(accept_encoding='foo,bar'),
+            {'accept-encoding': 'foo,bar'})
+
+        self.assertEqual(
+            make_headers(accept_encoding=['foo', 'bar']),
+            {'accept-encoding': 'foo,bar'})
+
+        self.assertEqual(
+            make_headers(accept_encoding=True, user_agent='banana'),
+            {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'})
+
+        self.assertEqual(
+            make_headers(user_agent='banana'),
+            {'user-agent': 'banana'})
+
+        self.assertEqual(
+            make_headers(keep_alive=True),
+            {'connection': 'keep-alive'})
+
+        self.assertEqual(
+            make_headers(basic_auth='foo:bar'),
+            {'authorization': 'Basic Zm9vOmJhcg=='})
+
+
+    def test_split_first(self):
+        test_cases = {
+            ('abcd', 'b'): ('a', 'cd', 'b'),
+            ('abcd', 'cb'): ('a', 'cd', 'b'),
+            ('abcd', ''): ('abcd', '', None),
+            ('abcd', 'a'): ('', 'bcd', 'a'),
+            ('abcd', 'ab'): ('', 'bcd', 'a'),
+        }
+        for input, expected in test_cases.items():
+            output = split_first(*input)
+            self.assertEqual(output, expected)
+
+    def test_add_stderr_logger(self):
+        handler = add_stderr_logger(level=logging.INFO) # Don't actually print debug
+        logger = logging.getLogger('urllib3')
+        self.assertTrue(handler in logger.handlers)
+
+        logger.debug('Testing add_stderr_logger')
+        logger.removeHandler(handler)