Imported Upstream version 1.8

10 files changed, 288 insertions, 124 deletions

diff --git a/test/__init__.py b/test/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/test/__init__.py
+++ /dev/null
diff --git a/test/benchmark.py b/test/benchmark.py
deleted file mode 100644
index e7049c4..0000000
--- a/test/benchmark.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Really simple rudimentary benchmark to compare ConnectionPool versus standard
-urllib to demonstrate the usefulness of connection re-using.
-"""
-from __future__ import print_function
-
-import sys
-import time
-import urllib
-
-sys.path.append('../')
-import urllib3
-
-
-# URLs to download. Doesn't matter as long as they're from the same host, so we
-# can take advantage of connection re-using.
-TO_DOWNLOAD = [
-    'http://code.google.com/apis/apps/',
-    'http://code.google.com/apis/base/',
-    'http://code.google.com/apis/blogger/',
-    'http://code.google.com/apis/calendar/',
-    'http://code.google.com/apis/codesearch/',
-    'http://code.google.com/apis/contact/',
-    'http://code.google.com/apis/books/',
-    'http://code.google.com/apis/documents/',
-    'http://code.google.com/apis/finance/',
-    'http://code.google.com/apis/health/',
-    'http://code.google.com/apis/notebook/',
-    'http://code.google.com/apis/picasaweb/',
-    'http://code.google.com/apis/spreadsheets/',
-    'http://code.google.com/apis/webmastertools/',
-    'http://code.google.com/apis/youtube/',
-]
-
-
-def urllib_get(url_list):
-    assert url_list
-    for url in url_list:
-        now = time.time()
-        r = urllib.urlopen(url)
-        elapsed = time.time() - now
-        print("Got in %0.3f: %s" % (elapsed, url))
-
-
-def pool_get(url_list):
-    assert url_list
-    pool = urllib3.connection_from_url(url_list[0])
-    for url in url_list:
-        now = time.time()
-        r = pool.get_url(url)
-        elapsed = time.time() - now
-        print("Got in %0.3fs: %s" % (elapsed, url))
-
-
-if __name__ == '__main__':
-    print("Running pool_get ...")
-    now = time.time()
-    pool_get(TO_DOWNLOAD)
-    pool_elapsed = time.time() - now
-
-    print("Running urllib_get ...")
-    now = time.time()
-    urllib_get(TO_DOWNLOAD)
-    urllib_elapsed = time.time() - now
-
-    print("Completed pool_get in %0.3fs" % pool_elapsed)
-    print("Completed urllib_get in %0.3fs" % urllib_elapsed)
-
-
-"""
-Example results:
-
-Completed pool_get in 1.163s
-Completed urllib_get in 2.318s
-"""
diff --git a/test/test_collections.py b/test/test_collections.py
index b44c58a..4d173ac 100644
--- a/test/test_collections.py
+++ b/test/test_collections.py
@@ -1,6 +1,9 @@
 import unittest
 
-from urllib3._collections import RecentlyUsedContainer as Container
+from urllib3._collections import (
+    HTTPHeaderDict,
+    RecentlyUsedContainer as Container
+)
 from urllib3.packages import six
 xrange = six.moves.xrange
 
@@ -121,5 +124,57 @@ class TestLRUContainer(unittest.TestCase):
 
         self.assertRaises(NotImplementedError, d.__iter__)
 
+
+class TestHTTPHeaderDict(unittest.TestCase):
+    def setUp(self):
+        self.d = HTTPHeaderDict(A='foo')
+        self.d.add('a', 'bar')
+
+    def test_overwriting_with_setitem_replaces(self):
+        d = HTTPHeaderDict()
+
+        d['A'] = 'foo'
+        self.assertEqual(d['a'], 'foo')
+
+        d['a'] = 'bar'
+        self.assertEqual(d['A'], 'bar')
+
+    def test_copy(self):
+        h = self.d.copy()
+        self.assertTrue(self.d is not h)
+        self.assertEqual(self.d, h)
+
+    def test_add(self):
+        d = HTTPHeaderDict()
+
+        d['A'] = 'foo'
+        d.add('a', 'bar')
+
+        self.assertEqual(d['a'], 'foo, bar')
+        self.assertEqual(d['A'], 'foo, bar')
+
+    def test_getlist(self):
+        self.assertEqual(self.d.getlist('a'), ['foo', 'bar'])
+        self.assertEqual(self.d.getlist('A'), ['foo', 'bar'])
+        self.assertEqual(self.d.getlist('b'), [])
+
+    def test_delitem(self):
+        del self.d['a']
+        self.assertFalse('a' in self.d)
+        self.assertFalse('A' in self.d)
+
+    def test_equal(self):
+        b = HTTPHeaderDict({'a': 'foo, bar'})
+        self.assertEqual(self.d, b)
+        c = [('a', 'foo, bar')]
+        self.assertNotEqual(self.d, c)
+
+    def test_len(self):
+        self.assertEqual(len(self.d), 1)
+
+    def test_repr(self):
+        rep = "HTTPHeaderDict({'A': 'foo, bar'})"
+        self.assertEqual(repr(self.d), rep)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_compatibility.py b/test/test_compatibility.py
new file mode 100644
index 0000000..05ee4de
--- /dev/null
+++ b/test/test_compatibility.py
@@ -0,0 +1,23 @@
+import unittest
+import warnings
+
+from urllib3.connection import HTTPConnection
+
+
+class TestVersionCompatibility(unittest.TestCase):
+    def test_connection_strict(self):
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+
+            # strict=True is deprecated in Py33+
+            conn = HTTPConnection('localhost', 12345, strict=True)
+
+            if w:
+                self.fail('HTTPConnection raised warning on strict=True: %r' % w[0].message)
+
+    def test_connection_source_address(self):
+        try:
+            # source_address does not exist in Py26-
+            conn = HTTPConnection('localhost', 12345, source_address='127.0.0.1')
+        except TypeError as e:
+            self.fail('HTTPConnection raised TypeError on source_adddress: %r' % e)
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
index ac1768e..02229cf 100644
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -13,10 +13,9 @@ from urllib3.exceptions import (
     HostChangedError,
     MaxRetryError,
     SSLError,
-    ReadTimeoutError,
 )
 
-from socket import error as SocketError, timeout as SocketTimeout
+from socket import error as SocketError
 from ssl import SSLError as BaseSSLError
 
 try:   # Python 3
@@ -39,6 +38,11 @@ class TestConnectionPool(unittest.TestCase):
             ('http://google.com/', 'http://google.com'),
             ('http://google.com/', 'http://google.com/abra/cadabra'),
             ('http://google.com:42/', 'http://google.com:42/abracadabra'),
+            # Test comparison using default ports
+            ('http://google.com:80/', 'http://google.com/abracadabra'),
+            ('http://google.com/', 'http://google.com:80/abracadabra'),
+            ('https://google.com:443/', 'https://google.com/abracadabra'),
+            ('https://google.com/', 'https://google.com:443/abracadabra'),
         ]
 
         for a, b in same_host:
@@ -51,11 +55,22 @@ class TestConnectionPool(unittest.TestCase):
             ('http://yahoo.com/', 'http://google.com/'),
             ('http://google.com:42', 'https://google.com/abracadabra'),
             ('http://google.com', 'https://google.net/'),
+            # Test comparison with default ports
+            ('http://google.com:42', 'http://google.com'),
+            ('https://google.com:42', 'https://google.com'),
+            ('http://google.com:443', 'http://google.com'),
+            ('https://google.com:80', 'https://google.com'),
+            ('http://google.com:443', 'https://google.com'),
+            ('https://google.com:80', 'http://google.com'),
+            ('https://google.com:443', 'http://google.com'),
+            ('http://google.com:80', 'https://google.com'),
         ]
 
         for a, b in not_same_host:
             c = connection_from_url(a)
             self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b))
+            c = connection_from_url(b)
+            self.assertFalse(c.is_same_host(a), "%s =? %s" % (b, a))
 
 
     def test_max_connections(self):
@@ -128,9 +143,8 @@ class TestConnectionPool(unittest.TestCase):
 
             self.assertEqual(pool.pool.qsize(), POOL_SIZE)
 
-        #make sure that all of the exceptions return the connection to the pool
-        _test(Empty, ReadTimeoutError)
-        _test(SocketTimeout, ReadTimeoutError)
+        # Make sure that all of the exceptions return the connection to the pool
+        _test(Empty, EmptyPoolError)
         _test(BaseSSLError, SSLError)
         _test(CertificateError, SSLError)
 
diff --git a/test/test_exceptions.py b/test/test_exceptions.py
index e20649b..4190a61 100644
--- a/test/test_exceptions.py
+++ b/test/test_exceptions.py
@@ -11,25 +11,36 @@ from urllib3.connectionpool import HTTPConnectionPool
 
 class TestPickle(unittest.TestCase):
 
-    def cycle(self, item):
+    def verify_pickling(self, item):
         return pickle.loads(pickle.dumps(item))
 
     def test_exceptions(self):
-        assert self.cycle(HTTPError(None))
-        assert self.cycle(MaxRetryError(None, None, None))
-        assert self.cycle(LocationParseError(None))
-        assert self.cycle(ConnectTimeoutError(None))
+        assert self.verify_pickling(HTTPError(None))
+        assert self.verify_pickling(MaxRetryError(None, None, None))
+        assert self.verify_pickling(LocationParseError(None))
+        assert self.verify_pickling(ConnectTimeoutError(None))
 
     def test_exceptions_with_objects(self):
-        assert self.cycle(HTTPError('foo'))
-        assert self.cycle(MaxRetryError(HTTPConnectionPool('localhost'),
-                                        '/', None))
-        assert self.cycle(LocationParseError('fake location'))
-        assert self.cycle(ClosedPoolError(HTTPConnectionPool('localhost'),
-                                          None))
-        assert self.cycle(EmptyPoolError(HTTPConnectionPool('localhost'),
-                                         None))
-        assert self.cycle(HostChangedError(HTTPConnectionPool('localhost'),
-                                           '/', None))
-        assert self.cycle(ReadTimeoutError(HTTPConnectionPool('localhost'),
-                                              '/', None))
+        assert self.verify_pickling(
+            HTTPError('foo'))
+
+        assert self.verify_pickling(
+            HTTPError('foo', IOError('foo')))
+
+        assert self.verify_pickling(
+            MaxRetryError(HTTPConnectionPool('localhost'), '/', None))
+
+        assert self.verify_pickling(
+            LocationParseError('fake location'))
+
+        assert self.verify_pickling(
+            ClosedPoolError(HTTPConnectionPool('localhost'), None))
+
+        assert self.verify_pickling(
+            EmptyPoolError(HTTPConnectionPool('localhost'), None))
+
+        assert self.verify_pickling(
+            HostChangedError(HTTPConnectionPool('localhost'), '/', None))
+
+        assert self.verify_pickling(
+            ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None))
diff --git a/test/test_fields.py b/test/test_fields.py
index 888c2d5..cdec68b 100644
--- a/test/test_fields.py
+++ b/test/test_fields.py
@@ -1,34 +1,39 @@
 import unittest
 
 from urllib3.fields import guess_content_type, RequestField
-from urllib3.packages.six import b, u
+from urllib3.packages.six import u
 
 
 class TestRequestField(unittest.TestCase):
 
     def test_guess_content_type(self):
-      self.assertEqual(guess_content_type('image.jpg'), 'image/jpeg')
-      self.assertEqual(guess_content_type('notsure'), 'application/octet-stream')
-      self.assertEqual(guess_content_type(None), 'application/octet-stream')
+        self.assertTrue(guess_content_type('image.jpg') in
+                        ['image/jpeg', 'image/pjpeg'])
+        self.assertEqual(guess_content_type('notsure'),
+                         'application/octet-stream')
+        self.assertEqual(guess_content_type(None), 'application/octet-stream')
 
     def test_create(self):
-      simple_field = RequestField('somename', 'data')
-      self.assertEqual(simple_field.render_headers(), '\r\n')
-      filename_field = RequestField('somename', 'data', filename='somefile.txt')
-      self.assertEqual(filename_field.render_headers(), '\r\n')
-      headers_field = RequestField('somename', 'data', headers={'Content-Length': 4})
-      self.assertEqual(headers_field.render_headers(),
-          'Content-Length: 4\r\n'
-          '\r\n')
+        simple_field = RequestField('somename', 'data')
+        self.assertEqual(simple_field.render_headers(), '\r\n')
+        filename_field = RequestField('somename', 'data',
+                                      filename='somefile.txt')
+        self.assertEqual(filename_field.render_headers(), '\r\n')
+        headers_field = RequestField('somename', 'data',
+                                     headers={'Content-Length': 4})
+        self.assertEqual(
+            headers_field.render_headers(), 'Content-Length: 4\r\n\r\n')
 
     def test_make_multipart(self):
-      field = RequestField('somename', 'data')
-      field.make_multipart(content_type='image/jpg', content_location='/test')
-      self.assertEqual(field.render_headers(),
-          'Content-Disposition: form-data; name="somename"\r\n'
-          'Content-Type: image/jpg\r\n'
-          'Content-Location: /test\r\n'
-          '\r\n')
+        field = RequestField('somename', 'data')
+        field.make_multipart(content_type='image/jpg',
+                             content_location='/test')
+        self.assertEqual(
+            field.render_headers(),
+            'Content-Disposition: form-data; name="somename"\r\n'
+            'Content-Type: image/jpg\r\n'
+            'Content-Location: /test\r\n'
+            '\r\n')
 
     def test_render_parts(self):
         field = RequestField('somename', 'data')
diff --git a/test/test_filepost.py b/test/test_filepost.py
index ca33d61..390dbb3 100644
--- a/test/test_filepost.py
+++ b/test/test_filepost.py
@@ -124,7 +124,7 @@ class TestMultipartEncoding(unittest.TestCase):
 
       encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
 
-      self.assertEquals(encoded,
+      self.assertEqual(encoded,
           b'--' + b(BOUNDARY) + b'\r\n'
           b'Content-Type: image/jpeg\r\n'
           b'\r\n'
diff --git a/test/test_response.py b/test/test_response.py
index 90d34eb..ecfcbee 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -5,6 +5,25 @@ from io import BytesIO, BufferedReader
 from urllib3.response import HTTPResponse
 from urllib3.exceptions import DecodeError
 
+
+from base64 import b64decode
+
+# A known random (i.e, not-too-compressible) payload generated with:
+#    "".join(random.choice(string.printable) for i in xrange(512))
+#    .encode("zlib").encode("base64")
+# Randomness in tests == bad, and fixing a seed may not be sufficient.
+ZLIB_PAYLOAD = b64decode(b"""\
+eJwFweuaoQAAANDfineQhiKLUiaiCzvuTEmNNlJGiL5QhnGpZ99z8luQfe1AHoMioB+QSWHQu/L+
+lzd7W5CipqYmeVTBjdgSATdg4l4Z2zhikbuF+EKn69Q0DTpdmNJz8S33odfJoVEexw/l2SS9nFdi
+pis7KOwXzfSqarSo9uJYgbDGrs1VNnQpT9f8zAorhYCEZronZQF9DuDFfNK3Hecc+WHLnZLQptwk
+nufw8S9I43sEwxsT71BiqedHo0QeIrFE01F/4atVFXuJs2yxIOak3bvtXjUKAA6OKnQJ/nNvDGKZ
+Khe5TF36JbnKVjdcL1EUNpwrWVfQpFYJ/WWm2b74qNeSZeQv5/xBhRdOmKTJFYgO96PwrHBlsnLn
+a3l0LwJsloWpMbzByU5WLbRE6X5INFqjQOtIwYz5BAlhkn+kVqJvWM5vBlfrwP42ifonM5yF4ciJ
+auHVks62997mNGOsM7WXNG3P98dBHPo2NhbTvHleL0BI5dus2JY81MUOnK3SGWLH8HeWPa1t5KcW
+S5moAj5HexY/g/F8TctpxwsvyZp38dXeLDjSQvEQIkF7XR3YXbeZgKk3V34KGCPOAeeuQDIgyVhV
+nP4HF2uWHA==""")
+
+
 class TestLegacyResponse(unittest.TestCase):
     def test_getheaders(self):
         headers = {'host': 'example.com'}
@@ -167,6 +186,23 @@ class TestResponse(unittest.TestCase):
         self.assertEqual(next(stream), b'o')
         self.assertRaises(StopIteration, next, stream)
 
+    def test_streaming_tell(self):
+        fp = BytesIO(b'foo')
+        resp = HTTPResponse(fp, preload_content=False)
+        stream = resp.stream(2, decode_content=False)
+
+        position = 0
+
+        position += len(next(stream))
+        self.assertEqual(2, position)
+        self.assertEqual(position, resp.tell())
+
+        position += len(next(stream))
+        self.assertEqual(3, position)
+        self.assertEqual(position, resp.tell())
+
+        self.assertRaises(StopIteration, next, stream)
+
     def test_gzipped_streaming(self):
         import zlib
         compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
@@ -182,6 +218,78 @@ class TestResponse(unittest.TestCase):
         self.assertEqual(next(stream), b'oo')
         self.assertRaises(StopIteration, next, stream)
 
+    def test_gzipped_streaming_tell(self):
+        import zlib
+        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+        uncompressed_data = b'foo'
+        data = compress.compress(uncompressed_data)
+        data += compress.flush()
+
+        fp = BytesIO(data)
+        resp = HTTPResponse(fp, headers={'content-encoding': 'gzip'},
+                         preload_content=False)
+        stream = resp.stream()
+
+        # Read everything
+        payload = next(stream)
+        self.assertEqual(payload, uncompressed_data)
+
+        self.assertEqual(len(data), resp.tell())
+
+        self.assertRaises(StopIteration, next, stream)
+
+    def test_deflate_streaming_tell_intermediate_point(self):
+        # Ensure that ``tell()`` returns the correct number of bytes when
+        # part-way through streaming compressed content.
+        import zlib
+
+        NUMBER_OF_READS = 10
+
+        class MockCompressedDataReading(BytesIO):
+            """
+            A ByteIO-like reader returning ``payload`` in ``NUMBER_OF_READS``
+            calls to ``read``.
+            """
+
+            def __init__(self, payload, payload_part_size):
+                self.payloads = [
+                    payload[i*payload_part_size:(i+1)*payload_part_size]
+                    for i in range(NUMBER_OF_READS+1)]
+
+                assert b"".join(self.payloads) == payload
+
+            def read(self, _):
+                # Amount is unused.
+                if len(self.payloads) > 0:
+                    return self.payloads.pop(0)
+                return b""
+
+        uncompressed_data = zlib.decompress(ZLIB_PAYLOAD)
+
+        payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS
+        fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size)
+        resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'},
+                            preload_content=False)
+        stream = resp.stream()
+
+        parts_positions = [(part, resp.tell()) for part in stream]
+        end_of_stream = resp.tell()
+
+        self.assertRaises(StopIteration, next, stream)
+
+        parts, positions = zip(*parts_positions)
+
+        # Check that the payload is equal to the uncompressed data
+        payload = b"".join(parts)
+        self.assertEqual(uncompressed_data, payload)
+
+        # Check that the positions in the stream are correct
+        expected = [(i+1)*payload_part_size for i in range(NUMBER_OF_READS)]
+        self.assertEqual(expected, list(positions))
+
+        # Check that the end of the stream is in the correct place
+        self.assertEqual(len(ZLIB_PAYLOAD), end_of_stream)
+
     def test_deflate_streaming(self):
         import zlib
         data = zlib.compress(b'foo')
@@ -244,6 +352,11 @@ class TestResponse(unittest.TestCase):
         self.assertEqual(next(stream), b'o')
         self.assertRaises(StopIteration, next, stream)
 
+    def test_get_case_insensitive_headers(self):
+        headers = {'host': 'example.com'}
+        r = HTTPResponse(headers=headers)
+        self.assertEqual(r.headers.get('host'), 'example.com')
+        self.assertEqual(r.headers.get('Host'), 'example.com')
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_util.py b/test/test_util.py
index b465fef..ebd3b5f 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -64,7 +64,7 @@ class TestUtil(unittest.TestCase):
         }
         for url, expected_host in url_host_map.items():
             returned_host = get_host(url)
-            self.assertEquals(returned_host, expected_host)
+            self.assertEqual(returned_host, expected_host)
 
     def test_invalid_host(self):
         # TODO: Add more tests
@@ -85,6 +85,8 @@ class TestUtil(unittest.TestCase):
             'http://google.com/': Url('http', host='google.com', path='/'),
             'http://google.com': Url('http', host='google.com'),
             'http://google.com?foo': Url('http', host='google.com', path='', query='foo'),
+
+            # Path/query/fragment
             '': Url(),
             '/': Url(path='/'),
             '?': Url(path='', query=''),
@@ -93,10 +95,22 @@ class TestUtil(unittest.TestCase):
             '/foo': Url(path='/foo'),
             '/foo?bar=baz': Url(path='/foo', query='bar=baz'),
             '/foo?bar=baz#banana?apple/orange': Url(path='/foo', query='bar=baz', fragment='banana?apple/orange'),
+
+            # Port
+            'http://google.com/': Url('http', host='google.com', path='/'),
+            'http://google.com:80/': Url('http', host='google.com', port=80, path='/'),
+            'http://google.com:/': Url('http', host='google.com', path='/'),
+            'http://google.com:80': Url('http', host='google.com', port=80),
+            'http://google.com:': Url('http', host='google.com'),
+
+            # Auth
+            'http://foo:bar@localhost/': Url('http', auth='foo:bar', host='localhost', path='/'),
+            'http://foo@localhost/': Url('http', auth='foo', host='localhost', path='/'),
+            'http://foo:bar@baz@localhost/': Url('http', auth='foo:bar@baz', host='localhost', path='/'),
         }
         for url, expected_url in url_host_map.items():
             returned_url = parse_url(url)
-            self.assertEquals(returned_url, expected_url)
+            self.assertEqual(returned_url, expected_url)
 
     def test_parse_url_invalid_IPv6(self):
         self.assertRaises(ValueError, parse_url, '[::1')
@@ -115,7 +129,7 @@ class TestUtil(unittest.TestCase):
         }
         for url, expected_request_uri in url_host_map.items():
             returned_url = parse_url(url)
-            self.assertEquals(returned_url.request_uri, expected_request_uri)
+            self.assertEqual(returned_url.request_uri, expected_request_uri)
 
     def test_netloc(self):
         url_netloc_map = {
@@ -126,7 +140,7 @@ class TestUtil(unittest.TestCase):
         }
 
         for url, expected_netloc in url_netloc_map.items():
-            self.assertEquals(parse_url(url).netloc, expected_netloc)
+            self.assertEqual(parse_url(url).netloc, expected_netloc)
 
     def test_make_headers(self):
         self.assertEqual(
@@ -157,6 +171,9 @@ class TestUtil(unittest.TestCase):
             make_headers(basic_auth='foo:bar'),
             {'authorization': 'Basic Zm9vOmJhcg=='})
 
+        self.assertEqual(
+            make_headers(proxy_basic_auth='foo:bar'),
+            {'proxy-authorization': 'Basic Zm9vOmJhcg=='})
 
     def test_split_first(self):
         test_cases = {
@@ -250,6 +267,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(timeout.read_timeout, None)
         self.assertEqual(timeout.total, None)
 
+        timeout = Timeout(5)
+        self.assertEqual(timeout.total, 5)
+
 
     def test_timeout_str(self):
         timeout = Timeout(connect=1, read=2, total=3)