Imported Upstream version 1.11

5 files changed, 234 insertions, 10 deletions

diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py
index d6cb162..741ae7b 100644
--- a/test/with_dummyserver/test_connectionpool.py
+++ b/test/with_dummyserver/test_connectionpool.py
@@ -36,7 +36,7 @@ from urllib3.util.timeout import Timeout
 
 import tornado
 from dummyserver.testcase import HTTPDummyServerTestCase
-from dummyserver.server import NoIPv6Warning
+from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS
 
 from nose.tools import timed
 
@@ -600,7 +600,7 @@ class TestConnectionPool(HTTPDummyServerTestCase):
 
     def test_source_address(self):
         for addr, is_ipv6 in VALID_SOURCE_ADDRESSES:
-            if is_ipv6 and not socket.has_ipv6:
+            if is_ipv6 and not HAS_IPV6_AND_DNS:
                 warnings.warn("No IPv6 support: skipping.",
                               NoIPv6Warning)
                 continue
@@ -647,6 +647,27 @@ class TestConnectionPool(HTTPDummyServerTestCase):
 
         self.assertEqual(b'123' * 4, response.read())
 
+    def test_cleanup_on_connection_error(self):
+        '''
+        Test that connections are recycled to the pool on 
+        connection errors where no http response is received.
+        '''
+        poolsize = 3
+        with HTTPConnectionPool(self.host, self.port, maxsize=poolsize, block=True) as http:
+            self.assertEqual(http.pool.qsize(), poolsize)
+
+            # force a connection error by supplying a non-existent 
+            # url. We won't get a response for this  and so the 
+            # conn won't be implicitly returned to the pool.
+            self.assertRaises(MaxRetryError,
+                http.request, 'GET', '/redirect', fields={'target': '/'}, release_conn=False, retries=0)
+
+            r = http.request('GET', '/redirect', fields={'target': '/'}, release_conn=False, retries=1)
+            r.release_conn()
+
+            # the pool should still contain poolsize elements
+            self.assertEqual(http.pool.qsize(), http.pool.maxsize)
+
 
 class TestRetry(HTTPDummyServerTestCase):
     def setUp(self):
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
index 992b8ef..63aea66 100644
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -419,6 +419,11 @@ class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
         self._pool.ca_certs = DEFAULT_CA
         self._pool.request('GET', '/')
 
+    def test_set_cert_default_cert_required(self):
+        conn = VerifiedHTTPSConnection(self.host, self.port)
+        conn.set_cert(ca_certs='/etc/ssl/certs/custom.pem')
+        self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED')
+
 
 class TestHTTPS_NoSAN(HTTPSDummyServerTestCase):
     certs = NO_SAN_CERTS
diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py
index 7e51c73..099ac52 100644
--- a/test/with_dummyserver/test_poolmanager.py
+++ b/test/with_dummyserver/test_poolmanager.py
@@ -1,6 +1,8 @@
 import unittest
 import json
 
+from nose.plugins.skip import SkipTest
+from dummyserver.server import HAS_IPV6
 from dummyserver.testcase import (HTTPDummyServerTestCase,
                                   IPv6HTTPDummyServerTestCase)
 from urllib3.poolmanager import PoolManager
@@ -128,6 +130,14 @@ class TestPoolManager(HTTPDummyServerTestCase):
     def test_headers(self):
         http = PoolManager(headers={'Foo': 'bar'})
 
+        r = http.request('GET', '%s/headers' % self.base_url)
+        returned_headers = json.loads(r.data.decode())
+        self.assertEqual(returned_headers.get('Foo'), 'bar')
+
+        r = http.request('POST', '%s/headers' % self.base_url)
+        returned_headers = json.loads(r.data.decode())
+        self.assertEqual(returned_headers.get('Foo'), 'bar')
+        
         r = http.request_encode_url('GET', '%s/headers' % self.base_url)
         returned_headers = json.loads(r.data.decode())
         self.assertEqual(returned_headers.get('Foo'), 'bar')
@@ -154,6 +164,9 @@ class TestPoolManager(HTTPDummyServerTestCase):
 
 
 class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase):
+    if not HAS_IPV6:
+        raise SkipTest("IPv6 is not supported on this system.")
+
     def setUp(self):
         self.base_url = 'http://[%s]:%d' % (self.host, self.port)
 
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
index df300fe..c593f2d 100644
--- a/test/with_dummyserver/test_proxy_poolmanager.py
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -4,11 +4,12 @@ import unittest
 
 from nose.tools import timed
 
-from dummyserver.testcase import HTTPDummyProxyTestCase
+from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase
 from dummyserver.server import (
     DEFAULT_CA, DEFAULT_CA_BAD, get_unreachable_address)
 from .. import TARPIT_HOST
 
+from urllib3._collections import HTTPHeaderDict
 from urllib3.poolmanager import proxy_from_url, ProxyManager
 from urllib3.exceptions import (
     MaxRetryError, SSLError, ProxyError, ConnectTimeoutError)
@@ -48,7 +49,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
 
     def test_proxy_conn_fail(self):
         host, port = get_unreachable_address()
-        http = proxy_from_url('http://%s:%s/' % (host, port), retries=1)
+        http = proxy_from_url('http://%s:%s/' % (host, port), retries=1, timeout=0.05)
         self.assertRaises(MaxRetryError, http.request, 'GET',
                           '%s/' % self.https_url)
         self.assertRaises(MaxRetryError, http.request, 'GET',
@@ -223,6 +224,22 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
         self.assertEqual(returned_headers.get('Host'),
                 '%s:%s'%(self.https_host,self.https_port))
 
+    def test_headerdict(self):
+        default_headers = HTTPHeaderDict(a='b')
+        proxy_headers = HTTPHeaderDict()
+        proxy_headers.add('foo', 'bar')
+
+        http = proxy_from_url(
+            self.proxy_url,
+            headers=default_headers,
+            proxy_headers=proxy_headers)
+
+        request_headers = HTTPHeaderDict(baz='quux')
+        r = http.request('GET', '%s/headers' % self.http_url, headers=request_headers)
+        returned_headers = json.loads(r.data.decode())
+        self.assertEqual(returned_headers.get('Foo'), 'bar')
+        self.assertEqual(returned_headers.get('Baz'), 'quux')
+
     def test_proxy_pooling(self):
         http = proxy_from_url(self.proxy_url)
 
@@ -283,5 +300,26 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
         except MaxRetryError as e:
             assert isinstance(e.reason, ConnectTimeoutError)
 
+
+class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase):
+
+    def setUp(self):
+        self.http_url = 'http://%s:%d' % (self.http_host, self.http_port)
+        self.http_url_alt = 'http://%s:%d' % (self.http_host_alt,
+                                              self.http_port)
+        self.https_url = 'https://%s:%d' % (self.https_host, self.https_port)
+        self.https_url_alt = 'https://%s:%d' % (self.https_host_alt,
+                                                self.https_port)
+        self.proxy_url = 'http://[%s]:%d' % (self.proxy_host, self.proxy_port)
+
+    def test_basic_ipv6_proxy(self):
+        http = proxy_from_url(self.proxy_url)
+
+        r = http.request('GET', '%s/' % self.http_url)
+        self.assertEqual(r.status, 200)
+
+        r = http.request('GET', '%s/' % self.https_url)
+        self.assertEqual(r.status, 200)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
index 6c99653..5af00e0 100644
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -10,17 +10,24 @@ from urllib3.exceptions import (
         SSLError,
         ProtocolError,
 )
+from urllib3.response import httplib
 from urllib3.util.ssl_ import HAS_SNI
 from urllib3.util.timeout import Timeout
 from urllib3.util.retry import Retry
+from urllib3._collections import HTTPHeaderDict
 
 from dummyserver.testcase import SocketDummyServerTestCase
 from dummyserver.server import (
     DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address)
 
-from .. import onlyPy3
+from .. import onlyPy3, LogRecorder
 
 from nose.plugins.skip import SkipTest
+try:
+    from mimetools import Message as MimeToolMessage
+except ImportError:
+    class MimeToolMessage(object):
+        pass
 from threading import Event
 import socket
 import ssl
@@ -119,8 +126,9 @@ class TestSocketClosing(SocketDummyServerTestCase):
     def test_connection_refused(self):
         # Does the pool retry if there is no listener on the port?
         host, port = get_unreachable_address()
-        pool = HTTPConnectionPool(host, port)
-        self.assertRaises(MaxRetryError, pool.request, 'GET', '/', retries=0)
+        http = HTTPConnectionPool(host, port, maxsize=3, block=True)
+        self.assertRaises(MaxRetryError, http.request, 'GET', '/', retries=0, release_conn=False)
+        self.assertEqual(http.pool.qsize(), http.pool.maxsize)
 
     def test_connection_read_timeout(self):
         timed_out = Event()
@@ -133,13 +141,15 @@ class TestSocketClosing(SocketDummyServerTestCase):
             sock.close()
 
         self._start_server(socket_handler)
-        pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
+        http = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False, maxsize=3, block=True)
 
         try:
-            self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/')
+            self.assertRaises(ReadTimeoutError, http.request, 'GET', '/', release_conn=False)
         finally:
             timed_out.set()
 
+        self.assertEqual(http.pool.qsize(), http.pool.maxsize)
+
     def test_https_connection_read_timeout(self):
         """ Handshake timeouts should fail with a Timeout"""
         timed_out = Event()
@@ -297,6 +307,63 @@ class TestSocketClosing(SocketDummyServerTestCase):
         self.assertEqual(response.status, 200)
         self.assertEqual(response.data, b'foo')
 
+    def test_connection_cleanup_on_read_timeout(self):
+        timed_out = Event()
+
+        def socket_handler(listener):
+            sock = listener.accept()[0]
+            buf = b''
+            body = 'Hi'
+            while not buf.endswith(b'\r\n\r\n'):
+                buf = sock.recv(65536)
+            sock.send(('HTTP/1.1 200 OK\r\n'
+                       'Content-Type: text/plain\r\n'
+                       'Content-Length: %d\r\n'
+                       '\r\n' % len(body)).encode('utf-8'))
+
+            timed_out.wait()
+            sock.close()
+
+        self._start_server(socket_handler)
+        with HTTPConnectionPool(self.host, self.port) as pool:
+            poolsize = pool.pool.qsize()
+            response = pool.urlopen('GET', '/', retries=0, preload_content=False,
+                                    timeout=Timeout(connect=1, read=0.001))
+            try:
+                self.assertRaises(ReadTimeoutError, response.read)
+                self.assertEqual(poolsize, pool.pool.qsize())
+            finally:
+                timed_out.set()
+
+    def test_connection_cleanup_on_protocol_error_during_read(self):
+        body = 'Response'
+        partial_body = body[:2]
+
+        def socket_handler(listener):
+            sock = listener.accept()[0]
+
+            # Consume request
+            buf = b''
+            while not buf.endswith(b'\r\n\r\n'):
+                buf = sock.recv(65536)
+
+            # Send partial response and close socket.
+            sock.send((
+                'HTTP/1.1 200 OK\r\n'
+                'Content-Type: text/plain\r\n'
+                'Content-Length: %d\r\n'
+                '\r\n'
+                '%s' % (len(body), partial_body)).encode('utf-8')
+            )
+            sock.close()
+
+        self._start_server(socket_handler)
+        with HTTPConnectionPool(self.host, self.port) as pool:
+            poolsize = pool.pool.qsize()
+            response = pool.request('GET', '/', retries=0, preload_content=False)
+
+            self.assertRaises(ProtocolError, response.read)
+            self.assertEqual(poolsize, pool.pool.qsize())
 
 
 class TestProxyManager(SocketDummyServerTestCase):
@@ -355,7 +422,7 @@ class TestProxyManager(SocketDummyServerTestCase):
         base_url = 'http://%s:%d' % (self.host, self.port)
 
         # Define some proxy headers.
-        proxy_headers = {'For The Proxy': 'YEAH!'}
+        proxy_headers = HTTPHeaderDict({'For The Proxy': 'YEAH!'})
         proxy = proxy_from_url(base_url, proxy_headers=proxy_headers)
 
         conn = proxy.connection_from_url('http://www.google.com/')
@@ -617,6 +684,86 @@ class TestHeaders(SocketDummyServerTestCase):
         r = pool.request('GET', '/')
         self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
 
+    def test_headers_are_sent_with_the_original_case(self):
+        headers = {'foo': 'bar', 'bAz': 'quux'}
+        parsed_headers = {}
+
+        def socket_handler(listener):
+            sock = listener.accept()[0]
+
+            buf = b''
+            while not buf.endswith(b'\r\n\r\n'):
+                buf += sock.recv(65536)
+
+            headers_list = [header for header in buf.split(b'\r\n')[1:] if header]
+
+            for header in headers_list:
+                (key, value) = header.split(b': ')
+                parsed_headers[key.decode()] = value.decode()
+
+            # Send incomplete message (note Content-Length)
+            sock.send((
+                'HTTP/1.1 204 No Content\r\n'
+                'Content-Length: 0\r\n'
+                '\r\n').encode('utf-8'))
+
+            sock.close()
+
+        self._start_server(socket_handler)
+        expected_headers = {'Accept-Encoding': 'identity',
+                            'Host': '{0}:{1}'.format(self.host, self.port)}
+        expected_headers.update(headers)
+
+        pool = HTTPConnectionPool(self.host, self.port, retries=False)
+        pool.request('GET', '/', headers=HTTPHeaderDict(headers))
+        self.assertEqual(expected_headers, parsed_headers)
+
+
+class TestBrokenHeaders(SocketDummyServerTestCase):
+    def setUp(self):
+        if issubclass(httplib.HTTPMessage, MimeToolMessage):
+            raise SkipTest('Header parsing errors not available')
+
+        super(TestBrokenHeaders, self).setUp()
+
+    def _test_broken_header_parsing(self, headers):
+        handler = create_response_handler((
+           b'HTTP/1.1 200 OK\r\n'
+           b'Content-Length: 0\r\n'
+           b'Content-type: text/plain\r\n'
+           ) + b'\r\n'.join(headers) + b'\r\n'
+        )
+
+        self._start_server(handler)
+        pool = HTTPConnectionPool(self.host, self.port, retries=False)
+
+        with LogRecorder() as logs:
+            pool.request('GET', '/')
+
+        for record in logs:
+            if 'Failed to parse headers' in record.msg and \
+                    pool._absolute_url('/') == record.args[0]:
+                return
+        self.fail('Missing log about unparsed headers')
+
+    def test_header_without_name(self):
+        self._test_broken_header_parsing([
+            b': Value\r\n',
+            b'Another: Header\r\n',
+        ])
+
+    def test_header_without_name_or_value(self):
+        self._test_broken_header_parsing([
+            b':\r\n',
+            b'Another: Header\r\n',
+        ])
+
+    def test_header_without_colon_or_value(self):
+        self._test_broken_header_parsing([
+            b'Broken Header',
+            b'Another: Header',
+        ])
+
 
 class TestHEAD(SocketDummyServerTestCase):
     def test_chunked_head_response_does_not_hang(self):