From 5f949ee35667a6065ab02a3e7ab8c98c9fcdcaed Mon Sep 17 00:00:00 2001 From: SVN-Git Migration Date: Thu, 8 Oct 2015 13:19:36 -0700 Subject: Imported Upstream version 1.8 --- test/__init__.py | 0 test/benchmark.py | 77 ------------------------------ test/test_collections.py | 57 +++++++++++++++++++++- test/test_compatibility.py | 23 +++++++++ test/test_connectionpool.py | 24 ++++++++-- test/test_exceptions.py | 45 +++++++++++------- test/test_fields.py | 43 +++++++++-------- test/test_filepost.py | 2 +- test/test_response.py | 113 ++++++++++++++++++++++++++++++++++++++++++++ test/test_util.py | 28 +++++++++-- 10 files changed, 288 insertions(+), 124 deletions(-) delete mode 100644 test/__init__.py delete mode 100644 test/benchmark.py create mode 100644 test/test_compatibility.py (limited to 'test') diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/benchmark.py b/test/benchmark.py deleted file mode 100644 index e7049c4..0000000 --- a/test/benchmark.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -""" -Really simple rudimentary benchmark to compare ConnectionPool versus standard -urllib to demonstrate the usefulness of connection re-using. -""" -from __future__ import print_function - -import sys -import time -import urllib - -sys.path.append('../') -import urllib3 - - -# URLs to download. Doesn't matter as long as they're from the same host, so we -# can take advantage of connection re-using. -TO_DOWNLOAD = [ - 'http://code.google.com/apis/apps/', - 'http://code.google.com/apis/base/', - 'http://code.google.com/apis/blogger/', - 'http://code.google.com/apis/calendar/', - 'http://code.google.com/apis/codesearch/', - 'http://code.google.com/apis/contact/', - 'http://code.google.com/apis/books/', - 'http://code.google.com/apis/documents/', - 'http://code.google.com/apis/finance/', - 'http://code.google.com/apis/health/', - 'http://code.google.com/apis/notebook/', - 'http://code.google.com/apis/picasaweb/', - 'http://code.google.com/apis/spreadsheets/', - 'http://code.google.com/apis/webmastertools/', - 'http://code.google.com/apis/youtube/', -] - - -def urllib_get(url_list): - assert url_list - for url in url_list: - now = time.time() - r = urllib.urlopen(url) - elapsed = time.time() - now - print("Got in %0.3f: %s" % (elapsed, url)) - - -def pool_get(url_list): - assert url_list - pool = urllib3.connection_from_url(url_list[0]) - for url in url_list: - now = time.time() - r = pool.get_url(url) - elapsed = time.time() - now - print("Got in %0.3fs: %s" % (elapsed, url)) - - -if __name__ == '__main__': - print("Running pool_get ...") - now = time.time() - pool_get(TO_DOWNLOAD) - pool_elapsed = time.time() - now - - print("Running urllib_get ...") - now = time.time() - urllib_get(TO_DOWNLOAD) - urllib_elapsed = time.time() - now - - print("Completed pool_get in %0.3fs" % pool_elapsed) - print("Completed urllib_get in %0.3fs" % urllib_elapsed) - - -""" -Example results: - -Completed pool_get in 1.163s -Completed urllib_get in 2.318s -""" diff --git a/test/test_collections.py b/test/test_collections.py index b44c58a..4d173ac 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -1,6 +1,9 @@ import unittest -from urllib3._collections import RecentlyUsedContainer as Container +from urllib3._collections import ( + HTTPHeaderDict, + RecentlyUsedContainer as Container +) from urllib3.packages import six xrange = six.moves.xrange @@ -121,5 +124,57 @@ class TestLRUContainer(unittest.TestCase): self.assertRaises(NotImplementedError, d.__iter__) + +class TestHTTPHeaderDict(unittest.TestCase): + def setUp(self): + self.d = HTTPHeaderDict(A='foo') + self.d.add('a', 'bar') + + def test_overwriting_with_setitem_replaces(self): + d = HTTPHeaderDict() + + d['A'] = 'foo' + self.assertEqual(d['a'], 'foo') + + d['a'] = 'bar' + self.assertEqual(d['A'], 'bar') + + def test_copy(self): + h = self.d.copy() + self.assertTrue(self.d is not h) + self.assertEqual(self.d, h) + + def test_add(self): + d = HTTPHeaderDict() + + d['A'] = 'foo' + d.add('a', 'bar') + + self.assertEqual(d['a'], 'foo, bar') + self.assertEqual(d['A'], 'foo, bar') + + def test_getlist(self): + self.assertEqual(self.d.getlist('a'), ['foo', 'bar']) + self.assertEqual(self.d.getlist('A'), ['foo', 'bar']) + self.assertEqual(self.d.getlist('b'), []) + + def test_delitem(self): + del self.d['a'] + self.assertFalse('a' in self.d) + self.assertFalse('A' in self.d) + + def test_equal(self): + b = HTTPHeaderDict({'a': 'foo, bar'}) + self.assertEqual(self.d, b) + c = [('a', 'foo, bar')] + self.assertNotEqual(self.d, c) + + def test_len(self): + self.assertEqual(len(self.d), 1) + + def test_repr(self): + rep = "HTTPHeaderDict({'A': 'foo, bar'})" + self.assertEqual(repr(self.d), rep) + if __name__ == '__main__': unittest.main() diff --git a/test/test_compatibility.py b/test/test_compatibility.py new file mode 100644 index 0000000..05ee4de --- /dev/null +++ b/test/test_compatibility.py @@ -0,0 +1,23 @@ +import unittest +import warnings + +from urllib3.connection import HTTPConnection + + +class TestVersionCompatibility(unittest.TestCase): + def test_connection_strict(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + # strict=True is deprecated in Py33+ + conn = HTTPConnection('localhost', 12345, strict=True) + + if w: + self.fail('HTTPConnection raised warning on strict=True: %r' % w[0].message) + + def test_connection_source_address(self): + try: + # source_address does not exist in Py26- + conn = HTTPConnection('localhost', 12345, source_address='127.0.0.1') + except TypeError as e: + self.fail('HTTPConnection raised TypeError on source_adddress: %r' % e) diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index ac1768e..02229cf 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -13,10 +13,9 @@ from urllib3.exceptions import ( HostChangedError, MaxRetryError, SSLError, - ReadTimeoutError, ) -from socket import error as SocketError, timeout as SocketTimeout +from socket import error as SocketError from ssl import SSLError as BaseSSLError try: # Python 3 @@ -39,6 +38,11 @@ class TestConnectionPool(unittest.TestCase): ('http://google.com/', 'http://google.com'), ('http://google.com/', 'http://google.com/abra/cadabra'), ('http://google.com:42/', 'http://google.com:42/abracadabra'), + # Test comparison using default ports + ('http://google.com:80/', 'http://google.com/abracadabra'), + ('http://google.com/', 'http://google.com:80/abracadabra'), + ('https://google.com:443/', 'https://google.com/abracadabra'), + ('https://google.com/', 'https://google.com:443/abracadabra'), ] for a, b in same_host: @@ -51,11 +55,22 @@ class TestConnectionPool(unittest.TestCase): ('http://yahoo.com/', 'http://google.com/'), ('http://google.com:42', 'https://google.com/abracadabra'), ('http://google.com', 'https://google.net/'), + # Test comparison with default ports + ('http://google.com:42', 'http://google.com'), + ('https://google.com:42', 'https://google.com'), + ('http://google.com:443', 'http://google.com'), + ('https://google.com:80', 'https://google.com'), + ('http://google.com:443', 'https://google.com'), + ('https://google.com:80', 'http://google.com'), + ('https://google.com:443', 'http://google.com'), + ('http://google.com:80', 'https://google.com'), ] for a, b in not_same_host: c = connection_from_url(a) self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b)) + c = connection_from_url(b) + self.assertFalse(c.is_same_host(a), "%s =? %s" % (b, a)) def test_max_connections(self): @@ -128,9 +143,8 @@ class TestConnectionPool(unittest.TestCase): self.assertEqual(pool.pool.qsize(), POOL_SIZE) - #make sure that all of the exceptions return the connection to the pool - _test(Empty, ReadTimeoutError) - _test(SocketTimeout, ReadTimeoutError) + # Make sure that all of the exceptions return the connection to the pool + _test(Empty, EmptyPoolError) _test(BaseSSLError, SSLError) _test(CertificateError, SSLError) diff --git a/test/test_exceptions.py b/test/test_exceptions.py index e20649b..4190a61 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -11,25 +11,36 @@ from urllib3.connectionpool import HTTPConnectionPool class TestPickle(unittest.TestCase): - def cycle(self, item): + def verify_pickling(self, item): return pickle.loads(pickle.dumps(item)) def test_exceptions(self): - assert self.cycle(HTTPError(None)) - assert self.cycle(MaxRetryError(None, None, None)) - assert self.cycle(LocationParseError(None)) - assert self.cycle(ConnectTimeoutError(None)) + assert self.verify_pickling(HTTPError(None)) + assert self.verify_pickling(MaxRetryError(None, None, None)) + assert self.verify_pickling(LocationParseError(None)) + assert self.verify_pickling(ConnectTimeoutError(None)) def test_exceptions_with_objects(self): - assert self.cycle(HTTPError('foo')) - assert self.cycle(MaxRetryError(HTTPConnectionPool('localhost'), - '/', None)) - assert self.cycle(LocationParseError('fake location')) - assert self.cycle(ClosedPoolError(HTTPConnectionPool('localhost'), - None)) - assert self.cycle(EmptyPoolError(HTTPConnectionPool('localhost'), - None)) - assert self.cycle(HostChangedError(HTTPConnectionPool('localhost'), - '/', None)) - assert self.cycle(ReadTimeoutError(HTTPConnectionPool('localhost'), - '/', None)) + assert self.verify_pickling( + HTTPError('foo')) + + assert self.verify_pickling( + HTTPError('foo', IOError('foo'))) + + assert self.verify_pickling( + MaxRetryError(HTTPConnectionPool('localhost'), '/', None)) + + assert self.verify_pickling( + LocationParseError('fake location')) + + assert self.verify_pickling( + ClosedPoolError(HTTPConnectionPool('localhost'), None)) + + assert self.verify_pickling( + EmptyPoolError(HTTPConnectionPool('localhost'), None)) + + assert self.verify_pickling( + HostChangedError(HTTPConnectionPool('localhost'), '/', None)) + + assert self.verify_pickling( + ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None)) diff --git a/test/test_fields.py b/test/test_fields.py index 888c2d5..cdec68b 100644 --- a/test/test_fields.py +++ b/test/test_fields.py @@ -1,34 +1,39 @@ import unittest from urllib3.fields import guess_content_type, RequestField -from urllib3.packages.six import b, u +from urllib3.packages.six import u class TestRequestField(unittest.TestCase): def test_guess_content_type(self): - self.assertEqual(guess_content_type('image.jpg'), 'image/jpeg') - self.assertEqual(guess_content_type('notsure'), 'application/octet-stream') - self.assertEqual(guess_content_type(None), 'application/octet-stream') + self.assertTrue(guess_content_type('image.jpg') in + ['image/jpeg', 'image/pjpeg']) + self.assertEqual(guess_content_type('notsure'), + 'application/octet-stream') + self.assertEqual(guess_content_type(None), 'application/octet-stream') def test_create(self): - simple_field = RequestField('somename', 'data') - self.assertEqual(simple_field.render_headers(), '\r\n') - filename_field = RequestField('somename', 'data', filename='somefile.txt') - self.assertEqual(filename_field.render_headers(), '\r\n') - headers_field = RequestField('somename', 'data', headers={'Content-Length': 4}) - self.assertEqual(headers_field.render_headers(), - 'Content-Length: 4\r\n' - '\r\n') + simple_field = RequestField('somename', 'data') + self.assertEqual(simple_field.render_headers(), '\r\n') + filename_field = RequestField('somename', 'data', + filename='somefile.txt') + self.assertEqual(filename_field.render_headers(), '\r\n') + headers_field = RequestField('somename', 'data', + headers={'Content-Length': 4}) + self.assertEqual( + headers_field.render_headers(), 'Content-Length: 4\r\n\r\n') def test_make_multipart(self): - field = RequestField('somename', 'data') - field.make_multipart(content_type='image/jpg', content_location='/test') - self.assertEqual(field.render_headers(), - 'Content-Disposition: form-data; name="somename"\r\n' - 'Content-Type: image/jpg\r\n' - 'Content-Location: /test\r\n' - '\r\n') + field = RequestField('somename', 'data') + field.make_multipart(content_type='image/jpg', + content_location='/test') + self.assertEqual( + field.render_headers(), + 'Content-Disposition: form-data; name="somename"\r\n' + 'Content-Type: image/jpg\r\n' + 'Content-Location: /test\r\n' + '\r\n') def test_render_parts(self): field = RequestField('somename', 'data') diff --git a/test/test_filepost.py b/test/test_filepost.py index ca33d61..390dbb3 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -124,7 +124,7 @@ class TestMultipartEncoding(unittest.TestCase): encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) - self.assertEquals(encoded, + self.assertEqual(encoded, b'--' + b(BOUNDARY) + b'\r\n' b'Content-Type: image/jpeg\r\n' b'\r\n' diff --git a/test/test_response.py b/test/test_response.py index 90d34eb..ecfcbee 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -5,6 +5,25 @@ from io import BytesIO, BufferedReader from urllib3.response import HTTPResponse from urllib3.exceptions import DecodeError + +from base64 import b64decode + +# A known random (i.e, not-too-compressible) payload generated with: +# "".join(random.choice(string.printable) for i in xrange(512)) +# .encode("zlib").encode("base64") +# Randomness in tests == bad, and fixing a seed may not be sufficient. +ZLIB_PAYLOAD = b64decode(b"""\ +eJwFweuaoQAAANDfineQhiKLUiaiCzvuTEmNNlJGiL5QhnGpZ99z8luQfe1AHoMioB+QSWHQu/L+ +lzd7W5CipqYmeVTBjdgSATdg4l4Z2zhikbuF+EKn69Q0DTpdmNJz8S33odfJoVEexw/l2SS9nFdi +pis7KOwXzfSqarSo9uJYgbDGrs1VNnQpT9f8zAorhYCEZronZQF9DuDFfNK3Hecc+WHLnZLQptwk +nufw8S9I43sEwxsT71BiqedHo0QeIrFE01F/4atVFXuJs2yxIOak3bvtXjUKAA6OKnQJ/nNvDGKZ +Khe5TF36JbnKVjdcL1EUNpwrWVfQpFYJ/WWm2b74qNeSZeQv5/xBhRdOmKTJFYgO96PwrHBlsnLn +a3l0LwJsloWpMbzByU5WLbRE6X5INFqjQOtIwYz5BAlhkn+kVqJvWM5vBlfrwP42ifonM5yF4ciJ +auHVks62997mNGOsM7WXNG3P98dBHPo2NhbTvHleL0BI5dus2JY81MUOnK3SGWLH8HeWPa1t5KcW +S5moAj5HexY/g/F8TctpxwsvyZp38dXeLDjSQvEQIkF7XR3YXbeZgKk3V34KGCPOAeeuQDIgyVhV +nP4HF2uWHA==""") + + class TestLegacyResponse(unittest.TestCase): def test_getheaders(self): headers = {'host': 'example.com'} @@ -167,6 +186,23 @@ class TestResponse(unittest.TestCase): self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream) + def test_streaming_tell(self): + fp = BytesIO(b'foo') + resp = HTTPResponse(fp, preload_content=False) + stream = resp.stream(2, decode_content=False) + + position = 0 + + position += len(next(stream)) + self.assertEqual(2, position) + self.assertEqual(position, resp.tell()) + + position += len(next(stream)) + self.assertEqual(3, position) + self.assertEqual(position, resp.tell()) + + self.assertRaises(StopIteration, next, stream) + def test_gzipped_streaming(self): import zlib compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) @@ -182,6 +218,78 @@ class TestResponse(unittest.TestCase): self.assertEqual(next(stream), b'oo') self.assertRaises(StopIteration, next, stream) + def test_gzipped_streaming_tell(self): + import zlib + compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + uncompressed_data = b'foo' + data = compress.compress(uncompressed_data) + data += compress.flush() + + fp = BytesIO(data) + resp = HTTPResponse(fp, headers={'content-encoding': 'gzip'}, + preload_content=False) + stream = resp.stream() + + # Read everything + payload = next(stream) + self.assertEqual(payload, uncompressed_data) + + self.assertEqual(len(data), resp.tell()) + + self.assertRaises(StopIteration, next, stream) + + def test_deflate_streaming_tell_intermediate_point(self): + # Ensure that ``tell()`` returns the correct number of bytes when + # part-way through streaming compressed content. + import zlib + + NUMBER_OF_READS = 10 + + class MockCompressedDataReading(BytesIO): + """ + A ByteIO-like reader returning ``payload`` in ``NUMBER_OF_READS`` + calls to ``read``. + """ + + def __init__(self, payload, payload_part_size): + self.payloads = [ + payload[i*payload_part_size:(i+1)*payload_part_size] + for i in range(NUMBER_OF_READS+1)] + + assert b"".join(self.payloads) == payload + + def read(self, _): + # Amount is unused. + if len(self.payloads) > 0: + return self.payloads.pop(0) + return b"" + + uncompressed_data = zlib.decompress(ZLIB_PAYLOAD) + + payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS + fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size) + resp = HTTPResponse(fp, headers={'content-encoding': 'deflate'}, + preload_content=False) + stream = resp.stream() + + parts_positions = [(part, resp.tell()) for part in stream] + end_of_stream = resp.tell() + + self.assertRaises(StopIteration, next, stream) + + parts, positions = zip(*parts_positions) + + # Check that the payload is equal to the uncompressed data + payload = b"".join(parts) + self.assertEqual(uncompressed_data, payload) + + # Check that the positions in the stream are correct + expected = [(i+1)*payload_part_size for i in range(NUMBER_OF_READS)] + self.assertEqual(expected, list(positions)) + + # Check that the end of the stream is in the correct place + self.assertEqual(len(ZLIB_PAYLOAD), end_of_stream) + def test_deflate_streaming(self): import zlib data = zlib.compress(b'foo') @@ -244,6 +352,11 @@ class TestResponse(unittest.TestCase): self.assertEqual(next(stream), b'o') self.assertRaises(StopIteration, next, stream) + def test_get_case_insensitive_headers(self): + headers = {'host': 'example.com'} + r = HTTPResponse(headers=headers) + self.assertEqual(r.headers.get('host'), 'example.com') + self.assertEqual(r.headers.get('Host'), 'example.com') if __name__ == '__main__': unittest.main() diff --git a/test/test_util.py b/test/test_util.py index b465fef..ebd3b5f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -64,7 +64,7 @@ class TestUtil(unittest.TestCase): } for url, expected_host in url_host_map.items(): returned_host = get_host(url) - self.assertEquals(returned_host, expected_host) + self.assertEqual(returned_host, expected_host) def test_invalid_host(self): # TODO: Add more tests @@ -85,6 +85,8 @@ class TestUtil(unittest.TestCase): 'http://google.com/': Url('http', host='google.com', path='/'), 'http://google.com': Url('http', host='google.com'), 'http://google.com?foo': Url('http', host='google.com', path='', query='foo'), + + # Path/query/fragment '': Url(), '/': Url(path='/'), '?': Url(path='', query=''), @@ -93,10 +95,22 @@ class TestUtil(unittest.TestCase): '/foo': Url(path='/foo'), '/foo?bar=baz': Url(path='/foo', query='bar=baz'), '/foo?bar=baz#banana?apple/orange': Url(path='/foo', query='bar=baz', fragment='banana?apple/orange'), + + # Port + 'http://google.com/': Url('http', host='google.com', path='/'), + 'http://google.com:80/': Url('http', host='google.com', port=80, path='/'), + 'http://google.com:/': Url('http', host='google.com', path='/'), + 'http://google.com:80': Url('http', host='google.com', port=80), + 'http://google.com:': Url('http', host='google.com'), + + # Auth + 'http://foo:bar@localhost/': Url('http', auth='foo:bar', host='localhost', path='/'), + 'http://foo@localhost/': Url('http', auth='foo', host='localhost', path='/'), + 'http://foo:bar@baz@localhost/': Url('http', auth='foo:bar@baz', host='localhost', path='/'), } for url, expected_url in url_host_map.items(): returned_url = parse_url(url) - self.assertEquals(returned_url, expected_url) + self.assertEqual(returned_url, expected_url) def test_parse_url_invalid_IPv6(self): self.assertRaises(ValueError, parse_url, '[::1') @@ -115,7 +129,7 @@ class TestUtil(unittest.TestCase): } for url, expected_request_uri in url_host_map.items(): returned_url = parse_url(url) - self.assertEquals(returned_url.request_uri, expected_request_uri) + self.assertEqual(returned_url.request_uri, expected_request_uri) def test_netloc(self): url_netloc_map = { @@ -126,7 +140,7 @@ class TestUtil(unittest.TestCase): } for url, expected_netloc in url_netloc_map.items(): - self.assertEquals(parse_url(url).netloc, expected_netloc) + self.assertEqual(parse_url(url).netloc, expected_netloc) def test_make_headers(self): self.assertEqual( @@ -157,6 +171,9 @@ class TestUtil(unittest.TestCase): make_headers(basic_auth='foo:bar'), {'authorization': 'Basic Zm9vOmJhcg=='}) + self.assertEqual( + make_headers(proxy_basic_auth='foo:bar'), + {'proxy-authorization': 'Basic Zm9vOmJhcg=='}) def test_split_first(self): test_cases = { @@ -250,6 +267,9 @@ class TestUtil(unittest.TestCase): self.assertEqual(timeout.read_timeout, None) self.assertEqual(timeout.total, None) + timeout = Timeout(5) + self.assertEqual(timeout.total, 5) + def test_timeout_str(self): timeout = Timeout(connect=1, read=2, total=3) -- cgit v1.2.3