Imported Upstream version 1.2.0

15 files changed, 206 insertions, 48 deletions

diff --git a/requests/packages/charade/__init__.py b/requests/packages/charade/__init__.py
index 5d580b3..1aadf3e 100644
--- a/requests/packages/charade/__init__.py
+++ b/requests/packages/charade/__init__.py
@@ -15,10 +15,15 @@
 # 02110-1301  USA

 ######################### END LICENSE BLOCK #########################

 

-__version__ = "1.0.1"

+__version__ = "1.0.3"

+from sys import version_info

 

 

 def detect(aBuf):

+    if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or

+            (version_info >= (3, 0) and not isinstance(aBuf, bytes))):

+        raise ValueError('Expected a bytes object, not a unicode object')

+

     from . import universaldetector

     u = universaldetector.UniversalDetector()

     u.reset()

diff --git a/requests/packages/charade/chardistribution.py b/requests/packages/charade/chardistribution.py
index 981bd1a..dfd3355 100644
--- a/requests/packages/charade/chardistribution.py
+++ b/requests/packages/charade/chardistribution.py
@@ -40,6 +40,7 @@ from .compat import wrap_ord
 ENOUGH_DATA_THRESHOLD = 1024

 SURE_YES = 0.99

 SURE_NO = 0.01

+MINIMUM_DATA_THRESHOLD = 3

 

 

 class CharDistributionAnalysis:

@@ -82,7 +83,7 @@ class CharDistributionAnalysis:
         """return confidence based on existing data"""

         # if we didn't receive any character in our consideration range,

         # return negative answer

-        if self._mTotalChars <= 0:

+        if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD:

             return SURE_NO

 

         if self._mTotalChars != self._mFreqChars:

diff --git a/requests/packages/charade/compat.py b/requests/packages/charade/compat.py
index f86c46b..d9e30ad 100644
--- a/requests/packages/charade/compat.py
+++ b/requests/packages/charade/compat.py
@@ -18,9 +18,17 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
+import sys
+
+
+if sys.version_info < (3, 0):
+    base_str = (str, unicode)
+else:
+    base_str = (bytes, str)
+
 
 def wrap_ord(a):
-    if isinstance(a, str):
+    if sys.version_info < (3, 0) and isinstance(a, base_str):
         return ord(a)
-    elif isinstance(a, int):
+    else:
         return a
diff --git a/requests/packages/charade/cp949prober.py b/requests/packages/charade/cp949prober.py
new file mode 100644
index 0000000..543501f
--- /dev/null
+++ b/requests/packages/charade/cp949prober.py
@@ -0,0 +1,44 @@
+######################## BEGIN LICENSE BLOCK ########################

+# The Original Code is mozilla.org code.

+#

+# The Initial Developer of the Original Code is

+# Netscape Communications Corporation.

+# Portions created by the Initial Developer are Copyright (C) 1998

+# the Initial Developer. All Rights Reserved.

+#

+# Contributor(s):

+#   Mark Pilgrim - port to Python

+#

+# This library is free software; you can redistribute it and/or

+# modify it under the terms of the GNU Lesser General Public

+# License as published by the Free Software Foundation; either

+# version 2.1 of the License, or (at your option) any later version.

+#

+# This library is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+# Lesser General Public License for more details.

+#

+# You should have received a copy of the GNU Lesser General Public

+# License along with this library; if not, write to the Free Software

+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA

+# 02110-1301  USA

+######################### END LICENSE BLOCK #########################

+

+from .mbcharsetprober import MultiByteCharSetProber

+from .codingstatemachine import CodingStateMachine

+from .chardistribution import EUCKRDistributionAnalysis

+from .mbcssm import CP949SMModel

+

+

+class CP949Prober(MultiByteCharSetProber):

+    def __init__(self):

+        MultiByteCharSetProber.__init__(self)

+        self._mCodingSM = CodingStateMachine(CP949SMModel)

+        # NOTE: CP949 is a superset of EUC-KR, so the distribution should be

+        #       not different.

+        self._mDistributionAnalyzer = EUCKRDistributionAnalysis()

+        self.reset()

+

+    def get_charset_name(self):

+        return "CP949"

diff --git a/requests/packages/charade/langcyrillicmodel.py b/requests/packages/charade/langcyrillicmodel.py
index 4b69c82..15e338f 100644
--- a/requests/packages/charade/langcyrillicmodel.py
+++ b/requests/packages/charade/langcyrillicmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA

 ######################### END LICENSE BLOCK #########################

 

-from . import constants

-

 # KOI8-R language model

 # Character Mapping Table:

 KOI8R_CharToOrderMap = (

diff --git a/requests/packages/charade/langgreekmodel.py b/requests/packages/charade/langgreekmodel.py
index 78e9ce6..93241ce 100644
--- a/requests/packages/charade/langgreekmodel.py
+++ b/requests/packages/charade/langgreekmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA

 ######################### END LICENSE BLOCK #########################

 

-from . import constants

-

 # 255: Control characters that usually does not exist in any text

 # 254: Carriage/Return

 # 253: symbol (punctuation) that does not belong to word

diff --git a/requests/packages/charade/langhebrewmodel.py b/requests/packages/charade/langhebrewmodel.py
index 4c6b3ce..d871324 100644
--- a/requests/packages/charade/langhebrewmodel.py
+++ b/requests/packages/charade/langhebrewmodel.py
@@ -27,8 +27,6 @@
 # 02110-1301  USA

 ######################### END LICENSE BLOCK #########################

 

-from . import constants

-

 # 255: Control characters that usually does not exist in any text

 # 254: Carriage/Return

 # 253: symbol (punctuation) that does not belong to word

diff --git a/requests/packages/charade/langhungarianmodel.py b/requests/packages/charade/langhungarianmodel.py
index bd7f505..6f59c61 100644
--- a/requests/packages/charade/langhungarianmodel.py
+++ b/requests/packages/charade/langhungarianmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA

 ######################### END LICENSE BLOCK #########################

 

-from . import constants

-

 # 255: Control characters that usually does not exist in any text

 # 254: Carriage/Return

 # 253: symbol (punctuation) that does not belong to word

diff --git a/requests/packages/charade/mbcsgroupprober.py b/requests/packages/charade/mbcsgroupprober.py
index ebe93d0..2f6f5e8 100644
--- a/requests/packages/charade/mbcsgroupprober.py
+++ b/requests/packages/charade/mbcsgroupprober.py
@@ -33,6 +33,7 @@ from .sjisprober import SJISProber
 from .eucjpprober import EUCJPProber

 from .gb2312prober import GB2312Prober

 from .euckrprober import EUCKRProber

+from .cp949prober import CP949Prober

 from .big5prober import Big5Prober

 from .euctwprober import EUCTWProber

 

@@ -46,6 +47,7 @@ class MBCSGroupProber(CharSetGroupProber):
             EUCJPProber(),

             GB2312Prober(),

             EUCKRProber(),

+            CP949Prober(),

             Big5Prober(),

             EUCTWProber()

         ]

diff --git a/requests/packages/charade/mbcssm.py b/requests/packages/charade/mbcssm.py
index 3a720c9..55c02f0 100644
--- a/requests/packages/charade/mbcssm.py
+++ b/requests/packages/charade/mbcssm.py
@@ -78,6 +78,46 @@ Big5SMModel = {'classTable': BIG5_cls,
                'charLenTable': Big5CharLenTable,

                'name': 'Big5'}

 

+# CP949

+

+CP949_cls  = (

+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0,  # 00 - 0f

+    1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1,  # 10 - 1f

+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 20 - 2f

+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 30 - 3f

+    1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,  # 40 - 4f

+    4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 50 - 5f

+    1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5,  # 60 - 6f

+    5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 70 - 7f

+    0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 80 - 8f

+    6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 90 - 9f

+    6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8,  # a0 - af

+    7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,  # b0 - bf

+    7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2,  # c0 - cf

+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # d0 - df

+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # e0 - ef

+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0,  # f0 - ff

+)

+

+CP949_st = (

+#cls=    0      1      2      3      4      5      6      7      8      9  # previous state =

+    eError,eStart,     3,eError,eStart,eStart,     4,     5,eError,     6, # eStart

+    eError,eError,eError,eError,eError,eError,eError,eError,eError,eError, # eError

+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe, # eItsMe

+    eError,eError,eStart,eStart,eError,eError,eError,eStart,eStart,eStart, # 3

+    eError,eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 4

+    eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 5

+    eError,eStart,eStart,eStart,eStart,eError,eError,eStart,eStart,eStart, # 6

+)

+

+CP949CharLenTable = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)

+

+CP949SMModel = {'classTable': CP949_cls,

+                'classFactor': 10,

+                'stateTable': CP949_st,

+                'charLenTable': CP949CharLenTable,

+                'name': 'CP949'}

+

 # EUC-JP

 

 EUCJP_cls = (

diff --git a/requests/packages/charade/universaldetector.py b/requests/packages/charade/universaldetector.py
index adaae72..6175bfb 100644
--- a/requests/packages/charade/universaldetector.py
+++ b/requests/packages/charade/universaldetector.py
@@ -28,6 +28,7 @@
 

 from . import constants

 import sys

+import codecs

 from .latin1prober import Latin1Prober  # windows-1252

 from .mbcsgroupprober import MBCSGroupProber  # multi-byte character sets

 from .sbcsgroupprober import SBCSGroupProber  # single-byte character sets

@@ -70,31 +71,31 @@ class UniversalDetector:
 

         if not self._mGotData:

             # If the data starts with BOM, we know it is UTF

-            if aBuf[:3] == '\xEF\xBB\xBF':

+            if aBuf[:3] == codecs.BOM:

                 # EF BB BF  UTF-8 with BOM

                 self.result = {'encoding': "UTF-8", 'confidence': 1.0}

-            elif aBuf[:4] == '\xFF\xFE\x00\x00':

+            elif aBuf[:4] == codecs.BOM_UTF32_LE:

                 # FF FE 00 00  UTF-32, little-endian BOM

                 self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}

-            elif aBuf[:4] == '\x00\x00\xFE\xFF':

+            elif aBuf[:4] == codecs.BOM_UTF32_BE:

                 # 00 00 FE FF  UTF-32, big-endian BOM

                 self.result = {'encoding': "UTF-32BE", 'confidence': 1.0}

-            elif aBuf[:4] == '\xFE\xFF\x00\x00':

+            elif aBuf[:4] == b'\xFE\xFF\x00\x00':

                 # FE FF 00 00  UCS-4, unusual octet order BOM (3412)

                 self.result = {

                     'encoding': "X-ISO-10646-UCS-4-3412",

                     'confidence': 1.0

                 }

-            elif aBuf[:4] == '\x00\x00\xFF\xFE':

+            elif aBuf[:4] == b'\x00\x00\xFF\xFE':

                 # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)

                 self.result = {

                     'encoding': "X-ISO-10646-UCS-4-2143",

                     'confidence': 1.0

                 }

-            elif aBuf[:2] == '\xFF\xFE':

+            elif aBuf[:2] == codecs.BOM_LE:

                 # FF FE  UTF-16, little endian BOM

                 self.result = {'encoding': "UTF-16LE", 'confidence': 1.0}

-            elif aBuf[:2] == '\xFE\xFF':

+            elif aBuf[:2] == codecs.BOM_BE:

                 # FE FF  UTF-16, big endian BOM

                 self.result = {'encoding': "UTF-16BE", 'confidence': 1.0}

 

diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py
index af8760d..51c87f5 100644
--- a/requests/packages/urllib3/connectionpool.py
+++ b/requests/packages/urllib3/connectionpool.py
@@ -9,6 +9,7 @@ import socket
 import errno
 
 from socket import error as SocketError, timeout as SocketTimeout
+from .util import resolve_cert_reqs, resolve_ssl_version
 
 try: # Python 3
     from http.client import HTTPConnection, HTTPException
@@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     ssl_version = None
 
     def set_cert(self, key_file=None, cert_file=None,
-                 cert_reqs='CERT_NONE', ca_certs=None):
-        ssl_req_scheme = {
-            'CERT_NONE': ssl.CERT_NONE,
-            'CERT_OPTIONAL': ssl.CERT_OPTIONAL,
-            'CERT_REQUIRED': ssl.CERT_REQUIRED
-        }
+                 cert_reqs=None, ca_certs=None):
 
         self.key_file = key_file
         self.cert_file = cert_file
-        self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE
+        self.cert_reqs = cert_reqs
         self.ca_certs = ca_certs
 
     def connect(self):
         # Add certificate verification
         sock = socket.create_connection((self.host, self.port), self.timeout)
 
+        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
+        resolved_ssl_version = resolve_ssl_version(self.ssl_version)
+
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
         self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
-                                    cert_reqs=self.cert_reqs,
+                                    cert_reqs=resolved_cert_reqs,
                                     ca_certs=self.ca_certs,
                                     server_hostname=self.host,
-                                    ssl_version=self.ssl_version)
+                                    ssl_version=resolved_ssl_version)
 
-        if self.ca_certs:
+        if resolved_cert_reqs != ssl.CERT_NONE:
             match_hostname(self.sock.getpeercert(), self.host)
 
 
@@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
                  strict=False, timeout=None, maxsize=1,
                  block=False, headers=None,
                  key_file=None, cert_file=None,
-                 cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None):
+                 cert_reqs=None, ca_certs=None, ssl_version=None):
 
         HTTPConnectionPool.__init__(self, host, port,
                                     strict, timeout, maxsize,
@@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
                             cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
 
-        if self.ssl_version is None:
-            connection.ssl_version = ssl.PROTOCOL_SSLv23
-        else:
-            connection.ssl_version = self.ssl_version
+        connection.ssl_version = self.ssl_version
 
         return connection
 
diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py
index a124202..6e7377c 100644
--- a/requests/packages/urllib3/poolmanager.py
+++ b/requests/packages/urllib3/poolmanager.py
@@ -58,6 +58,17 @@ class PoolManager(RequestMethods):
         self.pools = RecentlyUsedContainer(num_pools,
                                            dispose_func=lambda p: p.close())
 
+    def _new_pool(self, scheme, host, port):
+        """
+        Create a new :class:`ConnectionPool` based on host, port and scheme.
+
+        This method is used to actually create the connection pools handed out
+        by :meth:`connection_from_url` and companion methods. It is intended
+        to be overridden for customization.
+        """
+        pool_cls = pool_classes_by_scheme[scheme]
+        return pool_cls(host, port, **self.connection_pool_kw)
+
     def clear(self):
         """
         Empty our store of pools and direct them all to close.
@@ -74,6 +85,7 @@ class PoolManager(RequestMethods):
         If ``port`` isn't given, it will be derived from the ``scheme`` using
         ``urllib3.connectionpool.port_by_scheme``.
         """
+        scheme = scheme or 'http'
         port = port or port_by_scheme.get(scheme, 80)
 
         pool_key = (scheme, host, port)
@@ -85,11 +97,8 @@ class PoolManager(RequestMethods):
             return pool
 
         # Make a fresh ConnectionPool of the desired type
-        pool_cls = pool_classes_by_scheme[scheme]
-        pool = pool_cls(host, port, **self.connection_pool_kw)
-
+        pool = self._new_pool(scheme, host, port)
         self.pools[pool_key] = pool
-
         return pool
 
     def connection_from_url(self, url):
@@ -138,14 +147,24 @@ class PoolManager(RequestMethods):
 class ProxyManager(RequestMethods):
     """
     Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method
-    will make requests to any url through the defined proxy.
+    will make requests to any url through the defined proxy. The ProxyManager
+    class will automatically set the 'Host' header if it is not provided.
     """
 
     def __init__(self, proxy_pool):
         self.proxy_pool = proxy_pool
 
-    def _set_proxy_headers(self, headers=None):
+    def _set_proxy_headers(self, url, headers=None):
+        """
+        Sets headers needed by proxies: specifically, the Accept and Host
+        headers. Only sets headers not provided by the user.
+        """
         headers_ = {'Accept': '*/*'}
+
+        host = parse_url(url).host
+        if host:
+            headers_['Host'] = host
+
         if headers:
             headers_.update(headers)
 
@@ -154,7 +173,7 @@ class ProxyManager(RequestMethods):
     def urlopen(self, method, url, **kw):
         "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
         kw['assert_same_host'] = False
-        kw['headers'] = self._set_proxy_headers(kw.get('headers'))
+        kw['headers'] = self._set_proxy_headers(url, headers=kw.get('headers'))
         return self.proxy_pool.urlopen(method, url, **kw)
 
 
diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py
index 833be62..0761dc0 100644
--- a/requests/packages/urllib3/response.py
+++ b/requests/packages/urllib3/response.py
@@ -145,7 +145,17 @@ class HTTPResponse(object):
                 # cStringIO doesn't like amt=None
                 data = self._fp.read()
             else:
-                return self._fp.read(amt)
+                data = self._fp.read(amt)
+                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python.
+                    # Close the connection when no data is returned
+                    #
+                    # This is redundant to what httplib/http.client _should_
+                    # already do.  However, versions of python released before
+                    # December 15, 2012 (http://bugs.python.org/issue16298) do not
+                    # properly close the connection in all cases. There is no harm
+                    # in redundantly calling close.
+                    self._fp.close()
+                return data
 
             try:
                 if decode_content and decoder:
diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py
index 8d8654f..b827bc4 100644
--- a/requests/packages/urllib3/util.py
+++ b/requests/packages/urllib3/util.py
@@ -22,6 +22,7 @@ try:  # Test for SSL features
     SSLContext = None
     HAS_SNI = False
 
+    import ssl
     from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23
     from ssl import SSLContext  # Modern SSL?
     from ssl import HAS_SNI  # Has SNI?
@@ -263,10 +264,48 @@ def is_connection_dropped(conn):
             return True
 
 
+def resolve_cert_reqs(candidate):
+    """
+    Resolves the argument to a numeric constant, which can be passed to
+    the wrap_socket function/method from the ssl module.
+    Defaults to :data:`ssl.CERT_NONE`.
+    If given a string it is assumed to be the name of the constant in the
+    :mod:`ssl` module or its abbrevation.
+    (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
+    If it's neither `None` nor a string we assume it is already the numeric
+    constant which can directly be passed to wrap_socket.
+    """
+    if candidate is None:
+        return CERT_NONE
+
+    if isinstance(candidate, str):
+        res = getattr(ssl, candidate, None)
+        if res is None:
+            res = getattr(ssl, 'CERT_' + candidate)
+        return res
+
+    return candidate
+
+
+def resolve_ssl_version(candidate):
+    """
+    like resolve_cert_reqs
+    """
+    if candidate is None:
+        return PROTOCOL_SSLv23
+
+    if isinstance(candidate, str):
+        res = getattr(ssl, candidate, None)
+        if res is None:
+            res = getattr(ssl, 'PROTOCOL_' + candidate)
+        return res
+
+    return candidate
+
 if SSLContext is not None:  # Python 3.2+
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                         ca_certs=None, server_hostname=None,
-                        ssl_version=PROTOCOL_SSLv23):
+                        ssl_version=None):
         """
         All arguments except `server_hostname` have the same meaning as for
         :func:`ssl.wrap_socket`
@@ -279,8 +318,9 @@ if SSLContext is not None:  # Python 3.2+
         if ca_certs:
             try:
                 context.load_verify_locations(ca_certs)
-            except TypeError as e:  # Reraise as SSLError
-                # FIXME: This block needs a test.
+            # Py32 raises IOError
+            # Py33 raises FileNotFoundError
+            except Exception as e:  # Reraise as SSLError
                 raise SSLError(e)
         if certfile:
             # FIXME: This block needs a test.
@@ -290,9 +330,9 @@ if SSLContext is not None:  # Python 3.2+
         return context.wrap_socket(sock)
 
 else:  # Python 3.1 and earlier
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                         ca_certs=None, server_hostname=None,
-                        ssl_version=PROTOCOL_SSLv23):
+                        ssl_version=None):
         return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
                            ca_certs=ca_certs, cert_reqs=cert_reqs,
                            ssl_version=ssl_version)