Imported Upstream version 0.10.1

1 files changed, 269 insertions, 100 deletions

diff --git a/requests/models.py b/requests/models.py
index 0be3e89..c200896 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -7,28 +7,34 @@ requests.models
 This module contains the primary objects that power Requests.
 """
 
-import urllib
-import zlib
-
-from urlparse import urlparse, urlunparse, urljoin, urlsplit
+import os
 from datetime import datetime
 
-from .auth import dispatch as auth_dispatch
-from .hooks import dispatch_hook
+from .hooks import dispatch_hook, HOOKS
 from .structures import CaseInsensitiveDict
 from .status_codes import codes
-from .packages import oreos
+
+from .auth import HTTPBasicAuth, HTTPProxyAuth
+from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.exceptions import MaxRetryError
 from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .exceptions import (
-    Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError)
+    ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
+    URLRequired, SSLError)
 from .utils import (
     get_encoding_from_headers, stream_decode_response_unicode,
-    decode_gzip, stream_decode_gzip, guess_filename, requote_path)
+    stream_decompress, guess_filename, requote_path, dict_from_string)
+
+from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2
 
+# Import chardet if it is available.
+try:
+    import chardet
+except ImportError:
+    pass
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 
@@ -54,13 +60,20 @@ class Request(object):
         proxies=None,
         hooks=None,
         config=None,
-        _poolmanager=None):
+        _poolmanager=None,
+        verify=None,
+        session=None):
 
         #: Float describes the timeout of the request.
         #  (Use socket.setdefaulttimeout() as fallback)
         self.timeout = timeout
 
         #: Request URL.
+
+        # if isinstance(url, str):
+            # url = url.encode('utf-8')
+            # print(dir(url))
+
         self.url = url
 
         #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
@@ -79,7 +92,6 @@ class Request(object):
         #: Dictionary or byte of querystring data to attach to the
         #: :class:`Request <Request>`.
         self.params = None
-        self.params = dict(params or [])
 
         #: True if :class:`Request <Request>` is part of a redirect chain (disables history
         #: and HTTPError storage).
@@ -98,9 +110,8 @@ class Request(object):
         #: content and metadata of HTTP Response, once :attr:`sent <send>`.
         self.response = Response()
 
-        #: Authentication tuple to attach to :class:`Request <Request>`.
-        self._auth = auth
-        self.auth = auth_dispatch(auth)
+        #: Authentication tuple or object to attach to :class:`Request <Request>`.
+        self.auth = auth
 
         #: CookieJar to attach to :class:`Request <Request>`.
         self.cookies = dict(cookies or [])
@@ -112,17 +123,29 @@ class Request(object):
         self.sent = False
 
         #: Event-handling hooks.
-        self.hooks = hooks
+        self.hooks = {}
+
+        for event in HOOKS:
+            self.hooks[event] = []
+
+        hooks = hooks or {}
+
+        for (k, v) in list(hooks.items()):
+            self.register_hook(event=k, hook=v)
 
         #: Session.
-        self.session = None
+        self.session = session
+
+        #: SSL Verification.
+        self.verify = verify
 
         if headers:
             headers = CaseInsensitiveDict(self.headers)
         else:
             headers = CaseInsensitiveDict()
 
-        for (k, v) in self.config.get('base_headers', {}).items():
+        # Add configured base headers.
+        for (k, v) in list(self.config.get('base_headers', {}).items()):
             if k not in headers:
                 headers[k] = v
 
@@ -138,7 +161,7 @@ class Request(object):
         return '<Request [%s]>' % (self.method)
 
 
-    def _build_response(self, resp, is_error=False):
+    def _build_response(self, resp):
         """Build internal :class:`Response <Response>` object
         from given response.
         """
@@ -152,7 +175,7 @@ class Request(object):
 
             if resp:
 
-                # Fallback to None if there's no staus_code, for whatever reason.
+                # Fallback to None if there's no status_code, for whatever reason.
                 response.status_code = getattr(resp, 'status', None)
 
                 # Make headers case-insensitive.
@@ -167,17 +190,16 @@ class Request(object):
                 # Add new cookies from the server.
                 if 'set-cookie' in response.headers:
                     cookie_header = response.headers['set-cookie']
-                    cookies = oreos.dict_from_string(cookie_header)
+                    cookies = dict_from_string(cookie_header)
 
                 # Save cookies in Response.
                 response.cookies = cookies
 
-            # Save original resopnse for later.
-            response.raw = resp
-
-            if is_error:
-                response.error = resp
+                # No exceptions were harmed in the making of this request.
+                response.error = getattr(resp, 'error', None)
 
+            # Save original response for later.
+            response.raw = resp
             response.url = self.full_url
 
             return response
@@ -231,13 +253,15 @@ class Request(object):
                     files=self.files,
                     method=method,
                     params=self.session.params,
-                    auth=self._auth,
+                    auth=self.auth,
                     cookies=cookies,
                     redirect=True,
                     config=self.config,
                     timeout=self.timeout,
                     _poolmanager=self._poolmanager,
                     proxies = self.proxies,
+                    verify = self.verify,
+                    session = self.session
                 )
 
                 request.send()
@@ -264,16 +288,17 @@ class Request(object):
         returns it twice.
         """
 
-        if hasattr(data, '__iter__'):
+        if hasattr(data, '__iter__') and not isinstance(data, str):
             data = dict(data)
 
+
         if hasattr(data, 'items'):
             result = []
-            for k, vs in data.items():
+            for k, vs in list(data.items()):
                 for v in isinstance(vs, list) and vs or [vs]:
-                    result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
-                                   v.encode('utf-8') if isinstance(v, unicode) else v))
-            return result, urllib.urlencode(result, doseq=True)
+                    result.append((k.encode('utf-8') if isinstance(k, str) else k,
+                                   v.encode('utf-8') if isinstance(v, str) else v))
+            return result, urlencode(result, doseq=True)
         else:
             return data, data
 
@@ -284,20 +309,27 @@ class Request(object):
         if not self.url:
             raise URLRequired()
 
+        url = self.url
+
         # Support for unicode domain names and paths.
-        scheme, netloc, path, params, query, fragment = urlparse(self.url)
+        scheme, netloc, path, params, query, fragment = urlparse(url)
+
 
         if not scheme:
-            raise ValueError()
+            raise ValueError("Invalid URL %r: No schema supplied" % url)
+
+        netloc = netloc.encode('idna').decode('utf-8')
 
-        netloc = netloc.encode('idna')
+        if is_py2:
+            if isinstance(path, str):
+                path = path.encode('utf-8')
 
-        if isinstance(path, unicode):
-            path = path.encode('utf-8')
+            path = requote_path(path)
 
-        path = requote_path(path)
+        # print([ scheme, netloc, path, params, query, fragment ])
+        # print('---------------------')
 
-        url = str(urlunparse([ scheme, netloc, path, params, query, fragment ]))
+        url = (urlunparse([ scheme, netloc, path, params, query, fragment ]))
 
         if self._enc_params:
             if urlparse(url).query:
@@ -322,6 +354,10 @@ class Request(object):
         path = p.path
         if not path:
             path = '/'
+
+        # if is_py3:
+        path = quote(path.encode('utf-8'))
+
         url.append(path)
 
         query = p.query
@@ -329,9 +365,16 @@ class Request(object):
             url.append('?')
             url.append(query)
 
+        # print(url)
+
         return ''.join(url)
 
 
+    def register_hook(self, event, hook):
+        """Properly register a hook."""
+
+        return self.hooks[event].append(hook)
+
 
     def send(self, anyway=False, prefetch=False):
         """Sends the request. Returns True of successful, false if not.
@@ -344,29 +387,29 @@ class Request(object):
         already been sent.
         """
 
+        # Build the URL
+        url = self.full_url
+
         # Logging
         if self.config.get('verbose'):
             self.config.get('verbose').write('%s   %s   %s\n' % (
-                datetime.now().isoformat(), self.method, self.url
+                datetime.now().isoformat(), self.method, url
             ))
 
-        # Build the URL
-        url = self.full_url
-
         # Nottin' on you.
         body = None
         content_type = None
 
         # Multi-part file uploads.
         if self.files:
-            if not isinstance(self.data, basestring):
+            if not isinstance(self.data, str):
 
                 try:
                     fields = self.data.copy()
                 except AttributeError:
                     fields = dict(self.data)
 
-                for (k, v) in self.files.items():
+                for (k, v) in list(self.files.items()):
                     # support for explicit filename
                     if isinstance(v, (tuple, list)):
                         fn, fp = v
@@ -383,7 +426,7 @@ class Request(object):
             if self.data:
 
                 body = self._enc_data
-                if isinstance(self.data, basestring):
+                if isinstance(self.data, str):
                     content_type = None
                 else:
                     content_type = 'application/x-www-form-urlencoded'
@@ -392,12 +435,13 @@ class Request(object):
         if (content_type) and (not 'content-type' in self.headers):
             self.headers['Content-Type'] = content_type
 
-
         if self.auth:
-            auth_func, auth_args = self.auth
+            if isinstance(self.auth, tuple) and len(self.auth) == 2:
+                # special-case basic HTTP auth
+                self.auth = HTTPBasicAuth(*self.auth)
 
             # Allow auth to make its changes.
-            r = auth_func(self, *auth_args)
+            r = self.auth(self)
 
             # Update self to reflect the auth changes.
             self.__dict__.update(r.__dict__)
@@ -407,6 +451,12 @@ class Request(object):
 
         if proxy:
             conn = poolmanager.proxy_from_url(proxy)
+            _proxy = urlparse(proxy)
+            if '@' in _proxy.netloc:
+                auth, url = _proxy.netloc.split('@', 1)
+                self.proxy_auth = HTTPProxyAuth(*auth.split(':', 1))
+                r = self.proxy_auth(self)
+                self.__dict__.update(r.__dict__)
         else:
             # Check to see if keep_alive is allowed.
             if self.config.get('keep_alive'):
@@ -414,6 +464,33 @@ class Request(object):
             else:
                 conn = connectionpool.connection_from_url(url)
 
+        if url.startswith('https') and self.verify:
+
+            cert_loc = None
+
+            # Allow self-specified cert location.
+            if self.verify is not True:
+                cert_loc = self.verify
+
+
+            # Look for configuration.
+            if not cert_loc:
+                cert_loc = os.environ.get('REQUESTS_CA_BUNDLE')
+
+            # Curl compatiblity.
+            if not cert_loc:
+                cert_loc = os.environ.get('CURL_CA_BUNDLE')
+
+            # Use the awesome certifi list.
+            if not cert_loc:
+                cert_loc = __import__('certifi').where()
+
+            conn.cert_reqs = 'CERT_REQUIRED'
+            conn.ca_certs = cert_loc
+        else:
+            conn.cert_reqs = 'CERT_NONE'
+            conn.ca_certs = None
+
         if not self.sent or anyway:
 
             if self.cookies:
@@ -422,8 +499,8 @@ class Request(object):
                 if 'cookie' not in self.headers:
 
                     # Simple cookie with our dict.
-                    c = oreos.monkeys.SimpleCookie()
-                    for (k, v) in self.cookies.items():
+                    c = SimpleCookie()
+                    for (k, v) in list(self.cookies.items()):
                         c[k] = v
 
                     # Turn it into a header.
@@ -433,31 +510,43 @@ class Request(object):
                     self.headers['Cookie'] = cookie_header
 
             try:
-                # Send the request.
-                r = conn.urlopen(
-                    method=self.method,
-                    url=self.path_url,
-                    body=body,
-                    headers=self.headers,
-                    redirect=False,
-                    assert_same_host=False,
-                    preload_content=prefetch,
-                    decode_content=False,
-                    retries=self.config.get('max_retries', 0),
-                    timeout=self.timeout,
-                )
-
-
-            except MaxRetryError, e:
-                if not self.config.get('safe_mode', False):
+                # The inner try .. except re-raises certain exceptions as
+                # internal exception types; the outer suppresses exceptions
+                # when safe mode is set.
+                try:
+                    # Send the request.
+                    r = conn.urlopen(
+                        method=self.method,
+                        url=self.path_url,
+                        body=body,
+                        headers=self.headers,
+                        redirect=False,
+                        assert_same_host=False,
+                        preload_content=False,
+                        decode_content=True,
+                        retries=self.config.get('max_retries', 0),
+                        timeout=self.timeout,
+                    )
+                    self.sent = True
+
+                except MaxRetryError as e:
                     raise ConnectionError(e)
-                else:
-                    r = None
 
-            except (_SSLError, _HTTPError), e:
-                if not self.config.get('safe_mode', False):
+                except (_SSLError, _HTTPError) as e:
+                    if self.verify and isinstance(e, _SSLError):
+                        raise SSLError(e)
+
                     raise Timeout('Request timed out.')
 
+            except RequestException as e:
+                if self.config.get('safe_mode', False):
+                    # In safe mode, catch the exception and attach it to
+                    # a blank urllib3.HTTPResponse object.
+                    r = HTTPResponse()
+                    r.error = e
+                else:
+                    raise
+
             self._build_response(r)
 
             # Response manipulation hook.
@@ -469,7 +558,11 @@ class Request(object):
 
             # If prefetch is True, mark content as consumed.
             if prefetch:
-                self.response._content_consumed = True
+                # Save the response.
+                self.response.content
+
+            if self.config.get('danger_mode'):
+                self.response.raise_for_status()
 
             return self.sent
 
@@ -524,6 +617,10 @@ class Response(object):
     def __repr__(self):
         return '<Response [%s]>' % (self.status_code)
 
+    def __bool__(self):
+        """Returns true if :attr:`status_code` is 'OK'."""
+        return self.ok
+
     def __nonzero__(self):
         """Returns true if :attr:`status_code` is 'OK'."""
         return self.ok
@@ -537,7 +634,7 @@ class Response(object):
         return True
 
 
-    def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
+    def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
         """Iterates over the response data.  This avoids reading the content
         at once into memory for large responses.  The chunk size is the number
         of bytes it should read into memory.  This is not necessarily the
@@ -556,13 +653,40 @@ class Response(object):
                 yield chunk
             self._content_consumed = True
 
-        gen = generate()
+        def generate_chunked():
+            resp = self.raw._original_response
+            fp = resp.fp
+            if resp.chunk_left is not None:
+                pending_bytes = resp.chunk_left
+                while pending_bytes:
+                    chunk = fp.read(min(chunk_size, pending_bytes))
+                    pending_bytes-=len(chunk)
+                    yield chunk
+                fp.read(2) # throw away crlf
+            while 1:
+                #XXX correct line size? (httplib has 64kb, seems insane)
+                pending_bytes = fp.readline(40).strip()
+                pending_bytes = int(pending_bytes, 16)
+                if pending_bytes == 0:
+                    break
+                while pending_bytes:
+                    chunk = fp.read(min(chunk_size, pending_bytes))
+                    pending_bytes-=len(chunk)
+                    yield chunk
+                fp.read(2) # throw away crlf
+            self._content_consumed = True
+            fp.close()
 
-        if 'gzip' in self.headers.get('content-encoding', ''):
-            gen = stream_decode_gzip(gen)
 
-        if decode_unicode is None:
-            decode_unicode = self.config.get('decode_unicode')
+        if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
+            gen = generate_chunked()
+        else:
+            gen = generate()
+
+        if 'gzip' in self.headers.get('content-encoding', ''):
+            gen = stream_decompress(gen, mode='gzip')
+        elif 'deflate' in self.headers.get('content-encoding', ''):
+            gen = stream_decompress(gen, mode='deflate')
 
         if decode_unicode:
             gen = stream_decode_response_unicode(gen, self)
@@ -570,11 +694,44 @@ class Response(object):
         return gen
 
 
+    def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None):
+        """Iterates over the response data, one line at a time.  This
+        avoids reading the content at once into memory for large
+        responses.
+        """
+
+        #TODO: why rstrip by default
+        pending = None
+
+        for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
+
+            if pending is not None:
+                chunk = pending + chunk
+            lines = chunk.splitlines(True)
+
+            for line in lines[:-1]:
+                yield line.rstrip()
+
+            # Save the last part of the chunk for next iteration, to keep full line together
+            # lines may be empty for the last chunk of a chunked response
+
+            if lines:
+                pending = lines[-1]
+                #if pending is a complete line, give it baack
+                if pending[-1] == '\n':
+                    yield pending.rstrip()
+                    pending = None
+            else:
+                pending = None
+
+        # Yield the last line
+        if pending is not None:
+            yield pending.rstrip()
+
+
     @property
     def content(self):
-        """Content of the response, in bytes or unicode
-        (if available).
-        """
+        """Content of the response, in bytes."""
 
         if self._content is None:
             # Read the contents.
@@ -587,33 +744,45 @@ class Response(object):
             except AttributeError:
                 self._content = None
 
-        content = self._content
+        self._content_consumed = True
+        return self._content
 
-        # Decode GZip'd content.
-        if 'gzip' in self.headers.get('content-encoding', ''):
-            try:
-                content = decode_gzip(self._content)
-            except zlib.error:
-                pass
 
-        # Decode unicode content.
-        if self.config.get('decode_unicode'):
+    @property
+    def text(self):
+        """Content of the response, in unicode.
 
-            # Try charset from content-type
+        if Response.encoding is None and chardet module is available, encoding
+        will be guessed.
+        """
 
-            if self.encoding:
-                try:
-                    content = unicode(content, self.encoding)
-                except UnicodeError:
-                    pass
+        # Try charset from content-type
+        content = None
+        encoding = self.encoding
 
-            # Fall back:
+        # Fallback to auto-detected encoding if chardet is available.
+        if self.encoding is None:
             try:
-                content = unicode(content, self.encoding, errors='replace')
-            except TypeError:
+                detected = chardet.detect(self.content) or {}
+                encoding = detected.get('encoding')
+
+            # Trust that chardet isn't available or something went terribly wrong.
+            except Exception:
+                pass
+
+        # Decode unicode from given encoding.
+        try:
+            content = str(self.content, encoding)
+        except (UnicodeError, TypeError):
+            pass
+
+        # Try to fall back:
+        if not content:
+            try:
+                content = str(content, encoding, errors='replace')
+            except (UnicodeError, TypeError):
                 pass
 
-        self._content_consumed = True
         return content