Imported Upstream version 0.10.8

1 files changed, 122 insertions, 110 deletions

diff --git a/requests/models.py b/requests/models.py
index c200896..753e83a 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -21,14 +21,16 @@ from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
 from .packages.urllib3.filepost import encode_multipart_formdata
+from .defaults import SCHEMAS
 from .exceptions import (
     ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
     URLRequired, SSLError)
 from .utils import (
-    get_encoding_from_headers, stream_decode_response_unicode,
-    stream_decompress, guess_filename, requote_path, dict_from_string)
-
-from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2
+    get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
+    dict_from_string, stream_decode_response_unicode, get_netrc_auth)
+from .compat import (
+    urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
+    SimpleCookie, is_py2)
 
 # Import chardet if it is available.
 try:
@@ -39,7 +41,6 @@ except ImportError:
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 
 
-
 class Request(object):
     """The :class:`Request <Request>` object. It carries out all functionality of
     Requests. Recommended interface is with the Requests functions.
@@ -64,16 +65,14 @@ class Request(object):
         verify=None,
         session=None):
 
+        #: Dictionary of configurations for this request.
+        self.config = dict(config or [])
+
         #: Float describes the timeout of the request.
         #  (Use socket.setdefaulttimeout() as fallback)
         self.timeout = timeout
 
         #: Request URL.
-
-        # if isinstance(url, str):
-            # url = url.encode('utf-8')
-            # print(dir(url))
-
         self.url = url
 
         #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
@@ -103,6 +102,14 @@ class Request(object):
         # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
         self.proxies = dict(proxies or [])
 
+        # If no proxies are given, allow configuration by environment variables
+        # HTTP_PROXY and HTTPS_PROXY.
+        if not self.proxies and self.config.get('trust_env'):
+          if 'HTTP_PROXY' in os.environ:
+            self.proxies['http'] = os.environ['HTTP_PROXY']
+          if 'HTTPS_PROXY' in os.environ:
+            self.proxies['https'] = os.environ['HTTPS_PROXY']
+
         self.data, self._enc_data = self._encode_params(data)
         self.params, self._enc_params = self._encode_params(params)
 
@@ -116,9 +123,6 @@ class Request(object):
         #: CookieJar to attach to :class:`Request <Request>`.
         self.cookies = dict(cookies or [])
 
-        #: Dictionary of configurations for this request.
-        self.config = dict(config or [])
-
         #: True if Request has been sent.
         self.sent = False
 
@@ -152,15 +156,9 @@ class Request(object):
         self.headers = headers
         self._poolmanager = _poolmanager
 
-        # Pre-request hook.
-        r = dispatch_hook('pre_request', hooks, self)
-        self.__dict__.update(r.__dict__)
-
-
     def __repr__(self):
         return '<Request [%s]>' % (self.method)
 
-
     def _build_response(self, resp):
         """Build internal :class:`Response <Response>` object
         from given response.
@@ -200,26 +198,31 @@ class Request(object):
 
             # Save original response for later.
             response.raw = resp
-            response.url = self.full_url
+            if isinstance(self.full_url, bytes):
+                response.url = self.full_url.decode('utf-8')
+            else:
+                response.url = self.full_url
 
             return response
 
         history = []
 
         r = build(resp)
-        cookies = self.cookies
+
         self.cookies.update(r.cookies)
 
         if r.status_code in REDIRECT_STATI and not self.redirect:
+            while (('location' in r.headers) and
+                   ((r.status_code is codes.see_other) or (self.allow_redirects))):
 
-            while (
-                ('location' in r.headers) and
-                ((r.status_code is codes.see_other) or (self.allow_redirects))
-            ):
+                r.content  # Consume socket so it can be released
 
                 if not len(history) < self.config.get('max_redirects'):
                     raise TooManyRedirects()
 
+                # Release the connection back into the pool.
+                r.raw.release_conn()
+
                 history.append(r)
 
                 url = r.headers['location']
@@ -232,7 +235,10 @@ class Request(object):
                 # Facilitate non-RFC2616-compliant 'location' headers
                 # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
                 if not urlparse(url).netloc:
-                    url = urljoin(r.url, url)
+                    url = urljoin(r.url,
+                                  # Compliant with RFC3986, we percent
+                                  # encode the url.
+                                  requote_uri(url))
 
                 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
                 if r.status_code is codes.see_other:
@@ -254,18 +260,17 @@ class Request(object):
                     method=method,
                     params=self.session.params,
                     auth=self.auth,
-                    cookies=cookies,
+                    cookies=self.cookies,
                     redirect=True,
                     config=self.config,
                     timeout=self.timeout,
                     _poolmanager=self._poolmanager,
-                    proxies = self.proxies,
-                    verify = self.verify,
-                    session = self.session
+                    proxies=self.proxies,
+                    verify=self.verify,
+                    session=self.session
                 )
 
                 request.send()
-                cookies.update(request.response.cookies)
                 r = request.response
                 self.cookies.update(r.cookies)
 
@@ -275,7 +280,6 @@ class Request(object):
         self.response.request = self
         self.response.cookies.update(self.cookies)
 
-
     @staticmethod
     def _encode_params(data):
         """Encode parameters in a piece of data.
@@ -288,10 +292,12 @@ class Request(object):
         returns it twice.
         """
 
+        if isinstance(data, bytes):
+            return data, data
+
         if hasattr(data, '__iter__') and not isinstance(data, str):
             data = dict(data)
 
-
         if hasattr(data, 'items'):
             result = []
             for k, vs in list(data.items()):
@@ -314,30 +320,44 @@ class Request(object):
         # Support for unicode domain names and paths.
         scheme, netloc, path, params, query, fragment = urlparse(url)
 
-
         if not scheme:
             raise ValueError("Invalid URL %r: No schema supplied" % url)
 
+        if not scheme in SCHEMAS:
+            raise ValueError("Invalid scheme %r" % scheme)
+
         netloc = netloc.encode('idna').decode('utf-8')
 
+        if not path:
+            path = '/'
+
+
         if is_py2:
+            if isinstance(scheme, str):
+                scheme = scheme.encode('utf-8')
+            if isinstance(netloc, str):
+                netloc = netloc.encode('utf-8')
             if isinstance(path, str):
                 path = path.encode('utf-8')
+            if isinstance(params, str):
+                params = params.encode('utf-8')
+            if isinstance(query, str):
+                query = query.encode('utf-8')
+            if isinstance(fragment, str):
+                fragment = fragment.encode('utf-8')
 
-            path = requote_path(path)
-
-        # print([ scheme, netloc, path, params, query, fragment ])
-        # print('---------------------')
-
-        url = (urlunparse([ scheme, netloc, path, params, query, fragment ]))
+        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
 
         if self._enc_params:
             if urlparse(url).query:
-                return '%s&%s' % (url, self._enc_params)
+                url = '%s&%s' % (url, self._enc_params)
             else:
-                return '%s?%s' % (url, self._enc_params)
-        else:
-            return url
+                url = '%s?%s' % (url, self._enc_params)
+
+        if self.config.get('encode_uri', True):
+            url = requote_uri(url)
+
+        return url
 
     @property
     def path_url(self):
@@ -355,9 +375,6 @@ class Request(object):
         if not path:
             path = '/'
 
-        # if is_py3:
-        path = quote(path.encode('utf-8'))
-
         url.append(path)
 
         query = p.query
@@ -365,19 +382,15 @@ class Request(object):
             url.append('?')
             url.append(query)
 
-        # print(url)
-
         return ''.join(url)
 
-
     def register_hook(self, event, hook):
         """Properly register a hook."""
 
         return self.hooks[event].append(hook)
 
-
     def send(self, anyway=False, prefetch=False):
-        """Sends the request. Returns True of successful, false if not.
+        """Sends the request. Returns True of successful, False if not.
         If there was an HTTPError during transmission,
         self.response.status_code will contain the HTTPError code.
 
@@ -435,6 +448,10 @@ class Request(object):
         if (content_type) and (not 'content-type' in self.headers):
             self.headers['Content-Type'] = content_type
 
+        # Use .netrc auth if none was provided.
+        if not self.auth and self.config.get('trust_env'):
+            self.auth = get_netrc_auth(url)
+
         if self.auth:
             if isinstance(self.auth, tuple) and len(self.auth) == 2:
                 # special-case basic HTTP auth
@@ -472,13 +489,12 @@ class Request(object):
             if self.verify is not True:
                 cert_loc = self.verify
 
-
             # Look for configuration.
-            if not cert_loc:
+            if not cert_loc and self.config.get('trust_env'):
                 cert_loc = os.environ.get('REQUESTS_CA_BUNDLE')
 
             # Curl compatiblity.
-            if not cert_loc:
+            if not cert_loc and self.config.get('trust_env'):
                 cert_loc = os.environ.get('CURL_CA_BUNDLE')
 
             # Use the awesome certifi list.
@@ -509,6 +525,10 @@ class Request(object):
                     # Attach Cookie header to request.
                     self.headers['Cookie'] = cookie_header
 
+            # Pre-request hook.
+            r = dispatch_hook('pre_request', self.hooks, self)
+            self.__dict__.update(r.__dict__)
+
             try:
                 # The inner try .. except re-raises certain exceptions as
                 # internal exception types; the outer suppresses exceptions
@@ -523,7 +543,7 @@ class Request(object):
                         redirect=False,
                         assert_same_host=False,
                         preload_content=False,
-                        decode_content=True,
+                        decode_content=False,
                         retries=self.config.get('max_retries', 0),
                         timeout=self.timeout,
                     )
@@ -613,7 +633,6 @@ class Response(object):
         #: Dictionary of configurations for this request.
         self.config = {}
 
-
     def __repr__(self):
         return '<Response [%s]>' % (self.status_code)
 
@@ -633,7 +652,6 @@ class Response(object):
             return False
         return True
 
-
     def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
         """Iterates over the response data.  This avoids reading the content
         at once into memory for large responses.  The chunk size is the number
@@ -660,74 +678,64 @@ class Response(object):
                 pending_bytes = resp.chunk_left
                 while pending_bytes:
                     chunk = fp.read(min(chunk_size, pending_bytes))
-                    pending_bytes-=len(chunk)
+                    pending_bytes -= len(chunk)
                     yield chunk
-                fp.read(2) # throw away crlf
+                fp.read(2)  # throw away crlf
             while 1:
                 #XXX correct line size? (httplib has 64kb, seems insane)
                 pending_bytes = fp.readline(40).strip()
+                if not len(pending_bytes):
+                    # No content, like a HEAD request. Break out.
+                    break
                 pending_bytes = int(pending_bytes, 16)
                 if pending_bytes == 0:
                     break
                 while pending_bytes:
                     chunk = fp.read(min(chunk_size, pending_bytes))
-                    pending_bytes-=len(chunk)
+                    pending_bytes -= len(chunk)
                     yield chunk
-                fp.read(2) # throw away crlf
+                fp.read(2)  # throw away crlf
             self._content_consumed = True
             fp.close()
 
-
         if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
             gen = generate_chunked()
         else:
             gen = generate()
 
-        if 'gzip' in self.headers.get('content-encoding', ''):
-            gen = stream_decompress(gen, mode='gzip')
-        elif 'deflate' in self.headers.get('content-encoding', ''):
-            gen = stream_decompress(gen, mode='deflate')
+        gen = stream_untransfer(gen, self)
 
         if decode_unicode:
             gen = stream_decode_response_unicode(gen, self)
 
         return gen
 
-
     def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None):
         """Iterates over the response data, one line at a time.  This
         avoids reading the content at once into memory for large
         responses.
         """
 
-        #TODO: why rstrip by default
         pending = None
 
-        for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
+        for chunk in self.iter_content(
+            chunk_size=chunk_size,
+            decode_unicode=decode_unicode):
 
             if pending is not None:
                 chunk = pending + chunk
-            lines = chunk.splitlines(True)
+            lines = chunk.splitlines()
 
-            for line in lines[:-1]:
-                yield line.rstrip()
-
-            # Save the last part of the chunk for next iteration, to keep full line together
-            # lines may be empty for the last chunk of a chunked response
-
-            if lines:
-                pending = lines[-1]
-                #if pending is a complete line, give it baack
-                if pending[-1] == '\n':
-                    yield pending.rstrip()
-                    pending = None
+            if lines[-1][-1] == chunk[-1]:
+                pending = lines.pop()
             else:
                 pending = None
 
-        # Yield the last line
-        if pending is not None:
-            yield pending.rstrip()
+            for line in lines:
+                yield line
 
+        if pending is not None:
+            yield pending
 
     @property
     def content(self):
@@ -740,13 +748,26 @@ class Response(object):
                     raise RuntimeError(
                         'The content for this response was already consumed')
 
-                self._content = self.raw.read()
+                if self.status_code is 0:
+                    self._content = None
+                else:
+                    self._content = bytes().join(self.iter_content()) or bytes()
+
             except AttributeError:
                 self._content = None
 
         self._content_consumed = True
         return self._content
 
+    def _detected_encoding(self):
+        try:
+            detected = chardet.detect(self.content) or {}
+            return detected.get('encoding')
+
+        # Trust that chardet isn't available or something went terribly wrong.
+        except Exception:
+            pass
+
 
     @property
     def text(self):
@@ -762,43 +783,34 @@ class Response(object):
 
         # Fallback to auto-detected encoding if chardet is available.
         if self.encoding is None:
-            try:
-                detected = chardet.detect(self.content) or {}
-                encoding = detected.get('encoding')
-
-            # Trust that chardet isn't available or something went terribly wrong.
-            except Exception:
-                pass
+            encoding = self._detected_encoding()
 
         # Decode unicode from given encoding.
         try:
-            content = str(self.content, encoding)
+            content = str(self.content, encoding, errors='replace')
         except (UnicodeError, TypeError):
             pass
 
-        # Try to fall back:
-        if not content:
-            try:
-                content = str(content, encoding, errors='replace')
-            except (UnicodeError, TypeError):
-                pass
-
         return content
 
-
-    def raise_for_status(self):
+    def raise_for_status(self, allow_redirects=True):
         """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
 
         if self.error:
             raise self.error
 
-        if (self.status_code >= 300) and (self.status_code < 400):
-            raise HTTPError('%s Redirection' % self.status_code)
+        if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
+            http_error = HTTPError('%s Redirection' % self.status_code)
+            http_error.response = self
+            raise http_error
 
         elif (self.status_code >= 400) and (self.status_code < 500):
-            raise HTTPError('%s Client Error' % self.status_code)
-
-        elif (self.status_code >= 500) and (self.status_code < 600):
-            raise HTTPError('%s Server Error' % self.status_code)
+            http_error = HTTPError('%s Client Error' % self.status_code)
+            http_error.response = self
+            raise http_error
 
 
+        elif (self.status_code >= 500) and (self.status_code < 600):
+            http_error = HTTPError('%s Server Error' % self.status_code)
+            http_error.response = self
+            raise http_error