diff options
| -rw-r--r-- | HISTORY.rst | 108 | ||||
| -rw-r--r-- | LICENSE | 13 | ||||
| -rw-r--r-- | PKG-INFO | 284 | ||||
| -rw-r--r-- | README.rst | 156 | ||||
| -rw-r--r-- | requests/__init__.py | 4 | ||||
| -rw-r--r-- | requests/api.py | 121 | ||||
| -rw-r--r-- | requests/async.py | 41 | ||||
| -rw-r--r-- | requests/config.py | 56 | ||||
| -rw-r--r-- | requests/core.py | 25 | ||||
| -rw-r--r-- | requests/exceptions.py | 23 | ||||
| -rw-r--r-- | requests/models.py | 503 | ||||
| -rw-r--r-- | requests/monkeys.py | 91 | ||||
| -rw-r--r-- | requests/packages/__init__.py | 3 | ||||
| -rw-r--r-- | requests/packages/poster/__init__.py | 34 | ||||
| -rw-r--r-- | requests/packages/poster/encode.py | 414 | ||||
| -rw-r--r-- | requests/packages/poster/streaminghttp.py | 199 | ||||
| -rw-r--r-- | requests/patches.py | 5 | ||||
| -rw-r--r-- | requests/structures.py | 64 | ||||
| -rwxr-xr-x | setup.py | 53 |
19 files changed, 2197 insertions, 0 deletions
diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..c119319 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,108 @@ +History +------- + +0.4.1 (2011-05-22) +++++++++++++++++++ + +* Improved Redirection Handling +* New 'allow_redirects' param for following non-GET/HEAD Redirects +* Settings module refactoring + + +0.4.0 (2011-05-15) +++++++++++++++++++ + +* Response.history: list of redirected responses +* Case-Insensitive Header Dictionaries! +* Unicode URLs + + +0.3.4 (2011-05-14) +++++++++++++++++++ + +* Urllib2 HTTPAuthentication Recursion fix (Basic/Digest) +* Internal Refactor +* Bytes data upload Bugfix + + + +0.3.3 (2011-05-12) +++++++++++++++++++ + +* Request timeouts +* Unicode url-encoded data +* Settings context manager and module + + +0.3.2 (2011-04-15) +++++++++++++++++++ + +* Automatic Decompression of GZip Encoded Content +* AutoAuth Support for Tupled HTTP Auth + + +0.3.1 (2011-04-01) +++++++++++++++++++ + +* Cookie Changes +* Response.read() +* Poster fix + + +0.3.0 (2011-02-25) +++++++++++++++++++ + +* Automatic Authentication API Change +* Smarter Query URL Parameterization +* Allow file uploads and POST data together +* New Authentication Manager System + - Simpler Basic HTTP System + - Supports all build-in urllib2 Auths + - Allows for custom Auth Handlers + + +0.2.4 (2011-02-19) +++++++++++++++++++ + +* Python 2.5 Support +* PyPy-c v1.4 Support +* Auto-Authentication tests +* Improved Request object constructor + +0.2.3 (2011-02-15) +++++++++++++++++++ + +* New HTTPHandling Methods + - Reponse.__nonzero__ (false if bad HTTP Status) + - Response.ok (True if expected HTTP Status) + - Response.error (Logged HTTPError if bad HTTP Status) + - Reponse.raise_for_status() (Raises stored HTTPError) + + +0.2.2 (2011-02-14) +++++++++++++++++++ + +* Still handles request in the event of an HTTPError. (Issue #2) +* Eventlet and Gevent Monkeypatch support. +* Cookie Support (Issue #1) + + +0.2.1 (2011-02-14) +++++++++++++++++++ + +* Added file attribute to POST and PUT requests for multipart-encode file uploads. +* Added Request.url attribute for context and redirects + + +0.2.0 (2011-02-14) +++++++++++++++++++ + +* Birth! + + +0.0.1 (2011-02-13) +++++++++++++++++++ + +* Frustration +* Conception + @@ -0,0 +1,13 @@ +Copyright (c) 2011 Kenneth Reitz. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..ad13d55 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,284 @@ +Metadata-Version: 1.0 +Name: requests +Version: 0.4.1 +Summary: Awesome Python HTTP Library that's actually usable. +Home-page: http://python-requests.org +Author: Kenneth Reitz +Author-email: me@kennethreitz.com +License: ISC +Description: Requests: The Simple (e.g. usable) HTTP Module + ============================================== + + Most existing Python modules for dealing HTTP requests are insane. I have to look up *everything* that I want to do. Most of my worst Python experiences are a result of the various built-in HTTP libraries (yes, even worse than Logging). + + But this one's different. This one's going to be awesome. And simple. + + Really simple. + + Features + -------- + + - Extremely simple GET, HEAD, POST, PUT, DELETE Requests + + Simple HTTP Header Request Attachment + + Simple Data/Params Request Attachment + + Simple Multipart File Uploads + + CookieJar Support + + Redirection History + + Redirection Recursion Urllib Fix + + Auto Decompression of GZipped Content + + Unicode URL Support + + - Simple Authentication + + Simple URL + HTTP Auth Registry + + + Usage + ----- + + It couldn't be simpler. :: + + >>> import requests + >>> r = requests.get('http://google.com') + + + HTTPS? Basic Authentication? :: + + >>> r = requests.get('https://convore.com/api/account/verify.json') + >>> r.status_code + 401 + + + Uh oh, we're not authorized! Let's add authentication. :: + + >>> conv_auth = ('requeststest', 'requeststest') + >>> r = requests.get('https://convore.com/api/account/verify.json', auth=conv_auth) + + >>> r.status_code + 200 + + >>> r.headers['content-type'] + 'application/json' + + >>> r.content + '{"username": "requeststest", "url": "/users/requeststest/", "id": "9408", "img": "censored-long-url"}' + + + + API + --- + + **Requests:** + + All request functions return a Response object (see below). + + If a {filename: fileobject} dictionary is passed in (files=...), a multipart_encode upload will be performed. + If CookieJar object is is passed in (cookies=...), the cookies will be sent with the request. + + GET Requests + >>> requests.get(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + HEAD Requests + >>> requests.head(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + PUT Requests + >>> requests.put(url, data='', headers={}, files={}, cookies=None, auth=None) + <Response [200]> + + POST Requests + >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None) + <Response [200]> + + DELETE Requests + >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + + **Responses:** + + Response.status_code + (Integer) Received HTTP Status Code Response + + Response.headers + ((CaseInsensitive) Dictionary) Received HTTP Response Headers. + + Response.content + (Bytes) Received Content. + + Response.history + (List of Responses) Redirection History. + + Response.url + (String) URL of response. Useful for detecting redirects. + + Response.ok + (Bool) True if no errors occurred during the request, and the status_code is kosher. + + Response.cached + (Bool) True if Response.content is stored within the object. + + Response.error + (HTTPError) If an HTTPError occurred (e.g. status of 404), Otherwise this is None. + + Response.raise_for_status() + Raises HTTPError if a request is not kosher. + + + **HTTP Authentication Registry:** + + You can register AuthObjects to automatically enable HTTP Authentication on requests that contain a registered base URL string. + + >>> requests.auth_manager.add_auth(url, authobject) + + + + Installation + ------------ + + To install requests, simply: :: + + $ pip install requests + + Or, if you absolutely must: :: + + $ easy_install requests + + But, you really shouldn't do that. + + + + Contribute + ---------- + + If you'd like to contribute, simply fork `the repository`_, commit your changes to the **develop** branch (or branch off of it), and send a pull request. Make sure you add yourself to AUTHORS_. + + + + Roadmap + ------- + + - Sphinx Documentation + + .. _`the repository`: http://github.com/kennethreitz/requests + .. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS + + + History + ------- + + 0.4.1 (2011-05-22) + ++++++++++++++++++ + + * Improved Redirection Handling + * New 'allow_redirects' param for following non-GET/HEAD Redirects + * Settings module refactoring + + + 0.4.0 (2011-05-15) + ++++++++++++++++++ + + * Response.history: list of redirected responses + * Case-Insensitive Header Dictionaries! + * Unicode URLs + + + 0.3.4 (2011-05-14) + ++++++++++++++++++ + + * Urllib2 HTTPAuthentication Recursion fix (Basic/Digest) + * Internal Refactor + * Bytes data upload Bugfix + + + + 0.3.3 (2011-05-12) + ++++++++++++++++++ + + * Request timeouts + * Unicode url-encoded data + * Settings context manager and module + + + 0.3.2 (2011-04-15) + ++++++++++++++++++ + + * Automatic Decompression of GZip Encoded Content + * AutoAuth Support for Tupled HTTP Auth + + + 0.3.1 (2011-04-01) + ++++++++++++++++++ + + * Cookie Changes + * Response.read() + * Poster fix + + + 0.3.0 (2011-02-25) + ++++++++++++++++++ + + * Automatic Authentication API Change + * Smarter Query URL Parameterization + * Allow file uploads and POST data together + * New Authentication Manager System + - Simpler Basic HTTP System + - Supports all build-in urllib2 Auths + - Allows for custom Auth Handlers + + + 0.2.4 (2011-02-19) + ++++++++++++++++++ + + * Python 2.5 Support + * PyPy-c v1.4 Support + * Auto-Authentication tests + * Improved Request object constructor + + 0.2.3 (2011-02-15) + ++++++++++++++++++ + + * New HTTPHandling Methods + - Reponse.__nonzero__ (false if bad HTTP Status) + - Response.ok (True if expected HTTP Status) + - Response.error (Logged HTTPError if bad HTTP Status) + - Reponse.raise_for_status() (Raises stored HTTPError) + + + 0.2.2 (2011-02-14) + ++++++++++++++++++ + + * Still handles request in the event of an HTTPError. (Issue #2) + * Eventlet and Gevent Monkeypatch support. + * Cookie Support (Issue #1) + + + 0.2.1 (2011-02-14) + ++++++++++++++++++ + + * Added file attribute to POST and PUT requests for multipart-encode file uploads. + * Added Request.url attribute for context and redirects + + + 0.2.0 (2011-02-14) + ++++++++++++++++++ + + * Birth! + + + 0.0.1 (2011-02-13) + ++++++++++++++++++ + + * Frustration + * Conception + + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: ISC License (ISCL) +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..ce4eb58 --- /dev/null +++ b/README.rst @@ -0,0 +1,156 @@ +Requests: The Simple (e.g. usable) HTTP Module +============================================== + +Most existing Python modules for dealing HTTP requests are insane. I have to look up *everything* that I want to do. Most of my worst Python experiences are a result of the various built-in HTTP libraries (yes, even worse than Logging). + +But this one's different. This one's going to be awesome. And simple. + +Really simple. + +Features +-------- + +- Extremely simple GET, HEAD, POST, PUT, DELETE Requests + + Simple HTTP Header Request Attachment + + Simple Data/Params Request Attachment + + Simple Multipart File Uploads + + CookieJar Support + + Redirection History + + Redirection Recursion Urllib Fix + + Auto Decompression of GZipped Content + + Unicode URL Support + +- Simple Authentication + + Simple URL + HTTP Auth Registry + + +Usage +----- + +It couldn't be simpler. :: + + >>> import requests + >>> r = requests.get('http://google.com') + + +HTTPS? Basic Authentication? :: + + >>> r = requests.get('https://convore.com/api/account/verify.json') + >>> r.status_code + 401 + + +Uh oh, we're not authorized! Let's add authentication. :: + + >>> conv_auth = ('requeststest', 'requeststest') + >>> r = requests.get('https://convore.com/api/account/verify.json', auth=conv_auth) + + >>> r.status_code + 200 + + >>> r.headers['content-type'] + 'application/json' + + >>> r.content + '{"username": "requeststest", "url": "/users/requeststest/", "id": "9408", "img": "censored-long-url"}' + + + +API +--- + +**Requests:** + +All request functions return a Response object (see below). + +If a {filename: fileobject} dictionary is passed in (files=...), a multipart_encode upload will be performed. +If CookieJar object is is passed in (cookies=...), the cookies will be sent with the request. + + GET Requests + >>> requests.get(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + HEAD Requests + >>> requests.head(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + PUT Requests + >>> requests.put(url, data='', headers={}, files={}, cookies=None, auth=None) + <Response [200]> + + POST Requests + >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None) + <Response [200]> + + DELETE Requests + >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None) + <Response [200]> + + +**Responses:** + + Response.status_code + (Integer) Received HTTP Status Code Response + + Response.headers + ((CaseInsensitive) Dictionary) Received HTTP Response Headers. + + Response.content + (Bytes) Received Content. + + Response.history + (List of Responses) Redirection History. + + Response.url + (String) URL of response. Useful for detecting redirects. + + Response.ok + (Bool) True if no errors occurred during the request, and the status_code is kosher. + + Response.cached + (Bool) True if Response.content is stored within the object. + + Response.error + (HTTPError) If an HTTPError occurred (e.g. status of 404), Otherwise this is None. + + Response.raise_for_status() + Raises HTTPError if a request is not kosher. + + +**HTTP Authentication Registry:** + + You can register AuthObjects to automatically enable HTTP Authentication on requests that contain a registered base URL string. + + >>> requests.auth_manager.add_auth(url, authobject) + + + +Installation +------------ + +To install requests, simply: :: + + $ pip install requests + +Or, if you absolutely must: :: + + $ easy_install requests + +But, you really shouldn't do that. + + + +Contribute +---------- + +If you'd like to contribute, simply fork `the repository`_, commit your changes to the **develop** branch (or branch off of it), and send a pull request. Make sure you add yourself to AUTHORS_. + + + +Roadmap +------- + +- Sphinx Documentation + +.. _`the repository`: http://github.com/kennethreitz/requests +.. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS diff --git a/requests/__init__.py b/requests/__init__.py new file mode 100644 index 0000000..15a5050 --- /dev/null +++ b/requests/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from core import * +from core import __version__ diff --git a/requests/api.py b/requests/api.py new file mode 100644 index 0000000..0e27410 --- /dev/null +++ b/requests/api.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +""" +requests.api +~~~~~~~~~~~~ + +This module impliments the Requests API. + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +import requests +import config +from .models import Request, Response, AuthManager, AuthObject, auth_manager + + +__all__ = ('request', 'get', 'head', 'post', 'put', 'delete') + + + +def request(method, url, **kwargs): + """Constructs and sends a :class:`Request <models.Request>`. Returns :class:`Response <models.Response>` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of GET/HEAD/DELETE Parameters to send with the :class:`Request`. + :param data: (optional) Bytes/Dictionary of PUT/POST Data to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + """ + data = kwargs.pop('data', dict()) or kwargs.pop('params', dict()) + + r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', dict()), + cookiejar=kwargs.get('cookies', None), + files=kwargs.get('files', None), + auth=kwargs.get('auth', auth_manager.get_auth(url)), + timeout=kwargs.get('timeout', config.settings.timeout), + allow_redirects=kwargs.get('allow_redirects', None) + ) + + r.send() + + return r.response + + +def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs): + """Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + """ + + return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + + +def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs): + """Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + """ + + return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + + +def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs): + """Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary of POST data to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + """ + + return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + + +def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs): + """Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Bytes of PUT Data to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + """ + + return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + + +def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs): + """Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of DELETE Parameters to send with the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + """ + + return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) diff --git a/requests/async.py b/requests/async.py new file mode 100644 index 0000000..ab04084 --- /dev/null +++ b/requests/async.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +""" + requests.async + ~~~~~~~~~~~~~~ + + This module implements the main Requests system, after monkey-patching + the urllib2 module with eventlet or gevent.. + + :copyright: (c) 2011 by Kenneth Reitz. + :license: ISC, see LICENSE for more details. +""" + + +from __future__ import absolute_import + +import urllib +import urllib2 + +from urllib2 import HTTPError + + +try: + import eventlet + eventlet.monkey_patch() +except ImportError: + pass + +if not 'eventlet' in locals(): + try: + from gevent import monkey + monkey.patch_all() + except ImportError: + pass + + +if not 'eventlet' in locals(): + raise ImportError('No Async adaptations of urllib2 found!') + + +from .core import * diff --git a/requests/config.py b/requests/config.py new file mode 100644 index 0000000..63d3fa9 --- /dev/null +++ b/requests/config.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +""" +requests.config +~~~~~~~~~~~~~~~ + +This module provides the Requests settings feature set. + +""" + +class Settings(object): + _singleton = {} + + # attributes with defaults + __attrs__ = ('timeout',) + + def __init__(self, **kwargs): + super(Settings, self).__init__() + + self.__dict__ = self._singleton + + + def __call__(self, *args, **kwargs): + # new instance of class to call + r = self.__class__() + + # cache previous settings for __exit__ + r.__cache = self.__dict__.copy() + map(self.__cache.setdefault, self.__attrs__) + + # set new settings + self.__dict__.update(*args, **kwargs) + + return r + + + def __enter__(self): + pass + + + def __exit__(self, *args): + + # restore cached copy + self.__dict__.update(self.__cache.copy()) + del self.__cache + + + def __getattribute__(self, key): + if key in object.__getattribute__(self, '__attrs__'): + try: + return object.__getattribute__(self, key) + except AttributeError: + return None + return object.__getattribute__(self, key) + +settings = Settings()
\ No newline at end of file diff --git a/requests/core.py b/requests/core.py new file mode 100644 index 0000000..7f3d723 --- /dev/null +++ b/requests/core.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" +requests.core +~~~~~~~~~~~~~ + +This module implements the main Requests system. + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +__title__ = 'requests' +__version__ = '0.4.1' +__build__ = 0x000401 +__author__ = 'Kenneth Reitz' +__license__ = 'ISC' +__copyright__ = 'Copyright 2011 Kenneth Reitz' + + +from models import HTTPError, auth_manager +from api import * +from exceptions import * +from config import settings
\ No newline at end of file diff --git a/requests/exceptions.py b/requests/exceptions.py new file mode 100644 index 0000000..eff7512 --- /dev/null +++ b/requests/exceptions.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +""" +requests.exceptions +~~~~~~~~~~~~~~~ + +""" + +class RequestException(Exception): + """There was an ambiguous exception that occured while handling your + request.""" + +class AuthenticationError(RequestException): + """The authentication credentials provided were invalid.""" + +class Timeout(RequestException): + """The request timed out.""" + +class URLRequired(RequestException): + """A valid URL is required to make a request.""" + +class InvalidMethod(RequestException): + """An inappropriate method was attempted.""" diff --git a/requests/models.py b/requests/models.py new file mode 100644 index 0000000..2c3241d --- /dev/null +++ b/requests/models.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- + +""" +requests.models +~~~~~~~~~~~~~~~ + +""" + +import requests +import urllib +import urllib2 +import socket +import zlib + +from urllib2 import HTTPError +from urlparse import urlparse + +from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler +from .structures import CaseInsensitiveDict +from .packages.poster.encode import multipart_encode +from .packages.poster.streaminghttp import register_openers, get_handlers +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod + + + +class Request(object): + """The :class:`Request <models.Request>` object. It carries out all functionality of + Requests. Recommended interface is with the Requests functions. + """ + + _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE') + + def __init__(self, url=None, headers=dict(), files=None, method=None, + data=dict(), auth=None, cookiejar=None, timeout=None, + redirect=True, allow_redirects=False): + + socket.setdefaulttimeout(timeout) + + #: Request URL. + self.url = url + #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`. + self.headers = headers + #: Dictionary of files to multipart upload (``{filename: content}``). + self.files = files + #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. + self.method = method + #: Form or Byte data to attach to the :class:`Request <models.Request>`. + self.data = dict() + #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history + #: and HTTPError storage). + self.redirect = redirect + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) + self.allow_redirects = allow_redirects + + if hasattr(data, 'items'): + for (k, v) in data.items(): + self.data.update({ + k.encode('utf-8') if isinstance(k, unicode) else k: + v.encode('utf-8') if isinstance(v, unicode) else v + }) + self._enc_data = urllib.urlencode(self.data) + else: + self._enc_data = self.data = data + + #: :class:`Response <models.Response>` instance, containing + #: content and metadata of HTTP Response, once :attr:`sent <send>`. + self.response = Response() + + if isinstance(auth, (list, tuple)): + auth = AuthObject(*auth) + if not auth: + auth = auth_manager.get_auth(self.url) + #: :class:`AuthObject` to attach to :class:`Request <models.Request>`. + self.auth = auth + #: CookieJar to attach to :class:`Request <models.Request>`. + self.cookiejar = cookiejar + #: True if Request has been sent. + self.sent = False + + + def __repr__(self): + return '<Request [%s]>' % (self.method) + + + def __setattr__(self, name, value): + if (name == 'method') and (value): + if not value in self._METHODS: + raise InvalidMethod() + + object.__setattr__(self, name, value) + + + def _checks(self): + """Deterministic checks for consistency.""" + + if not self.url: + raise URLRequired + + + def _get_opener(self): + """Creates appropriate opener object for urllib2.""" + + _handlers = [] + + if self.cookiejar is not None: + _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) + + if self.auth: + if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) + self.auth.handler = self.auth.handler(auth_manager) + auth_manager.add_auth(self.url, self.auth) + + _handlers.append(self.auth.handler) + + + _handlers.append(HTTPRedirectHandler) + + if not _handlers: + return urllib2.urlopen + + if self.data or self.files: + _handlers.extend(get_handlers()) + + opener = urllib2.build_opener(*_handlers) + + if self.headers: + # Allow default headers in the opener to be overloaded + normal_keys = [k.capitalize() for k in self.headers] + for key, val in opener.addheaders[:]: + if key not in normal_keys: + continue + # Remove it, we have a value to take its place + opener.addheaders.remove((key, val)) + + return opener.open + + def _build_response(self, resp): + """Build internal :class:`Response <models.Response>` object from given response.""" + + def build(resp): + + response = Response() + response.status_code = getattr(resp, 'code', None) + + try: + response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) + response.content = resp.read() + except AttributeError: + pass + + if response.headers['content-encoding'] == 'gzip': + try: + response.content = zlib.decompress(response.content, 16+zlib.MAX_WBITS) + except zlib.error: + pass + + response.url = getattr(resp, 'url', None) + + return response + + + history = [] + + r = build(resp) + + if self.redirect: + + while ( + ('location' in r.headers) and + ((self.method in ('GET', 'HEAD')) or + (r.status_code is 303) or + (self.allow_redirects)) + ): + + history.append(r) + + url = r.headers['location'] + + # Facilitate for non-RFC2616-compliant 'location' headers + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if not urlparse(url).netloc: + parent_url_components = urlparse(self.url) + url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) + + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + if r.status_code is 303: + method = 'GET' + else: + method = self.method + + request = Request( + url, self.headers, self.files, method, + self.data, self.auth, self.cookiejar, redirect=False + ) + request.send() + r = request.response + + r.history = history + + self.response = r + + + @staticmethod + def _build_url(url, data=None): + """Build URLs.""" + + if urlparse(url).query: + return '%s&%s' % (url, data) + else: + if data: + return '%s?%s' % (url, data) + else: + return url + + + def send(self, anyway=False): + """Sends the request. Returns True of successful, false if not. + If there was an HTTPError during transmission, + self.response.status_code will contain the HTTPError code. + + Once a request is successfully sent, `sent` will equal True. + + :param anyway: If True, request will be sent, even if it has + already been sent. + """ + self._checks() + success = False + + if self.method in ('GET', 'HEAD', 'DELETE'): + req = _Request(self._build_url(self.url, self._enc_data), method=self.method) + else: + + if self.files: + register_openers() + + if self.data: + self.files.update(self.data) + + datagen, headers = multipart_encode(self.files) + req = _Request(self.url, data=datagen, headers=headers, method=self.method) + + else: + req = _Request(self.url, data=self._enc_data, method=self.method) + + if self.headers: + req.headers.update(self.headers) + + if not self.sent or anyway: + + try: + opener = self._get_opener() + resp = opener(req) + + if self.cookiejar is not None: + self.cookiejar.extract_cookies(resp, req) + + except urllib2.HTTPError, why: + self._build_response(why) + if not self.redirect: + self.response.error = why + except urllib2.URLError, error: + raise Timeout if isinstance(error.reason, socket.timeout) else error + else: + self._build_response(resp) + self.response.ok = True + + self.response.cached = False + else: + self.response.cached = True + + self.sent = self.response.ok + + return self.sent + + + def read(self, *args): + return self.response.read() + + + +class Response(object): + """The core :class:`Response <models.Response>` object. All + :class:`Request <models.Request>` objects contain a + :class:`response <models.Response>` attribute, which is an instance + of this class. + """ + + def __init__(self): + #: Raw content of the response, in bytes. + #: If ``content-encoding`` of response was set to ``gzip``, the + #: response data will be automatically deflated. + self.content = None + #: Integer Code of responded HTTP Status. + self.status_code = None + #: Case-insensitive Dictionary of Response Headers. + #: For example, ``headers['content-encoding']`` will return the + #: value of a ``'Content-Encoding'`` response header. + self.headers = CaseInsensitiveDict() + #: Final URL location of Response. + self.url = None + #: True if no :attr:`error` occured. + self.ok = False + #: Resulting :class:`HTTPError` of request, if one occured. + self.error = None + #: True, if the response :attr:`content` is cached locally. + self.cached = False + #: A list of :class:`Response <models.Response>` objects from + #: the history of the Request. Any redirect responses will end + #: up here. + self.history = [] + + + def __repr__(self): + return '<Response [%s]>' % (self.status_code) + + + def __nonzero__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return not self.error + + + def raise_for_status(self): + """Raises stored :class:`HTTPError`, if one occured.""" + if self.error: + raise self.error + + def read(self, *args): + """Returns :attr:`content`. Used for file-like object compatiblity.""" + + return self.content + + + +class AuthManager(object): + """Requests Authentication Manager.""" + + def __new__(cls): + singleton = cls.__dict__.get('__singleton__') + if singleton is not None: + return singleton + + cls.__singleton__ = singleton = object.__new__(cls) + + return singleton + + + def __init__(self): + self.passwd = {} + self._auth = {} + + + def __repr__(self): + return '<AuthManager [%s]>' % (self.method) + + + def add_auth(self, uri, auth): + """Registers AuthObject to AuthManager.""" + + uri = self.reduce_uri(uri, False) + + # try to make it an AuthObject + if not isinstance(auth, AuthObject): + try: + auth = AuthObject(*auth) + except TypeError: + pass + + self._auth[uri] = auth + + + def add_password(self, realm, uri, user, passwd): + """Adds password to AuthManager.""" + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) + + if reduced_uri not in self.passwd: + self.passwd[reduced_uri] = {} + self.passwd[reduced_uri] = (user, passwd) + + + def find_user_password(self, realm, authuri): + for uris, authinfo in self.passwd.iteritems(): + reduced_authuri = self.reduce_uri(authuri, False) + for uri in uris: + if self.is_suburi(uri, reduced_authuri): + return authinfo + + return (None, None) + + + def get_auth(self, uri): + (in_domain, in_path) = self.reduce_uri(uri, False) + + for domain, path, authority in ( + (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() + ): + if in_domain == domain: + if path in in_path: + return authority + + + def reduce_uri(self, uri, default_port=True): + """Accept authority or URI and extract only the authority and path.""" + # note HTTP URLs do not have a userinfo component + parts = urllib2.urlparse.urlsplit(uri) + if parts[1]: + # URI + scheme = parts[0] + authority = parts[1] + path = parts[2] or '/' + else: + # host or host:port + scheme = None + authority = uri + path = '/' + host, port = urllib2.splitport(authority) + if default_port and port is None and scheme is not None: + dport = {"http": 80, + "https": 443, + }.get(scheme) + if dport is not None: + authority = "%s:%d" % (host, dport) + + return authority, path + + + def is_suburi(self, base, test): + """Check if test is below base in a URI tree + + Both args must be URIs in reduced form. + """ + if base == test: + return True + if base[0] != test[0]: + return False + common = urllib2.posixpath.commonprefix((base[1], test[1])) + if len(common) == len(base[1]): + return True + return False + + + def empty(self): + self.passwd = {} + + + def remove(self, uri, realm=None): + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + for default_port in True, False: + reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) + del self.passwd[reduced_uri][realm] + + + def __contains__(self, uri): + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + uri = tuple([self.reduce_uri(u, False) for u in uri]) + + if uri in self.passwd: + return True + + return False + +auth_manager = AuthManager() + + + +class AuthObject(object): + """The :class:`AuthObject` is a simple HTTP Authentication token. When + given to a Requests function, it enables Basic HTTP Authentication for that + Request. You can also enable Authorization for domain realms with AutoAuth. + See AutoAuth for more details. + + :param username: Username to authenticate with. + :param password: Password for given username. + :param realm: (optional) the realm this auth applies to + :param handler: (optional) basic || digest || proxy_basic || proxy_digest + """ + + _handlers = { + 'basic': HTTPBasicAuthHandler, + 'digest': HTTPDigestAuthHandler, + 'proxy_basic': urllib2.ProxyBasicAuthHandler, + 'proxy_digest': urllib2.ProxyDigestAuthHandler + } + + def __init__(self, username, password, handler='basic', realm=None): + self.username = username + self.password = password + self.realm = realm + + if isinstance(handler, basestring): + self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler) + else: + self.handler = handler diff --git a/requests/monkeys.py b/requests/monkeys.py new file mode 100644 index 0000000..b8fe504 --- /dev/null +++ b/requests/monkeys.py @@ -0,0 +1,91 @@ +#-*- coding: utf-8 -*- + +""" +requests.monkeys +~~~~~~~~~~~~~~~~ + +Urllib2 Monkey patches. + +""" + +import urllib2 + + +class Request(urllib2.Request): + """Hidden wrapper around the urllib2.Request object. Allows for manual + setting of HTTP methods. + """ + + def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None): + urllib2.Request.__init__(self, url, data, headers, origin_req_host, unverifiable) + self.method = method + + def get_method(self): + if self.method: + return self.method + + return urllib2.Request.get_method(self) + + +class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): + + def http_error_301(self, req, fp, code, msg, headers): + pass + + http_error_302 = http_error_303 = http_error_307 = http_error_301 + + + +class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + + def __init__(self, *args, **kwargs): + urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) + self.retried_req = None + + def reset_retry_count(self): + # Python 2.6.5 will call this on 401 or 407 errors and thus loop + # forever. We disable reset_retry_count completely and reset in + # http_error_auth_reqed instead. + pass + + def http_error_auth_reqed(self, auth_header, host, req, headers): + # Reset the retry counter once for each request. + if req is not self.retried_req: + self.retried_req = req + self.retried = 0 + + return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed( + self, auth_header, host, req, headers + ) + + + +class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): + + def __init__(self, *args, **kwargs): + urllib2.HTTPDigestAuthHandler.__init__(self, *args, **kwargs) + self.retried_req = None + + def reset_retry_count(self): + # Python 2.6.5 will call this on 401 or 407 errors and thus loop + # forever. We disable reset_retry_count completely and reset in + # http_error_auth_reqed instead. + pass + + def http_error_auth_reqed(self, auth_header, host, req, headers): + # Reset the retry counter once for each request. + if req is not self.retried_req: + self.retried_req = req + self.retried = 0 + # In python < 2.5 AbstractDigestAuthHandler raises a ValueError if + # it doesn't know about the auth type requested. This can happen if + # somebody is using BasicAuth and types a bad password. + + try: + return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed( + self, auth_header, host, req, headers) + except ValueError, inst: + arg = inst.args[0] + if arg.startswith("AbstractDigestAuthHandler doesn't know "): + return + raise
\ No newline at end of file diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py new file mode 100644 index 0000000..ab2669e --- /dev/null +++ b/requests/packages/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from . import poster diff --git a/requests/packages/poster/__init__.py b/requests/packages/poster/__init__.py new file mode 100644 index 0000000..6e216fc --- /dev/null +++ b/requests/packages/poster/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) 2010 Chris AtLee +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +"""poster module + +Support for streaming HTTP uploads, and multipart/form-data encoding + +```poster.version``` is a 3-tuple of integers representing the version number. +New releases of poster will always have a version number that compares greater +than an older version of poster. +New in version 0.6.""" + +from __future__ import absolute_import + +from . import streaminghttp +from . import encode + +version = (0, 8, 0) # Thanks JP! diff --git a/requests/packages/poster/encode.py b/requests/packages/poster/encode.py new file mode 100644 index 0000000..cf2298d --- /dev/null +++ b/requests/packages/poster/encode.py @@ -0,0 +1,414 @@ +"""multipart/form-data encoding module + +This module provides functions that faciliate encoding name/value pairs +as multipart/form-data suitable for a HTTP POST or PUT request. + +multipart/form-data is the standard way to upload files over HTTP""" + +__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam', + 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers', + 'multipart_encode'] + +try: + import uuid + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + return uuid.uuid4().hex +except ImportError: + import random, sha + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + bits = random.getrandbits(160) + return sha.new(str(bits)).hexdigest() + +import urllib, re, os, mimetypes +try: + from email.header import Header +except ImportError: + # Python 2.4 + from email.Header import Header + +def encode_and_quote(data): + """If ``data`` is unicode, return urllib.quote_plus(data.encode("utf-8")) + otherwise return urllib.quote_plus(data)""" + if data is None: + return None + + if isinstance(data, unicode): + data = data.encode("utf-8") + return urllib.quote_plus(data) + +def _strify(s): + """If s is a unicode string, encode it to UTF-8 and return the results, + otherwise return str(s), or None if s is None""" + if s is None: + return None + if isinstance(s, unicode): + return s.encode("utf-8") + return str(s) + +class MultipartParam(object): + """Represents a single parameter in a multipart/form-data request + + ``name`` is the name of this parameter. + + If ``value`` is set, it must be a string or unicode object to use as the + data for this parameter. + + If ``filename`` is set, it is what to say that this parameter's filename + is. Note that this does not have to be the actual filename any local file. + + If ``filetype`` is set, it is used as the Content-Type for this parameter. + If unset it defaults to "text/plain; charset=utf8" + + If ``filesize`` is set, it specifies the length of the file ``fileobj`` + + If ``fileobj`` is set, it must be a file-like object that supports + .read(). + + Both ``value`` and ``fileobj`` must not be set, doing so will + raise a ValueError assertion. + + If ``fileobj`` is set, and ``filesize`` is not specified, then + the file's size will be determined first by stat'ing ``fileobj``'s + file descriptor, and if that fails, by seeking to the end of the file, + recording the current position as the size, and then by seeking back to the + beginning of the file. + + ``cb`` is a callable which will be called from iter_encode with (self, + current, total), representing the current parameter, current amount + transferred, and the total size. + """ + def __init__(self, name, value=None, filename=None, filetype=None, + filesize=None, fileobj=None, cb=None): + self.name = Header(name).encode() + self.value = _strify(value) + if filename is None: + self.filename = None + else: + if isinstance(filename, unicode): + # Encode with XML entities + self.filename = filename.encode("ascii", "xmlcharrefreplace") + else: + self.filename = str(filename) + self.filename = self.filename.encode("string_escape").\ + replace('"', '\\"') + self.filetype = _strify(filetype) + + self.filesize = filesize + self.fileobj = fileobj + self.cb = cb + + if self.value is not None and self.fileobj is not None: + raise ValueError("Only one of value or fileobj may be specified") + + if fileobj is not None and filesize is None: + # Try and determine the file size + try: + self.filesize = os.fstat(fileobj.fileno()).st_size + except (OSError, AttributeError): + try: + fileobj.seek(0, 2) + self.filesize = fileobj.tell() + fileobj.seek(0) + except: + raise ValueError("Could not determine filesize") + + def __cmp__(self, other): + attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj'] + myattrs = [getattr(self, a) for a in attrs] + oattrs = [getattr(other, a) for a in attrs] + return cmp(myattrs, oattrs) + + def reset(self): + if self.fileobj is not None: + self.fileobj.seek(0) + elif self.value is None: + raise ValueError("Don't know how to reset this parameter") + + @classmethod + def from_file(cls, paramname, filename): + """Returns a new MultipartParam object constructed from the local + file at ``filename``. + + ``filesize`` is determined by os.path.getsize(``filename``) + + ``filetype`` is determined by mimetypes.guess_type(``filename``)[0] + + ``filename`` is set to os.path.basename(``filename``) + """ + + return cls(paramname, filename=os.path.basename(filename), + filetype=mimetypes.guess_type(filename)[0], + filesize=os.path.getsize(filename), + fileobj=open(filename, "rb")) + + @classmethod + def from_params(cls, params): + """Returns a list of MultipartParam objects from a sequence of + name, value pairs, MultipartParam instances, + or from a mapping of names to values + + The values may be strings or file objects, or MultipartParam objects. + MultipartParam object names must match the given names in the + name,value pairs or mapping, if applicable.""" + if hasattr(params, 'items'): + params = params.items() + + retval = [] + for item in params: + if isinstance(item, cls): + retval.append(item) + continue + name, value = item + if isinstance(value, cls): + assert value.name == name + retval.append(value) + continue + if hasattr(value, 'read'): + # Looks like a file object + filename = getattr(value, 'name', None) + if filename is not None: + filetype = mimetypes.guess_type(filename)[0] + else: + filetype = None + + retval.append(cls(name=name, filename=filename, + filetype=filetype, fileobj=value)) + else: + retval.append(cls(name, value)) + return retval + + def encode_hdr(self, boundary): + """Returns the header of the encoding of this parameter""" + boundary = encode_and_quote(boundary) + + headers = ["--%s" % boundary] + + if self.filename: + disposition = 'form-data; name="%s"; filename="%s"' % (self.name, + self.filename) + else: + disposition = 'form-data; name="%s"' % self.name + + headers.append("Content-Disposition: %s" % disposition) + + if self.filetype: + filetype = self.filetype + else: + filetype = "text/plain; charset=utf-8" + + headers.append("Content-Type: %s" % filetype) + + headers.append("") + headers.append("") + + return "\r\n".join(headers) + + def encode(self, boundary): + """Returns the string encoding of this parameter""" + if self.value is None: + value = self.fileobj.read() + else: + value = self.value + + if re.search("^--%s$" % re.escape(boundary), value, re.M): + raise ValueError("boundary found in encoded string") + + return "%s%s\r\n" % (self.encode_hdr(boundary), value) + + def iter_encode(self, boundary, blocksize=4096): + """Yields the encoding of this parameter + If self.fileobj is set, then blocks of ``blocksize`` bytes are read and + yielded.""" + total = self.get_size(boundary) + current = 0 + if self.value is not None: + block = self.encode(boundary) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + else: + block = self.encode_hdr(boundary) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + last_block = "" + encoded_boundary = "--%s" % encode_and_quote(boundary) + boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary), + re.M) + while True: + block = self.fileobj.read(blocksize) + if not block: + current += 2 + yield "\r\n" + if self.cb: + self.cb(self, current, total) + break + last_block += block + if boundary_exp.search(last_block): + raise ValueError("boundary found in file data") + last_block = last_block[-len(encoded_boundary)-2:] + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + + def get_size(self, boundary): + """Returns the size in bytes that this param will be when encoded + with the given boundary.""" + if self.filesize is not None: + valuesize = self.filesize + else: + valuesize = len(self.value) + + return len(self.encode_hdr(boundary)) + 2 + valuesize + +def encode_string(boundary, name, value): + """Returns ``name`` and ``value`` encoded as a multipart/form-data + variable. ``boundary`` is the boundary string used throughout + a single request to separate variables.""" + + return MultipartParam(name, value).encode(boundary) + +def encode_file_header(boundary, paramname, filesize, filename=None, + filetype=None): + """Returns the leading data for a multipart/form-data field that contains + file data. + + ``boundary`` is the boundary string used throughout a single request to + separate variables. + + ``paramname`` is the name of the variable in this request. + + ``filesize`` is the size of the file data. + + ``filename`` if specified is the filename to give to this field. This + field is only useful to the server for determining the original filename. + + ``filetype`` if specified is the MIME type of this file. + + The actual file data should be sent after this header has been sent. + """ + + return MultipartParam(paramname, filesize=filesize, filename=filename, + filetype=filetype).encode_hdr(boundary) + +def get_body_size(params, boundary): + """Returns the number of bytes that the multipart/form-data encoding + of ``params`` will be.""" + size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params)) + return size + len(boundary) + 6 + +def get_headers(params, boundary): + """Returns a dictionary with Content-Type and Content-Length headers + for the multipart/form-data encoding of ``params``.""" + headers = {} + boundary = urllib.quote_plus(boundary) + headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary + headers['Content-Length'] = str(get_body_size(params, boundary)) + return headers + +class multipart_yielder: + def __init__(self, params, boundary, cb): + self.params = params + self.boundary = boundary + self.cb = cb + + self.i = 0 + self.p = None + self.param_iter = None + self.current = 0 + self.total = get_body_size(params, boundary) + + def __iter__(self): + return self + + def next(self): + """generator function to yield multipart/form-data representation + of parameters""" + if self.param_iter is not None: + try: + block = self.param_iter.next() + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + except StopIteration: + self.p = None + self.param_iter = None + + if self.i is None: + raise StopIteration + elif self.i >= len(self.params): + self.param_iter = None + self.p = None + self.i = None + block = "--%s--\r\n" % self.boundary + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + + self.p = self.params[self.i] + self.param_iter = self.p.iter_encode(self.boundary) + self.i += 1 + return self.next() + + def reset(self): + self.i = 0 + self.current = 0 + for param in self.params: + param.reset() + +def multipart_encode(params, boundary=None, cb=None): + """Encode ``params`` as multipart/form-data. + + ``params`` should be a sequence of (name, value) pairs or MultipartParam + objects, or a mapping of names to values. + Values are either strings parameter values, or file-like objects to use as + the parameter value. The file-like objects must support .read() and either + .fileno() or both .seek() and .tell(). + + If ``boundary`` is set, then it as used as the MIME boundary. Otherwise + a randomly generated boundary will be used. In either case, if the + boundary string appears in the parameter values a ValueError will be + raised. + + If ``cb`` is set, it should be a callback which will get called as blocks + of data are encoded. It will be called with (param, current, total), + indicating the current parameter being encoded, the current amount encoded, + and the total amount to encode. + + Returns a tuple of `datagen`, `headers`, where `datagen` is a + generator that will yield blocks of data that make up the encoded + parameters, and `headers` is a dictionary with the assoicated + Content-Type and Content-Length headers. + + Examples: + + >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> p = MultipartParam("key", "value2") + >>> datagen, headers = multipart_encode( [("key", "value1"), p] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> datagen, headers = multipart_encode( {"key": "value1"} ) + >>> s = "".join(datagen) + >>> assert "value2" not in s and "value1" in s + + """ + if boundary is None: + boundary = gen_boundary() + else: + boundary = urllib.quote_plus(boundary) + + headers = get_headers(params, boundary) + params = MultipartParam.from_params(params) + + return multipart_yielder(params, boundary, cb), headers diff --git a/requests/packages/poster/streaminghttp.py b/requests/packages/poster/streaminghttp.py new file mode 100644 index 0000000..1b591d4 --- /dev/null +++ b/requests/packages/poster/streaminghttp.py @@ -0,0 +1,199 @@ +"""Streaming HTTP uploads module. + +This module extends the standard httplib and urllib2 objects so that +iterable objects can be used in the body of HTTP requests. + +In most cases all one should have to do is call :func:`register_openers()` +to register the new streaming http handlers which will take priority over +the default handlers, and then you can use iterable objects in the body +of HTTP requests. + +**N.B.** You must specify a Content-Length header if using an iterable object +since there is no way to determine in advance the total size that will be +yielded, and there is no way to reset an interator. + +Example usage: + +>>> from StringIO import StringIO +>>> import urllib2, poster.streaminghttp + +>>> opener = poster.streaminghttp.register_openers() + +>>> s = "Test file data" +>>> f = StringIO(s) + +>>> req = urllib2.Request("http://localhost:5000", f, +... {'Content-Length': str(len(s))}) +""" + +import httplib, urllib2, socket +from httplib import NotConnected + +__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', + 'StreamingHTTPHandler', 'register_openers'] + +if hasattr(httplib, 'HTTPS'): + __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection']) + +class _StreamingHTTPMixin: + """Mixin class for HTTP and HTTPS connections that implements a streaming + send method.""" + def send(self, value): + """Send ``value`` to the server. + + ``value`` can be a string object, a file-like object that supports + a .read() method, or an iterable object that supports a .next() + method. + """ + # Based on python 2.6's httplib.HTTPConnection.send() + if self.sock is None: + if self.auto_open: + self.connect() + else: + raise NotConnected() + + # send the data to the server. if we get a broken pipe, then close + # the socket. we want to reconnect when somebody tries to send again. + # + # NOTE: we DO propagate the error, though, because we cannot simply + # ignore the error... the caller will know if they can retry. + if self.debuglevel > 0: + print "send:", repr(value) + try: + blocksize = 8192 + if hasattr(value, 'read') : + if hasattr(value, 'seek'): + value.seek(0) + if self.debuglevel > 0: + print "sendIng a read()able" + data = value.read(blocksize) + while data: + self.sock.sendall(data) + data = value.read(blocksize) + elif hasattr(value, 'next'): + if hasattr(value, 'reset'): + value.reset() + if self.debuglevel > 0: + print "sendIng an iterable" + for data in value: + self.sock.sendall(data) + else: + self.sock.sendall(value) + except socket.error, v: + if v[0] == 32: # Broken pipe + self.close() + raise + +class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): + """Subclass of `httplib.HTTPConnection` that overrides the `send()` method + to support iterable body objects""" + +class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """Subclass of `urllib2.HTTPRedirectHandler` that overrides the + `redirect_request` method to properly handle redirected POST requests + + This class is required because python 2.5's HTTPRedirectHandler does + not remove the Content-Type or Content-Length headers when requesting + the new resource, but the body of the original request is not preserved. + """ + + handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 + + # From python2.6 urllib2's HTTPRedirectHandler + def redirect_request(self, req, fp, code, msg, headers, newurl): + """Return a Request or None in response to a redirect. + + This is called by the http_error_30x methods when a + redirection response is received. If a redirection should + take place, return a new Request to allow http_error_30x to + perform the redirect. Otherwise, raise HTTPError if no-one + else should try to handle this url. Return None if you can't + but another Handler might. + """ + m = req.get_method() + if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + or code in (301, 302, 303) and m == "POST"): + # Strictly (according to RFC 2616), 301 or 302 in response + # to a POST MUST NOT cause a redirection without confirmation + # from the user (of urllib2, in this case). In practice, + # essentially all clients do redirect in this case, so we + # do the same. + # be conciliant with URIs containing a space + newurl = newurl.replace(' ', '%20') + newheaders = dict((k, v) for k, v in req.headers.items() + if k.lower() not in ( + "content-length", "content-type") + ) + return urllib2.Request(newurl, + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True) + else: + raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + +class StreamingHTTPHandler(urllib2.HTTPHandler): + """Subclass of `urllib2.HTTPHandler` that uses + StreamingHTTPConnection as its http connection class.""" + + handler_order = urllib2.HTTPHandler.handler_order - 1 + + def http_open(self, req): + """Open a StreamingHTTPConnection for the given request""" + return self.do_open(StreamingHTTPConnection, req) + + def http_request(self, req): + """Handle a HTTP request. Make sure that Content-Length is specified + if we're using an interable value""" + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPHandler.do_request_(self, req) + +if hasattr(httplib, 'HTTPS'): + class StreamingHTTPSConnection(_StreamingHTTPMixin, + httplib.HTTPSConnection): + """Subclass of `httplib.HTTSConnection` that overrides the `send()` + method to support iterable body objects""" + + class StreamingHTTPSHandler(urllib2.HTTPSHandler): + """Subclass of `urllib2.HTTPSHandler` that uses + StreamingHTTPSConnection as its http connection class.""" + + handler_order = urllib2.HTTPSHandler.handler_order - 1 + + def https_open(self, req): + return self.do_open(StreamingHTTPSConnection, req) + + def https_request(self, req): + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPSHandler.do_request_(self, req) + + +def get_handlers(): + handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler] + if hasattr(httplib, "HTTPS"): + handlers.append(StreamingHTTPSHandler) + return handlers + +def register_openers(): + """Register the streaming http handlers in the global urllib2 default + opener object. + + Returns the created OpenerDirector object.""" + opener = urllib2.build_opener(*get_handlers()) + + urllib2.install_opener(opener) + + return opener diff --git a/requests/patches.py b/requests/patches.py new file mode 100644 index 0000000..43a3b4c --- /dev/null +++ b/requests/patches.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +""" +requests.monkeys +""" diff --git a/requests/structures.py b/requests/structures.py new file mode 100644 index 0000000..0c82c7b --- /dev/null +++ b/requests/structures.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +""" +requests.structures +~~~~~~~~~~~~~~~~~~~ + +Datastructures that power Requests. + +""" + +from UserDict import DictMixin + + +class CaseInsensitiveDict(DictMixin): + """Case-insensitive Dictionary for :class:`Response <models.Response>` Headers. + + For example, ``headers['content-encoding']`` will return the + value of a ``'Content-Encoding'`` response header.""" + + def __init__(self, *args, **kwargs): + # super(CaseInsensitiveDict, self).__init__() + self.data = dict(*args, **kwargs) + + def __repr__(self): + return self.data.__repr__() + + def __getstate__(self): + return self.data.copy() + + def __setstate__(self, d): + self.data = d + + def _lower_keys(self): + return map(str.lower, self.data.keys()) + + + def __contains__(self, key): + return key.lower() in self._lower_keys() + + + def __getitem__(self, key): + + if key.lower() in self: + return self.items()[self._lower_keys().index(key.lower())][1] + + + def __setitem__(self, key, value): + return self.data.__setitem__(key, value) + + + def __delitem__(self, key): + return self.data.__delitem__(key) + + + def __keys__(self): + return self.data.__keys__() + + + def __iter__(self): + return self.data.__iter__() + + + def iteritems(self): + return self.data.iteritems() diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..f0cebcd --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +import requests + +from distutils.core import setup + + + +if sys.argv[-1] == "publish": + os.system("python setup.py sdist upload") + sys.exit() + +if sys.argv[-1] == "test": + os.system("python test_requests.py") + sys.exit() + +required = [] + +if sys.version_info[:2] < (2,6): + required.append('simplejson') + +setup( + name='requests', + version=requests.__version__, + description='Awesome Python HTTP Library that\'s actually usable.', + long_description=open('README.rst').read() + '\n\n' + + open('HISTORY.rst').read(), + author='Kenneth Reitz', + author_email='me@kennethreitz.com', + url='http://python-requests.org', + packages= [ + 'requests', + 'requests.packages', + 'requests.packages.poster' + ], + install_requires=required, + license='ISC', + classifiers=( + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'License :: OSI Approved :: ISC License (ISCL)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + # 'Programming Language :: Python :: 3.0', + # 'Programming Language :: Python :: 3.1', + ), +) |