Add XML-RPC interface and command line client

Introduce a new XML-RPC Patchwork interface inspired by the SOAP
interface from the old Patchwork.  The interface itself is fairly
lightweight and generic, and provides read-only access to a limited
subset of the Patchwork database, along with server-side search
and flexible filtering capabilities.

The command line client is modeled after the old one with some
additional filtering options.

The XML-RPC interface is disabled by default.  You can enable it
by setting ENABLE_XMLRPC = True in local_settings.py

This feature uses the django-xmlrpc package available from
http://django-xmlrpc.googlecode.com.

Signed-off-by: Nate Case <ncase@xes-inc.com>
Signed-off-by: Jeremy Kerr <jk@ozlabs.org>

5 files changed, 625 insertions, 0 deletions

diff --git a/apps/patchwork/bin/pwclient.py b/apps/patchwork/bin/pwclient.py
new file mode 100755
index 0000000..765e66b
--- /dev/null
+++ b/apps/patchwork/bin/pwclient.py
@@ -0,0 +1,330 @@
+#!/usr/bin/env python
+#
+# Patchwork command line client
+# Copyright (C) 2008 Nate Case <ncase@xes-inc.com>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import sys
+import xmlrpclib
+import getopt
+import string
+import tempfile
+import subprocess
+
+# Default Patchwork remote XML-RPC server URL
+# This script will check the PW_XMLRPC_URL environment variable
+# for the URL to access.  If that is unspecified, it will fallback to
+# the hardcoded default value specified here.
+DEFAULT_URL = "http://patchwork:80/xmlrpc/"
+
+PW_XMLRPC_URL = os.getenv("PW_XMLRPC_URL")
+if not PW_XMLRPC_URL:
+    PW_XMLRPC_URL = DEFAULT_URL
+
+class Filter:
+    """Filter for selecting patches."""
+    def __init__(self):
+        # These fields refer to specific objects, so they are special
+        # because we have to resolve them to IDs before passing the
+        # filter to the server
+        self.state = ""
+        self.project = ""
+
+        # The dictionary that gets passed to via XML-RPC
+        self.d = {}
+
+    def add(self, field, value):
+        if field == 'state':
+            self.state = value
+        elif field == 'project':
+            self.project = value
+        else:
+            # OK to add directly
+            self.d[field] = value
+
+    def resolve_ids(self, rpc):
+        """Resolve State, Project, and Person IDs based on filter strings."""
+        if self.state != "":
+            id = state_id_by_name(rpc, self.state)
+            if id == 0:
+                sys.stderr.write("Note: No State found matching %s*, " \
+                                 "ignoring filter\n" % self.state)
+            else:
+                self.d['state_id'] = id
+
+        if self.project != "":
+            id = project_id_by_name(rpc, self.project)
+            if id == 0:
+                sys.stderr.write("Note: No Project found matching %s, " \
+                                 "ignoring filter\n" % self.project)
+            else:
+                self.d['project_id'] = id
+
+    def __str__(self):
+        """Return human-readable description of the filter."""
+        return str(self.d)
+
+def usage():
+    sys.stderr.write("Usage: %s <action> [options]\n\n" % \
+                        (os.path.basename(sys.argv[0])))
+    sys.stderr.write("Where <action> is one of:\n")
+    sys.stderr.write(
+"""        apply <ID>    : Apply a patch (in the current dir, using -p1)
+        get <ID>      : Download a patch and save it locally
+        projects      : List all projects
+        states        : Show list of potential patch states
+        list [str]    : List patches, using the optional filters specified
+                        below and an optional substring to search for patches
+                        by name
+        search [str]  : Same as 'list'
+        view <ID>     : View a patch\n""")
+    sys.stderr.write("""\nFilter options for 'list' and 'search':
+        -s <state>    : Filter by patch state (e.g., 'New', 'Accepted', etc.)
+        -p <project>  : Filter by project name (see 'projects' for list)
+        -w <who>      : Filter by submitter (name, e-mail substring search)
+        -d <who>      : Filter by delegate (name, e-mail substring search)
+        -n <max #>    : Restrict number of results\n""")
+    sys.exit(1)
+
+def project_id_by_name(rpc, linkname):
+    """Given a project short name, look up the Project ID."""
+    if len(linkname) == 0:
+        return 0
+    # The search requires - instead of _
+    search = linkname.replace("_", "-")
+    projects = rpc.project_list(search, 0)
+    for project in projects:
+        if project['linkname'].replace("_", "-") == search:
+            return project['id']
+    return 0
+
+def state_id_by_name(rpc, name):
+    """Given a partial state name, look up the state ID."""
+    if len(name) == 0:
+        return 0
+    states = rpc.state_list(name, 0)
+    for state in states:
+        if state['name'].lower().startswith(name.lower()):
+            return state['id']
+    return 0
+
+def person_ids_by_name(rpc, name):
+    """Given a partial name or email address, return a list of the
+    person IDs that match."""
+    if len(name) == 0:
+        return []
+    people = rpc.person_list(name, 0)
+    return map(lambda x: x['id'], people)
+
+def list_patches(patches):
+    """Dump a list of patches to stdout."""
+    print("%-5s %-12s %s" % ("ID", "State", "Name"))
+    print("%-5s %-12s %s" % ("--", "-----", "----"))
+    for patch in patches:
+        print("%-5d %-12s %s" % (patch['id'], patch['state'], patch['name']))
+
+def action_list(rpc, filter, submitter_str, delegate_str):
+    filter.resolve_ids(rpc)
+
+    if submitter_str != "":
+        ids = person_ids_by_name(rpc, submitter_str)
+        if len(ids) == 0:
+            sys.stderr.write("Note: Nobody found matching *%s*\n", \
+                             submitter_str)
+        else:
+            for id in ids:
+                person = rpc.person_get(id)
+                print "Patches submitted by %s <%s>:" % \
+                        (person['name'], person['email'])
+                f = filter
+                f.add("submitter_id", id)
+                patches = rpc.patch_list(f.d)
+                list_patches(patches)
+        return
+
+    if delegate_str != "":
+        ids = person_ids_by_name(rpc, delegate_str)
+        if len(ids) == 0:
+            sys.stderr.write("Note: Nobody found matching *%s*\n", \
+                             delegate_str)
+        else:
+            for id in ids:
+                person = rpc.person_get(id)
+                print "Patches delegated to %s <%s>:" % \
+                        (person['name'], person['email'])
+                f = filter
+                f.add("delegate_id", id)
+                patches = rpc.patch_list(f.d)
+                list_patches(patches)
+        return
+
+    patches = rpc.patch_list(filter.d)
+    list_patches(patches)
+
+def action_projects(rpc):
+    projects = rpc.project_list("", 0)
+    print("%-5s %-24s %s" % ("ID", "Name", "Description"))
+    print("%-5s %-24s %s" % ("--", "----", "-----------"))
+    for project in projects:
+        print("%-5d %-24s %s" % (project['id'], \
+                project['linkname'].replace("_", "-"), \
+                project['name']))
+
+def action_states(rpc):
+    states = rpc.state_list("", 0)
+    print("%-5s %s" % ("ID", "Name"))
+    print("%-5s %s" % ("--", "----"))
+    for state in states:
+        print("%-5d %s" % (state['id'], state['name']))
+
+def action_get(rpc, patch_id):
+    patch = rpc.patch_get(patch_id)
+    s = rpc.patch_get_mbox(patch_id)
+
+    if patch == {} or len(s) == 0:
+        sys.stderr.write("Unable to get patch %d\n" % patch_id)
+        sys.exit(1)
+
+    base_fname = fname = os.path.basename(patch['filename'])
+    i = 0
+    while os.path.exists(fname):
+        fname = "%s.%d" % (base_fname, i)
+        i += 1
+
+    try:
+        f = open(fname, "w")
+    except:
+        sys.stderr.write("Unable to open %s for writing\n" % fname)
+        sys.exit(1)
+
+    try:
+        f.write(s)
+        f.close()
+        print "Saved patch to %s" % fname
+    except:
+        sys.stderr.write("Failed to write to %s\n" % fname)
+        sys.exit(1)
+
+def action_apply(rpc, patch_id):
+    patch = rpc.patch_get(patch_id)
+    if patch == {}:
+        sys.stderr.write("Error getting information on patch ID %d\n" % \
+                         patch_id)
+        sys.exit(1)
+    print "Applying patch #%d to current directory" % patch_id
+    print "Description: %s" % patch['name']
+    s = rpc.patch_get_mbox(patch_id)
+    if len(s) > 0:
+        proc = subprocess.Popen(['patch', '-p1'], stdin = subprocess.PIPE)
+        proc.communicate(s)
+    else:
+        sys.stderr.write("Error: No patch content found\n")
+        sys.exit(1)
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[2:], 's:p:w:d:n:')
+    except getopt.GetoptError, err:
+        print str(err)
+        usage()
+
+    if len(sys.argv) < 2:
+        usage()
+
+    action = sys.argv[1].lower()
+
+    filt = Filter()
+    submitter_str = ""
+    delegate_str = ""
+
+    for name, value in opts:
+        if name == '-s':
+            filt.add("state", value)
+        elif name == '-p':
+            filt.add("project", value)
+        elif name == '-w':
+            submitter_str = value
+        elif name == '-d':
+            delegate_str = value
+        elif name == '-n':
+            try:
+                filt.add("max_count", int(value))
+            except:
+                sys.stderr.write("Invalid maximum count '%s'\n" % value)
+                usage()
+        else:
+            sys.stderr.write("Unknown option '%s'\n" % name)
+            usage()
+
+    if len(args) > 1:
+        sys.stderr.write("Too many arguments specified\n")
+        usage()
+
+    try:
+        rpc = xmlrpclib.Server(PW_XMLRPC_URL)
+    except:
+        sys.stderr.write("Unable to connect to %s\n" % PW_XMLRPC_URL)
+        sys.exit(1)
+
+    if action == 'list' or action == 'search':
+        if len(args) > 0:
+            filt.add("name__icontains", args[0])
+        action_list(rpc, filt, submitter_str, delegate_str)
+
+    elif action.startswith('project'):
+        action_projects(rpc)
+
+    elif action.startswith('state'):
+        action_states(rpc)
+
+    elif action == 'view':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        s = rpc.patch_get_mbox(patch_id)
+        if len(s) > 0:
+            print s
+
+    elif action == 'get' or action == 'save':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        action_get(rpc, patch_id)
+
+    elif action == 'apply':
+        try:
+            patch_id = int(args[0])
+        except:
+            sys.stderr.write("Invalid patch ID given\n")
+            sys.exit(1)
+
+        action_apply(rpc, patch_id)
+
+    else:
+        sys.stderr.write("Unknown action '%s'\n" % action)
+        usage()
+
+if __name__ == "__main__":
+    main()
diff --git a/apps/patchwork/xmlrpc.py b/apps/patchwork/xmlrpc.py
new file mode 100644
index 0000000..a661611
--- /dev/null
+++ b/apps/patchwork/xmlrpc.py
@@ -0,0 +1,258 @@
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Nate Case <ncase@xes-inc.com>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# The XML-RPC interface provides a watered down, read-only interface to
+# the Patchwork database.  It's intended to be safe to export to the public
+# Internet.  A small subset of the object data is included, and the type
+# of requests/queries you can do is limited by the methods
+# that we export.
+
+from patchwork.models import Patch, Project, Person, Bundle, State
+
+# We allow most of the Django field lookup types for remote queries
+LOOKUP_TYPES = ["iexact", "contains", "icontains", "gt", "gte", "lt",
+                "in", "startswith", "istartswith", "endswith",
+                "iendswith", "range", "year", "month", "day", "isnull" ]
+
+#######################################################################
+# Helper functions
+#######################################################################
+
+def project_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Project
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'linkname'     : obj.linkname,
+         'name'         : obj.name,
+        }
+
+def person_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Person
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'email'        : obj.email,
+         'name'         : obj.name,
+         'user'         : str(obj.user),
+        }
+
+def patch_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Patch
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'date'         : str(obj.date),
+         'filename'     : obj.filename(),
+         'msgid'        : obj.msgid,
+         'name'         : obj.name,
+         'project'      : str(obj.project),
+         'project_id'   : obj.project_id,
+         'state'        : str(obj.state),
+         'state_id'     : obj.state_id,
+         'submitter'    : str(obj.submitter),
+         'submitter_id' : obj.submitter_id,
+         'delegate'     : str(obj.delegate),
+         'delegate_id'  : max(obj.delegate_id, 0),
+         'commit_ref'   : max(obj.commit_ref, ''),
+        }
+
+def bundle_to_dict(obj):
+    """Return a trimmed down dictionary representation of a Bundle
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'name'         : obj.name,
+         'n_patches'    : obj.n_patches(),
+         'public_url'   : obj.public_url(),
+        }
+
+def state_to_dict(obj):
+    """Return a trimmed down dictionary representation of a State
+    object which is OK to send to the client."""
+    return \
+        {
+         'id'           : obj.id,
+         'name'         : obj.name,
+        }
+
+#######################################################################
+# Public XML-RPC methods
+#######################################################################
+
+def pw_rpc_version():
+    """Return Patchwork XML-RPC interface version."""
+    return 1
+
+def project_list(search_str="", max_count=0):
+    """Get a list of projects matching the given filters."""
+    try:
+        if len(search_str) > 0:
+            projects = Project.objects.filter(name__icontains = search_str)
+        else:
+            projects = Project.objects.all()
+
+        if max_count > 0:
+            return map(project_to_dict, projects)[:max_count]
+        else:
+            return map(project_to_dict, projects)
+    except:
+        return []
+
+def project_get(project_id):
+    """Return structure for the given project ID."""
+    try:
+        project = Project.objects.filter(id = project_id)[0]
+        return project_to_dict(project)
+    except:
+        return {}
+
+def person_list(search_str="", max_count=0):
+    """Get a list of Person objects matching the given filters."""
+    try:
+        if len(search_str) > 0:
+            people = (Person.objects.filter(name__icontains = search_str) |
+                Person.objects.filter(email__icontains = search_str))
+        else:
+            people = Person.objects.all()
+
+        if max_count > 0:
+            return map(person_to_dict, people)[:max_count]
+        else:
+            return map(person_to_dict, people)
+
+    except:
+        return []
+
+def person_get(person_id):
+    """Return structure for the given person ID."""
+    try:
+        person = Person.objects.filter(id = person_id)[0]
+        return person_to_dict(person)
+    except:
+        return {}
+
+def patch_list(filter={}):
+    """Get a list of patches matching the given filters."""
+    try:
+        # We allow access to many of the fields.  But, some fields are
+        # filtered by raw object so we must lookup by ID instead over
+        # XML-RPC.
+        ok_fields = [
+            "id",
+            "name",
+            "project_id",
+            "submitter_id",
+            "delegate_id",
+            "state_id",
+            "date",
+            "commit_ref",
+            "hash",
+            "msgid",
+            "name",
+            "max_count",
+            ]
+
+        dfilter = {}
+        max_count = 0
+
+        for key in filter:
+            parts = key.split("__")
+            if ok_fields.count(parts[0]) == 0:
+                # Invalid field given
+                return []
+            if len(parts) > 1:
+                if LOOKUP_TYPES.count(parts[1]) == 0:
+                    # Invalid lookup type given
+                    return []
+
+            if parts[0] == 'project_id':
+                dfilter['project'] = Project.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'submitter_id':
+                dfilter['submitter'] = Person.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'state_id':
+                dfilter['state'] = State.objects.filter(id =
+                                        filter[key])[0]
+            elif parts[0] == 'max_count':
+                max_count = filter[key]
+            else:
+                dfilter[key] = filter[key]
+
+        patches = Patch.objects.filter(**dfilter)
+
+        if max_count > 0:
+            return map(patch_to_dict, patches)[:max_count]
+        else:
+            return map(patch_to_dict, patches)
+
+    except:
+        return []
+
+def patch_get(patch_id):
+    """Return structure for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch_to_dict(patch)
+    except:
+        return {}
+
+def patch_get_mbox(patch_id):
+    """Return mbox string for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch.mbox().as_string()
+    except:
+        return ""
+
+def patch_get_diff(patch_id):
+    """Return diff for the given patch ID."""
+    try:
+        patch = Patch.objects.filter(id = patch_id)[0]
+        return patch.content
+    except:
+        return ""
+
+def state_list(search_str="", max_count=0):
+    """Get a list of state structures matching the given search string."""
+    try:
+        if len(search_str) > 0:
+            states = State.objects.filter(name__icontains = search_str)
+        else:
+            states = State.objects.all()
+
+        if max_count > 0:
+            return map(state_to_dict, states)[:max_count]
+        else:
+            return map(state_to_dict, states)
+    except:
+        return []
+
+def state_get(state_id):
+    """Return structure for the given state ID."""
+    try:
+        state = State.objects.filter(id = state_id)[0]
+        return state_to_dict(state)
+    except:
+        return {}
diff --git a/apps/settings.py b/apps/settings.py
index d5fd7b1..f70ac2a 100644
--- a/apps/settings.py
+++ b/apps/settings.py
@@ -98,8 +98,28 @@ DEFAULT_FROM_EMAIL = 'Patchwork <patchwork@patchwork.example.com>'
 
 ACCOUNT_ACTIVATION_DAYS = 7
 
+# Set to True to enable the Patchwork XML-RPC interface
+ENABLE_XMLRPC = False
+
+XMLRPC_METHODS = (
+    # List methods to be exposed in the form (<method path>, <xml-rpcname>,)
+    ('patchwork.xmlrpc.pw_rpc_version', 'pw_rpc_version',),
+    ('patchwork.xmlrpc.patch_list',     'patch_list',),
+    ('patchwork.xmlrpc.patch_get',      'patch_get',),
+    ('patchwork.xmlrpc.patch_get_mbox', 'patch_get_mbox',),
+    ('patchwork.xmlrpc.patch_get_diff', 'patch_get_diff',),
+    ('patchwork.xmlrpc.project_list',   'project_list',),
+    ('patchwork.xmlrpc.project_get',    'project_get',),
+    ('patchwork.xmlrpc.person_list',    'person_list',),
+    ('patchwork.xmlrpc.person_get',     'person_get',),
+    ('patchwork.xmlrpc.state_list',     'state_list',),
+    ('patchwork.xmlrpc.state_get',      'state_get',),
+)
+
 try:
     from local_settings import *
+    if ENABLE_XMLRPC:
+        INSTALLED_APPS = INSTALLED_APPS + ('django_xmlrpc',)
 except ImportError, ex:
     import sys
     sys.stderr.write(\
diff --git a/apps/urls.py b/apps/urls.py
index 6e7f34e..3ccdd1a 100644
--- a/apps/urls.py
+++ b/apps/urls.py
@@ -46,4 +46,5 @@ urlpatterns = patterns('',
         {'document_root': '/srv/patchwork/htdocs/js'}),
      (r'^images/(?P<path>.*)$', 'django.views.static.serve',
         {'document_root': '/srv/patchwork/htdocs/images'}),
+     (r'xmlrpc/$', 'django_xmlrpc.views.handle_xmlrpc',),
 )
diff --git a/docs/INSTALL b/docs/INSTALL
index 1748601..cba69d5 100644
--- a/docs/INSTALL
+++ b/docs/INSTALL
@@ -62,6 +62,17 @@ in brackets):
          cd ../../apps
          ln -s ../lib/packages/django-registration ./registration
 
+	(OPTIONAL) If you want to enable the Patchwork XML-RPC interface,
+        which is required for pwclient to work, you'll need to set up the
+        django_xmlrpc package:
+
+         cd lib/packages/
+         wget \
+             http://django-xmlrpc.googlecode.com/files/django_xmlrpc-0.1.tar.gz
+         tar -zxf django_xmlrpc-0.1.tar.gz
+         cd ../../apps
+         ln -s ../lib/packages/django_xmlrpc ./django_xmlrpc
+
 	The settings.py file contains default settings for patchwork, you'll
 	need to configure settings for your own setup.
 
@@ -86,6 +97,11 @@ in brackets):
 	  MEDIA_ROOT
 	  TEMPLATE_DIRS
 
+        If you wish to enable the XML-RPC interface, add the following to
+        your local_settings.py file:
+
+	  ENABLE_XMLRPC = True
+
 	Then, get patchwork to create its tables in your configured database:
 
 	 cd apps/