Split directory/router parsing functionality into separate file from routerlist.c

svn:r1846

7 files changed, 1082 insertions, 1064 deletions

diff --git a/src/or/Makefile.am b/src/or/Makefile.am
index afee6a713..c833fa535 100644
--- a/src/or/Makefile.am
+++ b/src/or/Makefile.am
@@ -8,7 +8,7 @@ tor_SOURCES = buffers.c circuit.c command.c config.c \
 	connection.c connection_edge.c connection_or.c \
 	cpuworker.c directory.c dirserv.c dns.c main.c \
 	onion.c rendcommon.c rendclient.c rendmid.c \
-	rendservice.c rephist.c router.c routerlist.c \
+	rendservice.c rephist.c router.c routerlist.c routerparse.c \
 	tor_main.c
 
 tor_LDADD = ../common/libor.a
@@ -17,7 +17,7 @@ test_SOURCES = buffers.c circuit.c command.c config.c \
 	connection.c connection_edge.c connection_or.c \
 	cpuworker.c directory.c dirserv.c dns.c main.c \
 	onion.c rendcommon.c rendclient.c rendmid.c \
-	rendservice.c rephist.c router.c routerlist.c \
+	rendservice.c rephist.c router.c routerlist.c routerparse.c \
 	test.c
 
 test_LDADD = ../common/libor.a
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index 9959fe77b..ec5582a46 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -268,7 +268,7 @@ dirserv_add_descriptor(const char **desc)
   cp = desc_tmp = tor_strndup(start, desc_len);
 
   /* Check: is the descriptor syntactically valid? */
-  ri = router_get_entry_from_string(cp, NULL);
+  ri = router_parse_entry_from_string(cp, NULL);
   tor_free(desc_tmp);
   if (!ri) {
     log(LOG_WARN, "Couldn't parse descriptor");
diff --git a/src/or/or.h b/src/or/or.h
index 42ec606ca..aac00b5bb 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -1185,16 +1185,13 @@ routerinfo_t *router_choose_random_node(routerlist_t *dir,
 routerinfo_t *router_get_by_addr_port(uint32_t addr, uint16_t port);
 routerinfo_t *router_get_by_nickname(char *nickname);
 void router_get_routerlist(routerlist_t **prouterlist);
+void routerlist_free(routerlist_t *routerlist);
 void routerinfo_free(routerinfo_t *router);
 routerinfo_t *routerinfo_copy(const routerinfo_t *router);
 void router_mark_as_down(char *nickname);
 int router_set_routerlist_from_file(char *routerfile);
 int router_set_routerlist_from_string(const char *s);
-int router_get_dir_hash(const char *s, char *digest);
-int router_get_router_hash(const char *s, char *digest);
 int router_set_routerlist_from_directory(const char *s, crypto_pk_env_t *pkey);
-routerinfo_t *router_get_entry_from_string(const char *s, const char *end);
-int router_add_exit_policy_from_string(routerinfo_t *router, const char *s);
 int router_compare_addr_to_exit_policy(uint32_t addr, uint16_t port,
                                        struct exit_policy_t *policy);
 #define ADDR_POLICY_ACCEPTED 0
@@ -1203,6 +1200,20 @@ int router_compare_addr_to_exit_policy(uint32_t addr, uint16_t port,
 int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port);
 int router_exit_policy_rejects_all(routerinfo_t *router);
 
+/********************************* routerparse.c ************************/
+
+int router_get_router_hash(const char *s, char *digest);
+int router_get_dir_hash(const char *s, char *digest);
+int router_parse_list_from_string(const char **s,
+                                       routerlist_t **dest,
+                                       int n_good_nicknames,
+                                       const char **good_nickname_lst);
+int router_parse_routerlist_from_directory(const char *s,
+                                           routerlist_t **dest,
+                                           crypto_pk_env_t *pkey);
+routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
+int router_add_exit_policy_from_string(routerinfo_t *router, const char *s);
+
 /********************************* dirserv.c ***************************/
 int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk);
 int dirserv_parse_fingerprint_file(const char *fname);
diff --git a/src/or/router.c b/src/or/router.c
index a7a95abc1..ccf9235b3 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -652,7 +652,7 @@ int router_dump_router_to_string(char *s, int maxlen, routerinfo_t *router,
 
 #ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
   cp = s_tmp = s_dup = tor_strdup(s);
-  ri_tmp = router_get_entry_from_string(cp, NULL);
+  ri_tmp = router_parse_entry_from_string(cp, NULL);
   if (!ri_tmp) {
     log_fn(LOG_ERR, "We just generated a router descriptor we can't parse: <<%s>>",
            s);
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 214264b08..9ef7b3921 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -2,17 +2,12 @@
 /* See LICENSE for licensing information */
 /* $Id$ */
 
-#define _GNU_SOURCE
-/* XXX this is required on rh7 to make strptime not complain. how bad
- * is this for portability?
- */
-
 #include "or.h"
 
 /**
  * \file routerlist.c
  *
- * \brief Code to parse descriptors and directories, and to
+ * \brief Code to
  * maintain and access the global list of routerinfos for known
  * servers.
  **/
@@ -21,131 +16,11 @@
 
 extern or_options_t options; /**< command-line and config-file options */
 
-/****************************************************************************/
-
-/** Enumeration of possible token types.  The ones starting with K_
- * correspond to directory 'keywords'.  _UNRECOGNIZED is for an
- * unrecognized keyword; _ERR is an error in the tokenizing process,
- * _EOF is an end-of-file marker, and _NIL is used to encode
- * not-a-token.
- */
-typedef enum {
-  K_ACCEPT,
-  K_DIRECTORY_SIGNATURE,
-  K_RECOMMENDED_SOFTWARE,
-  K_REJECT,
-  K_ROUTER,
-  K_SIGNED_DIRECTORY,
-  K_SIGNING_KEY,
-  K_ONION_KEY,
-  K_LINK_KEY, /* XXXX obsolete */
-  K_ROUTER_SIGNATURE,
-  K_PUBLISHED,
-  K_RUNNING_ROUTERS,
-  K_PLATFORM,
-  K_OPT,
-  K_BANDWIDTH,
-  K_PORTS,
-  _UNRECOGNIZED,
-  _ERR,
-  _EOF,
-  _NIL
-} directory_keyword;
-
-/** Structure to hold a single directory token.
- *
- * We parse a directory by breaking it into "tokens", each consisting
- * of a keyword, a line full of arguments, and a binary object.  The
- * arguments and object are both optional, depending on the keyword
- * type.
- */
-typedef struct directory_token_t {
-  directory_keyword tp;        /**< Type of the token. */
-  int n_args;                  /**< Number of elements in args */
-  char **args;                 /**< Array of arguments from keyword line. */
-  char *object_type;           /**< -----BEGIN [object_type]-----*/
-  int object_size;             /**< Bytes in object_body */
-  char *object_body;           /**< Contents of object, base64-decoded. */
-  crypto_pk_env_t *key;        /**< For public keys only. */
-  char *error;                 /**< For _ERR tokens only. */
-} directory_token_t;
-
-/* ********************************************************************** */
-
-/** We use a table of rules to decide how to parse each token type. */
-
-/** Rules for how many arguments a keyword can take. */
-typedef enum {
-  NO_ARGS,     /**<    (1) no arguments, ever */
-  ARGS,        /**<    (2) a list of arguments separated by spaces */
-  CONCAT_ARGS, /**< or (3) the rest of the line, treated as a single argument. */
-} arg_syntax;
-
-/** Rules for whether the keyword needs an object. */
-typedef enum {
-  NO_OBJ,      /**<    (1) no object, ever */
-  NEED_OBJ,    /**<    (2) object is required */
-  NEED_KEY,    /**<    (3) object is required, and must be a public key. */
-  OBJ_OK,      /**< or (4) object is optional. */
-} obj_syntax;
-
-/** Rules for where a keyword can appear. */
-typedef enum {
-  ANY = 0,    /**< Appears in router descriptor or in directory sections. */
-  DIR_ONLY,   /**< Appears only in directory. */
-  RTR_ONLY,   /**< Appears only in router descriptor. */
-} where_syntax;
-
-/** Table mapping keywords to token value and to argument rules. */
-static struct {
-  char *t; int v; arg_syntax s; obj_syntax os; where_syntax ws;
-} token_table[] = {
-  { "accept",              K_ACCEPT,              ARGS,    NO_OBJ,  RTR_ONLY },
-  { "directory-signature", K_DIRECTORY_SIGNATURE, ARGS,    NEED_OBJ,DIR_ONLY},
-  { "reject",              K_REJECT,              ARGS,    NO_OBJ,  RTR_ONLY },
-  { "router",              K_ROUTER,              ARGS,    NO_OBJ,  RTR_ONLY },
-  { "recommended-software",K_RECOMMENDED_SOFTWARE,ARGS,    NO_OBJ,  DIR_ONLY },
-  { "signed-directory",    K_SIGNED_DIRECTORY,    NO_ARGS, NO_OBJ,  DIR_ONLY },
-  { "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY,RTR_ONLY },
-  { "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY,RTR_ONLY },
-  { "link-key",            K_LINK_KEY,            NO_ARGS, NEED_KEY,RTR_ONLY },
-  { "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ,RTR_ONLY },
-  { "running-routers",     K_RUNNING_ROUTERS,     ARGS,    NO_OBJ,  DIR_ONLY },
-  { "ports",               K_PORTS,               ARGS,    NO_OBJ,  RTR_ONLY },
-  { "bandwidth",           K_BANDWIDTH,           ARGS,    NO_OBJ,  RTR_ONLY },
-  { "platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ,  RTR_ONLY },
-  { "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ,  ANY },
-  { "opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK,  ANY },
-
-  { NULL, -1 }
-};
-
 /* ********************************************************************** */
 
 /* static function prototypes */
-static routerinfo_t *
-router_pick_directory_server_impl(void);
-static int
-router_get_list_from_string_impl(const char **s, routerlist_t **dest,
-                                 int n_good_nicknames,
-                                 const char **good_nickname_lst);
-int /* Exposed for unit tests */
-router_get_routerlist_from_directory_impl(const char *s, routerlist_t **dest,
-                                          crypto_pk_env_t *pkey);
-static int
-router_add_exit_policy(routerinfo_t *router, directory_token_t *tok);
-static int
-router_resolve_routerlist(routerlist_t *dir);
-
-static int router_get_hash_impl(const char *s, char *digest,
-                                const char *start_str, const char *end_str);
-static void token_free(directory_token_t *tok);
-static smartlist_t *find_all_exitpolicy(smartlist_t *s);
-static directory_token_t *find_first_by_keyword(smartlist_t *s,
-                                                directory_keyword keyword);
-static int tokenize_string(const char *start, const char *end,
-                           smartlist_t *out, int is_dir);
-static directory_token_t *get_next_token(const char **s, where_syntax where);
+static routerinfo_t *router_pick_directory_server_impl(void);
+static int router_resolve_routerlist(routerlist_t *dir);
 
 /****************************************************************************/
 
@@ -404,7 +279,7 @@ routerinfo_t *routerinfo_copy(const routerinfo_t *router)
 }
 
 /** Free all storage held by a routerlist <b>rl</b> */
-static void routerlist_free(routerlist_t *rl)
+void routerlist_free(routerlist_t *rl)
 {
   SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
                     routerinfo_free(r));
@@ -454,7 +329,7 @@ int router_set_routerlist_from_file(char *routerfile)
  * routerlist. */
 int router_set_routerlist_from_string(const char *s)
 {
-  if (router_get_list_from_string_impl(&s, &routerlist, -1, NULL)) {
+  if (router_parse_list_from_string(&s, &routerlist, -1, NULL)) {
     log(LOG_WARN, "Error parsing router file");
     return -1;
   }
@@ -467,24 +342,6 @@ int router_set_routerlist_from_string(const char *s)
   return 0;
 }
 
-/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
- * <b>s</b>.  Return 0 on success, nonzero on failure.
- */
-int router_get_dir_hash(const char *s, char *digest)
-{
-  return router_get_hash_impl(s,digest,
-                              "signed-directory","directory-signature");
-}
-
-/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
- * <b>s</b>. Return 0 on success, nonzero on failure.
- */
-int router_get_router_hash(const char *s, char *digest)
-{
-  return router_get_hash_impl(s,digest,
-                              "router ","router-signature");
-}
-
 /** Return 1 if myversion is in versionlist. Else return 0.
  * (versionlist is a comma-separated list of versions.) */
 int is_recommended_version(const char *myversion,
@@ -512,7 +369,7 @@ int is_recommended_version(const char *myversion,
  * signed with pkey. */
 int router_set_routerlist_from_directory(const char *s, crypto_pk_env_t *pkey)
 {
-  if (router_get_routerlist_from_directory_impl(s, &routerlist, pkey)) {
+  if (router_parse_routerlist_from_directory(s, &routerlist, pkey)) {
     log_fn(LOG_WARN, "Couldn't parse directory.");
     return -1;
   }
@@ -522,8 +379,8 @@ int router_set_routerlist_from_directory(const char *s, crypto_pk_env_t *pkey)
   }
   if (!is_recommended_version(VERSION, routerlist->software_versions)) {
     log(options.IgnoreVersion ? LOG_WARN : LOG_ERR,
-        "You are running Tor version %s, which will not work with this network.\n"
-       "Please use %s%s.",
+     "You are running Tor version %s, which will not work with this network.\n"
+     "Please use %s%s.",
         VERSION, strchr(routerlist->software_versions,',') ? "one of " : "",
         routerlist->software_versions);
     if(options.IgnoreVersion) {
@@ -674,906 +531,6 @@ int router_exit_policy_rejects_all(routerinfo_t *router) {
     == ADDR_POLICY_REJECTED;
 }
 
-/** Parse a date of the format "YYYY-MM-DD hh:mm:ss" and store the result into
- * *<b>t</b>.
- */
-/* XXX this should go in util.c, yes? -RD */
-static int parse_time(const char *cp, time_t *t)
-{
-  struct tm st_tm;
-#ifdef HAVE_STRPTIME
-  if (!strptime(cp, "%Y-%m-%d %H:%M:%S", &st_tm)) {
-    log_fn(LOG_WARN, "Published time was unparseable"); return -1;
-  }
-#else
-  unsigned int year=0, month=0, day=0, hour=100, minute=100, second=100;
-  if (sscanf(cp, "%u-%u-%u %u:%u:%u", &year, &month,
-                &day, &hour, &minute, &second) < 6) {
-        log_fn(LOG_WARN, "Published time was unparseable"); return -1;
-  }
-  if (year < 1970 || month < 1 || month > 12 || day < 1 || day > 31 ||
-          hour > 23 || minute > 59 || second > 61) {
-        log_fn(LOG_WARN, "Published time was nonsensical"); return -1;
-  }
-  st_tm.tm_year = year;
-  st_tm.tm_mon = month-1;
-  st_tm.tm_mday = day;
-  st_tm.tm_hour = hour;
-  st_tm.tm_min = minute;
-  st_tm.tm_sec = second;
-#endif
-  *t = tor_timegm(&st_tm);
-  return 0;
-}
-
-
-/** Helper function: parse a directory from <b>s</b> and, when done, store the
- * resulting routerlist in *<b>dest</b>, freeing the old value if necessary.
- * If <b>pkey</b> is provided, we check the directory signature with pkey.
- */
-int /* Should be static; exposed for unit tests */
-router_get_routerlist_from_directory_impl(const char *str,
-                                          routerlist_t **dest,
-                                          crypto_pk_env_t *pkey)
-{
-  directory_token_t *tok;
-  char digest[20];
-  char signed_digest[128];
-  routerlist_t *new_dir = NULL;
-  char *versions = NULL;
-  time_t published_on;
-  char *good_nickname_lst[1024];
-  int n_good_nicknames = 0;
-  int i, r;
-  const char *end;
-  smartlist_t *tokens = NULL;
-
-  if (router_get_dir_hash(str, digest)) {
-    log_fn(LOG_WARN, "Unable to compute digest of directory");
-    goto err;
-  }
-  log(LOG_DEBUG,"Received directory hashes to %s",hex_str(digest,4));
-
-  if ((end = strstr(str,"\nrouter "))) {
-    ++end;
-  } else if ((end = strstr(str, "\ndirectory-signature"))) {
-    ++end;
-  } else {
-    end = str + strlen(str);
-  }
-
-  tokens = smartlist_create();
-  if (tokenize_string(str,end,tokens,1)) {
-    log_fn(LOG_WARN, "Error tokenizing directory"); goto err;
-  }
-  if (smartlist_len(tokens) < 1) {
-    log_fn(LOG_WARN, "Impossibly short directory header"); goto err;
-  }
-  if ((tok = find_first_by_keyword(tokens, _UNRECOGNIZED))) {
-    log_fn(LOG_WARN, "Unrecognized keyword in \"%s\"; can't parse directory.",
-           tok->args[0]);
-    goto err;
-  }
-
-  tok = smartlist_get(tokens,0);
-  if (tok->tp != K_SIGNED_DIRECTORY) {
-    log_fn(LOG_WARN, "Directory doesn't start with signed-directory.");
-    goto err;
-  }
-
-  if (!(tok = find_first_by_keyword(tokens, K_PUBLISHED))) {
-    log_fn(LOG_WARN, "Missing published time on directory.");
-    goto err;
-  }
-  tor_assert(tok->n_args == 1);
-
-  if (parse_time(tok->args[0], &published_on) < 0) {
-     goto err;
-  }
-
-  if (!(tok = find_first_by_keyword(tokens, K_RECOMMENDED_SOFTWARE))) {
-    log_fn(LOG_WARN, "Missing recommended-software line from directory.");
-    goto err;
-  }
-  if (tok->n_args != 1) {
-    log_fn(LOG_WARN, "Invalid recommended-software line"); goto err;
-  }
-  versions = tor_strdup(tok->args[0]);
-
-  if (!(tok = find_first_by_keyword(tokens, K_RUNNING_ROUTERS))) {
-    log_fn(LOG_WARN, "Missing running-routers line from directory.");
-    goto err;
-  }
-
-  n_good_nicknames = tok->n_args;
-  memcpy(good_nickname_lst, tok->args, n_good_nicknames*sizeof(char *));
-  tok->n_args = 0; /* Don't free the strings in good_nickname_lst yet. */
-
-  /* Read the router list from s, advancing s up past the end of the last
-   * router. */
-  str = end;
-  if (router_get_list_from_string_impl(&str, &new_dir,
-                                       n_good_nicknames,
-                                       (const char**)good_nickname_lst)) {
-    log_fn(LOG_WARN, "Error reading routers from directory");
-    goto err;
-  }
-  for (i = 0; i < n_good_nicknames; ++i) {
-    tor_free(good_nickname_lst[i]); /* now free them */
-  }
-  new_dir->software_versions = versions; versions = NULL;
-  new_dir->published_on = published_on;
-
-  SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
-  smartlist_free(tokens);
-
-  tokens = smartlist_create();
-  if (tokenize_string(str,str+strlen(str),tokens,1)<0) {
-    log_fn(LOG_WARN, "Error tokenizing signature"); goto err;
-  }
-
-  if (smartlist_len(tokens) != 1 ||
-   ((directory_token_t*)smartlist_get(tokens,0))->tp != K_DIRECTORY_SIGNATURE){
-    log_fn(LOG_WARN,"Expected a single directory signature"); goto err;
-  }
-  tok = smartlist_get(tokens,0);
-  if (strcmp(tok->object_type, "SIGNATURE") || tok->object_size != 128) {
-    log_fn(LOG_WARN, "Bad object type or length on directory signature");
-    goto err;
-  }
-  if (pkey) {
-    if (crypto_pk_public_checksig(pkey, tok->object_body, 128, signed_digest)
-        != 20) {
-      log_fn(LOG_WARN, "Error reading directory: invalid signature.");
-      goto err;
-    }
-    log(LOG_DEBUG,"Signed directory hash starts %s", hex_str(signed_digest,4));
-
-    if (memcmp(digest, signed_digest, 20)) {
-      log_fn(LOG_WARN, "Error reading directory: signature does not match.");
-      goto err;
-    }
-  }
-
-  if (*dest)
-    routerlist_free(*dest);
-  *dest = new_dir;
-
-  r = 0;
-  goto done;
- err:
-  r = -1;
-  if (new_dir)
-    routerlist_free(new_dir);
-  tor_free(versions);
-  for (i = 0; i < n_good_nicknames; ++i) {
-    tor_free(good_nickname_lst[i]);
-  }
- done:
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
-    smartlist_free(tokens);
-  }
-  return r;
-}
-
-/** Helper function: Given a string *<b>s</b> containing a concatenated
- * sequence of router descriptors, parses them and stores the result
- * in *<b>dest</b>.  If good_nickname_lst is provided, then routers whose
- * nicknames are not listed are marked as nonrunning.  Advances *s to
- * a point immediately following the last router entry.  Returns 0 on
- * success and -1 on failure.
- */
-static int
-router_get_list_from_string_impl(const char **s, routerlist_t **dest,
-                                 int n_good_nicknames,
-                                 const char **good_nickname_lst)
-{
-  routerinfo_t *router;
-  smartlist_t *routers;
-  int rarray_len = 0;
-  int i;
-  const char *end;
-
-  tor_assert(s && *s);
-
-  routers = smartlist_create();
-
-  while (1) {
-    *s = eat_whitespace(*s);
-    /* Don't start parsing the rest of *s unless it contains a router. */
-    if (strncmp(*s, "router ", 7)!=0)
-      break;
-    if ((end = strstr(*s+1, "\nrouter "))) {
-      end++;
-    } else if ((end = strstr(*s+1, "\ndirectory-signature"))) {
-      end++;
-    } else {
-      end = *s+strlen(*s);
-    }
-
-    router = router_get_entry_from_string(*s, end);
-    *s = end;
-    if (!router) {
-      log_fn(LOG_WARN, "Error reading router; skipping");
-      continue;
-    }
-
-    if (n_good_nicknames>=0) {
-      router->is_running = 0;
-      for (i = 0; i < n_good_nicknames; ++i) {
-        if (0==strcasecmp(good_nickname_lst[i], router->nickname)) {
-          router->is_running = 1;
-          break;
-        }
-      }
-    } else {
-      router->is_running = 1; /* start out assuming all dirservers are up */
-    }
-    smartlist_add(routers, router);
-    log_fn(LOG_DEBUG,"just added router #%d.",rarray_len);
-  }
-
-  if (*dest)
-    routerlist_free(*dest);
-  *dest = tor_malloc(sizeof(routerlist_t));
-  (*dest)->routers = routers;
-  (*dest)->software_versions = NULL;
-
-  return 0;
-}
-
-
-/** Helper function: reads a single router entry from *<b>s</b> ...
- * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
- * returns NULL.
- */
-routerinfo_t *router_get_entry_from_string(const char *s,
-                                           const char *end) {
-  routerinfo_t *router = NULL;
-  char signed_digest[128];
-  char digest[128];
-  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
-  directory_token_t *tok;
-  int t;
-  int ports_set, bw_set;
-
-  if (!end) {
-    end = s + strlen(s);
-  }
-
-  if (router_get_router_hash(s, digest) < 0) {
-    log_fn(LOG_WARN, "Couldn't compute router hash.");
-    return NULL;
-  }
-  tokens = smartlist_create();
-  if (tokenize_string(s,end,tokens,0)) {
-    log_fn(LOG_WARN, "Error tokeninzing router descriptor."); goto err;
-  }
-
-  if (smartlist_len(tokens) < 2) {
-    log_fn(LOG_WARN, "Impossibly short router descriptor.");
-    goto err;
-  }
-  if ((tok = find_first_by_keyword(tokens, _UNRECOGNIZED))) {
-    log_fn(LOG_WARN, "Unrecognized keyword in \"%s\"; skipping descriptor.",
-           tok->args[0]);
-    goto err;
-  }
-
-  tok = smartlist_get(tokens,0);
-  if (tok->tp != K_ROUTER) {
-    log_fn(LOG_WARN,"Entry does not start with \"router\"");
-    goto err;
-  }
-
-  router = tor_malloc_zero(sizeof(routerinfo_t));
-  router->onion_pkey = router->identity_pkey = NULL;
-  ports_set = bw_set = 0;
-
-  if (tok->n_args == 2 || tok->n_args == 5 || tok->n_args == 6) {
-    router->nickname = tor_strdup(tok->args[0]);
-    if (strlen(router->nickname) > MAX_NICKNAME_LEN) {
-      log_fn(LOG_WARN,"Router nickname too long.");
-      goto err;
-    }
-    if (strspn(router->nickname, LEGAL_NICKNAME_CHARACTERS) !=
-        strlen(router->nickname)) {
-      log_fn(LOG_WARN, "Router nickname contains illegal characters.");
-      goto err;
-    }
-    router->address = tor_strdup(tok->args[1]);
-    router->addr = 0;
-
-    if (tok->n_args >= 5) {
-      router->or_port = atoi(tok->args[2]);
-      router->socks_port = atoi(tok->args[3]);
-      router->dir_port = atoi(tok->args[4]);
-      ports_set = 1;
-      /* XXXX Remove this after everyone has moved to 0.0.6 */
-      if (tok->n_args == 6) {
-        router->bandwidthrate = atoi(tok->args[5]);
-        router->bandwidthburst = router->bandwidthrate * 10;
-        bw_set = 1;
-      }
-    }
-  } else {
-    log_fn(LOG_WARN,"Wrong # of arguments to \"router\" (%d)",tok->n_args);
-    goto err;
-  }
-
-  tok = find_first_by_keyword(tokens, K_PORTS);
-  if (tok && ports_set) {
-    log_fn(LOG_WARN,"Redundant ports line");
-    goto err;
-  } else if (tok) {
-    if (tok->n_args != 3) {
-      log_fn(LOG_WARN,"Wrong # of arguments to \"ports\"");
-      goto err;
-    }
-    router->or_port = atoi(tok->args[0]);
-    router->socks_port = atoi(tok->args[1]);
-    router->dir_port = atoi(tok->args[2]);
-    ports_set = 1;
-  }
-
-  tok = find_first_by_keyword(tokens, K_BANDWIDTH);
-  if (tok && bw_set) {
-    log_fn(LOG_WARN,"Redundant bandwidth line");
-    goto err;
-  } else if (tok) {
-    if (tok->n_args < 2) {
-      log_fn(LOG_WARN,"Not enough arguments to \"bandwidth\"");
-      goto err;
-    }
-    router->bandwidthrate = atoi(tok->args[0]);
-    router->bandwidthburst = atoi(tok->args[1]);
-    bw_set = 1;
-  }
-
-  if (!(tok = find_first_by_keyword(tokens, K_PUBLISHED))) {
-    log_fn(LOG_WARN, "Missing published time"); goto err;
-  }
-  tor_assert(tok->n_args == 1);
-  if (parse_time(tok->args[0], &router->published_on) < 0)
-          goto err;
-
-  if (!(tok = find_first_by_keyword(tokens, K_ONION_KEY))) {
-    log_fn(LOG_WARN, "Missing onion key"); goto err;
-  }
-  /* XXX Check key length */
-  router->onion_pkey = tok->key;
-  tok->key = NULL; /* Prevent free */
-
-  if ((tok = find_first_by_keyword(tokens, K_LINK_KEY))) {
-    log_fn(LOG_INFO, "Skipping obsolete link-key");
-  }
-
-  if (!(tok = find_first_by_keyword(tokens, K_SIGNING_KEY))) {
-    log_fn(LOG_WARN, "Missing onion key"); goto err;
-  }
-  /* XXX Check key length */
-  router->identity_pkey = tok->key;
-  tok->key = NULL; /* Prevent free */
-
-  if ((tok = find_first_by_keyword(tokens, K_PLATFORM))) {
-    router->platform = tor_strdup(tok->args[0]);
-  }
-
-  exit_policy_tokens = find_all_exitpolicy(tokens);
-  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
-                    if (router_add_exit_policy(router,t)<0) {
-                      log_fn(LOG_WARN,"Error in exit policy"); goto err;}
-                    );
-
-  if (!(tok = find_first_by_keyword(tokens, K_ROUTER_SIGNATURE))) {
-    log_fn(LOG_WARN, "Missing router signature"); goto err;
-  }
-  if (strcmp(tok->object_type, "SIGNATURE") || tok->object_size != 128) {
-    log_fn(LOG_WARN, "Bad object type or length on router signature");
-    goto err;
-  }
-  if ((t=crypto_pk_public_checksig(router->identity_pkey, tok->object_body,
-                                   128, signed_digest)) != 20) {
-    log_fn(LOG_WARN, "Invalid signature %d",t); goto err;
-  }
-  if (memcmp(digest, signed_digest, 20)) {
-    log_fn(LOG_WARN, "Mismatched signature"); goto err;
-  }
-
-  if (!ports_set) {
-    log_fn(LOG_WARN,"No ports declared; failing."); goto err;
-  }
-  if (!bw_set) {
-    log_fn(LOG_WARN,"No bandwidth declared; failing."); goto err;
-  }
-  if(!router->or_port) {
-    log_fn(LOG_WARN,"or_port unreadable or 0. Failing.");
-    goto err;
-  }
-  if (!router->bandwidthrate) {
-    log_fn(LOG_WARN,"bandwidthrate unreadable or 0. Failing.");
-    goto err;
-  }
-  if (!router->platform) {
-    router->platform = tor_strdup("<unknown>");
-  }
-
-  log_fn(LOG_DEBUG,"or_port %d, socks_port %d, dir_port %d, bandwidthrate %u, bandwidthburst %u.",
-    router->or_port, router->socks_port, router->dir_port,
-    (unsigned) router->bandwidthrate, (unsigned) router->bandwidthburst);
-
-
-  goto done;
-  return router;
-
- err:
-  routerinfo_free(router);
-  router = NULL;
- done:
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
-    smartlist_free(tokens);
-  }
-  if (exit_policy_tokens) {
-    smartlist_free(exit_policy_tokens);
-  }
-  return router;
-}
-
-/** Parse the exit policy in the string <b>s</b> and add it to <b>router</b>.
- */
-int
-router_add_exit_policy_from_string(routerinfo_t *router, const char *s)
-{
-  directory_token_t *tok = NULL;
-  const char *cp;
-  char *tmp;
-  int r;
-  int len, idx;
-
-  /* *s might not end with \n, so we need to extend it with one. */
-  len = strlen(s);
-  cp = tmp = tor_malloc(len+2);
-  for (idx = 0; idx < len; ++idx) {
-    tmp[idx] = tolower(s[idx]);
-  }
-  tmp[len]='\n';
-  tmp[len+1]='\0';
-  tok = get_next_token(&cp, RTR_ONLY);
-  if (tok->tp == _ERR) {
-    log_fn(LOG_WARN, "Error reading exit policy: %s", tok->error);
-    goto err;
-  }
-  if (tok->tp != K_ACCEPT && tok->tp != K_REJECT) {
-    log_fn(LOG_WARN, "Expected 'accept' or 'reject'.");
-    goto err;
-  }
-
-  /* Now that we've gotten an exit policy, add it to the router. */
-  r = router_add_exit_policy(router, tok);
-  goto done;
- err:
-  r = -1;
- done:
-  free(tmp);
-  token_free(tok);
-  return r;
-}
-
-/** Given a K_ACCEPT or K_REJECT token and a router, create a new exit_policy_t
- * corresponding to the token, and add it to <b>router</b> */
-static int
-router_add_exit_policy(routerinfo_t *router, directory_token_t *tok) {
-
-  struct exit_policy_t *tmpe, *newe;
-  struct in_addr in;
-  char *arg, *address, *mask, *port, *endptr;
-  int bits;
-
-  tor_assert(tok->tp == K_REJECT || tok->tp == K_ACCEPT);
-
-  if (tok->n_args != 1)
-    return -1;
-  arg = tok->args[0];
-
-  newe = tor_malloc_zero(sizeof(struct exit_policy_t));
-
-  newe->string = tor_malloc(8+strlen(arg));
-  if (tok->tp == K_REJECT) {
-    strcpy(newe->string, "reject ");
-    newe->policy_type = EXIT_POLICY_REJECT;
-  } else {
-    strcpy(newe->string, "accept ");
-    newe->policy_type = EXIT_POLICY_ACCEPT;
-  }
-  strcat(newe->string, arg); /* can't overflow */
-
-  address = arg;
-  mask = strchr(arg,'/');
-  port = strchr(mask?mask:arg,':');
-  /* Break 'arg' into separate strings.  'arg' was already strdup'd by
-   * _router_get_next_token, so it's safe to modify.
-   */
-  if (mask)
-    *mask++ = 0;
-  if (port)
-    *port++ = 0;
-
-  if (strcmp(address, "*") == 0) {
-    newe->addr = 0;
-  } else if (tor_inet_aton(address, &in) != 0) {
-    newe->addr = ntohl(in.s_addr);
-  } else {
-    log_fn(LOG_WARN, "Malformed IP %s in exit policy; rejecting.",
-           address);
-    goto policy_read_failed;
-  }
-  if (!mask) {
-    if (strcmp(address, "*") == 0)
-      newe->msk = 0;
-    else
-      newe->msk = 0xFFFFFFFFu;
-  } else {
-    endptr = NULL;
-    bits = (int) strtol(mask, &endptr, 10);
-    if (!*endptr) {
-      /* strtol handled the whole mask. */
-      newe->msk = ~((1<<(32-bits))-1);
-    } else if (tor_inet_aton(mask, &in) != 0) {
-      newe->msk = ntohl(in.s_addr);
-    } else {
-      log_fn(LOG_WARN, "Malformed mask %s on exit policy; rejecting.",
-             mask);
-      goto policy_read_failed;
-    }
-  }
-  if (!port || strcmp(port, "*") == 0) {
-    newe->prt_min = 0;
-    newe->prt_max = 65535;
-  } else {
-    endptr = NULL;
-    newe->prt_min = (uint16_t) strtol(port, &endptr, 10);
-    if (*endptr == '-') {
-      port = endptr+1;
-      endptr = NULL;
-      newe->prt_max = (uint16_t) strtol(port, &endptr, 10);
-      if (*endptr) {
-      log_fn(LOG_WARN, "Malformed port %s on exit policy; rejecting.",
-             port);
-      }
-    } else if (*endptr) {
-      log_fn(LOG_WARN, "Malformed port %s on exit policy; rejecting.",
-             port);
-      goto policy_read_failed;
-    } else {
-      newe->prt_max = newe->prt_min;
-    }
-  }
-
-  in.s_addr = htonl(newe->addr);
-  address = tor_strdup(inet_ntoa(in));
-  in.s_addr = htonl(newe->msk);
-  log_fn(LOG_DEBUG,"%s %s/%s:%d-%d",
-         newe->policy_type == EXIT_POLICY_REJECT ? "reject" : "accept",
-         address, inet_ntoa(in), newe->prt_min, newe->prt_max);
-  tor_free(address);
-
-  /* now link newe onto the end of exit_policy */
-
-  if(!router->exit_policy) {
-    router->exit_policy = newe;
-    return 0;
-  }
-
-  for(tmpe=router->exit_policy; tmpe->next; tmpe=tmpe->next) ;
-  tmpe->next = newe;
-
-  return 0;
-
-policy_read_failed:
-  tor_assert(newe->string);
-  log_fn(LOG_WARN,"Couldn't parse line '%s'. Dropping", newe->string);
-  tor_free(newe->string);
-  free(newe);
-  return -1;
-}
-
-/*
- * Low-level tokenizer for router descriptors and directories.
- */
-
-
-/** Free all resources allocated for <b>tok</b> */
-static void
-token_free(directory_token_t *tok)
-{
-  int i;
-  tor_assert(tok);
-  if (tok->args) {
-    for (i = 0; i < tok->n_args; ++i) {
-      tor_free(tok->args[i]);
-    }
-    tor_free(tok->args);
-  }
-  tor_free(tok->object_type);
-  tor_free(tok->object_body);
-  if (tok->key)
-    crypto_free_pk_env(tok->key);
-  tor_free(tok);
-}
-
-/** Helper function: read the next token from *s, advance *s to the end
- * of the token, and return the parsed token.  If 'where' is DIR_ONLY
- * or RTR_ONLY, reject all tokens of the wrong type.
- */
-static directory_token_t *
-get_next_token(const char **s, where_syntax where) {
-  const char *next, *obstart;
-  int i, done, allocated;
-  directory_token_t *tok;
-  arg_syntax a_syn;
-  obj_syntax o_syn = NO_OBJ;
-
-#define RET_ERR(msg)                                    \
-  do { if (tok) token_free(tok);                        \
-       tok = tor_malloc_zero(sizeof(directory_token_t));\
-       tok->tp = _ERR;                                  \
-       tok->error = msg;                                \
-       goto done_tokenizing; } while (0)
-
-  tok = tor_malloc_zero(sizeof(directory_token_t));
-  tok->tp = _ERR;
-
-  *s = eat_whitespace(*s);
-  if (!**s) {
-    tok->tp = _EOF;
-    return tok;
-  }
-  next = find_whitespace(*s);
-  if (!next) {
-    tok->error = "Unexpected EOF"; return tok;
-  }
-  /* It's a keyword... but which one? */
-  for (i = 0 ; token_table[i].t ; ++i) {
-    if (!strncmp(token_table[i].t, *s, next-*s)) {
-      /* We've found the keyword. */
-      tok->tp = token_table[i].v;
-      a_syn = token_table[i].s;
-      o_syn = token_table[i].os;
-      if (token_table[i].ws != ANY && token_table[i].ws != where) {
-        if (where == DIR_ONLY) {
-          RET_ERR("Found a router-only token in a directory section");
-        } else {
-          RET_ERR("Found a directory-only token in a router descriptor");
-        }
-      }
-      if (a_syn == ARGS) {
-        /* This keyword takes multiple arguments. */
-        i = 0;
-        done = (*next == '\n');
-        allocated = 32;
-        tok->args = tor_malloc(sizeof(char*)*32);
-        *s = eat_whitespace_no_nl(next);
-        while (**s != '\n' && !done) {
-          next = find_whitespace(*s);
-          if (*next == '\n')
-            done = 1;
-          if (i == allocated) {
-            allocated *= 2;
-            tok->args = tor_realloc(tok->args,sizeof(char*)*allocated);
-          }
-          tok->args[i++] = tor_strndup(*s,next-*s);
-          *s = eat_whitespace_no_nl(next+1);
-        }
-        tok->n_args = i;
-      } else if (a_syn == CONCAT_ARGS) {
-        /* The keyword takes the line as a single argument */
-        *s = eat_whitespace_no_nl(next);
-        next = strchr(*s, '\n');
-        if (!next)
-          RET_ERR("Unexpected EOF");
-        tok->args = tor_malloc(sizeof(char*));
-        tok->args[0] = tor_strndup(*s,next-*s);
-        tok->n_args = 1;
-        *s = eat_whitespace_no_nl(next+1);
-      } else {
-        /* The keyword takes no arguments. */
-        tor_assert(a_syn == NO_ARGS);
-        *s = eat_whitespace_no_nl(next);
-        if (**s != '\n') {
-          RET_ERR("Unexpected arguments");
-        }
-        tok->n_args = 0;
-        *s = eat_whitespace_no_nl(*s+1);
-      }
-      break;
-    }
-  }
-  if (tok->tp == _ERR) {
-    tok->tp = _UNRECOGNIZED;
-    next = strchr(*s, '\n');
-    if (!next) {
-      RET_ERR("Unexpected EOF");
-    }
-    tok->args = tor_malloc(sizeof(char*));
-    tok->args[0] = tor_strndup(*s,next-*s);
-    tok->n_args = 1;
-    *s = next+1;
-    o_syn = OBJ_OK;
-  }
-  *s = eat_whitespace(*s);
-  if (strncmp(*s, "-----BEGIN ", 11)) {
-    goto done_tokenizing;
-  }
-  obstart = *s;
-  *s += 11; /* length of "-----BEGIN ". */
-  next = strchr(*s, '\n');
-  if (next-*s < 6 || strncmp(next-5, "-----\n", 6)) {
-    RET_ERR("Malformed object: bad begin line");
-  }
-  tok->object_type = tor_strndup(*s, next-*s-5);
-  *s = next+1;
-  next = strstr(*s, "-----END ");
-  if (!next) {
-    RET_ERR("Malformed object: missing end line");
-  }
-  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) {
-    if (strncmp(next, "-----END RSA PUBLIC KEY-----\n", 29))
-      RET_ERR("Malformed object: mismatched end line");
-    next = strchr(next,'\n')+1;
-    tok->key = crypto_new_pk_env();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, next-obstart))
-      RET_ERR("Couldn't parse public key.");
-    *s = next;
-  } else {
-    tok->object_body = tor_malloc(next-*s); /* really, this is too much RAM. */
-    i = base64_decode(tok->object_body, 256, *s, next-*s);
-    if (i<0) {
-      RET_ERR("Malformed object: bad base64-encoded data");
-    }
-    tok->object_size = i;
-    *s = next + 9; /* length of "-----END ". */
-    i = strlen(tok->object_type);
-    if (strncmp(*s, tok->object_type, i) || strncmp(*s+i, "-----\n", 6)) {
-      RET_ERR("Malformed object: mismatched end tag");
-    }
-    *s += i+6;
-  }
-  switch(o_syn)
-    {
-    case NO_OBJ:
-      if (tok->object_body)
-        RET_ERR("Unexpected object for keyword");
-      if (tok->key)
-        RET_ERR("Unexpected public key for keyword");
-      break;
-    case NEED_OBJ:
-      if (!tok->object_body)
-        RET_ERR("Missing object for keyword");
-      break;
-    case NEED_KEY:
-      if (!tok->key)
-        RET_ERR("Missing publid key for keyword");
-      break;
-    case OBJ_OK:
-      break;
-    }
-
- done_tokenizing:
-
-#if 0
-  for (i = 0; token_table[i].t ; ++i) {
-    if (token_table[i].v == tok->tp) {
-      fputs(token_table[i].t, stdout);
-      break;
-      i = -1;
-    }
-  }
-  if (i) {
-    if (tok->tp == _UNRECOGNIZED) fputs("UNRECOGNIZED", stdout);
-    if (tok->tp == _ERR) fputs("ERR",stdout);
-    if (tok->tp == _EOF) fputs("EOF",stdout);
-    if (tok->tp == _NIL) fputs("_NIL",stdout);
-  }
-  for(i = 0; i < tok->n_args; ++i) {
-    fprintf(stdout," \"%s\"", tok->args[i]);
-  }
-  if (tok->error) { fprintf(stdout," *%s*", tok->error); }
-  fputs("\n",stdout);
-#endif
-
-
-  return tok;
-#undef RET_ERR
-}
-
-/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
- * them to <b>out</b>.  If <b>is_dir</b> is true, reject all non-directory
- * tokens; else reject all non-routerdescriptor tokens.
- */
-static int
-tokenize_string(const char *start, const char *end, smartlist_t *out,
-                int is_dir)
-{
-  const char **s;
-  directory_token_t *tok = NULL;
-  where_syntax where = is_dir ? DIR_ONLY : RTR_ONLY;
-  s = &start;
-  while (*s < end && (!tok || tok->tp != _EOF)) {
-    tok = get_next_token(s, where);
-    if (tok->tp == _ERR) {
-      log_fn(LOG_WARN, "parse error: %s", tok->error);
-      return -1;
-    }
-    smartlist_add(out, tok);
-    *s = eat_whitespace(*s);
-  }
-
-  return 0;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
- * NULL if no such keyword is found.
- */
-static directory_token_t *
-find_first_by_keyword(smartlist_t *s, directory_keyword keyword)
-{
-  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
-  return NULL;
-}
-
-/** Return a newly allocated smartlist of all accept or reject tokens in
- * <b>s</b>.
- */
-static smartlist_t *
-find_all_exitpolicy(smartlist_t *s)
-{
-  smartlist_t *out = smartlist_create();
-  SMARTLIST_FOREACH(s, directory_token_t *, t,
-                    if (t->tp == K_ACCEPT || t->tp == K_REJECT)
-                      smartlist_add(out,t));
-  return out;
-}
-
-
-/** Compute the SHA digest of the substring of <b>s</b> taken from the first
- * occurrence of <b>start_str</b> through the first newline after the first
- * subsequent occurrence of <b>end_str</b>; store the 20-byte result in
- * <b>digest</b>; return 0 on success.
- *
- * If no such substring exists, return -1.
- */
-static int router_get_hash_impl(const char *s, char *digest,
-                                const char *start_str,
-                                const char *end_str)
-{
-  char *start, *end;
-  start = strstr(s, start_str);
-  if (!start) {
-    log_fn(LOG_WARN,"couldn't find \"%s\"",start_str);
-    return -1;
-  }
-  end = strstr(start+strlen(start_str), end_str);
-  if (!end) {
-    log_fn(LOG_WARN,"couldn't find \"%s\"",end_str);
-    return -1;
-  }
-  end = strchr(end, '\n');
-  if (!end) {
-    log_fn(LOG_WARN,"couldn't find EOL");
-    return -1;
-  }
-  ++end;
-
-  if (crypto_digest(start, end-start, digest)) {
-    log_fn(LOG_WARN,"couldn't compute digest");
-    return -1;
-  }
-
-  return 0;
-}
-
 /*
   Local Variables:
   mode:c
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
new file mode 100644
index 000000000..48bb72388
--- /dev/null
+++ b/src/or/routerparse.c
@@ -0,0 +1,1052 @@
+/* Copyright 2001-2003 Roger Dingledine, Matej Pfajfar. */
+/* See LICENSE for licensing information */
+/* $Id$ */
+
+/**
+ * \file routerparse.c
+ *
+ * \brief Code to parse and validate router descriptors and directories.
+ **/
+
+#define _GNU_SOURCE
+/* XXX this is required on rh7 to make strptime not complain. how bad
+ * is this for portability?
+ */
+
+#include "or.h"
+
+/****************************************************************************/
+
+/** Enumeration of possible token types.  The ones starting with K_
+ * correspond to directory 'keywords'.  _UNRECOGNIZED is for an
+ * unrecognized keyword; _ERR is an error in the tokenizing process,
+ * _EOF is an end-of-file marker, and _NIL is used to encode
+ * not-a-token.
+ */
+typedef enum {
+  K_ACCEPT,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_LINK_KEY, /* XXXX obsolete */
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_PLATFORM,
+  K_OPT,
+  K_BANDWIDTH,
+  K_PORTS,
+  _UNRECOGNIZED,
+  _ERR,
+  _EOF,
+  _NIL
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args;                  /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  int object_size;             /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+  crypto_pk_env_t *key;        /**< For public keys only. */
+  char *error;                 /**< For _ERR tokens only. */
+} directory_token_t;
+
+/* ********************************************************************** */
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for how many arguments a keyword can take. */
+typedef enum {
+  NO_ARGS,     /**<    (1) no arguments, ever */
+  ARGS,        /**<    (2) a list of arguments separated by spaces */
+  CONCAT_ARGS, /**< or (3) the rest of the line, treated as a single argument. */
+} arg_syntax;
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,      /**<    (1) no object, ever */
+  NEED_OBJ,    /**<    (2) object is required */
+  NEED_KEY,    /**<    (3) object is required, and must be a public key. */
+  OBJ_OK,      /**< or (4) object is optional. */
+} obj_syntax;
+
+/** Rules for where a keyword can appear. */
+typedef enum {
+  ANY = 0,    /**< Appears in router descriptor or in directory sections. */
+  DIR_ONLY,   /**< Appears only in directory. */
+  RTR_ONLY,   /**< Appears only in router descriptor. */
+} where_syntax;
+
+/** Table mapping keywords to token value and to argument rules. */
+static struct {
+  char *t; int v; arg_syntax s; obj_syntax os; where_syntax ws;
+} token_table[] = {
+  { "accept",              K_ACCEPT,              ARGS,    NO_OBJ,  RTR_ONLY },
+  { "directory-signature", K_DIRECTORY_SIGNATURE, ARGS,    NEED_OBJ,DIR_ONLY},
+  { "reject",              K_REJECT,              ARGS,    NO_OBJ,  RTR_ONLY },
+  { "router",              K_ROUTER,              ARGS,    NO_OBJ,  RTR_ONLY },
+  { "recommended-software",K_RECOMMENDED_SOFTWARE,ARGS,    NO_OBJ,  DIR_ONLY },
+  { "signed-directory",    K_SIGNED_DIRECTORY,    NO_ARGS, NO_OBJ,  DIR_ONLY },
+  { "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY,RTR_ONLY },
+  { "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY,RTR_ONLY },
+  { "link-key",            K_LINK_KEY,            NO_ARGS, NEED_KEY,RTR_ONLY },
+  { "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ,RTR_ONLY },
+  { "running-routers",     K_RUNNING_ROUTERS,     ARGS,    NO_OBJ,  DIR_ONLY },
+  { "ports",               K_PORTS,               ARGS,    NO_OBJ,  RTR_ONLY },
+  { "bandwidth",           K_BANDWIDTH,           ARGS,    NO_OBJ,  RTR_ONLY },
+  { "platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ,  RTR_ONLY },
+  { "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ,  ANY },
+  { "opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK,  ANY },
+
+  { NULL, -1 }
+};
+
+/* static function prototypes */
+static int router_add_exit_policy(routerinfo_t *router,directory_token_t *tok);
+static int router_get_hash_impl(const char *s, char *digest,
+                                const char *start_str, const char *end_str);
+static void token_free(directory_token_t *tok);
+static smartlist_t *find_all_exitpolicy(smartlist_t *s);
+static directory_token_t *find_first_by_keyword(smartlist_t *s,
+                                                directory_keyword keyword);
+static int tokenize_string(const char *start, const char *end,
+                           smartlist_t *out, int is_dir);
+static directory_token_t *get_next_token(const char **s, where_syntax where);
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
+ * <b>s</b>.  Return 0 on success, nonzero on failure.
+ */
+int router_get_dir_hash(const char *s, char *digest)
+{
+  return router_get_hash_impl(s,digest,
+                              "signed-directory","directory-signature");
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
+ * <b>s</b>. Return 0 on success, nonzero on failure.
+ */
+int router_get_router_hash(const char *s, char *digest)
+{
+  return router_get_hash_impl(s,digest,
+                              "router ","router-signature");
+}
+
+/** Parse a date of the format "YYYY-MM-DD hh:mm:ss" and store the result into
+ * *<b>t</b>.
+ */
+/* XXX this should go in util.c, yes? -RD */
+static int parse_time(const char *cp, time_t *t)
+{
+  struct tm st_tm;
+#ifdef HAVE_STRPTIME
+  if (!strptime(cp, "%Y-%m-%d %H:%M:%S", &st_tm)) {
+    log_fn(LOG_WARN, "Published time was unparseable"); return -1;
+  }
+#else
+  unsigned int year=0, month=0, day=0, hour=100, minute=100, second=100;
+  if (sscanf(cp, "%u-%u-%u %u:%u:%u", &year, &month,
+                &day, &hour, &minute, &second) < 6) {
+        log_fn(LOG_WARN, "Published time was unparseable"); return -1;
+  }
+  if (year < 1970 || month < 1 || month > 12 || day < 1 || day > 31 ||
+          hour > 23 || minute > 59 || second > 61) {
+        log_fn(LOG_WARN, "Published time was nonsensical"); return -1;
+  }
+  st_tm.tm_year = year;
+  st_tm.tm_mon = month-1;
+  st_tm.tm_mday = day;
+  st_tm.tm_hour = hour;
+  st_tm.tm_min = minute;
+  st_tm.tm_sec = second;
+#endif
+  *t = tor_timegm(&st_tm);
+  return 0;
+}
+
+
+/** Parse a directory from <b>s</b> and, when done, store the
+ * resulting routerlist in *<b>dest</b>, freeing the old value if necessary.
+ * If <b>pkey</b> is provided, we check the directory signature with pkey.
+ */
+int /* Should be static; exposed for unit tests */
+router_parse_routerlist_from_directory(const char *str,
+				       routerlist_t **dest,
+				       crypto_pk_env_t *pkey)
+{
+  directory_token_t *tok;
+  char digest[20];
+  char signed_digest[128];
+  routerlist_t *new_dir = NULL;
+  char *versions = NULL;
+  time_t published_on;
+  char *good_nickname_lst[1024];
+  int n_good_nicknames = 0;
+  int i, r;
+  const char *end;
+  smartlist_t *tokens = NULL;
+
+  if (router_get_dir_hash(str, digest)) {
+    log_fn(LOG_WARN, "Unable to compute digest of directory");
+    goto err;
+  }
+  log(LOG_DEBUG,"Received directory hashes to %s",hex_str(digest,4));
+
+  if ((end = strstr(str,"\nrouter "))) {
+    ++end;
+  } else if ((end = strstr(str, "\ndirectory-signature"))) {
+    ++end;
+  } else {
+    end = str + strlen(str);
+  }
+
+  tokens = smartlist_create();
+  if (tokenize_string(str,end,tokens,1)) {
+    log_fn(LOG_WARN, "Error tokenizing directory"); goto err;
+  }
+  if (smartlist_len(tokens) < 1) {
+    log_fn(LOG_WARN, "Impossibly short directory header"); goto err;
+  }
+  if ((tok = find_first_by_keyword(tokens, _UNRECOGNIZED))) {
+    log_fn(LOG_WARN, "Unrecognized keyword in \"%s\"; can't parse directory.",
+           tok->args[0]);
+    goto err;
+  }
+
+  tok = smartlist_get(tokens,0);
+  if (tok->tp != K_SIGNED_DIRECTORY) {
+    log_fn(LOG_WARN, "Directory doesn't start with signed-directory.");
+    goto err;
+  }
+
+  if (!(tok = find_first_by_keyword(tokens, K_PUBLISHED))) {
+    log_fn(LOG_WARN, "Missing published time on directory.");
+    goto err;
+  }
+  tor_assert(tok->n_args == 1);
+
+  if (parse_time(tok->args[0], &published_on) < 0) {
+     goto err;
+  }
+
+  if (!(tok = find_first_by_keyword(tokens, K_RECOMMENDED_SOFTWARE))) {
+    log_fn(LOG_WARN, "Missing recommended-software line from directory.");
+    goto err;
+  }
+  if (tok->n_args != 1) {
+    log_fn(LOG_WARN, "Invalid recommended-software line"); goto err;
+  }
+  versions = tor_strdup(tok->args[0]);
+
+  if (!(tok = find_first_by_keyword(tokens, K_RUNNING_ROUTERS))) {
+    log_fn(LOG_WARN, "Missing running-routers line from directory.");
+    goto err;
+  }
+
+  n_good_nicknames = tok->n_args;
+  memcpy(good_nickname_lst, tok->args, n_good_nicknames*sizeof(char *));
+  tok->n_args = 0; /* Don't free the strings in good_nickname_lst yet. */
+
+  /* Read the router list from s, advancing s up past the end of the last
+   * router. */
+  str = end;
+  if (router_parse_list_from_string(&str, &new_dir,
+				    n_good_nicknames,
+				    (const char**)good_nickname_lst)) {
+    log_fn(LOG_WARN, "Error reading routers from directory");
+    goto err;
+  }
+  for (i = 0; i < n_good_nicknames; ++i) {
+    tor_free(good_nickname_lst[i]); /* now free them */
+  }
+  new_dir->software_versions = versions; versions = NULL;
+  new_dir->published_on = published_on;
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
+  smartlist_free(tokens);
+
+  tokens = smartlist_create();
+  if (tokenize_string(str,str+strlen(str),tokens,1)<0) {
+    log_fn(LOG_WARN, "Error tokenizing signature"); goto err;
+  }
+
+  if (smartlist_len(tokens) != 1 ||
+   ((directory_token_t*)smartlist_get(tokens,0))->tp != K_DIRECTORY_SIGNATURE){
+    log_fn(LOG_WARN,"Expected a single directory signature"); goto err;
+  }
+  tok = smartlist_get(tokens,0);
+  if (strcmp(tok->object_type, "SIGNATURE") || tok->object_size != 128) {
+    log_fn(LOG_WARN, "Bad object type or length on directory signature");
+    goto err;
+  }
+  if (pkey) {
+    if (crypto_pk_public_checksig(pkey, tok->object_body, 128, signed_digest)
+        != 20) {
+      log_fn(LOG_WARN, "Error reading directory: invalid signature.");
+      goto err;
+    }
+    log(LOG_DEBUG,"Signed directory hash starts %s", hex_str(signed_digest,4));
+
+    if (memcmp(digest, signed_digest, 20)) {
+      log_fn(LOG_WARN, "Error reading directory: signature does not match.");
+      goto err;
+    }
+  }
+
+  if (*dest)
+    routerlist_free(*dest);
+  *dest = new_dir;
+
+  r = 0;
+  goto done;
+ err:
+  r = -1;
+  if (new_dir)
+    routerlist_free(new_dir);
+  tor_free(versions);
+  for (i = 0; i < n_good_nicknames; ++i) {
+    tor_free(good_nickname_lst[i]);
+  }
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
+    smartlist_free(tokens);
+  }
+  return r;
+}
+
+/** Given a string *<b>s</b> containing a concatenated
+ * sequence of router descriptors, parses them and stores the result
+ * in *<b>dest</b>.  If good_nickname_lst is provided, then routers whose
+ * nicknames are not listed are marked as nonrunning.  Advances *s to
+ * a point immediately following the last router entry.  Returns 0 on
+ * success and -1 on failure.
+ */
+int
+router_parse_list_from_string(const char **s, routerlist_t **dest,
+			      int n_good_nicknames,
+			      const char **good_nickname_lst)
+{
+  routerinfo_t *router;
+  smartlist_t *routers;
+  int rarray_len = 0;
+  int i;
+  const char *end;
+
+  tor_assert(s && *s);
+
+  routers = smartlist_create();
+
+  while (1) {
+    *s = eat_whitespace(*s);
+    /* Don't start parsing the rest of *s unless it contains a router. */
+    if (strncmp(*s, "router ", 7)!=0)
+      break;
+    if ((end = strstr(*s+1, "\nrouter "))) {
+      end++;
+    } else if ((end = strstr(*s+1, "\ndirectory-signature"))) {
+      end++;
+    } else {
+      end = *s+strlen(*s);
+    }
+
+    router = router_parse_entry_from_string(*s, end);
+    *s = end;
+    if (!router) {
+      log_fn(LOG_WARN, "Error reading router; skipping");
+      continue;
+    }
+
+    if (n_good_nicknames>=0) {
+      router->is_running = 0;
+      for (i = 0; i < n_good_nicknames; ++i) {
+        if (0==strcasecmp(good_nickname_lst[i], router->nickname)) {
+          router->is_running = 1;
+          break;
+        }
+      }
+    } else {
+      router->is_running = 1; /* start out assuming all dirservers are up */
+    }
+    smartlist_add(routers, router);
+    log_fn(LOG_DEBUG,"just added router #%d.",rarray_len);
+  }
+
+  if (*dest)
+    routerlist_free(*dest);
+  *dest = tor_malloc(sizeof(routerlist_t));
+  (*dest)->routers = routers;
+  (*dest)->software_versions = NULL;
+
+  return 0;
+}
+
+
+/** Helper function: reads a single router entry from *<b>s</b> ...
+ * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
+ * returns NULL.
+ */
+routerinfo_t *router_parse_entry_from_string(const char *s,
+                                           const char *end) {
+  routerinfo_t *router = NULL;
+  char signed_digest[128];
+  char digest[128];
+  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
+  directory_token_t *tok;
+  int t;
+  int ports_set, bw_set;
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  if (router_get_router_hash(s, digest) < 0) {
+    log_fn(LOG_WARN, "Couldn't compute router hash.");
+    return NULL;
+  }
+  tokens = smartlist_create();
+  if (tokenize_string(s,end,tokens,0)) {
+    log_fn(LOG_WARN, "Error tokeninzing router descriptor."); goto err;
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_fn(LOG_WARN, "Impossibly short router descriptor.");
+    goto err;
+  }
+  if ((tok = find_first_by_keyword(tokens, _UNRECOGNIZED))) {
+    log_fn(LOG_WARN, "Unrecognized keyword in \"%s\"; skipping descriptor.",
+           tok->args[0]);
+    goto err;
+  }
+
+  tok = smartlist_get(tokens,0);
+  if (tok->tp != K_ROUTER) {
+    log_fn(LOG_WARN,"Entry does not start with \"router\"");
+    goto err;
+  }
+
+  router = tor_malloc_zero(sizeof(routerinfo_t));
+  router->onion_pkey = router->identity_pkey = NULL;
+  ports_set = bw_set = 0;
+
+  if (tok->n_args == 2 || tok->n_args == 5 || tok->n_args == 6) {
+    router->nickname = tor_strdup(tok->args[0]);
+    if (strlen(router->nickname) > MAX_NICKNAME_LEN) {
+      log_fn(LOG_WARN,"Router nickname too long.");
+      goto err;
+    }
+    if (strspn(router->nickname, LEGAL_NICKNAME_CHARACTERS) !=
+        strlen(router->nickname)) {
+      log_fn(LOG_WARN, "Router nickname contains illegal characters.");
+      goto err;
+    }
+    router->address = tor_strdup(tok->args[1]);
+    router->addr = 0;
+
+    if (tok->n_args >= 5) {
+      router->or_port = atoi(tok->args[2]);
+      router->socks_port = atoi(tok->args[3]);
+      router->dir_port = atoi(tok->args[4]);
+      ports_set = 1;
+      /* XXXX Remove this after everyone has moved to 0.0.6 */
+      if (tok->n_args == 6) {
+        router->bandwidthrate = atoi(tok->args[5]);
+        router->bandwidthburst = router->bandwidthrate * 10;
+        bw_set = 1;
+      }
+    }
+  } else {
+    log_fn(LOG_WARN,"Wrong # of arguments to \"router\" (%d)",tok->n_args);
+    goto err;
+  }
+
+  tok = find_first_by_keyword(tokens, K_PORTS);
+  if (tok && ports_set) {
+    log_fn(LOG_WARN,"Redundant ports line");
+    goto err;
+  } else if (tok) {
+    if (tok->n_args != 3) {
+      log_fn(LOG_WARN,"Wrong # of arguments to \"ports\"");
+      goto err;
+    }
+    router->or_port = atoi(tok->args[0]);
+    router->socks_port = atoi(tok->args[1]);
+    router->dir_port = atoi(tok->args[2]);
+    ports_set = 1;
+  }
+
+  tok = find_first_by_keyword(tokens, K_BANDWIDTH);
+  if (tok && bw_set) {
+    log_fn(LOG_WARN,"Redundant bandwidth line");
+    goto err;
+  } else if (tok) {
+    if (tok->n_args < 2) {
+      log_fn(LOG_WARN,"Not enough arguments to \"bandwidth\"");
+      goto err;
+    }
+    router->bandwidthrate = atoi(tok->args[0]);
+    router->bandwidthburst = atoi(tok->args[1]);
+    bw_set = 1;
+  }
+
+  if (!(tok = find_first_by_keyword(tokens, K_PUBLISHED))) {
+    log_fn(LOG_WARN, "Missing published time"); goto err;
+  }
+  tor_assert(tok->n_args == 1);
+  if (parse_time(tok->args[0], &router->published_on) < 0)
+          goto err;
+
+  if (!(tok = find_first_by_keyword(tokens, K_ONION_KEY))) {
+    log_fn(LOG_WARN, "Missing onion key"); goto err;
+  }
+  /* XXX Check key length */
+  router->onion_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+
+  if ((tok = find_first_by_keyword(tokens, K_LINK_KEY))) {
+    log_fn(LOG_INFO, "Skipping obsolete link-key");
+  }
+
+  if (!(tok = find_first_by_keyword(tokens, K_SIGNING_KEY))) {
+    log_fn(LOG_WARN, "Missing onion key"); goto err;
+  }
+  /* XXX Check key length */
+  router->identity_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+
+  if ((tok = find_first_by_keyword(tokens, K_PLATFORM))) {
+    router->platform = tor_strdup(tok->args[0]);
+  }
+
+  exit_policy_tokens = find_all_exitpolicy(tokens);
+  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
+                    if (router_add_exit_policy(router,t)<0) {
+                      log_fn(LOG_WARN,"Error in exit policy"); goto err;}
+                    );
+
+  if (!(tok = find_first_by_keyword(tokens, K_ROUTER_SIGNATURE))) {
+    log_fn(LOG_WARN, "Missing router signature"); goto err;
+  }
+  if (strcmp(tok->object_type, "SIGNATURE") || tok->object_size != 128) {
+    log_fn(LOG_WARN, "Bad object type or length on router signature");
+    goto err;
+  }
+  if ((t=crypto_pk_public_checksig(router->identity_pkey, tok->object_body,
+                                   128, signed_digest)) != 20) {
+    log_fn(LOG_WARN, "Invalid signature %d",t); goto err;
+  }
+  if (memcmp(digest, signed_digest, 20)) {
+    log_fn(LOG_WARN, "Mismatched signature"); goto err;
+  }
+
+  if (!ports_set) {
+    log_fn(LOG_WARN,"No ports declared; failing."); goto err;
+  }
+  if (!bw_set) {
+    log_fn(LOG_WARN,"No bandwidth declared; failing."); goto err;
+  }
+  if(!router->or_port) {
+    log_fn(LOG_WARN,"or_port unreadable or 0. Failing.");
+    goto err;
+  }
+  if (!router->bandwidthrate) {
+    log_fn(LOG_WARN,"bandwidthrate unreadable or 0. Failing.");
+    goto err;
+  }
+  if (!router->platform) {
+    router->platform = tor_strdup("<unknown>");
+  }
+
+  log_fn(LOG_DEBUG,"or_port %d, socks_port %d, dir_port %d, bandwidthrate %u, bandwidthburst %u.",
+    router->or_port, router->socks_port, router->dir_port,
+    (unsigned) router->bandwidthrate, (unsigned) router->bandwidthburst);
+
+
+  goto done;
+  return router;
+
+ err:
+  routerinfo_free(router);
+  router = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
+    smartlist_free(tokens);
+  }
+  if (exit_policy_tokens) {
+    smartlist_free(exit_policy_tokens);
+  }
+  return router;
+}
+
+/** Parse the exit policy in the string <b>s</b> and add it to <b>router</b>.
+ */
+int
+router_add_exit_policy_from_string(routerinfo_t *router, const char *s)
+{
+  directory_token_t *tok = NULL;
+  const char *cp;
+  char *tmp;
+  int r;
+  int len, idx;
+
+  /* *s might not end with \n, so we need to extend it with one. */
+  len = strlen(s);
+  cp = tmp = tor_malloc(len+2);
+  for (idx = 0; idx < len; ++idx) {
+    tmp[idx] = tolower(s[idx]);
+  }
+  tmp[len]='\n';
+  tmp[len+1]='\0';
+  tok = get_next_token(&cp, RTR_ONLY);
+  if (tok->tp == _ERR) {
+    log_fn(LOG_WARN, "Error reading exit policy: %s", tok->error);
+    goto err;
+  }
+  if (tok->tp != K_ACCEPT && tok->tp != K_REJECT) {
+    log_fn(LOG_WARN, "Expected 'accept' or 'reject'.");
+    goto err;
+  }
+
+  /* Now that we've gotten an exit policy, add it to the router. */
+  r = router_add_exit_policy(router, tok);
+  goto done;
+ err:
+  r = -1;
+ done:
+  free(tmp);
+  token_free(tok);
+  return r;
+}
+
+/** Given a K_ACCEPT or K_REJECT token and a router, create a new exit_policy_t
+ * corresponding to the token, and add it to <b>router</b> */
+static int
+router_add_exit_policy(routerinfo_t *router, directory_token_t *tok) {
+
+  struct exit_policy_t *tmpe, *newe;
+  struct in_addr in;
+  char *arg, *address, *mask, *port, *endptr;
+  int bits;
+
+  tor_assert(tok->tp == K_REJECT || tok->tp == K_ACCEPT);
+
+  if (tok->n_args != 1)
+    return -1;
+  arg = tok->args[0];
+
+  newe = tor_malloc_zero(sizeof(struct exit_policy_t));
+
+  newe->string = tor_malloc(8+strlen(arg));
+  if (tok->tp == K_REJECT) {
+    strcpy(newe->string, "reject ");
+    newe->policy_type = EXIT_POLICY_REJECT;
+  } else {
+    strcpy(newe->string, "accept ");
+    newe->policy_type = EXIT_POLICY_ACCEPT;
+  }
+  strcat(newe->string, arg); /* can't overflow */
+
+  address = arg;
+  mask = strchr(arg,'/');
+  port = strchr(mask?mask:arg,':');
+  /* Break 'arg' into separate strings.  'arg' was already strdup'd by
+   * _router_get_next_token, so it's safe to modify.
+   */
+  if (mask)
+    *mask++ = 0;
+  if (port)
+    *port++ = 0;
+
+  if (strcmp(address, "*") == 0) {
+    newe->addr = 0;
+  } else if (tor_inet_aton(address, &in) != 0) {
+    newe->addr = ntohl(in.s_addr);
+  } else {
+    log_fn(LOG_WARN, "Malformed IP %s in exit policy; rejecting.",
+           address);
+    goto policy_read_failed;
+  }
+  if (!mask) {
+    if (strcmp(address, "*") == 0)
+      newe->msk = 0;
+    else
+      newe->msk = 0xFFFFFFFFu;
+  } else {
+    endptr = NULL;
+    bits = (int) strtol(mask, &endptr, 10);
+    if (!*endptr) {
+      /* strtol handled the whole mask. */
+      newe->msk = ~((1<<(32-bits))-1);
+    } else if (tor_inet_aton(mask, &in) != 0) {
+      newe->msk = ntohl(in.s_addr);
+    } else {
+      log_fn(LOG_WARN, "Malformed mask %s on exit policy; rejecting.",
+             mask);
+      goto policy_read_failed;
+    }
+  }
+  if (!port || strcmp(port, "*") == 0) {
+    newe->prt_min = 0;
+    newe->prt_max = 65535;
+  } else {
+    endptr = NULL;
+    newe->prt_min = (uint16_t) strtol(port, &endptr, 10);
+    if (*endptr == '-') {
+      port = endptr+1;
+      endptr = NULL;
+      newe->prt_max = (uint16_t) strtol(port, &endptr, 10);
+      if (*endptr) {
+      log_fn(LOG_WARN, "Malformed port %s on exit policy; rejecting.",
+             port);
+      }
+    } else if (*endptr) {
+      log_fn(LOG_WARN, "Malformed port %s on exit policy; rejecting.",
+             port);
+      goto policy_read_failed;
+    } else {
+      newe->prt_max = newe->prt_min;
+    }
+  }
+
+  in.s_addr = htonl(newe->addr);
+  address = tor_strdup(inet_ntoa(in));
+  in.s_addr = htonl(newe->msk);
+  log_fn(LOG_DEBUG,"%s %s/%s:%d-%d",
+         newe->policy_type == EXIT_POLICY_REJECT ? "reject" : "accept",
+         address, inet_ntoa(in), newe->prt_min, newe->prt_max);
+  tor_free(address);
+
+  /* now link newe onto the end of exit_policy */
+
+  if(!router->exit_policy) {
+    router->exit_policy = newe;
+    return 0;
+  }
+
+  for(tmpe=router->exit_policy; tmpe->next; tmpe=tmpe->next) ;
+  tmpe->next = newe;
+
+  return 0;
+
+policy_read_failed:
+  tor_assert(newe->string);
+  log_fn(LOG_WARN,"Couldn't parse line '%s'. Dropping", newe->string);
+  tor_free(newe->string);
+  free(newe);
+  return -1;
+}
+
+/*
+ * Low-level tokenizer for router descriptors and directories.
+ */
+
+
+/** Free all resources allocated for <b>tok</b> */
+static void
+token_free(directory_token_t *tok)
+{
+  int i;
+  tor_assert(tok);
+  if (tok->args) {
+    for (i = 0; i < tok->n_args; ++i) {
+      tor_free(tok->args[i]);
+    }
+    tor_free(tok->args);
+  }
+  tor_free(tok->object_type);
+  tor_free(tok->object_body);
+  if (tok->key)
+    crypto_free_pk_env(tok->key);
+  tor_free(tok);
+}
+
+/** Helper function: read the next token from *s, advance *s to the end
+ * of the token, and return the parsed token.  If 'where' is DIR_ONLY
+ * or RTR_ONLY, reject all tokens of the wrong type.
+ */
+static directory_token_t *
+get_next_token(const char **s, where_syntax where) {
+  const char *next, *obstart;
+  int i, done, allocated;
+  directory_token_t *tok;
+  arg_syntax a_syn;
+  obj_syntax o_syn = NO_OBJ;
+
+#define RET_ERR(msg)                                    \
+  do { if (tok) token_free(tok);                        \
+       tok = tor_malloc_zero(sizeof(directory_token_t));\
+       tok->tp = _ERR;                                  \
+       tok->error = msg;                                \
+       goto done_tokenizing; } while (0)
+
+  tok = tor_malloc_zero(sizeof(directory_token_t));
+  tok->tp = _ERR;
+
+  *s = eat_whitespace(*s);
+  if (!**s) {
+    tok->tp = _EOF;
+    return tok;
+  }
+  next = find_whitespace(*s);
+  if (!next) {
+    tok->error = "Unexpected EOF"; return tok;
+  }
+  /* It's a keyword... but which one? */
+  for (i = 0 ; token_table[i].t ; ++i) {
+    if (!strncmp(token_table[i].t, *s, next-*s)) {
+      /* We've found the keyword. */
+      tok->tp = token_table[i].v;
+      a_syn = token_table[i].s;
+      o_syn = token_table[i].os;
+      if (token_table[i].ws != ANY && token_table[i].ws != where) {
+        if (where == DIR_ONLY) {
+          RET_ERR("Found a router-only token in a directory section");
+        } else {
+          RET_ERR("Found a directory-only token in a router descriptor");
+        }
+      }
+      if (a_syn == ARGS) {
+        /* This keyword takes multiple arguments. */
+        i = 0;
+        done = (*next == '\n');
+        allocated = 32;
+        tok->args = tor_malloc(sizeof(char*)*32);
+        *s = eat_whitespace_no_nl(next);
+        while (**s != '\n' && !done) {
+          next = find_whitespace(*s);
+          if (*next == '\n')
+            done = 1;
+          if (i == allocated) {
+            allocated *= 2;
+            tok->args = tor_realloc(tok->args,sizeof(char*)*allocated);
+          }
+          tok->args[i++] = tor_strndup(*s,next-*s);
+          *s = eat_whitespace_no_nl(next+1);
+        }
+        tok->n_args = i;
+      } else if (a_syn == CONCAT_ARGS) {
+        /* The keyword takes the line as a single argument */
+        *s = eat_whitespace_no_nl(next);
+        next = strchr(*s, '\n');
+        if (!next)
+          RET_ERR("Unexpected EOF");
+        tok->args = tor_malloc(sizeof(char*));
+        tok->args[0] = tor_strndup(*s,next-*s);
+        tok->n_args = 1;
+        *s = eat_whitespace_no_nl(next+1);
+      } else {
+        /* The keyword takes no arguments. */
+        tor_assert(a_syn == NO_ARGS);
+        *s = eat_whitespace_no_nl(next);
+        if (**s != '\n') {
+          RET_ERR("Unexpected arguments");
+        }
+        tok->n_args = 0;
+        *s = eat_whitespace_no_nl(*s+1);
+      }
+      break;
+    }
+  }
+  if (tok->tp == _ERR) {
+    tok->tp = _UNRECOGNIZED;
+    next = strchr(*s, '\n');
+    if (!next) {
+      RET_ERR("Unexpected EOF");
+    }
+    tok->args = tor_malloc(sizeof(char*));
+    tok->args[0] = tor_strndup(*s,next-*s);
+    tok->n_args = 1;
+    *s = next+1;
+    o_syn = OBJ_OK;
+  }
+  *s = eat_whitespace(*s);
+  if (strncmp(*s, "-----BEGIN ", 11)) {
+    goto done_tokenizing;
+  }
+  obstart = *s;
+  *s += 11; /* length of "-----BEGIN ". */
+  next = strchr(*s, '\n');
+  if (next-*s < 6 || strncmp(next-5, "-----\n", 6)) {
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = tor_strndup(*s, next-*s-5);
+  *s = next+1;
+  next = strstr(*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing end line");
+  }
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) {
+    if (strncmp(next, "-----END RSA PUBLIC KEY-----\n", 29))
+      RET_ERR("Malformed object: mismatched end line");
+    next = strchr(next,'\n')+1;
+    tok->key = crypto_new_pk_env();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, next-obstart))
+      RET_ERR("Couldn't parse public key.");
+    *s = next;
+  } else {
+    tok->object_body = tor_malloc(next-*s); /* really, this is too much RAM. */
+    i = base64_decode(tok->object_body, 256, *s, next-*s);
+    if (i<0) {
+      RET_ERR("Malformed object: bad base64-encoded data");
+    }
+    tok->object_size = i;
+    *s = next + 9; /* length of "-----END ". */
+    i = strlen(tok->object_type);
+    if (strncmp(*s, tok->object_type, i) || strncmp(*s+i, "-----\n", 6)) {
+      RET_ERR("Malformed object: mismatched end tag");
+    }
+    *s += i+6;
+  }
+  switch(o_syn)
+    {
+    case NO_OBJ:
+      if (tok->object_body)
+        RET_ERR("Unexpected object for keyword");
+      if (tok->key)
+        RET_ERR("Unexpected public key for keyword");
+      break;
+    case NEED_OBJ:
+      if (!tok->object_body)
+        RET_ERR("Missing object for keyword");
+      break;
+    case NEED_KEY:
+      if (!tok->key)
+        RET_ERR("Missing publid key for keyword");
+      break;
+    case OBJ_OK:
+      break;
+    }
+
+ done_tokenizing:
+
+#if 0
+  for (i = 0; token_table[i].t ; ++i) {
+    if (token_table[i].v == tok->tp) {
+      fputs(token_table[i].t, stdout);
+      break;
+      i = -1;
+    }
+  }
+  if (i) {
+    if (tok->tp == _UNRECOGNIZED) fputs("UNRECOGNIZED", stdout);
+    if (tok->tp == _ERR) fputs("ERR",stdout);
+    if (tok->tp == _EOF) fputs("EOF",stdout);
+    if (tok->tp == _NIL) fputs("_NIL",stdout);
+  }
+  for(i = 0; i < tok->n_args; ++i) {
+    fprintf(stdout," \"%s\"", tok->args[i]);
+  }
+  if (tok->error) { fprintf(stdout," *%s*", tok->error); }
+  fputs("\n",stdout);
+#endif
+
+
+  return tok;
+#undef RET_ERR
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  If <b>is_dir</b> is true, reject all non-directory
+ * tokens; else reject all non-routerdescriptor tokens.
+ */
+static int
+tokenize_string(const char *start, const char *end, smartlist_t *out,
+                int is_dir)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  where_syntax where = is_dir ? DIR_ONLY : RTR_ONLY;
+  s = &start;
+  while (*s < end && (!tok || tok->tp != _EOF)) {
+    tok = get_next_token(s, where);
+    if (tok->tp == _ERR) {
+      log_fn(LOG_WARN, "parse error: %s", tok->error);
+      return -1;
+    }
+    smartlist_add(out, tok);
+    *s = eat_whitespace(*s);
+  }
+
+  return 0;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+static directory_token_t *
+find_first_by_keyword(smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** Return a newly allocated smartlist of all accept or reject tokens in
+ * <b>s</b>.
+ */
+static smartlist_t *
+find_all_exitpolicy(smartlist_t *s)
+{
+  smartlist_t *out = smartlist_create();
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == K_ACCEPT || t->tp == K_REJECT)
+                      smartlist_add(out,t));
+  return out;
+}
+
+/** Compute the SHA digest of the substring of <b>s</b> taken from the first
+ * occurrence of <b>start_str</b> through the first newline after the first
+ * subsequent occurrence of <b>end_str</b>; store the 20-byte result in
+ * <b>digest</b>; return 0 on success.
+ *
+ * If no such substring exists, return -1.
+ */
+static int router_get_hash_impl(const char *s, char *digest,
+                                const char *start_str,
+                                const char *end_str)
+{
+  char *start, *end;
+  start = strstr(s, start_str);
+  if (!start) {
+    log_fn(LOG_WARN,"couldn't find \"%s\"",start_str);
+    return -1;
+  }
+  end = strstr(start+strlen(start_str), end_str);
+  if (!end) {
+    log_fn(LOG_WARN,"couldn't find \"%s\"",end_str);
+    return -1;
+  }
+  end = strchr(end, '\n');
+  if (!end) {
+    log_fn(LOG_WARN,"couldn't find EOL");
+    return -1;
+  }
+  ++end;
+
+  if (crypto_digest(start, end-start, digest)) {
+    log_fn(LOG_WARN,"couldn't compute digest");
+    return -1;
+  }
+
+  return 0;
+}
+
+/*
+  Local Variables:
+  mode:c
+  indent-tabs-mode:nil
+  c-basic-offset:2
+  End:
+*/
diff --git a/src/or/test.c b/src/or/test.c
index 2e34d2fc4..d0b5fcdcc 100644
--- a/src/or/test.c
+++ b/src/or/test.c
@@ -20,8 +20,6 @@ extern or_options_t options;
 int have_failed = 0;
 
 /* These functions are file-local, but are exposed so we can test. */
-int router_get_routerlist_from_directory_impl(
-        const char *s, routerlist_t **dest, crypto_pk_env_t *pkey);
 void add_fingerprint_to_dir(const char *nickname, const char *fp);
 void get_platform_str(char *platform, int len);
 
@@ -727,7 +725,7 @@ test_dir_format()
 
   test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0);
   cp = buf;
-  rp1 = router_get_entry_from_string((const char*)cp,NULL);
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL);
   test_assert(rp1);
   test_streq(rp1->address, r1.address);
   test_eq(rp1->or_port, r1.or_port);
@@ -750,7 +748,7 @@ test_dir_format()
   test_streq(buf, buf2);
 
   cp = buf;
-  rp2 = router_get_entry_from_string(&cp);
+  rp2 = router_parse_entry_from_string(&cp);
   test_assert(rp2);
   test_streq(rp2->address, r2.address);
   test_eq(rp2->or_port, r2.or_port);
@@ -787,7 +785,7 @@ test_dir_format()
   options.Nickname = "DirServer";
   test_assert(!dirserv_dump_directory_to_string(buf,8192,pk3));
   cp = buf;
-  test_assert(!router_get_routerlist_from_directory_impl(buf, &dir1, pk3));
+  test_assert(!router_parse_routerlist_from_directory(buf, &dir1, pk3));
   test_eq(2, smartlist_len(dir1->routers));
   dirserv_free_fingerprint_list();