r15400@catbus: nickm | 2007-09-26 12:13:12 -0400

 Backend for descriptor annotations: parse annotations as keywords; only allow them to appear in the cache; do not serve them as part of the descriptor if we are a dirserver.  Still need mechanism to set annotations.  Still need to rename cache file.


svn:r11654

6 files changed, 141 insertions, 97 deletions

diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index b0bc210f1..c55e8cd34 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -536,7 +536,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
 
   s = desc;
   list = smartlist_create();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0)) {
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0)) {
     SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
         msg_out = NULL;
 
@@ -561,7 +561,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   smartlist_clear(list);
 
   s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1)) {
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0)) {
     SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
         msg_out = NULL;
 
diff --git a/src/or/or.h b/src/or/or.h
index dd098fa96..780f537bf 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -20,23 +20,15 @@
 #define WIN32_LEAN_AND_MEAN
 #endif
 
-// #include <stdio.h>
-// #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
-//#ifdef HAVE_STRING_H
-//#include <string.h>
-//#endif
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
 #endif
-//#ifdef HAVE_CTYPE_H
-//#include <ctype.h>
-//#endif
 #ifdef HAVE_SYS_PARAM_H
 #include <sys/param.h> /* FreeBSD needs this to know what version it is */
 #endif
@@ -53,21 +45,12 @@
 #ifdef HAVE_SYS_IOCTL_H
 #include <sys/ioctl.h>
 #endif
-//#ifdef HAVE_SYS_SOCKET_H
-//#include <sys/socket.h>
-//#endif
 #ifdef HAVE_SYS_UN_H
 #include <sys/un.h>
 #endif
-//#ifdef HAVE_SYS_TIME_H
-//#include <sys/time.h>
-//#endif
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-//#ifdef HAVE_NETINET_IN_H
-//#include <netinet/in.h>
-//#endif
 #ifdef HAVE_ARPA_INET_H
 #include <arpa/inet.h>
 #endif
@@ -91,25 +74,6 @@
 #define MAXCONNECTIONS 15000
 #endif
 
-#if 0
-#ifdef MS_WINDOWS
-/* No, we don't need to redefine FD_SETSIZE before including winsock:
- * we use libevent now, and libevent handles the select() stuff.  Yes,
- * some documents imply that we need to redefine anyway if we're using
- * select() anywhere in our application or in anything it links to: these
- * documents are either the holy texts of a cargo cult of network
- * programmers, or more likely a simplification of what's going on for
- * people who haven't read winsock[2].h for themselves.
- */
-#if defined(_MSC_VER) && (_MSC_VER <= 1300)
-#include <winsock.h>
-#else
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#endif
-#endif
-#endif
-
 #ifdef MS_WINDOWS
 #include <io.h>
 #include <process.h>
@@ -1118,9 +1082,12 @@ typedef struct download_status_t {
 
 /** Information need to cache an onion router's descriptor. */
 typedef struct signed_descriptor_t {
-  /** Pointer to the raw server descriptor.  Not necessarily NUL-terminated.
-   * If saved_location is SAVED_IN_CACHE, this pointer is null.  */
+  /** Pointer to the raw server descriptor, preceeded by annotatinos.  Not
+   * necessarily NUL-terminated.  If saved_location is SAVED_IN_CACHE, this
+   * pointer is null.  */
   char *signed_descriptor_body;
+  /** Length of the annotations preceeding the server descriptor. */
+  size_t annotations_len;
   /** Length of the server descriptor. */
   size_t signed_descriptor_len;
   /** Digest of the server descriptor, computed as specified in dir-spec.txt */
@@ -1135,7 +1102,7 @@ typedef struct signed_descriptor_t {
    * extrainfo. */
   download_status_t ei_dl_status;
   /** Where is the descriptor saved? */
-  saved_location_t saved_location ;
+  saved_location_t saved_location;
   /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
    * this descriptor in the corresponding file. */
   off_t saved_offset;
@@ -2894,10 +2861,6 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id(
                                        const char *identity);
 int networkstatus_check_consensus_signature(networkstatus_vote_t *consensus,
                                             int warn);
-#if 0
-int networkstatus_add_consensus_signatures(networkstatus_vote_t *target,
-                                           networkstatus_vote_t *src);
-#endif
 int networkstatus_add_detached_signatures(networkstatus_vote_t *target,
                                           ns_detached_signatures_t *sigs);
 char *networkstatus_get_detached_signatures(networkstatus_vote_t *consensus);
@@ -3639,7 +3602,8 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
 int router_parse_list_from_string(const char **s, const char *eos,
                                   smartlist_t *dest,
                                   saved_location_t saved_location,
-                                  int is_extrainfo);
+                                  int is_extrainfo,
+                                  int allow_annotations);
 int router_parse_routerlist_from_directory(const char *s,
                                            routerlist_t **dest,
                                            crypto_pk_env_t *pkey,
@@ -3648,7 +3612,8 @@ int router_parse_routerlist_from_directory(const char *s,
 int router_parse_runningrouters(const char *str);
 int router_parse_directory(const char *str);
 routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
-                                             int cache_copy);
+                                             int cache_copy,
+                                             int allow_annotations);
 extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end,
                          int cache_copy, struct digest_ri_map_t *routermap);
 addr_policy_t *router_parse_addr_policy_from_string(const char *s,
diff --git a/src/or/router.c b/src/or/router.c
index 57625fadb..5358ccdf9 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -485,7 +485,7 @@ init_keys(void)
       return -1;
     }
     if (mydesc) {
-      ri = router_parse_entry_from_string(mydesc, NULL, 1);
+      ri = router_parse_entry_from_string(mydesc, NULL, 1, 0);
       if (!ri) {
         log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse.");
         return -1;
@@ -1632,7 +1632,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
     const char *cp;
     routerinfo_t *ri_tmp;
     cp = s_dup = tor_strdup(s);
-    ri_tmp = router_parse_entry_from_string(cp, NULL, 1);
+    ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0);
     if (!ri_tmp) {
       log_err(LD_BUG,
               "We just generated a router descriptor we can't parse.");
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 02c6c851f..0ed2c0c48 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -43,6 +43,8 @@ static local_routerstatus_t *router_get_combined_status_by_nickname(
                                                 int warn_if_unnamed);
 static void update_router_have_minimum_dir_info(void);
 static void router_dir_info_changed(void);
+static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc,
+                                                   int with_annotations);
 
 DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
 DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
@@ -522,8 +524,8 @@ signed_desc_append_to_journal(signed_descriptor_t *desc,
   or_options_t *options = get_options();
   size_t fname_len = strlen(options->DataDirectory)+32;
   char *fname;
-  const char *body = signed_descriptor_get_body(desc);
-  size_t len = desc->signed_descriptor_len;
+  const char *body = signed_descriptor_get_body_impl(desc,1);
+  size_t len = desc->signed_descriptor_len + desc->annotations_len;
 
   if (purpose != ROUTER_PURPOSE_GENERAL &&
       purpose != EXTRAINFO_PURPOSE_GENERAL) {
@@ -629,7 +631,7 @@ router_rebuild_store(int force, desc_store_t *store)
   SMARTLIST_FOREACH(signed_descriptors, signed_descriptor_t *, sd,
     {
       sized_chunk_t *c;
-      const char *body = signed_descriptor_get_body(sd);
+      const char *body = signed_descriptor_get_body_impl(sd, 1);
       if (!body) {
         log_warn(LD_BUG, "No descriptor available for router.");
         goto done;
@@ -638,7 +640,7 @@ router_rebuild_store(int force, desc_store_t *store)
         continue;
       c = tor_malloc(sizeof(sized_chunk_t));
       c->bytes = body;
-      c->len = sd->signed_descriptor_len;
+      c->len = sd->signed_descriptor_len + sd->annotations_len;
       smartlist_add(chunk_list, c);
     });
 
@@ -1965,21 +1967,29 @@ extrainfo_get_by_descriptor_digest(const char *digest)
 /** Return a pointer to the signed textual representation of a descriptor.
  * The returned string is not guaranteed to be NUL-terminated: the string's
  * length will be in desc-\>signed_descriptor_len. */
-const char *
-signed_descriptor_get_body(signed_descriptor_t *desc)
+static const char *
+signed_descriptor_get_body_impl(signed_descriptor_t *desc,
+                                int with_annotations)
 {
   const char *r = NULL;
   size_t len = desc->signed_descriptor_len;
+  off_t offset = desc->saved_offset;
+  if (with_annotations)
+    len += desc->annotations_len;
+  else
+    offset += desc->annotations_len;
+
   tor_assert(len > 32);
   if (desc->saved_location == SAVED_IN_CACHE && routerlist) {
     desc_store_t *store = desc_get_store(router_get_routerlist(), desc);
     if (store && store->mmap) {
       tor_assert(desc->saved_offset + len <= store->mmap->size);
-      r = store->mmap->data + desc->saved_offset;
+      r = store->mmap->data + offset;
     }
   }
   if (!r) /* no mmap, or not in cache. */
-    r = desc->signed_descriptor_body;
+    r = desc->signed_descriptor_body +
+      (with_annotations ? 0 : desc->annotations_len);
 
   tor_assert(r);
   if (memcmp("router ", r, 7) && memcmp("extra-info ", r, 11)) {
@@ -1991,6 +2001,13 @@ signed_descriptor_get_body(signed_descriptor_t *desc)
   return r;
 }
 
+/** DOCDOC */
+const char *
+signed_descriptor_get_body(signed_descriptor_t *desc)
+{
+  return signed_descriptor_get_body_impl(desc, 0);
+}
+
 /** Return the current list of all known routers. */
 routerlist_t *
 router_get_routerlist(void)
@@ -3089,7 +3106,7 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
   tor_assert(msg);
   *msg = NULL;
 
-  if (!(ri = router_parse_entry_from_string(s, NULL, 1))) {
+  if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0))) {
     log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
     *msg = "Couldn't parse router descriptor.";
     return -1;
@@ -3147,8 +3164,10 @@ router_load_routers_from_string(const char *s, const char *eos,
   char fp[HEX_DIGEST_LEN+1];
   const char *msg;
   int from_cache = (saved_location != SAVED_NOWHERE);
+  int allow_annotations = (saved_location != SAVED_NOWHERE);
 
-  router_parse_list_from_string(&s, eos, routers, saved_location, 0);
+  router_parse_list_from_string(&s, eos, routers, saved_location, 0,
+                                allow_annotations);
 
   routers_update_status_from_networkstatus(routers, !from_cache);
 
@@ -3209,7 +3228,7 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
   const char *msg;
   int from_cache = (saved_location != SAVED_NOWHERE);
 
-  router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1);
+  router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0);
 
   log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list));
 
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index ef87caee7..34111a76b 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -1,4 +1,4 @@
-/* oCpyright (c) 2001 Matej Pfajfar.
+/* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2007, Roger Dingledine, Nick Mathewson. */
 /* See LICENSE for licensing information */
@@ -15,11 +15,9 @@ const char routerparse_c_id[] =
 
 /****************************************************************************/
 
-/** Enumeration of possible token types.  The ones starting with K_
- * correspond to directory 'keywords'.  _UNRECOGNIZED is for an
- * unrecognized keyword; _ERR is an error in the tokenizing process,
- * _EOF is an end-of-file marker, and _NIL is used to encode
- * not-a-token.
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+ * to directory 'keywords'. _ERR is an error in the tokenizing process, _EOF
+ * is an end-of-file marker, and _NIL is used to encode not-a-token.
  */
 typedef enum {
   K_ACCEPT = 0,
@@ -77,12 +75,18 @@ typedef enum {
   K_CONSENSUS_DIGEST,
   K_CONSENSUS_METHODS,
 
+  A_PURPOSE,
+  _A_UNKNOWN,
+
   _UNRECOGNIZED,
   _ERR,
   _EOF,
   _NIL
 } directory_keyword;
 
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION _A_UNKNOWN
+
 /** Structure to hold a single directory token.
  *
  * We parse a directory by breaking it into "tokens", each consisting
@@ -139,6 +143,8 @@ typedef struct token_rule_t {
   /** One or more of AT_START/AT_END to limit where the item may appear in a
    * document. */
   int pos;
+  /** DOCDOC */
+  int is_annotation;
 } token_rule_t;
 
 /*
@@ -149,21 +155,23 @@ typedef struct token_rule_t {
  */
 
 /** Appears to indicate the end of a table. */
-#define END_OF_TABLE { NULL, _NIL, 0,0,0, NO_OBJ, 0, INT_MAX, 0 }
+#define END_OF_TABLE { NULL, _NIL, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
 /** An item with no restrictions: used for obsolete document types */
-#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0 }
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
 /** An item with no restrictions on multiplicity or location. */
-#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0 }
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
 /** An item that must appear exactly once */
-#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0 }
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
 /** An item that must appear exactly once, at the start of the document */
-#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START }
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
 /** An item that must appear exactly once, at the end of the document */
-#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END }
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
 /** An item that must appear one or more times */
-#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0 }
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
 /** An item that must appear no more than once */
-#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0 }
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
 
 /* Argument multiplicity: any number of arguments. */
 #define ARGS        0,INT_MAX,0
@@ -200,6 +208,7 @@ static token_rule_t routerdesc_token_table[] = {
 
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
+  A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
 
   END_OF_TABLE
 };
@@ -365,7 +374,8 @@ static directory_token_t *find_first_by_keyword(smartlist_t *s,
                                                 directory_keyword keyword);
 static int tokenize_string(const char *start, const char *end,
                            smartlist_t *out,
-                           token_rule_t *table);
+                           token_rule_t *table,
+                           int allow_annotations);
 static directory_token_t *get_next_token(const char **s,
                                          const char *eos,
                                          token_rule_t *table);
@@ -614,7 +624,7 @@ router_parse_directory(const char *str)
   }
   ++cp;
   tokens = smartlist_create();
-  if (tokenize_string(cp,strchr(cp,'\0'),tokens,dir_token_table)) {
+  if (tokenize_string(cp,strchr(cp,'\0'),tokens,dir_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing directory signature"); goto err;
   }
   if (smartlist_len(tokens) != 1) {
@@ -643,7 +653,7 @@ router_parse_directory(const char *str)
   }
 
   tokens = smartlist_create();
-  if (tokenize_string(str,end,tokens,dir_token_table)) {
+  if (tokenize_string(str,end,tokens,dir_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing directory"); goto err;
   }
 
@@ -692,7 +702,7 @@ router_parse_runningrouters(const char *str)
     goto err;
   }
   tokens = smartlist_create();
-  if (tokenize_string(str,eos,tokens,dir_token_table)) {
+  if (tokenize_string(str,eos,tokens,dir_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing running-routers"); goto err;
   }
   tok = smartlist_get(tokens,0);
@@ -858,7 +868,8 @@ int
 router_parse_list_from_string(const char **s, const char *eos,
                               smartlist_t *dest,
                               saved_location_t saved_location,
-                              int want_extrainfo)
+                              int want_extrainfo,
+                              int allow_annotations)
 {
   routerinfo_t *router;
   extrainfo_t *extrainfo;
@@ -923,7 +934,8 @@ router_parse_list_from_string(const char **s, const char *eos,
       }
     } else if (!have_extrainfo && !want_extrainfo) {
       router = router_parse_entry_from_string(*s, end,
-                                          saved_location != SAVED_IN_CACHE);
+                                              saved_location != SAVED_IN_CACHE,
+                                              allow_annotations);
       if (router) {
         signed_desc = &router->cache_info;
         elt = router;
@@ -977,13 +989,14 @@ dump_distinct_digest_count(int severity)
  */
 routerinfo_t *
 router_parse_entry_from_string(const char *s, const char *end,
-                               int cache_copy)
+                               int cache_copy, int allow_annotations)
 {
   routerinfo_t *router = NULL;
   char digest[128];
   smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
   directory_token_t *tok;
   struct in_addr in;
+  const char *start_of_annotations, *cp;
 
   if (!end) {
     end = s + strlen(s);
@@ -993,12 +1006,23 @@ router_parse_entry_from_string(const char *s, const char *end,
   while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
     --end;
 
+  start_of_annotations = s;
+  cp = tor_memstr(s, end-s, "\nrouter ");
+  if (!cp) {
+    if (end-s < 7 || strcmpstart(s, "router ")) {
+      log_warn(LD_DIR, "No router keyword found.");
+      goto err;
+    }
+  } else {
+    s = cp+1;
+  }
+
   if (router_get_router_hash(s, digest) < 0) {
     log_warn(LD_DIR, "Couldn't compute router hash.");
     return NULL;
   }
   tokens = smartlist_create();
-  if (tokenize_string(s,end,tokens,routerdesc_token_table)) {
+  if (tokenize_string(s,end,tokens,routerdesc_token_table,allow_annotations)) {
     log_warn(LD_DIR, "Error tokenizing router descriptor.");
     goto err;
   }
@@ -1019,6 +1043,7 @@ router_parse_entry_from_string(const char *s, const char *end,
   router->routerlist_index = -1;
   if (cache_copy)
     router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
+  router->cache_info.signed_descriptor_len = s-start_of_annotations;
   router->cache_info.signed_descriptor_len = end-s;
   memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
 
@@ -1218,13 +1243,13 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
     return NULL;
   }
   tokens = smartlist_create();
-  if (tokenize_string(s,end,tokens,extrainfo_token_table)) {
-    log_warn(LD_DIR, "Error tokenizing router descriptor.");
+  if (tokenize_string(s,end,tokens,extrainfo_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing extra-info document.");
     goto err;
   }
 
   if (smartlist_len(tokens) < 2) {
-    log_warn(LD_DIR, "Impossibly short router descriptor.");
+    log_warn(LD_DIR, "Impossibly short extra-info document.");
     goto err;
   }
 
@@ -1328,7 +1353,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
   len = eos - s;
 
   tokens = smartlist_create();
-  if (tokenize_string(s, eos, tokens, dir_key_certificate_table) < 0) {
+  if (tokenize_string(s, eos, tokens, dir_key_certificate_table,0) < 0) {
     log_warn(LD_DIR, "Error tokenizing key certificate");
     goto err;
   }
@@ -1468,7 +1493,7 @@ routerstatus_parse_entry_from_string(const char **s, smartlist_t *tokens,
 
   eos = find_start_of_next_routerstatus(*s);
 
-  if (tokenize_string(*s, eos, tokens, rtrstatus_token_table)) {
+  if (tokenize_string(*s, eos, tokens, rtrstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing router status");
     goto err;
   }
@@ -1640,7 +1665,7 @@ networkstatus_parse_from_string(const char *s)
   }
 
   eos = find_start_of_next_routerstatus(s);
-  if (tokenize_string(s, eos, tokens, netstatus_token_table)) {
+  if (tokenize_string(s, eos, tokens, netstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing network-status header.");
     goto err;
   }
@@ -1752,7 +1777,7 @@ networkstatus_parse_from_string(const char *s)
   smartlist_uniq(ns->entries, _compare_routerstatus_entries,
                  _free_duplicate_routerstatus_entry);
 
-  if (tokenize_string(s, NULL, footer_tokens, dir_footer_token_table)) {
+  if (tokenize_string(s, NULL, footer_tokens, dir_footer_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing network-status footer.");
     goto err;
   }
@@ -1812,7 +1837,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   end_of_header = find_start_of_next_routerstatus(s);
   if (tokenize_string(s, end_of_header, tokens,
                       is_vote ? networkstatus_vote_token_table :
-                                networkstatus_consensus_token_table)) {
+                                networkstatus_consensus_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote header.");
     goto err;
   }
@@ -2032,7 +2057,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   else
     end_of_footer = s + strlen(s);
   if (tokenize_string(s, end_of_footer, footer_tokens,
-                      networkstatus_vote_footer_token_table)) {
+                      networkstatus_vote_footer_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
     goto err;
   }
@@ -2143,7 +2168,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos)
     eos = s + strlen(s);
 
   if (tokenize_string(s, eos, tokens,
-                      networkstatus_detached_signature_token_table)) {
+                      networkstatus_detached_signature_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing detached networkstatus signatures");
     goto err;
   }
@@ -2569,8 +2594,12 @@ get_next_token(const char **s, const char *eos, token_rule_t *table)
     }
   }
 
-  if (tok->tp == _ERR) { /* No keyword matched; call it an "opt". */
-    tok->tp = K_OPT;
+  if (tok->tp == _ERR) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (**s == '@')
+      tok->tp = _A_UNKNOWN;
+    else
+      tok->tp = K_OPT;
     tok->args = tor_malloc(sizeof(char*));
     tok->args[0] = tor_strndup(*s, eol-*s);
     tok->n_args = 1;
@@ -2637,12 +2666,13 @@ get_next_token(const char **s, const char *eos, token_rule_t *table)
  */
 static int
 tokenize_string(const char *start, const char *end, smartlist_t *out,
-                token_rule_t *table)
+                token_rule_t *table, int allow_annotations)
 {
   const char **s;
   directory_token_t *tok = NULL;
   int counts[_NIL];
   int i;
+  int first_nonannotation;
 
   s = &start;
   if (!end)
@@ -2661,6 +2691,36 @@ tokenize_string(const char *start, const char *end, smartlist_t *out,
     *s = eat_whitespace_eos(*s, end);
   }
 
+  if (allow_annotations) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
   for (i = 0; table[i].t; ++i) {
     if (counts[table[i].v] < table[i].min_cnt) {
       log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
@@ -2672,7 +2732,7 @@ tokenize_string(const char *start, const char *end, smartlist_t *out,
     }
     if (table[i].pos & AT_START) {
       if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, 0))->tp != table[i].v) {
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
         log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
         return -1;
       }
diff --git a/src/or/test.c b/src/or/test.c
index 96578d41c..f9d817d10 100644
--- a/src/or/test.c
+++ b/src/or/test.c
@@ -2189,7 +2189,7 @@ test_dir_format(void)
 
   test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0);
   cp = buf;
-  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1);
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0);
   test_assert(rp1);
   test_streq(rp1->address, r1.address);
   test_eq(rp1->or_port, r1.or_port);