From d61b5df9c1bed57cb39888a1f256cf6c234c29eb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 16:05:08 -0400 Subject: Fix various bugs in microdescriptor caching. --- src/or/microdesc.c | 105 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 22 deletions(-) (limited to 'src/or/microdesc.c') diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0128fbbab..2533564dd 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -40,19 +40,28 @@ HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, _tor_malloc, _tor_realloc, _tor_free); +/* returns n bytes written */ static int -dump_microdescriptor(FILE *f, microdesc_t *md) +dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) { + int r = 0; /* XXXX drops unkown annotations. */ if (md->last_listed) { char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; format_iso_time(buf, md->last_listed); - fprintf(f, "@last-listed %s\n", buf); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; } md->off = (off_t) ftell(f); fwrite(md->body, 1, md->bodylen, f); - return 0; + r += md->bodylen; + return r; } static microdesc_cache_t *the_microdesc_cache = NULL; @@ -72,7 +81,7 @@ get_microdesc_cache(void) } /* There are three sources of microdescriptors: - 1) Generated us while acting as a directory authority. + 1) Generated by us while acting as a directory authority. 2) Loaded from the cache on disk. 3) Downloaded. */ @@ -98,7 +107,8 @@ microdescs_add_to_cache(microdesc_cache_t *cache, return added; } -/* Returns list of added microdesc_t. Frees any not added. */ +/* Returns list of added microdesc_t. Frees any not added. Updates last_listed. + */ smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, @@ -108,13 +118,17 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, open_file_t *open_file = NULL; FILE *f = NULL; // int n_added = 0; + size_t size = 0; if (where == SAVED_NOWHERE && !no_save) { - f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND, + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, 0600, &open_file); - if (!f) - log_warn(LD_DIR, "Couldn't append to journal in %s", - cache->journal_fname); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } } added = smartlist_create(); @@ -131,8 +145,10 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, /* Okay, it's a new one. */ if (f) { - dump_microdescriptor(f, md); + int annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; } else { md->saved_location = where; } @@ -143,7 +159,18 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_add(added, md); } SMARTLIST_FOREACH_END(md); - finish_writing_to_file(open_file); /*XXX Check me.*/ + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + return added; } @@ -152,9 +179,11 @@ microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); - microdesc_free(*entry); + microdesc_free(md); } + HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { tor_munmap_file(cache->cache_content); cache->cache_content = NULL; @@ -176,8 +205,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) if (mm) { added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } } journal_content = read_file_to_str(cache->journal_fname, @@ -186,8 +217,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } tor_free(journal_content); } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", @@ -202,8 +235,16 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) FILE *f; microdesc_t **mdp; smartlist_t *wrote; + int size; + off_t off = 0; + int orig_size, new_size; - f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE, + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, 0600, &open_file); if (!f) return -1; @@ -212,15 +253,17 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) HT_FOREACH(mdp, microdesc_map, &cache->map) { microdesc_t *md = *mdp; + int annotation_len; if (md->no_save) continue; - dump_microdescriptor(f, md); + size = dump_microdescriptor(f, md, &annotation_len); + md->off = off + annotation_len; + off += size; if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); md->saved_location = SAVED_IN_CACHE; } - smartlist_add(wrote, md); } @@ -229,21 +272,29 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) if (cache->cache_content) tor_munmap_file(cache->cache_content); cache->cache_content = tor_mmap_file(cache->cache_fname); + if (!cache->cache_content && smartlist_len(wrote)) { log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", cache->cache_fname); + smartlist_free(wrote); return -1; } SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { - if (md->no_save) - continue; tor_assert(md->saved_location == SAVED_IN_CACHE); md->body = (char*)cache->cache_content->data + md->off; tor_assert(!memcmp(md->body, "onion-key", 9)); - } SMARTLIST_FOREACH_END(wrote); + } SMARTLIST_FOREACH_END(md); smartlist_free(wrote); + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + return 0; } @@ -265,3 +316,13 @@ microdesc_free(microdesc_t *md) tor_free(md); } +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} -- cgit v1.2.3