diff options
Diffstat (limited to 'src/or')
43 files changed, 1521 insertions, 879 deletions
diff --git a/src/or/Makefile.am b/src/or/Makefile.am index a95b9eb92..a9ac3cdee 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -82,5 +82,7 @@ or_sha1.i: $(tor_SOURCES) touch or_sha1.i; \ fi +CLEANFILES = micro-revision.i + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 3ba2760bf..e8422637c 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -270,14 +270,25 @@ buf_shrink_freelists(int free_all) (freelists[i].lowest_length - slack); int n_to_skip = freelists[i].cur_length - n_to_free; int orig_n_to_free = n_to_free, n_freed=0; + int orig_n_to_skip = n_to_skip; int new_length = n_to_skip; chunk_t **chp = &freelists[i].head; chunk_t *chunk; - log_info(LD_MM, "Cleaning freelist for %d-byte chunks: keeping %d, " - "dropping %d.", - (int)freelists[i].alloc_size, n_to_skip, n_to_free); + log_info(LD_MM, "Cleaning freelist for %d-byte chunks: length %d, " + "keeping %d, dropping %d.", + (int)freelists[i].alloc_size, freelists[i].cur_length, + n_to_skip, n_to_free); + tor_assert(n_to_skip + n_to_free == freelists[i].cur_length); while (n_to_skip) { - tor_assert((*chp)->next); + if (! (*chp)->next) { + log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " + "%d-byte chunks, but only found %d. (Length %d)", + orig_n_to_skip, (int)freelists[i].alloc_size, + orig_n_to_skip-n_to_skip, freelists[i].cur_length); + assert_freelist_ok(&freelists[i]); + return; + } + // tor_assert((*chp)->next); chp = &(*chp)->next; --n_to_skip; } diff --git a/src/or/buffers.h b/src/or/buffers.h index 42d92dd89..8fd403d95 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -23,7 +23,6 @@ void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 5bb9d70d5..5567b246a 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -605,19 +605,19 @@ circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) * after we do so. Use this result to estimate parameters and * calculate the timeout. * - * Returns -1 and sets msg on error. Msg must be freed by the caller. + * Return -1 on error. */ int circuit_build_times_parse_state(circuit_build_times_t *cbt, - or_state_t *state, char **msg) + or_state_t *state) { int tot_values = 0; uint32_t loaded_cnt = 0, N = 0; config_line_t *line; unsigned int i; build_time_t *loaded_times; + int err = 0; circuit_build_times_init(cbt); - *msg = NULL; if (circuit_build_times_disabled()) { return 0; @@ -631,8 +631,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, smartlist_split_string(args, line->value, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (smartlist_len(args) < 2) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Too few arguments to CircuitBuildTime"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -645,8 +646,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, CBT_BUILD_TIME_MAX, &ok, NULL); if (!ok) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Unparsable bin number"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin number"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -654,8 +656,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, count = (uint32_t)tor_parse_ulong(count_str, 0, 0, UINT32_MAX, &ok, NULL); if (!ok) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Unparsable bin count"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin count"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -692,10 +695,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, "Corrupt state file? Build times count mismatch. " "Read %d times, but file says %d", loaded_cnt, state->TotalBuildTimes); - *msg = tor_strdup("Build times count mismatch."); + err = 1; circuit_build_times_reset(cbt); - tor_free(loaded_times); - return -1; + goto done; } circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); @@ -716,10 +718,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, "Corrupt state file? Shuffled build times mismatch. " "Read %d times, but file says %d", tot_values, state->TotalBuildTimes); - *msg = tor_strdup("Build times count mismatch."); + err = 1; circuit_build_times_reset(cbt); - tor_free(loaded_times); - return -1; + goto done; } circuit_build_times_set_timeout(cbt); @@ -728,8 +729,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, circuit_build_times_filter_timeouts(cbt); } + done: tor_free(loaded_times); - return *msg ? -1 : 0; + return err ? -1 : 0; } /** @@ -1512,7 +1514,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int r; -again: + again: r = onion_extend_cpath(circ); if (r < 0) { log_info(LD_CIRC,"Generating cpath hop failed."); @@ -1750,7 +1752,7 @@ circuit_deliver_create_cell(circuit_t *circ, uint8_t cell_type, cell.circ_id = circ->n_circ_id; memcpy(cell.payload, payload, ONIONSKIN_CHALLENGE_LEN); - append_cell_to_circuit_queue(circ, circ->n_conn, &cell, CELL_DIRECTION_OUT); + append_cell_to_circuit_queue(circ, circ->n_conn, &cell, CELL_DIRECTION_OUT, 0); if (CIRCUIT_IS_ORIGIN(circ)) { /* mark it so it gets better rate limiting treatment. */ @@ -2327,7 +2329,7 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, circ->is_first_hop = (cell_type == CELL_CREATED_FAST); append_cell_to_circuit_queue(TO_CIRCUIT(circ), - circ->p_conn, &cell, CELL_DIRECTION_IN); + circ->p_conn, &cell, CELL_DIRECTION_IN, 0); log_debug(LD_CIRC,"Finished sending 'created' cell."); if (!is_local_addr(&circ->p_conn->_base.addr) && diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index 3a02f0420..f4cc2a904 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -81,7 +81,7 @@ extern circuit_build_times_t circ_times; void circuit_build_times_update_state(circuit_build_times_t *cbt, or_state_t *state); int circuit_build_times_parse_state(circuit_build_times_t *cbt, - or_state_t *state, char **msg); + or_state_t *state); void circuit_build_times_count_timeout(circuit_build_times_t *cbt, int did_onehop); int circuit_build_times_count_close(circuit_build_times_t *cbt, @@ -107,8 +107,6 @@ void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, double quantile, double time_ms); int circuit_build_times_update_alpha(circuit_build_times_t *cbt); double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); -void circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, - double quantile_cutoff); void circuitbuild_running_unit_tests(void); void circuit_build_times_reset(circuit_build_times_t *cbt); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index a3f10a884..e9335b18d 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -719,7 +719,7 @@ circuit_expire_old_circuits_clientside(time_t now) } for (circ = global_circuitlist; circ; circ = circ->next) { - if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) + if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. @@ -746,13 +746,23 @@ circuit_expire_old_circuits_clientside(time_t now) (long)(now - circ->timestamp_created)); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { - log_notice(LD_CIRC, - "Ancient non-dirty circuit %d is still around after " - "%ld seconds. Purpose: %d", - TO_ORIGIN_CIRCUIT(circ)->global_identifier, - (long)(now - circ->timestamp_created), - circ->purpose); - TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld seconds. Purpose: %d", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + (long)(now - circ->timestamp_created), + circ->purpose); + /* FFFF implement a new circuit_purpose_to_string() so we don't + * just print out a number for circ->purpose */ + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } } } } @@ -1184,13 +1194,13 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, * as loudly. the user doesn't even know it's happening. */ if (options->UseBridges && bridges_known_but_down()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying known bridges again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying known bridges again."); bridges_retry_all(); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying directory fetches again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying directory fetches again."); routerlist_retry_directory_downloads(time(NULL)); } } diff --git a/src/or/command.c b/src/or/command.c index 0460e25c2..ea0bbea1e 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -288,7 +288,14 @@ command_process_create_cell(cell_t *cell, or_connection_t *conn) /* hand it off to the cpuworkers, and then return. */ if (assign_onionskin_to_cpuworker(NULL, circ, onionskin) < 0) { - log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing."); +#define WARN_HANDOFF_FAILURE_INTERVAL (6*60*60) + static ratelim_t handoff_warning = + RATELIM_INIT(WARN_HANDOFF_FAILURE_INTERVAL); + char *m; + if ((m = rate_limit_log(&handoff_warning, approx_time()))) { + log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing.%s",m); + tor_free(m); + } circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return; } diff --git a/src/or/config.c b/src/or/config.c index a4e4f89c1..6b3bcf6da 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -17,6 +17,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "connection_or.h" #include "control.h" #include "cpuworker.h" #include "dirserv.h" @@ -440,6 +441,12 @@ static config_var_t _state_vars[] = { V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryDirReadEnds, ISOTIME, NULL), + V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirWriteEnds, ISOTIME, NULL), + V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteValues, CSV, ""), V(TorVersion, STRING, NULL), @@ -447,7 +454,7 @@ static config_var_t _state_vars[] = { V(LastWritten, ISOTIME, NULL), V(TotalBuildTimes, UINT, NULL), - V(CircuitBuildAbandonedCount, UINT, "0"), + V(CircuitBuildAbandonedCount, UINT, "0"), VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), @@ -950,10 +957,12 @@ options_act_reversible(or_options_t *old_options, char **msg) } /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) */ - if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { - *msg = tor_strdup("Failed to bind one of the listener ports."); - goto rollback; + * ports under 1024.) We don't want to rebind if we're hibernating. */ + if (!we_are_hibernating()) { + if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { + *msg = tor_strdup("Failed to bind one of the listener ports."); + goto rollback; + } } } @@ -1243,9 +1252,30 @@ options_act(or_options_t *old_options) circuit_expire_all_dirty_circs(); } +/* How long should we delay counting bridge stats after becoming a bridge? + * We use this so we don't count people who used our bridge thinking it is + * a relay. If you change this, don't forget to change the log message + * below. It's 4 hours (the time it takes to stop being used by clients) + * plus some extra time for clock skew. */ +#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60) + if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) { - log_info(LD_GENERAL, "Bridge status changed. Forgetting GeoIP stats."); - geoip_remove_old_clients(time(NULL)+(2*60*60)); + int was_relay = 0; + if (options->BridgeRelay) { + time_t int_start = time(NULL); + if (old_options->ORPort == options->ORPort) { + int_start += RELAY_BRIDGE_STATS_DELAY; + was_relay = 1; + } + geoip_bridge_stats_init(int_start); + log_info(LD_CONFIG, "We are acting as a bridge now. Starting new " + "GeoIP stats interval%s.", was_relay ? " in 6 " + "hours from now" : ""); + } else { + geoip_bridge_stats_term(); + log_info(LD_GENERAL, "We are no longer acting as a bridge. " + "Forgetting GeoIP stats."); + } } if (options_transition_affects_workers(old_options, options)) { @@ -1270,6 +1300,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -1315,6 +1349,40 @@ options_act(or_options_t *old_options) } } + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + time_t now = time(NULL); + if ((!old_options || !old_options->CellStatistics) && + options->CellStatistics) + rep_hist_buffer_stats_init(now); + if ((!old_options || !old_options->DirReqStatistics) && + options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if ((!old_options || !old_options->EntryStatistics) && + options->EntryStatistics) + geoip_entry_stats_init(now); + if ((!old_options || !old_options->ExitPortStatistics) && + options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + if (!old_options) + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in 24 hours from now."); + } + + if (old_options && old_options->CellStatistics && + !options->CellStatistics) + rep_hist_buffer_stats_term(); + if (old_options && old_options->DirReqStatistics && + !options->DirReqStatistics) + geoip_dirreq_stats_term(); + if (old_options && old_options->EntryStatistics && + !options->EntryStatistics) + geoip_entry_stats_term(); + if (old_options && old_options->ExitPortStatistics && + !options->ExitPortStatistics) + rep_hist_exit_stats_term(); + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || @@ -1560,6 +1628,16 @@ config_find_option(config_format_t *fmt, const char *key) return NULL; } +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(config_format_t *fmt) +{ + int i; + for (i=0; fmt->vars[i].name; ++i) + ; + return i; +} + /* * Functions to assign config options. */ @@ -1704,7 +1782,7 @@ config_assign_value(config_format_t *fmt, or_options_t *options, static int config_assign_line(config_format_t *fmt, or_options_t *options, config_line_t *c, int use_defaults, - int clear_first, char **msg) + int clear_first, bitarray_t *options_seen, char **msg) { config_var_t *var; @@ -1724,6 +1802,7 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return -1; } } + /* Put keyword into canonical case. */ if (strcmp(var->name, c->key)) { tor_free(c->key); @@ -1746,6 +1825,18 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return 0; } + if (options_seen && (var->type != CONFIG_TYPE_LINELIST && + var->type != CONFIG_TYPE_LINELIST_S)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + int var_index = (int)(var - fmt->vars); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", var->name); + } + bitarray_set(options_seen, var_index); + } + if (config_assign_value(fmt, options, c, msg) < 0) return -2; return 0; @@ -2014,6 +2105,8 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, int use_defaults, int clear_first, char **msg) { config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(fmt); CHECK(fmt, options); @@ -2033,14 +2126,18 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, config_reset_line(fmt, options, p->key, use_defaults); } + options_seen = bitarray_init_zero(n_options); /* pass 3: assign. */ while (list) { int r; if ((r=config_assign_line(fmt, options, list, use_defaults, - clear_first, msg))) + clear_first, options_seen, msg))) { + bitarray_free(options_seen); return r; + } list = list->next; } + bitarray_free(options_seen); return 0; } @@ -3404,6 +3501,13 @@ options_validate(or_options_t *old_options, or_options_t *options, "upgrade your Tor controller as soon as possible."); } + if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { + log_warn(LD_CONFIG, "You set the CookieAuthFileGroupReadable but did " + "not configure a the path for the cookie file via " + "CookieAuthFile. This means your cookie will not be group " + "readable."); + } + if (options->UseEntryGuards && ! options->NumEntryGuards) REJECT("Cannot enable UseEntryGuards with NumEntryGuards set to 0"); @@ -3578,6 +3682,21 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AccelDir && !options->AccelName) REJECT("Can't use hardware crypto accelerator dir without engine name."); + if (options->PublishServerDescriptor) + SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, { + if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0")) + if (smartlist_len(options->PublishServerDescriptor) > 1) { + COMPLAIN("You have passed a list of multiple arguments to the " + "PublishServerDescriptor option that includes 0 or 1. " + "0 or 1 should only be used as the sole argument. " + "This configuration will be rejected in a future release."); + break; + } + }); + + if (options->BridgeRelay == 1 && options->ORPort == 0) + REJECT("BridgeRelay is 1, ORPort is 0. This is an invalid combination."); + return 0; #undef REJECT #undef COMPLAIN @@ -3642,16 +3761,6 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (old->CellStatistics != new_val->CellStatistics || - old->DirReqStatistics != new_val->DirReqStatistics || - old->EntryStatistics != new_val->EntryStatistics || - old->ExitPortStatistics != new_val->ExitPortStatistics) { - *msg = tor_strdup("While Tor is running, changing either " - "CellStatistics, DirReqStatistics, EntryStatistics, " - "or ExitPortStatistics is not allowed."); - return -1; - } - if (old->DisableAllSwap != new_val->DisableAllSwap) { *msg = tor_strdup("While Tor is running, changing DisableAllSwap " "is not allowed."); @@ -3723,7 +3832,7 @@ get_windows_conf_root(void) { static int is_set = 0; static char path[MAX_PATH+1]; - WCHAR wpath[MAX_PATH] = {0}; + TCHAR tpath[MAX_PATH] = {0}; LPITEMIDLIST idl; IMalloc *m; @@ -3750,8 +3859,12 @@ get_windows_conf_root(void) return path; } /* Convert the path from an "ID List" (whatever that is!) to a path. */ - result = SHGetPathFromIDListW(idl, wpath); - wcstombs(path,wpath,MAX_PATH); + result = SHGetPathFromIDList(idl, tpath); +#ifdef UNICODE + wcstombs(path,tpath,MAX_PATH); +#else + strlcpy(path,tpath,sizeof(path)); +#endif /* Now we need to free the memory that the path-idl was stored in. In * typical Windows fashion, we can't just call 'free()' on it. */ @@ -4876,9 +4989,7 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (circuit_build_times_parse_state(&circ_times, global_state, &err) < 0) { - log_warn(LD_GENERAL,"%s",err); - tor_free(err); + if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { ret = -1; } return ret; diff --git a/src/or/connection.c b/src/or/connection.c index 55d2fa814..91ce74b5b 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -839,13 +839,13 @@ static void warn_too_many_conns(void) { #define WARN_TOO_MANY_CONNS_INTERVAL (6*60*60) - static time_t last_warned = 0; - time_t now = time(NULL); - int n_conns = get_n_open_sockets(); - if (last_warned + WARN_TOO_MANY_CONNS_INTERVAL < now) { + static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CONNS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.", n_conns); - last_warned = now; + "raise your ulimit -n.%s", n_conns, m); + tor_free(m); control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", n_conns); } @@ -2082,8 +2082,6 @@ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - if (!connection_is_rate_limited(conn)) - return; /* local IPs are free */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -2095,16 +2093,24 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } + /* Count bytes of answering direct and tunneled directory requests */ + if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { + if (num_read > 0) + rep_hist_note_dir_bytes_read(num_read, now); + if (num_written > 0) + rep_hist_note_dir_bytes_written(num_written, now); + } + + if (!connection_is_rate_limited(conn)) + return; /* local IPs are free */ if (num_read > 0) { - if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_read(conn->port, num_read); rep_hist_note_bytes_read(num_read, now); } if (num_written > 0) { - if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_written(conn->port, num_written); rep_hist_note_bytes_written(num_written, now); } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -2355,7 +2361,7 @@ connection_handle_read_impl(connection_t *conn) return 0; } -loop_again: + loop_again: try_to_read = max_to_read; tor_assert(!conn->marked_for_close); @@ -2401,8 +2407,12 @@ loop_again: connection_t *linked = conn->linked_conn; if (n_read) { - /* Probably a no-op, but hey. */ - connection_buckets_decrement(linked, approx_time(), n_read, 0); + /* Probably a no-op, since linked conns typically don't count for + * bandwidth rate limiting. But do it anyway so we can keep stats + * accurately. Note that since we read the bytes from conn, and + * we're writing the bytes onto the linked connection, we count + * these as <i>written</i> bytes. */ + connection_buckets_decrement(linked, approx_time(), 0, n_read); if (connection_flushed_some(linked) < 0) connection_mark_for_close(linked); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 7522368c5..6a3a5ef0a 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -147,7 +147,7 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) return 0; case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - if (connection_edge_package_raw_inbuf(conn, package_partial) < 0) { + if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return -1; @@ -1408,6 +1408,26 @@ consider_plaintext_ports(edge_connection_t *conn, uint16_t port) * different one? */ #define TRACKHOSTEXITS_RETRIES 5 +/** Call connection_ap_handshake_rewrite_and_attach() unless a controller + * asked us to leave streams unattached. Return 0 in that case. + * + * See connection_ap_handshake_rewrite_and_attach()'s + * documentation for arguments and return value. + */ +int +connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + or_options_t *options = get_options(); + + if (options->LeaveStreamsUnattached) { + conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; + return 0; + } + return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); +} + /** Connection <b>conn</b> just finished its socks handshake, or the * controller asked us to take care of it. If <b>circ</b> is defined, * then that's where we'll want to attach it. Otherwise we have to @@ -1908,11 +1928,7 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) else control_event_stream_status(conn, STREAM_EVENT_NEW_RESOLVE, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_init_accepted_conn() found a new trans AP conn. @@ -1926,7 +1942,6 @@ int connection_ap_process_transparent(edge_connection_t *conn) { socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -1950,11 +1965,7 @@ connection_ap_process_transparent(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_edge_process_inbuf() found a conn in state natd_wait. See if @@ -1975,7 +1986,6 @@ connection_ap_process_natd(edge_connection_t *conn) size_t tlen = 30; int err, port_ok; socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -2031,13 +2041,9 @@ connection_ap_process_natd(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** Iterate over the two bytes of stream_id until we get one that is not @@ -2050,7 +2056,7 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) streamid_t test_stream_id; uint32_t attempts=0; -again: + again: test_stream_id = circ->next_stream_id++; if (++attempts > 1<<16) { /* Make sure we don't loop forever if all stream_id's are used. */ @@ -2614,7 +2620,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_REND,"Finished assigning addr/port"); n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */ - /* add it into the linked list of n_streams on this circuit */ + /* add it into the linked list of p_streams on this circuit */ n_stream->next_stream = origin_circ->p_streams; n_stream->on_circuit = circ; origin_circ->p_streams = n_stream; @@ -2988,7 +2994,7 @@ parse_extended_hostname(char *address, int allowdotexit) if (rend_valid_service_id(query)) { return ONION_HOSTNAME; /* success */ } -failed: + failed: /* otherwise, return to previous state and return 0 */ *s = '.'; return BAD_HOSTNAME; diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index c3d6098c5..762af5172 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -79,10 +79,12 @@ void client_dns_set_addressmap(const char *address, uint32_t val, const char *addressmap_register_virtual_address(int type, char *new_address); void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, time_t max_expires, int want_expiry); +int connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, origin_circuit_t *circ, crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); /** Possible return values for parse_extended_hostname. */ typedef enum hostname_type_t { diff --git a/src/or/connection_or.c b/src/or/connection_or.c index c94325a5b..4d82da63e 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -252,8 +252,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { - ssize_t n = (OR_CONN_HIGHWATER - datalen + CELL_NETWORK_SIZE-1) - / CELL_NETWORK_SIZE; + ssize_t n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE); time_t now = approx_time(); while (conn->active_circuits && n > 0) { int flushed; @@ -349,21 +348,18 @@ connection_or_digest_is_known_relay(const char *id_digest) return 0; } -/** If we don't necessarily know the router we're connecting to, but we - * have an addr/port/id_digest, then fill in as much as we can. Start - * by checking to see if this describes a router we know. */ +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + */ static void -connection_or_init_conn_from_address(or_connection_t *conn, - const tor_addr_t *addr, uint16_t port, - const char *id_digest, - int started_here) +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) { - or_options_t *options = get_options(); int rate, burst; /* per-connection rate limiting params */ - routerinfo_t *r = router_get_by_digest(id_digest); - connection_or_set_identity_digest(conn, id_digest); - - if (connection_or_digest_is_known_relay(id_digest)) { + if (connection_or_digest_is_known_relay(conn->identity_digest)) { /* It's in the consensus, or we have a descriptor for it meaning it * was probably in a recent consensus. It's a recognized relay: * give it full bandwidth. */ @@ -382,7 +378,43 @@ connection_or_init_conn_from_address(or_connection_t *conn, } conn->bandwidthrate = rate; - conn->read_bucket = conn->write_bucket = conn->bandwidthburst = burst; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + +/** If we don't necessarily know the router we're connecting to, but we + * have an addr/port/id_digest, then fill in as much as we can. Start + * by checking to see if this describes a router we know. */ +static void +connection_or_init_conn_from_address(or_connection_t *conn, + const tor_addr_t *addr, uint16_t port, + const char *id_digest, + int started_here) +{ + routerinfo_t *r = router_get_by_digest(id_digest); + connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 8e3723c15..717630217 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -26,6 +26,8 @@ int connection_or_flushed_some(or_connection_t *conn); int connection_or_finished_flushing(or_connection_t *conn); int connection_or_finished_connecting(or_connection_t *conn); int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg); @@ -44,10 +46,6 @@ void connection_or_write_var_cell_to_buf(const var_cell_t *cell, int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason); int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); int is_or_protocol_version_known(uint16_t version); void cell_pack(packed_cell_t *dest, const cell_t *src); diff --git a/src/or/control.c b/src/or/control.c index 7cbb1bd1f..7eead0e18 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -2259,7 +2259,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, circ->_base.purpose = new_purpose; connection_write_str_to_buf("250 OK\r\n", conn); -done: + done: if (args) { SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 2760d9666..6f943d78b 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -192,7 +192,7 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(0); /* don't ask me to do handshakes yet */ } -done_processing: + done_processing: conn->state = CPUWORKER_STATE_IDLE; num_cpuworkers_busy--; if (conn->timestamp_created < last_rotation_time) { diff --git a/src/or/directory.c b/src/or/directory.c index a3e575ac9..ffa312bc4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -67,8 +67,10 @@ static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_networkstatus_failed( dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); +static void connection_dir_retry_bridges(smartlist_t *descs); static void dir_networkstatus_download_failed(smartlist_t *failed, int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, @@ -582,7 +584,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_is_constrained(get_options())) + if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -592,6 +594,8 @@ connection_dir_request_failed(dir_connection_t *conn) conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", conn->_base.address); + if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) + connection_dir_bridge_routerdesc_failed(conn); connection_dir_download_routerdesc_failed(conn); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { networkstatus_consensus_download_failed(0); @@ -645,6 +649,24 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, } } +/** Helper: Attempt to fetch directly the descriptors of each bridge + * listed in <b>failed</b>. + */ +static void +connection_dir_retry_bridges(smartlist_t *descs) +{ + char digest[DIGEST_LEN]; + SMARTLIST_FOREACH(descs, const char *, cp, + { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", + escaped(cp)); + continue; + } + retry_bridge_descriptor_fetch_directly(digest); + }); +} + /** Called when an attempt to download one or more router descriptors * or extra-info documents on connection <b>conn</b> failed. */ @@ -662,6 +684,33 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn) (void) conn; } +/** Called when an attempt to download a bridge's routerdesc from + * one of the authorities failed due to a network error. If + * possible attempt to download descriptors from the bridge directly. + */ +static void +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn) +{ + smartlist_t *which = NULL; + + /* Requests for bridge descriptors are in the form 'fp/', so ignore + anything else. */ + if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/")) + return; + + which = smartlist_create(); + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen("fp/"), + which, NULL, 0); + + tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO); + if (smartlist_len(which)) { + connection_dir_retry_bridges(which); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + } + smartlist_free(which); +} + /** Called when an attempt to fetch a certificate fails. */ static void connection_dir_download_cert_failed(dir_connection_t *conn, int status) @@ -3484,6 +3533,14 @@ download_status_reset(download_status_t *dls) dls->next_attempt_at = time(NULL) + schedule[0]; } +/** Return the number of failures on <b>dls</b> since the last success (if + * any). */ +int +download_status_get_n_failures(const download_status_t *dls) +{ + return dls->n_download_failures; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -3499,16 +3556,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int server = directory_fetches_from_authorities(get_options()); if (!was_descriptor_digests) { if (router_purpose == ROUTER_PURPOSE_BRIDGE) { - tor_assert(!was_extrainfo); /* not supported yet */ - SMARTLIST_FOREACH(failed, const char *, cp, - { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { - log_warn(LD_BUG, "Malformed fingerprint in list: %s", - escaped(cp)); - continue; - } - retry_bridge_descriptor_fetch_directly(digest); - }); + tor_assert(!was_extrainfo); + connection_dir_retry_bridges(failed); } return; /* FFFF should implement for other-than-router-purpose someday */ } diff --git a/src/or/directory.h b/src/or/directory.h index 36b4cf2b1..6fd2c0bef 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -104,5 +104,7 @@ download_status_mark_impossible(download_status_t *dl) dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; } +int download_status_get_n_failures(const download_status_t *dls); + #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 86cd18611..3fcf1783d 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -730,6 +730,10 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); + /* Tell if we're about to need to launch a test if we add this. */ + ri->needs_retest_if_added = + dirserv_should_launch_reachability_test(ri, ri_old); + r = router_add_to_routerlist(ri, msg, 0, 0); if (!WRA_WAS_ADDED(r)) { /* unless the routerinfo was fine, just out-of-date */ @@ -744,7 +748,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) changed = smartlist_create(); smartlist_add(changed, ri); - control_event_descriptors_changed(changed); + routerlist_descriptors_added(changed, 0); smartlist_free(changed); if (!*msg) { *msg = ri->is_valid ? "Descriptor for valid server accepted" : @@ -923,6 +927,11 @@ running_long_enough_to_decide_unreachable(void) * the directory. */ #define REACHABLE_TIMEOUT (45*60) +/** If we tested a router and found it reachable _at least this long_ after it + * declared itself hibernating, it is probably done hibernating and we just + * missed a descriptor from it. */ +#define HIBERNATION_PUBLICATION_SKEW (60*60) + /** Treat a router as alive if * - It's me, and I'm not hibernating. * or - We've found it reachable recently. */ @@ -935,11 +944,23 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) */ int answer; - if (router_is_me(router) && !we_are_hibernating()) + if (router_is_me(router)) { + /* We always know if we are down ourselves. */ + answer = ! we_are_hibernating(); + } else if (router->is_hibernating && + (router->cache_info.published_on + + HIBERNATION_PUBLICATION_SKEW) > router->last_reachable) { + /* A hibernating router is down unless we (somehow) had contact with it + * since it declared itself to be hibernating. */ + answer = 0; + } else if (get_options()->AssumeReachable) { + /* If AssumeReachable, everybody is up unless they say they are down! */ answer = 1; - else - answer = get_options()->AssumeReachable || - now < router->last_reachable + REACHABLE_TIMEOUT; + } else { + /* Otherwise, a router counts as up if we found it reachable in the last + REACHABLE_TIMEOUT seconds. */ + answer = (now < router->last_reachable + REACHABLE_TIMEOUT); + } if (!answer && running_long_enough_to_decide_unreachable()) { /* not considered reachable. tell rephist. */ @@ -3098,6 +3119,30 @@ dirserv_orconn_tls_done(const char *address, * skip testing. */ } +/** Called when we, as an authority, receive a new router descriptor either as + * an upload or a download. Used to decide whether to relaunch reachability + * testing for the server. */ +int +dirserv_should_launch_reachability_test(routerinfo_t *ri, routerinfo_t *ri_old) +{ + if (!authdir_mode_handles_descs(get_options(), ri->purpose)) + return 0; + if (!ri_old) { + /* New router: Launch an immediate reachability test, so we will have an + * opinion soon in case we're generating a consensus soon */ + return 1; + } + if (ri_old->is_hibernating && !ri->is_hibernating) { + /* It just came out of hibernation; launch a reachability test */ + return 1; + } + if (! routers_have_same_or_addr(ri, ri_old)) { + /* Address or port changed; launch a reachability test */ + return 1; + } + return 0; +} + /** Helper function for dirserv_test_reachability(). Start a TLS * connection to <b>router</b>, and annotate it with when we started * the test. */ diff --git a/src/or/dirserv.h b/src/or/dirserv.h index fc5a5549c..94e4e811d 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -55,7 +55,6 @@ enum was_router_added_t dirserv_add_multiple_descriptors( enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source); -void dirserv_free_descriptors(void); void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); @@ -100,6 +99,8 @@ void dirserv_orconn_tls_done(const char *address, uint16_t or_port, const char *digest_rcvd, int as_advertised); +int dirserv_should_launch_reachability_test(routerinfo_t *ri, + routerinfo_t *ri_old); void dirserv_single_reachability_test(time_t now, routerinfo_t *router); void dirserv_test_reachability(time_t now); int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, diff --git a/src/or/dirvote.c b/src/or/dirvote.c index fd4d742cc..eae3bc8a4 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -762,7 +762,7 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, } } -out: + out: if (berr) { log_info(LD_DIR, "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT @@ -2942,6 +2942,7 @@ dirvote_compute_consensuses(void) strlen(pending_consensus_signatures), 0); log_notice(LD_DIR, "Signature(s) posted."); + smartlist_free(votes); return 0; err: smartlist_free(votes); @@ -3008,6 +3009,7 @@ dirvote_add_signatures_to_pending_consensus( networkstatus_vote_free(v); } *msg_out = "Signatures added"; + tor_free(new_signatures); } else if (r == 0) { *msg_out = "Signatures ignored"; } else { @@ -3137,7 +3139,7 @@ void dirvote_free_all(void) { dirvote_clear_votes(1); - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_vote_list); pending_vote_list = NULL; smartlist_free(previous_vote_list); @@ -3146,7 +3148,7 @@ dirvote_free_all(void) dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); if (pending_consensus_signature_list) { - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_consensus_signature_list); pending_consensus_signature_list = NULL; } diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index e231b655f..ad4f4122b 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -141,16 +141,17 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -186,16 +187,17 @@ dnsserv_launch_request(const char *name, int reverse) return -1; } - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", escaped_safe_str_client(q_name)); tor_free(q_name); return 0; diff --git a/src/or/eventdns.c b/src/or/eventdns.c index 4e44d1516..14c5d88df 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -3132,7 +3132,7 @@ load_nameservers_with_getnetworkparams(void) GetNetworkParams_fn_t fn; /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(handle = LoadLibraryW(L"iphlpapi.dll"))) { + if (!(handle = LoadLibrary(TEXT("iphlpapi.dll"))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); /* right now status = 0, doesn't that mean "good" - mikec */ status = -1; @@ -3201,46 +3201,44 @@ load_nameservers_with_getnetworkparams(void) } static int -config_nameserver_from_reg_key(HKEY key, const char *subkey) +config_nameserver_from_reg_key(HKEY key, const TCHAR *subkey) { char *buf; DWORD bufsz = 0, type = 0; - WCHAR wsubkey[MAX_PATH] = {0}; - char ansibuf[MAX_PATH] = {0}; int status = 0; - mbstowcs(wsubkey,subkey,MAX_PATH); - if (RegQueryValueExW(key, wsubkey, 0, &type, NULL, &bufsz) + if (RegQueryValueEx(key, subkey, 0, &type, NULL, &bufsz) != ERROR_MORE_DATA) return -1; if (!(buf = mm_malloc(bufsz))) return -1; - if (RegQueryValueExW(key, wsubkey, 0, &type, (LPBYTE)buf, &bufsz) + if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { - wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH); - status = evdns_nameserver_ip_add_line(ansibuf); + wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH);/*XXXX UNICODE */ + status = evdns_nameserver_ip_add_line(buf); } mm_free(buf); return status; } -#define SERVICES_KEY L"System\\CurrentControlSet\\Services\\" -#define WIN_NS_9X_KEY SERVICES_KEY L"VxD\\MSTCP" -#define WIN_NS_NT_KEY SERVICES_KEY L"Tcpip\\Parameters" +#define SERVICES_KEY TEXT("System\\CurrentControlSet\\Services\\") +#define WIN_NS_9X_KEY SERVICES_KEY TEXT("VxD\\MSTCP") +#define WIN_NS_NT_KEY SERVICES_KEY TEXT("Tcpip\\Parameters") static int load_nameservers_from_registry(void) { int found = 0; int r; - OSVERSIONINFO info = {0}; + OSVERSIONINFO info; + memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof (info); - GetVersionExW((LPOSVERSIONINFO)&info); + GetVersionEx(&info); #define TRY(k, name) \ - if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ + if (!found && config_nameserver_from_reg_key(k,TEXT(name)) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ found = 1; \ } else if (!found) { \ @@ -3251,12 +3249,12 @@ load_nameservers_from_registry(void) if (info.dwMajorVersion >= 5) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; - if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, KEY_READ, &nt_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } - r = RegOpenKeyExW(nt_key, L"Interfaces", 0, + r = RegOpenKeyEx(nt_key, Text("Interfaces"), 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { @@ -3271,7 +3269,7 @@ load_nameservers_from_registry(void) RegCloseKey(nt_key); } else { HKEY win_key = 0; - if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, KEY_READ, &win_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG, "Couldn't open registry key, %d", (int)GetLastError()); return -1; diff --git a/src/or/geoip.c b/src/or/geoip.c index eae927522..d9c8a0151 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,8 +3,10 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE @@ -17,6 +19,7 @@ #include "routerlist.h" static void clear_geoip_db(void); +static void init_geoip_countries(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -25,16 +28,11 @@ typedef struct geoip_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_entry_t; -/** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 1 -/** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (24*60*60) - /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v2_ns_requests[REQUEST_HIST_LEN]; - uint32_t n_v3_ns_requests[REQUEST_HIST_LEN]; + uint32_t n_v2_ns_requests; + uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -106,11 +104,11 @@ geoip_parse_entry(const char *line) { unsigned int low, high; char b[3]; - if (!geoip_countries) { - geoip_countries = smartlist_create(); + if (!geoip_countries) + init_geoip_countries(); + if (!geoip_entries) geoip_entries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + while (TOR_ISSPACE(*line)) ++line; if (*line == '#') @@ -165,6 +163,24 @@ should_record_bridge_info(or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + /** Clear the GeoIP database and reload it from the file * <b>filename</b>. Return 0 on success, -1 on failure. * @@ -190,17 +206,8 @@ geoip_load_file(const char *filename, or_options_t *options) filename, msg); return -1; } - if (!geoip_countries) { - geoip_country_t *geoip_unresolved; - geoip_countries = smartlist_create(); - /* Add a geoip_country_t for requests that could not be resolved to a - * country as first element (index 0) to geoip_countries. */ - geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); - strlcpy(geoip_unresolved->countrycode, "??", - sizeof(geoip_unresolved->countrycode)); - smartlist_add(geoip_countries, geoip_unresolved); - country_idxplus1_by_lc_code = strmap_new(); - } + if (!geoip_countries) + init_geoip_countries(); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e)); smartlist_free(geoip_entries); @@ -227,9 +234,10 @@ geoip_load_file(const char *filename, or_options_t *options) } /** Given an IP address in host order, return a number representing the - * country to which that address belongs, or -1 for unknown. The return value - * will always be less than geoip_get_n_countries(). To decode it, - * call geoip_get_country_name(). + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). */ int geoip_get_country_by_ip(uint32_t ipaddr) @@ -238,7 +246,7 @@ geoip_get_country_by_ip(uint32_t ipaddr) if (!geoip_entries) return -1; ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry); - return ent ? (int)ent->country : -1; + return ent ? (int)ent->country : 0; } /** Return the number of countries recognized by the GeoIP database. */ @@ -282,14 +290,6 @@ typedef struct clientmap_entry_t { /** Map from client IP address to last time seen. */ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); -/** Time at which we started tracking client IP history. */ -static time_t client_history_starts = 0; - -/** When did the current period of checking per-country request history - * start? */ -static time_t current_request_period_starts = 0; -/** How many older request periods are we remembering? */ -static int n_old_request_periods = 0; /** Hashtable helper: compute a hash of a clientmap_entry_t. */ static INLINE unsigned @@ -309,6 +309,23 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + /** How often do we update our estimate which share of v2 and v3 directory * requests is sent to us? We could as well trigger updates of shares from * network status updates, but that means adding a lot of calls into code @@ -370,25 +387,6 @@ geoip_get_mean_shares(time_t now, double *v2_share_out, return 0; } -/* Rotate period of v2 and v3 network status requests. */ -static void -rotate_request_period(void) -{ - SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) { -#if REQUEST_HIST_LEN > 1 - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); -#endif - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - } SMARTLIST_FOREACH_END(c); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; -} - /** Note that we've seen a client connect from the IP <b>addr</b> (host order) * at time <b>now</b>. Ignored by all but bridges and directories if * configured accordingly. */ @@ -403,35 +401,12 @@ geoip_note_client_seen(geoip_client_action_t action, if (!options->EntryStatistics && (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; - /* Did we recently switch from bridge to relay or back? */ - if (client_history_starts > now) - return; } else { if (options->BridgeRelay || options->BridgeAuthoritativeDir || !options->DirReqStatistics) return; } - /* As a bridge that doesn't rotate request periods every 24 hours, - * possibly rotate now. */ - if (options->BridgeRelay) { - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - /* Also discard all items in the client history that are too old. - * (This only works here because bridge and directory stats are - * independent. Otherwise, we'd only want to discard those items - * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ - geoip_remove_old_clients(current_request_period_starts); - /* Now rotate request period */ - rotate_request_period(); - } - } - lookup.ipaddr = addr; lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); @@ -453,20 +428,15 @@ geoip_note_client_seen(geoip_client_action_t action, if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) - ++country->n_v3_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v3_ns_requests; else - ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v2_ns_requests; } /* Periodically determine share of requests that we should see */ if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) geoip_determine_shares(now); } - - if (!client_history_starts) { - client_history_starts = now; - current_request_period_starts = now; - } } /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's @@ -483,18 +453,13 @@ _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) } } -/** Forget about all clients that haven't connected since <b>cutoff</b>. - * If <b>cutoff</b> is in the future, clients won't be added to the history - * until this time is reached. This is useful to prevent relays that switch - * to bridges from reporting unbelievable numbers of clients. */ +/** Forget about all clients that haven't connected since <b>cutoff</b>. */ void geoip_remove_old_clients(time_t cutoff) { clientmap_HT_FOREACH_FN(&client_history, _remove_old_client_helper, &cutoff); - if (client_history_starts < cutoff) - client_history_starts = cutoff; } /** How many responses are we giving to clients requesting v2 network @@ -540,13 +505,6 @@ geoip_note_ns_response(geoip_client_action_t action, * multiple of this value. */ #define IP_GRANULARITY 8 -/** Return the time at which we started recording geoip data. */ -time_t -geoip_get_history_start(void) -{ - return client_history_starts; -} - /** Helper type: used to sort per-country totals by value. */ typedef struct c_hist_t { char country[3]; /**< Two-letter country code. */ @@ -622,7 +580,7 @@ HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, dirreq_map_ent_eq, 0.6, malloc, realloc, free); /** Helper: Put <b>entry</b> into map of directory requests using - * <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there is + * <b>type</b> and <b>dirreq_id</b> as key parts. If there is * already an entry for that key, print out a BUG warning and return. */ static void _dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, @@ -643,7 +601,7 @@ _dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, } /** Helper: Look up and return an entry in the map of directory requests - * using <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there + * using <b>type</b> and <b>dirreq_id</b> as key parts. If there * is no such entry, return NULL. */ static dirreq_map_entry_t * _dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) @@ -806,124 +764,94 @@ geoip_get_dirreq_history(geoip_client_action_t action, return result; } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) - -/** Helper for geoip_get_client_history_dirreq() and - * geoip_get_client_history_bridge(). */ -static char * -geoip_get_client_history(time_t now, geoip_client_action_t action, - int min_observation_time, unsigned granularity) +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ +char * +geoip_get_client_history(geoip_client_action_t action) { char *result = NULL; + unsigned granularity = IP_GRANULARITY; + smartlist_t *chunks = NULL; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **ent; + unsigned *counts = NULL; + unsigned total = 0; + if (!geoip_is_loaded()) return NULL; - if (client_history_starts < (now - min_observation_time)) { - smartlist_t *chunks = NULL; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **ent; - unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); - unsigned total = 0; - HT_FOREACH(ent, clientmap, &client_history) { - int country; - if ((*ent)->action != (int)action) - continue; - country = geoip_get_country_by_ip((*ent)->ipaddr); - if (country < 0) - country = 0; /** unresolved requests are stored at index 0. */ - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - } - /* Don't record anything if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) - goto done; - /* Make a list of c_hist_t */ - entries = smartlist_create(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); - } - } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, _c_hist_compare); - - /* Build the result. */ - chunks = smartlist_create(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - char *buf=NULL; - tor_asprintf(&buf, "%s=%u", ch->country, ch->total); - smartlist_add(chunks, buf); - }); - result = smartlist_join_strings(chunks, ",", 0, NULL); - done: - tor_free(counts); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - if (entries) { - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); + + counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + HT_FOREACH(ent, clientmap, &client_history) { + int country; + if ((*ent)->action != (int)action) + continue; + country = geoip_get_country_by_ip((*ent)->ipaddr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + } + /* Don't record anything if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) + goto done; + /* Make a list of c_hist_t */ + entries = smartlist_create(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); } } - return result; -} - -/** Return a newly allocated comma-separated string containing entries for - * all the countries from which we've seen enough clients connect as a - * directory. The entry format is cc=num where num is the number of IPs - * we've seen connecting from that country, and cc is a lowercased country - * code. Returns NULL if we don't want to export geoip data yet. */ -char * -geoip_get_client_history_dirreq(time_t now, - geoip_client_action_t action) -{ - return geoip_get_client_history(now, action, - DIR_RECORD_USAGE_MIN_OBSERVATION_TIME, - DIR_RECORD_USAGE_GRANULARITY); -} + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, _c_hist_compare); -/** Return a newly allocated comma-separated string containing entries for - * all the countries from which we've seen enough clients connect as a - * bridge. The entry format is cc=num where num is the number of IPs - * we've seen connecting from that country, and cc is a lowercased country - * code. Returns NULL if we don't want to export geoip data yet. */ -char * -geoip_get_client_history_bridge(time_t now, - geoip_client_action_t action) -{ - return geoip_get_client_history(now, action, - GEOIP_MIN_OBSERVATION_TIME, - IP_GRANULARITY); + /* Build the result. */ + chunks = smartlist_create(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + char *buf=NULL; + tor_asprintf(&buf, "%s=%u", ch->country, ch->total); + smartlist_add(chunks, buf); + }); + result = smartlist_join_strings(chunks, ",", 0, NULL); + done: + tor_free(counts); + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + if (entries) { + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + } + return result; } /** Return a newly allocated string holding the per-country request history * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ char * -geoip_get_request_history(time_t now, geoip_client_action_t action) +geoip_get_request_history(geoip_client_action_t action) { smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; - int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; - if (client_history_starts >= (now - min_observation_time)) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; @@ -932,13 +860,10 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) entries = smartlist_create(); SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - uint32_t *n = (action == GEOIP_CLIENT_NETWORKSTATUS) - ? c->n_v3_ns_requests : c->n_v2_ns_requests; uint32_t tot = 0; - int i; c_hist_t *ent; - for (i=0; i < REQUEST_HIST_LEN; ++i) - tot += n[i]; + tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ? + c->n_v3_ns_requests : c->n_v2_ns_requests; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -962,7 +887,8 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Start time of directory request stats. */ +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ static time_t start_of_dirreq_stats_interval; /** Initialize directory request stats. */ @@ -972,8 +898,47 @@ geoip_dirreq_stats_init(time_t now) start_of_dirreq_stats_interval = now; } -/** Write dirreq statistics to $DATADIR/stats/dirreq-stats. */ +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ void +geoip_dirreq_stats_term(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS || + (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + v2_share_times_seconds = v3_share_times_seconds = 0.0; + last_time_determined_shares = 0; + share_seconds = 0; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = 0; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when + * we would next want to write. */ +time_t geoip_dirreq_stats_write(time_t now) { char *statsdir = NULL, *filename = NULL; @@ -984,8 +949,10 @@ geoip_dirreq_stats_write(time_t now) FILE *out; int i; - if (!get_options()->DirReqStatistics) - goto done; + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_dirreq_stats_interval); @@ -994,10 +961,8 @@ geoip_dirreq_stats_write(time_t now) if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; filename = get_datadir_fname2("stats", "dirreq-stats"); - data_v2 = geoip_get_client_history_dirreq(now, - GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history_dirreq(now, - GEOIP_CLIENT_NETWORKSTATUS); + data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); @@ -1011,13 +976,16 @@ geoip_dirreq_stats_write(time_t now) tor_free(data_v2); tor_free(data_v3); - data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); + data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS); if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); #define RESPONSE_GRANULARITY 8 for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { ns_v2_responses[i] = round_uint32_to_next_multiple_of( @@ -1072,9 +1040,6 @@ geoip_dirreq_stats_write(time_t now) finish_writing_to_file(open_file); open_file = NULL; - /* Rotate request period */ - rotate_request_period(); - start_of_dirreq_stats_interval = now; done: @@ -1084,9 +1049,11 @@ geoip_dirreq_stats_write(time_t now) tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; } -/** Start time of bridge stats. */ +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ static time_t start_of_bridge_stats_interval; /** Initialize bridge stats. */ @@ -1096,6 +1063,15 @@ geoip_bridge_stats_init(time_t now) start_of_bridge_stats_interval = now; } +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + /** Parse the bridge statistics as they are written to extra-info * descriptors for being returned to controller clients. Return the * controller string if successful, or NULL otherwise. */ @@ -1175,16 +1151,9 @@ geoip_bridge_stats_write(time_t now) written[ISO_TIME_LEN+1], *out = NULL, *controller_str; size_t len; - /* If we changed from relay to bridge recently, adapt starting time - * of current measurements. */ - if (start_of_bridge_stats_interval < client_history_starts) - start_of_bridge_stats_interval = client_history_starts; - /* Check if 24 hours have passed since starting measurements. */ - if (now < start_of_bridge_stats_interval + - DIR_ENTRY_RECORD_USAGE_RETAIN_IPS) - return start_of_bridge_stats_interval + - DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_bridge_stats_interval); @@ -1193,7 +1162,7 @@ geoip_bridge_stats_write(time_t now) if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; filename = get_datadir_fname2("stats", "bridge-stats"); - data = geoip_get_client_history_bridge(now, GEOIP_CLIENT_CONNECT); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); format_iso_time(written, now); len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; @@ -1219,7 +1188,7 @@ geoip_bridge_stats_write(time_t now) tor_free(data); tor_free(out); return start_of_bridge_stats_interval + - DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; + WRITE_STATS_INTERVAL; } /** Try to load the most recent bridge statistics from disk, unless we @@ -1267,7 +1236,8 @@ geoip_get_bridge_stats_controller(time_t now) return bridge_stats_controller; } -/** Start time of entry stats. */ +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ static time_t start_of_entry_stats_interval; /** Initialize entry stats. */ @@ -1277,8 +1247,18 @@ geoip_entry_stats_init(time_t now) start_of_entry_stats_interval = now; } -/** Write entry statistics to $DATADIR/stats/entry-stats. */ +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ void +geoip_entry_stats_term(void) +{ + client_history_clear(); + start_of_entry_stats_interval = 0; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats and return time + * when we would next want to write. */ +time_t geoip_entry_stats_write(time_t now) { char *statsdir = NULL, *filename = NULL; @@ -1287,8 +1267,10 @@ geoip_entry_stats_write(time_t now) open_file_t *open_file = NULL; FILE *out; - if (!get_options()->EntryStatistics) - goto done; + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_entry_stats_interval); @@ -1297,7 +1279,7 @@ geoip_entry_stats_write(time_t now) if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; filename = get_datadir_fname2("stats", "entry-stats"); - data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); @@ -1318,6 +1300,7 @@ geoip_entry_stats_write(time_t now) tor_free(filename); tor_free(statsdir); tor_free(data); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } /** Helper used to implement GETINFO ip-to-country/... controller command. */ diff --git a/src/or/geoip.h b/src/or/geoip.h index c3a4fbcdf..68e01deec 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -29,12 +29,8 @@ void geoip_remove_old_clients(time_t cutoff); void geoip_note_ns_response(geoip_client_action_t action, geoip_ns_response_t response); -time_t geoip_get_history_start(void); -char *geoip_get_client_history_dirreq(time_t now, - geoip_client_action_t action); -char *geoip_get_client_history_bridge(time_t now, - geoip_client_action_t action); -char *geoip_get_request_history(time_t now, geoip_client_action_t action); +char *geoip_get_client_history(geoip_client_action_t action); +char *geoip_get_request_history(geoip_client_action_t action); int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); @@ -46,11 +42,14 @@ void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state); void geoip_dirreq_stats_init(time_t now); -void geoip_dirreq_stats_write(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); void geoip_entry_stats_init(time_t now); -void geoip_entry_stats_write(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); void geoip_bridge_stats_init(time_t now); time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); const char *geoip_get_bridge_stats_controller(time_t); diff --git a/src/or/hibernate.h b/src/or/hibernate.h index 8192ac5d8..687fadb66 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -24,7 +24,6 @@ void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); #endif diff --git a/src/or/main.c b/src/or/main.c index ff674f386..f33dc2f6b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -791,12 +791,13 @@ run_connection_housekeeping(int i, time_t now) "Tor gave up on the connection"); connection_mark_for_close(conn); conn->hold_open_until_flushed = 1; - } else if (past_keepalive && !connection_state_is_open(conn)) { - /* We never managed to actually get this connection open and happy. */ - log_info(LD_OR,"Expiring non-open OR connection to fd %d (%s:%d).", - conn->s,conn->address, conn->port); - connection_mark_for_close(conn); - conn->hold_open_until_flushed = 1; + } else if (!connection_state_is_open(conn)) { + if (past_keepalive) { + /* We never managed to actually get this connection open and happy. */ + log_info(LD_OR,"Expiring non-open OR connection to fd %d (%s:%d).", + conn->s,conn->address, conn->port); + connection_mark_for_close(conn); + } } else if (we_are_hibernating() && !or_conn->n_circuits && !buf_datalen(conn->outbuf)) { /* We're hibernating, there's no circuits, and nothing to flush.*/ @@ -997,42 +998,32 @@ run_scheduled_events(time_t now) /* 1g. Check whether we should write statistics to disk. */ - if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) { -#define WRITE_STATS_INTERVAL (24*60*60) - if (options->CellStatistics || options->DirReqStatistics || - options->EntryStatistics || options->ExitPortStatistics) { - if (!time_to_write_stats_files) { - /* Initialize stats. We're doing this here and not in options_act, - * so that we know exactly when the 24 hours interval ends. */ - if (options->CellStatistics) - rep_hist_buffer_stats_init(now); - if (options->DirReqStatistics) - geoip_dirreq_stats_init(now); - if (options->EntryStatistics) - geoip_entry_stats_init(now); - if (options->ExitPortStatistics) - rep_hist_exit_stats_init(now); - log_notice(LD_CONFIG, "Configured to measure statistics. Look for " - "the *-stats files that will first be written to the " - "data directory in %d hours from now.", - WRITE_STATS_INTERVAL / (60 * 60)); - time_to_write_stats_files = now + WRITE_STATS_INTERVAL; - } else { - /* Write stats to disk. */ - if (options->CellStatistics) + if (time_to_write_stats_files < now) { +#define CHECK_WRITE_STATS_INTERVAL (60*60) + time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? + time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = rep_hist_buffer_stats_write(time_to_write_stats_files); - if (options->DirReqStatistics) - geoip_dirreq_stats_write(time_to_write_stats_files); - if (options->EntryStatistics) - geoip_entry_stats_write(time_to_write_stats_files); - if (options->ExitPortStatistics) - rep_hist_exit_stats_write(time_to_write_stats_files); - time_to_write_stats_files += WRITE_STATS_INTERVAL; - } - } else { - /* Never write stats to disk */ - time_to_write_stats_files = -1; + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; } + time_to_write_stats_files = next_time_to_write_stats_files; } /* 1h. Check whether we should write bridge statistics to disk. diff --git a/src/or/microdesc.c b/src/or/microdesc.c index f56ccd9ee..e8f3e7c59 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -53,7 +53,7 @@ HT_PROTOTYPE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq); HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, - _tor_malloc, _tor_realloc, _tor_free); + malloc, realloc, free); /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. * On success, return the total number of bytes written, and set diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index a9a9c78b8..9bb5546d9 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -14,10 +14,12 @@ #include "circuitbuild.h" #include "config.h" #include "connection.h" +#include "connection_or.h" #include "control.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" +#include "main.h" #include "networkstatus.h" #include "relay.h" #include "router.h" @@ -351,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -458,7 +464,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn) { int n_good = 0; - int n_missing_key = 0; + int n_missing_key = 0, n_dl_failed_key = 0; int n_bad = 0; int n_unknown = 0; int n_no_signature = 0; @@ -476,7 +482,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, voter) { int good_here = 0; int bad_here = 0; - int missing_key_here = 0; + int missing_key_here = 0, dl_failed_key_here = 0; SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { if (!sig->good_signature && !sig->bad_signature && sig->signature) { @@ -496,11 +502,15 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, } else if (!cert || cert->expires < now) { smartlist_add(need_certs_from, voter); ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; continue; } if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { smartlist_add(need_certs_from, voter); ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; continue; } } @@ -513,9 +523,11 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, ++n_good; else if (bad_here) ++n_bad; - else if (missing_key_here) + else if (missing_key_here) { ++n_missing_key; - else + if (dl_failed_key_here) + ++n_dl_failed_key; + } else ++n_no_signature; } SMARTLIST_FOREACH_END(voter); @@ -528,39 +540,71 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, smartlist_add(missing_authorities, ds); }); - if (warn > 1 || (warn >= 0 && n_good < n_required)) + if (warn > 1 || (warn >= 0 && + (n_good + n_missing_key - n_dl_failed_key < n_required))) { severity = LOG_WARN; - else + } else { severity = LOG_INFO; + } if (warn >= 0) { SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Consensus includes unrecognized authority '%s' " - "at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Consensus includes unrecognized authority " + "'%s' at %s:%d (contact %s; identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Looks like we need to download a new certificate " - "from authority '%s' at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Looks like we need to download a new " + "certificate from authority '%s' at %s:%d (contact %s; " + "identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(missing_authorities, trusted_dir_server_t *, ds, { - log_info(LD_DIR, "Consensus does not include configured " + log(severity, LD_DIR, "Consensus does not include configured " "authority '%s' at %s:%d (identity %s)", ds->nickname, ds->address, (int)ds->dir_port, hex_str(ds->v3_identity_digest, DIGEST_LEN)); }); - log(severity, LD_DIR, - "%d unknown, %d missing key, %d good, %d bad, %d no signature, " - "%d required", n_unknown, n_missing_key, n_good, n_bad, - n_no_signature, n_required); + { + smartlist_t *sl = smartlist_create(); + char *cp; + tor_asprintf(&cp, "A consensus needs %d good signatures from recognized " + "authorities for us to accept it. This one has %d.", + n_required, n_good); + smartlist_add(sl,cp); + if (n_no_signature) { + tor_asprintf(&cp, "%d of the authorities we know didn't sign it.", + n_no_signature); + smartlist_add(sl,cp); + } + if (n_unknown) { + tor_asprintf(&cp, "It has %d signatures from authorities we don't " + "recognize.", n_unknown); + smartlist_add(sl,cp); + } + if (n_bad) { + tor_asprintf(&cp, "%d of the signatures on it didn't verify " + "correctly.", n_bad); + smartlist_add(sl,cp); + } + if (n_missing_key) { + tor_asprintf(&cp, "We were unable to check %d of the signatures, " + "because we were missing the keys.", n_missing_key); + smartlist_add(sl,cp); + } + cp = smartlist_join_strings(sl, " ", 0, NULL); + log(severity, LD_DIR, "%s", cp); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } } smartlist_free(unrecognized); @@ -1162,14 +1206,11 @@ update_v2_networkstatus_cache_downloads(time_t now) static void update_consensus_networkstatus_downloads(time_t now) { - or_options_t *options = get_options(); int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ - if (authdir_mode_v3(options)) - return; /* Authorities never fetch a consensus */ /* XXXXNM Microdescs: may need to download more types. */ if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) @@ -1224,14 +1265,26 @@ update_consensus_networkstatus_fetch_time(time_t now) if (c) { long dl_interval; long interval = c->fresh_until - c->valid_after; + long min_sec_before_caching = CONSENSUS_MIN_SECONDS_BEFORE_CACHING; time_t start; + + if (min_sec_before_caching > interval/16) { + /* Usually we allow 2-minutes slop factor in case clocks get + desynchronized a little. If we're on a private network with + a crazy-fast voting interval, though, 2 minutes may be too + much. */ + min_sec_before_caching = interval/16; + } + if (directory_fetches_dir_info_early(options)) { /* We want to cache the next one at some point after this one * is no longer fresh... */ - start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + start = c->fresh_until + min_sec_before_caching; /* Some clients may need the consensus sooner than others. */ - if (options->FetchDirInfoExtraEarly) { + if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { dl_interval = 60; + if (min_sec_before_caching + dl_interval > interval) + dl_interval = interval/2; } else { /* But only in the first half-interval after that. */ dl_interval = interval/2; @@ -1247,10 +1300,9 @@ update_consensus_networkstatus_fetch_time(time_t now) * to choose the rest of the interval *after* them. */ if (directory_fetches_dir_info_later(options)) { /* Give all the *clients* enough time to download the consensus. */ - start = start + dl_interval + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + start = start + dl_interval + min_sec_before_caching; /* But try to get it before ours actually expires. */ - dl_interval = (c->valid_until - start) - - CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + dl_interval = (c->valid_until - start) - min_sec_before_caching; } } if (dl_interval < 1) @@ -1502,6 +1554,7 @@ networkstatus_set_current_consensus(const char *consensus, networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; @@ -1539,7 +1592,7 @@ networkstatus_set_current_consensus(const char *consensus, } if (flav != USABLE_CONSENSUS_FLAVOR && - !directory_caches_dir_info(get_options())) { + !directory_caches_dir_info(options)) { /* This consensus is totally boring to us: we won't use it, and we won't * serve it. Drop it. */ goto done; @@ -1674,7 +1727,7 @@ networkstatus_set_current_consensus(const char *consensus, download_status_failed(&consensus_dl_status[flav], 0); } - if (directory_caches_dir_info(get_options())) { + if (directory_caches_dir_info(options)) { dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, @@ -1687,9 +1740,13 @@ networkstatus_set_current_consensus(const char *consensus, /* XXXXNM Microdescs: needs a non-ns variant. */ update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); + dirvote_recalculate_timing(options, now); routerstatus_list_update_named_server_map(); - cell_ewma_set_scale_factor(get_options(), current_consensus); + cell_ewma_set_scale_factor(options, current_consensus); + + /* XXX022 where is the right place to put this call? */ + connection_or_update_token_buckets(get_connection_array(), options); + circuit_build_times_new_consensus_params(&circ_times, current_consensus); } @@ -1924,6 +1981,15 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->is_bad_directory = rs->is_bad_directory; router->is_bad_exit = rs->is_bad_exit; router->is_hs_dir = rs->is_hs_dir; + } else { + /* If we _are_ an authority, we should check whether this router + * is one that will cause us to need a reachability test. */ + routerinfo_t *old_router = + router_get_by_digest(router->cache_info.identity_digest); + if (old_router != router) { + router->needs_retest_if_added = + dirserv_should_launch_reachability_test(router, old_router); + } } if (router->is_running && ds) { download_status_reset(&ds->v2_ns_dl_status); diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 4059dead5..32b71a9ce 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -73,7 +73,6 @@ int networkstatus_set_current_consensus(const char *consensus, unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures); void signed_descs_update_status_from_consensus_networkstatus( diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 9bcb7047e..0b611f0bf 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -7,6 +7,7 @@ #include "or.h" #include "config.h" #include "main.h" +#include "ntmain.h" #ifdef HAVE_EVENT2_EVENT_H #include <event2/event.h> @@ -14,12 +15,12 @@ #include <event.h> #endif -#include <tchar.h> -#define GENSRV_SERVICENAME TEXT("tor") -#define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") +#include <windows.h> +#define GENSRV_SERVICENAME "tor" +#define GENSRV_DISPLAYNAME "Tor Win32 Service" #define GENSRV_DESCRIPTION \ - TEXT("Provides an anonymous Internet communication system") -#define GENSRV_USERACCT TEXT("NT AUTHORITY\\LocalService") + "Provides an anonymous Internet communication system" +#define GENSRV_USERACCT "NT AUTHORITY\\LocalService" // Cheating: using the pre-defined error codes, tricks Windows into displaying // a semi-related human-readable error message if startup fails as @@ -35,7 +36,6 @@ static SERVICE_STATUS_HANDLE hStatus; * to the NT service functions. */ static char **backup_argv; static int backup_argc; -static char* nt_strerror(uint32_t errnum); static void nt_service_control(DWORD request); static void nt_service_body(int argc, char **argv); @@ -69,30 +69,30 @@ struct service_fns { SC_HANDLE (WINAPI *CreateServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, - LPCTSTR lpDisplayName, + LPCSTR lpServiceName, + LPCSTR lpDisplayName, DWORD dwDesiredAccess, DWORD dwServiceType, DWORD dwStartType, DWORD dwErrorControl, - LPCTSTR lpBinaryPathName, - LPCTSTR lpLoadOrderGroup, + LPCSTR lpBinaryPathName, + LPCSTR lpLoadOrderGroup, LPDWORD lpdwTagId, - LPCTSTR lpDependencies, - LPCTSTR lpServiceStartName, - LPCTSTR lpPassword); + LPCSTR lpDependencies, + LPCSTR lpServiceStartName, + LPCSTR lpPassword); BOOL (WINAPI *DeleteService_fn)( SC_HANDLE hService); SC_HANDLE (WINAPI *OpenSCManagerA_fn)( - LPCTSTR lpMachineName, - LPCTSTR lpDatabaseName, + LPCSTR lpMachineName, + LPCSTR lpDatabaseName, DWORD dwDesiredAccess); SC_HANDLE (WINAPI *OpenServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, + LPCSTR lpServiceName, DWORD dwDesiredAccess); BOOL (WINAPI *QueryServiceStatus_fn)( @@ -100,23 +100,23 @@ struct service_fns { LPSERVICE_STATUS lpServiceStatus); SERVICE_STATUS_HANDLE (WINAPI *RegisterServiceCtrlHandlerA_fn)( - LPCTSTR lpServiceName, + LPCSTR lpServiceName, LPHANDLER_FUNCTION lpHandlerProc); BOOL (WINAPI *SetServiceStatus_fn)(SERVICE_STATUS_HANDLE, LPSERVICE_STATUS); BOOL (WINAPI *StartServiceCtrlDispatcherA_fn)( - const SERVICE_TABLE_ENTRY* lpServiceTable); + const SERVICE_TABLE_ENTRYA* lpServiceTable); BOOL (WINAPI *StartServiceA_fn)( SC_HANDLE hService, DWORD dwNumServiceArgs, - LPCTSTR* lpServiceArgVectors); + LPCSTR* lpServiceArgVectors); BOOL (WINAPI *LookupAccountNameA_fn)( - LPCTSTR lpSystemName, - LPCTSTR lpAccountName, + LPCSTR lpSystemName, + LPCSTR lpAccountName, PSID Sid, LPDWORD cbSid, LPTSTR ReferencedDomainName, @@ -139,7 +139,7 @@ nt_service_loadlibrary(void) return; /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(library = LoadLibrary("advapi32.dll"))) { + if (!(library = LoadLibrary(TEXT("advapi32.dll")))) { log_err(LD_GENERAL, "Couldn't open advapi32.dll. Are you trying to use " "NT services on Windows 98? That doesn't work."); goto err; @@ -283,20 +283,20 @@ nt_service_body(int argc, char **argv) static void nt_service_main(void) { - SERVICE_TABLE_ENTRY table[2]; + SERVICE_TABLE_ENTRYA table[2]; DWORD result = 0; char *errmsg; nt_service_loadlibrary(); table[0].lpServiceName = (char*)GENSRV_SERVICENAME; - table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTION)nt_service_body; + table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTIONA)nt_service_body; table[1].lpServiceName = NULL; table[1].lpServiceProc = NULL; if (!service_fns.StartServiceCtrlDispatcherA_fn(table)) { result = GetLastError(); - errmsg = nt_strerror(result); + errmsg = format_win32_error(result); printf("Service error %d : %s\n", (int) result, errmsg); - LocalFree(errmsg); + tor_free(errmsg); if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { if (tor_init(backup_argc, backup_argv) < 0) return; @@ -331,9 +331,9 @@ nt_service_open_scm(void) nt_service_loadlibrary(); if ((hSCManager = service_fns.OpenSCManagerA_fn( NULL, NULL, SC_MANAGER_CREATE_SERVICE)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenSCManager() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hSCManager; } @@ -348,9 +348,9 @@ nt_service_open(SC_HANDLE hSCManager) nt_service_loadlibrary(); if ((hService = service_fns.OpenServiceA_fn(hSCManager, GENSRV_SERVICENAME, SERVICE_ALL_ACCESS)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hService; } @@ -382,14 +382,14 @@ nt_service_start(SC_HANDLE hService) printf("Service started successfully\n"); return 0; } else { - errmsg = nt_strerror(service_status.dwWin32ExitCode); + errmsg = format_win32_error(service_status.dwWin32ExitCode); printf("Service failed to start : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("StartService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -426,14 +426,14 @@ nt_service_stop(SC_HANDLE hService) } else if (wait_time == MAX_SERVICE_WAIT_TIME) { printf("Service did not stop within %d seconds.\n", wait_time); } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("QueryServiceStatus() failed : %s\n",errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("ControlService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -447,6 +447,7 @@ static char * nt_service_command_line(int *using_default_torrc) { TCHAR tor_exe[MAX_PATH+1]; + char tor_exe_ascii[MAX_PATH+1]; char *command, *options=NULL; smartlist_t *sl; int i, cmdlen; @@ -472,18 +473,25 @@ nt_service_command_line(int *using_default_torrc) options = smartlist_join_strings(sl,"\" \"",0,NULL); smartlist_free(sl); +#ifdef UNICODE + wcstombs(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#else + strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#endif + /* Allocate a string for the NT service command line */ - cmdlen = strlen(tor_exe) + (options?strlen(options):0) + 32; + cmdlen = strlen(tor_exe_ascii) + (options?strlen(options):0) + 32; command = tor_malloc(cmdlen); /* Format the service command */ if (options) { if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service \"%s\"", - tor_exe, options)<0) { + tor_exe_ascii, options)<0) { tor_free(command); /* sets command to NULL. */ } } else { /* ! options */ - if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", tor_exe)<0) { + if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", + tor_exe_ascii)<0) { tor_free(command); /* sets command to NULL. */ } } @@ -508,7 +516,7 @@ nt_service_install(int argc, char **argv) SC_HANDLE hSCManager = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION sdBuff; + SERVICE_DESCRIPTIONA sdBuff; char *command; char *errmsg; const char *user_acct = GENSRV_USERACCT; @@ -598,10 +606,10 @@ nt_service_install(int argc, char **argv) SERVICE_AUTO_START, SERVICE_ERROR_IGNORE, command, NULL, NULL, NULL, user_acct, password)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("CreateService() failed : %s\n", errmsg); service_fns.CloseServiceHandle_fn(hSCManager); - LocalFree(errmsg); + tor_free(errmsg); tor_free(command); return -1; } @@ -642,9 +650,9 @@ nt_service_remove(void) nt_service_stop(hService); if (service_fns.DeleteService_fn(hService) == FALSE) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("DeleteService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); service_fns.CloseServiceHandle_fn(hService); service_fns.CloseServiceHandle_fn(hSCManager); return -1; @@ -701,20 +709,6 @@ nt_service_cmd_stop(void) return stop; } -/** Given a Win32 error code, this attempts to make Windows - * return a human-readable error message. The char* returned - * is allocated by Windows, but should be freed with LocalFree() - * when finished with it. */ -static char* -nt_strerror(uint32_t errnum) -{ - char *msgbuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&msgbuf, 0, NULL); - return msgbuf; -} - int nt_service_parse_options(int argc, char **argv, int *should_exit) { diff --git a/src/or/onion.c b/src/or/onion.c index ebc358364..fa001656e 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -63,15 +63,16 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) if (ol_length >= get_options()->MaxOnionsPending) { #define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) - static time_t last_warned = 0; - time_t now = time(NULL); - if (last_warned + WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL < now) { + static ratelim_t last_warned = + RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { log_warn(LD_GENERAL, "Your computer is too slow to handle this many circuit " "creation requests! Please consider using the " "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); - last_warned = now; + "restricted exit policy.%s",m); + tor_free(m); } tor_free(tmp); return -1; diff --git a/src/or/or.h b/src/or/or.h index 572dc8b96..3c109738d 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -59,6 +59,9 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif #ifdef HAVE_ARPA_INET_H #include <arpa/inet.h> #endif @@ -551,7 +554,7 @@ typedef enum { #define END_STREAM_REASON_DESTROY 5 #define END_STREAM_REASON_DONE 6 #define END_STREAM_REASON_TIMEOUT 7 -/* 8 is unallocated for historical reasons. */ +#define END_STREAM_REASON_NOROUTE 8 #define END_STREAM_REASON_HIBERNATING 9 #define END_STREAM_REASON_INTERNAL 10 #define END_STREAM_REASON_RESOURCELIMIT 11 @@ -1453,6 +1456,9 @@ typedef struct { * directory according to the authorities. */ unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this * router rejects everything. */ + /** True if, after we have added this router, we should re-launch + * tests for it. */ + unsigned int needs_retest_if_added:1; /** Tor can use this router for general positions in circuits. */ #define ROUTER_PURPOSE_GENERAL 0 @@ -2842,6 +2848,12 @@ typedef struct { time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + time_t BWHistoryDirReadEnds; + int BWHistoryDirReadInterval; + smartlist_t *BWHistoryDirReadValues; + time_t BWHistoryDirWriteEnds; + int BWHistoryDirWriteInterval; + smartlist_t *BWHistoryDirWriteValues; /** Build time histogram */ config_line_t * BuildtimeHistogram; @@ -3265,6 +3277,8 @@ typedef enum { DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 } dirreq_state_t; +#define WRITE_STATS_INTERVAL (24*60*60) + /********************************* microdesc.c *************************/ typedef struct microdesc_cache_t microdesc_cache_t; diff --git a/src/or/policies.c b/src/or/policies.c index db3c6d886..4fd090415 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -381,7 +381,7 @@ validate_addr_policies(or_options_t *options, char **msg) ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); -err: + err: addr_policy_list_free(addr_policy); return *msg ? -1 : 0; #undef REJECT @@ -1272,7 +1272,7 @@ policy_summarize(smartlist_t *policy) result = tor_malloc(final_size); tor_snprintf(result, final_size, "%s %s", prefix, shorter_str); -cleanup: + cleanup: /* cleanup */ SMARTLIST_FOREACH(summary, policy_summary_item_t *, s, tor_free(s)); smartlist_free(summary); diff --git a/src/or/reasons.c b/src/or/reasons.c index 2dd5fe946..ade9a3abf 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -28,6 +28,7 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_DESTROY: return "DESTROY"; case END_STREAM_REASON_DONE: return "DONE"; case END_STREAM_REASON_TIMEOUT: return "TIMEOUT"; + case END_STREAM_REASON_NOROUTE: return "NOROUTE"; case END_STREAM_REASON_HIBERNATING: return "HIBERNATING"; case END_STREAM_REASON_INTERNAL: return "INTERNAL"; case END_STREAM_REASON_RESOURCELIMIT: return "RESOURCELIMIT"; @@ -62,6 +63,7 @@ stream_end_reason_to_string(int reason) case END_STREAM_REASON_DESTROY: return "destroyed"; case END_STREAM_REASON_DONE: return "closed normally"; case END_STREAM_REASON_TIMEOUT: return "gave up (timeout)"; + case END_STREAM_REASON_NOROUTE: return "no route to host"; case END_STREAM_REASON_HIBERNATING: return "server is hibernating"; case END_STREAM_REASON_INTERNAL: return "internal error at server"; case END_STREAM_REASON_RESOURCELIMIT: return "server out of resources"; @@ -104,6 +106,8 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_SUCCEEDED; case END_STREAM_REASON_TIMEOUT: return SOCKS5_TTL_EXPIRED; + case END_STREAM_REASON_NOROUTE: + return SOCKS5_HOST_UNREACHABLE; case END_STREAM_REASON_RESOURCELIMIT: return SOCKS5_GENERAL_ERROR; case END_STREAM_REASON_HIBERNATING: @@ -164,6 +168,14 @@ errno_to_stream_end_reason(int e) S_CASE(ENOTCONN): S_CASE(ENETUNREACH): return END_STREAM_REASON_INTERNAL; + S_CASE(EHOSTUNREACH): + /* XXXX022 + * The correct behavior is END_STREAM_REASON_NOROUTE, but older + * clients don't recognize it. So we're going to continue sending + * "MISC" until 0.2.1.27 or later is "well established". + */ + /* return END_STREAM_REASON_NOROUTE; */ + return END_STREAM_REASON_MISC; S_CASE(ECONNREFUSED): return END_STREAM_REASON_CONNECTREFUSED; S_CASE(ECONNRESET): diff --git a/src/or/relay.c b/src/or/relay.c index 6d51f18a3..a4f7718d9 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -38,26 +38,32 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int -connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); -static void -circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); -static void -circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); -static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, - circuit_t *circ, - crypt_path_t *layer_hint); -static int -circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); +static void circuit_consider_sending_sendme(circuit_t *circ, + crypt_path_t *layer_hint); +static void circuit_resume_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_resume_edge_reading_helper(edge_connection_t *conn, + circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_consider_stop_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_queue_streams_are_blocked(circuit_t *circ); /** Cache the current hi-res time; the cache gets reset when libevent * calls us. */ static struct timeval cached_time_hires = {0, 0}; +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +#define CELL_QUEUE_HIGHWATER_SIZE 256 +/** Start reading from edge connections again when we get down to this many + * cells. */ +#define CELL_QUEUE_LOWWATER_SIZE 64 + static void tor_gettimeofday_cached(struct timeval *tv) { @@ -268,7 +274,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * we might kill the circ before we relay * the cells. */ - append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction, 0); return 0; } @@ -365,7 +371,7 @@ relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, static int circuit_package_relay_cell(cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, - crypt_path_t *layer_hint) + crypt_path_t *layer_hint, streamid_t on_stream) { or_connection_t *conn; /* where to send the cell */ @@ -409,7 +415,7 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } ++stats_n_relay_cells_relayed; - append_cell_to_circuit_queue(circ, conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, conn, cell, cell_direction, on_stream); return 0; } @@ -536,7 +542,7 @@ relay_command_to_string(uint8_t command) * return 0. */ int -relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, +relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, uint8_t relay_command, const char *payload, size_t payload_len, crypt_path_t *cpath_layer) { @@ -624,8 +630,8 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, } } - if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer) - < 0) { + if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, + stream_id) < 0) { log_warn(LD_BUG,"circuit_package_relay_cell failed. Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); return -1; @@ -692,7 +698,8 @@ edge_reason_is_retriable(int reason) reason == END_STREAM_REASON_RESOURCELIMIT || reason == END_STREAM_REASON_EXITPOLICY || reason == END_STREAM_REASON_RESOLVEFAILED || - reason == END_STREAM_REASON_MISC; + reason == END_STREAM_REASON_MISC || + reason == END_STREAM_REASON_NOROUTE; } /** Called when we receive an END cell on a stream that isn't open yet, @@ -787,6 +794,7 @@ connection_ap_process_end_not_open( case END_STREAM_REASON_RESOLVEFAILED: case END_STREAM_REASON_TIMEOUT: case END_STREAM_REASON_MISC: + case END_STREAM_REASON_NOROUTE: if (client_dns_incr_failures(conn->socks_request->address) < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ @@ -945,7 +953,7 @@ connection_edge_process_relay_cell_not_open( } /* handle anything that might have queued */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -1235,9 +1243,13 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); + if (circuit_queue_streams_are_blocked(circ)) { + /* Still waiting for queue to flush; don't touch conn */ + return 0; + } connection_start_reading(TO_CONN(conn)); /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -1303,15 +1315,19 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; -/** While conn->inbuf has an entire relay payload of bytes on it, - * and the appropriate package windows aren't empty, grab a cell - * and send it down the circuit. +/** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or + * <b>package_partial</b> is true), and the appropriate package windows aren't + * empty, grab a cell and send it down the circuit. + * + * If *<b>max_cells</b> is given, package no more than max_cells. Decrement + * *<b>max_cells</b> by the number of cells packaged. * * Return -1 (and send a RELAY_COMMAND_END cell if necessary) if conn should * be marked for close, else return 0. */ int -connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) +connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, + int *max_cells) { size_t amount_to_process, length; char payload[CELL_PAYLOAD_SIZE]; @@ -1327,7 +1343,10 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) return 0; } -repeat_connection_edge_package_raw_inbuf: + if (max_cells && *max_cells <= 0) + return 0; + + repeat_connection_edge_package_raw_inbuf: circ = circuit_get_by_edge_conn(conn); if (!circ) { @@ -1388,6 +1407,12 @@ repeat_connection_edge_package_raw_inbuf: } log_debug(domain,"conn->package_window is now %d",conn->package_window); + if (max_cells) { + *max_cells -= 1; + if (*max_cells <= 0) + return 0; + } + /* handle more if there's more, or return 0 if there isn't */ goto repeat_connection_edge_package_raw_inbuf; } @@ -1435,7 +1460,10 @@ connection_edge_consider_sending_sendme(edge_connection_t *conn) static void circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) { - + if (circuit_queue_streams_are_blocked(circ)) { + log_debug(layer_hint?LD_APP:LD_EXIT,"Too big queue, no resuming"); + return; + } log_debug(layer_hint?LD_APP:LD_EXIT,"resuming"); if (CIRCUIT_IS_ORIGIN(circ)) @@ -1451,31 +1479,100 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) * of a linked list of edge streams that should each be considered. */ static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, +circuit_resume_edge_reading_helper(edge_connection_t *first_conn, circuit_t *circ, crypt_path_t *layer_hint) { - for ( ; conn; conn=conn->next_stream) { - if (conn->_base.marked_for_close) + edge_connection_t *conn; + int n_streams, n_streams_left; + int packaged_this_round; + int cells_on_queue; + int cells_per_conn; + + /* How many cells do we have space for? It will be the minimum of + * the number needed to exhaust the package window, and the minimum + * needed to fill the cell queue. */ + int max_to_package = circ->package_window; + if (CIRCUIT_IS_ORIGIN(circ)) { + cells_on_queue = circ->n_conn_cells.n; + } else { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + cells_on_queue = or_circ->p_conn_cells.n; + } + if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) + max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + + /* Count how many non-marked streams there are that have anything on + * their inbuf, and enable reading on all of the connections. */ + n_streams = 0; + for (conn=first_conn; conn; conn=conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) continue; - if ((!layer_hint && conn->package_window > 0) || - (layer_hint && conn->package_window > 0 && - conn->cpath_layer == layer_hint)) { + if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_streams; + } + } + + if (n_streams == 0) /* avoid divide-by-zero */ + return 0; + + again: + + cells_per_conn = CEIL_DIV(max_to_package, n_streams); + + packaged_this_round = 0; + n_streams_left = 0; + + /* Iterate over all connections. Package up to cells_per_conn cells on + * each. Update packaged_this_round with the total number of cells + * packaged, and n_streams_left with the number that still have data to + * package. + */ + for (conn=first_conn; conn; conn=conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + int n = cells_per_conn, r; /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1)<0) { - /* (We already sent an end cell if possible) */ + r = connection_edge_package_raw_inbuf(conn, 1, &n); + + /* Note how many we packaged */ + packaged_this_round += (cells_per_conn-n); + + if (r<0) { + /* Problem while packaging. (We already sent an end cell if + * possible) */ connection_mark_for_close(TO_CONN(conn)); continue; } + /* If there's still data to read, we'll be coming back to this stream. */ + if (buf_datalen(conn->_base.inbuf)) + ++n_streams_left; + /* If the circuit won't accept any more data, return without looking * at any more of the streams. Any connections that should be stopped * have already been stopped by connection_edge_package_raw_inbuf. */ if (circuit_consider_stop_edge_reading(circ, layer_hint)) return -1; + /* XXXX should we also stop immediately if we fill up the cell queue? + * Probably. */ } } + + /* If we made progress, and we are willing to package more, and there are + * any streams left that want to package stuff... try again! + */ + if (packaged_this_round && packaged_this_round < max_to_package && + n_streams_left) { + max_to_package -= packaged_this_round; + n_streams = n_streams_left; + goto again; + } + return 0; } @@ -1544,13 +1641,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - #ifdef ACTIVE_CIRCUITS_PARANOIA #define assert_active_circuits_ok_paranoid(conn) \ assert_active_circuits_ok(conn) @@ -2091,12 +2181,19 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) * every edge connection that is using <b>circ</b> to write to <b>orconn</b>, - * and start or stop reading as appropriate. */ -static void + * and start or stop reading as appropriate. + * + * If <b>stream_id</b> is nonzero, block only the edge connection whose + * stream_id matches it. + * + * Returns the number of streams whose status we changed. + */ +static int set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, - int block) + int block, streamid_t stream_id) { edge_connection_t *edge = NULL; + int n = 0; if (circ->n_conn == orconn) { circ->streams_blocked_on_n_conn = block; if (CIRCUIT_IS_ORIGIN(circ)) @@ -2109,7 +2206,13 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, for (; edge; edge = edge->next_stream) { connection_t *conn = TO_CONN(edge); - edge->edge_blocked_on_circ = block; + if (stream_id && edge->stream_id != stream_id) + continue; + + if (edge->edge_blocked_on_circ != block) { + ++n; + edge->edge_blocked_on_circ = block; + } if (!conn->read_event) { /* This connection is a placeholder for something; probably a DNS @@ -2126,6 +2229,8 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, connection_start_reading(conn); } } + + return n; } /** Pull as many cells as possible (but no more than <b>max</b>) from the @@ -2251,7 +2356,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) - set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ + set_streams_blocked_on_circ(circ, conn, 0, 0); /* unblock streams */ /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { @@ -2268,7 +2373,8 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, * transmitting in <b>direction</b>. */ void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction) + cell_t *cell, cell_direction_t direction, + streamid_t fromstream) { cell_queue_t *queue; int streams_blocked; @@ -2293,7 +2399,12 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) - set_streams_blocked_on_circ(circ, orconn, 1); /* block streams */ + set_streams_blocked_on_circ(circ, orconn, 1, 0); /* block streams */ + + if (streams_blocked && fromstream) { + /* This edge connection is apparently not blocked; block it. */ + set_streams_blocked_on_circ(circ, orconn, 1, fromstream); + } if (queue->n == 1) { /* This was the first cell added to the queue. We need to make this @@ -2426,3 +2537,15 @@ assert_active_circuits_ok(or_connection_t *orconn) tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); } +/** Return 1 if we shouldn't restart reading on this circuit, even if + * we get a SENDME. Else return 0. +*/ +static int +circuit_queue_streams_are_blocked(circuit_t *circ) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + return circ->streams_blocked_on_n_conn; + } else { + return circ->streams_blocked_on_p_conn; + } +} diff --git a/src/or/relay.h b/src/or/relay.h index 7fb0655ef..08a1ffe78 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -27,7 +27,8 @@ int connection_edge_send_command(edge_connection_t *fromconn, uint8_t relay_command, const char *payload, size_t payload_len); int connection_edge_package_raw_inbuf(edge_connection_t *conn, - int package_partial); + int package_partial, + int *max_cells); void connection_edge_consider_sending_sendme(edge_connection_t *conn); extern uint64_t stats_n_data_cells_packaged; @@ -45,7 +46,8 @@ void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction); + cell_t *cell, cell_direction_t direction, + streamid_t fromstream); void connection_or_unlink_all_active_circs(or_connection_t *conn); int connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, time_t now); diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 0377f121c..68abb886a 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -209,7 +209,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, introcirc->_base.purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT; return 0; -err: + err: circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL); return -1; diff --git a/src/or/rendservice.h b/src/or/rendservice.h index f979a3941..1767714c6 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -15,7 +15,6 @@ int num_rend_services(void); int rend_config_services(or_options_t *options, int validate_only); int rend_service_load_keys(void); -void rend_services_init(void); void rend_services_introduce(void); void rend_consider_services_upload(time_t now); void rend_hsdir_routers_changed(void); diff --git a/src/or/rephist.c b/src/or/rephist.c index a419f31e7..056fc5cc1 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -6,7 +6,8 @@ * \file rephist.c * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've - * been using, which ports we tend to want, and so on. + * been using, which ports we tend to want, and so on; further, + * exit port statistics and cell statistics. **/ #include "or.h" @@ -1283,13 +1284,21 @@ bw_array_new(void) static bw_array_t *read_array = NULL; /** Recent history of bandwidth observations for write operations. */ static bw_array_t *write_array = NULL; - -/** Set up read_array and write_array. */ +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up [dir-]read_array and [dir-]write_array. */ static void bw_arrays_init(void) { read_array = bw_array_new(); write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); } /** We read <b>num_bytes</b> more bytes in second <b>when</b>. @@ -1323,200 +1332,22 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } -/* Some constants */ -/** To what multiple should byte numbers be rounded up? */ -#define EXIT_STATS_ROUND_UP_BYTES 1024 -/** To what multiple should stream counts be rounded up? */ -#define EXIT_STATS_ROUND_UP_STREAMS 4 -/** Number of TCP ports */ -#define EXIT_STATS_NUM_PORTS 65536 -/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to - * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 - -/* The following data structures are arrays and no fancy smartlists or maps, - * so that all write operations can be done in constant time. This comes at - * the price of some memory (1.25 MB) and linear complexity when writing - * stats for measuring relays. */ -/** Number of bytes read in current period by exit port */ -static uint64_t *exit_bytes_read = NULL; -/** Number of bytes written in current period by exit port */ -static uint64_t *exit_bytes_written = NULL; -/** Number of streams opened in current period by exit port */ -static uint32_t *exit_streams = NULL; - -/** When does the current exit stats period end? */ -static time_t start_of_exit_stats_interval; - -/** Initialize exit port stats. */ -void -rep_hist_exit_stats_init(time_t now) -{ - start_of_exit_stats_interval = now; - exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint32_t)); -} - -/** Write exit stats to $DATADIR/stats/exit-stats and reset counters. */ -void -rep_hist_exit_stats_write(time_t now) -{ - char t[ISO_TIME_LEN+1]; - int r, i, comma; - uint64_t *b, total_bytes, threshold_bytes, other_bytes; - uint32_t other_streams; - - char *statsdir = NULL, *filename = NULL; - open_file_t *open_file = NULL; - FILE *out = NULL; - - if (!exit_streams) - return; /* Not initialized */ - - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE) < 0) - goto done; - filename = get_datadir_fname2("stats", "exit-stats"); - format_iso_time(t, now); - log_info(LD_HIST, "Writing exit port statistics to disk for period " - "ending at %s.", t); - - if (!open_file) { - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, - 0600, &open_file); - if (!out) { - log_warn(LD_HIST, "Couldn't open '%s'.", filename); - goto done; - } - } - - /* written yyyy-mm-dd HH:MM:SS (n s) */ - if (fprintf(out, "exit-stats-end %s (%d s)\n", t, - (unsigned) (now - start_of_exit_stats_interval)) < 0) - goto done; - - /* Count the total number of bytes, so that we can attribute all - * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ - total_bytes = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; - } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - - /* exit-kibibytes-(read|written) port=kibibytes,.. */ - for (r = 0; r < 2; r++) { - b = r ? exit_bytes_read : exit_bytes_written; - tor_assert(b); - if (fprintf(out, "%s ", - r ? "exit-kibibytes-read" - : "exit-kibibytes-written") < 0) - goto done; - - comma = 0; - other_bytes = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (b[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - uint64_t num = round_uint64_to_next_multiple_of(b[i], - EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - if (fprintf(out, "%s%d="U64_FORMAT, - comma++ ? "," : "", i, - U64_PRINTF_ARG(num)) < 0) - goto done; - } else - other_bytes += b[i]; - } - } - other_bytes = round_uint64_to_next_multiple_of(other_bytes, - EXIT_STATS_ROUND_UP_BYTES); - other_bytes /= 1024; - if (fprintf(out, "%sother="U64_FORMAT"\n", - comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) - goto done; - } - /* exit-streams-opened port=num,.. */ - if (fprintf(out, "exit-streams-opened ") < 0) - goto done; - comma = 0; - other_streams = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_streams[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%s%d=%u", - comma++ ? "," : "", i, num)<0) - goto done; - } else - other_streams += exit_streams[i]; - } - } - other_streams = round_uint32_to_next_multiple_of(other_streams, - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%sother=%u\n", - comma ? "," : "", other_streams)<0) - goto done; - /* Reset counters */ - memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); - memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); - memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); - start_of_exit_stats_interval = now; - - if (open_file) - finish_writing_to_file(open_file); - open_file = NULL; - done: - if (open_file) - abort_writing_to_file(open_file); - tor_free(filename); - tor_free(statsdir); -} - -/** Note that we wrote <b>num_bytes</b> to an exit connection to - * <b>port</b>. */ -void -rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) -{ - if (!get_options()->ExitPortStatistics) - return; - if (!exit_bytes_written) - return; /* Not initialized */ - exit_bytes_written[port] += num_bytes; - log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", - (unsigned long)num_bytes, port); -} - -/** Note that we read <b>num_bytes</b> from an exit connection to - * <b>port</b>. */ +/** We wrote <b>num_bytes</b> more directory bytes in second <b>when</b>. + * (like rep_hist_note_bytes_written() above) + */ void -rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) +rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when) { - if (!get_options()->ExitPortStatistics) - return; - if (!exit_bytes_read) - return; /* Not initialized */ - exit_bytes_read[port] += num_bytes; - log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", - (unsigned long)num_bytes, port); + add_obs(dir_write_array, when, num_bytes); } -/** Note that we opened an exit stream to <b>port</b>. */ +/** We read <b>num_bytes</b> more directory bytes in second <b>when</b>. + * (like rep_hist_note_bytes_written() above) + */ void -rep_hist_note_exit_stream_opened(uint16_t port) +rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when) { - if (!get_options()->ExitPortStatistics) - return; - if (!exit_streams) - return; /* Not initialized */ - exit_streams[port]++; - log_debug(LD_HIST, "Opened exit stream to port %d", port); + add_obs(dir_read_array, when, num_bytes); } /** Helper: Return the largest value in b->maxima. (This is equal to the @@ -1554,9 +1385,9 @@ rep_hist_bandwidth_assess(void) return (int)(U64_TO_DBL(r)/NUM_SECS_ROLLING_MEASURE); } -/** Print the bandwidth history of b (either read_array or write_array) - * into the buffer pointed to by buf. The format is simply comma - * separated numbers, from oldest to newest. +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. * * It returns the number of bytes written. */ @@ -1614,20 +1445,37 @@ rep_hist_get_bandwidth_lines(int for_extrainfo) char *buf, *cp; char t[ISO_TIME_LEN+1]; int r; - bw_array_t *b; + bw_array_t *b = NULL; + const char *desc = NULL; size_t len; - /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+21*NUM_TOTALS)*2; + /* opt [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ + len = (67+21*NUM_TOTALS)*4; buf = tor_malloc_zero(len); cp = buf; - for (r=0;r<2;++r) { - b = r?read_array:write_array; + for (r=0;r<4;++r) { + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } tor_assert(b); format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ", - for_extrainfo ? "" : "opt ", - r ? "read-history" : "write-history", t, + for_extrainfo ? "" : "opt ", desc, t, NUM_SECS_BW_SUM_INTERVAL); cp += strlen(cp); cp += rep_hist_fill_bandwidth_history(cp, len-(cp-buf), b); @@ -1643,20 +1491,41 @@ rep_hist_update_state(or_state_t *state) { int len, r; char *buf, *cp; - smartlist_t **s_values; - time_t *s_begins; - int *s_interval; - bw_array_t *b; + smartlist_t **s_values = NULL; + time_t *s_begins = NULL; + int *s_interval = NULL; + bw_array_t *b = NULL; len = 20*NUM_TOTALS+1; buf = tor_malloc_zero(len); - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?&state->BWHistoryReadEnds :&state->BWHistoryWriteEnds; - s_interval= r?&state->BWHistoryReadInterval:&state->BWHistoryWriteInterval; - s_values = r?&state->BWHistoryReadValues :&state->BWHistoryWriteValues; - + for (r=0;r<4;++r) { + switch (r) { + case 0: + b = write_array; + s_begins = &state->BWHistoryWriteEnds; + s_interval = &state->BWHistoryWriteInterval; + s_values = &state->BWHistoryWriteValues; + break; + case 1: + b = read_array; + s_begins = &state->BWHistoryReadEnds; + s_interval = &state->BWHistoryReadInterval; + s_values = &state->BWHistoryReadValues; + break; + case 2: + b = dir_write_array; + s_begins = &state->BWHistoryDirWriteEnds; + s_interval = &state->BWHistoryDirWriteInterval; + s_values = &state->BWHistoryDirWriteValues; + break; + case 3: + b = dir_read_array; + s_begins = &state->BWHistoryDirReadEnds; + s_interval = &state->BWHistoryDirReadInterval; + s_values = &state->BWHistoryDirReadValues; + break; + } if (*s_values) { SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); smartlist_free(*s_values); @@ -1696,23 +1565,45 @@ rep_hist_update_state(or_state_t *state) int rep_hist_load_state(or_state_t *state, char **err) { - time_t s_begins, start; + time_t s_begins = 0, start; time_t now = time(NULL); uint64_t v; int r,i,ok; int all_ok = 1; - int s_interval; - smartlist_t *s_values; - bw_array_t *b; + int s_interval = 0; + smartlist_t *s_values = NULL; + bw_array_t *b = NULL; /* Assert they already have been malloced */ tor_assert(read_array && write_array); - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?state->BWHistoryReadEnds:state->BWHistoryWriteEnds; - s_interval = r?state->BWHistoryReadInterval:state->BWHistoryWriteInterval; - s_values = r?state->BWHistoryReadValues:state->BWHistoryWriteValues; + for (r=0;r<4;++r) { + switch (r) { + case 0: + b = write_array; + s_begins = state->BWHistoryWriteEnds; + s_interval = state->BWHistoryWriteInterval; + s_values = state->BWHistoryWriteValues; + break; + case 1: + b = read_array; + s_begins = state->BWHistoryReadEnds; + s_interval = state->BWHistoryReadInterval; + s_values = state->BWHistoryReadValues; + break; + case 2: + b = dir_write_array; + s_begins = state->BWHistoryDirWriteEnds; + s_interval = state->BWHistoryDirWriteInterval; + s_values = state->BWHistoryDirWriteValues; + break; + case 3: + b = dir_read_array; + s_begins = state->BWHistoryDirReadEnds; + s_interval = state->BWHistoryDirReadInterval; + s_values = state->BWHistoryDirReadValues; + break; + } if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { start = s_begins - s_interval*(smartlist_len(s_values)); if (start > now) @@ -2044,25 +1935,242 @@ dump_pk_ops(int severity) pk_op_counts.n_rend_server_ops); } -/** Free all storage held by the OR/link history caches, by the - * bandwidth history arrays, or by the port history. */ +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to + * see in order to be included in exit stats. */ +#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ void -rep_hist_free_all(void) +rep_hist_exit_stats_init(time_t now) { - digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(last_stability_doc); + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); +} + +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) +{ + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); +} + +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) +{ + start_of_exit_stats_interval = 0; tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); - built_last_stability_doc_at = 0; - predicted_ports_free(); +} + +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) +{ + int i; + uint64_t total_bytes = 0, threshold_bytes, other_read = 0, + other_written = 0; + uint32_t other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; + + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ + + /* Count total number of bytes, so that we can attribute observations + * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; + } + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; + + /* Add observations of all ports above the threshold to smartlists and + * join them to single strings. Also count bytes and streams of ports + * below or equal to the threshold. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + if (exit_bytes_written[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + } + if (exit_bytes_read[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + } + if (exit_streams[i] > 0) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", i, num); + smartlist_add(streams_strings, buf); + } + } else { + other_read += exit_bytes_read[i]; + other_written += exit_bytes_written[i]; + other_streams += exit_streams[i]; + } + } + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; +} + +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + log_info(LD_HIST, "Writing exit port statistics to disk."); + + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); + + /* Reset counters. */ + rep_hist_reset_exit_stats(now); + + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; + } + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); + + done: + tor_free(str); + tor_free(statsdir); + tor_free(filename); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ +void +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) +{ + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); +} + +/** Note that we opened an exit stream to <b>port</b>. */ +void +rep_hist_note_exit_stream_opened(uint16_t port) +{ + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); } /*** cell statistics ***/ -/** Start of the current buffer stats interval. */ +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ static time_t start_of_buffer_stats_interval; /** Initialize buffer stats. */ @@ -2132,8 +2240,22 @@ _buffer_stats_compare_entries(const void **_a, const void **_b) return 0; } -/** Write buffer statistics to $DATADIR/stats/buffer-stats. */ +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ void +rep_hist_buffer_stats_term(void) +{ + start_of_buffer_stats_interval = 0; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); +} + +/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when + * we would next want to write exit stats. */ +time_t rep_hist_buffer_stats_write(time_t now) { char *statsdir = NULL, *filename = NULL; @@ -2147,6 +2269,12 @@ rep_hist_buffer_stats_write(time_t now) smartlist_t *str_build = smartlist_create(); char *str = NULL, *buf=NULL; circuit_t *circ; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + /* add current circuits to stats */ for (circ = _circuit_get_global_list(); circ; circ = circ->next) rep_hist_buffer_stats_add_circ(circ, now); @@ -2244,5 +2372,22 @@ rep_hist_buffer_stats_write(time_t now) } tor_free(str); #undef SHARES + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ +void +rep_hist_free_all(void) +{ + digestmap_free(history_map, free_or_history); + tor_free(read_array); + tor_free(write_array); + tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + built_last_stability_doc_at = 0; + predicted_ports_free(); } diff --git a/src/or/rephist.h b/src/or/rephist.h index a84589221..c3914dcaf 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -23,11 +23,10 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes); -void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes); -void rep_hist_note_exit_stream_opened(uint16_t port); -void rep_hist_exit_stats_init(time_t now); -void rep_hist_exit_stats_write(time_t now); + +void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); + int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); @@ -62,18 +61,20 @@ void dump_pk_ops(int severity); void rep_hist_free_all(void); -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); void rep_hist_buffer_stats_init(time_t now); void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval); -void rep_hist_buffer_stats_write(time_t now); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); #endif diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 5f98abe01..b77107ca0 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -440,6 +440,23 @@ authority_cert_dl_failed(const char *id_digest, int status) download_status_failed(&cl->dl_status, status); } +/** Return true iff when we've been getting enough failures when trying to + * download the certificate with ID digest <b>id_digest</b> that we're willing + * to start bugging the user about it. */ +int +authority_cert_dl_looks_uncertain(const char *id_digest) +{ +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 + cert_list_t *cl; + int n_failures; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return 0; + + n_failures = download_status_get_n_failures(&cl->dl_status); + return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; +} + /** How many times will we try to fetch a certificate before giving up? */ #define MAX_CERT_DL_FAILURES 8 @@ -1257,6 +1274,13 @@ mark_all_trusteddirservers_up(void) router_dir_info_changed(); } +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port; +} + /** Reset all internal variables used to count failed downloads of network * status objects. */ void @@ -3151,15 +3175,25 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, id_digest = router->cache_info.identity_digest; + old_router = router_get_by_digest(id_digest); + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { - log_info(LD_DIR, - "Dropping descriptor that we already have for router '%s'", - router->nickname); - *msg = "Router descriptor was not new."; - routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + tor_assert(old_router); + if (! (routerinfo_is_a_configured_bridge(router) && + (router->purpose == ROUTER_PURPOSE_BRIDGE || + old_router->purpose != ROUTER_PURPOSE_BRIDGE))) { + log_info(LD_DIR, + "Dropping descriptor that we already have for router '%s'", + router->nickname); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_WAS_NOT_NEW; + } } if (authdir) { @@ -3196,15 +3230,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, { routerstatus_t *rs = - networkstatus_v2_find_entry(ns, router->cache_info.identity_digest); + networkstatus_v2_find_entry(ns, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) rs->need_to_mirror = 0; }); if (consensus) { - routerstatus_t *rs = networkstatus_vote_find_entry(consensus, - router->cache_info.identity_digest); + routerstatus_t *rs = networkstatus_vote_find_entry(consensus, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { @@ -3226,8 +3259,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } /* If we have a router with the same identity key, choose the newer one. */ - old_router = rimap_get(routerlist->identity_map, - router->cache_info.identity_digest); if (old_router) { if (!in_consensus && (router->cache_info.published_on <= old_router->cache_info.published_on)) { @@ -3246,8 +3277,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, log_debug(LD_DIR, "Replacing entry for router '%s/%s' [%s]", router->nickname, old_router->nickname, hex_str(id_digest,DIGEST_LEN)); - if (router->addr == old_router->addr && - router->or_port == old_router->or_port) { + if (routers_have_same_or_addr(router, old_router)) { /* these carry over when the address and orport are unchanged. */ router->last_reachable = old_router->last_reachable; router->testing_since = old_router->testing_since; @@ -3276,11 +3306,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, * the list. */ routerlist_insert(routerlist, router); if (!from_cache) { - if (authdir) { - /* launch an immediate reachability test, so we will have an opinion - * soon in case we're generating a consensus soon */ - dirserv_single_reachability_test(time(NULL), router); - } signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } @@ -3600,15 +3625,19 @@ routerlist_remove_old_routers(void) /** We just added a new set of descriptors. Take whatever extra steps * we need. */ -static void +void routerlist_descriptors_added(smartlist_t *sl, int from_cache) { tor_assert(sl); control_event_descriptors_changed(sl); - SMARTLIST_FOREACH(sl, routerinfo_t *, ri, + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) learned_bridge_descriptor(ri, from_cache); - ); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); } /** @@ -4191,7 +4220,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; } - n_per_request = (n_downloadable+MIN_REQUESTS-1) / MIN_REQUESTS; + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); if (n_per_request > MAX_DL_PER_REQUEST) n_per_request = MAX_DL_PER_REQUEST; if (n_per_request < MIN_DL_PER_REQUEST) @@ -4204,7 +4233,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, log_info(LD_DIR, "Launching %d request%s for %d router%s, %d at a time", - (n_downloadable+n_per_request-1)/n_per_request, + CEIL_DIV(n_downloadable, n_per_request), req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { diff --git a/src/or/routerlist.h b/src/or/routerlist.h index e31b07aef..574bce7ff 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -24,6 +24,7 @@ void authority_cert_get_all(smartlist_t *certs_out); void authority_cert_dl_failed(const char *id_digest, int status); void authority_certs_fetch_missing(networkstatus_t *status, time_t now); int router_reload_router_list(void); +int authority_cert_dl_looks_uncertain(const char *id_digest); smartlist_t *router_get_trusted_dir_servers(void); routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); @@ -35,6 +36,7 @@ int router_get_my_share_of_directory_requests(double *v2_share_out, void router_reset_status_download_failures(void); void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); +int routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2); void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, int must_be_running); int router_nickname_is_in_list(routerinfo_t *router, const char *list); @@ -115,6 +117,7 @@ was_router_added_t router_add_to_routerlist(routerinfo_t *router, was_router_added_t router_add_extrainfo_to_routerlist( extrainfo_t *ei, const char **msg, int from_cache, int from_fetch); +void routerlist_descriptors_added(smartlist_t *sl, int from_cache); void routerlist_remove_old_routers(void); int router_load_single_router(const char *s, uint8_t purpose, int cache, const char **msg); diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 2313f68e0..e5ebf0761 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -30,11 +30,6 @@ int router_parse_list_from_string(const char **s, const char *eos, int is_extrainfo, int allow_annotations, const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); int router_parse_runningrouters(const char *str); int router_parse_directory(const char *str); |