diff options
42 files changed, 472 insertions, 356 deletions
@@ -1,12 +1,22 @@ -Changes in version 0.2.2.15-alpha - 2010-08-?? +Changes in version 0.2.2.15-alpha - 2010-08-1? o Major bugfixes: - Stop assigning the HSDir flag to relays that disable their DirPort (and thus will refuse to answer directory requests). This fix should dramatically improve the reachability of hidden services: - hidden services and hidden service clients pick three HSDir relays + hidden services and hidden service clients pick six HSDir relays to store and retrieve the hidden service descriptor, and currently about half of the HSDir relays will refuse to work. Bugfix on 0.2.0.10-alpha; fixes part of bug 1693. + - The PerConnBWRate and Burst config options, along with the + bwconnrate and bwconnburst consensus params, initialized each conn's + token bucket values only when the connection is established. Now we + update them if the config options change, and update them every time + we get a new consensus. Otherwise we can encounter an ugly edge + case where we initialize an OR conn to client-level bandwidth, + but then later the relay joins the consensus and we leave it + throttled. Bugfix on 0.2.2.7-alpha; fixes bug 1830. + - Fix a regression that caused Tor to rebind its ports if it receives + SIGHUP while hibernating. Bugfix in 0.1.1.6-alpha; closes bug 919. o Major features: - Lower the maximum weighted-fractional-uptime cutoff to 98%. This @@ -35,19 +45,19 @@ Changes in version 0.2.2.15-alpha - 2010-08-?? - Take a first step towards making or.h smaller by splitting out function definitions for all source files in src/or/. Leave structures and defines in or.h for now. + - New unit tests for exit-port history statistics; refactored exit + statistics code to be more easily tested. o Minor bugfixes (on 0.2.1.x and earlier): - Complain if PublishServerDescriptor is given multiple arguments that - include 0 or 1. This configuration will be rejected in future. + include 0 or 1. This configuration will be rejected in the future. Bugfix on 0.2.0.1-alpha; closes bug 1107. + - Disallow BridgeRelay 1 and ORPort 0 at once in the configuration. + Bugfix on 0.2.0.13-alpha; closes bug 928. - Change "Application request when we're believed to be offline." notice to "Application request when we haven't used client functionality lately.", to clarify that it's not an error. Bugfix on 0.0.9.3; fixes bug 1222. - - Fix a regression that caused Tor to rebind its ports if it receives - SIGHUP while hibernating. Bugfix in 0.1.1.6-alpha; closes bug 919. - - Disallow BridgeRelay 1 and ORPort 0 at once in the configuration. - Bugfix on 0.2.0.13-alpha; closes bug 928. - Fix a bug in the controller interface where "GETINFO ns/asdaskljkl" would return "551 Internal error" rather than "552 Unrecognized key ns/asdaskljkl". Bugfix on 0.1.2.3-alpha. @@ -60,6 +70,13 @@ Changes in version 0.2.2.15-alpha - 2010-08-?? - Fix a rare bug in rend_fn unit tests: we would fail a test when a randomly generated port is 0. Diagnosed by Matt Edman. Bugfix on 0.2.0.10-alpha; fixes bug 1808. + - Exit nodes didn't recognize EHOSTUNREACH as a plausible error code, + and so sent back END_STREAM_REASON_MISC. They now send a new stream + ending reason: END_STREAM_REASON_NOROUTE. Also update the spec to + reflect this new reason. Bugfix on 0.1.0.1-rc; fixes bug 1793. + - Instead of giving an assertion failure on an internal mismatch + on estimated freelist size, just log a BUG warning and try later. + Mitigates but does not fix bug 1125. o Minor bugfixes (on 0.2.2.x): - Alter directory authorities to always consider Exit-flagged nodes @@ -74,6 +91,20 @@ Changes in version 0.2.2.15-alpha - 2010-08-?? old and broken version of libevent. Treat 1.4.14b-stable like 1.4.14-stable when parsing the version. Fixes bug 1731; bugfix on 0.2.2.1-alpha. + - Don't use substitution references like $(VAR:MOD) when + $(asciidoc_files) is empty -- make(1) on NetBSD transforms + '$(:x)' to 'x' rather than the empty string. This bites us in + doc/ when configured with --disable-asciidoc. Bugfix on + 0.2.2.9-alpha; fixes bug 1773. + - Fix to remove a spurious hidden service server-side log notice about + "Ancient non-dirty circuits". Bugfix on 0.2.2.14-alpha; fixes + bug 1741. + - Fix compilation with --with-dmalloc set. Bugfix on 0.2.2.6-alpha; + fixes bug 1832. + - Fix three memory leaks: one in circuit_build_times_parse_state(), + one in dirvote_add_signatures_to_pending_consensus(), and one every + time we parse a v3 network consensus. Bugfixes on 0.2.2.14-alpha, + 0.2.2.6-alpha, and 0.2.2.10-alpha respectively; fixes bug 1831. Changes in version 0.2.2.14-alpha - 2010-07-12 diff --git a/changes/bug1141 b/changes/bug1141 new file mode 100644 index 000000000..9975e418d --- /dev/null +++ b/changes/bug1141 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix an assertion failure that could occur in caches or bridge users + when using a very short voting interval on a testing network. + Diagnosed by Robert Hogan. Fixes bug 1141; bugfix on 0.2.0.8-alpha. + diff --git a/changes/headercleanup b/changes/headercleanup new file mode 100644 index 000000000..25a58fcda --- /dev/null +++ b/changes/headercleanup @@ -0,0 +1,3 @@ + o Code simplifications and refactoring: + - Remove a bunch of unused function declarations as well as a block of + #if 0'd code from the unit tests. Closes bug 1824. diff --git a/changes/makefile-var-exp b/changes/makefile-var-exp deleted file mode 100644 index 14e6a725b..000000000 --- a/changes/makefile-var-exp +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes: - - Change $(VAR:MOD) to ${VAR:MOD} -- make(1) on NetBSD substitutes - '$(:x)' to 'x' rather than the empty string. This bites us in doc/ - when configured with --disable-asciidoc. Curly braces should work - in all implementations of make(1) but this patch changes only the - places where we use the VAR:MOD expansion. Doesn't fix bug 1773; - bugfix on 0.2.2.9-alpha. - diff --git a/contrib/checkSpace.pl b/contrib/checkSpace.pl index b694abff6..6eb32e562 100755 --- a/contrib/checkSpace.pl +++ b/contrib/checkSpace.pl @@ -20,6 +20,10 @@ for $fn (@ARGV) { if (/\t/) { print " TAB:$fn:$.\n"; } + ## Warn about markers that don't have a space in front of them + if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) { + print "nosplabel:$fn:$.\n"; + } ## Warn about trailing whitespace. if (/ +$/) { print "Space\@EOL:$fn:$.\n"; diff --git a/doc/HACKING b/doc/HACKING index ac68a35a0..d8538b1f2 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -50,15 +50,16 @@ When you do a commit that needs a ChangeLog entry, add a new file to the "changes" toplevel subdirectory. It should have the format of a one-entry changelog section from the current ChangeLog file, as in - o Major bugfixes: - - Fix a potential buffer overflow. Fixes bug 9999. Bugfix on - Tor 0.3.1.4-beta. + o Major bugfixes: + - Fix a potential buffer overflow. Fixes bug 9999; bugfix on + 0.3.1.4-beta. To write a changes file, first categorize the change. Some common categories are: Minor bugfixes, Major bugfixes, Minor features, Major features, Code -simplifications and refactoring. Then say what the change does. Then, if -it's a bugfix, then mention what bug it fixes and when the bug was -introduced. +simplifications and refactoring. Then say what the change does. If +it's a bugfix, mention what bug it fixes and when the bug was +introduced. To find out which Git tag the change was introduced in, +you can use "git describe --contains <sha1 of commit>". If at all possible, try to create this file in the same commit where you are making the change. Please give it a distinctive name that no diff --git a/doc/Makefile.am b/doc/Makefile.am index 2e70fd747..68747c8d2 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -15,25 +15,27 @@ if USE_ASCIIDOC asciidoc_files = tor tor-gencert tor-resolve torify +html_in = $(asciidoc_files:=.html.in) +man_in = $(asciidoc_files:=.1.in) +txt_in = $(asciidoc_files:=.1.txt) +nodist_man_MANS = $(asciidoc_files:=.1) +doc_DATA = $(asciidoc_files:=.html) else asciidoc_files = +html_in = +man_in = +txt_in = +nodist_man_MANS = +doc_DATA = endif -html_in = ${asciidoc_files:=.html.in} - -man_in = ${asciidoc_files:=.1.in} - EXTRA_DIST = HACKING asciidoc-helper.sh \ - $(html_in) $(man_in) ${asciidoc_files:=.1.txt} \ + $(html_in) $(man_in) $(txt_in) \ tor-osx-dmg-creation.txt tor-rpm-creation.txt \ tor-win32-mingw-creation.txt -nodist_man_MANS = ${asciidoc_files:=.1} - docdir = @docdir@ -doc_DATA = ${asciidoc_files:=.html} - asciidoc_product = $(nodist_man_MANS) $(doc_DATA) SUBDIRS = spec diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 5a68864b2..31ce35074 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -92,7 +92,7 @@ 2.4. General-use tokens - ; CRLF means, "the ASCII Carriage Return character (decimal value value 13) + ; CRLF means, "the ASCII Carriage Return character (decimal value 13) ; followed by the ASCII Linefeed character (decimal value 10)." CRLF = CR LF @@ -1049,7 +1049,7 @@ Reason = "MISC" / "RESOLVEFAILED" / "CONNECTREFUSED" / "EXITPOLICY" / "DESTROY" / "DONE" / "TIMEOUT" / - "HIBERNATING" / "INTERNAL"/ "RESOURCELIMIT" / + "NOROUTE" / "HIBERNATING" / "INTERNAL"/ "RESOURCELIMIT" / "CONNRESET" / "TORPROTOCOL" / "NOTDIRECTORY" / "END" The "REASON" field is provided only for FAILED, CLOSED, and DETACHED diff --git a/doc/spec/rend-spec.txt b/doc/spec/rend-spec.txt index d440c8a81..12e20df65 100644 --- a/doc/spec/rend-spec.txt +++ b/doc/spec/rend-spec.txt @@ -9,7 +9,7 @@ RFC 2119. Read - https://www.torproject.org/doc/design-paper/tor-design.html#sec:rendezvous + https://svn.torproject.org/svn/projects/design-paper/tor-design.html#sec:rendezvous before you read this specification. It will make more sense. Rendezvous points provide location-hidden services (server diff --git a/doc/spec/tor-spec.txt b/doc/spec/tor-spec.txt index f448f6da2..2b1223ba7 100644 --- a/doc/spec/tor-spec.txt +++ b/doc/spec/tor-spec.txt @@ -835,7 +835,8 @@ see tor-design.pdf. 6 -- REASON_DONE (Anonymized TCP connection was closed) 7 -- REASON_TIMEOUT (Connection timed out, or OR timed out while connecting) - 8 -- (unallocated) [**] + 8 -- REASON_NOROUTE (Routing error while attempting to + contact destination) 9 -- REASON_HIBERNATING (OR is temporarily hibernating) 10 -- REASON_INTERNAL (Internal error at the OR) 11 -- REASON_RESOURCELIMIT (OR has no resources to fulfill request) @@ -857,8 +858,6 @@ see tor-design.pdf. [*] Older versions of Tor also send this reason when connections are reset. - [**] Due to a bug in versions of Tor through 0095, error reason 8 must - remain allocated until that version is obsolete. --- [The rest of this section describes unimplemented functionality.] diff --git a/doc/tor.1.txt b/doc/tor.1.txt index db623c8da..3b7e30bdf 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -965,23 +965,20 @@ is non-zero): **CellStatistics** **0**|**1**:: When this option is enabled, Tor writes statistics on the mean time that - cells spend in circuit queues to disk every 24 hours. Cannot be changed - while Tor is running. (Default: 0) + cells spend in circuit queues to disk every 24 hours. (Default: 0) **DirReqStatistics** **0**|**1**:: When this option is enabled, Tor writes statistics on the number and - response time of network status requests to disk every 24 hours. Cannot be - changed while Tor is running. (Default: 0) + response time of network status requests to disk every 24 hours. + (Default: 0) **EntryStatistics** **0**|**1**:: When this option is enabled, Tor writes statistics on the number of - directly connecting clients to disk every 24 hours. Cannot be changed while - Tor is running. (Default: 0) + directly connecting clients to disk every 24 hours. (Default: 0) **ExitPortStatistics** **0**|**1**:: When this option is enabled, Tor writes statistics on the number of relayed - bytes and opened stream per exit port to disk every 24 hours. Cannot be - changed while Tor is running. (Default: 0) + bytes and opened stream per exit port to disk every 24 hours. (Default: 0) **ExtraInfoStatistics** **0**|**1**:: When this option is enabled, Tor includes previously gathered statistics in diff --git a/src/or/buffers.c b/src/or/buffers.c index 3ba2760bf..e8422637c 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -270,14 +270,25 @@ buf_shrink_freelists(int free_all) (freelists[i].lowest_length - slack); int n_to_skip = freelists[i].cur_length - n_to_free; int orig_n_to_free = n_to_free, n_freed=0; + int orig_n_to_skip = n_to_skip; int new_length = n_to_skip; chunk_t **chp = &freelists[i].head; chunk_t *chunk; - log_info(LD_MM, "Cleaning freelist for %d-byte chunks: keeping %d, " - "dropping %d.", - (int)freelists[i].alloc_size, n_to_skip, n_to_free); + log_info(LD_MM, "Cleaning freelist for %d-byte chunks: length %d, " + "keeping %d, dropping %d.", + (int)freelists[i].alloc_size, freelists[i].cur_length, + n_to_skip, n_to_free); + tor_assert(n_to_skip + n_to_free == freelists[i].cur_length); while (n_to_skip) { - tor_assert((*chp)->next); + if (! (*chp)->next) { + log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " + "%d-byte chunks, but only found %d. (Length %d)", + orig_n_to_skip, (int)freelists[i].alloc_size, + orig_n_to_skip-n_to_skip, freelists[i].cur_length); + assert_freelist_ok(&freelists[i]); + return; + } + // tor_assert((*chp)->next); chp = &(*chp)->next; --n_to_skip; } diff --git a/src/or/buffers.h b/src/or/buffers.h index 42d92dd89..8fd403d95 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -23,7 +23,6 @@ void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 5bb9d70d5..e5e7d2219 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -605,19 +605,19 @@ circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) * after we do so. Use this result to estimate parameters and * calculate the timeout. * - * Returns -1 and sets msg on error. Msg must be freed by the caller. + * Return -1 on error. */ int circuit_build_times_parse_state(circuit_build_times_t *cbt, - or_state_t *state, char **msg) + or_state_t *state) { int tot_values = 0; uint32_t loaded_cnt = 0, N = 0; config_line_t *line; unsigned int i; build_time_t *loaded_times; + int err = 0; circuit_build_times_init(cbt); - *msg = NULL; if (circuit_build_times_disabled()) { return 0; @@ -631,8 +631,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, smartlist_split_string(args, line->value, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (smartlist_len(args) < 2) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Too few arguments to CircuitBuildTime"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -645,8 +646,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, CBT_BUILD_TIME_MAX, &ok, NULL); if (!ok) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Unparsable bin number"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin number"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -654,8 +656,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, count = (uint32_t)tor_parse_ulong(count_str, 0, 0, UINT32_MAX, &ok, NULL); if (!ok) { - *msg = tor_strdup("Unable to parse circuit build times: " - "Unparsable bin count"); + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin count"); + err = 1; SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); smartlist_free(args); break; @@ -692,10 +695,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, "Corrupt state file? Build times count mismatch. " "Read %d times, but file says %d", loaded_cnt, state->TotalBuildTimes); - *msg = tor_strdup("Build times count mismatch."); + err = 1; circuit_build_times_reset(cbt); - tor_free(loaded_times); - return -1; + goto done; } circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); @@ -716,10 +718,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, "Corrupt state file? Shuffled build times mismatch. " "Read %d times, but file says %d", tot_values, state->TotalBuildTimes); - *msg = tor_strdup("Build times count mismatch."); + err = 1; circuit_build_times_reset(cbt); - tor_free(loaded_times); - return -1; + goto done; } circuit_build_times_set_timeout(cbt); @@ -728,8 +729,9 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, circuit_build_times_filter_timeouts(cbt); } + done: tor_free(loaded_times); - return *msg ? -1 : 0; + return err ? -1 : 0; } /** @@ -1512,7 +1514,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int r; -again: + again: r = onion_extend_cpath(circ); if (r < 0) { log_info(LD_CIRC,"Generating cpath hop failed."); diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index 3a02f0420..f4cc2a904 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -81,7 +81,7 @@ extern circuit_build_times_t circ_times; void circuit_build_times_update_state(circuit_build_times_t *cbt, or_state_t *state); int circuit_build_times_parse_state(circuit_build_times_t *cbt, - or_state_t *state, char **msg); + or_state_t *state); void circuit_build_times_count_timeout(circuit_build_times_t *cbt, int did_onehop); int circuit_build_times_count_close(circuit_build_times_t *cbt, @@ -107,8 +107,6 @@ void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, double quantile, double time_ms); int circuit_build_times_update_alpha(circuit_build_times_t *cbt); double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); -void circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, - double quantile_cutoff); void circuitbuild_running_unit_tests(void); void circuit_build_times_reset(circuit_build_times_t *cbt); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 4503e1d81..e9335b18d 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -746,13 +746,23 @@ circuit_expire_old_circuits_clientside(time_t now) (long)(now - circ->timestamp_created)); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { - log_notice(LD_CIRC, - "Ancient non-dirty circuit %d is still around after " - "%ld seconds. Purpose: %d", - TO_ORIGIN_CIRCUIT(circ)->global_identifier, - (long)(now - circ->timestamp_created), - circ->purpose); - TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld seconds. Purpose: %d", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + (long)(now - circ->timestamp_created), + circ->purpose); + /* FFFF implement a new circuit_purpose_to_string() so we don't + * just print out a number for circ->purpose */ + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } } } } diff --git a/src/or/config.c b/src/or/config.c index 1c3326d0c..d020c889c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -17,6 +17,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "connection_or.h" #include "control.h" #include "cpuworker.h" #include "dirserv.h" @@ -1298,6 +1299,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -4972,9 +4977,7 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (circuit_build_times_parse_state(&circ_times, global_state, &err) < 0) { - log_warn(LD_GENERAL,"%s",err); - tor_free(err); + if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { ret = -1; } return ret; diff --git a/src/or/connection.c b/src/or/connection.c index 99ce7fffe..c040be041 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2104,15 +2104,13 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ if (num_read > 0) { - if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_read(conn->port, num_read); rep_hist_note_bytes_read(num_read, now); } if (num_written > 0) { - if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_written(conn->port, num_written); rep_hist_note_bytes_written(num_written, now); } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -2363,7 +2361,7 @@ connection_handle_read_impl(connection_t *conn) return 0; } -loop_again: + loop_again: try_to_read = max_to_read; tor_assert(!conn->marked_for_close); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index e83028fae..64f342993 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2050,7 +2050,7 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) streamid_t test_stream_id; uint32_t attempts=0; -again: + again: test_stream_id = circ->next_stream_id++; if (++attempts > 1<<16) { /* Make sure we don't loop forever if all stream_id's are used. */ @@ -2988,7 +2988,7 @@ parse_extended_hostname(char *address, int allowdotexit) if (rend_valid_service_id(query)) { return ONION_HOSTNAME; /* success */ } -failed: + failed: /* otherwise, return to previous state and return 0 */ *s = '.'; return BAD_HOSTNAME; diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index c3d6098c5..d1bce48e8 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -82,7 +82,6 @@ void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, origin_circuit_t *circ, crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); /** Possible return values for parse_extended_hostname. */ typedef enum hostname_type_t { diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 405df1578..0ba1bca31 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -349,21 +349,18 @@ connection_or_digest_is_known_relay(const char *id_digest) return 0; } -/** If we don't necessarily know the router we're connecting to, but we - * have an addr/port/id_digest, then fill in as much as we can. Start - * by checking to see if this describes a router we know. */ +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + */ static void -connection_or_init_conn_from_address(or_connection_t *conn, - const tor_addr_t *addr, uint16_t port, - const char *id_digest, - int started_here) +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) { - or_options_t *options = get_options(); int rate, burst; /* per-connection rate limiting params */ - routerinfo_t *r = router_get_by_digest(id_digest); - connection_or_set_identity_digest(conn, id_digest); - - if (connection_or_digest_is_known_relay(id_digest)) { + if (connection_or_digest_is_known_relay(conn->identity_digest)) { /* It's in the consensus, or we have a descriptor for it meaning it * was probably in a recent consensus. It's a recognized relay: * give it full bandwidth. */ @@ -382,7 +379,43 @@ connection_or_init_conn_from_address(or_connection_t *conn, } conn->bandwidthrate = rate; - conn->read_bucket = conn->write_bucket = conn->bandwidthburst = burst; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + +/** If we don't necessarily know the router we're connecting to, but we + * have an addr/port/id_digest, then fill in as much as we can. Start + * by checking to see if this describes a router we know. */ +static void +connection_or_init_conn_from_address(or_connection_t *conn, + const tor_addr_t *addr, uint16_t port, + const char *id_digest, + int started_here) +{ + routerinfo_t *r = router_get_by_digest(id_digest); + connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 8e3723c15..717630217 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -26,6 +26,8 @@ int connection_or_flushed_some(or_connection_t *conn); int connection_or_finished_flushing(or_connection_t *conn); int connection_or_finished_connecting(or_connection_t *conn); int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg); @@ -44,10 +46,6 @@ void connection_or_write_var_cell_to_buf(const var_cell_t *cell, int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason); int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); int is_or_protocol_version_known(uint16_t version); void cell_pack(packed_cell_t *dest, const cell_t *src); diff --git a/src/or/control.c b/src/or/control.c index 7cbb1bd1f..7eead0e18 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -2259,7 +2259,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, circ->_base.purpose = new_purpose; connection_write_str_to_buf("250 OK\r\n", conn); -done: + done: if (args) { SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 2760d9666..6f943d78b 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -192,7 +192,7 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(0); /* don't ask me to do handshakes yet */ } -done_processing: + done_processing: conn->state = CPUWORKER_STATE_IDLE; num_cpuworkers_busy--; if (conn->timestamp_created < last_rotation_time) { diff --git a/src/or/dirserv.h b/src/or/dirserv.h index fc5a5549c..5ebcb8b2b 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -55,7 +55,6 @@ enum was_router_added_t dirserv_add_multiple_descriptors( enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source); -void dirserv_free_descriptors(void); void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index fd4d742cc..eae3bc8a4 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -762,7 +762,7 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, } } -out: + out: if (berr) { log_info(LD_DIR, "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT @@ -2942,6 +2942,7 @@ dirvote_compute_consensuses(void) strlen(pending_consensus_signatures), 0); log_notice(LD_DIR, "Signature(s) posted."); + smartlist_free(votes); return 0; err: smartlist_free(votes); @@ -3008,6 +3009,7 @@ dirvote_add_signatures_to_pending_consensus( networkstatus_vote_free(v); } *msg_out = "Signatures added"; + tor_free(new_signatures); } else if (r == 0) { *msg_out = "Signatures ignored"; } else { @@ -3137,7 +3139,7 @@ void dirvote_free_all(void) { dirvote_clear_votes(1); - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_vote_list); pending_vote_list = NULL; smartlist_free(previous_vote_list); @@ -3146,7 +3148,7 @@ dirvote_free_all(void) dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); if (pending_consensus_signature_list) { - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_consensus_signature_list); pending_consensus_signature_list = NULL; } diff --git a/src/or/geoip.c b/src/or/geoip.c index 6ed14e173..d9c8a0151 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,9 +3,10 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country to entry guards, bridges, and - * directories as well as statistics on answering network status requests. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE @@ -765,10 +766,10 @@ geoip_get_dirreq_history(geoip_client_action_t action, /** Return a newly allocated comma-separated string containing entries for * all the countries from which we've seen enough clients connect as a - * bridge, directory, or entry guard. The entry format is cc=num where num - * is the number of IPs we've seen connecting from that country, and cc is - * a lowercased country code. Returns NULL if we don't want to export - * geoip data yet. */ + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ char * geoip_get_client_history(geoip_client_action_t action) { @@ -828,7 +829,7 @@ geoip_get_client_history(geoip_client_action_t action) smartlist_add(chunks, buf); }); result = smartlist_join_strings(chunks, ",", 0, NULL); -done: + done: tor_free(counts); if (chunks) { SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); diff --git a/src/or/hibernate.h b/src/or/hibernate.h index 8192ac5d8..687fadb66 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -24,7 +24,6 @@ void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); #endif diff --git a/src/or/microdesc.c b/src/or/microdesc.c index f56ccd9ee..e8f3e7c59 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -53,7 +53,7 @@ HT_PROTOTYPE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq); HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, - _tor_malloc, _tor_realloc, _tor_free); + malloc, realloc, free); /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. * On success, return the total number of bytes written, and set diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index a9a9c78b8..c0a3a28e4 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -14,10 +14,12 @@ #include "circuitbuild.h" #include "config.h" #include "connection.h" +#include "connection_or.h" #include "control.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" +#include "main.h" #include "networkstatus.h" #include "relay.h" #include "router.h" @@ -351,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -1224,14 +1230,26 @@ update_consensus_networkstatus_fetch_time(time_t now) if (c) { long dl_interval; long interval = c->fresh_until - c->valid_after; + long min_sec_before_caching = CONSENSUS_MIN_SECONDS_BEFORE_CACHING; time_t start; + + if (min_sec_before_caching > interval/16) { + /* Usually we allow 2-minutes slop factor in case clocks get + desynchronized a little. If we're on a private network with + a crazy-fast voting interval, though, 2 minutes may be too + much. */ + min_sec_before_caching = interval/16; + } + if (directory_fetches_dir_info_early(options)) { /* We want to cache the next one at some point after this one * is no longer fresh... */ - start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + start = c->fresh_until + min_sec_before_caching; /* Some clients may need the consensus sooner than others. */ if (options->FetchDirInfoExtraEarly) { dl_interval = 60; + if (min_sec_before_caching + dl_interval > interval) + dl_interval = interval/2; } else { /* But only in the first half-interval after that. */ dl_interval = interval/2; @@ -1247,10 +1265,9 @@ update_consensus_networkstatus_fetch_time(time_t now) * to choose the rest of the interval *after* them. */ if (directory_fetches_dir_info_later(options)) { /* Give all the *clients* enough time to download the consensus. */ - start = start + dl_interval + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + start = start + dl_interval + min_sec_before_caching; /* But try to get it before ours actually expires. */ - dl_interval = (c->valid_until - start) - - CONSENSUS_MIN_SECONDS_BEFORE_CACHING; + dl_interval = (c->valid_until - start) - min_sec_before_caching; } } if (dl_interval < 1) @@ -1502,6 +1519,7 @@ networkstatus_set_current_consensus(const char *consensus, networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; @@ -1539,7 +1557,7 @@ networkstatus_set_current_consensus(const char *consensus, } if (flav != USABLE_CONSENSUS_FLAVOR && - !directory_caches_dir_info(get_options())) { + !directory_caches_dir_info(options)) { /* This consensus is totally boring to us: we won't use it, and we won't * serve it. Drop it. */ goto done; @@ -1674,7 +1692,7 @@ networkstatus_set_current_consensus(const char *consensus, download_status_failed(&consensus_dl_status[flav], 0); } - if (directory_caches_dir_info(get_options())) { + if (directory_caches_dir_info(options)) { dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, @@ -1687,9 +1705,13 @@ networkstatus_set_current_consensus(const char *consensus, /* XXXXNM Microdescs: needs a non-ns variant. */ update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); + dirvote_recalculate_timing(options, now); routerstatus_list_update_named_server_map(); - cell_ewma_set_scale_factor(get_options(), current_consensus); + cell_ewma_set_scale_factor(options, current_consensus); + + /* XXX022 where is the right place to put this call? */ + connection_or_update_token_buckets(get_connection_array(), options); + circuit_build_times_new_consensus_params(&circ_times, current_consensus); } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 4059dead5..32b71a9ce 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -73,7 +73,6 @@ int networkstatus_set_current_consensus(const char *consensus, unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures); void signed_descs_update_status_from_consensus_networkstatus( diff --git a/src/or/or.h b/src/or/or.h index 78c04eada..e6307e3ee 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -551,7 +551,7 @@ typedef enum { #define END_STREAM_REASON_DESTROY 5 #define END_STREAM_REASON_DONE 6 #define END_STREAM_REASON_TIMEOUT 7 -/* 8 is unallocated for historical reasons. */ +#define END_STREAM_REASON_NOROUTE 8 #define END_STREAM_REASON_HIBERNATING 9 #define END_STREAM_REASON_INTERNAL 10 #define END_STREAM_REASON_RESOURCELIMIT 11 diff --git a/src/or/policies.c b/src/or/policies.c index db3c6d886..4fd090415 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -381,7 +381,7 @@ validate_addr_policies(or_options_t *options, char **msg) ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); -err: + err: addr_policy_list_free(addr_policy); return *msg ? -1 : 0; #undef REJECT @@ -1272,7 +1272,7 @@ policy_summarize(smartlist_t *policy) result = tor_malloc(final_size); tor_snprintf(result, final_size, "%s %s", prefix, shorter_str); -cleanup: + cleanup: /* cleanup */ SMARTLIST_FOREACH(summary, policy_summary_item_t *, s, tor_free(s)); smartlist_free(summary); diff --git a/src/or/reasons.c b/src/or/reasons.c index 2dd5fe946..8bce3625d 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -28,6 +28,7 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_DESTROY: return "DESTROY"; case END_STREAM_REASON_DONE: return "DONE"; case END_STREAM_REASON_TIMEOUT: return "TIMEOUT"; + case END_STREAM_REASON_NOROUTE: return "NOROUTE"; case END_STREAM_REASON_HIBERNATING: return "HIBERNATING"; case END_STREAM_REASON_INTERNAL: return "INTERNAL"; case END_STREAM_REASON_RESOURCELIMIT: return "RESOURCELIMIT"; @@ -62,6 +63,7 @@ stream_end_reason_to_string(int reason) case END_STREAM_REASON_DESTROY: return "destroyed"; case END_STREAM_REASON_DONE: return "closed normally"; case END_STREAM_REASON_TIMEOUT: return "gave up (timeout)"; + case END_STREAM_REASON_NOROUTE: return "no route to host"; case END_STREAM_REASON_HIBERNATING: return "server is hibernating"; case END_STREAM_REASON_INTERNAL: return "internal error at server"; case END_STREAM_REASON_RESOURCELIMIT: return "server out of resources"; @@ -104,6 +106,8 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_SUCCEEDED; case END_STREAM_REASON_TIMEOUT: return SOCKS5_TTL_EXPIRED; + case END_STREAM_REASON_NOROUTE: + return SOCKS5_HOST_UNREACHABLE; case END_STREAM_REASON_RESOURCELIMIT: return SOCKS5_GENERAL_ERROR; case END_STREAM_REASON_HIBERNATING: @@ -164,6 +168,8 @@ errno_to_stream_end_reason(int e) S_CASE(ENOTCONN): S_CASE(ENETUNREACH): return END_STREAM_REASON_INTERNAL; + E_CASE(EHOSTUNREACH): + return END_STREAM_REASON_NOROUTE; S_CASE(ECONNREFUSED): return END_STREAM_REASON_CONNECTREFUSED; S_CASE(ECONNRESET): diff --git a/src/or/relay.c b/src/or/relay.c index 22ecdaafa..b85cc84c5 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -692,7 +692,8 @@ edge_reason_is_retriable(int reason) reason == END_STREAM_REASON_RESOURCELIMIT || reason == END_STREAM_REASON_EXITPOLICY || reason == END_STREAM_REASON_RESOLVEFAILED || - reason == END_STREAM_REASON_MISC; + reason == END_STREAM_REASON_MISC || + reason == END_STREAM_REASON_NOROUTE; } /** Called when we receive an END cell on a stream that isn't open yet, @@ -787,6 +788,7 @@ connection_ap_process_end_not_open( case END_STREAM_REASON_RESOLVEFAILED: case END_STREAM_REASON_TIMEOUT: case END_STREAM_REASON_MISC: + case END_STREAM_REASON_NOROUTE: if (client_dns_incr_failures(conn->socks_request->address) < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ @@ -1326,7 +1328,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) return 0; } -repeat_connection_edge_package_raw_inbuf: + repeat_connection_edge_package_raw_inbuf: circ = circuit_get_by_edge_conn(conn); if (!circ) { diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 0377f121c..68abb886a 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -209,7 +209,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, introcirc->_base.purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT; return 0; -err: + err: circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL); return -1; diff --git a/src/or/rendservice.h b/src/or/rendservice.h index f979a3941..1767714c6 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -15,7 +15,6 @@ int num_rend_services(void); int rend_config_services(or_options_t *options, int validate_only); int rend_service_load_keys(void); -void rend_services_init(void); void rend_services_introduce(void); void rend_consider_services_upload(time_t now); void rend_hsdir_routers_changed(void); diff --git a/src/or/rephist.c b/src/or/rephist.c index 28248c321..056fc5cc1 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1975,6 +1975,16 @@ rep_hist_exit_stats_init(time_t now) sizeof(uint32_t)); } +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) +{ + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); +} + /** Stop collecting exit port stats in a way that we can re-start doing * so in rep_hist_exit_stats_init(). */ void @@ -1986,164 +1996,173 @@ rep_hist_exit_stats_term(void) tor_free(exit_streams); } -/** Write exit stats to $DATADIR/stats/exit-stats, reset counters, and - * return when we would next want to write exit stats. */ -time_t -rep_hist_exit_stats_write(time_t now) +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) { + int i; + uint64_t total_bytes = 0, threshold_bytes, other_read = 0, + other_written = 0; + uint32_t other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; char t[ISO_TIME_LEN+1]; - int r, i, comma; - uint64_t *b, total_bytes, threshold_bytes, other_bytes; - uint32_t other_streams; - - char *statsdir = NULL, *filename = NULL; - open_file_t *open_file = NULL; - FILE *out = NULL; + char *result; if (!start_of_exit_stats_interval) - return 0; /* Not initialized. */ - if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ + return NULL; /* Not initialized. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE) < 0) - goto done; - filename = get_datadir_fname2("stats", "exit-stats"); - format_iso_time(t, now); - log_info(LD_HIST, "Writing exit port statistics to disk for period " - "ending at %s.", t); - - if (!open_file) { - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, - 0600, &open_file); - if (!out) { - log_warn(LD_HIST, "Couldn't open '%s'.", filename); - goto done; - } - } - - /* written yyyy-mm-dd HH:MM:SS (n s) */ - if (fprintf(out, "exit-stats-end %s (%d s)\n", t, - (unsigned) (now - start_of_exit_stats_interval)) < 0) - goto done; - - /* Count the total number of bytes, so that we can attribute all - * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + /* Count total number of bytes, so that we can attribute observations + * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL * of all bytes to a special port 'other'. */ - total_bytes = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { total_bytes += exit_bytes_read[i]; total_bytes += exit_bytes_written[i]; } threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - /* exit-kibibytes-(read|written) port=kibibytes,.. */ - for (r = 0; r < 2; r++) { - b = r ? exit_bytes_read : exit_bytes_written; - tor_assert(b); - if (fprintf(out, "%s ", - r ? "exit-kibibytes-read" - : "exit-kibibytes-written") < 0) - goto done; - - comma = 0; - other_bytes = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (b[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - uint64_t num = round_uint64_to_next_multiple_of(b[i], - EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - if (fprintf(out, "%s%d="U64_FORMAT, - comma++ ? "," : "", i, - U64_PRINTF_ARG(num)) < 0) - goto done; - } else - other_bytes += b[i]; - } - } - other_bytes = round_uint64_to_next_multiple_of(other_bytes, - EXIT_STATS_ROUND_UP_BYTES); - other_bytes /= 1024; - if (fprintf(out, "%sother="U64_FORMAT"\n", - comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) - goto done; - } - /* exit-streams-opened port=num,.. */ - if (fprintf(out, "exit-streams-opened ") < 0) - goto done; - comma = 0; - other_streams = 0; + /* Add observations of all ports above the threshold to smartlists and + * join them to single strings. Also count bytes and streams of ports + * below or equal to the threshold. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_streams[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + if (exit_bytes_written[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + } + if (exit_bytes_read[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + } + if (exit_streams[i] > 0) { uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%s%d=%u", - comma++ ? "," : "", i, num)<0) - goto done; - } else - other_streams += exit_streams[i]; + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", i, num); + smartlist_add(streams_strings, buf); + } + } else { + other_read += exit_bytes_read[i]; + other_written += exit_bytes_written[i]; + other_streams += exit_streams[i]; } } + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); other_streams = round_uint32_to_next_multiple_of(other_streams, - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%sother=%u\n", - comma ? "," : "", other_streams)<0) + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; +} + +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + log_info(LD_HIST, "Writing exit port statistics to disk."); + + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); + + /* Reset counters. */ + rep_hist_reset_exit_stats(now); + + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); goto done; - /* Reset counters */ - memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); - memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); - memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); - start_of_exit_stats_interval = now; + } + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); - if (open_file) - finish_writing_to_file(open_file); - open_file = NULL; done: - if (open_file) - abort_writing_to_file(open_file); - tor_free(filename); + tor_free(str); tor_free(statsdir); + tor_free(filename); return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } -/** Note that we wrote <b>num_bytes</b> to an exit connection to - * <b>port</b>. */ +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ void -rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) { - if (!get_options()->ExitPortStatistics) - return; - if (!exit_bytes_written) - return; /* Not initialized */ - exit_bytes_written[port] += num_bytes; - log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", - (unsigned long)num_bytes, port); -} - -/** Note that we read <b>num_bytes</b> from an exit connection to - * <b>port</b>. */ -void -rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) -{ - if (!get_options()->ExitPortStatistics) - return; - if (!exit_bytes_read) - return; /* Not initialized */ - exit_bytes_read[port] += num_bytes; - log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", - (unsigned long)num_bytes, port); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); } /** Note that we opened an exit stream to <b>port</b>. */ void rep_hist_note_exit_stream_opened(uint16_t port) { - if (!get_options()->ExitPortStatistics) - return; - if (!exit_streams) - return; /* Not initialized */ + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ exit_streams[port]++; log_debug(LD_HIST, "Opened exit stream to port %d", port); } diff --git a/src/or/rephist.h b/src/or/rephist.h index dc756df50..c3914dcaf 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -23,14 +23,10 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); + void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); -void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes); -void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes); -void rep_hist_note_exit_stream_opened(uint16_t port); -void rep_hist_exit_stats_init(time_t now); -time_t rep_hist_exit_stats_write(time_t now); -void rep_hist_exit_stats_term(void); + int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); @@ -65,13 +61,14 @@ void dump_pk_ops(int severity); void rep_hist_free_all(void); -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); void rep_hist_buffer_stats_init(time_t now); void rep_hist_buffer_stats_add_circ(circuit_t *circ, diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 2313f68e0..e5ebf0761 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -30,11 +30,6 @@ int router_parse_list_from_string(const char **s, const char *eos, int is_extrainfo, int allow_annotations, const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); int router_parse_runningrouters(const char *str); int router_parse_directory(const char *str); diff --git a/src/test/test.c b/src/test/test.c index 9948ecf99..16ac1e4a6 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -61,6 +61,7 @@ double fabs(double x); #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> +#include "main.h" #endif /** Set to true if any unit test has failed. Mostly, this is set by the macros @@ -70,6 +71,7 @@ int have_failed = 0; /** Temporary directory (set up by setup_directory) under which we store all * our files during testing. */ static char temp_dir[256]; +static pid_t temp_dir_setup_in_pid = 0; /** Select and create the temporary directory we'll use to run our unit tests. * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. @@ -96,6 +98,7 @@ setup_directory(void) exit(1); } is_setup = 1; + temp_dir_setup_in_pid = getpid(); } /** Return a filename relative to our testing temporary directory */ @@ -109,11 +112,16 @@ get_fname(const char *name) } /** Remove all files stored under the temporary directory, and the directory - * itself. */ + * itself. Called by atexit(). */ static void remove_directory(void) { - smartlist_t *elements = tor_listdir(temp_dir); + smartlist_t *elements; + if (getpid() != temp_dir_setup_in_pid) { + /* Only clean out the tempdir when the main process is exiting. */ + return; + } + elements = tor_listdir(temp_dir); if (elements) { SMARTLIST_FOREACH(elements, const char *, cp, { @@ -337,76 +345,6 @@ test_buffers(void) buf_free(buf); buf = NULL; -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - done: if (buf) buf_free(buf); @@ -478,7 +416,6 @@ test_circuit_timeout(void) circuit_build_times_t final; double timeout1, timeout2; or_state_t state; - char *msg; int i, runs; double close_ms; circuit_build_times_init(&initial); @@ -518,7 +455,7 @@ test_circuit_timeout(void) test_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); circuit_build_times_update_state(&estimate, &state); - test_assert(circuit_build_times_parse_state(&final, &state, &msg) == 0); + test_assert(circuit_build_times_parse_state(&final, &state) == 0); circuit_build_times_update_alpha(&final); timeout2 = circuit_build_times_calculate_timeout(&final, @@ -620,7 +557,7 @@ test_circuit_timeout(void) circuit_build_times_count_timeout(&final, 1); } -done: + done: return; } @@ -1150,6 +1087,57 @@ test_geoip(void) tor_free(s); } +/** Run unit tests for stats code. */ +static void +test_stats(void) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + + /* We shouldn't collect exit stats without initializing them. */ + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note some streams and bytes, and generate history + * string. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_note_exit_stream_opened(443); + rep_hist_note_exit_bytes(443, 100, 10000); + rep_hist_note_exit_bytes(443, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 80=1,443=1,other=0\n" + "exit-kibibytes-read 80=10,443=20,other=0\n" + "exit-streams-opened 80=4,443=4,other=0\n", s); + tor_free(s); + + /* Stop collecting stats, add some bytes, and ensure we don't generate + * a history string. */ + rep_hist_exit_stats_term(); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add some bytes, reset stats, and see what history we + * get when observing no streams or bytes at all. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_reset_exit_stats(now); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written other=0\n" + "exit-kibibytes-read other=0\n" + "exit-streams-opened other=0\n", s); + + done: + tor_free(s); +} + static void * legacy_test_setup(const struct testcase_t *testcase) { @@ -1182,6 +1170,8 @@ const struct testcase_setup_t legacy_setup = { test_ ## group ## _ ## name } #define DISABLED(name) \ { #name, legacy_test_helper, TT_SKIP, &legacy_setup, name } +#define FORK(name) \ + { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_ ## name } static struct testcase_t test_array[] = { ENT(buffers), @@ -1190,6 +1180,7 @@ static struct testcase_t test_array[] = { ENT(policies), ENT(rend_fns), ENT(geoip), + FORK(stats), DISABLED(bench_aes), DISABLED(bench_dmap), diff --git a/src/test/test_dir.c b/src/test/test_dir.c index a129bf977..80d2379de 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -580,7 +580,7 @@ test_dir_measured_bw(void) "557365204145532d32353620696e73746561642e") == 0); } -done: + done: return; } |