diff options
Diffstat (limited to 'src/or')
39 files changed, 10300 insertions, 9280 deletions
diff --git a/src/or/Makefile.am b/src/or/Makefile.am index ad2476ff1..cfa8a035a 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,76 +41,40 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c - -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + -lz -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i + tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. \([0-9][0-9]*\)/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: \(.*\), Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ + mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 571f86de1..4dbd9a7a0 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,8 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "../common/util.h" +#include "../common/log.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +147,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +198,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -401,7 +406,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -447,7 +452,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -481,7 +486,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -520,6 +525,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -1400,7 +1407,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "Tor only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ @@ -1488,7 +1495,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1513,7 +1521,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ @@ -1611,6 +1619,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, 128, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 3d0f728d5..cd5ada8dc 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,25 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// FIXME: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +63,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +79,959 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +static int32_t +circuit_build_times_max_timeouts(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmaxtimeouts", + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT); + return num; +} + +static int32_t +circuit_build_times_min_circs_to_observe(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmincircs", + CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE); + return num; +} + +double +circuit_build_times_quantile_cutoff(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtquantile", + CBT_DEFAULT_QUANTILE_CUTOFF); + return num/100.0; +} + +static int32_t +circuit_build_times_test_frequency(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbttestfreq", + CBT_DEFAULT_TEST_FREQUENCY); + return num; +} + +static int32_t +circuit_build_times_min_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmintimeout", + CBT_DEFAULT_TIMEOUT_MIN_VALUE); + return num; +} + +int32_t +circuit_build_times_initial_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtinitialtimeout", + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE); + return num; +} + +static int32_t +circuit_build_times_recent_circuit_count(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS); + return num; +} + +/** + * This function is called when we get a consensus update. + * + * It checks to see if we have changed any consensus parameters + * that require reallocation or discard of previous stats. + */ +void +circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns) +{ + int32_t num = networkstatus_get_param(ns, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS); + + if (num != cbt->liveness.num_recent_circs) { + int8_t *recent_circs; + log_notice(LD_CIRC, "Changing recent timeout size from %d to %d", + cbt->liveness.num_recent_circs, num); + + tor_assert(num > 0); + tor_assert(cbt->liveness.timeouts_after_firsthop); + + /* + * Technically this is a circular array that we are reallocating + * and memcopying. However, since it only consists of either 1s + * or 0s, and is only used in a statistical test to determine when + * we should discard our history after a sufficient number of 1's + * have been reached, it is fine if order is not preserved or + * elements are lost. + * + * cbtrecentcount should only be changing in cases of severe network + * distress anyway, so memory correctness here is paramount over + * doing acrobatics to preserve the array. + */ + recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, + sizeof(int8_t)*MIN(num, cbt->liveness.num_recent_circs)); + + // Adjust the index if it needs it. + if (num < cbt->liveness.num_recent_circs) { + cbt->liveness.after_firsthop_idx = MIN(num-1, + cbt->liveness.after_firsthop_idx); + } + + tor_free(cbt->liveness.timeouts_after_firsthop); + cbt->liveness.timeouts_after_firsthop = recent_circs; + cbt->liveness.num_recent_circs = num; + } +} + +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + circuit_build_times_min_timeout()/1000); + timeout = circuit_build_times_min_timeout(); + } + } else { + timeout = circuit_build_times_initial_timeout(); + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->pre_timeouts = 0; + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout value based to either the + * config setting or BUILD_TIMEOUT_INITIAL_VALUE. + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->liveness.num_recent_circs = circuit_build_times_recent_circuit_count(); + cbt->liveness.timeouts_after_firsthop = tor_malloc_zero(sizeof(int8_t)* + cbt->liveness.num_recent_circs); + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); +} + +/** + * Rewind our timeout history by n timeout positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + if (cbt->pre_timeouts) { + /* If we have pre-timeouts, it means we're not yet storing + * timeouts in our normal array. Only rewind the counter. */ + if (cbt->pre_timeouts > n) { + cbt->pre_timeouts -= n; + } else { + cbt->pre_timeouts = 0; + } + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. Total: %d. " + "Pre-timeouts: %d", n, cbt->build_times_idx, + cbt->total_build_times, cbt->pre_timeouts); + + return; + } + + cbt->build_times_idx -= n; + cbt->build_times_idx %= CBT_NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx) + %CBT_NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} + +/** + * Add a new timeout value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</a> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + tor_assert(time <= CBT_BUILD_TIME_MAX); + if (time <= 0) { + log_warn(LD_CIRC, "Circuit build time is %u!", time); + return -1; + } + + // XXX: Probably want to demote this to debug for the release. + log_info(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % CBT_NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % CBT_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = CBT_BUILD_TIME_MAX; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == CBT_BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than CBT_BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*CBT_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0) continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / CBT_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the most frequent build time (rounded to CBT_BIN_WIDTH ms). + * + * Ties go in favor of the slower time. + */ +static build_time_t +circuit_build_times_mode(circuit_build_times_t *cbt) +{ + build_time_t i, nbins, max_bin=0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[max_bin]) { + max_bin = i; + } + } + + tor_free(histogram); + + return max_bin*CBT_BIN_WIDTH+CBT_BIN_WIDTH/2; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + i*CBT_BIN_WIDTH+CBT_BIN_WIDTH/2, histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Stolen from http://en.wikipedia.org/wiki/Fisher\u2013Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + int num_times) +{ + int n = num_times; + if (num_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "Decreasing circuit_build_times size from %d to %d", + num_times, CBT_NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random CBT_NCIRCUITS_TO_OBSERVE + * subset (ie the first CBT_NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, CBT_NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Returns -1 and sets msg on error. Msg must be freed by the caller. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + int i; + build_time_t *loaded_times = tor_malloc(sizeof(build_time_t) + * state->TotalBuildTimes); + circuit_build_times_init(cbt); + *msg = NULL; + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + CBT_BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin number"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin count"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d, file says %d", loaded_cnt, state->TotalBuildTimes); + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + tor_assert(cbt->total_build_times == tot_values); + tor_assert(cbt->total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); + circuit_build_times_set_timeout(cbt); + tor_free(loaded_times); + return *msg ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +void +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_mode(cbt); + + for (i=0; i< CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else { + a += tor_mathlog(x[i]); + } + n++; + } + + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + a -= n*tor_mathlog(cbt->Xm); + a = n/a; + + cbt->alpha = a; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + uint64_t r = crypto_rand_uint64(UINT64_MAX-1); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + ((q_hi-q_lo)*r)/(1.0*UINT64_MAX); + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** Generate points in [cutoff, 1.0) on the CDF. */ +void +circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, + double quantile_cutoff) +{ + // XXX: This may be failing when the number of samples is small? + // Keep getting values for the largest timeout bucket over and over + // again... Probably because alpha is very very large in that case.. + build_time_t gentime = circuit_build_times_generate_sample(cbt, + quantile_cutoff, CBT_MAX_SYNTHETIC_QUANTILE); + + if (gentime < (build_time_t)tor_lround(cbt->timeout_ms)) { + log_warn(LD_CIRC, + "Generated a synthetic timeout LESS than the current timeout: " + "%ums vs %lfms using Xm: %d a: %lf, q: %lf", + gentime, cbt->timeout_ms, cbt->Xm, cbt->alpha, quantile_cutoff); + } else if (gentime > CBT_BUILD_TIME_MAX) { + log_info(LD_CIRC, + "Generated a synthetic timeout larger than the max: %u", + gentime); + gentime = CBT_BUILD_TIME_MAX; + } else { + log_info(LD_CIRC, "Generated synthetic circuit build time %u for timeout", + gentime); + } + + circuit_build_times_add_time(cbt, gentime); +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Generate synthetic timeout values for the timeouts + * that have happened before we estimated our parameters. + */ +static void +circuit_build_times_count_pretimeouts(circuit_build_times_t *cbt) +{ + /* Store a timeout as a random position past the current + * cutoff on the pareto curve */ + if (cbt->pre_timeouts) { + double timeout_quantile = 1.0- + ((double)cbt->pre_timeouts)/ + (cbt->pre_timeouts+cbt->total_build_times); + /* Make sure it doesn't exceed the synthetic max */ + timeout_quantile *= CBT_MAX_SYNTHETIC_QUANTILE; + cbt->Xm = circuit_build_times_mode(cbt); + tor_assert(cbt->Xm > 0); + /* Use current timeout to get an estimate on alpha */ + circuit_build_times_initial_alpha(cbt, timeout_quantile, + cbt->timeout_ms); + while (cbt->pre_timeouts-- != 0) { + circuit_build_times_add_timeout_worker(cbt, timeout_quantile); + } + cbt->pre_timeouts = 0; + } +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + if (cbt->total_build_times < circuit_build_times_min_circs_to_observe()) + return 1; + return 0; +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); +} + +/** + * Called to indicate that the network showed some signs of liveness. + * + * This function is called every time we receive a cell. Avoid + * syscalls, events, and other high-intensity work. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + cbt->liveness.network_last_live = approx_time(); + cbt->liveness.nonlive_discarded = 0; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; +} + +/** + * A circuit just timed out. If there has been no recent network activity + * at all, but this circuit was launched back when we thought the network + * was live, increment the number of "nonlive" circuit timeouts. + * + * Also distinguish between whether it failed before the first hop + * and record that in our history for later deciding if the network has + * changed. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + * + * Also double check that it is a valid timeout after we have possibly + * just recently reset cbt->timeout_ms. + */ + if (cbt->liveness.network_last_live <= start_time && + start_time <= (now - cbt->timeout_ms/1000.0)) { + cbt->liveness.nonlive_timeouts++; + } else if (did_onehop) { + /* Count a one-hop timeout */ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; + } +} + +/** + * Returns false if the network has not received a cell or tls handshake + * in the past NETWORK_NOTLIVE_TIMEOUT_COUNT circuits. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts >= CBT_NETWORK_NONLIVE_DISCARD_COUNT) { + if (!cbt->liveness.nonlive_discarded) { + cbt->liveness.nonlive_discarded = 1; + log_notice(LD_CIRC, "Network is no longer live (too many recent " + "circuit timeouts). Dead for %ld seconds.", + (long int)(now - cbt->liveness.network_last_live)); + /* Only discard NETWORK_NONLIVE_TIMEOUT_COUNT-1 because we stopped + * counting after that */ + circuit_build_times_rewind_history(cbt, + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT-1); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_DISCARD); + } + return 0; + } else if (cbt->liveness.nonlive_timeouts >= + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT) { + if (cbt->timeout_ms < circuit_build_times_get_initial_timeout()) { + log_notice(LD_CIRC, + "Network is flaky. No activity for %ld seconds. " + "Temporarily raising timeout to %lds.", + (long int)(now - cbt->liveness.network_last_live), + tor_lround(circuit_build_times_get_initial_timeout()/1000)); + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + cbt->liveness.net_suspended = 1; + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_SUSPENDED); + } + + return 0; + } else if (cbt->liveness.net_suspended) { + log_notice(LD_CIRC, + "Network activity has resumed. " + "Resuming circuit timeout calculations."); + cbt->liveness.net_suspended = 0; + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESUME); + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < cbt->liveness.num_recent_circs; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < circuit_build_times_max_timeouts()) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(*cbt->liveness.timeouts_after_firsthop)* + cbt->liveness.num_recent_circs); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + cbt->timeout_ms *= 2; + } else { + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + + log_notice(LD_CIRC, + "Network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_add_timeout(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + circuit_build_times_network_timeout(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + /* If there are a ton of timeouts, we should reduce + * the circuit build timeout */ + if (circuit_build_times_network_check_changed(cbt)) { + return 0; + } + + if (!cbt->have_computed_timeout) { + /* Store a timeout before we have enough data */ + cbt->pre_timeouts++; + log_info(LD_CIRC, + "Not enough circuits yet to calculate a new build timeout." + " Need %d more.", circuit_build_times_min_circs_to_observe() + - cbt->total_build_times); + return 0; + } + + circuit_build_times_count_pretimeouts(cbt); + circuit_build_times_add_timeout_worker(cbt, + circuit_build_times_quantile_cutoff()); + + return 1; +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + if (cbt->total_build_times < circuit_build_times_min_circs_to_observe()) { + return; + } + + circuit_build_times_count_pretimeouts(cbt); + circuit_build_times_update_alpha(cbt); + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_quantile_cutoff()); + + cbt->have_computed_timeout = 1; + + if (cbt->timeout_ms < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, circuit_build_times_min_timeout()); + cbt->timeout_ms = circuit_build_times_min_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, Xm: %d, a: %lf) " + "based on %d circuit times", tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->Xm, cbt->alpha, cbt->total_build_times); +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -148,8 +1121,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1189,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -361,9 +1333,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s ", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -536,7 +1509,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -642,6 +1615,17 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (!circ->build_state->onehop_tunnel) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.highres_created, &end); + if (timediff > INT32_MAX) + timediff = INT32_MAX; + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_network_circ_success(&circ_times); + circuit_build_times_set_timeout(&circ_times); + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) @@ -650,7 +1634,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) or_options_t *options = get_options(); has_completed_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -705,7 +1689,7 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); @@ -942,10 +1926,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1149,6 +2132,8 @@ circuit_all_predicted_ports_handled(time_t now, int *need_uptime, smartlist_t *LongLivedServices = get_options()->LongLivedPorts; tor_assert(need_uptime); tor_assert(need_capacity); + // Always predict need_capacity + *need_capacity = 1; enough = (smartlist_len(sl) == 0); for (i = 0; i < smartlist_len(sl); ++i) { port = smartlist_get(sl, i); @@ -1253,9 +2238,16 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } - if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { + if (router_is_unreliable(router, need_uptime, need_capacity, 0) && + (!options->ExitNodes || + !routerset_contains_router(options->ExitNodes, router))) { + /* FFFF Someday, differentiate between a routerset that names + * routers, and a routerset that names countries, and only do this + * check if they've asked for specific exit relays. Or if the country + * they ask for is rare. Or something. */ n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable, unless we have + * ExitNodes set, in which case we asked for it */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1280,7 +2272,7 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, { if (!ap_stream_wants_exit_attention(conn)) continue; /* Skip everything but APs in CIRCUIT_WAIT */ - if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router)) { + if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router, 1)) { ++n_supported[i]; // log_fn(LOG_DEBUG,"%s is supported. n_supported[%d] now %d.", // router->nickname, i, n_supported[i]); @@ -1322,7 +2314,8 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } @@ -1347,8 +2340,9 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); use = smartlist_create(); @@ -1368,12 +2362,14 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ + /* FFF sometimes the above results in null, when the requested + * exit node is considered down by the consensus. we should pick + * it anyway, since the user asked for it. */ router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); if (router) break; @@ -1391,10 +2387,10 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes && options->StrictNodes) { log_warn(LD_CIRC, "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "StrictNodes is set: can't choose an exit."); } return NULL; } @@ -1425,15 +2421,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1553,8 +2547,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1676,8 +2669,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1741,11 +2733,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1866,9 +2854,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1991,35 +2979,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2028,11 +3039,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2061,10 +3073,15 @@ log_entry_guards(int severity) SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); + const char *msg = NULL; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_snprintf(buf, sizeof(buf), "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_snprintf(buf, sizeof(buf), "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); smartlist_add(elements, tor_strdup(buf)); }); @@ -2089,12 +3106,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2180,7 +3198,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2298,10 +3317,13 @@ entry_guards_compute_status(void) int severity = LOG_DEBUG; or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; options = get_options(); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); now = time(NULL); @@ -2328,13 +3350,16 @@ entry_guards_compute_status(void) if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2405,6 +3430,7 @@ entry_guard_register_connect_status(const char *digest, int succeeded, "Removing from the list. %d/%d entry guards usable/new.", entry->nickname, buf, num_live_entry_guards()-1, smartlist_len(entry_guards)-1); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); entry_guard_free(entry); smartlist_del_keeporder(entry_guards, idx); log_entry_guards(LOG_INFO); @@ -2441,7 +3467,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2472,7 +3499,8 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } @@ -2496,7 +3524,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2540,8 +3568,9 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { + /* Finally, the remaining previously configured guards that are not in + * EntryNodes, unless we're strict in which case we drop them */ + if (options->StrictNodes) { SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, entry_guard_free(e)); } else { @@ -2555,16 +3584,30 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we'd prefer to stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; +} + +/* Are we dead set against changing our entry guard list, or would we + * change it if it means keeping Tor usable? */ +static int +entry_list_is_totally_static(or_options_t *options) +{ + if (options->EntryNodes && options->StrictNodes) + return 1; + if (options->UseBridges) + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2582,7 +3625,7 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { smartlist_add(exit_family, chosen_exit); @@ -2596,36 +3639,60 @@ choose_random_entry(cpath_build_state_t *state) if (should_add_entry_nodes) entry_guards_prepend_from_config(); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) pick_entry_guards(); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_totally_static(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2650,7 +3717,7 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { + if (!r && entry_list_is_constrained(options) && consider_exit_family) { /* still no? if we're using bridges or have strictentrynodes * set, and our chosen exit is in the same family as all our * bridges/entry guards, then be flexible about families. */ @@ -2661,15 +3728,15 @@ choose_random_entry(cpath_build_state_t *state) } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2904,8 +3971,7 @@ int getinfo_helper_entry_guards(control_connection_t *conn, const char *question, char **answer) { - int use_long_names = conn->use_long_names; - + (void) conn; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { smartlist_t *sl = smartlist_create(); @@ -2913,12 +3979,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2927,19 +3994,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2947,7 +4012,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index b84d7f762..73e2e06cc 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -380,10 +380,17 @@ static void init_circuit_base(circuit_t *circ) { circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->highres_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +415,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +438,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +458,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +504,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +514,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +557,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +568,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -1083,9 +1096,9 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } @@ -1236,11 +1249,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 6ae2769a6..9eda9e248 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -20,6 +20,8 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ static void circuit_expire_old_circuits(time_t now); static void circuit_increment_failure_count(void); +long int lround(double x); + /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). * Else return 0. */ @@ -113,7 +115,7 @@ circuit_is_acceptable(circuit_t *circ, edge_connection_t *conn, return 0; } } - if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter)) { + if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter, 0)) { /* can't exit from this router */ return 0; } @@ -263,16 +265,18 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(time_t now) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout is BUILD_TIMEOUT_INITIAL_VALUE if we haven't + * decided on a customized one yet */ + time_t general_cutoff = now - lround(circ_times.timeout_ms/1000); + time_t begindir_cutoff = now - lround(circ_times.timeout_ms/2000); time_t introcirc_cutoff = begindir_cutoff; cpath_build_state_t *build_state; - while (circ) { + while (next_circ) { time_t cutoff; - victim = circ; - circ = circ->next; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -343,6 +347,12 @@ circuit_expire_building(time_t now) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + if (circuit_build_times_add_timeout(&circ_times, first_hop_succeeded, + victim->timestamp_created)) + circuit_build_times_set_timeout(&circ_times); } if (victim->n_conn) @@ -414,7 +424,7 @@ circuit_stream_is_being_handled(edge_connection_t *conn, if (exitrouter && (!need_uptime || build_state->need_uptime)) { int ok; if (conn) { - ok = connection_ap_can_use_exit(conn, exitrouter); + ok = connection_ap_can_use_exit(conn, exitrouter, 0); } else { addr_policy_result_t r = compare_addr_to_addr_policy( 0, port, exitrouter->exit_policy); @@ -431,11 +441,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +527,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -624,6 +647,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ @@ -631,7 +659,15 @@ static void circuit_expire_old_circuits(time_t now) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + time_t cutoff; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff = now - IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff = now - get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) @@ -806,6 +842,9 @@ circuit_build_failed(origin_circuit_t *circ) "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } if (n_conn_id) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); @@ -889,8 +928,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -1064,7 +1103,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1072,7 +1111,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); @@ -1105,19 +1144,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1146,7 +1180,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1188,8 +1222,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { /* help predict this next time */ @@ -1371,7 +1404,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1398,7 +1431,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } return -1; } - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); diff --git a/src/or/command.c b/src/or/command.c index 181bad8bc..8d838a0a2 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -395,15 +395,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -511,7 +514,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -625,8 +629,8 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/config.c b/src/or/config.c index c7c1e19bc..0e92fbf7b 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -66,6 +66,7 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +84,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +137,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,10 +166,14 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -186,18 +194,19 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,12 +214,16 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), @@ -222,6 +235,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,11 +248,15 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), + OBSOLETE("HSAuthorityRecordStats"), V(HttpProxy, STRING, NULL), V(HttpProxyAuthenticator, STRING, NULL), V(HttpsProxy, STRING, NULL), V(HttpsProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), @@ -265,6 +284,8 @@ static config_var_t _option_vars[] = { V(ORPort, UINT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -288,7 +309,7 @@ static config_var_t _option_vars[] = { OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -304,8 +325,7 @@ static config_var_t _option_vars[] = { V(SocksPort, UINT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -330,6 +350,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -340,6 +361,7 @@ static config_var_t _option_vars[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -396,6 +418,10 @@ static config_var_t _state_vars[] = { V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +436,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +454,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -700,20 +517,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +529,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +549,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +613,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -828,10 +629,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+8; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (r%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } @@ -844,8 +645,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -854,43 +657,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1103,6 +935,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1233,7 +1074,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1311,14 +1151,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1351,11 +1191,26 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + (options->ExcludeNodes && + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)) || + (options->ExcludeExitNodes && + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes)) || + (options->EntryNodes && + !routerset_equal(old_options->EntryNodes, options->EntryNodes)) || + (options->ExitNodes && + !routerset_equal(old_options->ExitNodes, options->ExitNodes)) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards, or changed preferred or " + "excluded node lists. Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); } @@ -1409,13 +1264,29 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || @@ -1488,7 +1359,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1533,7 +1407,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1625,19 +1502,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -2305,20 +2169,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2367,7 +2221,7 @@ resolve_my_address(int warn_severity, or_options_t *options, if (tor_inet_aton(hostname, &in) == 0) { /* then we have to resolve it */ explicit_ip = 0; - if(!tor_lookup_hostname(hostname, &addr)) { + if (!tor_lookup_hostname(hostname, &addr)) { uint32_t interface_ip; if (explicit_hostname) { @@ -2564,7 +2418,10 @@ config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2666,6 +2523,8 @@ is_listening_on_low_port(uint16_t port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2715,7 +2574,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2743,14 +2601,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; @@ -2791,7 +2643,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2892,15 +2744,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2937,7 +2788,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -3105,19 +2956,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3155,6 +2998,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3166,9 +3013,9 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); if (options->ConnLimit <= 0) { r = tor_snprintf(buf, sizeof(buf), @@ -3301,6 +3148,21 @@ options_validate(or_options_t *old_options, or_options_t *options, }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + r = tor_snprintf(buf, sizeof(buf), + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + *msg = tor_strdup(r >= 0 ? buf : "internal error"); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { r = tor_snprintf(buf, sizeof(buf), "Unrecognized value in PublishServerDescriptor"); @@ -3324,29 +3186,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3365,6 +3228,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ensure_bandwidth_cap(&options->RelayBandwidthBurst, "RelayBandwidthBurst", msg) < 0) return -1; + if (ensure_bandwidth_cap(&options->PerConnBWRate, + "PerConnBWRate", msg) < 0) + return -1; + if (ensure_bandwidth_cap(&options->PerConnBWBurst, + "PerConnBWBurst", msg) < 0) + return -1; if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { @@ -3418,7 +3287,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to parse accounting options. See logs for details."); if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, + if (tor_addr_port_parse(options->HttpProxy, &options->HttpProxyAddr, &options->HttpProxyPort) < 0) REJECT("HttpProxy failed to parse or resolve. Please fix."); if (options->HttpProxyPort == 0) { /* give it a default */ @@ -3432,7 +3301,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, + if (tor_addr_port_parse(options->HttpsProxy, &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) REJECT("HttpsProxy failed to parse or resolve. Please fix."); if (options->HttpsProxyPort == 0) { /* give it a default */ @@ -3445,6 +3314,45 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3593,6 +3501,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HttpProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HttpProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3672,6 +3586,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + return 0; #undef REJECT #undef COMPLAIN @@ -3729,9 +3648,11 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } @@ -3741,6 +3662,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } + if (old->CellStatistics != new_val->CellStatistics || + old->DirReqStatistics != new_val->DirReqStatistics || + old->EntryStatistics != new_val->EntryStatistics || + old->ExitPortStatistics != new_val->ExitPortStatistics) { + *msg = tor_strdup("While Tor is running, changing either " + "CellStatistics, DirReqStatistics, EntryStatistics, " + "or ExitPortStatistics is not allowed."); + return -1; + } + + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " + "is not allowed."); + return -1; + } + return 0; } @@ -3908,13 +3845,7 @@ find_torrc_filename(int argc, char **argv, log(LOG_WARN, LD_CONFIG, "Duplicate -f options on command line."); tor_free(fname); } -#ifdef MS_WINDOWS - /* XXX one day we might want to extend expand_filename to work - * under Windows as well. */ - fname = tor_strdup(argv[i+1]); -#else fname = expand_filename(argv[i+1]); -#endif *using_default_torrc = 0; ++i; } else if (!strcmp(argv[i],"--ignore-missing-torrc")) { @@ -4020,6 +3951,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4580,7 +4517,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4691,15 +4628,12 @@ write_configuration_file(const char *fname, or_options_t *options) int options_save_current(void) { - if (torrc_fname) { - /* This fails if we can't write to our configuration file. - * - * If we try falling back to datadirectory or something, we have a better - * chance of saving the configuration, but a better chance of doing - * something the user never expected. Let's just warn instead. */ - return write_configuration_file(torrc_fname, get_options()); - } - return write_configuration_file(get_default_conf_file(), get_options()); + /* This fails if we can't write to our configuration file. + * + * If we try falling back to datadirectory or something, we have a better + * chance of saving the configuration, but a better chance of doing + * something the user never expected. */ + return write_configuration_file(get_torrc_fname(), get_options()); } /** Mapping from a unit name to a multiplier for converting that unit into a @@ -4764,30 +4698,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4797,7 +4748,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4819,256 +4771,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; - -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5155,8 +4888,7 @@ or_state_set(or_state_t *new_state) { char *err = NULL; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); @@ -5166,6 +4898,10 @@ or_state_set(or_state_t *new_state) log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); } + if (circuit_build_times_parse_state(&circ_times, global_state, &err) < 0) { + log_warn(LD_GENERAL,"%s",err); + tor_free(err); + } } /** Reload the persistent state from disk, generating a new state as needed. @@ -5298,6 +5034,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); @@ -5363,10 +5100,9 @@ getinfo_helper_config(control_connection_t *conn, int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; @@ -5388,13 +5124,8 @@ getinfo_helper_config(control_connection_t *conn, if (!type) continue; len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_snprintf(line, len, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 000000000..be9eaa331 --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 5036be450..3747f2edf 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -21,7 +21,8 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -32,6 +33,10 @@ static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +97,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +181,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +209,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +307,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +316,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +376,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +390,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +404,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +427,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +436,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -544,13 +533,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -589,7 +571,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -840,9 +822,9 @@ warn_too_many_conns(void) log_warn(LD_NET,"Failing because we have %d connections already. Please " "raise your ulimit -n.", n_conns); last_warned = now; + control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", + n_conns); } - control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", - n_conns); } /** Bind a new non-blocking socket listening to the socket described @@ -1254,7 +1236,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1267,8 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1276,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1291,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1300,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HttpsProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1643,6 +1975,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +1983,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,8 +2052,8 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) @@ -1728,10 +2071,16 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + if (num_read > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_read(conn->port, num_read); rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_written(conn->port, num_written); rep_hist_note_bytes_written(num_written, now); + } if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2088,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2130,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2229,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2262,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2274,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,8 +2302,8 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { int max_to_read=-1, try_to_read; size_t before, n_read = 0; @@ -2016,13 +2372,13 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { /* Probably a no-op, but hey. */ - connection_buckets_decrement(linked, approx_time(), 0, n_read); + connection_buckets_decrement(linked, approx_time(), n_read, 0); if (connection_flushed_some(linked) < 0) connection_mark_for_close(linked); @@ -2033,7 +2389,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2398,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2075,7 +2441,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2243,8 +2609,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2625,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2669,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2688,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2714,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2404,6 +2777,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +2959,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +2978,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2734,7 +3112,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -3033,7 +3411,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index ded02e936..861482d2a 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -330,11 +330,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +377,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +428,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +446,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +456,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +466,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -577,8 +584,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +690,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +703,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -782,14 +797,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +828,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +838,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +860,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +924,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +945,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +966,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1230,8 +1246,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1252,7 +1270,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1400,7 +1419,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1417,7 +1436,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, RESOLVED_TYPE_IPV4, tor_strdup(socks->address)); tor_assert(new_addr); log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1473,7 +1493,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1481,11 +1501,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1494,7 +1515,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); tor_assert(!automap); if (s) { @@ -1505,7 +1526,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, *s = 0; } else { log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1521,7 +1542,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, } else { log_warn(LD_APP, "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1535,7 +1556,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1544,18 +1565,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1585,7 +1594,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1649,12 +1658,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1676,32 +1685,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1889,14 +1881,6 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else @@ -2160,7 +2144,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2168,7 +2152,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2178,12 +2162,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2215,7 +2193,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2566,6 +2545,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2623,7 +2607,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (rh.command == RELAY_COMMAND_BEGIN_DIR) { tor_assert(or_circ); if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr)) - tor_addr_assign(&n_stream->_base.addr, &or_circ->p_conn->real_addr); + tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr); return connection_exit_connect_dir(n_stream); } @@ -2713,7 +2697,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2735,7 +2719,7 @@ connection_exit_connect(edge_connection_t *edge_conn) case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2748,7 +2732,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2795,13 +2779,17 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn = dir_connection_new(AF_INET); - tor_addr_assign(&dirconn->_base.addr, &exitconn->_base.addr); + tor_addr_copy(&dirconn->_base.addr, &exitconn->_base.addr); dirconn->_base.port = 0; dirconn->_base.address = tor_strdup(exitconn->_base.address); dirconn->_base.type = CONN_TYPE_DIR; dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2852,10 +2840,16 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) * to exit from it, or 0 if it probably will not allow it. * (We might be uncertain if conn's destination address has not yet been * resolved.) + * + * If <b>excluded_means_no</b> is 1 and Exclude*Nodes is set and excludes + * this relay, return 0. */ int -connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) +connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2901,20 +2895,35 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + (options->StrictNodes || excluded_means_no) && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* If we are trying to avoid this node as exit, and we have StrictNodes + * set, then this is not a suitable exit. Refuse it. + * + * If we don't have StrictNodes set, then this function gets called in + * two contexts. First, we've got a circuit open and we want to know + * whether we can use it. In that case, we somehow built this circuit + * despite having the last hop in ExcludeExitNodes, so we should be + * willing to use it. Second, we are evaluating whether this is an + * acceptable exit for a new circuit. In that case, skip it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2923,8 +2932,16 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } else { + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable " + "it."); + /* FFFF send a controller event too to notify Vidalia users */ + return BAD_HOSTNAME; + } } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2943,11 +2960,3 @@ failed: return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 3ac0bf545..dfd0a965b 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -80,10 +80,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -187,66 +185,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +193,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -312,11 +263,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +281,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HttpsProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +320,19 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +static int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -386,11 +343,31 @@ connection_or_init_conn_from_address(or_connection_t *conn, int started_here) { or_options_t *options = get_options(); + int rate, burst; /* per-connection rate limiting params */ routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + if (connection_or_digest_is_known_relay(id_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + (int)networkstatus_get_param(NULL, "bwconnrate", + (int)options->BandwidthRate); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + (int)networkstatus_get_param(NULL, "bwconnburst", + (int)options->BandwidthBurst); + } + + conn->bandwidthrate = rate; + conn->read_bucket = conn->write_bucket = conn->bandwidthburst = burst; + conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); tor_addr_copy(&conn->real_addr, addr); @@ -753,6 +730,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +749,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* use a proxy server if available */ if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); + using_proxy = 1; + tor_addr_copy(&addr, &options->HttpsProxyAddr); port = options->HttpsProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +778,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +805,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,7 +919,7 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for @@ -959,19 +946,20 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +968,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -1015,7 +1004,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); } @@ -1031,7 +1020,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1073,8 +1062,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1071,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1108,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1126,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1151,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1234,6 +1227,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1243,6 +1237,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the diff --git a/src/or/control.c b/src/or/control.c index 7eb2e042f..771beaeb5 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -43,7 +43,8 @@ #define EVENT_STREAM_BANDWIDTH_USED 0x0014 #define EVENT_CLIENTS_SEEN 0x0015 #define EVENT_NEWCONSENSUS 0x0016 -#define _EVENT_MAX 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define _EVENT_MAX 0x0017 /* If _EVENT_MAX ever hits 0x0020, we need to make the mask wider. */ /** Bitfield: The bit 1<<e is set if <b>any</b> open control @@ -54,13 +55,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +65,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +86,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +102,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +115,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +148,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +188,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +280,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -542,28 +510,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +534,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +542,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +568,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +583,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +829,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +871,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +880,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1002,6 +923,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, event_code = EVENT_CLIENTS_SEEN; else if (!strcasecmp(ev, "NEWCONSENSUS")) event_code = EVENT_NEWCONSENSUS; + else if (!strcasecmp(ev, "BUILDTIMEOUT_SET")) + event_code = EVENT_BUILDTIMEOUT_SET; else { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); @@ -1016,8 +939,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1328,7 +1249,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1266,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1381,6 +1303,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1463,6 +1387,7 @@ static int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1587,7 +1512,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1523,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1637,6 +1560,7 @@ static int getinfo_helper_events(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1571,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1651,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1666,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1691,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1846,21 +1758,10 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + const char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + *answer = tor_strdup(bridge_stats); } else { return 0; } @@ -1870,8 +1771,9 @@ getinfo_helper_events(control_connection_t *control_conn, /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in - * *<b>a</b>. If there's no answer, or an error occurs, just don't set - * <b>a</b>. Return 0. + * *<b>a</b>. If an internal error occurs, return -1. On success, or if + * the key is not recognized, return 0. Do not set <b>a</b> if the key is + * not recognized. */ typedef int (*getinfo_helper_t)(control_connection_t *, const char *q, char **a); @@ -1894,6 +1796,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1933,7 +1837,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +1848,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -2108,12 +2003,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, static uint8_t circuit_purpose_from_string(const char *string) { - if (!strcmpstart(string, "purpose=")) + if (!strcasecmpstart(string, "purpose=")) string += strlen("purpose="); - if (!strcmp(string, "general")) + if (!strcasecmp(string, "general")) return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcmp(string, "controller")) + else if (!strcasecmp(string, "controller")) return CIRCUIT_PURPOSE_CONTROLLER; else return CIRCUIT_PURPOSE_UNKNOWN; @@ -2145,6 +2040,31 @@ getargs_helper(const char *command, control_connection_t *conn, return NULL; } +/** Helper. Return the first element of <b>sl</b> at index <b>start_at</b> or + * higher that starts with <b>prefix</b>, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int @@ -2160,33 +2080,57 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, router_nicknames = smartlist_create(); - args = getargs_helper("EXTENDCIRCUIT", conn, body, 2, -1); + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); if (!args) goto done; zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - if (zero_circ && smartlist_len(args)>2) { - char *purp = smartlist_get(args,2); - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch_by_router(intended_purpose, NULL, + CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); goto done; } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!zero_circ && !circ) { + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); goto done; } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + routers = smartlist_create(); SMARTLIST_FOREACH(router_nicknames, const char *, n, { @@ -2244,8 +2188,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2271,7 +2214,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, } { - char *purp = smartlist_get(args,1); + const char *purp = find_element_starting_with(args,1,"PURPOSE="); new_purpose = circuit_purpose_from_string(purp); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); @@ -2316,9 +2259,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", (char*)smartlist_get(args, 1)); - } else if (circ && smartlist_len(args) > 2) { - char *hopstring = smartlist_get(args, 2); - if (!strcasecmpstart(hopstring, "HOP=")) { + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { hopstring += strlen("HOP="); hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, &hop_line_ok, NULL); @@ -2365,7 +2308,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2426,9 +2369,9 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, } } else if (!strcasecmpstart(option, "cache=")) { option += strlen("cache="); - if (!strcmp(option, "no")) + if (!strcasecmp(option, "no")) cache = 0; - else if (!strcmp(option, "yes")) + else if (!strcasecmp(option, "yes")) cache = 1; else { connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", @@ -2610,17 +2553,17 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, args = smartlist_create(); smartlist_split_string(args, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) && - !strcasecmp(smartlist_get(args, 0), "mode=reverse")) { - char *cp = smartlist_get(args, 0); - smartlist_del_keeporder(args, 0); - tor_free(cp); - is_reverse = 1; + { + const char *modearg = find_element_starting_with(args, 0, "mode="); + if (modearg && !strcasecmp(modearg, "mode=reverse")) + is_reverse = 1; } failed = smartlist_create(); SMARTLIST_FOREACH(args, const char *, arg, { - if (dnsserv_launch_request(arg, is_reverse)<0) - smartlist_add(failed, (char*)arg); + if (!is_keyval_pair(arg)) { + if (dnsserv_launch_request(arg, is_reverse)<0) + smartlist_add(failed, (char*)arg); + } }); send_control_done(conn); @@ -2710,7 +2653,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2660,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2672,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -3039,20 +2975,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3188,8 +3115,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s %s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3129,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) -{ - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3182,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3203,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3232,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3252,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3322,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3335,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3353,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3375,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3425,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3465,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3608,6 +3492,51 @@ control_event_newconsensus(const networkstatus_t *consensus) consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); } +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + const char *type_string = NULL; + double qnt = circuit_build_times_quantile_cutoff(); + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, + "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%lf CUTOFF_QUANTILE=%lf\r\n", + type_string, (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt); + + return 0; +} + /** Called when a single local_routerstatus_t has changed: Sends an NS event * to any controller that cares. */ int @@ -3631,7 +3560,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3608,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3672,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3680,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -4008,10 +3933,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/directory.c b/src/or/directory.c index 4153bca51..39e67c957 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -92,11 +92,13 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { @@ -228,7 +230,7 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * (length 'payload_len'). dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * * <b>type</b> specifies what sort of dir authorities (V1, V2, @@ -556,7 +558,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -610,7 +612,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -647,7 +649,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -663,7 +665,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -742,6 +744,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HttpProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -767,7 +778,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* then we want to connect to dirport directly */ if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); + tor_addr_copy(&addr, &options->HttpProxyAddr); dir_port = options->HttpProxyPort; } @@ -789,7 +800,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -828,7 +839,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -876,7 +887,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -888,10 +899,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -899,8 +910,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -909,7 +920,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -1045,31 +1056,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1158,7 +1148,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1474,21 +1464,22 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { - routerstatus_t *rs; - trusted_dir_server_t *ds; - log_info(LD_DIR,"Received http status code %d (%s) from server " - "'%s:%d'. I'll try again soon.", - status_code, escaped(reason), conn->_base.address, - conn->_base.port); - if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) - rs->last_dir_503_at = now; - if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) - ds->fake_status.last_dir_503_at = now; + if (status_code == 503) { + if (body_len < 16) { + routerstatus_t *rs; + trusted_dir_server_t *ds; + log_info(LD_DIR,"Received http status code %d (%s) from server " + "'%s:%d'. I'll try again soon.", + status_code, escaped(reason), conn->_base.address, + conn->_base.port); + if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) + rs->last_dir_503_at = now; + if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) + ds->fake_status.last_dir_503_at = now; - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } else if (status_code == 503) { + tor_free(body); tor_free(headers); tor_free(reason); + return -1; + } /* XXXX022 Remove this once every server with bug 539 is obsolete. */ log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " "a body anyway. We'll pretend it gave us a 200.", @@ -1560,7 +1551,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1576,7 +1567,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1634,8 +1625,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1662,9 +1653,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1675,7 +1668,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1695,11 +1688,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1727,7 +1720,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1915,7 +1908,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1962,7 +1955,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2005,12 +1998,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2317,7 +2304,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2338,31 +2325,32 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { - log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d); + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "Failed to decode requested authority digest %s.", d); continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2509,9 +2497,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2526,18 +2517,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2545,6 +2562,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2552,11 +2570,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2568,18 +2588,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2615,7 +2642,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2641,7 +2668,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2694,6 +2722,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2775,7 +2838,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2785,7 +2849,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2887,18 +2952,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3175,8 +3231,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3212,6 +3268,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3511,19 +3577,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3535,22 +3619,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3560,26 +3647,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 851fc5cbd..5f1cb8543 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -41,7 +41,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +63,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +102,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +184,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,10 +370,10 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, + * and there aren't any critical relay-side vulnerabilities. Once more + * of them die off, we should raise this minimum to 0.2.0.x. */ + if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; @@ -520,7 +522,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -835,46 +837,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,6 +895,13 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ @@ -944,6 +913,10 @@ list_single_server_status(routerinfo_t *desc, int is_live) void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX022 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; if (router_is_me(router) && !we_are_hibernating()) @@ -952,7 +925,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) answer = get_options()->AssumeReachable || now < router->last_reachable + REACHABLE_TIMEOUT; - if (!answer) { + if (!answer && running_long_enough_to_decide_unreachable()) { /* not considered reachable. tell rephist. */ rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); } @@ -965,7 +938,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +957,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1090,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1238,14 +1210,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1291,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1389,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1577,7 +1562,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1591,9 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1790,7 +1776,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl) if (router_is_active(ri, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = exit_policy_is_general_exit(ri->exit_policy); + ri->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1897,16 +1884,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +1916,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +1929,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +1971,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2232,6 +2254,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2256,15 +2449,12 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2309,11 +2499,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; @@ -2328,12 +2520,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,15 +2602,22 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { @@ -2537,7 +2763,7 @@ generate_v2_networkstatus_opinion(void) if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2785,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2818,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2867,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2707,10 +2933,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2775,7 +3003,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2788,7 +3017,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2904,17 +3134,20 @@ dirserv_test_reachability(time_t now, int try_all) ctr = (ctr + 1) % 128; } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX022 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3000,6 +3233,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3031,6 +3276,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3094,6 +3350,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3117,6 +3375,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3199,8 +3495,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3224,6 +3519,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3245,10 +3542,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 64e07cc0a..ecf236e8f 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,21 +11,55 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 9 + +/** Lowest consensus method that contains a 'directory-footer' marker */ +#define MIN_METHOD_FOR_FOOTER 9 + +/** Lowest consensus method that contains bandwidth weights */ +#define MIN_METHOD_FOR_BW_WEIGHTS 9 + +/** Lowest consensus method that contains consensus params */ +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -44,6 +78,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, uint32_t addr; routerlist_t *rl = router_get_routerlist(); char *version_lines = NULL; + int r; networkstatus_voter_info_t *voter; tor_assert(private_signing_key); @@ -70,20 +105,30 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, version_lines = tor_malloc(v_len); cp = version_lines; if (client_versions) { - tor_snprintf(cp, v_len-(cp-version_lines), + r = tor_snprintf(cp, v_len-(cp-version_lines), "client-versions %s\n", client_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for client-versions line"); + tor_assert(0); + } cp += strlen(cp); } - if (server_versions) - tor_snprintf(cp, v_len-(cp-version_lines), + if (server_versions) { + r = tor_snprintf(cp, v_len-(cp-version_lines), "server-versions %s\n", server_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for server-versions line"); + tor_assert(0); + } + } } else { version_lines = tor_strdup(""); } len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); + len += strlen("\ndirectory-footer\n"); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +138,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); - tor_snprintf(status, len, + r = tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,24 +164,38 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for network status line"); + tor_assert(0); + } + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN); - tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + r = tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for legacy-dir-key line"); + tor_assert(0); + } outp += strlen(outp); } @@ -139,15 +206,32 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); + + r = tor_snprintf(outp, endp-outp, "directory-footer\n"); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for directory-footer line"); + tor_assert(0); + } + outp += strlen(outp); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +254,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +300,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +373,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -344,7 +416,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -380,18 +453,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -455,7 +555,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -475,6 +599,468 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + +#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ + ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \ + (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \ + (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \ + (g) >= 0 && (g) <= (mx)) + +#define CHECK_EQ(a, b, margin) \ + ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin)) + +typedef enum { + BW_WEIGHTS_NO_ERROR = 0, + BW_WEIGHTS_RANGE_ERROR = 1, + BW_WEIGHTS_SUMG_ERROR = 2, + BW_WEIGHTS_SUME_ERROR = 3, + BW_WEIGHTS_SUMD_ERROR = 4, + BW_WEIGHTS_BALANCE_MID_ERROR = 5, + BW_WEIGHTS_BALANCE_EG_ERROR = 6 +} bw_weights_error_t; + +/** + * Verify that any weightings satisfy the balanced formulas. + */ +static bw_weights_error_t +networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, + int64_t Wme, int64_t Wmd, int64_t Wee, + int64_t Wed, int64_t scale, int64_t G, + int64_t M, int64_t E, int64_t D, int64_t T, + int64_t margin, int do_balance) { + bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR; + + // Wed + Wmd + Wgd == 1 + if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) { + berr = BW_WEIGHTS_SUMD_ERROR; + goto out; + } + + // Wmg + Wgg == 1 + if (!CHECK_EQ(Wmg + Wgg, scale, margin)) { + berr = BW_WEIGHTS_SUMG_ERROR; + goto out; + } + + // Wme + Wee == 1 + if (!CHECK_EQ(Wme + Wee, scale, margin)) { + berr = BW_WEIGHTS_SUME_ERROR; + goto out; + } + + // Verify weights within range 0->1 + if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) { + berr = BW_WEIGHTS_RANGE_ERROR; + goto out; + } + + if (do_balance) { + // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_EG_ERROR; + goto out; + } + + // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G, + (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_MID_ERROR; + goto out; + } + } + +out: + if (berr) { + log_info(LD_DIR, + "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + return berr; +} + +static void +networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, + int64_t E, int64_t D, int64_t T, + int64_t weight_scale) +{ + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + bw_weights_error_t berr = 0; + /* Case 1: Neither are scarce. + * + * Attempt to ensure that we have a large amount of exit bandwidth + * in the middle position. + */ + casename = "Case 1 (Wme*E = Wmd*D)"; + Wgg = (weight_scale*(D+E+G+M))/(3*G); + if (D==0) Wmd = 0; + else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D > S + bw_weights_error_t berr = 0; + casename = "Case 2b (Wme*E == Wmd*D)"; + if (D != 0) { + Wgg = weight_scale; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + } + + if (D == 0 || berr) { // Can happen if M > T/3 + casename = "Case 2b (E=G)"; + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (D == 0) Wgd = 0; + else Wgd = (weight_scale*(D+E-G))/(2*D); + Wed = weight_scale - Wgd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + bw_weights_error_t berr = 0; + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); + Wgg = weight_scale; + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; + Wgg = (weight_scale*(D + E + G + M))/(3*G); + Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + if (Wgg < 0 || Wgg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + + Wgg = MAX(MIN(Wgg, weight_scale), 0); + } + if (Wgd < 0 || Wgd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wgd = MAX(MIN(Wgd, weight_scale), 0); + } + if (Wmg < 0 || Wmg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmg = MAX(MIN(Wmg, weight_scale), 0); + } + if (Wme < 0 || Wme > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wme), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wme = MAX(MIN(Wme, weight_scale), 0); + } + if (Wmd < 0 || Wmd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmd = MAX(MIN(Wmd, weight_scale), 0); + } + if (Wee < 0 || Wee > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wee), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wee = MAX(MIN(Wee, weight_scale), 0); + } + if (Wed < 0 || Wed > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wed), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wed = MAX(MIN(Wed, weight_scale), 0); + } + + // Add consensus weight keywords + smartlist_add(chunks, tor_strdup("bandwidth-weights ")); + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + } + smartlist_add(chunks, tor_strdup(buf)); + log_notice(LD_CIRC, "Computed bandwidth weights for %s: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -489,18 +1075,26 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */ + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + char *params = NULL; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -602,8 +1196,12 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_snprintf(buf, sizeof(buf), "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, tor_strdup(buf)); if (consensus_method >= 2) { tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", @@ -627,13 +1225,21 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -647,7 +1253,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, @@ -701,7 +1307,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -813,10 +1422,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0; + int is_guard = 0, is_exit = 0; int naming_conflict = 0; int n_listing = 0; int i; char buf[256]; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -835,6 +1446,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +1480,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -879,7 +1494,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -925,7 +1542,11 @@ networkstatus_compute_consensus(smartlist_t *votes, } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); - if (!strcmp(fl, "Running")) + if (!strcmp(fl, "Exit")) + is_exit = 1; + else if (!strcmp(fl, "Guard")) + is_guard = 1; + else if (!strcmp(fl, "Running")) is_running = 1; } } @@ -945,11 +1566,31 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + if (rs_out.has_bandwidth) { + T += rs_out.bandwidth; + if (is_exit && is_guard) + D += rs_out.bandwidth; + else if (is_exit) + E += rs_out.bandwidth; + else if (is_guard) + G += rs_out.bandwidth; + else + M += rs_out.bandwidth; + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs_out.nickname); + } + } + /* Ok, we already picked a descriptor digest we want to list * previously. Now we want to use the exit policy summary from * that descriptor. If everybody plays nice all the voters who @@ -1036,9 +1677,20 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Okay!! Now we can write the descriptor... */ /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + const size_t mlen = BASE64_DIGEST256_LEN+5; + digest256_to_base64(m, microdesc_digest); + cp = tor_malloc(mlen); + tor_snprintf(cp, mlen, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1055,10 +1707,12 @@ networkstatus_compute_consensus(smartlist_t *votes, log_warn(LD_BUG, "Not enough space in buffer for weight line."); *buf = '\0'; } + smartlist_add(chunks, tor_strdup(buf)); }; + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { + if (rs_out.has_exitsummary && flavor == FLAV_NS) { char buf[MAX_POLICY_LINE_LEN+1]; int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); if (r<0) { @@ -1087,29 +1741,80 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); + } + + if (consensus_method >= MIN_METHOD_FOR_FOOTER) { + /* Starting with consensus method 9, we clearly mark the directory + * footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + int64_t weight_scale = BW_WEIGHT_SCALE; + char *bw_weight_param = NULL; + + // Parse params, extract BW_WEIGHT_SCALE if present + // DO NOT use consensus_param_bw_weight_scale() in this code! + // The consensus is not formed yet! + if (strcmpstart(params, "bwweightscale=") == 0) + bw_weight_param = params; + else + bw_weight_param = strstr(params, " bwweightscale="); + + if (bw_weight_param) { + int ok=0; + char *eq = strchr(bw_weight_param, '='); + if (eq) { + weight_scale = tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } + + networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1122,9 +1827,15 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1146,11 +1857,15 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *c; if (!(c = networkstatus_parse_vote_from_string(result, NULL, NS_TYPE_CONSENSUS))) { - log_err(LD_BUG,"Generated a networkstatus consensus we couldn't " + log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); tor_free(result); return NULL; } + // Verify balancing parameters + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + networkstatus_verify_bw_weights(c); + } networkstatus_vote_free(c); } @@ -1171,10 +1886,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1191,79 +1910,179 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); + + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; + + elements = smartlist_create(); - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1272,10 +2091,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1285,45 +2105,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); - smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1513,7 +2377,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1550,14 +2414,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1651,15 +2514,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1697,12 +2584,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1760,7 +2643,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1856,8 +2745,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1884,14 +2772,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1903,6 +2795,19 @@ dirvote_compute_consensus(void) "%d of %d", n_votes, n_voters/2); goto err; } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); + goto err; + } if (!(my_cert = get_my_v3_authority_cert())) { log_warn(LD_DIR, "Can't generate consensus without a certificate."); @@ -1931,6 +2836,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -1939,39 +2845,58 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1979,7 +2904,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2005,8 +2930,7 @@ dirvote_compute_consensus(void) return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2015,75 +2939,60 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2096,8 +3005,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2110,10 +3073,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2132,17 +3095,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2158,12 +3129,8 @@ dirvote_free_all(void) smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { /* now empty as a result of clear_pending_votes. */ smartlist_free(pending_consensus_signature_list); @@ -2177,13 +3144,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2229,15 +3197,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dns.c b/src/or/dns.c index 98eb27f8b..54acef4be 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -13,7 +13,51 @@ #include "or.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option_(base, opt, val) \ + evdns_set_option((opt),(val),DNS_OPTIONS_ALL) +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) + +#elif defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER < 0x02000303 +#define evdns_base_set_option_(base, opt, val) \ + evdns_base_set_option((base), (opt),(val),DNS_OPTIONS_ALL) + +#else +#define evdns_base_set_option_ evdns_base_set_option + +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +72,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +136,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +260,8 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +311,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +354,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +377,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +400,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -611,7 +663,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * know the answer. */ if (tor_addr_from_str(&addr, exitconn->_base.address) >= 0) { if (tor_addr_family(&addr) == AF_INET) { - tor_addr_assign(&exitconn->_base.addr, &addr); + tor_addr_copy(&exitconn->_base.addr, &addr); exitconn->address_ttl = DEFAULT_DNS_TTL; return 1; } else { @@ -711,6 +763,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +860,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1162,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1122,16 +1184,13 @@ configure_nameservers(int force) log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1205,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,38 +1223,47 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) evdns_base_set_option_(the_evdns_base, (k), (v)) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1292,6 +1361,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1310,25 +1380,29 @@ launch_resolve(edge_connection_t *exitconn) if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1449,8 +1523,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1540,19 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1564,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1576,18 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1628,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1636,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1657,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1704,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1748,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index bceaa462b..2e5301eae 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,14 @@ **/ #include "or.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX022 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +92,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -136,13 +138,13 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -181,13 +183,13 @@ dnsserv_launch_request(const char *name, int reverse) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } @@ -297,8 +299,8 @@ dnsserv_configure_listener(connection_t *conn) tor_assert(conn->s >= 0); tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER); - conn->dns_server_port = evdns_add_server_port(conn->s, 0, - evdns_server_callback, NULL); + conn->dns_server_port = + tor_evdns_add_server_port(conn->s, 0, evdns_server_callback, NULL); } /** Free the evdns server port for <b>conn</b>, which must be an diff --git a/src/or/eventdns.c b/src/or/eventdns.c index a889e803e..06add11b1 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -1342,8 +1293,8 @@ server_port_read(struct evdns_server_port *s) { static void server_port_flush(struct evdns_server_port *port) { - while (port->pending_replies) { - struct server_request *req = port->pending_replies; + struct server_request *req = port->pending_replies; + while (req) { ssize_t r = sendto(port->socket, req->response, req->response_len, 0, (struct sockaddr*) &req->addr, (socklen_t)req->addrlen); if (r < 0) { @@ -1355,6 +1306,9 @@ server_port_flush(struct evdns_server_port *port) if (server_request_free(req)) { /* we released the last reference to req->port. */ return; + } else { + assert(port->pending_replies != req); + req = port->pending_replies; } } @@ -2046,7 +2000,6 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { * request_finished; that one already deletes the timeout event. * XXXX021 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2109,8 +2062,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2177,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2194,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2242,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2283,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2297,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2520,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2909,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2985,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3384,8 +3344,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/geoip.c b/src/or/geoip.c index eb8460e24..0f4805ec9 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -21,9 +21,9 @@ typedef struct geoip_entry_t { } geoip_entry_t; /** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 +#define REQUEST_HIST_LEN 1 /** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) +#define REQUEST_HIST_PERIOD (24*60*60) /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { @@ -42,7 +42,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -186,7 +186,14 @@ geoip_load_file(const char *filename, or_options_t *options) return -1; } if (!geoip_countries) { + geoip_country_t *geoip_unresolved; geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); country_idxplus1_by_lc_code = strmap_new(); } if (geoip_entries) { @@ -261,8 +268,8 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; #define ACTION_MASK 3 @@ -289,7 +296,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +304,89 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + +/* Rotate period of v2 and v3 network status requests. */ +static void +rotate_request_period(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { +#if REQUEST_HIST_LEN > 1 + memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); +#endif + c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; + c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; + }); + current_request_period_starts += REQUEST_HIST_PERIOD; + if (n_old_request_periods < REQUEST_HIST_LEN-1) + ++n_old_request_periods; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,60 +394,57 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; /* Did we recently switch from bridge to relay or back? */ if (client_history_starts > now) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; + /* As a bridge that doesn't rotate request periods every 24 hours, + * possibly rotate now. */ + if (options->BridgeRelay) { + while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { + if (!geoip_countries) + geoip_countries = smartlist_create(); + if (!current_request_period_starts) { + current_request_period_starts = now; + break; + } + /* Also discard all items in the client history that are too old. + * (This only works here because bridge and directory stats are + * independent. Otherwise, we'd only want to discard those items + * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ + geoip_remove_old_clients(current_request_period_starts); + /* Now rotate request period */ + rotate_request_period(); } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); + } + lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); if (ent) { - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; } else { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) @@ -367,6 +452,10 @@ geoip_note_client_seen(geoip_client_action_t action, else ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; } + + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } if (!client_history_starts) { @@ -380,8 +469,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -403,6 +492,38 @@ geoip_remove_old_clients(time_t cutoff) client_history_starts = cutoff; } +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; +} + /** Do not mention any country from which fewer than this number of IPs have * connected. This conceivably avoids reporting information that could * deanonymize users, though analysis is lacking. */ @@ -442,32 +563,258 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX022 once we're sure the bug case never happens, we can switch + * to HT_INSERT */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) +{ + char *result = NULL; + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) + return NULL; + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); + } + } + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** How long do we have to have observed per-country request history before we + * are willing to talk about it? */ +#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) + +/** Helper for geoip_get_client_history_dirreq() and + * geoip_get_client_history_bridge(). */ +static char * +geoip_get_client_history(time_t now, geoip_client_action_t action, + int min_observation_time, unsigned granularity) { char *result = NULL; if (!geoip_is_loaded()) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { + if (client_history_starts < (now - min_observation_time)) { char buf[32]; smartlist_t *chunks = NULL; smartlist_t *entries = NULL; @@ -476,18 +823,13 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) clientmap_entry_t **ent; unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif HT_FOREACH(ent, clientmap, &client_history) { int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) + if ((*ent)->action != (int)action) continue; country = geoip_get_country_by_ip((*ent)->ipaddr); if (country < 0) - continue; + country = 0; /** unresolved requests are stored at index 0. */ tor_assert(0 <= country && country < n_countries); ++counts[country]; ++total; @@ -536,6 +878,34 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) return result; } +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * directory. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + DIR_RECORD_USAGE_MIN_OBSERVATION_TIME, + DIR_RECORD_USAGE_GRANULARITY); +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + GEOIP_MIN_OBSERVATION_TIME, + IP_GRANULARITY); +} + /** Return a newly allocated string holding the per-country request history * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ @@ -545,12 +915,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif + int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) + if (client_history_starts >= (now - min_observation_time)) return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) @@ -590,60 +957,370 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ +void +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_write(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; + + if (!get_options()->DirReqStatistics) + goto done; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + /* Rotate request period */ + rotate_request_period(); + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif +} + +/** Start time of bridge stats. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Parse the bridge statistics as they are written to extra-info + * descriptors for being returned to controller clients. Return the + * controller string if successful, or NULL otherwise. */ +static char * +parse_bridge_stats_controller(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *controller_str, *eos, *eol, *summary; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + size_t controller_len; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return NULL; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return NULL; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return NULL; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return NULL; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return NULL; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (tmp) { + tmp += strlen(BRIDGE_IPS); + tmp = eat_whitespace_no_nl(tmp); + eol = strchr(tmp, '\n'); + if (eol) + summary = tor_strndup(tmp, eol-tmp); + else + summary = tor_strdup(tmp); + } else { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return NULL; + summary = tor_strdup(""); + } + + controller_len = strlen("TimeStarted=\"\" CountrySummary=") + + strlen(summary) + 42; + controller_str = tor_malloc(controller_len); + if (tor_snprintf(controller_str, controller_len, + "TimeStarted=\"%s\" CountrySummary=%s", + stats_start_str, summary) < 0) { + tor_free(controller_str); + tor_free(summary); + return NULL; + } + tor_free(summary); + return controller_str; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Most recent bridge statistics formatted to be returned to controller + * clients. */ +static char *bridge_stats_controller = NULL; + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *data = NULL, + written[ISO_TIME_LEN+1], *out = NULL, *controller_str; + size_t len; + + /* If we changed from relay to bridge recently, adapt starting time + * of current measurements. */ + if (start_of_bridge_stats_interval < client_history_starts) + start_of_bridge_stats_interval = client_history_starts; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS) + return start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + data = geoip_get_client_history_bridge(now, GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + + ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; + out = tor_malloc(len); + if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n", + written, (unsigned) (now - start_of_bridge_stats_interval), + data ? data : "") < 0) + goto done; + write_str_to_file(filename, out, 0); + controller_str = parse_bridge_stats_controller(out, now); + if (!controller_str) + goto done; + start_of_bridge_stats_interval = now; + tor_free(bridge_stats_extrainfo); + tor_free(bridge_stats_controller); + bridge_stats_extrainfo = out; + out = NULL; + bridge_stats_controller = controller_str; + control_event_clients_seen(controller_str); + done: + tor_free(filename); + tor_free(statsdir); + tor_free(data); + tor_free(out); + return start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately. */ +static void +load_bridge_stats(time_t now) +{ + char *statsdir, *fname=NULL, *contents, *controller_str; + if (bridge_stats_extrainfo) + return; + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents) { + controller_str = parse_bridge_stats_controller(contents, now); + if (controller_str) { + bridge_stats_extrainfo = contents; + bridge_stats_controller = controller_str; + } else { + tor_free(contents); + } + } + done: + tor_free(fname); + tor_free(statsdir); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return most recent bridge statistics to be returned to controller + * clients, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_controller(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_controller; +} + +/** Start time of entry stats. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats. */ +void +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!get_options()->EntryStatistics) + goto done; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); } /** Helper used to implement GETINFO ip-to-country/... controller command. */ @@ -674,8 +1351,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1366,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 5ebe5b1c5..3c52a3172 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -182,6 +182,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } diff --git a/src/or/main.c b/src/or/main.c index 9b57ce872..74075b625 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -18,6 +18,12 @@ #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -127,12 +133,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +146,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +269,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +416,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +563,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +616,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +646,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -708,6 +731,7 @@ run_connection_housekeeping(int i, time_t now) return; /* we're all done here, the rest is just for OR conns */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -774,7 +798,7 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) @@ -800,14 +824,15 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -935,11 +960,66 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) { +#define WRITE_STATS_INTERVAL (24*60*60) + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + if (!time_to_write_stats_files) { + /* Initialize stats. We're doing this here and not in options_act, + * so that we know exactly when the 24 hours interval ends. */ + if (options->CellStatistics) + rep_hist_buffer_stats_init(now); + if (options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if (options->EntryStatistics) + geoip_entry_stats_init(now); + if (options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in %d hours from now.", + WRITE_STATS_INTERVAL / (60 * 60)); + time_to_write_stats_files = now + WRITE_STATS_INTERVAL; + } else { + /* Write stats to disk. */ + if (options->CellStatistics) + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (options->DirReqStatistics) + geoip_dirreq_stats_write(time_to_write_stats_files); + if (options->EntryStatistics) + geoip_entry_stats_write(time_to_write_stats_files); + if (options->ExitPortStatistics) + rep_hist_exit_stats_write(time_to_write_stats_files); + time_to_write_stats_files += WRITE_STATS_INTERVAL; + } + } else { + /* Never write stats to disk */ + time_to_write_stats_files = -1; + } + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -1109,12 +1189,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1148,8 +1222,8 @@ second_elapsed_callback(int fd, short event, void *args) (void)event; (void)args; if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); + timeout_event = tor_evtimer_new(tor_libevent_get_base(), + second_elapsed_callback, NULL); one_second.tv_sec = 1; one_second.tv_usec = 0; } @@ -1222,15 +1296,7 @@ second_elapsed_callback(int fd, short event, void *args) current_second = now; /* remember which second it is, for next time */ -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) + if (event_add(timeout_event, &one_second)) log_err(LD_NET, "Error from libevent when setting one-second timeout event"); } @@ -1393,8 +1459,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1441,20 +1509,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1597,6 +1661,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1623,7 +1688,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1720,7 +1786,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1732,12 +1798,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1826,7 +1893,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1920,7 +1989,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1928,6 +1996,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1938,12 +2007,10 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); tor_free(timeout_event); if (!postfork) { release_lockfile(); diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 000000000..f29611f93 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,411 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + _tor_malloc, _tor_realloc, _tor_free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static ssize_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + ssize_t r = 0; + size_t written; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + written = fwrite(md->body, 1, md->bodylen, f); + if (written != md->bodylen) { + log_warn(LD_DIR, + "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", + (unsigned long)written, (unsigned long)md->bodylen, + strerror(ferror(f))); + return -1; + } + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + ssize_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + ssize_t size; + off_t off = 0; + int orig_size, new_size; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index a43ed5254..8110c7838 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -35,16 +35,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +62,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +95,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +104,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +178,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +187,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); } - tor_free(s); + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +234,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +265,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +282,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +292,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +308,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -295,18 +348,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +374,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,34 +396,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -401,37 +460,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int missing_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++n_unknown; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; + else if (missing_key_here) + ++n_missing_key; else ++n_no_signature; - }); + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -784,8 +858,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -798,7 +872,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -807,7 +881,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -819,7 +893,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -872,7 +946,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,27 +1151,32 @@ static void update_consensus_networkstatus_downloads(time_t now) { or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ if (authdir_mode_v3(options)) return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1113,7 +1192,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1137,8 +1217,13 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly) { + dl_interval = 60; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1158,7 +1243,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1179,7 +1264,6 @@ update_consensus_networkstatus_fetch_time(time_t now) time_to_download_next_consensus = now; log_info(LD_DIR, "No live consensus; we should fetch one immediately."); } - } /** Return 1 if there's a reason we shouldn't try any directory @@ -1214,10 +1298,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1225,7 +1313,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1394,16 +1483,29 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1413,33 +1515,70 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(get_options())) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + result = -1; + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { /* XXX022 when we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1448,16 +1587,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1486,56 +1625,67 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (directory_caches_dir_info(get_options())) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + c = NULL; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(get_options(), now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(get_options(), current_consensus); + circuit_build_times_new_consensus_params(&circ_times, current_consensus); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); - if (ftime_definitely_before(now, current_consensus->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; @@ -1554,8 +1704,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) - networkstatus_vote_free(c); + networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); return result; @@ -1566,13 +1715,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1658,10 +1811,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1670,8 +1821,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1685,11 +1836,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1826,7 +1975,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1888,6 +2037,25 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +int32_t +get_net_param_from_list(smartlist_t *net_params, const char *param_name, + int default_val) +{ + size_t name_len = strlen(param_name); + + SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); + if (ok) + return (int32_t) v; + } + } SMARTLIST_FOREACH_END(p); + + return default_val; +} + /** Return the value of a integer parameter from the networkstatus <b>ns</b> * whose name is <b>param_name</b>. If <b>ns</b> is NULL, try loading the * latest consensus ourselves. Return <b>default_val</b> if no latest @@ -1896,27 +2064,59 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val) { - size_t name_len; - if (!ns) /* if they pass in null, go find it ourselves */ ns = networkstatus_get_latest_consensus(); if (!ns || !ns->net_params) return default_val; - name_len = strlen(param_name); + return get_net_param_from_list(ns->net_params, param_name, default_val); +} - SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { - if (!strcmpstart(p, param_name) && p[name_len] == '=') { - int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); - if (ok) - return (int32_t) v; - } - } SMARTLIST_FOREACH_END(p); +/** Return the value of a integer bw weight parameter from the networkstatus + * <b>ns</b> whose name is <b>weight_name</b>. If <b>ns</b> is NULL, try + * loading the latest consensus ourselves. Return <b>default_val</b> if no + * latest consensus, or if it has no parameter called <b>param_name</b>. */ +int32_t +networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name, + int32_t default_val) +{ + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); - return default_val; + if (!ns || !ns->weight_params) + return default_val; + + return get_net_param_from_list(ns->weight_params, weight_name, default_val); +} + +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; } /** If <b>question</b> is a string beginning with "ns/" in a format the @@ -1970,30 +2170,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/ntmain.c b/src/or/ntmain.c index ad6698e72..6f349cca6 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -6,6 +6,12 @@ #define MAIN_PRIVATE #include "or.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + #include <tchar.h> #define GENSRV_SERVICENAME TEXT("tor") #define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") @@ -218,7 +224,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); diff --git a/src/or/onion.c b/src/or/onion.c index 8ee59f284..887087424 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -58,11 +58,17 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static time_t last_warned = 0; + time_t now = time(NULL); + if (last_warned + WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL < now) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy."); + last_warned = now; + } tor_free(tmp); return -1; } @@ -253,8 +259,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -304,8 +311,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/or.h b/src/or/or.h index 74ab7e78d..cf27520f1 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -85,14 +82,14 @@ #include "crypto.h" #include "tortls.h" -#include "log.h" +#include "../common/log.h" #include "compat.h" #include "container.h" #include "util.h" #include "torgzip.h" #include "address.h" - -#include <event.h> +#include "compat_libevent.h" +#include "ht.h" /* These signals are defined to help control_signal_act work. */ @@ -221,6 +218,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +253,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -483,7 +493,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -641,10 +651,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +745,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -851,12 +851,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +928,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +953,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -980,6 +999,8 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1041,12 +1062,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1076,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1150,7 +1183,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,12 +1212,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1267,6 +1295,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1516,6 +1545,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1572,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1660,11 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +typedef struct vote_microdesc_hash_t { + struct vote_microdesc_hash_t *next; + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1673,45 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1721,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1659,6 +1771,10 @@ typedef struct networkstatus_t { * consensus, sorted by key. */ smartlist_t *net_params; + /** List of key=value strings for the bw weight parameters in the + * consensus. */ + smartlist_t *weight_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -1666,8 +1782,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1795,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2008,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1960,6 +2100,7 @@ typedef struct circuit_t { time_t timestamp_created; /**< When was this circuit created? */ time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the * circuit is clean. */ + struct timeval highres_created; /**< When exactly was the circuit created? */ uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2117,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2098,6 +2247,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2169,13 +2331,13 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ @@ -2183,6 +2345,9 @@ typedef struct { /** Union of ExcludeNodes and ExcludeExitNodes */ struct routerset_t *_ExcludeExitNodesUnion; + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ + /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; /** Bitmask; derived from AllowInvalidNodes. */ @@ -2237,8 +2402,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2328,10 +2491,15 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int CircuitBuildTimeout; /**< If non-zero, cull non-open circuits that + * were born at least this many seconds ago. If + * zero, use the internal adaptive algorithm. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,6 +2517,8 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ int NumCpus; /**< How many CPUs should we try to use? */ int RunTesting; /**< If true, create testing circuits to measure how well the * other ORs are running. */ @@ -2359,15 +2529,25 @@ typedef struct { char *ContactInfo; /**< Contact info to be published in the directory. */ char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + tor_addr_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ char *HttpProxyAuthenticator; /**< username:password string, if any. */ char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ + tor_addr_t HttpsProxyAddr; /**< Parsed addr for https proxy, if any. */ uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ + /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, * use the "Alternate*Authority" options below instead. */ @@ -2429,8 +2609,13 @@ typedef struct { * or not (1)? */ int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2626,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2638,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,6 +2689,29 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -2517,6 +2730,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2773,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2780,21 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2609,6 +2831,10 @@ typedef struct { int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + uint16_t TotalBuildTimes; + /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2703,6 +2929,7 @@ int fetch_from_buf_http(buf_t *buf, int force_complete); int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); @@ -2756,7 +2983,7 @@ void entry_guards_compute_status(void); int entry_guard_register_connect_status(const char *digest, int succeeded, int mark_relay_status, time_t now); void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); +int entry_list_is_constrained(or_options_t *options); routerinfo_t *choose_random_entry(cpath_build_state_t *state); int entry_guards_parse_state(or_state_t *state, int set, char **msg); void entry_guards_update_state(or_state_t *state); @@ -2777,6 +3004,170 @@ void bridges_retry_all(void); void entry_guards_free_all(void); +/* Circuit Build Timeout "public" functions and structures. */ + +/** Total size of the circuit timeout history to accumulate. + * 1000 is approx 2.5 days worth of continual-use circuits. */ +#define CBT_NCIRCUITS_TO_OBSERVE 1000 + +/** Maximum quantile to use to generate synthetic timeouts. + * We want to stay a bit short of 1.0, because longtail is + * loooooooooooooooooooooooooooooooooooooooooooooooooooong. */ +#define CBT_MAX_SYNTHETIC_QUANTILE 0.985 + +/** Width of the histogram bins in milliseconds */ +#define CBT_BIN_WIDTH ((build_time_t)50) + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; +#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Save state every 10 circuits */ +#define CBT_SAVE_STATE_EVERY 10 + +/* Circuit Build Timeout network liveness constants */ + +/** + * Have we received a cell in the last N circ attempts? + * + * This tells us when to temporarily switch back to + * BUILD_TIMEOUT_INITIAL_VALUE until we start getting cells, + * at which point we switch back to computing the timeout from + * our saved history. + */ +#define CBT_NETWORK_NONLIVE_TIMEOUT_COUNT (3) + +/** + * This tells us when to toss out the last streak of N timeouts. + * + * If instead we start getting cells, we switch back to computing the timeout + * from our saved history. + */ +#define CBT_NETWORK_NONLIVE_DISCARD_COUNT (CBT_NETWORK_NONLIVE_TIMEOUT_COUNT*2) + +/* Circuit build times consensus parameters */ + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define CBT_DEFAULT_RECENT_CIRCUITS 20 + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us to abandon timeout history and set + * the timeout back to BUILD_TIMEOUT_INITIAL_VALUE. + */ +#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) + +/** Minimum circuits before estimating a timeout */ +#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 + +/** Cutoff percentile on the CDF for our timeout estimation. */ +#define CBT_DEFAULT_QUANTILE_CUTOFF 80 +double circuit_build_times_quantile_cutoff(void); + +/** How often in seconds should we build a test circuit */ +#define CBT_DEFAULT_TEST_FREQUENCY 60 + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (2*1000) + +/** Initial circuit build timeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +int32_t circuit_build_times_initial_timeout(void); + +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < 1 || \ + CBT_NETWORK_NONLIVE_DISCARD_COUNT < 1 || \ + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT < 1 +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** If the network is not live, have we yet discarded our history? */ + int nonlive_discarded; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t *timeouts_after_firsthop; + /** Number of elements allocated for the above array */ + int num_recent_circs; + /** Index into circular array. */ + int after_firsthop_idx; + /** The network is not live. Timeout gathering is suspended */ + int net_suspended; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** Number of timeouts that have happened before estimating pareto + * parameters */ + int pre_timeouts; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds */ + double timeout_ms; +} circuit_build_times_t; + +extern circuit_build_times_t circ_times; +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg); +int circuit_build_times_add_timeout(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); +void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +void circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, + double quantile_cutoff); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + /********************************* circuitlist.c ***********************/ circuit_t * _circuit_get_global_list(void); @@ -2886,7 +3277,9 @@ const char *get_dirportfrontpage(void); or_options_t *get_options(void); int set_options(or_options_t *new_val, char **msg); void config_free_all(void); +const char *safe_str_client(const char *address); const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); const char *get_version(void); @@ -2898,6 +3291,7 @@ int resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr, char **hostname_out); int is_local_addr(const tor_addr_t *addr) ATTR_PURE; void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); int options_init_from_torrc(int argc, char **argv); setopt_err_t options_init_from_string(const char *cf, int command, const char *command_arg, char **msg); @@ -2931,6 +3325,7 @@ int options_need_geoip_info(or_options_t *options, const char **reason_out); int getinfo_helper_config(control_connection_t *conn, const char *question, char **answer); +const char *tor_get_digests(void); uint32_t get_effective_bwrate(or_options_t *options); uint32_t get_effective_bwburst(or_options_t *options); @@ -2951,7 +3346,6 @@ control_connection_t *control_connection_new(int socket_family); connection_t *connection_new(int type, int socket_family); void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); void connection_free(connection_t *conn); void connection_free_all(void); void connection_about_to_close_connection(connection_t *conn); @@ -2966,6 +3360,10 @@ void connection_expire_held_open(void); int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + int retry_all_listeners(smartlist_t *replaced_conns, smartlist_t *new_conns); @@ -3008,8 +3406,7 @@ connection_t *connection_get_by_type_addr_port_purpose(int type, uint16_t port, int purpose); connection_t *connection_get_by_type_state(int type, int state); connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); + const char *rendquery); #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); @@ -3059,7 +3456,8 @@ int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); void connection_exit_connect(edge_connection_t *conn); int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no); void connection_ap_expire_beginning(void); void connection_ap_attach_pending(void); void connection_ap_fail_onehop(const char *failed_digest, @@ -3113,7 +3511,7 @@ int hostname_is_noconnect_address(const char *address); typedef enum hostname_type_t { NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME } hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) int get_pf_socket(void); @@ -3199,6 +3597,15 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; + void control_update_global_event_mask(void); void control_adjust_event_log_severity(void); @@ -3264,6 +3671,8 @@ int control_event_server_status(int severity, const char *format, ...) CHECK_PRINTF(2,3); int control_event_guard(const char *nickname, const char *digest, const char *status); +int control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); int init_cookie_authentication(int enabled); smartlist_t *decode_hashed_passwords(config_line_t *passwords); @@ -3293,8 +3702,7 @@ typedef enum { void control_event_bootstrap(bootstrap_status_t status, int progress); void control_event_bootstrap_problem(const char *warn, int reason); -void control_event_clients_seen(const char *timestarted, - const char *countries); +void control_event_clients_seen(const char *controller_str); #ifdef CONTROL_PRIVATE /* Used only by control.c and test.c */ @@ -3363,9 +3771,13 @@ void directory_initiate_command(const char *address, const tor_addr_t *addr, const char *payload, size_t payload_len, time_t if_modified_since); +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); + smartlist_t *fp_out, int *compressed_out, + int flags); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; @@ -3417,8 +3829,8 @@ download_status_mark_impossible(download_status_t *dl) * Running Stable Unnamed V2Dir Valid\n". */ #define MAX_FLAG_LINE_LEN 96 /** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) /** Maximum length of an exit policy summary line. */ #define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) /** Amount of space to allocate for each entry: r, s, and v lines. */ @@ -3450,8 +3862,6 @@ enum was_router_added_t dirserv_add_multiple_descriptors( enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); void dirserv_free_descriptors(void); void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -3472,14 +3882,16 @@ int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); void directory_set_dirty(void); cached_dir_t *dirserv_get_directory(void); cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_directory(const char *directory, time_t when, int is_running_routers); void dirserv_set_cached_networkstatus_v2(const char *directory, const char *identity, time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); void dirserv_clear_old_v1_info(time_t now); void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); @@ -3501,15 +3913,38 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int dirserv_would_reject_router(routerstatus_t *rs); int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +typedef enum { + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); + routerstatus_format_type_t format); void dirserv_free_all(void); void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; + +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + /********************************* dirvote.c ************************/ /** Lowest allowable value for VoteSeconds. */ @@ -3519,6 +3954,9 @@ cached_dir_t *new_cached_dir(char *s, time_t published); /** Smallest allowable voting interval. */ #define MIN_VOTE_INTERVAL 300 +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 + void dirvote_free_all(void); /* vote manipulation */ @@ -3527,11 +3965,12 @@ char *networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); void ns_detached_signatures_free(ns_detached_signatures_t *s); /* cert manipulation */ @@ -3559,7 +3998,7 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char **msg_out); /* Item access */ -const char *dirvote_get_pending_consensus(void); +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); const char *dirvote_get_pending_detached_signatures(void); #define DGV_BY_ID 1 #define DGV_INCLUDE_PENDING 2 @@ -3574,6 +4013,17 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, networkstatus_t *v3_ns); @@ -3595,6 +4045,7 @@ int dns_resolve(edge_connection_t *exitconn); void dns_launch_correctness_checks(void); int dns_seems_to_be_broken(void); void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); /********************************* dnsserv.c ************************/ @@ -3610,6 +4061,15 @@ int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + #ifdef GEOIP_PRIVATE int geoip_parse_entry(const char *line); #endif @@ -3625,7 +4085,7 @@ country_t geoip_get_country(const char *countrycode); * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, @@ -3635,13 +4095,78 @@ typedef enum { void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now); void geoip_remove_old_clients(time_t cutoff); +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); +char *geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action); +char *geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action); char *geoip_get_request_history(time_t now, geoip_client_action_t action); int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer); void geoip_free_all(void); -void dump_geoip_stats(void); + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; + +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +void geoip_dirreq_stats_write(time_t now); +void geoip_entry_stats_init(time_t now); +void geoip_entry_stats_write(time_t now); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +const char *geoip_get_bridge_stats_extrainfo(time_t); +const char *geoip_get_bridge_stats_controller(time_t); /********************************* hibernate.c **********************/ @@ -3664,13 +4189,18 @@ extern int has_completed_circuit; int connection_add(connection_t *conn); int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); int connection_in_array(connection_t *conn); void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -void connection_watch_events(connection_t *conn, short events); +typedef enum watchable_events { + READ_EVENT=0x02, + WRITE_EVENT=0x04 +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); void connection_stop_reading(connection_t *conn); void connection_start_reading(connection_t *conn); @@ -3706,6 +4236,31 @@ void do_hash_password(void); int tor_init(int argc, char **argv); #endif +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + /********************************* networkstatus.c *********************/ /** How old do we allow a v2 network-status to get before removing it @@ -3743,14 +4298,16 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id( const char *identity); int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); char *networkstatus_get_cache_filename(const char *identity_digest); int router_set_networkstatus_v2(const char *s, time_t arrived_at, v2_networkstatus_source_t source, smartlist_t *requested_fingerprints); void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest); routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, @@ -3780,7 +4337,10 @@ networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 #define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routerstatus_list_update_from_consensus_networkstatus(time_t now); @@ -3792,10 +4352,18 @@ void signed_descs_update_status_from_consensus_networkstatus( char *networkstatus_getinfo_helper_single(routerstatus_t *rs); char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t get_net_param_from_list(smartlist_t *net_params, const char *name, + int default_val); int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val); int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); +int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight, + int32_t default_val); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); /********************************* ntmain.c ***************************/ @@ -3888,7 +4456,8 @@ addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, uint16_t port, const smartlist_t *policy); int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); + int rejectprivate, const char *local_address, + int add_default_policy); void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); int exit_policy_is_general_exit(smartlist_t *policy); int policy_is_reject_star(const smartlist_t *policy); @@ -3915,6 +4484,8 @@ int tls_error_to_orconn_end_reason(int e); int errno_to_orconn_end_reason(int e); const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); /********************************* relay.c ***************************/ @@ -3963,6 +4534,9 @@ int append_address_to_payload(char *payload_out, const tor_addr_t *addr); const char *decode_address_from_payload(tor_addr_t *addr_out, const char *payload, int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); /********************************* rephist.c ***************************/ @@ -3977,6 +4551,11 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); +void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_stream_opened(uint16_t port); +void rep_hist_exit_stats_init(time_t now); +void rep_hist_exit_stats_write(time_t now); int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); @@ -4028,13 +4607,17 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now); void hs_usage_write_statistics_to_file(time_t now); void hs_usage_free_all(void); +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +void rep_hist_buffer_stats_write(time_t now); + /********************************* rendclient.c ***************************/ void rend_client_introcirc_has_opened(origin_circuit_t *circ); void rend_client_rendcirc_has_opened(origin_circuit_t *circ); int rend_client_introduction_acked(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_refetch_renddesc(const char *query); void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); int rend_client_remove_intro_point(extend_info_t *failed_intro, const rend_data_t *rend_query); @@ -4042,7 +4625,7 @@ int rend_client_rendezvous_acked(origin_circuit_t *circ, const char *request, size_t request_len); int rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); +void rend_client_desc_trynow(const char *query); extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); @@ -4109,10 +4692,6 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, int command, size_t length, const char *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, size_t len); int rend_get_service_id(crypto_pk_env_t *pk, char *out); @@ -4255,7 +4834,6 @@ int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, crypto_pk_env_t *ident_key); int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); @@ -4380,11 +4958,13 @@ uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ typedef enum { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_GUARD + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR } bandwidth_weight_rule_t; routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule); -routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl); +routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); /** Flags to be passed to control router_choose_random_node() to indicate what * kind of nodes to pick according to what algorithm. */ @@ -4394,13 +4974,10 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, struct routerset_t *excludedset, router_crn_flags_t flags); @@ -4564,16 +5141,22 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; + + int git_tag_len; + char git_tag[DIGEST_LEN]; } tor_version_t; int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, + size_t digest_len, crypto_pk_env_t *private_key); int router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, @@ -4605,13 +5188,19 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); +int compare_routerstatus_entries(const void **_a, const void **_b); networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); +int networkstatus_verify_bw_weights(networkstatus_t *ns); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_type_t ns_type); ns_detached_signatures_t *networkstatus_parse_detached_signatures( const char *s, const char *eos); +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, @@ -4630,5 +5219,7 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); +void tor_gettimeofday_cache_clear(void); + #endif diff --git a/src/or/policies.c b/src/or/policies.c index 7299b7db3..90e159a88 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -344,7 +344,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,12 +376,6 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); err: addr_policy_list_free(addr_policy); @@ -829,14 +824,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,8 +845,10 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; @@ -866,6 +865,49 @@ policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) smartlist_add(r->exit_policy, item); } +/** Return 1 if there is at least one /8 subnet in <b>policy</b> that + * allows exiting to <b>port</b>. Otherwise, return 0. */ +static int +exit_policy_is_general_exit_helper(smartlist_t *policy, int port) +{ + uint32_t mask, ip, i; + /* Is this /8 rejected (1), or undecided (0)? */ + char subnet_status[256]; + + memset(subnet_status, 0, sizeof(subnet_status)); + SMARTLIST_FOREACH(policy, addr_policy_t *, p, { + if (p->prt_min > port || p->prt_max < port) + continue; /* Doesn't cover our port. */ + mask = 0; + tor_assert(p->maskbits <= 32); + + if (p->maskbits) + mask = UINT32_MAX<<(32-p->maskbits); + ip = tor_addr_to_ipv4h(&p->addr); + + /* Calculate the first and last subnet that this exit policy touches + * and set it as loop boundaries. */ + for (i = ((mask & ip)>>24); i <= (~((mask & ip) ^ mask)>>24); ++i) { + tor_addr_t addr; + if (subnet_status[i] != 0) + continue; /* We already reject some part of this /8 */ + tor_addr_from_ipv4h(&addr, i<<24); + if (tor_addr_is_internal(&addr, 0)) + continue; /* Local or non-routable addresses */ + if (p->policy_type == ADDR_POLICY_ACCEPT) { + if (p->maskbits > 8) + continue; /* Narrower than a /8. */ + /* We found an allowed subnet of at least size /8. Done + * for this port! */ + return 1; + } else if (p->policy_type == ADDR_POLICY_REJECT) { + subnet_status[i] = 1; + } + } + }); + return 0; +} + /** Return true iff <b>ri</b> is "useful as an exit node", meaning * it allows exit to at least one /8 address space for at least * two of ports 80, 443, and 6667. */ @@ -879,19 +921,7 @@ exit_policy_is_general_exit(smartlist_t *policy) return 0; for (i = 0; i < 3; ++i) { - SMARTLIST_FOREACH(policy, addr_policy_t *, p, { - if (p->prt_min > ports[i] || p->prt_max < ports[i]) - continue; /* Doesn't cover our port. */ - if (p->maskbits > 8) - continue; /* Narrower than a /8. */ - if (tor_addr_is_loopback(&p->addr)) - continue; /* 127.x or ::1. */ - /* We have a match that is at least a /8. */ - if (p->policy_type == ADDR_POLICY_ACCEPT) { - ++n_allowed; - break; /* stop considering this port */ - } - }); + n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]); } return n_allowed >= 2; } @@ -1271,7 +1301,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1280,19 +1311,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/reasons.c b/src/or/reasons.c index 45b592367..46483b277 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -326,3 +326,49 @@ circuit_end_reason_to_control_string(int reason) } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + diff --git a/src/or/relay.c b/src/or/relay.c index 2082da94f..599d3d9c8 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,6 +10,7 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" #include "mempool.h" @@ -35,6 +36,26 @@ circuit_resume_edge_reading_helper(edge_connection_t *conn, static int circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ + +static struct timeval cached_time_hires = {0, 0}; + +static void +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} + /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? */ @@ -533,6 +554,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -550,9 +577,9 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, * Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ + * relay cells on rendezvous circuits. See bug 1038. Once no relays + * (and thus no rendezvous points) are running 0.2.1.3-alpha through + * 0.2.1.18, we can take out that exception. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -999,7 +1026,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1038,6 +1066,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1508,6 +1546,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1516,7 +1558,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1525,6 +1568,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1537,7 +1584,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1599,7 +1646,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1610,11 +1688,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1666,8 +1752,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && + (halflife_ms = networkstatus_get_param( + consensus, "CircPriorityHalflifeMsec", -1) >= 0)) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1679,6 +1983,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1690,10 +1996,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1707,6 +2022,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1720,6 +2037,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1739,6 +2065,10 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) @@ -1781,7 +2111,7 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1792,9 +2122,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1808,10 +2164,58 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_warn(LD_BUG, "Cannot determine insertion time of cell."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1831,7 +2235,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1954,16 +2358,31 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); } diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 7023b7f27..9a31c2d73 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -63,7 +63,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -76,48 +76,26 @@ rend_client_send_introduction(origin_circuit_t *introcirc, &entry) < 1) { log_warn(LD_REND, "query %s didn't have valid rend desc in cache. Failing.", - escaped_safe_str(introcirc->rend_data->onion_address)); + escaped_safe_str_client(introcirc->rend_data->onion_address)); goto err; } - /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto err; - } + /* first 20 bytes of payload are the hash of the intro key */ + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; } + }); + if (!intro_key) { + log_info(LD_REND, "Our introduction point knowledge changed in " + "mid-connect! Could not find intro key; we only have a " + "v2 rend desc with %d intro points. Giving up.", + smartlist_len(entry->parsed->intro_nodes)); + goto err; } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); @@ -291,7 +269,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, extend_info = rend_client_get_random_intro(circ->rend_data); if (!extend_info) { log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); + escaped_safe_str_client(circ->rend_data->onion_address)); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return -1; } @@ -299,7 +277,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Re-extending circ %d, " "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, circ->_base.n_circ_id, extend_info->nickname); result = circuit_extend_to_new_exit(circ, extend_info); @@ -307,7 +285,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Building a new introduction " "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, extend_info->nickname); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); @@ -470,45 +448,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -517,15 +471,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -551,8 +503,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -573,18 +525,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -601,18 +548,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, if (smartlist_len(ent->parsed->intro_nodes) == 0) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -620,7 +562,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -682,8 +624,9 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, request, DH_KEY_LEN, - keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->dh_handshake_state, + request, DH_KEY_LEN, keys, + DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); goto err; } @@ -721,24 +664,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -770,17 +707,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -798,7 +730,7 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { log_warn(LD_REND, "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); return NULL; } @@ -948,8 +880,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 1f9596de2..814abd8cc 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -22,6 +22,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +127,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +141,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +406,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +416,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +426,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +623,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,61 +661,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, cp, *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -826,22 +777,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -862,7 +818,7 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } @@ -888,9 +844,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -966,6 +922,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX022 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (smartlist_len((*e)->parsed->intro_nodes) == 0) + return 0; return 1; } @@ -1044,7 +1005,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1059,13 +1019,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published) now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1073,25 +1035,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published) tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && !memcmp(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1099,10 +1058,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1114,7 +1069,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published) memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1165,7 +1120,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1294,7 +1249,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1329,14 +1285,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1344,7 +1300,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1354,14 +1310,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1379,12 +1335,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 289d64d39..7795db0d7 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -10,8 +10,7 @@ #include "or.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +41,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +55,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +87,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +107,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +120,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +136,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +155,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +266,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +276,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +401,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +429,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +452,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (!memcmp(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +484,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +498,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +739,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (!memcmp(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -944,21 +884,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -988,7 +923,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1082,7 +1017,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1157,7 +1092,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1167,7 +1103,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1183,14 +1119,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1200,11 +1138,10 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1286,7 +1223,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1299,7 +1236,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1322,18 +1259,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1344,7 +1279,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1374,9 +1308,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1386,8 +1319,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /* If we already have enough introduction circuits for this service, * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { log_info(LD_CIRC|LD_REND, "We have just finished an introduction " "circuit, but we already have enough. Redefining purpose to " "general."); @@ -1400,13 +1332,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1465,9 +1392,8 @@ rend_service_intro_established(origin_circuit_t *circuit, const char *request, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1517,9 +1443,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1575,13 +1500,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1590,8 +1514,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1601,8 +1524,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1635,6 +1557,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1656,15 +1579,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1694,9 +1620,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1708,35 +1633,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1875,8 +1773,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1928,7 +1825,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1940,10 +1837,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2036,8 +1931,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2063,10 +1957,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2097,9 +1990,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rephist.c b/src/or/rephist.c index 71ec0bd53..c8da8dfe2 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -14,7 +14,6 @@ static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -185,7 +184,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -1320,6 +1318,202 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to + * see in order to be included in exit stats. */ +#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** When does the current exit stats period end? */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ +void +rep_hist_exit_stats_init(time_t now) +{ + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); +} + +/** Write exit stats to $DATADIR/stats/exit-stats and reset counters. */ +void +rep_hist_exit_stats_write(time_t now) +{ + char t[ISO_TIME_LEN+1]; + int r, i, comma; + uint64_t *b, total_bytes, threshold_bytes, other_bytes; + uint32_t other_streams; + + char *statsdir = NULL, *filename = NULL; + open_file_t *open_file = NULL; + FILE *out = NULL; + + if (!exit_streams) + return; /* Not initialized */ + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "exit-stats"); + format_iso_time(t, now); + log_info(LD_HIST, "Writing exit port statistics to disk for period " + "ending at %s.", t); + + if (!open_file) { + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) { + log_warn(LD_HIST, "Couldn't open '%s'.", filename); + goto done; + } + } + + /* written yyyy-mm-dd HH:MM:SS (n s) */ + if (fprintf(out, "exit-stats-end %s (%d s)\n", t, + (unsigned) (now - start_of_exit_stats_interval)) < 0) + goto done; + + /* Count the total number of bytes, so that we can attribute all + * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ + total_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; + } + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; + + /* exit-kibibytes-(read|written) port=kibibytes,.. */ + for (r = 0; r < 2; r++) { + b = r ? exit_bytes_read : exit_bytes_written; + tor_assert(b); + if (fprintf(out, "%s ", + r ? "exit-kibibytes-read" + : "exit-kibibytes-written") < 0) + goto done; + + comma = 0; + other_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (b[i] > 0) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint64_t num = round_uint64_to_next_multiple_of(b[i], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + if (fprintf(out, "%s%d="U64_FORMAT, + comma++ ? "," : "", i, + U64_PRINTF_ARG(num)) < 0) + goto done; + } else + other_bytes += b[i]; + } + } + other_bytes = round_uint64_to_next_multiple_of(other_bytes, + EXIT_STATS_ROUND_UP_BYTES); + other_bytes /= 1024; + if (fprintf(out, "%sother="U64_FORMAT"\n", + comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) + goto done; + } + /* exit-streams-opened port=num,.. */ + if (fprintf(out, "exit-streams-opened ") < 0) + goto done; + comma = 0; + other_streams = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_streams[i] > 0) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%s%d=%u", + comma++ ? "," : "", i, num)<0) + goto done; + } else + other_streams += exit_streams[i]; + } + } + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%sother=%u\n", + comma ? "," : "", other_streams)<0) + goto done; + /* Reset counters */ + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); + start_of_exit_stats_interval = now; + + if (open_file) + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); +} + +/** Note that we wrote <b>num_bytes</b> to an exit connection to + * <b>port</b>. */ +void +rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_bytes_written) + return; /* Not initialized */ + exit_bytes_written[port] += num_bytes; + log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we read <b>num_bytes</b> from an exit connection to + * <b>port</b>. */ +void +rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_bytes_read) + return; /* Not initialized */ + exit_bytes_read[port] += num_bytes; + log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we opened an exit stream to <b>port</b>. */ +void +rep_hist_note_exit_stream_opened(uint16_t port) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_streams) + return; /* Not initialized */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); +} + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) @@ -1419,7 +1613,7 @@ rep_hist_get_bandwidth_lines(int for_extrainfo) size_t len; /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + len = (60+21*NUM_TOTALS)*2; buf = tor_malloc_zero(len); cp = buf; for (r=0;r<2;++r) { @@ -1715,8 +1909,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, return 0; /* too long ago */ if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) *need_uptime = 1; - if (predicted_internal_capacity_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) - *need_capacity = 1; + // Always predict that we need capacity. + *need_capacity = 1; return 1; } @@ -1854,555 +2048,197 @@ rep_hist_free_all(void) tor_free(read_array); tor_free(write_array); tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); built_last_stability_doc_at = 0; predicted_ports_free(); } -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) -{ - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; -} - -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) -{ - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; -} +/*** cell statistics ***/ -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) -{ - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; -} +/** Start of the current buffer stats interval. */ +static time_t start_of_buffer_stats_interval; -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) +/** Initialize buffer stats. */ +void +rep_hist_buffer_stats_init(time_t now) { - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; + start_of_buffer_stats_interval = now; } -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) -{ - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} +typedef struct circ_buffer_stats_t { + uint32_t processed_cells; + double mean_num_cells_in_queue; + double mean_time_cells_in_queue; + uint32_t local_circ_id; +} circ_buffer_stats_t; -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} +/** Holds stats. */ +smartlist_t *circuits_for_buffer_stats = NULL; -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; - } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} - -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ +void +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) { - if (!lst) + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); -} - -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) -{ - if (!s) + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} - -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} - -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created > + start_of_buffer_stats_interval ? + circ->timestamp_created : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Frees all memory that was used for collecting hidden service usage data. */ +/** Write buffer statistics to $DATADIR/stats/buffer-stats. */ void -hs_usage_free_all(void) +rep_hist_buffer_stats_write(time_t now) { - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} - -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL; + char buf[32]; + circuit_t *circ; + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } -} - -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} - -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } -} - -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) -{ - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); -} - -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ -void -hs_usage_note_publish_total(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(publish_total, now, service_id); -} - -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ -void -hs_usage_note_publish_novel(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(publish_novel, now, service_id); -} - -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ -void -hs_usage_note_fetch_total(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(fetch_total, now, service_id); -} - -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ -void -hs_usage_note_fetch_successful(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(fetch_successful, now, service_id); -} - -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) -{ - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } - return cp-buf; -} - -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) -{ - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); } - return buf; -} - -/** Write current statistics about hidden service usage to file. */ -void -hs_usage_write_statistics_to_file(time_t now) -{ - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + tor_free(str); +#undef SHARES } diff --git a/src/or/router.c b/src/or/router.c index 6db9a7c0a..347d7f9ef 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -61,8 +61,7 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; onionkey_set_at = time(NULL); tor_mutex_release(key_lock); @@ -111,8 +110,7 @@ get_onion_key_set_at(void) void set_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); + crypto_free_pk_env(identitykey); identitykey = k; crypto_pk_get_digest(identitykey, identitykey_digest); } @@ -201,8 +199,7 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); @@ -331,10 +328,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +340,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -442,7 +436,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1267,6 +1263,7 @@ router_rebuild_descriptor(int force) uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); + size_t ei_size; or_options_t *options = get_options(); if (desc_clean_since && !force) @@ -1309,7 +1306,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->ExitPolicyRejectPrivate, - ri->address); + ri->address, !options->BridgeRelay); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1380,9 +1377,10 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + ei_size = options->ExtraInfoStatistics ? MAX_EXTRAINFO_UPLOAD_SIZE : 8192; + ei->cache_info.signed_descriptor_body = tor_malloc(ei_size); + if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, + ei_size, ei, get_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1425,11 +1423,9 @@ router_rebuild_descriptor(int force) tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1610,8 +1606,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on. @@ -1670,7 +1664,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1791,7 +1785,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1826,6 +1820,57 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + /** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, * negative on failure. */ @@ -1840,6 +1885,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char *bandwidth_usage; int result; size_t len; + static int write_stats_to_extrainfo = 1; + time_t now = time(NULL); base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -1851,23 +1898,79 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "published %s\n%s", extrainfo->nickname, identity, published, bandwidth_usage); + tor_free(bandwidth_usage); if (result<0) return -1; - if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *contents = NULL; + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write entry-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write buffer-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write exit-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + } + + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + size_t pos = strlen(s); + if (strlcpy(s + pos, bridge_stats, maxlen - strlen(s)) != + strlen(bridge_stats)) { + log_warn(LD_DIR, "Could not write bridge-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } } } @@ -1876,10 +1979,10 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, len += strlen(s+len); if (router_get_extrainfo_hash(s, digest)<0) return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) + if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN, + ident_key)<0) return -1; -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING { char *cp, *s_dup; extrainfo_t *ei_tmp; @@ -1895,31 +1998,26 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(s_dup); extrainfo_free(ei_tmp); } -#endif - - return (int)strlen(s)+1; -} -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *cp, *s_dup; + extrainfo_t *ei_tmp; + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + log_warn(LD_GENERAL, + "We just generated an extra-info descriptor with " + "statistics that we can't parse. Not adding statistics to " + "this or any future extra-info descriptors. Descriptor " + "was:\n%s", s); + write_stats_to_extrainfo = 0; + extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key); + } + tor_free(s_dup); + extrainfo_free(ei_tmp); } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + + return (int)strlen(s)+1; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2046,26 +2144,16 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 2fc80738f..f0ae53abe 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -156,21 +156,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -224,7 +227,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +244,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -442,17 +448,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +470,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (!ftime_definitely_after(now, cert->expires)) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_notice(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -749,8 +755,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -1089,9 +1094,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) } SMARTLIST_FOREACH_END(status); if (smartlist_len(tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(tunnel); + result = routerstatus_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel); + result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); } else if (smartlist_len(trusted_tunnel)) { /* FFFF We don't distinguish between trusteds and overloaded trusteds * yet. Maybe one day we should. */ @@ -1099,9 +1105,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) * is a feature, but it could easily be a bug. -RD */ result = smartlist_choose(trusted_tunnel); } else if (smartlist_len(direct)) { - result = routerstatus_sl_choose_by_bandwidth(direct); + result = routerstatus_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_direct)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_direct); + result = routerstatus_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); } else { result = smartlist_choose(trusted_direct); } @@ -1531,6 +1538,188 @@ kb_to_bytes(uint32_t bw) /** Helper function: * choose a random element of smartlist <b>sl</b>, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>statuses</b> is zero, then <b>sl</b> is a list of + * routerinfo_t's. Otherwise it's a list of routerstatus_t's. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static void * +smartlist_choose_by_bandwidth_weights(smartlist_t *sl, + bandwidth_weight_rule_t rule, + int statuses) +{ + int64_t weight_scale; + int64_t rand_bw; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + double weighted_bw = 0; + double *bandwidths; + double tmp = 0; + unsigned int i; + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + weight_scale = networkstatus_get_param(NULL, "bwweightscale", + BW_WEIGHT_SCALE); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to old selection" + " algorithm."); + return NULL; // Use old algorithm. + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_malloc_zero(sizeof(double)*smartlist_len(sl)); + + // Cycle through smartlist and total the bandwidth. + for (i = 0; i < (unsigned)smartlist_len(sl); ++i) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; + double weight = 1; + if (statuses) { + routerstatus_t *status = smartlist_get(sl, i); + is_exit = status->is_exit; + is_guard = status->is_possible_guard; + is_dir = (status->dir_port != 0); + if (!status->has_bandwidth) { + tor_free(bandwidths); + /* This should never happen, unless all the authorites downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + log_warn(LD_BUG, + "Consensus is not listing bandwidths. Defaulting back to " + "old router selection algorithm."); + return NULL; + } + this_bw = kb_to_bytes(status->bandwidth); + } else { + routerstatus_t *rs; + routerinfo_t *router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); + is_exit = router->is_exit; + is_guard = router->is_possible_guard; + is_dir = (router->dir_port != 0); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else { /* bridge or other descriptor not in our consensus */ + this_bw = router_get_advertised_bandwidth_capped(router); + } + } + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + + bandwidths[i] = weight*this_bw; + weighted_bw += weight*this_bw; + } + + log_debug(LD_CIRC, "Choosing node for rule %d based on weights " + "Wg=%lf Wm=%lf We=%lf Wd=%lf with total bw %lf", rule, + Wg, Wm, We, Wd, weighted_bw); + + rand_bw = crypto_rand_uint64(DBL_TO_U64(weighted_bw)); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ + + /* Last, count through sl until we get to the element we picked */ + tmp = 0.0; + for (i=0; i < (unsigned)smartlist_len(sl); i++) { + tmp += bandwidths[i]; + if (tmp >= rand_bw) + break; + } + + if (i == (unsigned)smartlist_len(sl)) { + /* This was once possible due to round-off error, but shouldn't be able + * to occur any longer. */ + tor_fragile_assert(); + --i; + log_warn(LD_BUG, "Round-off error in computing bandwidth had an effect on " + " which router we chose. Please tell the developers. " + "%lf " U64_FORMAT " %lf", tmp, U64_PRINTF_ARG(rand_bw), + weighted_bw); + } + tor_free(bandwidths); + return smartlist_get(sl, i); +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b>, weighted by * the advertised bandwidth of each element. * * If <b>statuses</b> is zero, then <b>sl</b> is a list of @@ -1565,6 +1754,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_t *guard_bits; int me_idx = -1; + // This function does not support WEIGHT_FOR_DIR + // or WEIGHT_FOR_MID + if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { + rule = NO_WEIGHTING; + } + /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || rule == WEIGHT_FOR_EXIT || @@ -1691,12 +1886,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1745,6 +1940,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1788,26 +1985,34 @@ routerinfo_t * routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule) { - return smartlist_choose_by_bandwidth(sl, rule, 0); + routerinfo_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 0))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 0); + } } /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each status. */ routerstatus_t * -routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) +routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule) { /* We are choosing neither exit nor guard here. Weight accordingly. */ - return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); + routerstatus_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 1))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 1); + } } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1820,8 +2025,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1829,18 +2033,16 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; tor_assert(!(weight_for_exit && need_guard)); rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - - excludednodes = smartlist_create(); + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ @@ -1857,60 +2059,35 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + // Always weight by bandwidth + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2370,6 +2547,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2401,7 +2581,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2440,46 +2621,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2849,8 +2990,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -3739,12 +3879,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3759,10 +3895,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3771,6 +3905,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3824,7 +3961,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3926,7 +4063,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -4814,8 +4951,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4950,9 +5087,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5026,7 +5162,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5194,9 +5332,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5266,35 +5408,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5305,8 +5427,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } diff --git a/src/or/routerparse.c b/src/or/routerparse.c index aa1aba423..bde45a512 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -11,6 +11,8 @@ #include "or.h" #include "memarea.h" +#undef log +#include <math.h> /****************************************************************************/ @@ -55,6 +57,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +65,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -78,13 +106,17 @@ typedef enum { K_KNOWN_FLAGS, K_PARAMS, + K_BW_WEIGHTS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +154,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +290,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +324,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -375,7 +433,7 @@ static token_rule_t client_keys_token_table[] = { /** List of tokens allowed in V3 networkstatus votes. */ static token_rule_t networkstatus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), @@ -403,7 +461,7 @@ static token_rule_t networkstatus_token_table[] = { /** List of tokens allowed in V3 networkstatus consensuses. */ static token_rule_t networkstatus_consensus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), @@ -430,6 +488,7 @@ static token_rule_t networkstatus_consensus_token_table[] = { /** List of tokens allowable in the footer of v1/v2 directory/networkstatus * footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { + T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; @@ -437,10 +496,20 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,9 +522,13 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, - char end_char); - -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, size_t s_len, + digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -479,6 +552,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -499,6 +573,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -506,7 +608,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -516,7 +619,8 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest) { return router_get_hash_impl(s, s_len, digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -526,7 +630,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -536,18 +641,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,strlen(s),digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version", "\ndirectory-signature", - ' '); + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -556,7 +674,7 @@ int router_get_extrainfo_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "extra-info", - "\nrouter-signature",'\n'); + "\nrouter-signature",'\n', DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -568,14 +686,15 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i; + int siglen; signature = tor_malloc(crypto_pk_keysize(private_key)); - if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -583,7 +702,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -690,7 +809,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -726,11 +845,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -763,11 +882,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -789,7 +909,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -818,7 +938,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -830,9 +950,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -882,7 +1003,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -918,6 +1039,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -948,14 +1070,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1141,7 +1263,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1413,7 +1535,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1431,16 +1553,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1465,6 +1586,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1539,7 +1661,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1553,12 +1676,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1582,6 +1705,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1606,7 +1730,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1699,7 +1823,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1708,6 +1832,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1727,7 +1852,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1735,8 +1860,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1747,22 +1873,30 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /** Helper: given a string <b>s</b>, return the start of the next router-status * object (starting with "r " at the start of a line). If none is found, - * return the start of the next directory signature. If none is found, return - * the end of the string. */ + * return the start of the directory footer, or the next directory signature. + * If none is found, return the end of the string. */ static INLINE const char * find_start_of_next_routerstatus(const char *s) { const char *eos = strstr(s, "\nr "); if (eos) { - const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); + const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-footer\n"); + if (eos2) eos2 += strlen("\ndirectory-footer"); + else eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); + /* Technically we are returning either the start of the next + * routerstatus AFTER the \n, or the start of the next consensus + * line AT the \n. This seems asymmetric, but leaving it that way + * for now for stability. */ if (eos2 && eos2 < eos) return eos2; else return eos+1; } else { + if ((eos = strstr(s, "\ndirectory-footer\n"))) + return eos+strlen("\ndirectory-footer\n"); if ((eos = strstr(s, "\ndirectory-signature"))) return eos+1; - return s + strlen(s); + return s + strlen(s); } } @@ -1777,22 +1911,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1804,7 +1945,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1825,29 +1974,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1933,6 +2087,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1954,16 +2119,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -1975,8 +2153,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2000,7 +2178,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2114,17 +2292,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2143,19 +2321,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2164,6 +2342,394 @@ networkstatus_v2_parse_from_string(const char *s) return ns; } +/** Verify the bandwidth weights of a network status document */ +int +networkstatus_verify_bw_weights(networkstatus_t *ns) +{ + int64_t weight_scale; + int64_t G=0, M=0, E=0, D=0, T=0; + double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed; + double Gtotal=0, Mtotal=0, Etotal=0; + const char *casename = NULL; + int valid = 1; + + weight_scale = networkstatus_get_param(ns, "bwweightscale", BW_WEIGHT_SCALE); + Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); + Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1); + Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1); + Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1); + Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1); + Wme = networkstatus_get_bw_weight(ns, "Wme", -1); + Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1); + Weg = networkstatus_get_bw_weight(ns, "Weg", -1); + Wem = networkstatus_get_bw_weight(ns, "Wem", -1); + Wee = networkstatus_get_bw_weight(ns, "Wee", -1); + Wed = networkstatus_get_bw_weight(ns, "Wed", -1); + + if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0 + || Wem<0 || Wee<0 || Wed<0) { + log_warn(LD_BUG, "No bandwidth weights produced in consensus!"); + return 0; + } + + // First, sanity check basic summing properties that hold for all cases + // We use > 1 as the check for these because they are computed as integers. + // Sometimes there are rounding errors. + if (fabs(Wmm - weight_scale) > 1) { + log_warn(LD_BUG, "Wmm=%lf != "I64_FORMAT, + Wmm, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + if (fabs(Wem - Wee) > 1) { + log_warn(LD_BUG, "Wem=%lf != Wee=%lf", Wem, Wee); + valid = 0; + } + + if (fabs(Wgm - Wgg) > 1) { + log_warn(LD_BUG, "Wgm=%lf != Wgg=%lf", Wgm, Wgg); + valid = 0; + } + + if (fabs(Weg - Wed) > 1) { + log_warn(LD_BUG, "Wed=%lf != Weg=%lf", Wed, Weg); + valid = 0; + } + + if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgg=%lf != "I64_FORMAT" - Wmg=%lf", Wgg, + I64_PRINTF_ARG(weight_scale), Wmg); + valid = 0; + } + + if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wee=%lf != "I64_FORMAT" - Wme=%lf", Wee, + I64_PRINTF_ARG(weight_scale), Wme); + valid = 0; + } + + if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgd=%lf + Wmd=%lf + Wed=%lf != "I64_FORMAT, + Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + Wgg /= weight_scale; + Wgm /= weight_scale; + Wgd /= weight_scale; + + Wmg /= weight_scale; + Wmm /= weight_scale; + Wme /= weight_scale; + Wmd /= weight_scale; + + Weg /= weight_scale; + Wem /= weight_scale; + Wee /= weight_scale; + Wed /= weight_scale; + + // Then, gather G, M, E, D, T to determine case + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (rs->has_bandwidth) { + T += rs->bandwidth; + if (rs->is_exit && rs->is_possible_guard) { + D += rs->bandwidth; + Gtotal += Wgd*rs->bandwidth; + Mtotal += Wmd*rs->bandwidth; + Etotal += Wed*rs->bandwidth; + } else if (rs->is_exit) { + E += rs->bandwidth; + Mtotal += Wme*rs->bandwidth; + Etotal += Wee*rs->bandwidth; + } else if (rs->is_possible_guard) { + G += rs->bandwidth; + Gtotal += Wgg*rs->bandwidth; + Mtotal += Wmg*rs->bandwidth; + } else { + M += rs->bandwidth; + Mtotal += Wmm*rs->bandwidth; + } + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); + } + } SMARTLIST_FOREACH_END(rs); + + // Finally, check equality conditions depending upon case 1, 2 or 3 + // Full equality cases: 1, 3b + // Partial equality cases: 2b (E=G), 3a (M=E) + // Fully unknown: 2a + if (3*E >= T && 3*G >= T) { + // Case 1: Neither are scarce + casename = "Case 1"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else if (3*E < T && 3*G < T) { + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. Devote no extra + * bandwidth to middle nodes. + */ + if (R+D < S) { // Subcase a + double Rtotal, Stotal; + if (E < G) { + Rtotal = Etotal; + Stotal = Gtotal; + } else { + Rtotal = Gtotal; + Stotal = Etotal; + } + casename = "Case 2a"; + // Rtotal < Stotal + if (Rtotal > Stotal) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Rtotal %lf > Stotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal, Stotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Rtotal < T/3 + if (3*Rtotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Rtotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Mtotal > T/3 + if (3*Mtotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Mtotal %lf < T " + I64_FORMAT". " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { // Subcase b: R+D > S + casename = "Case 2b"; + + /* Check the rare-M redirect case. */ + if (D != 0 && 3*M < T) { + casename = "Case 2b (balanced)"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + int64_t NS = MAX(E, G); + if (3*(S+D) < T) { // Subcase a: + double Stotal; + double NStotal; + if (G < E) { + casename = "Case 3a (G scarce)"; + Stotal = Gtotal; + NStotal = Etotal; + } else { // if (G >= E) { + casename = "Case 3a (E scarce)"; + NStotal = Gtotal; + Stotal = Etotal; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (NS >= M) { + if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: NStotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + // if NS < M, NStotal > T/3 because only one of G or E is scarce + if (3*NStotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*NStotal %lf < T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } else { // Subcase b: S+D >= T/3 + casename = "Case 3b"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + + if (valid) + log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.", + casename); + + return valid; +} /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ @@ -2175,19 +2741,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2203,7 +2771,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consenus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2357,8 +2941,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2450,7 +3035,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2460,7 +3045,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2493,14 +3079,66 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, { + int found_sig = 0; + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) + found_sig = 1; + else if (found_sig) { + log_warn(LD_DIR, "Extraneous token after first directory-signature"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); + } + + tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS); + if (tok) { + ns->weight_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in weight params", + escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + smartlist_add(ns->weight_params, tor_strdup(tok->args[i])); + } + } + + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2509,11 +3147,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2521,11 +3159,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2534,35 +3176,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX) + if (tok->object_size >= INT_MAX) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2570,27 +3226,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2603,6 +3263,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2612,10 +3301,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2627,18 +3319,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2658,57 +3389,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2764,7 +3540,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2887,9 +3663,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2901,7 +3676,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3179,7 +3954,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3293,17 +4068,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, size_t s_len, char *digest, +router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); @@ -3330,14 +4099,214 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, size_t s_len, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } + + return 0; +} + +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); return -1; } return 0; } +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3362,7 +4331,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3458,6 +4427,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3483,8 +4469,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3573,7 +4565,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, strlen(desc), desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3692,7 +4684,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3706,12 +4698,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3869,7 +4860,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -3942,7 +4933,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -3981,7 +4972,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4053,7 +5044,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index b451638c2..000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4900 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(); - crypto_dh_env_t *dh2 = crypto_dh_new(); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_free(_thread_test_mutex); - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/test_data.c b/src/or/test_data.c deleted file mode 100644 index f926ee17d..000000000 --- a/src/or/test_data.c +++ /dev/null @@ -1,173 +0,0 @@ -/* Copyright 2001-2004 Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** First of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_1[] = -"dir-key-certificate-version 3\n" -"fingerprint D867ACF56A9D229B35C25F0090BC9867E906BE69\n" -"dir-key-published 2008-12-12 18:07:24\n" -"dir-key-expires 2009-12-12 18:07:24\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAveMpKlw8oD1YqFqpJchuwSR82BDhutbqgHiez3QO9FmzOctJpV+Y\n" -"mpTYIJLS/qC+4GBKFF1VK0C4SoBrS3zri0qdXdE+vBGcyrxrjMklpxoqSKRY2011\n" -"4eqYPghKlo5RzuqteBclGCHyNxWjUJeRKDWgvh+U/gr2uYM6fRm5q0fCzg4aECE7\n" -"VP6fDGZrMbQI8jHpiMSoC9gkUASNEa6chLInlnP8/H5qUEW4TB9CN/q095pefuwL\n" -"P+F+1Nz5hnM7fa5XmeMB8iM4RriUmOQlLBZgpQBMpEfWMIPcR9F1Gh3MxERqqUcH\n" -"tmij+IZdeXg9OkCXykcabaYIhZD3meErn9Tax4oA/THduLfgli9zM0ExwzH1OooN\n" -"L8rIcJ+2eBo3bQiQUbdYW71sl9w7nSPtircbJUa1mUvWYLPWQxFliPiQSetgJLMj\n" -"VQqtPmV2hvN2Xk3lLfJO50qMTK7w7Gsaw8UtV4YDM1Hcjp/hQaIB1xfwhXgl+eUU\n" -"btUa4c+cUTjHAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBALPSUInyuEu6NV3NjozplaniIEBzQXEjv1x9/+mqnwZABpYVmuy9A8nx\n" -"eoyY3sZFsnYwNW/IZjAgG23pEmevu3F+L4myMjjaa6ORl3MgRYQ4gmuFqpefrGdm\n" -"ywRCleh2JerkQ4VxOuq10dn/abITzLyaZzMw30KXWp5pxKXOLtxFAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-crosscert\n" -"-----BEGIN ID SIGNATURE-----\n" -"FTBJNR/Hlt4T53yUMp1r/QCSMCpkHJCbYBT0R0pvYqhqFfYN5qHRSICRXaFFImIF\n" -"0DGWmwRza6DxPKNzkm5/b7I0de9zJW1jNNdQAQK5xppAtQcAafRdu8cBonnmh9KX\n" -"k1NrAK/X00FYywju3yl/SxCn1GddVNkHYexEudmJMPM=\n" -"-----END ID SIGNATURE-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"pjWguLFBfELZDc6DywL6Do21SCl7LcutfpM92MEn4WYeSNcTXNR6lRX7reOEJk4e\n" -"NwEaMt+Hl7slgeR5wjnW3OmMmRPZK9bquNWbfD+sAOV9bRFZTpXIdleAQFPlwvMF\n" -"z/Gzwspzn4i2Yh6hySShrctMmW8YL3OM8LsBXzBhp/rG2uHlsxmIsc13DA6HWt61\n" -"ffY72uNE6KckDGsQ4wPGP9q69y6g+X+TNio1KPbsILbePv6EjbO+rS8FiS4njPlg\n" -"SPYry1RaUvxzxTkswIzdE1tjJrUiqpbWlTGxrH9N4OszoLm45Pc784KLULrjKIoi\n" -"Q+vRsGrcMBAa+kDowWU6H1ryKR7KOhzRTcf2uqLE/W3ezaRwmOG+ETmoVFwbhk2X\n" -"OlbXEM9fWP+INvFkr6Z93VYL2jGkCjV7e3xXmre/Lb92fUcYi6t5dwzfV8gJnIoG\n" -"eCHd0K8NrQK0ipVk/7zcPDKOPeo9Y5aj/f6X/pDHtb+Dd5sT+l82G/Tqy4DIYUYR\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_1 */ -const char AUTHORITY_SIGNKEY_1[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICWwIBAAKBgQCz0lCJ8rhLujVdzY6M6ZWp4iBAc0FxI79cff/pqp8GQAaWFZrs\n" -"vQPJ8XqMmN7GRbJ2MDVvyGYwIBtt6RJnr7txfi+JsjI42mujkZdzIEWEOIJrhaqX\n" -"n6xnZssEQpXodiXq5EOFcTrqtdHZ/2myE8y8mmczMN9Cl1qeacSlzi7cRQIDAQAB\n" -"AoGASpzUkDinIbzU0eQt5ugxEnliOnvYRpK3nzAk1JbYPyan1PSIAPz4qn1JBTeV\n" -"EB3xS7r7ITO8uvFHkFZqLZ2sH1uE6e4sAytJGO+kyqnlkiDTPEXpcGe99j8PH1yj\n" -"xUOrHRlAYWjG8NEkQi+APA+HZkswE3L/viFwR2AARoE2ac0CQQDsOLdNJa+mqn6N\n" -"1L76nEl/YgXHtKUks+beOR3IgknKEjcsJJEUHyiu0wjbXZV6gTtyQvcAePglUUD1\n" -"R2OkOOADAkEAwuCxvHEAPeQbVt8fSvxw74vqew6LITP2Utb1dQK0E26IRPF36BsJ\n" -"buO/gqMZv6ALq+/KxpA/pUsApbgog9uUFwJAYvHCvbrKX1pM1iXFtP1fv86UMzlU\n" -"bxI34t8zvXftZonIuGG8rxv6E3hr3k7NvNmCx/KKuZTyA9eMCPFVKEV2dwJACn8j\n" -"06yagLrqphE6lEVop953cM1lvRIZcHjXm8fbfzhy6pO/C6d5KJnn1NeIKYQrXMV7\n" -"vJpEc1jI3iQ/Omr3XQJAEBIt5MlP2wlrX9om7B+32XBygUssY3cw/bXybZrtSU0/\n" -"Yx4lqK0ca5IkTp3HevwnlWaJgbaOTUspCVshzJBhDA==\n" -"-----END RSA PRIVATE KEY-----\n"; - -/** Second of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_2[] = -"dir-key-certificate-version 3\n" -"fingerprint 4D44AE0470B9E88FD4558EFEC82698FB33715400\n" -"dir-key-published 2007-06-13 16:52:32\n" -"dir-key-expires 2008-06-13 16:52:32\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAqukDwQRm1Oy1pPY+7GNRnRNFJzEVPUBfJwC4tBH19tkvdRQPuIGI\n" -"2jiTy/rmZ6CLcl1G0oulSgxfKEX75QdptOasZu+rKUrRRSxx0QrXhs9a7up0rpXh\n" -"13fw3mh1Vl/As3rJYF30Hjk01BTOJMxi/HY2y0ALQytFWjiMGY74A9Y6+uDcHkB2\n" -"KflBjxIl8zpCsXsTTnUhN5kXqaOOnK46XaUShSpXsyOxTMJXuJEtgLz9XCyA8XjW\n" -"d75QLHucEnlTqxUAdI5YSN2KIlIJiySCVnAorDpJey2mE9VncpHQWMCv/FPFdnSU\n" -"EMMPUc4bBShcoNFf0mMJeV2sv+dBkgKAL0GLM19PuJIThJhfN/B6+YQTxw4HEpPV\n" -"plfUqYRN0fYC+5hCTS6rroO/uCfDR7NBtoeDNm9dQrvjfk3b/Mywah1rdWNjnVqs\n" -"tPJaz3fc/CVBOUUexhmyktgLuwSNEYIQilQ+BydkWN/4RObhV+YSV5BgekEDVaoS\n" -"RHw4IbYBDHVxAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBAOu3dgrQth3iqvi/UzfywaANw0bBUuMOBhnMBeiLEcRLneJHUJkVvrpR\n" -"/EDQkdMov1e7CX6aqBKygVnbDNYjJ+bcQej8MKpuuW+zIknnz5lfnAVZO5uAmo3Y\n" -"DpG574oQ2FFMdkWHSBloIRxSj/E4Jn1M2qJjElBXP0E33Ka/Noo7AgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"Fv0Li68QUdAiChY3OklZOakHzwXAUfCzDNxkqe+HLC0n6ZECE9ZCvLVo69XmgVhH\n" -"L5qYr2rxT6QpF+9yuOHbN9gWn8EsDcli06MlhX9TUt/IYVxHa/9tJwNoTfEw2w2D\n" -"tyHhWm94IfOK7/Sea6jHnjckl80X+kk0ZNtAGs3/6fP4iltKNGXnvBwfgLpEgW7X\n" -"NpDl0OLeDuA79zem2GogwQZQdoDbePByU0TJVx9jYi2Bzx2Nb2H0hRTPP6+dY0HQ\n" -"MHb7yyyTQRad5iAUnExKhhyt22p7X3a6lgkAhq4YrNn/zVPkpnT2dzjsOydTHOW8\n" -"2BQs33QlGNe095i47pJBDYsUgmJaXfqB/RG6dFg7jwIsc3/7dZcvcqfxY7wKcD/T\n" -"wtogCIKxDvWbZn7f0hqYkT6uQC8Zom8bcnedmyzufOZCyA2SqQ2wvio6lznR4RIB\n" -"a8qDHR0tPS9/VkqTPcvUWCZeY3UiDeWPjoK1nea1pz6DHDWglKPx86a0amjjayZQ\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_2 */ -const char AUTHORITY_SIGNKEY_2[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDrt3YK0LYd4qr4v1M38sGgDcNGwVLjDgYZzAXoixHES53iR1CZ\n" -"Fb66UfxA0JHTKL9Xuwl+mqgSsoFZ2wzWIyfm3EHo/DCqbrlvsyJJ58+ZX5wFWTub\n" -"gJqN2A6Rue+KENhRTHZFh0gZaCEcUo/xOCZ9TNqiYxJQVz9BN9ymvzaKOwIDAQAB\n" -"AoGAJ+I9/ex8tCfTSA2PdisEKiHKBeHWNYb870Z/RW6qje1BhLUOZSixwfL3XLwt\n" -"wG3nml+SZrKid69uhZaz4FPIf0tqCgURf6dDrF5vuzzr7VLVqkZHYSBp0vE6bu0R\n" -"Sgc5QNxI2talgc4bsp0O0C+Zd4n3Yto0pXl/I6NHVAxlFBECQQD2mahkY+QEHWPV\n" -"yRY3w3HhRmWBcrkY2zVyvPpqfn/sdHRPYW/yj4Xr/d1CO9VyFmEs4k324lIvu6LT\n" -"WDdpPlcJAkEA9LOZv5aNeAm8ckvvXH7iv8KiONiSz0n9wlisxMhNYTEkOCo1g7jG\n" -"AX5ZknRC9s4sWCPOBpMhloUvemdQ5FCEIwJBAMqCFwoSCf7jD8hRcUBr7QodoF/0\n" -"kVJ7OeI2lMJ9jZnlbFp/3snn2Qeam2e38SnWfQi582KKKwnt4eIDMMXpntkCQQDI\n" -"v1Lh11wl3y7nQZ6T7lCNatp08k+2mQgCWYcbRQweMRd6sD4I2xwt+372ZETPfyLo\n" -"CC+sOyYx+v+RVpMJS3irAkEA6l98nMteZKmhOgyKSjdolP+ahpZunb+WnCdAtP97\n" -"rjZyXmEZS3oe7TRCDD28GAGMmxSDvNfOOpyn14ishEs5AQ==\n" -"-----END RSA PRIVATE KEY-----\n"; - -/** Third of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_3[] = -"dir-key-certificate-version 3\n" -"fingerprint ED3719BF554DE9D7D59F5CA5A4F5AD121D020ED9\n" -"dir-key-published 2007-06-13 16:52:40\n" -"dir-key-expires 2008-06-13 16:52:40\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAtB+yw4BNxtZAG4cPaedkhWNmeij7IuNWmXjh58ZYEGurvGyHs1w4\n" -"QlwNYI2UftSIeIGdWZ5fJ17h9P3xvO6eeJuOt4KPrNOxUbSGrELEx1Lje1fDAJ1X\n" -"SvN+dvptusxtyFUr8afgTPrFIvYuazQ6q/Rw+NDagjmDx3h/A/enihpBnjwzeH8j\n" -"Xzu7b+HKnzFnNfveTDdvSy0NSC6tCOnrfXo31XbXRXtlesnMIpbJClUcAv55eyai\n" -"/PrVPCCUz8mk0sQnn2Xhv1YJmwOlQTGMfg0a0kWLmh+UWcHsGQ4VWxBZJcuzgFHG\n" -"hu2/Fz6DXSpX5Q6B9HKoGmnH1oBh24l0kUW1jL8BxPY4YDU1Lt5t3qgcDn9dXYcI\n" -"o8VvyI0ecSc26Q2PYFWX1hpN4VIBZ8uGaW3IpyTdNiRq0g3iMGRFEXcDlWuyMB9E\n" -"EbSM7m/79V/z7SjDd75EP8Z0qDPESEVB8a8LbuSJtzFVE0KHd7RzkIEN5sorXspZ\n" -"/THukftSmkIvAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBANrSZlUq38Boz3iuUOydYTJV57rTbq1bz805FP2QG2Z+2bwpgKIOZag/\n" -"gN2A1ySJaIYLgZIg9irxrLkqlY/UAjC23y6V9fJXP1S3TXoqLmHleW8PsaDLuwTo\n" -"hCWaR61Mx9WG7IXcodn2Z7RiCfZpSW4Rztbk5WtjQa5jPXSFOuBJAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"UNXZy+4OQ8iat+gw+vg2ynvKj2BYbqZt+EAZAV3rmw6gux44U9TLRECRd6LsA08N\n" -"4+Vz01TU81xqMgfrUy94ei2YvcfpO8art9/muWHTP9SmOX8S1uqDqLWA+n723C9A\n" -"HyVXn4aINncO2081gJcIW5+Ul8WTCeZe/n3LVPTCKbTdqxvmrPUdCWlJTQUmb19M\n" -"T+kcCjaEfgQGLC+Y2MHqYe/nxz+aBKqpjiWUDdjc35va6r/2e3c0jGi1B1xRZxN1\n" -"xThPZ+CifjDoWBxJdDGlIfZRK1lMnOCJY9w9ibTXQ1UnvE4whFvmB55/t9/XLq4q\n" -"3pnZz0H7funey3+ilmTxDohoAYT1GX+4a+3xYH07UmAFqlTzqKClj84XEHn+Cer7\n" -"Nun9kJlJFuBgUpQjwCkzedFZKKLOHgB2h7trJfnqcBpAM8Rup1Bb5u/RcBx9gy1q\n" -"pMc65FviIrc/Q5TUku6NNbCbnGll1599PvWuUzkG42lJ17V6psKHIsqGtVdHlCUc\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_3 */ -const char AUTHORITY_SIGNKEY_3[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDa0mZVKt/AaM94rlDsnWEyVee6026tW8/NORT9kBtmftm8KYCi\n" -"DmWoP4DdgNckiWiGC4GSIPYq8ay5KpWP1AIwtt8ulfXyVz9Ut016Ki5h5XlvD7Gg\n" -"y7sE6IQlmketTMfVhuyF3KHZ9me0Ygn2aUluEc7W5OVrY0GuYz10hTrgSQIDAQAB\n" -"AoGBAIyoeG1AnQmildKeQpiGZackf0uhg2BeRwpFKg//5Q0Sd0Wza+M/2+q1v1Ei\n" -"86ihxxV7KfPTykk6hmuUSwVkI28Z+5J9NYTr35EzPiUlqpo0iclTkFqrlbqSPULx\n" -"9fQhvcOGv1c0m5CnYrHsM8eu3tagLg+6OE4abLOYX4Az5pkxAkEA/NwHhVaVJrXH\n" -"lGDrRAfGtaD5Tzeeg1H9DNZi5lmFiSNR0O11sgDLkiZNP5oM8knyqo8Gq08hwxEb\n" -"yqMXM3XtJQJBAN2KJbFhOjDIkvJyYvbmcP6P7vV2c9j+oUTKkFMF7vvfWunxMi9j\n" -"ghbdUKgl7tU0VFpw7ufDDD0pkN6sua3gp1UCQQCvNzTK861U7p/GtMYyFQVf9JTt\n" -"jMf9jYHBNInBvwTme6AFG5bz6tMlif77dJ9GAXHzODrR2Hq3thJA/3RjR3M1AkBg\n" -"+6M4ncmtpYC+5lhwob0Bk90WU/6vFflfdhXsYoKWfNb95vsDR9qhS82Nbt25NClh\n" -"VmMfzoFDHTkwYgj/F4PpAkEA+RaaSRP7BmbvFNqvlm8J/m0RVdAH4+p/Q5Z5u6Yo\n" -"N7xC/gFi0qFPGKsDvD2CncAYmt+KNsd8S0JGDN4eieKn+Q==\n" -"-----END RSA PRIVATE KEY-----\n"; - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index bb1bf37b4..117369c56 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif |