diff options
40 files changed, 1093 insertions, 163 deletions
@@ -22,3 +22,6 @@ Frequently Asked Questions: https://www.torproject.org/faq.html https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ + +To get started working on Tor development: + See the doc/HACKING file. diff --git a/changes/bug10169 b/changes/bug10169 new file mode 100644 index 000000000..979c4e428 --- /dev/null +++ b/changes/bug10169 @@ -0,0 +1,4 @@ + o Major features: + - Also consider stream buffer sizes when calculating OOM + conditions. Rename MaxMemInCellQueues to MaxMemInQueues. Fixes + bug 10169. diff --git a/changes/bug8788 b/changes/bug8788 new file mode 100644 index 000000000..26fb5bc4f --- /dev/null +++ b/changes/bug8788 @@ -0,0 +1,3 @@ + o Minor bugfixes: + - Avoid crashing on a malformed resolv.conf file when running a + server using Libevent 1. Fixes bug 8788; bugfix on 0.1.1.23. diff --git a/changes/bug9099 b/changes/bug9099 new file mode 100644 index 000000000..5cd963ea8 --- /dev/null +++ b/changes/bug9099 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix a comment about the rend_server_descriptor_t.protocols field + to more accurately describe its range. Also, make that + field unsigned, to more accurately reflect its usage. + Fixes bug 9099; bugfix on 0.2.1.5-alpha. diff --git a/changes/bug9222 b/changes/bug9222 new file mode 100644 index 000000000..bd6627711 --- /dev/null +++ b/changes/bug9222 @@ -0,0 +1,5 @@ + o Documentation: + + - Document in the manpage that "KBytes" may also be written as + "kilobytes" or "KB", that "Kbits" may also be written as + "kilobits", and so forth. Closes ticket #9222. diff --git a/changes/bug9686 b/changes/bug9686 new file mode 100644 index 000000000..82a8f4447 --- /dev/null +++ b/changes/bug9686 @@ -0,0 +1,3 @@ + o Minor changes: + - Decrease the lower limit of MaxMemInQueues to 256 MBytes, to + appease raspberry pi users. Fixes bug 9686.
\ No newline at end of file diff --git a/changes/ticket5528 b/changes/ticket5528 new file mode 100644 index 000000000..69b2c1d16 --- /dev/null +++ b/changes/ticket5528 @@ -0,0 +1,4 @@ + o Code simplifications and refactoring: + - Get rid of router->address, since in all cases it was just the + string representation of router->addr. Resolves ticket 5528. + diff --git a/changes/ticket9176 b/changes/ticket9176 new file mode 100644 index 000000000..53c30e330 --- /dev/null +++ b/changes/ticket9176 @@ -0,0 +1,5 @@ + o Minor features: + + - Made PREDICTED_CIRCS_RELEVANCE_TIME configurable from config + file with a new option, PredictedPortsRelevanceTime. Implements + ticket #9176. Patch by unixninja92. diff --git a/doc/HACKING b/doc/HACKING index 39eafec20..3c1d0d957 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -93,7 +93,7 @@ down bugs. Jenkins ~~~~~~~ -http://jenkins.torproject.org +https://jenkins.torproject.org Dmalloc ~~~~~~~ diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4b686ca88..c49692c69 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -143,7 +143,17 @@ GENERAL OPTIONS the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be _at the very least_ 30 KBytes (that is, - 30720 bytes). (Default: 1 GByte) + 30720 bytes). (Default: 1 GByte) + + + + With this option, and in other options that take arguments in bytes, + KBytes, and so on, other formats are also supported. Notably, "KBytes" can + also be written as "kilobytes" or "kb"; "MBytes" can be written as + "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. + Tor also accepts "byte" and "bit" in the singular. + The prefixes "tera" and "T" are also recognized. + If no units are given, we default to bytes. + To avoid confusion, we recommend writing "bytes" or "bits" explicitly, + since it's easy to forget that "B" means bytes, not bits. [[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**:: Limit the maximum token bucket size (also known as the burst) to the given @@ -534,6 +544,12 @@ GENERAL OPTIONS following the Tor specification. Otherwise, they are logged with severity \'info'. (Default: 0) +[[PredictedPortsRelevanceTime]] **PredictedPortsRelevanceTime** __NUM__:: + Set how long, after the client has mad an anonymized connection to a + given port, we will try to make sure that we build circuits to + exits that support that port. The maximum value for this option is 1 + hour. (Default: 1 hour) + [[RunAsDaemon]] **RunAsDaemon** **0**|**1**:: If 1, Tor forks and daemonizes to the background. This option has no effect on Windows; instead you should use the --service command-line option. @@ -1728,13 +1744,13 @@ is non-zero): localhost, RFC1918 addresses, and so on. This can create security issues; you should probably leave it off. (Default: 0) -[[MaxMemInCellQueues]] **MaxMemInCellQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: +[[MaxMemInQueues]] **MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: This option configures a threshold above which Tor will assume that it - needs to stop queueing cells because it's about to run out of memory. - If it hits this threshold, it will begin killing circuits until it - has recovered at least 10% of this memory. Do not set this option too + needs to stop queueing or buffering data because it's about to run out of + memory. If it hits this threshold, it will begin killing circuits until + it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only - affects circuit queues, so the actual process size will be larger than + affects some queues, so the actual process size will be larger than this. (Default: 8GB) DIRECTORY SERVER OPTIONS diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4d0fff833..8525b4a72 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -626,7 +626,8 @@ tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, } #endif -#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \ + && !defined(TOR_UNIT_TESTS) void tor_gettimeofday_cached(struct timeval *tv) { @@ -659,5 +660,45 @@ tor_gettimeofday_cache_clear(void) { cached_time_hires.tv_sec = 0; } + +#ifdef TOR_UNIT_TESTS +/** For testing: force-update the cached time to a given value. */ +void +tor_gettimeofday_cache_set(const struct timeval *tv) +{ + tor_assert(tv); + memcpy(&cached_time_hires, tv, sizeof(*tv)); +} #endif +#endif + +/** + * As tor_gettimeofday_cached, but can never move backwards in time. + * + * The returned value may diverge from wall-clock time, since wall-clock time + * can trivially be adjusted backwards, and this can't. Don't mix wall-clock + * time with these values in the same calculation. + * + * Depending on implementation, this function may or may not "smooth out" huge + * jumps forward in wall-clock time. It may or may not keep its results + * advancing forward (as opposed to stalling) if the wall-clock time goes + * backwards. The current implementation does neither of of these. + * + * This function is not thread-safe; do not call it outside the main thread. + * + * In future versions of Tor, this may return a time does not have its + * origin at the Unix epoch. + */ +void +tor_gettimeofday_cached_monotonic(struct timeval *tv) +{ + struct timeval last_tv = { 0, 0 }; + + tor_gettimeofday_cached(tv); + if (timercmp(tv, &last_tv, <)) { + memcpy(tv, &last_tv, sizeof(struct timeval)); + } else { + memcpy(&last_tv, tv, sizeof(struct timeval)); + } +} diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index fda873359..f0d1828b7 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -91,6 +91,10 @@ int tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); +#ifdef TOR_UNIT_TESTS +void tor_gettimeofday_cache_set(const struct timeval *tv); +#endif +void tor_gettimeofday_cached_monotonic(struct timeval *tv); #endif diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index 8b934c443..5ac9c1230 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -3014,7 +3014,8 @@ resolv_conf_parse_line(char *const start, int flags) { if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { const char *const nameserver = NEXT_TOKEN; - evdns_nameserver_ip_add(nameserver); + if (nameserver) + evdns_nameserver_ip_add(nameserver); } else if (!strcmp(first_token, "domain") && (flags & DNS_OPTION_SEARCH)) { const char *const domain = NEXT_TOKEN; if (domain) { diff --git a/src/or/buffers.c b/src/or/buffers.c index 50016d3a8..012ced6d3 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -62,6 +62,8 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, int state, char **reason, ssize_t *drain_out); +#define DEBUG_CHUNK_ALLOC + /* Chunk manipulation functions */ /** A single chunk on a buffer or in a freelist. */ @@ -69,7 +71,12 @@ typedef struct chunk_t { struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ size_t datalen; /**< The number of bytes stored in this chunk */ size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in * this chunk. */ } chunk_t; @@ -141,6 +148,9 @@ static chunk_freelist_t freelists[] = { * could help with? */ static uint64_t n_freelist_miss = 0; +/** DOCDOC */ +static size_t total_bytes_allocated_in_chunks = 0; + static void assert_freelist_ok(chunk_freelist_t *fl); /** Return the freelist to hold chunks of size <b>alloc</b>, or NULL if @@ -174,6 +184,11 @@ chunk_free_unchecked(chunk_t *chunk) } else { if (freelist) ++freelist->n_free; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(alloc == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= alloc); + total_bytes_allocated_in_chunks -= alloc; tor_free(chunk); } } @@ -200,6 +215,10 @@ chunk_new_with_alloc_size(size_t alloc) else ++n_freelist_miss; ch = tor_malloc(alloc); +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + total_bytes_allocated_in_chunks += alloc; } ch->next = NULL; ch->datalen = 0; @@ -211,6 +230,14 @@ chunk_new_with_alloc_size(size_t alloc) static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } static INLINE chunk_t * @@ -220,7 +247,11 @@ chunk_new_with_alloc_size(size_t alloc) ch = tor_malloc(alloc); ch->next = NULL; ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; return ch; } @@ -232,11 +263,18 @@ static INLINE chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + size_t memlen_orig = chunk->memlen; tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(memlen_orig)); + chunk->DBG_alloc = CHUNK_ALLOC_SIZE(sz); +#endif + total_bytes_allocated_in_chunks += + CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig); return chunk; } @@ -261,12 +299,14 @@ preferred_chunk_size(size_t target) } /** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void + * last call to buf_shrink_freelists(). Return the amount of memory + * freed. */ +size_t buf_shrink_freelists(int free_all) { #ifdef ENABLE_BUF_FREELISTS int i; + size_t total_freed = 0; disable_control_logging(); for (i = 0; freelists[i].alloc_size; ++i) { int slack = freelists[i].slack; @@ -298,6 +338,13 @@ buf_shrink_freelists(int free_all) *chp = NULL; while (chunk) { chunk_t *next = chunk->next; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(chunk->memlen)); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -315,18 +362,21 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; + tor_assert(orig_n_to_skip == new_length); log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", + "length %d, kept %d, dropped %d. New length is %d", (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); + orig_n_to_skip, orig_n_to_free, new_length); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } done: enable_control_logging(); + return total_freed; #else (void) free_all; + return 0; #endif } @@ -376,9 +426,10 @@ struct buf_t { * * If <b>nulterminate</b> is true, ensure that there is a 0 byte in * buf->head->mem right after all the data. */ -static void +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate) { + /* XXXX nothing uses nulterminate; remove it. */ chunk_t *dest, *src; size_t capacity; if (!buf->head) @@ -450,6 +501,20 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) check(); } +#ifdef TOR_UNIT_TESTS +void +buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) +{ + if (!buf || !buf->head) { + *cp = NULL; + *sz = 0; + } else { + *cp = buf->head->data; + *sz = buf->head->datalen; + } +} +#endif + /** Resize buf so it won't hold extra memory that we haven't been * using lately. */ @@ -504,6 +569,12 @@ buf_new(void) return buf; } +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + /** Remove all data from <b>buf</b>. */ void buf_clear(buf_t *buf) @@ -531,7 +602,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } @@ -564,6 +635,10 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem; @@ -599,6 +674,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -606,6 +682,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached_monotonic(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -618,6 +698,26 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, diff --git a/src/or/buffers.h b/src/or/buffers.h index 48b118520..a201282da 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -16,17 +16,21 @@ buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); +size_t buf_shrink_freelists(int free_all); void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); @@ -100,6 +104,8 @@ void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate); +void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); #endif #endif diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index ffcca4666..98fef4c14 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -606,27 +606,30 @@ int inform_testing_reachability(void) { char dirbuf[128]; + char *address; const routerinfo_t *me = router_get_my_routerinfo(); if (!me) return 0; + address = tor_dup_ip(me->addr); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); if (me->dir_port) { tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); } log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", - me->address, me->or_port, + address, me->or_port, me->dir_port ? dirbuf : "", me->dir_port ? "are" : "is", TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60); + tor_free(address); return 1; } diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 947489636..b2eb730c8 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1435,9 +1435,9 @@ circuit_mark_all_dirty_circs_as_unusable(void) * - If circ->rend_splice is set (we are the midpoint of a joined * rendezvous stream), then mark the other circuit to close as well. */ -void -circuit_mark_for_close_(circuit_t *circ, int reason, int line, - const char *file) +MOCK_IMPL(void, +circuit_mark_for_close_, (circuit_t *circ, int reason, int line, + const char *file)) { int orig_reason = reason; /* Passed to the controller */ assert_circuit_ok(circ); @@ -1612,6 +1612,38 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at <b>stream</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit <b>c</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + /** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ STATIC size_t n_cells_in_circ_queues(const circuit_t *c) @@ -1632,7 +1664,7 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; @@ -1652,20 +1684,68 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list <b>stream</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + } + + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit <b>c</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + TO_ORIGIN_CIRCUIT((circuit_t*)c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + TO_OR_CIRCUIT((circuit_t*)c)->n_streams, now); + } +} -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in + * truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; if (age_a < age_b) return 1; @@ -1675,67 +1755,88 @@ circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); + + { + const size_t recovered = buf_shrink_freelists(1); + if (recovered >= current_allocation) { + log_warn(LD_BUG, "We somehow recovered more memory from freelists " + "than we thought we had allocated"); + current_allocation = 0; + } else { + current_allocation -= recovered; + } + } { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } + tor_gettimeofday_cached_monotonic(&now); + now_ms = (uint32_t)tv_to_msec(&now); + /* This algorithm itself assumes that you've got enough memory slack * to actually run it. */ - TOR_LIST_FOREACH(circ, &global_circuitlist, head) + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); smartlist_add(circlist, circ); - - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + } /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); /* Okay, now the worst circuits are at the front of the list. Let's mark * them, and reclaim their storage aggressively. */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { size_t n = n_cells_in_circ_queues(circ); + size_t freed; if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) + + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) break; } SMARTLIST_FOREACH_END(circ); clean_cell_pool(); /* In case this helps. */ - - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), - n_circuits_killed); + buf_shrink_freelists(1); /* This is necessary to actually release buffer + chunks. */ + + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive.", + U64_PRINTF_ARG(mem_recovered), + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed); smartlist_free(circlist); } diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index 1c8cf7de2..a29c29a49 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -53,8 +53,8 @@ origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags); void circuit_mark_all_unused_circs(void); void circuit_mark_all_dirty_circs_as_unusable(void); -void circuit_mark_for_close_(circuit_t *circ, int reason, - int line, const char *file); +MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, + int line, const char *file)); int circuit_get_cpath_len(origin_circuit_t *circ); void circuit_clear_cpath(origin_circuit_t *circ); crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); @@ -76,6 +76,9 @@ void channel_note_destroy_not_pending(channel_t *chan, circid_t id); #ifdef CIRCUITLIST_PRIVATE STATIC void circuit_free(circuit_t *circ); STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); #endif #endif diff --git a/src/or/config.c b/src/or/config.c index 5633696d0..4c292d82b 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -85,6 +85,7 @@ static config_abbrev_t option_abbrevs_[] = { { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -306,7 +307,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + V(MaxMemInQueues, MEMUNIT, "8 GB"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -317,6 +318,7 @@ static config_var_t option_vars_[] = { V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), + V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), @@ -2380,6 +2382,11 @@ compute_publishserverdescriptor(or_options_t *options) * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +/** Higest allowable value for PredictedPortsRelevanceTime; if this is + * too high, our selection of exits will decrease for an extended + * period of time to an uncomfortable level .*/ +#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) + /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) @@ -2758,10 +2765,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInCellQueues < (500 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 500 MB for now. " + if (options->MaxMemInQueues < (256 << 20)) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (500 << 20); + options->MaxMemInQueues = (256 << 20); } options->AllowInvalid_ = 0; @@ -2840,6 +2847,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } + if (options->PredictedPortsRelevanceTime > + MAX_PREDICTED_CIRCS_RELEVANCE) { + log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " + "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); + options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; + } + if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: diff --git a/src/or/control.c b/src/or/control.c index 05ff9a659..1da996992 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -3176,6 +3176,10 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } +static void lost_owning_controller(const char *owner_type, + const char *loss_manner) + ATTR_NORETURN; + /** Shut down this Tor instance in the same way that SIGINT would, but * with a log message appropriate for the loss of an owning controller. */ static void @@ -4670,6 +4674,8 @@ static char *owning_controller_process_spec = NULL; * if this Tor instance is not currently owned by a process. */ static tor_process_monitor_t *owning_controller_process_monitor = NULL; +static void owning_controller_procmon_cb(void *unused) ATTR_NORETURN; + /** Process-termination monitor callback for Tor's owning controller * process. */ static void diff --git a/src/or/directory.c b/src/or/directory.c index 739885c76..5eccb2cab 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -82,8 +82,7 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const char *address, - const tor_addr_t *addr, +static void directory_initiate_command_rend(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, @@ -464,7 +463,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, tor_addr_t addr; routerinfo_t *ri = node->ri; node_get_addr(node, &addr); - directory_initiate_command(ri->address, &addr, + directory_initiate_command(&addr, ri->or_port, 0/*no dirport*/, ri->cache_info.identity_digest, dir_purpose, @@ -593,9 +592,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - char address_buf[INET_NTOA_BUF_LEN+1]; - struct in_addr in; - const char *address; tor_addr_t addr; const int anonymized_connection = dirind_is_anon(indirection); node = node_get_by_id(status->identity_digest); @@ -605,13 +601,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, "don't have its router descriptor.", routerstatus_describe(status)); return; - } else if (node) { - node_get_address_string(node, address_buf, sizeof(address_buf)); - address = address_buf; - } else { - in.s_addr = htonl(status->addr); - tor_inet_ntoa(&in, address_buf, sizeof(address_buf)); - address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); @@ -625,7 +614,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(address, &addr, + directory_initiate_command_rend(&addr, status->or_port, status->dir_port, status->identity_digest, dir_purpose, router_purpose, @@ -867,7 +856,7 @@ directory_command_should_use_begindir(const or_options_t *options, * <b>supports_begindir</b>, and whose identity key digest is * <b>digest</b>. */ void -directory_initiate_command(const char *address, const tor_addr_t *_addr, +directory_initiate_command(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -875,7 +864,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(address, _addr, or_port, dir_port, + directory_initiate_command_rend(_addr, or_port, dir_port, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -897,7 +886,7 @@ is_sensitive_dir_purpose(uint8_t dir_purpose) /** Same as directory_initiate_command(), but accepts rendezvous data to * fetch a hidden service descriptor. */ static void -directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, +directory_initiate_command_rend(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -915,7 +904,6 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, const int anonymized_connection = dirind_is_anon(indirection); tor_addr_t addr; - tor_assert(address); tor_assert(_addr); tor_assert(or_port || dir_port); tor_assert(digest); @@ -948,7 +936,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = use_begindir ? or_port : dir_port; - conn->base_.address = tor_strdup(address); + conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; diff --git a/src/or/directory.h b/src/or/directory.h index 3de69329a..bc200797d 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -63,7 +63,7 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, +void directory_initiate_command(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index cd0e23cc2..984b47d2f 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -79,7 +79,6 @@ static int add_fingerprint_to_dir(const char *nickname, const char *fp, struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log); @@ -327,7 +326,6 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) } return dirserv_get_status_impl(d, router->nickname, - router->address, router->addr, router->or_port, router->platform, router->contact_info, msg, 1); @@ -341,7 +339,6 @@ dirserv_would_reject_router(const routerstatus_t *rs) uint32_t res; res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, - "", /* address is only used in logs */ rs->addr, rs->or_port, NULL, NULL, NULL, 0); @@ -380,7 +377,6 @@ dirserv_get_name_status(const char *id_digest, const char *nickname) */ static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log) @@ -452,14 +448,14 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad directory because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADDIR; } if (authdir_policy_badexit_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADEXIT; } @@ -467,7 +463,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_permits_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); if (msg) *msg = "Authdir is rejecting routers in this range."; return FP_REJECT; @@ -475,7 +471,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_valid_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_INVALID; } if (reject_unlisted) { @@ -524,19 +520,12 @@ dirserv_free_fingerprint_list(void) static int dirserv_router_has_valid_address(routerinfo_t *ri) { - struct in_addr iaddr; if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ - if (!tor_inet_aton(ri->address, &iaddr)) { - log_info(LD_DIRSERV,"Router %s published non-IP address '%s'. Refusing.", - router_describe(ri), - ri->address); - return -1; - } - if (is_internal_IP(ntohl(iaddr.s_addr), 0)) { + if (is_internal_IP(ri->addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address '%s'. Refusing.", - router_describe(ri), ri->address); + "Router %s published internal IP address. Refusing.", + router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } return 0; @@ -588,12 +577,10 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, } if (dirserv_router_has_valid_address(ri) < 0) { log_fn(severity, LD_DIRSERV, - "Router %s has invalid address '%s'. " - "Not adding (%s).", + "Router %s has invalid address. Not adding (%s).", router_describe(ri), - ri->address, esc_router_info(ri)); - *msg = "Rejected: Address is not an IP, or IP is a private address."; + *msg = "Rejected: Address is a private address."; return -1; } @@ -3279,7 +3266,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); + router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, router->cache_info.identity_digest); diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index d463303fc..37d5fb974 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1967,7 +1967,6 @@ get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { - char *address; const or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( @@ -1982,15 +1981,12 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - address = tor_dup_addr(&bridge->addr); - - directory_initiate_command(address, &bridge->addr, + directory_initiate_command(&bridge->addr, bridge->port, 0/*no dirport*/, bridge->identity, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, DIRIND_ONEHOP, "authority.z", NULL, 0, 0); - tor_free(address); } /** Fetching the bridge descriptor from the bridge authority returned a @@ -2108,13 +2104,11 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } else { if (tor_addr_family(&bridge->addr) == AF_INET) { ri->addr = tor_addr_to_ipv4h(&bridge->addr); - tor_free(ri->address); - ri->address = tor_dup_ip(ri->addr); ri->or_port = bridge->port; log_info(LD_DIR, "Adjusted bridge routerinfo for '%s' to match configured " "address %s:%d.", - ri->nickname, ri->address, ri->or_port); + ri->nickname, fmt_addr32(ri->addr), ri->or_port); } else if (tor_addr_family(&bridge->addr) == AF_INET6) { tor_addr_copy(&ri->ipv6_addr, &bridge->addr); ri->ipv6_orport = bridge->port; diff --git a/src/or/main.c b/src/or/main.c index b0529cde8..7294c8955 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1670,24 +1670,28 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* every 20 minutes, check and complain if necessary */ const routerinfo_t *me = router_get_my_routerinfo(); if (me && !check_whether_orport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " "its ORPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->or_port); + address, me->or_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } if (me && !check_whether_dirport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG, "Your server (%s:%d) has not managed to confirm that its " "DirPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 03fa836d4..737de92ee 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -785,7 +785,7 @@ void node_get_address_string(const node_t *node, char *buf, size_t len) { if (node->ri) { - strlcpy(buf, node->ri->address, len); + strlcpy(buf, fmt_addr32(node->ri->addr), len); } else if (node->rs) { tor_addr_t addr; tor_addr_from_ipv4h(&addr, node->rs->addr); diff --git a/src/or/or.h b/src/or/or.h index c47ae23ed..546adaa3a 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2009,7 +2009,6 @@ typedef int16_t country_t; /** Information about another onion router in the network. */ typedef struct { signed_descriptor_t cache_info; - char *address; /**< Location of OR: either a hostname or an IP address. */ char *nickname; /**< Human-readable OR name. */ uint32_t addr; /**< IPv4 address of OR, in host order. */ @@ -2816,6 +2815,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ struct create_cell_t *n_chan_create_cell; @@ -3469,9 +3471,8 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + uint64_t MaxMemInQueues; /**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ /** @name port booleans * @@ -3637,6 +3638,10 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ + int PredictedPortsRelevanceTime; /** How long after we've requested a + * connection for a given port, do we want + * to continue to pick exits that support + * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -4836,9 +4841,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes; diff --git a/src/or/policies.c b/src/or/policies.c index 469a203e2..42dc46b7f 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -438,7 +438,7 @@ validate_addr_policies(const or_options_t *options, char **msg) if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, options->IPv6Exit, - options->ExitPolicyRejectPrivate, NULL, + options->ExitPolicyRejectPrivate, 0, !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); @@ -970,7 +970,7 @@ exit_policy_remove_redundancies(smartlist_t *dest) int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6_exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy) { if (!ipv6_exit) { @@ -980,7 +980,7 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, append_exit_policy_string(dest, "reject private:*"); if (local_address) { char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject %s:*", local_address); + tor_snprintf(buf, sizeof(buf), "reject %s:*", fmt_addr32(local_address)); append_exit_policy_string(dest, buf); } } diff --git a/src/or/policies.h b/src/or/policies.h index facbbb6b5..91ac42749 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -45,7 +45,7 @@ addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, diff --git a/src/or/relay.c b/src/or/relay.c index dc234c1f2..d6742d25e 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2153,7 +2153,8 @@ cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, (void)circ; (void)exitward; (void)use_stats; - tor_gettimeofday_cached(&now); + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); cell_queue_append(queue, copy); @@ -2201,13 +2202,21 @@ packed_cell_mem_cost(void) return sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD; } +/** DOCDOC */ +STATIC size_t +cell_queues_get_total_allocation(void) +{ + return total_cells_allocated * packed_cell_mem_cost(); +} + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ -static int +STATIC int cell_queues_check_size(void) { - size_t alloc = total_cells_allocated * packed_cell_mem_cost(); - if (alloc >= get_options()->MaxMemInCellQueues) { + size_t alloc = cell_queues_get_total_allocation(); + alloc += buf_get_total_allocation(); + if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; } diff --git a/src/or/relay.h b/src/or/relay.h index 20eecfb40..2c7d0d8ae 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -85,6 +85,8 @@ STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); STATIC packed_cell_t *packed_cell_new(void); STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); +STATIC size_t cell_queues_get_total_allocation(void); +STATIC int cell_queues_check_size(void); #endif #endif diff --git a/src/or/rephist.c b/src/or/rephist.c index 66dc5f611..16e16ab65 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1862,22 +1862,20 @@ rep_hist_note_used_port(time_t now, uint16_t port) add_predicted_port(now, port); } -/** For this long after we've seen a request for a given port, assume that - * we'll want to make connections to the same port in the future. */ -#define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) - /** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { + int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { - if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + if (pp->time + predicted_circs_relevance_time < now) { log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); rephist_total_alloc -= sizeof(predicted_port_t); @@ -1944,14 +1942,17 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; predicted_internal_uptime_time = now; predicted_internal_capacity_time = now; } - if (predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) + if (predicted_internal_time + predicted_circs_relevance_time < now) return 0; /* too long ago */ - if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) *need_uptime = 1; // Always predict that we need capacity. *need_capacity = 1; @@ -1963,8 +1964,11 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + return smartlist_len(predicted_ports_list) || - predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now; + predicted_internal_time + predicted_circs_relevance_time >= now; } /** Return 1 if we have no need for circuits currently, else return 0. */ diff --git a/src/or/router.c b/src/or/router.c index 6fa9f65e1..4828a8df6 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1177,7 +1177,7 @@ consider_testing_reachability(int test_or, int test_dir) /* XXX IPv6 self testing */ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", - me->address, me->or_port); + fmt_addr32(me->addr), me->or_port); circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); extend_info_free(ei); @@ -1189,7 +1189,7 @@ consider_testing_reachability(int test_or, int test_dir) CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(me->address, &addr, + directory_initiate_command(&addr, me->or_port, me->dir_port, me->cache_info.identity_digest, DIR_PURPOSE_FETCH_SERVERDESC, @@ -1204,6 +1204,7 @@ router_orport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", get_options()->PublishServerDescriptor_ != NO_DIRINFO ? @@ -1212,7 +1213,8 @@ router_orport_found_reachable(void) mark_my_descriptor_dirty("ORPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } } @@ -1222,6 +1224,7 @@ router_dirport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; @@ -1229,7 +1232,8 @@ router_dirport_found_reachable(void) mark_my_descriptor_dirty("DirPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1800,7 +1804,6 @@ router_rebuild_descriptor(int force) ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; - ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; ri->or_port = router_get_advertised_or_port(options); @@ -1865,7 +1868,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->IPv6Exit, options->ExitPolicyRejectPrivate, - ri->address, !options->BridgeRelay); + ri->addr, !options->BridgeRelay); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -2271,8 +2274,7 @@ char * router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key) { - /* XXXX025 Make this look entirely at its arguments, and not at globals. - */ + char *address = NULL; char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ char digest[DIGEST_LEN]; @@ -2346,7 +2348,9 @@ router_dump_router_to_string(routerinfo_t *router, } } + address = tor_dup_ip(router->addr); chunks = smartlist_new(); + /* Generate the easy portion of the router descriptor. */ smartlist_add_asprintf(chunks, "router %s %s %d 0 %d\n" @@ -2362,7 +2366,7 @@ router_dump_router_to_string(routerinfo_t *router, "signing-key\n%s" "%s%s%s%s", router->nickname, - router->address, + address, router->or_port, decide_to_advertise_dirport(options, router->dir_port), extra_or_address ? extra_or_address : "", @@ -2476,6 +2480,7 @@ router_dump_router_to_string(routerinfo_t *router, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); } + tor_free(address); tor_free(family_line); tor_free(onion_pkey); tor_free(identity_pkey); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index d636a1e25..49ea752ba 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -2709,7 +2709,6 @@ routerinfo_free(routerinfo_t *router) return; tor_free(router->cache_info.signed_descriptor_body); - tor_free(router->address); tor_free(router->nickname); tor_free(router->platform); tor_free(router->contact_info); @@ -4720,7 +4719,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) } /* If any key fields differ, they're different. */ - if (strcasecmp(r1->address, r2->address) || + if (r1->addr != r2->addr || strcasecmp(r1->nickname, r2->nickname) || r1->or_port != r2->or_port || !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 9414a533b..ad3cf3b38 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1195,8 +1195,7 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR,"Router nickname is invalid"); goto err; } - router->address = tor_strdup(tok->args[1]); - if (!tor_inet_aton(router->address, &in)) { + if (!tor_inet_aton(tok->args[1], &in)) { log_warn(LD_DIR,"Router address is not an IP address."); goto err; } diff --git a/src/test/include.am b/src/test/include.am index e7aebac38..c6743a19b 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -32,6 +32,7 @@ src_test_test_SOURCES = \ src/test/test_introduce.c \ src/test/test_logging.c \ src/test/test_microdesc.c \ + src/test/test_oom.c \ src/test/test_options.c \ src/test/test_pt.c \ src/test/test_replay.c \ diff --git a/src/test/test.c b/src/test/test.c index 2529e2902..456dde141 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -638,7 +638,7 @@ test_policy_summary_helper(const char *policy_str, line.value = (char *)policy_str; line.next = NULL; - r = policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1); + r = policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1); test_eq(r, 0); summary = policy_summarize(policy, AF_INET); @@ -695,7 +695,7 @@ test_policies(void) test_assert(ADDR_POLICY_REJECTED == compare_tor_addr_to_addr_policy(&tar, 2, policy)); - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, NULL, 1)); + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, 0, 1)); test_assert(policy2); policy3 = smartlist_new(); @@ -782,7 +782,7 @@ test_policies(void) line.key = (char*)"foo"; line.value = (char*)"accept *:80,reject private:*,reject *:*"; line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1)); + test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1)); test_assert(policy); //test_streq(policy->string, "accept *:80"); //test_streq(policy->next->string, "reject *:*"); @@ -1630,6 +1630,7 @@ extern struct testcase_t backtrace_tests[]; extern struct testcase_t hs_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t routerkeys_tests[]; +extern struct testcase_t oom_tests[]; static struct testgroup_t testgroups[] = { { "", test_array }, @@ -1656,6 +1657,7 @@ static struct testgroup_t testgroups[] = { { "hs/", hs_tests }, { "nodelist/", nodelist_tests }, { "routerkeys/", routerkeys_tests }, + { "oom/", oom_tests }, END_OF_GROUPS }; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index a009faa0b..6dd771593 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -193,7 +193,120 @@ test_buffers_basic(void *arg) buf_free(buf); if (buf2) buf_free(buf2); + buf_shrink_freelists(1); } + +static void +test_buffer_pullup(void *arg) +{ + buf_t *buf; + char *stuff, *tmp; + const char *cp; + size_t sz; + (void)arg; + stuff = tor_malloc(16384); + tmp = tor_malloc(16384); + + /* Note: this test doesn't check the nulterminate argument to buf_pullup, + since nothing actually uses it. We should remove it some time. */ + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + + tt_assert(buf); + tt_int_op(buf_get_default_chunk_size(buf), ==, 4096); + + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* There are a bunch of cases for pullup. One is the trivial case. Let's + mess around with an empty buffer. */ + buf_pullup(buf, 16, 1); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, ==, NULL); + tt_ptr_op(sz, ==, 0); + + /* Let's make sure nothing got allocated */ + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Case 1: everything puts into the first chunk with some moving. */ + + /* Let's add some data. */ + crypto_rand(stuff, 16384); + write_to_buf(stuff, 3000, buf); + write_to_buf(stuff+3000, 3000, buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + /* Make room for 3000 bytes in the first chunk, so that the pullup-move code + * can get tested. */ + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 3000); + test_memeq(tmp, stuff, 3000); + buf_pullup(buf, 2048, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 2048); + test_memeq(cp, stuff+3000, 2048); + tt_int_op(3000, ==, buf_datalen(buf)); + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 0); + test_memeq(tmp, stuff+3000, 2048); + + buf_free(buf); + + /* Now try the large-chunk case. */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + write_to_buf(stuff+8000, 4000, buf); + write_to_buf(stuff+12000, 4000, buf); + tt_int_op(buf_datalen(buf), ==, 16000); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + buf_pullup(buf, 12500, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 12500); + test_memeq(cp, stuff, 12500); + tt_int_op(buf_datalen(buf), ==, 16000); + + fetch_from_buf(tmp, 12400, buf); + test_memeq(tmp, stuff, 12400); + tt_int_op(buf_datalen(buf), ==, 3600); + fetch_from_buf(tmp, 3500, buf); + test_memeq(tmp, stuff+12400, 3500); + fetch_from_buf(tmp, 100, buf); + test_memeq(tmp, stuff+15900, 10); + + buf_free(buf); + + /* Make sure that the pull-up-whole-buffer case works */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */ + buf_pullup(buf, 16000, 0); /* Way too much. */ + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, ==, 7900); + test_memeq(cp, stuff+100, 7900); + + buf_free(buf); + buf = NULL; + + buf_shrink_freelists(1); + + tt_int_op(buf_get_total_allocation(), ==, 0); + done: + buf_free(buf); + buf_shrink_freelists(1); + tor_free(stuff); + tor_free(tmp); +} + static void test_buffer_copy(void *arg) { @@ -257,6 +370,7 @@ test_buffer_copy(void *arg) generic_buffer_free(buf); if (buf2) generic_buffer_free(buf2); + buf_shrink_freelists(1); } static void @@ -331,12 +445,156 @@ test_buffer_ext_or_cmd(void *arg) ext_or_cmd_free(cmd); generic_buffer_free(buf); tor_free(tmp); + buf_shrink_freelists(1); +} + +static void +test_buffer_allocation_tracking(void *arg) +{ + char *junk = tor_malloc(16384); + buf_t *buf1 = NULL, *buf2 = NULL; + int i; + + (void)arg; + + crypto_rand(junk, 16384); + tt_int_op(buf_get_total_allocation(), ==, 0); + + buf1 = buf_new(); + tt_assert(buf1); + buf2 = buf_new(); + tt_assert(buf2); + + tt_int_op(buf_allocation(buf1), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); + fetch_from_buf(junk, 100, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); /* still 4 4k chunks */ + + tt_int_op(buf_get_total_allocation(), ==, 16384); + + fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */ + tt_int_op(buf_allocation(buf1), ==, 3*4096); /* now 3 4k chunks */ + + tt_int_op(buf_get_total_allocation(), ==, 16384); /* that chunk went onto + the freelist. */ + + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 4096); /* another 4k chunk. */ + tt_int_op(buf_get_total_allocation(), ==, 16384); /* that chunk came from + the freelist. */ + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 8192); /* another 4k chunk. */ + tt_int_op(buf_get_total_allocation(), ==, 5*4096); /* that chunk was new. */ + + /* Make a really huge buffer */ + for (i = 0; i < 1000; ++i) { + write_to_buf(junk, 4000, buf2); + } + tt_int_op(buf_allocation(buf2), >=, 4008000); + tt_int_op(buf_get_total_allocation(), >=, 4008000); + buf_free(buf2); + buf2 = NULL; + + tt_int_op(buf_get_total_allocation(), <, 4008000); + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, buf_allocation(buf1)); + buf_free(buf1); + buf1 = NULL; + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, 0); + + done: + buf_free(buf1); + buf_free(buf2); + buf_shrink_freelists(1); +} + +static void +test_buffer_time_tracking(void *arg) +{ + buf_t *buf=NULL, *buf2=NULL; + struct timeval tv0; + const time_t START = 1389288246; + const uint32_t START_MSEC = (uint32_t) ((uint64_t)START * 1000); + int i; + char tmp[4096]; + (void)arg; + + crypto_rand(tmp, sizeof(tmp)); + + tv0.tv_sec = START; + tv0.tv_usec = 0; + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + tt_assert(buf); + + /* Empty buffer means the timestamp is 0. */ + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC)); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + tor_gettimeofday_cache_set(&tv0); + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(1000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + buf2 = buf_copy(buf); + tt_assert(buf2); + tt_int_op(1234, ==, buf_get_oldest_chunk_timestamp(buf2, START_MSEC+1234)); + + /* Now add more bytes; enough to overflow the first chunk. */ + tv0.tv_usec += 123 * 1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4207, ==, buf_datalen(buf)); + + /* The oldest bytes are still in the front. */ + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* Once those bytes are dropped, the chunk is still on the first + * timestamp. */ + fetch_from_buf(tmp, 100, buf); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* But once we discard the whole first chunk, we get the data in the second + * chunk. */ + fetch_from_buf(tmp, 4000, buf); + tt_int_op(107, ==, buf_datalen(buf)); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + + /* This time we'll be grabbing a chunk from the freelist, and making sure + its time gets updated */ + tv0.tv_sec += 5; + tv0.tv_usec = 617*1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4307, ==, buf_datalen(buf)); + + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + fetch_from_buf(tmp, 4000, buf); + fetch_from_buf(tmp, 306, buf); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617)); + tt_int_op(383, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000)); + + done: + buf_free(buf); + buf_free(buf2); } struct testcase_t buffer_tests[] = { - { "basic", test_buffers_basic, 0, NULL, NULL }, - { "copy", test_buffer_copy, 0, NULL, NULL }, - { "ext_or_cmd", test_buffer_ext_or_cmd, 0, NULL, NULL }, + { "basic", test_buffers_basic, TT_FORK, NULL, NULL }, + { "copy", test_buffer_copy, TT_FORK, NULL, NULL }, + { "pullup", test_buffer_pullup, TT_FORK, NULL, NULL }, + { "ext_or_cmd", test_buffer_ext_or_cmd, TT_FORK, NULL, NULL }, + { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK, + NULL, NULL }, + { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index dcfe98dd1..9e01bdbd4 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -97,7 +97,6 @@ test_dir_formats(void) get_platform_str(platform, sizeof(platform)); r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); r1->addr = 0xc0a80001u; /* 192.168.0.1 */ r1->cache_info.published_on = 0; r1->or_port = 9000; @@ -124,7 +123,6 @@ test_dir_formats(void) ex2->maskbits = 8; ex2->prt_min = ex2->prt_max = 24; r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); r2->addr = 0x0a030201u; /* 10.3.2.1 */ r2->platform = tor_strdup(platform); r2->cache_info.published_on = 5; @@ -153,7 +151,7 @@ test_dir_formats(void) tor_free(options->ContactInfo); test_assert(buf); - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n" "or-address [1:2:3:4::]:9999\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); @@ -187,7 +185,7 @@ test_dir_formats(void) cp = buf; rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp1); - test_streq(rp1->address, r1->address); + test_eq(rp1->addr, r1->addr); test_eq(rp1->or_port, r1->or_port); //test_eq(rp1->dir_port, r1->dir_port); test_eq(rp1->bandwidthrate, r1->bandwidthrate); @@ -198,7 +196,7 @@ test_dir_formats(void) //test_assert(rp1->exit_policy == NULL); strlcpy(buf2, - "router Fred 1.1.1.1 9005 0 0\n" + "router Fred 10.3.2.1 9005 0 0\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); strlcat(buf2, "\n" @@ -231,7 +229,7 @@ test_dir_formats(void) cp = buf; rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp2); - test_streq(rp2->address, r2->address); + test_eq(rp2->addr, r2->addr); test_eq(rp2->or_port, r2->or_port); test_eq(rp2->dir_port, r2->dir_port); test_eq(rp2->bandwidthrate, r2->bandwidthrate); diff --git a/src/test/test_oom.c b/src/test/test_oom.c new file mode 100644 index 000000000..cc6e53235 --- /dev/null +++ b/src/test/test_oom.c @@ -0,0 +1,348 @@ +/* Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Unit tests for OOM handling logic */ + +#define RELAY_PRIVATE +#define BUFFERS_PRIVATE +#define CIRCUITLIST_PRIVATE +#include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "compat_libevent.h" +#include "connection.h" +#include "config.h" +#include "mempool.h" +#include "relay.h" +#include "test.h" + +/* small replacement mock for circuit_mark_for_close_ to avoid doing all + * the other bookkeeping that comes with marking circuits. */ +static void +circuit_mark_for_close_dummy_(circuit_t *circ, int reason, int line, + const char *file) +{ + (void) reason; + if (circ->marked_for_close) { + TT_FAIL(("Circuit already marked for close at %s:%d, but we are marking " + "it again at %s:%d", + circ->marked_for_close_file, (int)circ->marked_for_close, + file, line)); + } + + circ->marked_for_close = line; + circ->marked_for_close_file = file; +} + +static circuit_t * +dummy_or_circuit_new(int n_p_cells, int n_n_cells) +{ + or_circuit_t *circ = or_circuit_new(0, NULL); + int i; + cell_t cell; + + for (i=0; i < n_p_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), &circ->p_chan_cells, + 0, &cell, 1, 0); + } + + for (i=0; i < n_n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_OR; + return TO_CIRCUIT(circ); +} + +static circuit_t * +dummy_origin_circuit_new(int n_cells) +{ + origin_circuit_t *circ = origin_circuit_new(); + int i; + cell_t cell; + + for (i=0; i < n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + return TO_CIRCUIT(circ); +} + +static void +add_bytes_to_buf(generic_buffer_t *buf, size_t n_bytes) +{ + char b[3000]; + + while (n_bytes) { + size_t this_add = n_bytes > sizeof(buf) ? sizeof(buf) : n_bytes; + crypto_rand(b, sizeof(b)); + generic_buffer_add(buf, b, this_add); + n_bytes -= this_add; + } +} + +static edge_connection_t * +dummy_edge_conn_new(circuit_t *circ, + int type, size_t in_bytes, size_t out_bytes) +{ + edge_connection_t *conn; + + if (type == CONN_TYPE_EXIT) + conn = edge_connection_new(type, AF_INET); + else + conn = ENTRY_TO_EDGE_CONN(entry_connection_new(type, AF_INET)); + + /* We add these bytes directly to the buffers, to avoid all the + * edge connection read/write machinery. */ + add_bytes_to_buf(TO_CONN(conn)->inbuf, in_bytes); + add_bytes_to_buf(TO_CONN(conn)->outbuf, out_bytes); + + conn->on_circuit = circ; + if (type == CONN_TYPE_EXIT) { + or_circuit_t *oc = TO_OR_CIRCUIT(circ); + conn->next_stream = oc->n_streams; + oc->n_streams = conn; + } else { + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + conn->next_stream = oc->p_streams; + oc->p_streams = conn; + } + + return conn; +} + +/** Run unit tests for buffers.c */ +static void +test_oom_circbuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL; + struct timeval tv = { 1389631048, 0 }; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + init_cell_pool(); + + /* Far too low for real life. */ + options->MaxMemInQueues = 256*packed_cell_mem_cost(); + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Now we're going to fake up some circuits and get them added to the global + circuit list. */ + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + c1 = dummy_origin_circuit_new(30); + tv.tv_usec = 10*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_or_circuit_new(20, 20); + + tt_int_op(packed_cell_mem_cost(), ==, + sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 70); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + + tv.tv_usec = 20*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(100, 85); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 255); + + tv.tv_usec = 30*1000; + tor_gettimeofday_cache_set(&tv); + /* Adding this cell will trigger our OOM handler. */ + c4 = dummy_or_circuit_new(2, 0); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 257); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + circuit_free(c1); + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); /* go back in time */ + c1 = dummy_or_circuit_new(90, 0); + + tv.tv_usec = 40*1000; /* go back to the future */ + tor_gettimeofday_cache_set(&tv); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + + UNMOCK(circuit_mark_for_close_); +} + +/** Run unit tests for buffers.c */ +static void +test_oom_streambuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL; + struct timeval tv = { 1389641159, 0 }; + uint32_t tvms; + int i; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + init_cell_pool(); + + /* Far too low for real life. */ + options->MaxMemInQueues = 81*packed_cell_mem_cost() + 4096 * 34; + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Start all circuits with a bit of data queued in cells */ + tv.tv_usec = 500*1000; /* go halfway into the second. */ + tor_gettimeofday_cache_set(&tv); + c1 = dummy_or_circuit_new(10,10); + tv.tv_usec = 510*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_origin_circuit_new(20); + tv.tv_usec = 520*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(20,20); + tv.tv_usec = 530*1000; + tor_gettimeofday_cache_set(&tv); + c4 = dummy_or_circuit_new(0,0); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + + tv.tv_usec = 600*1000; + tor_gettimeofday_cache_set(&tv); + + /* Add some connections to c1...c4. */ + for (i = 0; i < 4; ++i) { + edge_connection_t *ec; + /* link it to a circuit */ + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c1, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c2, CONN_TYPE_AP, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); /* Yes, 4 twice*/ + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + + tv.tv_sec += 1; + tv.tv_usec = 0; + tvms = (uint32_t) tv_to_msec(&tv); + + tt_int_op(circuit_max_queued_cell_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_cell_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_cell_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_cell_age(c4, tvms), ==, 0); + + tt_int_op(circuit_max_queued_data_age(c1, tvms), ==, 390); + tt_int_op(circuit_max_queued_data_age(c2, tvms), ==, 380); + tt_int_op(circuit_max_queued_data_age(c3, tvms), ==, 0); + tt_int_op(circuit_max_queued_data_age(c4, tvms), ==, 370); + + tt_int_op(circuit_max_queued_item_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_item_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_item_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 370); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + tt_int_op(buf_get_total_allocation(), ==, 4096*16*2); + + /* Now give c4 a very old buffer of modest size */ + { + edge_connection_t *ec; + tv.tv_sec -= 1; + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 1000); + + tt_int_op(cell_queues_check_size(), ==, 0); + + /* And run over the limit. */ + tv.tv_usec = 800*1000; + tor_gettimeofday_cache_set(&tv); + c5 = dummy_or_circuit_new(0,5); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + /* C4 should have died. */ + tt_assert(! c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(c4->marked_for_close); + tt_assert(! c5->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*8*2); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + circuit_free(c5); + + UNMOCK(circuit_mark_for_close_); +} + +struct testcase_t oom_tests[] = { + { "circbuf", test_oom_circbuf, TT_FORK, NULL, NULL }, + { "streambuf", test_oom_streambuf, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + |