diff options
Diffstat (limited to 'src/or')
94 files changed, 9519 insertions, 6695 deletions
diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 79e4b7c5e..998770a3d 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -45,7 +45,7 @@ typedef struct { char *new_address; time_t expires; - ENUM_BF(addressmap_entry_source_t) source:3; + addressmap_entry_source_bitfield_t source:3; unsigned src_wildcard:1; unsigned dst_wildcard:1; short num_resolve_failures; @@ -798,7 +798,7 @@ address_is_in_virtual_range(const char *address) /** Return a random address conforming to the virtual address configuration * in <b>conf</b>. */ -/* private */ void +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out) { uint8_t tmp[4]; diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 40210ee99..417832b31 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -7,6 +7,8 @@ #ifndef TOR_ADDRESSMAP_H #define TOR_ADDRESSMAP_H +#include "testsupport.h" + void addressmap_init(void); void addressmap_clear_excluded_trackexithosts(const or_options_t *options); void addressmap_clear_invalid_automaps(const or_options_t *options); @@ -52,8 +54,8 @@ typedef struct virtual_addr_conf_t { maskbits_t bits; } virtual_addr_conf_t; -void get_random_virtual_addr(const virtual_addr_conf_t *conf, - tor_addr_t *addr_out); +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, + tor_addr_t *addr_out); #endif #endif diff --git a/src/or/buffers.c b/src/or/buffers.c index c4c847ec8..033f86288 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -19,6 +19,7 @@ #include "connection_or.h" #include "control.h" #include "reasons.h" +#include "ext_orport.h" #include "../common/util.h" #include "../common/torlog.h" #ifdef HAVE_UNISTD_H @@ -63,16 +64,6 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, /* Chunk manipulation functions */ -/** A single chunk on a buffer or in a freelist. */ -typedef struct chunk_t { - struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ - size_t datalen; /**< The number of bytes stored in this chunk */ - size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ - char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ - char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in - * this chunk. */ -} chunk_t; - #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0]) /** Return the number of bytes needed to allocate a chunk to hold @@ -109,6 +100,9 @@ chunk_repack(chunk_t *chunk) chunk->data = &chunk->mem[0]; } +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; + #if defined(ENABLE_BUF_FREELISTS) || defined(RUNNING_DOXYGEN) /** A freelist of chunks. */ typedef struct chunk_freelist_t { @@ -173,6 +167,11 @@ chunk_free_unchecked(chunk_t *chunk) } else { if (freelist) ++freelist->n_free; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(alloc == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= alloc); + total_bytes_allocated_in_chunks -= alloc; tor_free(chunk); } } @@ -199,6 +198,10 @@ chunk_new_with_alloc_size(size_t alloc) else ++n_freelist_miss; ch = tor_malloc(alloc); +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + total_bytes_allocated_in_chunks += alloc; } ch->next = NULL; ch->datalen = 0; @@ -210,6 +213,14 @@ chunk_new_with_alloc_size(size_t alloc) static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } static INLINE chunk_t * @@ -219,7 +230,11 @@ chunk_new_with_alloc_size(size_t alloc) ch = tor_malloc(alloc); ch->next = NULL; ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; return ch; } @@ -231,11 +246,18 @@ static INLINE chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + size_t memlen_orig = chunk->memlen; tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(memlen_orig)); + chunk->DBG_alloc = CHUNK_ALLOC_SIZE(sz); +#endif + total_bytes_allocated_in_chunks += + CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig); return chunk; } @@ -260,12 +282,14 @@ preferred_chunk_size(size_t target) } /** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void + * last call to buf_shrink_freelists(). Return the amount of memory + * freed. */ +size_t buf_shrink_freelists(int free_all) { #ifdef ENABLE_BUF_FREELISTS int i; + size_t total_freed = 0; disable_control_logging(); for (i = 0; freelists[i].alloc_size; ++i) { int slack = freelists[i].slack; @@ -281,7 +305,7 @@ buf_shrink_freelists(int free_all) chunk_t **chp = &freelists[i].head; chunk_t *chunk; while (n_to_skip) { - if (! (*chp)->next) { + if (!(*chp) || ! (*chp)->next) { log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " "%d-byte chunks, but only found %d. (Length %d)", orig_n_to_skip, (int)freelists[i].alloc_size, @@ -297,6 +321,13 @@ buf_shrink_freelists(int free_all) *chp = NULL; while (chunk) { chunk_t *next = chunk->next; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(chunk->memlen)); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -314,18 +345,21 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; + tor_assert(orig_n_to_skip == new_length); log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", + "length %d, kept %d, dropped %d. New length is %d", (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); + orig_n_to_skip, orig_n_to_free, new_length); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } done: enable_control_logging(); + return total_freed; #else (void) free_all; + return 0; #endif } @@ -356,28 +390,16 @@ buf_dump_freelist_sizes(int severity) #endif } -/** Magic value for buf_t.magic, to catch pointer errors. */ -#define BUFFER_MAGIC 0xB0FFF312u -/** A resizeable buffer, optimized for reading and writing. */ -struct buf_t { - uint32_t magic; /**< Magic cookie for debugging: Must be set to - * BUFFER_MAGIC. */ - size_t datalen; /**< How many bytes is this buffer holding right now? */ - size_t default_chunk_size; /**< Don't allocate any chunks smaller than - * this for this buffer. */ - chunk_t *head; /**< First chunk in the list, or NULL for none. */ - chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ -}; - /** Collapse data from the first N chunks from <b>buf</b> into buf->head, * growing it as necessary, until buf->head has the first <b>bytes</b> bytes * of data from the buffer, or until buf->head has all the data in <b>buf</b>. * * If <b>nulterminate</b> is true, ensure that there is a 0 byte in * buf->head->mem right after all the data. */ -static void +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate) { + /* XXXX nothing uses nulterminate; remove it. */ chunk_t *dest, *src; size_t capacity; if (!buf->head) @@ -449,6 +471,20 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) check(); } +#ifdef TOR_UNIT_TESTS +void +buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) +{ + if (!buf || !buf->head) { + *cp = NULL; + *sz = 0; + } else { + *cp = buf->head->data; + *sz = buf->head->datalen; + } +} +#endif + /** Resize buf so it won't hold extra memory that we haven't been * using lately. */ @@ -503,6 +539,12 @@ buf_new(void) return buf; } +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + /** Remove all data from <b>buf</b>. */ void buf_clear(buf_t *buf) @@ -530,7 +572,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } @@ -563,6 +605,10 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem; @@ -598,6 +644,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -605,6 +652,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached_monotonic(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -617,6 +668,26 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, @@ -1294,7 +1365,7 @@ buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) /** Return the first position in <b>buf</b> at which the <b>n</b>-character * string <b>s</b> occurs, or -1 if it does not occur. */ -/*private*/ int +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n) { buf_pos_t pos; @@ -1702,6 +1773,64 @@ fetch_from_evbuffer_socks(struct evbuffer *buf, socks_request_t *req, } #endif +/** The size of the header of an Extended ORPort message: 2 bytes for + * COMMAND, 2 bytes for BODYLEN */ +#define EXT_OR_CMD_HEADER_SIZE 4 + +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + + check(); + if (buf->datalen < EXT_OR_CMD_HEADER_SIZE) + return 0; + peek_from_buf(hdr, sizeof(hdr), buf); + len = ntohs(get_uint16(hdr+2)); + if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE); + fetch_from_buf((*out)->body, len, buf); + return 1; +} + +#ifdef USE_BUFFEREVENTS +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + size_t buf_len = evbuffer_get_length(buf); + + if (buf_len < EXT_OR_CMD_HEADER_SIZE) + return 0; + evbuffer_copyout(buf, hdr, EXT_OR_CMD_HEADER_SIZE); + len = ntohs(get_uint16(hdr+2)); + if (buf_len < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + evbuffer_drain(buf, EXT_OR_CMD_HEADER_SIZE); + evbuffer_remove(buf, (*out)->body, len); + return 1; +} +#endif + /** Implementation helper to implement fetch_from_*_socks. Instead of looking * at a buffer's contents, we look at the <b>datalen</b> bytes of data in * <b>data</b>. Instead of removing data from the buffer, we set @@ -2348,6 +2477,7 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, char *next; size_t old_avail, avail; int over = 0; + do { int need_new_chunk = 0; if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { diff --git a/src/or/buffers.h b/src/or/buffers.h index c947f0ba9..c90e14750 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -12,19 +12,25 @@ #ifndef TOR_BUFFERS_H #define TOR_BUFFERS_H +#include "testsupport.h" + buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); +size_t buf_shrink_freelists(int free_all); void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); @@ -51,6 +57,8 @@ int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); +int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out); + #ifdef USE_BUFFEREVENTS int fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out, int linkproto); @@ -66,6 +74,8 @@ int peek_evbuffer_has_control0_command(struct evbuffer *buf); int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, const char *data, size_t data_len, int done); +int fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, + ext_or_cmd_t **out); #endif #ifdef USE_BUFFEREVENTS @@ -75,6 +85,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) evbuffer_remove((b),(buf),(buflen)) #define generic_buffer_clear(b) evbuffer_drain((b), evbuffer_get_length((b))) #define generic_buffer_free(b) evbuffer_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_evbuffer((b), (out)) #else #define generic_buffer_new() buf_new() #define generic_buffer_len(b) buf_datalen((b)) @@ -82,6 +94,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) fetch_from_buf((buf),(buflen),(b)) #define generic_buffer_clear(b) buf_clear((b)) #define generic_buffer_free(b) buf_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_buf((b), (out)) #endif int generic_buffer_set_to_copy(generic_buffer_t **output, const generic_buffer_t *input); @@ -89,7 +103,38 @@ int generic_buffer_set_to_copy(generic_buffer_t **output, void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate); +void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); + +#define DEBUG_CHUNK_ALLOC +/** A single chunk on a buffer or in a freelist. */ +typedef struct chunk_t { + struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ + size_t datalen; /**< The number of bytes stored in this chunk */ + size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif + char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ + char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in + * this chunk. */ +} chunk_t; + +/** Magic value for buf_t.magic, to catch pointer errors. */ +#define BUFFER_MAGIC 0xB0FFF312u +/** A resizeable buffer, optimized for reading and writing. */ +struct buf_t { + uint32_t magic; /**< Magic cookie for debugging: Must be set to + * BUFFER_MAGIC. */ + size_t datalen; /**< How many bytes is this buffer holding right now? */ + size_t default_chunk_size; /**< Don't allocate any chunks smaller than + * this for this buffer. */ + chunk_t *head; /**< First chunk in the list, or NULL for none. */ + chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ +}; #endif #endif diff --git a/src/or/channel.c b/src/or/channel.c index 1270eace7..63af2f91c 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -95,12 +95,7 @@ typedef struct channel_idmap_entry_s { static INLINE unsigned channel_idmap_hash(const channel_idmap_entry_t *ent) { - const unsigned *a = (const unsigned *)ent->digest; -#if SIZEOF_INT == 4 - return a[0] ^ a[1] ^ a[2] ^ a[3] ^ a[4]; -#elif SIZEOF_INT == 8 - return a[0] ^ a[1]; -#endif + return (unsigned) siphash24g(ent->digest, DIGEST_LEN); } static INLINE int @@ -122,6 +117,8 @@ static cell_queue_entry_t * cell_queue_entry_new_fixed(cell_t *cell); static cell_queue_entry_t * cell_queue_entry_new_var(var_cell_t *var_cell); +static int is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out); /* Functions to maintain the digest map */ static void channel_add_to_digest_map(channel_t *chan); @@ -731,8 +728,8 @@ channel_init(channel_t *chan) /* Init timestamp */ chan->timestamp_last_added_nonpadding = time(NULL); - /* Init next_circ_id */ - chan->next_circ_id = crypto_rand_int(1 << 15); + /* Warn about exhausted circuit IDs no more than hourly. */ + chan->last_warned_circ_ids_exhausted.rate = 3600; /* Initialize queues. */ TOR_SIMPLEQ_INIT(&chan->incoming_queue); @@ -803,7 +800,8 @@ channel_free(channel_t *chan) /* Get rid of cmux */ if (chan->cmux) { - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, NULL); + circuitmux_mark_destroyed_circids_usable(chan->cmux, chan); circuitmux_free(chan->cmux); chan->cmux = NULL; } @@ -1688,6 +1686,13 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) chan->timestamp_last_added_nonpadding = approx_time(); } + { + circid_t circ_id; + if (is_destroy_cell(chan, q, &circ_id)) { + channel_note_destroy_not_pending(chan, circ_id); + } + } + /* Can we send it right out? If so, try */ if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue) && chan->state == CHANNEL_STATE_OPEN) { @@ -2355,7 +2360,7 @@ channel_do_open_actions(channel_t *chan) started_here = channel_is_outgoing(chan); if (started_here) { - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); rep_hist_note_connect_succeeded(chan->identity_digest, now); if (entry_guard_register_connect_status( chan->identity_digest, 1, 0, now) < 0) { @@ -2373,8 +2378,14 @@ channel_do_open_actions(channel_t *chan) /* only report it to the geoip module if it's not a known router */ if (!router_get_by_id_digest(chan->identity_digest)) { if (channel_get_addr_if_possible(chan, &remote_addr)) { - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &remote_addr, + char *transport_name = NULL; + if (chan->get_transport_name(chan, &transport_name) < 0) + transport_name = NULL; + + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, + &remote_addr, transport_name, now); + tor_free(transport_name); } /* Otherwise the underlying transport can't tell us this, so skip it */ } @@ -2611,6 +2622,54 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) } } +/** If <b>packed_cell</b> on <b>chan</b> is a destroy cell, then set + * *<b>circid_out</b> to its circuit ID, and return true. Otherwise, return + * false. */ +/* XXXX Move this function. */ +int +packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out) +{ + if (chan->wide_circ_ids) { + if (packed_cell->body[4] == CELL_DESTROY) { + *circid_out = ntohl(get_uint32(packed_cell->body)); + return 1; + } + } else { + if (packed_cell->body[2] == CELL_DESTROY) { + *circid_out = ntohs(get_uint16(packed_cell->body)); + return 1; + } + } + return 0; +} + +/** DOCDOC */ +static int +is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out) +{ + *circid_out = 0; + switch (q->type) { + case CELL_QUEUE_FIXED: + if (q->u.fixed.cell->command == CELL_DESTROY) { + *circid_out = q->u.fixed.cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_VAR: + if (q->u.var.var_cell->command == CELL_DESTROY) { + *circid_out = q->u.var.var_cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_PACKED: + return packed_cell_is_destroy(chan, q->u.packed.packed_cell, circid_out); + } + return 0; +} + /** * Send destroy cell on a channel * @@ -2622,25 +2681,20 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) int channel_send_destroy(circid_t circ_id, channel_t *chan, int reason) { - cell_t cell; - tor_assert(chan); /* Check to make sure we can send on this channel first */ if (!(chan->state == CHANNEL_STATE_CLOSING || chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { - memset(&cell, 0, sizeof(cell_t)); - cell.circ_id = circ_id; - cell.command = CELL_DESTROY; - cell.payload[0] = (uint8_t) reason; + chan->state == CHANNEL_STATE_ERROR) && + chan->cmux) { + channel_note_destroy_pending(chan, circ_id); + circuitmux_append_destroy_cell(chan, chan->cmux, circ_id, reason); log_debug(LD_OR, "Sending destroy (circID %u) on channel %p " "(global ID " U64_FORMAT ")", (unsigned)circ_id, chan, U64_PRINTF_ARG(chan->global_identifier)); - - channel_write_cell(chan, &cell); } else { log_warn(LD_BUG, "Someone called channel_send_destroy() for circID %u " @@ -2806,7 +2860,7 @@ channel_free_list(smartlist_t *channels, int mark_for_close) channel_state_to_string(curr->state), curr->state); /* Detach circuits early so they can find the channel */ if (curr->cmux) { - circuitmux_detach_all_circuits(curr->cmux); + circuitmux_detach_all_circuits(curr->cmux, NULL); } channel_unregister(curr); if (mark_for_close) { diff --git a/src/or/channel.h b/src/or/channel.h index 29ba40e32..bd9a02f32 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -10,7 +10,6 @@ #define TOR_CHANNEL_H #include "or.h" -#include "tor_queue.h" #include "circuitmux.h" /* Channel handler function pointer typedefs */ @@ -22,7 +21,7 @@ struct cell_queue_entry_s; TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s) incoming_queue; typedef struct chan_cell_queue chan_cell_queue_t; -/* +/** * Channel struct; see the channel_t typedef in or.h. A channel is an * abstract interface for the OR-to-OR connection, similar to connection_or_t, * but without the strong coupling to the underlying TLS implementation. They @@ -32,18 +31,18 @@ typedef struct chan_cell_queue chan_cell_queue_t; */ struct channel_s { - /* Magic number for type-checking cast macros */ + /** Magic number for type-checking cast macros */ uint32_t magic; - /* Current channel state */ + /** Current channel state */ channel_state_t state; - /* Globally unique ID number for a channel over the lifetime of a Tor + /** Globally unique ID number for a channel over the lifetime of a Tor * process. */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** has this channel ever been open? */ @@ -58,28 +57,28 @@ struct channel_s { CHANNEL_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_t *, int); - /* Registered handlers for incoming cells */ + /** Registered handlers for incoming cells */ channel_cell_handler_fn_ptr cell_handler; channel_var_cell_handler_fn_ptr var_cell_handler; /* Methods implemented by the lower layer */ - /* + /** * Ask the underlying transport what the remote endpoint address is, in * a tor_addr_t. This is optional and subclasses may leave this NULL. * If they implement it, they should write the address out to the @@ -87,79 +86,74 @@ struct channel_s { * available. */ int (*get_remote_addr)(channel_t *, tor_addr_t *); + int (*get_transport_name)(channel_t *chan, char **transport_out); + #define GRD_FLAG_ORIGINAL 1 #define GRD_FLAG_ADDR_ONLY 2 - /* + /** * Get a text description of the remote endpoint; canonicalized if the flag * GRD_FLAG_ORIGINAL is not set, or the one we originally connected * to/received from if it is. If GRD_FLAG_ADDR_ONLY is set, we return only * the original address. */ const char * (*get_remote_descr)(channel_t *, int); - /* Check if the lower layer has queued writes */ + /** Check if the lower layer has queued writes */ int (*has_queued_writes)(channel_t *); - /* + /** * If the second param is zero, ask the lower layer if this is * 'canonical', for a transport-specific definition of canonical; if * it is 1, ask if the answer to the preceding query is safe to rely * on. */ int (*is_canonical)(channel_t *, int); - /* Check if this channel matches a specified extend_info_t */ + /** Check if this channel matches a specified extend_info_t */ int (*matches_extend_info)(channel_t *, extend_info_t *); - /* Check if this channel matches a target address when extending */ + /** Check if this channel matches a target address when extending */ int (*matches_target)(channel_t *, const tor_addr_t *); - /* Write a cell to an open channel */ + /** Write a cell to an open channel */ int (*write_cell)(channel_t *, cell_t *); - /* Write a packed cell to an open channel */ + /** Write a packed cell to an open channel */ int (*write_packed_cell)(channel_t *, packed_cell_t *); - /* Write a variable-length cell to an open channel */ + /** Write a variable-length cell to an open channel */ int (*write_var_cell)(channel_t *, var_cell_t *); - /* + /** * Hash of the public RSA key for the other side's identity key, or * zeroes if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; - /* Nickname of the OR on the other side, or NULL if none. */ + /** Nickname of the OR on the other side, or NULL if none. */ char *nickname; - /* + /** * Linked list of channels with the same identity digest, for the * digest->channel map */ TOR_LIST_ENTRY(channel_s) next_with_same_id; - /* List of incoming cells to handle */ + /** List of incoming cells to handle */ chan_cell_queue_t incoming_queue; - /* List of queued outgoing cells */ + /** List of queued outgoing cells */ chan_cell_queue_t outgoing_queue; - /* Circuit mux for circuits sending on this channel */ + /** Circuit mux for circuits sending on this channel */ circuitmux_t *cmux; - /* Circuit ID generation stuff for use by circuitbuild.c */ + /** Circuit ID generation stuff for use by circuitbuild.c */ - /* + /** * When we send CREATE cells along this connection, which half of the * space should we use? */ - ENUM_BF(circ_id_type_t) circ_id_type:2; + circ_id_type_bitfield_t circ_id_type:2; /** DOCDOC*/ unsigned wide_circ_ids:1; - /** Have we logged a warning about circID exhaustion on this channel? */ - unsigned warned_circ_ids_exhausted:1; - /* - * Which circ_id do we try to use next on this connection? This is - * always in the range 0..1<<15-1. - */ - circid_t next_circ_id; - /* For how many circuits are we n_chan? What about p_chan? */ + /** For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; - /* + /** * True iff this channel shouldn't get any new circs attached to it, * because the connection is too old, or because there's a better one. * More generally, this flag is used to note an unhealthy connection; @@ -183,6 +177,10 @@ struct channel_s { */ unsigned int is_local:1; + /** Have we logged a warning about circID exhaustion on this channel? + * If so, when? */ + ratelim_t last_warned_circ_ids_exhausted; + /** Channel timestamps for cell channels */ time_t timestamp_client; /* Client used this, according to relay.c */ time_t timestamp_drained; /* Output queue empty */ @@ -211,7 +209,7 @@ struct channel_listener_s { */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** Why did we close? @@ -223,31 +221,31 @@ struct channel_listener_s { CHANNEL_LISTENER_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_listener_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_listener_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_listener_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_listener_t *, int); - /* Registered listen handler to call on incoming connection */ + /** Registered listen handler to call on incoming connection */ channel_listener_fn_ptr listener; - /* List of pending incoming connections */ + /** List of pending incoming connections */ smartlist_t *incoming_list; - /* Timestamps for listeners */ + /** Timestamps for listeners */ time_t timestamp_accepted; - /* Counters for listeners */ + /** Counters for listeners */ uint64_t n_accepted; }; @@ -482,5 +480,9 @@ uint64_t channel_count_xmitted(channel_t *chan); uint64_t channel_listener_count_accepted(channel_listener_t *chan_l); +int packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out); + #endif diff --git a/src/or/channeltls.c b/src/or/channeltls.c index d5428c1ab..539ead193 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -56,6 +56,8 @@ static const char * channel_tls_describe_transport_method(channel_t *chan); static void channel_tls_free_method(channel_t *chan); static int channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out); static const char * channel_tls_get_remote_descr_method(channel_t *chan, int flags); static int channel_tls_has_queued_writes_method(channel_t *chan); @@ -116,6 +118,7 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->free = channel_tls_free_method; chan->get_remote_addr = channel_tls_get_remote_addr_method; chan->get_remote_descr = channel_tls_get_remote_descr_method; + chan->get_transport_name = channel_tls_get_transport_name_method; chan->has_queued_writes = channel_tls_has_queued_writes_method; chan->is_canonical = channel_tls_is_canonical_method; chan->matches_extend_info = channel_tls_matches_extend_info_method; @@ -286,8 +289,8 @@ channel_tls_handle_incoming(or_connection_t *orconn) if (is_local_addr(&(TO_CONN(orconn)->addr))) channel_mark_local(chan); channel_mark_incoming(chan); - /* If we got one, we should register it */ - if (chan) channel_register(chan); + /* Register it */ + channel_register(chan); return chan; } @@ -435,6 +438,30 @@ channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out) } /** + * Get the name of the pluggable transport used by a channel_tls_t. + * + * This implements the get_transport_name for channel_tls_t. If the + * channel uses a pluggable transport, copy its name to + * <b>transport_out</b> and return 0. If the channel did not use a + * pluggable transport, return -1. */ + +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(transport_out); + tor_assert(tlschan->conn); + + if (!tlschan->conn->ext_or_transport) + return -1; + + *transport_out = tor_strdup(tlschan->conn->ext_or_transport); + return 0; +} + +/** * Get endpoint description of a channel_tls_t * * This implements the get_remote_descr method for channel_tls_t; it returns @@ -1255,13 +1282,20 @@ static void channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) { int highest_supported_version = 0; - const uint8_t *cp, *end; int started_here = 0; tor_assert(cell); tor_assert(chan); tor_assert(chan->conn); + if ((cell->payload_len % 2) == 1) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a VERSION cell with odd payload length %d; " + "closing connection.",cell->payload_len); + connection_or_close_for_error(chan->conn, 0); + return; + } + started_here = connection_or_nonopen_was_started_here(chan->conn); if (chan->conn->link_proto != 0 || @@ -1287,11 +1321,15 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } tor_assert(chan->conn->handshake_state); - end = cell->payload + cell->payload_len; - for (cp = cell->payload; cp+1 < end; cp += 2) { - uint16_t v = ntohs(get_uint16(cp)); - if (is_or_protocol_version_known(v) && v > highest_supported_version) - highest_supported_version = v; + + { + int i; + const uint8_t *cp = cell->payload; + for (i = 0; i < cell->payload_len / 2; ++i, cp += 2) { + uint16_t v = ntohs(get_uint16(cp)); + if (is_or_protocol_version_known(v) && v > highest_supported_version) + highest_supported_version = v; + } } if (!highest_supported_version) { log_fn(LOG_PROTOCOL_WARN, LD_OR, @@ -1489,12 +1527,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) my_addr_ptr = (uint8_t*) cell->payload + 6; end = cell->payload + CELL_PAYLOAD_SIZE; cp = cell->payload + 6 + my_addr_len; - if (cp >= end) { - log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Addresses too long in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; - } else if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { + + /* We used to check: + * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { + * + * This is actually never going to happen, since my_addr_len is at most 255, + * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ + + if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); @@ -1648,12 +1688,16 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) for (i = 0; i < n_certs; ++i) { uint8_t cert_type; uint16_t cert_len; - if (ptr + 3 > cell->payload + cell->payload_len) { + if (cell->payload_len < 3) + goto truncated; + if (ptr > cell->payload + cell->payload_len - 3) { goto truncated; } cert_type = *ptr; cert_len = ntohs(get_uint16(ptr+1)); - if (ptr + 3 + cert_len > cell->payload + cell->payload_len) { + if (cell->payload_len < 3 + cert_len) + goto truncated; + if (ptr > cell->payload + cell->payload_len - cert_len - 3) { goto truncated; } if (cert_type == OR_CERT_TYPE_TLS_LINK || diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c new file mode 100644 index 000000000..51a75cf50 --- /dev/null +++ b/src/or/circpathbias.c @@ -0,0 +1,1538 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "channel.h" +#include "circpathbias.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "circuitstats.h" +#include "connection_edge.h" +#include "config.h" +#include "entrynodes.h" +#include "networkstatus.h" +#include "relay.h" + +static void pathbias_count_successful_close(origin_circuit_t *circ); +static void pathbias_count_collapse(origin_circuit_t *circ); +static void pathbias_count_use_failed(origin_circuit_t *circ); +static void pathbias_measure_use_rate(entry_guard_t *guard); +static void pathbias_measure_close_rate(entry_guard_t *guard); +static void pathbias_scale_use_rates(entry_guard_t *guard); +static void pathbias_scale_close_rates(entry_guard_t *guard); +static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); + +/** Increment the number of times we successfully extended a circuit to + * <b>guard</b>, first checking if the failure rate is high enough that + * we should eliminate the guard. Return -1 if the guard looks no good; + * return 0 if the guard looks fine. + */ +static int +entry_guard_inc_circ_attempt_count(entry_guard_t *guard) +{ + entry_guards_changed(); + + pathbias_measure_close_rate(guard); + + if (guard->path_bias_disabled) + return -1; + + pathbias_scale_close_rates(guard); + guard->circ_attempts++; + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + return 0; +} + +/** The minimum number of circuit attempts before we start + * thinking about warning about path bias and dropping guards */ +static int +pathbias_get_min_circs(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_CIRC 150 + if (options->PathBiasCircThreshold >= 5) + return options->PathBiasCircThreshold; + else + return networkstatus_get_param(NULL, "pb_mincircs", + DFLT_PATH_BIAS_MIN_CIRC, + 5, INT32_MAX); +} + +/** The circuit success rate below which we issue a notice */ +static double +pathbias_get_notice_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_PCT 70 + if (options->PathBiasNoticeRate >= 0.0) + return options->PathBiasNoticeRate; + else + return networkstatus_get_param(NULL, "pb_noticepct", + DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** The circuit success rate below which we issue a warn */ +static double +pathbias_get_warn_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_WARN_PCT 50 + if (options->PathBiasWarnRate >= 0.0) + return options->PathBiasWarnRate; + else + return networkstatus_get_param(NULL, "pb_warnpct", + DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * The extreme rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_PCT 30 + if (options->PathBiasExtremeRate >= 0.0) + return options->PathBiasExtremeRate; + else + return networkstatus_get_param(NULL, "pb_extremepct", + DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * If 1, we actually disable use of guards that fall below + * the extreme_pct. + */ +int +pathbias_get_dropguards(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_DROP_GUARDS 0 + if (options->PathBiasDropGuards >= 0) + return options->PathBiasDropGuards; + else + return networkstatus_get_param(NULL, "pb_dropguards", + DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); +} + +/** + * This is the number of circuits at which we scale our + * counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 + if (options->PathBiasScaleThreshold >= 10) + return options->PathBiasScaleThreshold; + else + return networkstatus_get_param(NULL, "pb_scalecircs", + DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, + INT32_MAX); +} + +/** + * Compute the path bias scaling ratio from the consensus + * parameters pb_multfactor/pb_scalefactor. + * + * Returns a value in (0, 1.0] which we multiply our pathbias + * counts with to scale them down. + */ +static double +pathbias_get_scale_ratio(const or_options_t *options) +{ + /* + * The scale factor is the denominator for our scaling + * of circuit counts for our path bias window. + * + * Note that our use of doubles for the path bias state + * file means that powers of 2 work best here. + */ + int denominator = networkstatus_get_param(NULL, "pb_scalefactor", + 2, 2, INT32_MAX); + (void) options; + /** + * The mult factor is the numerator for our scaling + * of circuit counts for our path bias window. It + * allows us to scale by fractions. + */ + return networkstatus_get_param(NULL, "pb_multfactor", + 1, 1, denominator)/((double)denominator); +} + +/** The minimum number of circuit usage attempts before we start + * thinking about warning about path use bias and dropping guards */ +static int +pathbias_get_min_use(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_USE 20 + if (options->PathBiasUseThreshold >= 3) + return options->PathBiasUseThreshold; + else + return networkstatus_get_param(NULL, "pb_minuse", + DFLT_PATH_BIAS_MIN_USE, + 3, INT32_MAX); +} + +/** The circuit use success rate below which we issue a notice */ +static double +pathbias_get_notice_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 + if (options->PathBiasNoticeUseRate >= 0.0) + return options->PathBiasNoticeUseRate; + else + return networkstatus_get_param(NULL, "pb_noticeusepct", + DFLT_PATH_BIAS_NOTICE_USE_PCT, + 0, 100)/100.0; +} + +/** + * The extreme use rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 + if (options->PathBiasExtremeUseRate >= 0.0) + return options->PathBiasExtremeUseRate; + else + return networkstatus_get_param(NULL, "pb_extremeusepct", + DFLT_PATH_BIAS_EXTREME_USE_PCT, + 0, 100)/100.0; +} + +/** + * This is the number of circuits at which we scale our + * use counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_use_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 + if (options->PathBiasScaleUseThreshold >= 10) + return options->PathBiasScaleUseThreshold; + else + return networkstatus_get_param(NULL, "pb_scaleuse", + DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, + 10, INT32_MAX); +} + +/** + * Convert a Guard's path state to string. + */ +const char * +pathbias_state_to_string(path_state_t state) +{ + switch (state) { + case PATH_STATE_NEW_CIRC: + return "new"; + case PATH_STATE_BUILD_ATTEMPTED: + return "build attempted"; + case PATH_STATE_BUILD_SUCCEEDED: + return "build succeeded"; + case PATH_STATE_USE_ATTEMPTED: + return "use attempted"; + case PATH_STATE_USE_SUCCEEDED: + return "use succeeded"; + case PATH_STATE_USE_FAILED: + return "use failed"; + case PATH_STATE_ALREADY_COUNTED: + return "already counted"; + } + + return "unknown"; +} + +/** + * This function decides if a circuit has progressed far enough to count + * as a circuit "attempt". As long as end-to-end tagging is possible, + * we assume the adversary will use it over hop-to-hop failure. Therefore, + * we only need to account bias for the last hop. This should make us + * much more resilient to ambient circuit failure, and also make that + * failure easier to measure (we only need to measure Exit failure rates). + */ +static int +pathbias_is_new_circ_attempt(origin_circuit_t *circ) +{ +#define N2N_TAGGING_IS_POSSIBLE +#ifdef N2N_TAGGING_IS_POSSIBLE + /* cpath is a circular list. We want circs with more than one hop, + * and the second hop must be waiting for keys still (it's just + * about to get them). */ + return circ->cpath && + circ->cpath->next != circ->cpath && + circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; +#else + /* If tagging attacks are no longer possible, we probably want to + * count bias from the first hop. However, one could argue that + * timing-based tagging is still more useful than per-hop failure. + * In which case, we'd never want to use this. + */ + return circ->cpath && + circ->cpath->state == CPATH_STATE_AWAITING_KEYS; +#endif +} + +/** + * Decide if the path bias code should count a circuit. + * + * @returns 1 if we should count it, 0 otherwise. + */ +static int +pathbias_should_count(origin_circuit_t *circ) +{ +#define PATHBIAS_COUNT_INTERVAL (600) + static ratelim_t count_limit = + RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); + char *rate_msg = NULL; + + /* We can't do path bias accounting without entry guards. + * Testing and controller circuits also have no guards. + * + * We also don't count server-side rends, because their + * endpoint could be chosen maliciously. + * Similarly, we can't count client-side intro attempts, + * because clients can be manipulated into connecting to + * malicious intro points. */ + if (get_options()->UseEntryGuards == 0 || + circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || + circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || + circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || + circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || + (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results. + * + * The reason we check the path state too here is because for the + * cannibalized versions of these purposes, we count them as successful + * before their purpose change. + */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED + && circ->path_state != PATH_STATE_ALREADY_COUNTED) { + log_info(LD_BUG, + "Circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Completely ignore one hop circuits */ + if (circ->build_state->onehop_tunnel || + circ->build_state->desired_path_len == 1) { + /* Check for inconsistency */ + if (circ->build_state->desired_path_len != 1 || + !circ->build_state->onehop_tunnel) { + if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { + log_info(LD_BUG, + "One-hop circuit has length %d. Path state is %s. " + "Circuit is a %s currently %s.%s", + circ->build_state->desired_path_len, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + tor_fragile_assert(); + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { + log_info(LD_BUG, + "One-hop circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { + log_info(LD_BUG, + "Circuit %d is now being counted despite being ignored " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; + + return 1; +} + +/** + * Check our circuit state to see if this is a successful circuit attempt. + * If so, record it in the current guard's path bias circ_attempt count. + * + * Also check for several potential error cases for bug #6475. + */ +int +pathbias_count_build_attempt(origin_circuit_t *circ) +{ +#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) + static ratelim_t circ_attempt_notice_limit = + RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); + char *rate_msg = NULL; + + if (!pathbias_should_count(circ)) { + return 0; + } + + if (pathbias_is_new_circ_attempt(circ)) { + /* Help track down the real cause of bug #6475: */ + if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + /* Don't re-count cannibalized circs.. */ + if (!circ->has_opened) { + entry_guard_t *guard = NULL; + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } else if (circ->base_.n_chan) { + guard = + entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_NEW_CIRC) { + circ->path_state = PATH_STATE_BUILD_ATTEMPTED; + + if (entry_guard_inc_circ_attempt_count(guard) < 0) { + /* Bogus guard; we already warned. */ + return -END_CIRC_REASON_TORPROTOCOL; + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Unopened circuit has strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Unopened circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } + } + + return 0; +} + +/** + * Check our circuit state to see if this is a successful circuit + * completion. If so, record it in the current guard's path bias + * success count. + * + * Also check for several potential error cases for bug #6475. + */ +void +pathbias_count_build_success(origin_circuit_t *circ) +{ +#define SUCCESS_NOTICE_INTERVAL (600) + static ratelim_t success_notice_limit = + RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); + char *rate_msg = NULL; + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* Don't count cannibalized/reused circs for path bias + * "build" success, since they get counted under "use" success. */ + if (!circ->has_opened) { + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { + circ->path_state = PATH_STATE_BUILD_SUCCEEDED; + guard->circ_successes++; + entry_guards_changed(); + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } else { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Succeeded circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + if (guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Completed circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } +} + +/** + * Record an attempt to use a circuit. Changes the circuit's + * path state and update its guard's usage counter. + * + * Used for path bias usage accounting. + */ +void +pathbias_count_use_attempt(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + log_notice(LD_BUG, + "Used circuit is in strange path state %s. " + "Circuit is a %s currently %s.", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + pathbias_measure_use_rate(guard); + pathbias_scale_use_rates(guard); + guard->use_attempts++; + entry_guards_changed(); + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used for guard %s ($%s).", + circ->global_identifier, + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } else { + /* Harmless but educational log message */ + log_info(LD_CIRC, + "Used circuit %d is already in path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } + + return; +} + +/** + * Check the circuit's path state is appropriate and mark it as + * successfully used. Used for path bias usage accounting. + * + * We don't actually increment the guard's counters until + * pathbias_check_close(), because the circuit can still transition + * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this + * is done so we can probe the circuit for liveness at close). + */ +void +pathbias_mark_use_success(origin_circuit_t *circ) +{ + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + log_notice(LD_BUG, + "Used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + + pathbias_count_use_attempt(circ); + } + + /* We don't do any accounting at the guard until actual circuit close */ + circ->path_state = PATH_STATE_USE_SUCCEEDED; + + return; +} + +/** + * If a stream ever detatches from a circuit in a retriable way, + * we need to mark this circuit as still needing either another + * successful stream, or in need of a probe. + * + * An adversary could let the first stream request succeed (ie the + * resolve), but then tag and timeout the remainder (via cell + * dropping), forcing them on new circuits. + * + * Rolling back the state will cause us to probe such circuits, which + * should lead to probe failures in the event of such tagging due to + * either unrecognized cells coming in while we wait for the probe, + * or the cipher state getting out of sync in the case of dropped cells. + */ +void +pathbias_mark_use_rollback(origin_circuit_t *circ) +{ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + log_info(LD_CIRC, + "Rolling back pathbias use state to 'attempted' for detached " + "circuit %d", circ->global_identifier); + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } +} + +/** + * Actually count a circuit success towards a guard's usage counters + * if the path state is appropriate. + */ +static void +pathbias_count_use_success(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { + log_notice(LD_BUG, + "Successfully used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + guard->use_successes++; + entry_guards_changed(); + + if (guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " + "for guard %s=%s", + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used successfully for guard " + "%s ($%s).", + circ->global_identifier, guard->use_successes, + guard->use_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } + + return; +} + +/** + * Send a probe down a circuit that the client attempted to use, + * but for which the stream timed out/failed. The probe is a + * RELAY_BEGIN cell with a 0.a.b.c destination address, which + * the exit will reject and reply back, echoing that address. + * + * The reason for such probes is because it is possible to bias + * a user's paths simply by causing timeouts, and these timeouts + * are not possible to differentiate from unresponsive servers. + * + * The probe is sent at the end of the circuit lifetime for two + * reasons: to prevent cryptographic taggers from being able to + * drop cells to cause timeouts, and to prevent easy recognition + * of probes before any real client traffic happens. + * + * Returns -1 if we couldn't probe, 0 otherwise. + */ +static int +pathbias_send_usable_probe(circuit_t *circ) +{ + /* Based on connection_ap_handshake_send_begin() */ + char payload[CELL_PAYLOAD_SIZE]; + int payload_len; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + crypt_path_t *cpath_layer = NULL; + char *probe_nonce = NULL; + + tor_assert(ocirc); + + cpath_layer = ocirc->cpath->prev; + + if (cpath_layer->state != CPATH_STATE_OPEN) { + /* This can happen for cannibalized circuits. Their + * last hop isn't yet open */ + log_info(LD_CIRC, + "Got pathbias probe request for unopened circuit %d. " + "Opened %d, len %d", ocirc->global_identifier, + ocirc->has_opened, ocirc->build_state->desired_path_len); + return -1; + } + + /* We already went down this road. */ + if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && + ocirc->pathbias_probe_id) { + log_info(LD_CIRC, + "Got pathbias probe request for circuit %d with " + "outstanding probe", ocirc->global_identifier); + return -1; + } + + /* Can't probe if the channel isn't open */ + if (circ->n_chan == NULL || + (circ->n_chan->state != CHANNEL_STATE_OPEN + && circ->n_chan->state != CHANNEL_STATE_MAINT)) { + log_info(LD_CIRC, + "Skipping pathbias probe for circuit %d: Channel is not open.", + ocirc->global_identifier); + return -1; + } + + circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + /* Update timestamp for when circuit_expire_building() should kill us */ + tor_gettimeofday(&circ->timestamp_began); + + /* Generate a random address for the nonce */ + crypto_rand((char*)ô->pathbias_probe_nonce, + sizeof(ocirc->pathbias_probe_nonce)); + ocirc->pathbias_probe_nonce &= 0x00ffffff; + probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); + + tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); + payload_len = (int)strlen(payload)+1; + + // XXX: need this? Can we assume ipv4 will always be supported? + // If not, how do we tell? + //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { + // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); + // payload_len += 4; + //} + + /* Generate+Store stream id, make sure it's non-zero */ + ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); + + if (ocirc->pathbias_probe_id==0) { + log_warn(LD_CIRC, + "Ran out of stream IDs on circuit %u during " + "pathbias probe attempt.", ocirc->global_identifier); + tor_free(probe_nonce); + return -1; + } + + log_info(LD_CIRC, + "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", + probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); + tor_free(probe_nonce); + + /* Send a test relay cell */ + if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, + RELAY_COMMAND_BEGIN, payload, + payload_len, cpath_layer) < 0) { + log_notice(LD_CIRC, + "Failed to send pathbias probe cell on circuit %d.", + ocirc->global_identifier); + return -1; + } + + /* Mark it freshly dirty so it doesn't get expired in the meantime */ + circ->timestamp_dirty = time(NULL); + + return 0; +} + +/** + * Check the response to a pathbias probe, to ensure the + * cell is recognized and the nonce and other probe + * characteristics are as expected. + * + * If the response is valid, return 0. Otherwise return < 0. + */ +int +pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) +{ + /* Based on connection_edge_process_relay_cell() */ + relay_header_t rh; + int reason; + uint32_t ipv4_host; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + + tor_assert(cell); + tor_assert(ocirc); + tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + relay_header_unpack(&rh, cell->payload); + + reason = rh.length > 0 ? + get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; + + if (rh.command == RELAY_COMMAND_END && + reason == END_STREAM_REASON_EXITPOLICY && + ocirc->pathbias_probe_id == rh.stream_id) { + + /* Check length+extract host: It is in network order after the reason code. + * See connection_edge_end(). */ + if (rh.length < 9) { /* reason+ipv4+dns_ttl */ + log_notice(LD_PROTOCOL, + "Short path bias probe response length field (%d).", rh.length); + return - END_CIRC_REASON_TORPROTOCOL; + } + + ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); + + /* Check nonce */ + if (ipv4_host == ocirc->pathbias_probe_nonce) { + pathbias_mark_use_success(ocirc); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + log_info(LD_CIRC, + "Got valid path bias probe back for circ %d, stream %d.", + ocirc->global_identifier, ocirc->pathbias_probe_id); + return 0; + } else { + log_notice(LD_CIRC, + "Got strange probe value 0x%x vs 0x%x back for circ %d, " + "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, + ocirc->global_identifier, ocirc->pathbias_probe_id); + return -1; + } + } + log_info(LD_CIRC, + "Got another cell back back on pathbias probe circuit %d: " + "Command: %d, Reason: %d, Stream-id: %d", + ocirc->global_identifier, rh.command, reason, rh.stream_id); + return -1; +} + +/** + * Check if a circuit was used and/or closed successfully. + * + * If we attempted to use the circuit to carry a stream but failed + * for whatever reason, or if the circuit mysteriously died before + * we could attach any streams, record these two cases. + * + * If we *have* successfully used the circuit, or it appears to + * have been closed by us locally, count it as a success. + * + * Returns 0 if we're done making decisions with the circ, + * or -1 if we want to probe it first. + */ +int +pathbias_check_close(origin_circuit_t *ocirc, int reason) +{ + circuit_t *circ = ô->base_; + + if (!pathbias_should_count(ocirc)) { + return 0; + } + + switch (ocirc->path_state) { + /* If the circuit was closed after building, but before use, we need + * to ensure we were the ones who tried to close it (and not a remote + * actor). */ + case PATH_STATE_BUILD_SUCCEEDED: + if (reason & END_CIRC_REASON_FLAG_REMOTE) { + /* Remote circ close reasons on an unused circuit all could be bias */ + log_info(LD_CIRC, + "Circuit %d remote-closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) + == END_CIRC_REASON_CHANNEL_CLOSED && + circ->n_chan && + circ->n_chan->reason_for_closing + != CHANNEL_CLOSE_REQUESTED) { + /* If we didn't close the channel ourselves, it could be bias */ + /* XXX: Only count bias if the network is live? + * What about clock jumps/suspends? */ + log_info(LD_CIRC, + "Circuit %d's channel closed without successful use for reason " + "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " + "%d.", ocirc->global_identifier, + reason, circ->n_chan->reason_for_closing, + circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else { + pathbias_count_successful_close(ocirc); + } + break; + + /* If we tried to use a circuit but failed, we should probe it to ensure + * it has not been tampered with. */ + case PATH_STATE_USE_ATTEMPTED: + /* XXX: Only probe and/or count failure if the network is live? + * What about clock jumps/suspends? */ + if (pathbias_send_usable_probe(circ) == 0) + return -1; + else + pathbias_count_use_failed(ocirc); + + /* Any circuit where there were attempted streams but no successful + * streams could be bias */ + log_info(LD_CIRC, + "Circuit %d closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + break; + + case PATH_STATE_USE_SUCCEEDED: + pathbias_count_successful_close(ocirc); + pathbias_count_use_success(ocirc); + break; + + case PATH_STATE_USE_FAILED: + pathbias_count_use_failed(ocirc); + break; + + case PATH_STATE_NEW_CIRC: + case PATH_STATE_BUILD_ATTEMPTED: + case PATH_STATE_ALREADY_COUNTED: + default: + // Other states are uninteresting. No stats to count. + break; + } + + ocirc->path_state = PATH_STATE_ALREADY_COUNTED; + + return 0; +} + +/** + * Count a successfully closed circuit. + */ +static void +pathbias_count_successful_close(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + /* In the long run: circuit_success ~= successful_circuit_close + + * circ_failure + stream_failure */ + guard->successful_circuits_closed++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Successfully closed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a circuit that fails after it is built, but before it can + * carry any traffic. + * + * This is needed because there are ways to destroy a + * circuit after it has successfully completed. Right now, this is + * used for purely informational/debugging purposes. + */ +static void +pathbias_count_collapse(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->collapsed_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Destroyed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a known failed circuit (because we could not probe it). + * + * This counter is informational. + */ +static void +pathbias_count_use_failed(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->unusable_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + /* XXX note cut-and-paste code in this function compared to nearby + * functions. Would be nice to refactor. -RD */ + log_info(LD_CIRC, + "Stream-failing circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count timeouts for path bias log messages. + * + * These counts are purely informational. + */ +void +pathbias_count_timeout(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* For hidden service circs, they can actually be used + * successfully and then time out later (because + * the other side declines to use them). */ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->timeouts++; + entry_guards_changed(); + } +} + +/** + * Helper function to count all of the currently opened circuits + * for a guard that are in a given path state range. The state + * range is inclusive on both ends. + */ +static int +pathbias_count_circs_in_states(entry_guard_t *guard, + path_state_t from, + path_state_t to) +{ + circuit_t *circ; + int open_circuits = 0; + + /* Count currently open circuits. Give them the benefit of the doubt. */ + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + origin_circuit_t *ocirc = NULL; + if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ + circ->marked_for_close) /* already counted */ + continue; + + ocirc = TO_ORIGIN_CIRCUIT(circ); + + if (!ocirc->cpath || !ocirc->cpath->extend_info) + continue; + + if (ocirc->path_state >= from && + ocirc->path_state <= to && + pathbias_should_count(ocirc) && + fast_memeq(guard->identity, + ocirc->cpath->extend_info->identity_digest, + DIGEST_LEN)) { + log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", + ocirc->global_identifier, + pathbias_state_to_string(ocirc->path_state)); + open_circuits++; + } + } + + return open_circuits; +} + +/** + * Return the number of circuits counted as successfully closed for + * this guard. + * + * Also add in the currently open circuits to give them the benefit + * of the doubt. + */ +double +pathbias_get_close_success_count(entry_guard_t *guard) +{ + return guard->successful_circuits_closed + + pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Return the number of circuits counted as successfully used + * this guard. + * + * Also add in the currently open circuits that we are attempting + * to use to give them the benefit of the doubt. + */ +double +pathbias_get_use_success_count(entry_guard_t *guard) +{ + return guard->use_successes + + pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Check the path bias use rate against our consensus parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + */ +static void +pathbias_measure_use_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->use_attempts > pathbias_get_min_use(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_extreme_use_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of stream on its circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_use_extreme) { + guard->path_bias_use_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of streams on its circuits. " + "This could indicate a route manipulation attack, network " + "overload, bad local network connectivity, or a bug. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_notice_use_rate(options)) { + if (!guard->path_bias_use_noticed) { + guard->path_bias_use_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing to carry more streams on its " + "circuits than usual. " + "Most likely this means the Tor network is overloaded " + "or your network connection is poor. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * Check the path bias circuit close status rates against our consensus + * parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + * + * XXX: This function shares similar log messages and checks to + * pathbias_measure_use_rate(). It may be possible to combine them + * eventually, especially if we can ever remove the need for 3 + * levels of closure warns (if the overall circuit failure rate + * goes down with ntor). One way to do so would be to multiply + * the build rate with the use rate to get an idea of the total + * fraction of the total network paths the user is able to use. + * See ticket #8159. + */ +static void +pathbias_measure_close_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->circ_attempts > pathbias_get_min_circs(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_extreme_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_extreme) { + guard->path_bias_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "This could indicate a route manipulation attack, " + "extreme network overload, or a bug. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_warn_rate(options)) { + if (!guard->path_bias_warned) { + guard->path_bias_warned = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing a very large " + "amount of circuits. " + "Most likely this means the Tor network is " + "overloaded, but it could also mean an attack against " + "you or potentially the guard itself. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_notice_rate(options)) { + if (!guard->path_bias_noticed) { + guard->path_bias_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing more circuits than " + "usual. " + "Most likely this means the Tor network is overloaded. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * This function scales the path bias use rates if we have + * more data than the scaling threshold. This allows us to + * be more sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +static void +pathbias_scale_close_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); + int opened_built = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_FAILED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); + + guard->circ_attempts -= (opened_attempts+opened_built); + guard->circ_successes -= opened_built; + + guard->circ_attempts *= scale_ratio; + guard->circ_successes *= scale_ratio; + guard->timeouts *= scale_ratio; + guard->successful_circuits_closed *= scale_ratio; + guard->collapsed_circuits *= scale_ratio; + guard->unusable_circuits *= scale_ratio; + + guard->circ_attempts += (opened_attempts+opened_built); + guard->circ_successes += opened_built; + + entry_guards_changed(); + + log_info(LD_CIRC, + "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " + "%s ($%s)", + guard->circ_successes, guard->successful_circuits_closed, + guard->circ_attempts, opened_built, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, opened_built, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } +} + +/** + * This function scales the path bias circuit close rates if we have + * more data than the scaling threshold. This allows us to be more + * sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +void +pathbias_scale_use_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->use_attempts >= guard->use_successes); + + guard->use_attempts -= opened_attempts; + + guard->use_attempts *= scale_ratio; + guard->use_successes *= scale_ratio; + + guard->use_attempts += opened_attempts; + + log_info(LD_CIRC, + "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", + guard->use_successes, guard->use_attempts, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias usage counts to %f/%f " + "(%d open) for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + + entry_guards_changed(); + } +} + diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h new file mode 100644 index 000000000..c95d801a4 --- /dev/null +++ b/src/or/circpathbias.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef TOR_CIRCPATHBIAS_H +#define TOR_CIRCPATHBIAS_H + +double pathbias_get_extreme_rate(const or_options_t *options); +double pathbias_get_extreme_use_rate(const or_options_t *options); +int pathbias_get_dropguards(const or_options_t *options); +void pathbias_count_timeout(origin_circuit_t *circ); +void pathbias_count_build_success(origin_circuit_t *circ); +int pathbias_count_build_attempt(origin_circuit_t *circ); +int pathbias_check_close(origin_circuit_t *circ, int reason); +int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); +void pathbias_count_use_attempt(origin_circuit_t *circ); +void pathbias_mark_use_success(origin_circuit_t *circ); +void pathbias_mark_use_rollback(origin_circuit_t *circ); +const char *pathbias_state_to_string(path_state_t state); + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 2b4d3c311..9d06759de 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,8 +9,11 @@ * \brief The actual details of building circuits. **/ +#define CIRCUITBUILD_PRIVATE + #include "or.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -40,19 +43,11 @@ #include "routerparse.h" #include "routerset.h" #include "crypto.h" -#include "connection_edge.h" #ifndef MIN #define MIN(a,b) ((a)<(b)?(a):(b)) #endif -/********* START VARIABLES **********/ - -/** A global list of all circuits at this hop. */ -extern circuit_t *global_circuitlist; - -/********* END VARIABLES ************/ - static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, const char *id_digest); @@ -64,14 +59,6 @@ static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int onion_extend_cpath(origin_circuit_t *circ); static int count_acceptable_nodes(smartlist_t *routers); static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); -static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); -static void pathbias_count_build_success(origin_circuit_t *circ); -static void pathbias_count_successful_close(origin_circuit_t *circ); -static void pathbias_count_collapse(origin_circuit_t *circ); -static void pathbias_count_use_failed(origin_circuit_t *circ); -static void pathbias_measure_use_rate(entry_guard_t *guard); -static void pathbias_measure_close_rate(entry_guard_t *guard); -static void pathbias_scale_use_rates(entry_guard_t *guard); #ifdef CURVE25519_ENABLED static int circuits_can_use_ntor(void); #endif @@ -92,18 +79,27 @@ channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, return chan; } -/** Iterate over values of circ_id, starting from conn-\>next_circ_id, - * and with the high bit specified by conn-\>circ_id_type, until we get - * a circ_id that is not in use by any other circuit on that conn. +/** Search for a value for circ_id that we can use on <b>chan</b> for an + * outbound circuit, until we get a circ_id that is not in use by any other + * circuit on that conn. * * Return it, or 0 if can't get a unique circ_id. */ -static circid_t +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan) { +/* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ +#define MAX_CIRCID_ATTEMPTS 64 + int in_use; + unsigned n_with_circ = 0, n_pending_destroy = 0; circid_t test_circ_id; circid_t attempts=0; - circid_t high_bit, max_range; + circid_t high_bit, max_range, mask; tor_assert(chan); @@ -113,32 +109,52 @@ get_unique_circ_id_by_chan(channel_t *chan) "a client with no identity."); return 0; } - max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + mask = max_range - 1; high_bit = (chan->circ_id_type == CIRC_ID_TYPE_HIGHER) ? max_range : 0; do { - /* Sequentially iterate over test_circ_id=1...max_range until we find a - * circID such that (high_bit|test_circ_id) is not already used. */ - test_circ_id = chan->next_circ_id++; - if (test_circ_id == 0 || test_circ_id >= max_range) { - test_circ_id = 1; - chan->next_circ_id = 2; - } - if (++attempts > max_range) { - /* Make sure we don't loop forever if all circ_id's are used. This - * matters because it's an external DoS opportunity. + if (++attempts > MAX_CIRCID_ATTEMPTS) { + /* Make sure we don't loop forever because all circuit IDs are used. + * + * Once, we would try until we had tried every possible circuit ID. But + * that's quite expensive. Instead, we try MAX_CIRCID_ATTEMPTS random + * circuit IDs, and then give up. + * + * This potentially causes us to give up early if our circuit ID space + * is nearly full. If we have N circuit IDs in use, then we will reject + * a new circuit with probability (N / max_range) ^ MAX_CIRCID_ATTEMPTS. + * This means that in practice, a few percent of our circuit ID capacity + * will go unused. + * + * The alternative here, though, is to do a linear search over the + * whole circuit ID space every time we extend a circuit, which is + * not so great either. */ - if (! chan->warned_circ_ids_exhausted) { - chan->warned_circ_ids_exhausted = 1; - log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " + log_fn_ratelim(&chan->last_warned_circ_ids_exhausted, LOG_WARN, + LD_CIRC,"No unused circIDs found on channel %s wide " "circID support, with %u inbound and %u outbound circuits. " + "Found %u circuit IDs in use by circuits, and %u with " + "pending destroy cells." "Failing a circuit.", chan->wide_circ_ids ? "with" : "without", - chan->num_p_circuits, chan->num_n_circuits); - } + chan->num_p_circuits, chan->num_n_circuits, + n_with_circ, n_pending_destroy); return 0; } + + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; - } while (circuit_id_in_use_on_channel(test_circ_id, chan)); + + in_use = circuit_id_in_use_on_channel(test_circ_id, chan); + if (in_use == 1) + ++n_with_circ; + else if (in_use == 2) + ++n_pending_destroy; + } while (in_use); return test_circ_id; } @@ -299,9 +315,9 @@ circuit_rep_hist_note_result(origin_circuit_t *circ) static int circuit_cpath_supports_ntor(const origin_circuit_t *circ) { - crypt_path_t *head = circ->cpath, *cpath = circ->cpath; + crypt_path_t *head, *cpath; - cpath = head; + cpath = head = circ->cpath; do { if (cpath->extend_info && !tor_mem_is_zero( @@ -583,7 +599,9 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, id = get_unique_circ_id_by_chan(circ->n_chan); if (!id) { - log_warn(LD_CIRC,"failed to get unique circID."); + static ratelim_t circid_warning_limit = RATELIM_INIT(9600); + log_fn_ratelim(&circid_warning_limit, LOG_WARN, LD_CIRC, + "failed to get unique circID."); return -1; } log_debug(LD_CIRC,"Chosen circID %u.", (unsigned)id); @@ -628,27 +646,30 @@ int inform_testing_reachability(void) { char dirbuf[128]; + char *address; const routerinfo_t *me = router_get_my_routerinfo(); if (!me) return 0; + address = tor_dup_ip(me->addr); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); if (me->dir_port) { tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); } log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", - me->address, me->or_port, + address, me->or_port, me->dir_port ? dirbuf : "", me->dir_port ? "are" : "is", TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60); + tor_free(address); return 1; } @@ -663,16 +684,18 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) if (!circ->cpath->extend_info->onion_key) return 1; /* our hand is forced: only a create_fast will work. */ - if (!options->FastFirstHopPK) - return 0; /* we prefer to avoid create_fast */ if (public_server_mode(options)) { /* We're a server, and we know an onion key. We can choose. * Prefer to blend our circuit into the other circuits we are * creating on behalf of others. */ return 0; } + if (options->FastFirstHopPK == -1) { + /* option is "auto", so look at the consensus. */ + return networkstatus_get_param(NULL, "usecreatefast", 1, 0, 1); + } - return 1; + return options->FastFirstHopPK; } /** Return true if <b>circ</b> is the type of circuit we want to count @@ -842,20 +865,24 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) * it off at, we probably had a suspend event along this codepath, * and we should discard the value. */ - if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + if (timediff < 0 || + timediff > 2*get_circuit_build_close_time_ms()+1000) { log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " "Assuming clock jump. Purpose %d (%s)", timediff, circ->base_.purpose, circuit_purpose_to_string(circ->base_.purpose)); } else if (!circuit_build_times_disabled()) { /* Only count circuit times if the network is live */ - if (circuit_build_times_network_check_live(&circ_times)) { - circuit_build_times_add_time(&circ_times, (build_time_t)timediff); - circuit_build_times_set_timeout(&circ_times); + if (circuit_build_times_network_check_live( + get_circuit_build_times())) { + circuit_build_times_add_time(get_circuit_build_times_mutable(), + (build_time_t)timediff); + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - circuit_build_times_network_circ_success(&circ_times); + circuit_build_times_network_circ_success( + get_circuit_build_times_mutable()); } } } @@ -1150,1516 +1177,6 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, return 0; } -/** The minimum number of circuit attempts before we start - * thinking about warning about path bias and dropping guards */ -static int -pathbias_get_min_circs(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_CIRC 150 - if (options->PathBiasCircThreshold >= 5) - return options->PathBiasCircThreshold; - else - return networkstatus_get_param(NULL, "pb_mincircs", - DFLT_PATH_BIAS_MIN_CIRC, - 5, INT32_MAX); -} - -/** The circuit success rate below which we issue a notice */ -static double -pathbias_get_notice_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_PCT 70 - if (options->PathBiasNoticeRate >= 0.0) - return options->PathBiasNoticeRate; - else - return networkstatus_get_param(NULL, "pb_noticepct", - DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** The circuit success rate below which we issue a warn */ -static double -pathbias_get_warn_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_WARN_PCT 50 - if (options->PathBiasWarnRate >= 0.0) - return options->PathBiasWarnRate; - else - return networkstatus_get_param(NULL, "pb_warnpct", - DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * The extreme rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_PCT 30 - if (options->PathBiasExtremeRate >= 0.0) - return options->PathBiasExtremeRate; - else - return networkstatus_get_param(NULL, "pb_extremepct", - DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * If 1, we actually disable use of guards that fall below - * the extreme_pct. - */ -int -pathbias_get_dropguards(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_DROP_GUARDS 0 - if (options->PathBiasDropGuards >= 0) - return options->PathBiasDropGuards; - else - return networkstatus_get_param(NULL, "pb_dropguards", - DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); -} - -/** - * This is the number of circuits at which we scale our - * counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 - if (options->PathBiasScaleThreshold >= 10) - return options->PathBiasScaleThreshold; - else - return networkstatus_get_param(NULL, "pb_scalecircs", - DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, - INT32_MAX); -} - -/** - * Compute the path bias scaling ratio from the consensus - * parameters pb_multfactor/pb_scalefactor. - * - * Returns a value in (0, 1.0] which we multiply our pathbias - * counts with to scale them down. - */ -static double -pathbias_get_scale_ratio(const or_options_t *options) -{ - /* - * The scale factor is the denominator for our scaling - * of circuit counts for our path bias window. - * - * Note that our use of doubles for the path bias state - * file means that powers of 2 work best here. - */ - int denominator = networkstatus_get_param(NULL, "pb_scalefactor", - 2, 2, INT32_MAX); - (void) options; - /** - * The mult factor is the numerator for our scaling - * of circuit counts for our path bias window. It - * allows us to scale by fractions. - */ - return networkstatus_get_param(NULL, "pb_multfactor", - 1, 1, denominator)/((double)denominator); -} - -/** The minimum number of circuit usage attempts before we start - * thinking about warning about path use bias and dropping guards */ -static int -pathbias_get_min_use(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_USE 20 - if (options->PathBiasUseThreshold >= 3) - return options->PathBiasUseThreshold; - else - return networkstatus_get_param(NULL, "pb_minuse", - DFLT_PATH_BIAS_MIN_USE, - 3, INT32_MAX); -} - -/** The circuit use success rate below which we issue a notice */ -static double -pathbias_get_notice_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 - if (options->PathBiasNoticeUseRate >= 0.0) - return options->PathBiasNoticeUseRate; - else - return networkstatus_get_param(NULL, "pb_noticeusepct", - DFLT_PATH_BIAS_NOTICE_USE_PCT, - 0, 100)/100.0; -} - -/** - * The extreme use rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 - if (options->PathBiasExtremeUseRate >= 0.0) - return options->PathBiasExtremeUseRate; - else - return networkstatus_get_param(NULL, "pb_extremeusepct", - DFLT_PATH_BIAS_EXTREME_USE_PCT, - 0, 100)/100.0; -} - -/** - * This is the number of circuits at which we scale our - * use counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_use_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 - if (options->PathBiasScaleUseThreshold >= 10) - return options->PathBiasScaleUseThreshold; - else - return networkstatus_get_param(NULL, "pb_scaleuse", - DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, - 10, INT32_MAX); -} - -/** - * Convert a Guard's path state to string. - */ -const char * -pathbias_state_to_string(path_state_t state) -{ - switch (state) { - case PATH_STATE_NEW_CIRC: - return "new"; - case PATH_STATE_BUILD_ATTEMPTED: - return "build attempted"; - case PATH_STATE_BUILD_SUCCEEDED: - return "build succeeded"; - case PATH_STATE_USE_ATTEMPTED: - return "use attempted"; - case PATH_STATE_USE_SUCCEEDED: - return "use succeeded"; - case PATH_STATE_USE_FAILED: - return "use failed"; - case PATH_STATE_ALREADY_COUNTED: - return "already counted"; - } - - return "unknown"; -} - -/** - * This function decides if a circuit has progressed far enough to count - * as a circuit "attempt". As long as end-to-end tagging is possible, - * we assume the adversary will use it over hop-to-hop failure. Therefore, - * we only need to account bias for the last hop. This should make us - * much more resilient to ambient circuit failure, and also make that - * failure easier to measure (we only need to measure Exit failure rates). - */ -static int -pathbias_is_new_circ_attempt(origin_circuit_t *circ) -{ -#define N2N_TAGGING_IS_POSSIBLE -#ifdef N2N_TAGGING_IS_POSSIBLE - /* cpath is a circular list. We want circs with more than one hop, - * and the second hop must be waiting for keys still (it's just - * about to get them). */ - return circ->cpath && - circ->cpath->next != circ->cpath && - circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; -#else - /* If tagging attacks are no longer possible, we probably want to - * count bias from the first hop. However, one could argue that - * timing-based tagging is still more useful than per-hop failure. - * In which case, we'd never want to use this. - */ - return circ->cpath && - circ->cpath->state == CPATH_STATE_AWAITING_KEYS; -#endif -} - -/** - * Decide if the path bias code should count a circuit. - * - * @returns 1 if we should count it, 0 otherwise. - */ -static int -pathbias_should_count(origin_circuit_t *circ) -{ -#define PATHBIAS_COUNT_INTERVAL (600) - static ratelim_t count_limit = - RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); - char *rate_msg = NULL; - - /* We can't do path bias accounting without entry guards. - * Testing and controller circuits also have no guards. - * - * We also don't count server-side rends, because their - * endpoint could be chosen maliciously. - * Similarly, we can't count client-side intro attempts, - * because clients can be manipulated into connecting to - * malicious intro points. */ - if (get_options()->UseEntryGuards == 0 || - circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || - circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || - circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || - circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || - (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && - circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results. - * - * The reason we check the path state too here is because for the - * cannibalized versions of these purposes, we count them as successful - * before their purpose change. - */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED - && circ->path_state != PATH_STATE_ALREADY_COUNTED) { - log_info(LD_BUG, - "Circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Completely ignore one hop circuits */ - if (circ->build_state->onehop_tunnel || - circ->build_state->desired_path_len == 1) { - /* Check for inconsistency */ - if (circ->build_state->desired_path_len != 1 || - !circ->build_state->onehop_tunnel) { - if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { - log_info(LD_BUG, - "One-hop circuit has length %d. Path state is %s. " - "Circuit is a %s currently %s.%s", - circ->build_state->desired_path_len, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - tor_fragile_assert(); - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { - log_info(LD_BUG, - "One-hop circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { - log_info(LD_BUG, - "Circuit %d is now being counted despite being ignored " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; - - return 1; -} - -/** - * Check our circuit state to see if this is a successful circuit attempt. - * If so, record it in the current guard's path bias circ_attempt count. - * - * Also check for several potential error cases for bug #6475. - */ -static int -pathbias_count_build_attempt(origin_circuit_t *circ) -{ -#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) - static ratelim_t circ_attempt_notice_limit = - RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); - char *rate_msg = NULL; - - if (!pathbias_should_count(circ)) { - return 0; - } - - if (pathbias_is_new_circ_attempt(circ)) { - /* Help track down the real cause of bug #6475: */ - if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - /* Don't re-count cannibalized circs.. */ - if (!circ->has_opened) { - entry_guard_t *guard = NULL; - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } else if (circ->base_.n_chan) { - guard = - entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_NEW_CIRC) { - circ->path_state = PATH_STATE_BUILD_ATTEMPTED; - - if (entry_guard_inc_circ_attempt_count(guard) < 0) { - /* Bogus guard; we already warned. */ - return -END_CIRC_REASON_TORPROTOCOL; - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Unopened circuit has strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Unopened circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } - } - - return 0; -} - -/** - * Check our circuit state to see if this is a successful circuit - * completion. If so, record it in the current guard's path bias - * success count. - * - * Also check for several potential error cases for bug #6475. - */ -static void -pathbias_count_build_success(origin_circuit_t *circ) -{ -#define SUCCESS_NOTICE_INTERVAL (600) - static ratelim_t success_notice_limit = - RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); - char *rate_msg = NULL; - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* Don't count cannibalized/reused circs for path bias - * "build" success, since they get counted under "use" success. */ - if (!circ->has_opened) { - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { - circ->path_state = PATH_STATE_BUILD_SUCCEEDED; - guard->circ_successes++; - entry_guards_changed(); - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } else { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Succeeded circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - if (guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Completed circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } -} - -/** - * Record an attempt to use a circuit. Changes the circuit's - * path state and update its guard's usage counter. - * - * Used for path bias usage accounting. - */ -void -pathbias_count_use_attempt(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - log_notice(LD_BUG, - "Used circuit is in strange path state %s. " - "Circuit is a %s currently %s.", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - pathbias_measure_use_rate(guard); - pathbias_scale_use_rates(guard); - guard->use_attempts++; - entry_guards_changed(); - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used for guard %s ($%s).", - circ->global_identifier, - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } else { - /* Harmless but educational log message */ - log_info(LD_CIRC, - "Used circuit %d is already in path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } - - return; -} - -/** - * Check the circuit's path state is appropriate and mark it as - * successfully used. Used for path bias usage accounting. - * - * We don't actually increment the guard's counters until - * pathbias_check_close(), because the circuit can still transition - * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this - * is done so we can probe the circuit for liveness at close). - */ -void -pathbias_mark_use_success(origin_circuit_t *circ) -{ - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - log_notice(LD_BUG, - "Used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - - pathbias_count_use_attempt(circ); - } - - /* We don't do any accounting at the guard until actual circuit close */ - circ->path_state = PATH_STATE_USE_SUCCEEDED; - - return; -} - -/** - * If a stream ever detatches from a circuit in a retriable way, - * we need to mark this circuit as still needing either another - * successful stream, or in need of a probe. - * - * An adversary could let the first stream request succeed (ie the - * resolve), but then tag and timeout the remainder (via cell - * dropping), forcing them on new circuits. - * - * Rolling back the state will cause us to probe such circuits, which - * should lead to probe failures in the event of such tagging due to - * either unrecognized cells coming in while we wait for the probe, - * or the cipher state getting out of sync in the case of dropped cells. - */ -void -pathbias_mark_use_rollback(origin_circuit_t *circ) -{ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - log_info(LD_CIRC, - "Rolling back pathbias use state to 'attempted' for detached " - "circuit %d", circ->global_identifier); - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } -} - -/** - * Actually count a circuit success towards a guard's usage counters - * if the path state is appropriate. - */ -static void -pathbias_count_use_success(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { - log_notice(LD_BUG, - "Successfully used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - guard->use_successes++; - entry_guards_changed(); - - if (guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " - "for guard %s=%s", - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used successfully for guard " - "%s ($%s).", - circ->global_identifier, guard->use_successes, - guard->use_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } - - return; -} - -/** - * Send a probe down a circuit that the client attempted to use, - * but for which the stream timed out/failed. The probe is a - * RELAY_BEGIN cell with a 0.a.b.c destination address, which - * the exit will reject and reply back, echoing that address. - * - * The reason for such probes is because it is possible to bias - * a user's paths simply by causing timeouts, and these timeouts - * are not possible to differentiate from unresponsive servers. - * - * The probe is sent at the end of the circuit lifetime for two - * reasons: to prevent cryptographic taggers from being able to - * drop cells to cause timeouts, and to prevent easy recognition - * of probes before any real client traffic happens. - * - * Returns -1 if we couldn't probe, 0 otherwise. - */ -static int -pathbias_send_usable_probe(circuit_t *circ) -{ - /* Based on connection_ap_handshake_send_begin() */ - char payload[CELL_PAYLOAD_SIZE]; - int payload_len; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - crypt_path_t *cpath_layer = NULL; - char *probe_nonce = NULL; - - tor_assert(ocirc); - - cpath_layer = ocirc->cpath->prev; - - if (cpath_layer->state != CPATH_STATE_OPEN) { - /* This can happen for cannibalized circuits. Their - * last hop isn't yet open */ - log_info(LD_CIRC, - "Got pathbias probe request for unopened circuit %d. " - "Opened %d, len %d", ocirc->global_identifier, - ocirc->has_opened, ocirc->build_state->desired_path_len); - return -1; - } - - /* We already went down this road. */ - if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && - ocirc->pathbias_probe_id) { - log_info(LD_CIRC, - "Got pathbias probe request for circuit %d with " - "outstanding probe", ocirc->global_identifier); - return -1; - } - - /* Can't probe if the channel isn't open */ - if (circ->n_chan == NULL || - (circ->n_chan->state != CHANNEL_STATE_OPEN - && circ->n_chan->state != CHANNEL_STATE_MAINT)) { - log_info(LD_CIRC, - "Skipping pathbias probe for circuit %d: Channel is not open.", - ocirc->global_identifier); - return -1; - } - - circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - /* Update timestamp for when circuit_expire_building() should kill us */ - tor_gettimeofday(&circ->timestamp_began); - - /* Generate a random address for the nonce */ - crypto_rand((char*)ô->pathbias_probe_nonce, - sizeof(ocirc->pathbias_probe_nonce)); - ocirc->pathbias_probe_nonce &= 0x00ffffff; - probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); - - tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); - payload_len = (int)strlen(payload)+1; - - // XXX: need this? Can we assume ipv4 will always be supported? - // If not, how do we tell? - //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { - // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); - // payload_len += 4; - //} - - /* Generate+Store stream id, make sure it's non-zero */ - ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); - - if (ocirc->pathbias_probe_id==0) { - log_warn(LD_CIRC, - "Ran out of stream IDs on circuit %u during " - "pathbias probe attempt.", ocirc->global_identifier); - tor_free(probe_nonce); - return -1; - } - - log_info(LD_CIRC, - "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", - probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); - tor_free(probe_nonce); - - /* Send a test relay cell */ - if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, - RELAY_COMMAND_BEGIN, payload, - payload_len, cpath_layer) < 0) { - log_notice(LD_CIRC, - "Failed to send pathbias probe cell on circuit %d.", - ocirc->global_identifier); - return -1; - } - - /* Mark it freshly dirty so it doesn't get expired in the meantime */ - circ->timestamp_dirty = time(NULL); - - return 0; -} - -/** - * Check the response to a pathbias probe, to ensure the - * cell is recognized and the nonce and other probe - * characteristics are as expected. - * - * If the response is valid, return 0. Otherwise return < 0. - */ -int -pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) -{ - /* Based on connection_edge_process_relay_cell() */ - relay_header_t rh; - int reason; - uint32_t ipv4_host; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - - tor_assert(cell); - tor_assert(ocirc); - tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - relay_header_unpack(&rh, cell->payload); - - reason = rh.length > 0 ? - get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; - - if (rh.command == RELAY_COMMAND_END && - reason == END_STREAM_REASON_EXITPOLICY && - ocirc->pathbias_probe_id == rh.stream_id) { - - /* Check length+extract host: It is in network order after the reason code. - * See connection_edge_end(). */ - if (rh.length < 9) { /* reason+ipv4+dns_ttl */ - log_notice(LD_PROTOCOL, - "Short path bias probe response length field (%d).", rh.length); - return - END_CIRC_REASON_TORPROTOCOL; - } - - ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); - - /* Check nonce */ - if (ipv4_host == ocirc->pathbias_probe_nonce) { - pathbias_mark_use_success(ocirc); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - log_info(LD_CIRC, - "Got valid path bias probe back for circ %d, stream %d.", - ocirc->global_identifier, ocirc->pathbias_probe_id); - return 0; - } else { - log_notice(LD_CIRC, - "Got strange probe value 0x%x vs 0x%x back for circ %d, " - "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, - ocirc->global_identifier, ocirc->pathbias_probe_id); - return -1; - } - } - log_info(LD_CIRC, - "Got another cell back back on pathbias probe circuit %d: " - "Command: %d, Reason: %d, Stream-id: %d", - ocirc->global_identifier, rh.command, reason, rh.stream_id); - return -1; -} - -/** - * Check if a circuit was used and/or closed successfully. - * - * If we attempted to use the circuit to carry a stream but failed - * for whatever reason, or if the circuit mysteriously died before - * we could attach any streams, record these two cases. - * - * If we *have* successfully used the circuit, or it appears to - * have been closed by us locally, count it as a success. - * - * Returns 0 if we're done making decisions with the circ, - * or -1 if we want to probe it first. - */ -int -pathbias_check_close(origin_circuit_t *ocirc, int reason) -{ - circuit_t *circ = ô->base_; - - if (!pathbias_should_count(ocirc)) { - return 0; - } - - switch (ocirc->path_state) { - /* If the circuit was closed after building, but before use, we need - * to ensure we were the ones who tried to close it (and not a remote - * actor). */ - case PATH_STATE_BUILD_SUCCEEDED: - if (reason & END_CIRC_REASON_FLAG_REMOTE) { - /* Remote circ close reasons on an unused circuit all could be bias */ - log_info(LD_CIRC, - "Circuit %d remote-closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) - == END_CIRC_REASON_CHANNEL_CLOSED && - circ->n_chan && - circ->n_chan->reason_for_closing - != CHANNEL_CLOSE_REQUESTED) { - /* If we didn't close the channel ourselves, it could be bias */ - /* XXX: Only count bias if the network is live? - * What about clock jumps/suspends? */ - log_info(LD_CIRC, - "Circuit %d's channel closed without successful use for reason " - "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " - "%d.", ocirc->global_identifier, - reason, circ->n_chan->reason_for_closing, - circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else { - pathbias_count_successful_close(ocirc); - } - break; - - /* If we tried to use a circuit but failed, we should probe it to ensure - * it has not been tampered with. */ - case PATH_STATE_USE_ATTEMPTED: - /* XXX: Only probe and/or count failure if the network is live? - * What about clock jumps/suspends? */ - if (pathbias_send_usable_probe(circ) == 0) - return -1; - else - pathbias_count_use_failed(ocirc); - - /* Any circuit where there were attempted streams but no successful - * streams could be bias */ - log_info(LD_CIRC, - "Circuit %d closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - break; - - case PATH_STATE_USE_SUCCEEDED: - pathbias_count_successful_close(ocirc); - pathbias_count_use_success(ocirc); - break; - - case PATH_STATE_USE_FAILED: - pathbias_count_use_failed(ocirc); - break; - - case PATH_STATE_NEW_CIRC: - case PATH_STATE_BUILD_ATTEMPTED: - case PATH_STATE_ALREADY_COUNTED: - default: - // Other states are uninteresting. No stats to count. - break; - } - - ocirc->path_state = PATH_STATE_ALREADY_COUNTED; - - return 0; -} - -/** - * Count a successfully closed circuit. - */ -static void -pathbias_count_successful_close(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - /* In the long run: circuit_success ~= successful_circuit_close + - * circ_failure + stream_failure */ - guard->successful_circuits_closed++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Successfully closed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a circuit that fails after it is built, but before it can - * carry any traffic. - * - * This is needed because there are ways to destroy a - * circuit after it has successfully completed. Right now, this is - * used for purely informational/debugging purposes. - */ -static void -pathbias_count_collapse(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->collapsed_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Destroyed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a known failed circuit (because we could not probe it). - * - * This counter is informational. - */ -static void -pathbias_count_use_failed(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->unusable_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - /* XXX note cut-and-paste code in this function compared to nearby - * functions. Would be nice to refactor. -RD */ - log_info(LD_CIRC, - "Stream-failing circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count timeouts for path bias log messages. - * - * These counts are purely informational. - */ -void -pathbias_count_timeout(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* For hidden service circs, they can actually be used - * successfully and then time out later (because - * the other side declines to use them). */ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->timeouts++; - entry_guards_changed(); - } -} - -/** - * Helper function to count all of the currently opened circuits - * for a guard that are in a given path state range. The state - * range is inclusive on both ends. - */ -static int -pathbias_count_circs_in_states(entry_guard_t *guard, - path_state_t from, - path_state_t to) -{ - circuit_t *circ; - int open_circuits = 0; - - /* Count currently open circuits. Give them the benefit of the doubt. */ - for (circ = global_circuitlist; circ; circ = circ->next) { - origin_circuit_t *ocirc = NULL; - if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ - circ->marked_for_close) /* already counted */ - continue; - - ocirc = TO_ORIGIN_CIRCUIT(circ); - - if (!ocirc->cpath || !ocirc->cpath->extend_info) - continue; - - if (ocirc->path_state >= from && - ocirc->path_state <= to && - pathbias_should_count(ocirc) && - fast_memeq(guard->identity, - ocirc->cpath->extend_info->identity_digest, - DIGEST_LEN)) { - log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", - ocirc->global_identifier, - pathbias_state_to_string(ocirc->path_state)); - open_circuits++; - } - } - - return open_circuits; -} - -/** - * Return the number of circuits counted as successfully closed for - * this guard. - * - * Also add in the currently open circuits to give them the benefit - * of the doubt. - */ -double -pathbias_get_close_success_count(entry_guard_t *guard) -{ - return guard->successful_circuits_closed + - pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Return the number of circuits counted as successfully used - * this guard. - * - * Also add in the currently open circuits that we are attempting - * to use to give them the benefit of the doubt. - */ -double -pathbias_get_use_success_count(entry_guard_t *guard) -{ - return guard->use_successes + - pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Check the path bias use rate against our consensus parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - */ -static void -pathbias_measure_use_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->use_attempts > pathbias_get_min_use(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_extreme_use_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of stream on its circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_use_extreme) { - guard->path_bias_use_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of streams on its circuits. " - "This could indicate a route manipulation attack, network " - "overload, bad local network connectivity, or a bug. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_notice_use_rate(options)) { - if (!guard->path_bias_use_noticed) { - guard->path_bias_use_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing to carry more streams on its " - "circuits than usual. " - "Most likely this means the Tor network is overloaded " - "or your network connection is poor. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * Check the path bias circuit close status rates against our consensus - * parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - * - * XXX: This function shares similar log messages and checks to - * pathbias_measure_use_rate(). It may be possible to combine them - * eventually, especially if we can ever remove the need for 3 - * levels of closure warns (if the overall circuit failure rate - * goes down with ntor). One way to do so would be to multiply - * the build rate with the use rate to get an idea of the total - * fraction of the total network paths the user is able to use. - * See ticket #8159. - */ -static void -pathbias_measure_close_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->circ_attempts > pathbias_get_min_circs(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_extreme_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_extreme) { - guard->path_bias_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "This could indicate a route manipulation attack, " - "extreme network overload, or a bug. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_warn_rate(options)) { - if (!guard->path_bias_warned) { - guard->path_bias_warned = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing a very large " - "amount of circuits. " - "Most likely this means the Tor network is " - "overloaded, but it could also mean an attack against " - "you or potentially the guard itself. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_notice_rate(options)) { - if (!guard->path_bias_noticed) { - guard->path_bias_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing more circuits than " - "usual. " - "Most likely this means the Tor network is overloaded. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * This function scales the path bias use rates if we have - * more data than the scaling threshold. This allows us to - * be more sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -static void -pathbias_scale_close_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); - int opened_built = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_FAILED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); - - guard->circ_attempts -= (opened_attempts+opened_built); - guard->circ_successes -= opened_built; - - guard->circ_attempts *= scale_ratio; - guard->circ_successes *= scale_ratio; - guard->timeouts *= scale_ratio; - guard->successful_circuits_closed *= scale_ratio; - guard->collapsed_circuits *= scale_ratio; - guard->unusable_circuits *= scale_ratio; - - guard->circ_attempts += (opened_attempts+opened_built); - guard->circ_successes += opened_built; - - entry_guards_changed(); - - log_info(LD_CIRC, - "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " - "%s ($%s)", - guard->circ_successes, guard->successful_circuits_closed, - guard->circ_attempts, opened_built, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, opened_built, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } -} - -/** - * This function scales the path bias circuit close rates if we have - * more data than the scaling threshold. This allows us to be more - * sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -void -pathbias_scale_use_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->use_attempts >= guard->use_successes); - - guard->use_attempts -= opened_attempts; - - guard->use_attempts *= scale_ratio; - guard->use_successes *= scale_ratio; - - guard->use_attempts += opened_attempts; - - log_info(LD_CIRC, - "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", - guard->use_successes, guard->use_attempts, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias usage counts to %f/%f " - "(%d open) for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - - entry_guards_changed(); - } -} - -/** Increment the number of times we successfully extended a circuit to - * <b>guard</b>, first checking if the failure rate is high enough that - * we should eliminate the guard. Return -1 if the guard looks no good; - * return 0 if the guard looks fine. - */ -static int -entry_guard_inc_circ_attempt_count(entry_guard_t *guard) -{ - entry_guards_changed(); - - pathbias_measure_close_rate(guard); - - if (guard->path_bias_disabled) - return -1; - - pathbias_scale_close_rates(guard); - guard->circ_attempts++; - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - return 0; -} - /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>. * (The body of <b>reply</b> varies depending on what sort of handshake * this is.) @@ -2828,11 +1345,7 @@ onionskin_answer(or_circuit_t *circ, * number of endpoints that would give something away about our destination. * * If the routerlist <b>nodes</b> doesn't have enough routers - * to handle the desired path length, return as large a path length as - * is feasible, except if it's less than 2, in which case return -1. - * XXX ^^ I think this behavior is a hold-over from back when we had only a - * few relays in the network, and certainly back before guards existed. - * We should very likely get rid of it. -RD + * to handle the desired path length, return -1. */ static int new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) @@ -2853,19 +1366,13 @@ new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", routelen, num_acceptable_routers, smartlist_len(nodes)); - if (num_acceptable_routers < 2) { + if (num_acceptable_routers < routelen) { log_info(LD_CIRC, - "Not enough acceptable routers (%d). Discarding this circuit.", - num_acceptable_routers); + "Not enough acceptable routers (%d/%d). Discarding this circuit.", + num_acceptable_routers, routelen); return -1; } - if (num_acceptable_routers < routelen) { - log_info(LD_CIRC,"Not enough routers: cutting routelen from %d to %d.", - routelen, num_acceptable_routers); - routelen = num_acceptable_routers; - } - return routelen; } diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index a3091707e..71caea94e 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -57,16 +57,10 @@ const char *build_state_get_exit_nickname(cpath_build_state_t *state); const node_t *choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state); -double pathbias_get_extreme_rate(const or_options_t *options); -double pathbias_get_extreme_use_rate(const or_options_t *options); -int pathbias_get_dropguards(const or_options_t *options); -void pathbias_count_timeout(origin_circuit_t *circ); -int pathbias_check_close(origin_circuit_t *circ, int reason); -int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); -void pathbias_count_use_attempt(origin_circuit_t *circ); -void pathbias_mark_use_success(origin_circuit_t *circ); -void pathbias_mark_use_rollback(origin_circuit_t *circ); -const char *pathbias_state_to_string(path_state_t state); + +#ifdef CIRCUITBUILD_PRIVATE +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); +#endif #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c7b15e40b..6238e08e1 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -8,9 +8,10 @@ * \file circuitlist.c * \brief Manage the global circuit list. **/ - +#define CIRCUITLIST_PRIVATE #include "or.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -31,20 +32,23 @@ #include "rephist.h" #include "routerlist.h" #include "routerset.h" + #include "ht.h" /********* START VARIABLES **********/ /** A global list of all circuits at this hop. */ -circuit_t *global_circuitlist=NULL; +struct global_circuitlist_s global_circuitlist = + TOR_LIST_HEAD_INITIALIZER(global_circuitlist); /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */ static smartlist_t *circuits_pending_chans = NULL; -static void circuit_free(circuit_t *circ); -static void circuit_free_cpath(crypt_path_t *cpath); static void circuit_free_cpath_node(crypt_path_t *victim); static void cpath_ref_decref(crypt_path_reference_t *cpath_ref); +//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, +// const uint8_t *token); +static void circuit_clear_rend_token(or_circuit_t *circ); /********* END VARIABLES ************/ @@ -72,7 +76,15 @@ chan_circid_entries_eq_(chan_circid_circuit_map_t *a, static INLINE unsigned int chan_circid_entry_hash_(chan_circid_circuit_map_t *a) { - return ((unsigned)a->circ_id) ^ (unsigned)(uintptr_t)(a->chan); + /* Try to squeze the siphash input into 8 bytes to save any extra siphash + * rounds. This hash function is in the critical path. */ + uintptr_t chan = (uintptr_t) (void*) a->chan; + uint32_t array[2]; + array[0] = a->circ_id; + /* The low bits of the channel pointer are uninteresting, since the channel + * is a pretty big structure. */ + array[1] = (uint32_t) (chan >> 6); + return (unsigned) siphash24g(array, sizeof(array)); } /** Map from [chan,circid] to circuit. */ @@ -207,18 +219,123 @@ circuit_set_circid_chan_helper(circuit_t *circ, int direction, } } +/** Mark that circuit id <b>id</b> shouldn't be used on channel <b>chan</b>, + * even if there is no circuit on the channel. We use this to keep the + * circuit id from getting re-used while we have queued but not yet sent + * a destroy cell. */ +void +channel_mark_circid_unusable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_FIND(chan_circid_map, &chan_circid_map, &search); + + if (ent && ent->circuit) { + /* we have a problem. */ + log_warn(LD_BUG, "Tried to mark %u unusable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + } else if (ent) { + /* It's already marked. */ + } else { + ent = tor_malloc_zero(sizeof(chan_circid_circuit_map_t)); + ent->chan = chan; + ent->circ_id = id; + /* leave circuit at NULL */ + HT_INSERT(chan_circid_map, &chan_circid_map, ent); + } +} + +/** Mark that a circuit id <b>id</b> can be used again on <b>chan</b>. + * We use this to re-enable the circuit ID after we've sent a destroy cell. + */ +void +channel_mark_circid_usable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_REMOVE(chan_circid_map, &chan_circid_map, &search); + if (ent && ent->circuit) { + log_warn(LD_BUG, "Tried to mark %u usable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + return; + } + if (_last_circid_chan_ent == ent) + _last_circid_chan_ent = NULL; + tor_free(ent); +} + +/** Called to indicate that a DESTROY is pending on <b>chan</b> with + * circuit ID <b>id</b>, but hasn't been sent yet. */ +void +channel_note_destroy_pending(channel_t *chan, circid_t id) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 1; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 1; + } + } + return; + } + channel_mark_circid_unusable(chan, id); +} + +/** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with + * circuit ID <b>id</b> -- typically, because it has been sent. */ +void +channel_note_destroy_not_pending(channel_t *chan, circid_t id) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 0; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 0; + } + } + /* XXXX this shouldn't happen; log a bug here. */ + return; + } + channel_mark_circid_usable(chan, id); +} + /** Set the p_conn field of a circuit <b>circ</b>, along * with the corresponding circuit ID, and add the circuit as appropriate * to the (chan,id)-\>circuit map. */ void -circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, +circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id, channel_t *chan) { - circuit_set_circid_chan_helper(TO_CIRCUIT(circ), CELL_DIRECTION_IN, - id, chan); + circuit_t *circ = TO_CIRCUIT(or_circ); + channel_t *old_chan = or_circ->p_chan; + circid_t old_id = or_circ->p_circ_id; + + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan); if (chan) - tor_assert(bool_eq(circ->p_chan_cells.n, circ->next_active_on_p_chan)); + tor_assert(bool_eq(or_circ->p_chan_cells.n, + or_circ->next_active_on_p_chan)); + + if (circ->p_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->p_delete_pending = 0; + } } /** Set the n_conn field of a circuit <b>circ</b>, along @@ -228,10 +345,18 @@ void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan) { + channel_t *old_chan = circ->n_chan; + circid_t old_id = circ->n_circ_id; + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan); if (chan) tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan)); + + if (circ->n_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->n_delete_pending = 0; + } } /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing @@ -257,21 +382,6 @@ circuit_set_state(circuit_t *circ, uint8_t state) circ->state = state; } -/** Add <b>circ</b> to the global list of circuits. This is called only from - * within circuit_new. - */ -static void -circuit_add(circuit_t *circ) -{ - if (!global_circuitlist) { /* first one */ - global_circuitlist = circ; - circ->next = NULL; - } else { - circ->next = global_circuitlist; - global_circuitlist = circ; - } -} - /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for * the given connection. */ void @@ -329,33 +439,17 @@ circuit_count_pending_on_channel(channel_t *chan) void circuit_close_all_marked(void) { - circuit_t *tmp,*m; - - while (global_circuitlist && global_circuitlist->marked_for_close) { - tmp = global_circuitlist->next; - circuit_free(global_circuitlist); - global_circuitlist = tmp; - } - - tmp = global_circuitlist; - while (tmp && tmp->next) { - if (tmp->next->marked_for_close) { - m = tmp->next->next; - circuit_free(tmp->next); - tmp->next = m; - /* Need to check new tmp->next; don't advance tmp. */ - } else { - /* Advance tmp. */ - tmp = tmp->next; - } - } + circuit_t *circ, *tmp; + TOR_LIST_FOREACH_SAFE(circ, &global_circuitlist, head, tmp) + if (circ->marked_for_close) + circuit_free(circ); } /** Return the head of the global linked list of circuits. */ -circuit_t * -circuit_get_global_list_(void) +MOCK_IMPL(struct global_circuitlist_s *, +circuit_get_global_list,(void)) { - return global_circuitlist; + return &global_circuitlist; } /** Function to make circ-\>state human-readable */ @@ -570,8 +664,9 @@ init_circuit_base(circuit_t *circ) circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + cell_queue_init(&circ->n_chan_cells); - circuit_add(circ); + TOR_LIST_INSERT_HEAD(&global_circuitlist, circ, head); } /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> @@ -595,7 +690,7 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); - circ_times.last_circ_at = approx_time(); + circuit_build_times_update_last_circ(get_circuit_build_times_mutable()); return circ; } @@ -615,6 +710,7 @@ or_circuit_new(circid_t p_circ_id, channel_t *p_chan) circuit_set_p_circid_chan(circ, p_circ_id, p_chan); circ->remaining_relay_early_cells = MAX_RELAY_EARLY_CELLS_PER_CIRCUIT; + cell_queue_init(&circ->p_chan_cells); init_circuit_base(TO_CIRCUIT(circ)); @@ -623,7 +719,7 @@ or_circuit_new(circid_t p_circ_id, channel_t *p_chan) /** Deallocate space associated with circ. */ -static void +STATIC void circuit_free(circuit_t *circ) { void *mem; @@ -643,7 +739,7 @@ circuit_free(circuit_t *circ) } tor_free(ocirc->build_state); - circuit_free_cpath(ocirc->cpath); + circuit_clear_cpath(ocirc); crypto_pk_free(ocirc->intro_key); rend_data_free(ocirc->rend_data); @@ -672,6 +768,8 @@ circuit_free(circuit_t *circ) crypto_cipher_free(ocirc->n_crypto); crypto_digest_free(ocirc->n_digest); + circuit_clear_rend_token(ocirc); + if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; tor_assert(other->base_.magic == OR_CIRCUIT_MAGIC); @@ -689,6 +787,8 @@ circuit_free(circuit_t *circ) extend_info_free(circ->n_hop); tor_free(circ->n_chan_create_cell); + TOR_LIST_REMOVE(circ, head); + /* Remove from map. */ circuit_set_n_circid_chan(circ, 0, NULL); @@ -700,11 +800,14 @@ circuit_free(circuit_t *circ) tor_free(mem); } -/** Deallocate space associated with the linked list <b>cpath</b>. */ -static void -circuit_free_cpath(crypt_path_t *cpath) +/** Deallocate the linked list circ-><b>cpath</b>, and remove the cpath from + * <b>circ</b>. */ +void +circuit_clear_cpath(origin_circuit_t *circ) { - crypt_path_t *victim, *head=cpath; + crypt_path_t *victim, *head, *cpath; + + head = cpath = circ->cpath; if (!cpath) return; @@ -718,13 +821,7 @@ circuit_free_cpath(crypt_path_t *cpath) } circuit_free_cpath_node(cpath); -} -/** Remove all the items in the cpath on <b>circ</b>.*/ -void -circuit_clear_cpath(origin_circuit_t *circ) -{ - circuit_free_cpath(circ->cpath); circ->cpath = NULL; } @@ -732,11 +829,11 @@ circuit_clear_cpath(origin_circuit_t *circ) void circuit_free_all(void) { - circuit_t *next; - while (global_circuitlist) { - next = global_circuitlist->next; - if (! CIRCUIT_IS_ORIGIN(global_circuitlist)) { - or_circuit_t *or_circ = TO_OR_CIRCUIT(global_circuitlist); + circuit_t *tmp, *tmp2; + + TOR_LIST_FOREACH_SAFE(tmp, &global_circuitlist, head, tmp2) { + if (! CIRCUIT_IS_ORIGIN(tmp)) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(tmp); while (or_circ->resolving_streams) { edge_connection_t *next_conn; next_conn = or_circ->resolving_streams->next_stream; @@ -744,13 +841,24 @@ circuit_free_all(void) or_circ->resolving_streams = next_conn; } } - circuit_free(global_circuitlist); - global_circuitlist = next; + circuit_free(tmp); } smartlist_free(circuits_pending_chans); circuits_pending_chans = NULL; + { + chan_circid_circuit_map_t **elt, **next, *c; + for (elt = HT_START(chan_circid_map, &chan_circid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(chan_circid_map, &chan_circid_map, elt); + + tor_assert(c->circuit == NULL); + tor_free(c); + } + } HT_CLEAR(chan_circid_map, &chan_circid_map); } @@ -815,7 +923,7 @@ circuit_dump_by_conn(connection_t *conn, int severity) circuit_t *circ; edge_connection_t *tmpconn; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; if (circ->marked_for_close) { @@ -848,79 +956,13 @@ circuit_dump_by_conn(connection_t *conn, int severity) } } -/** A helper function for circuit_dump_by_chan() below. Log a bunch - * of information about circuit <b>circ</b>. - */ -static void -circuit_dump_chan_details(int severity, - circuit_t *circ, - channel_t *chan, - const char *type, - circid_t this_circid, - circid_t other_circid) -{ - tor_log(severity, LD_CIRC, "Conn %p has %s circuit: circID %u " - "(other side %u), state %d (%s), born %ld:", - chan, type, (unsigned)this_circid, (unsigned)other_circid, circ->state, - circuit_state_to_string(circ->state), - (long)circ->timestamp_began.tv_sec); - if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ - circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); - } -} - -/** Log, at severity <b>severity</b>, information about each circuit - * that is connected to <b>chan</b>. - */ -void -circuit_dump_by_chan(channel_t *chan, int severity) -{ - circuit_t *circ; - - tor_assert(chan); - - for (circ = global_circuitlist; circ; circ = circ->next) { - circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; - - if (circ->marked_for_close) { - continue; - } - - if (!CIRCUIT_IS_ORIGIN(circ)) { - p_circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - if (! CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_chan && - TO_OR_CIRCUIT(circ)->p_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "App-ward", - p_circ_id, n_circ_id); - } - - if (circ->n_chan && circ->n_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "Exit-ward", - n_circ_id, p_circ_id); - } - - if (!circ->n_chan && circ->n_hop && - channel_matches_extend_info(chan, circ->n_hop) && - tor_memeq(chan->identity_digest, - circ->n_hop->identity_digest, DIGEST_LEN)) { - circuit_dump_chan_details(severity, circ, chan, - (circ->state == CIRCUIT_STATE_OPEN && - !CIRCUIT_IS_ORIGIN(circ)) ? - "Endpoint" : "Pending", - n_circ_id, p_circ_id); - } - } -} - /** Return the circuit whose global ID is <b>id</b>, or NULL if no * such circuit exists. */ origin_circuit_t * circuit_get_by_global_id(uint32_t id) { circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && TO_ORIGIN_CIRCUIT(circ)->global_identifier == id) { if (circ->marked_for_close) @@ -936,9 +978,13 @@ circuit_get_by_global_id(uint32_t id) * - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and * - circ is attached to <b>chan</b>, either as p_chan or n_chan. * Return NULL if no such circuit exists. + * + * If <b>found_entry_out</b> is provided, set it to true if we have a + * placeholder entry for circid/chan, and leave it unset otherwise. */ static INLINE circuit_t * -circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) +circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan, + int *found_entry_out) { chan_circid_circuit_map_t search; chan_circid_circuit_map_t *found; @@ -959,21 +1005,27 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) " circ_id %u, channel ID " U64_FORMAT " (%p)", found->circuit, (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = 1; return found->circuit; } log_debug(LD_CIRC, - "circuit_get_by_circid_channel_impl() found nothing for" + "circuit_get_by_circid_channel_impl() found %s for" " circ_id %u, channel ID " U64_FORMAT " (%p)", + found ? "placeholder" : "nothing", (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = found ? 1 : 0; + return NULL; /* The rest of this checks for bugs. Disabled by default. */ /* We comment it out because coverity complains otherwise. { circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->p_chan == chan && or_circ->p_circ_id == circ_id) { @@ -1001,7 +1053,7 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) circuit_t * circuit_get_by_circid_channel(circid_t circ_id, channel_t *chan) { - circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan); + circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan, NULL); if (!circ || circ->marked_for_close) return NULL; else @@ -1017,15 +1069,25 @@ circuit_t * circuit_get_by_circid_channel_even_if_marked(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan); + return circuit_get_by_circid_channel_impl(circ_id, chan, NULL); } /** Return true iff the circuit ID <b>circ_id</b> is currently used by a - * circuit, marked or not, on <b>chan</b>. */ + * circuit, marked or not, on <b>chan</b>, or if the circ ID is reserved until + * a queued destroy cell can be sent. + * + * (Return 1 if the circuit is present, marked or not; Return 2 + * if the circuit ID is pending a destroy.) + **/ int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan) != NULL; + int found = 0; + if (circuit_get_by_circid_channel_impl(circ_id, chan, &found) != NULL) + return 1; + if (found) + return 2; + return 0; } /** Return the circuit that a given edge connection is using. */ @@ -1049,13 +1111,59 @@ circuit_get_by_edge_conn(edge_connection_t *conn) void circuit_unlink_all_from_channel(channel_t *chan, int reason) { - circuit_t *circ; + smartlist_t *detached = smartlist_new(); + +/* #define DEBUG_CIRCUIT_UNLINK_ALL */ - channel_unlink_all_circuits(chan); + channel_unlink_all_circuits(chan, detached); - for (circ = global_circuitlist; circ; circ = circ->next) { +#ifdef DEBUG_CIRCUIT_UNLINK_ALL + { + circuit_t *circ; + smartlist_t *detached_2 = smartlist_new(); + int mismatch = 0, badlen = 0; + + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + if (circ->n_chan == chan || + (!CIRCUIT_IS_ORIGIN(circ) && + TO_OR_CIRCUIT(circ)->p_chan == chan)) { + smartlist_add(detached_2, circ); + } + } + + if (smartlist_len(detached) != smartlist_len(detached_2)) { + log_warn(LD_BUG, "List of detached circuits had the wrong length! " + "(got %d, should have gotten %d)", + (int)smartlist_len(detached), + (int)smartlist_len(detached_2)); + badlen = 1; + } + smartlist_sort_pointers(detached); + smartlist_sort_pointers(detached_2); + + SMARTLIST_FOREACH(detached, circuit_t *, c, + if (c != smartlist_get(detached_2, c_sl_idx)) + mismatch = 1; + ); + + if (mismatch) + log_warn(LD_BUG, "Mismatch in list of detached circuits."); + + if (badlen || mismatch) { + smartlist_free(detached); + detached = detached_2; + } else { + log_notice(LD_CIRC, "List of %d circuits was as expected.", + (int)smartlist_len(detached)); + smartlist_free(detached_2); + } + } +#endif + + SMARTLIST_FOREACH_BEGIN(detached, circuit_t *, circ) { int mark = 0; if (circ->n_chan == chan) { + circuit_set_n_circid_chan(circ, 0, NULL); mark = 1; @@ -1071,9 +1179,16 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) mark = 1; } } - if (mark && !circ->marked_for_close) + if (!mark) { + log_warn(LD_BUG, "Circuit on detached list which I had no reason " + "to mark"); + continue; + } + if (!circ->marked_for_close) circuit_mark_for_close(circ, reason); - } + } SMARTLIST_FOREACH_END(circ); + + smartlist_free(detached); } /** Return a circ such that @@ -1089,8 +1204,7 @@ origin_circuit_t * circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) { circuit_t *circ; - - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (!circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1118,11 +1232,11 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, circuit_t *circ; tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose)); if (start == NULL) - circ = global_circuitlist; + circ = TOR_LIST_FIRST(&global_circuitlist); else - circ = TO_CIRCUIT(start)->next; + circ = TOR_LIST_NEXT(TO_CIRCUIT(start), head); - for ( ; circ; circ = circ->next) { + for ( ; circ; circ = TOR_LIST_NEXT(circ, head)) { if (circ->marked_for_close) continue; if (circ->purpose != purpose) @@ -1137,43 +1251,175 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, return NULL; } -/** Return the first OR circuit in the global list whose purpose is - * <b>purpose</b>, and whose rend_token is the <b>len</b>-byte - * <b>token</b>. */ +/** Map from rendezvous cookie to or_circuit_t */ +static digestmap_t *rend_cookie_map = NULL; + +/** Map from introduction point digest to or_circuit_t */ +static digestmap_t *intro_digest_map = NULL; + +/** Return the OR circuit whose purpose is <b>purpose</b>, and whose + * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>, + * look for rendezvous point circuits; otherwise look for introduction point + * circuits. */ static or_circuit_t * -circuit_get_by_rend_token_and_purpose(uint8_t purpose, const char *token, - size_t len) +circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ, + const char *token) { - circuit_t *circ; - for (circ = global_circuitlist; circ; circ = circ->next) { - if (! circ->marked_for_close && - circ->purpose == purpose && - tor_memeq(TO_OR_CIRCUIT(circ)->rend_token, token, len)) - return TO_OR_CIRCUIT(circ); + or_circuit_t *circ; + digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) + return NULL; + + circ = digestmap_get(map, token); + if (!circ || + circ->base_.purpose != purpose || + circ->base_.marked_for_close) + return NULL; + + if (!circ->rendinfo) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a " + "circuit with no rendinfo set.", + safe_str(t), is_rend_circ); + tor_free(t); + return NULL; } - return NULL; + + if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) || + tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d", + safe_str(t), is_rend_circ, + safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)), + (int)circ->rendinfo->is_rend_circ); + tor_free(t); + return NULL; + } + + return circ; +} + +/** Clear the rendezvous cookie or introduction point key digest that's + * configured on <b>circ</b>, if any, and remove it from any such maps. */ +static void +circuit_clear_rend_token(or_circuit_t *circ) +{ + or_circuit_t *found_circ; + digestmap_t *map; + + if (!circ || !circ->rendinfo) + return; + + map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map"); + return; + } + + found_circ = digestmap_get(map, circ->rendinfo->rend_token); + if (found_circ == circ) { + /* Great, this is the right one. */ + digestmap_remove(map, circ->rendinfo->rend_token); + } else if (found_circ) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it was already replaced in the map."); + } else { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it not in the map at all."); + } + + tor_free(circ->rendinfo); /* Sets it to NULL too */ +} + +/** Set the rendezvous cookie (if is_rend_circ), or the introduction point + * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value + * in <b>token</b>, and add it to the appropriate map. If it previously had a + * token, clear it. If another circuit previously had the same + * cookie/intro-digest, mark that circuit and remove it from the map. */ +static void +circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, + const uint8_t *token) +{ + digestmap_t **map_p, *map; + or_circuit_t *found_circ; + + /* Find the right map, creating it as needed */ + map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map; + + if (!*map_p) + *map_p = digestmap_new(); + + map = *map_p; + + /* If this circuit already has a token, we need to remove that. */ + if (circ->rendinfo) + circuit_clear_rend_token(circ); + + if (token == NULL) { + /* We were only trying to remove this token, not set a new one. */ + return; + } + + found_circ = digestmap_get(map, (const char *)token); + if (found_circ) { + tor_assert(found_circ != circ); + circuit_clear_rend_token(found_circ); + if (! found_circ->base_.marked_for_close) { + circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED); + if (is_rend_circ) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Duplicate rendezvous cookie (%s...) used on two circuits", + hex_str((const char*)token, 4)); /* only log first 4 chars */ + } + } + } + + /* Now set up the rendinfo */ + circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo)); + memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN); + circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0; + + digestmap_set(map, (const char *)token, circ); } /** Return the circuit waiting for a rendezvous with the provided cookie. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_rendezvous(const char *cookie) +circuit_get_rendezvous(const uint8_t *cookie) { return circuit_get_by_rend_token_and_purpose( CIRCUIT_PURPOSE_REND_POINT_WAITING, - cookie, REND_COOKIE_LEN); + 1, (const char*)cookie); } /** Return the circuit waiting for intro cells of the given digest. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_intro_point(const char *digest) +circuit_get_intro_point(const uint8_t *digest) { return circuit_get_by_rend_token_and_purpose( - CIRCUIT_PURPOSE_INTRO_POINT, digest, - DIGEST_LEN); + CIRCUIT_PURPOSE_INTRO_POINT, 0, + (const char *)digest); +} + +/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that cookie, mark it. */ +void +circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie) +{ + circuit_set_rend_token(circ, 1, cookie); +} + +/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that intro point digest, mark it. */ +void +circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest) +{ + circuit_set_rend_token(circ, 0, digest); } /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL, @@ -1207,7 +1453,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, "capacity %d, internal %d", purpose, need_uptime, need_capacity, internal); - for (circ_=global_circuitlist; circ_; circ_ = circ_->next) { + TOR_LIST_FOREACH(circ_, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ_) && circ_->state == CIRCUIT_STATE_OPEN && !circ_->marked_for_close && @@ -1297,8 +1543,7 @@ void circuit_mark_all_unused_circs(void) { circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && !circ->timestamp_dirty) @@ -1317,8 +1562,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void) { circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) { @@ -1344,9 +1588,9 @@ circuit_mark_all_dirty_circs_as_unusable(void) * - If circ->rend_splice is set (we are the midpoint of a joined * rendezvous stream), then mark the other circuit to close as well. */ -void -circuit_mark_for_close_(circuit_t *circ, int reason, int line, - const char *file) +MOCK_IMPL(void, +circuit_mark_for_close_, (circuit_t *circ, int reason, int line, + const char *file)) { int orig_reason = reason; /* Passed to the controller */ assert_circuit_ok(circ); @@ -1450,6 +1694,7 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, channel_send_destroy(circ->n_circ_id, circ->n_chan, reason); } circuitmux_detach_circuit(circ->n_chan->cmux, circ); + circuit_set_n_circid_chan(circ, 0, NULL); } if (! CIRCUIT_IS_ORIGIN(circ)) { @@ -1483,6 +1728,7 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, channel_send_destroy(or_circ->p_circ_id, or_circ->p_chan, reason); } circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); + circuit_set_p_circid_chan(or_circ, 0, NULL); } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1521,8 +1767,40 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } -/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at <b>stream</b>. Return the number of bytes recovered. */ static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit <b>c</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + +/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ +STATIC size_t n_cells_in_circ_queues(const circuit_t *c) { size_t n = c->n_chan_cells.n; @@ -1541,17 +1819,19 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; - if (c->n_chan_cells.head) - age = now - c->n_chan_cells.head->inserted_time; + packed_cell_t *cell; + + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&c->n_chan_cells.head))) + age = now - cell->inserted_time; if (! CIRCUIT_IS_ORIGIN(c)) { - const or_circuit_t *orcirc = TO_OR_CIRCUIT((circuit_t*)c); - if (orcirc->p_chan_cells.head) { - uint32_t age2 = now - orcirc->p_chan_cells.head->inserted_time; + const or_circuit_t *orcirc = CONST_TO_OR_CIRCUIT(c); + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&orcirc->p_chan_cells.head))) { + uint32_t age2 = now - cell->inserted_time; if (age2 > age) return age2; } @@ -1559,20 +1839,68 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list <b>stream</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + } + + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit <b>c</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + CONST_TO_ORIGIN_CIRCUIT(c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + CONST_TO_OR_CIRCUIT(c)->n_streams, now); + } +} -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in + * truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; if (age_a < age_b) return 1; @@ -1582,67 +1910,90 @@ circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); + + { + const size_t recovered = buf_shrink_freelists(1); + if (recovered >= current_allocation) { + log_warn(LD_BUG, "We somehow recovered more memory from freelists " + "than we thought we had allocated"); + current_allocation = 0; + } else { + current_allocation -= recovered; + } + } { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } + tor_gettimeofday_cached_monotonic(&now); + now_ms = (uint32_t)tv_to_msec(&now); + /* This algorithm itself assumes that you've got enough memory slack * to actually run it. */ - for (circ = global_circuitlist; circ; circ = circ->next) + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); smartlist_add(circlist, circ); - - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + } /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); /* Okay, now the worst circuits are at the front of the list. Let's mark * them, and reclaim their storage aggressively. */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { size_t n = n_cells_in_circ_queues(circ); + size_t freed; if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) + + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) break; } SMARTLIST_FOREACH_END(circ); +#ifdef ENABLE_MEMPOOLS clean_cell_pool(); /* In case this helps. */ +#endif /* ENABLE_MEMPOOLS */ + buf_shrink_freelists(1); /* This is necessary to actually release buffer + chunks. */ - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), - n_circuits_killed); + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive.", + U64_PRINTF_ARG(mem_recovered), + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed); smartlist_free(circlist); } @@ -1716,15 +2067,10 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c->purpose >= CIRCUIT_PURPOSE_MIN_ && c->purpose <= CIRCUIT_PURPOSE_MAX_); - { - /* Having a separate variable for this pleases GCC 4.2 in ways I hope I - * never understand. -NM. */ - circuit_t *nonconst_circ = (circuit_t*) c; - if (CIRCUIT_IS_ORIGIN(c)) - origin_circ = TO_ORIGIN_CIRCUIT(nonconst_circ); - else - or_circ = TO_OR_CIRCUIT(nonconst_circ); - } + if (CIRCUIT_IS_ORIGIN(c)) + origin_circ = CONST_TO_ORIGIN_CIRCUIT(c); + else + or_circ = CONST_TO_OR_CIRCUIT(c); if (c->n_chan) { tor_assert(!c->n_hop); @@ -1733,15 +2079,16 @@ assert_circuit_ok(const circuit_t *c) /* We use the _impl variant here to make sure we don't fail on marked * circuits, which would not be returned by the regular function. */ circuit_t *c2 = circuit_get_by_circid_channel_impl(c->n_circ_id, - c->n_chan); + c->n_chan, NULL); tor_assert(c == c2); } } if (or_circ && or_circ->p_chan) { if (or_circ->p_circ_id) { /* ibid */ - circuit_t *c2 = circuit_get_by_circid_channel_impl(or_circ->p_circ_id, - or_circ->p_chan); + circuit_t *c2 = + circuit_get_by_circid_channel_impl(or_circ->p_circ_id, + or_circ->p_chan, NULL); tor_assert(c == c2); } } diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index acc4b81fc..916afba21 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -12,17 +12,22 @@ #ifndef TOR_CIRCUITLIST_H #define TOR_CIRCUITLIST_H -circuit_t * circuit_get_global_list_(void); +#include "testsupport.h" + +TOR_LIST_HEAD(global_circuitlist_s, circuit_t); + +MOCK_DECL(struct global_circuitlist_s*, circuit_get_global_list, (void)); const char *circuit_state_to_string(int state); const char *circuit_purpose_to_controller_string(uint8_t purpose); const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose); const char *circuit_purpose_to_string(uint8_t purpose); void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_dump_by_chan(channel_t *chan, int severity); void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, channel_t *chan); void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan); +void channel_mark_circid_unusable(channel_t *chan, circid_t id); +void channel_mark_circid_usable(channel_t *chan, circid_t id); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); @@ -41,14 +46,16 @@ origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data( const rend_data_t *rend_data); origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); +or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie); +or_circuit_t *circuit_get_intro_point(const uint8_t *digest); +void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie); +void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest); origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags); void circuit_mark_all_unused_circs(void); void circuit_mark_all_dirty_circs_as_unusable(void); -void circuit_mark_for_close_(circuit_t *circ, int reason, - int line, const char *file); +MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, + int line, const char *file)); int circuit_get_cpath_len(origin_circuit_t *circ); void circuit_clear_cpath(origin_circuit_t *circ); crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); @@ -64,5 +71,16 @@ void assert_circuit_ok(const circuit_t *c); void circuit_free_all(void); void circuits_handle_oom(size_t current_allocation); +void channel_note_destroy_pending(channel_t *chan, circid_t id); +void channel_note_destroy_not_pending(channel_t *chan, circid_t id); + +#ifdef CIRCUITLIST_PRIVATE +STATIC void circuit_free(circuit_t *circ); +STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); +#endif + #endif diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index 545cfd065..52ebfef08 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -10,6 +10,7 @@ #include "channel.h" #include "circuitlist.h" #include "circuitmux.h" +#include "relay.h" /* * Private typedefs for circuitmux.c @@ -115,6 +116,22 @@ struct circuitmux_s { */ struct circuit_t *active_circuits_head, *active_circuits_tail; + /** List of queued destroy cells */ + cell_queue_t destroy_cell_queue; + /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit + * returned the destroy queue. Used to force alternation between + * destroy/non-destroy cells. + * + * XXXX There is no reason to think that alternating is a particularly good + * approach -- it's just designed to prevent destroys from starving other + * cells completely. + */ + unsigned int last_cell_was_destroy : 1; + /** Destroy counter: increment this when a destroy gets queued, decrement + * when we unqueue it, so we can test to make sure they don't starve. + */ + int64_t destroy_ctr; + /* * Circuitmux policy; if this is non-NULL, it can override the built- * in round-robin active circuits behavior. This is how EWMA works in @@ -193,6 +210,11 @@ static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); +/* Static global variables */ + +/** Count the destroy balance to debug destroy queue logic */ +static int64_t global_destroy_ctr = 0; + /* Function definitions */ /** @@ -361,16 +383,20 @@ circuitmux_alloc(void) rv = tor_malloc_zero(sizeof(*rv)); rv->chanid_circid_map = tor_malloc_zero(sizeof(*( rv->chanid_circid_map))); HT_INIT(chanid_circid_muxinfo_map, rv->chanid_circid_map); + cell_queue_init(&rv->destroy_cell_queue); return rv; } /** * Detach all circuits from a circuitmux (use before circuitmux_free()) + * + * If <b>detached_out</b> is non-NULL, add every detached circuit_t to + * detached_out. */ void -circuitmux_detach_all_circuits(circuitmux_t *cmux) +circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) { chanid_circid_muxinfo_t **i = NULL, *to_remove; channel_t *chan = NULL; @@ -386,7 +412,11 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); while (i) { to_remove = *i; - if (to_remove) { + + if (! to_remove) { + log_warn(LD_BUG, "Somehow, an HT iterator gave us a NULL pointer."); + break; + } else { /* Find a channel and circuit */ chan = channel_find_by_global_id(to_remove->chan_id); if (chan) { @@ -407,6 +437,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) /* Clear n_mux */ circ->n_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else if (circ->magic == OR_CIRCUIT_MAGIC) { /* * Update active_circuits et al.; this does policy notifies, so @@ -422,6 +455,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) * so clear p_mux. */ TO_OR_CIRCUIT(circ)->p_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else { /* Complain and move on */ log_warn(LD_CIRC, @@ -476,6 +512,31 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) cmux->n_cells = 0; } +/** Reclaim all circuit IDs currently marked as unusable on <b>chan</b> because + * of pending destroy cells in <b>cmux</b>. + * + * This function must be called AFTER circuits are unlinked from the (channel, + * circuid-id) map with circuit_unlink_all_from_channel(), but before calling + * circuitmux_free(). + */ +void +circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan) +{ + packed_cell_t *cell; + int n_bad = 0; + TOR_SIMPLEQ_FOREACH(cell, &cmux->destroy_cell_queue.head, next) { + circid_t circid = 0; + if (packed_cell_is_destroy(chan, cell, &circid)) { + channel_mark_circid_usable(chan, circid); + } else { + ++n_bad; + } + } + if (n_bad) + log_warn(LD_BUG, "%d cell(s) on destroy queue did not look like a " + "DESTROY cell.", n_bad); +} + /** * Free a circuitmux_t; the circuits must be detached first with * circuitmux_detach_all_circuits(). @@ -508,6 +569,30 @@ circuitmux_free(circuitmux_t *cmux) tor_free(cmux->chanid_circid_map); } + /* + * We're throwing away some destroys; log the counter and + * adjust the global counter by the queue size. + */ + if (cmux->destroy_cell_queue.n > 0) { + cmux->destroy_ctr -= cmux->destroy_cell_queue.n; + global_destroy_ctr -= cmux->destroy_cell_queue.n; + log_debug(LD_CIRC, + "Freeing cmux at %p with %u queued destroys; the last cmux " + "destroy balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, cmux->destroy_cell_queue.n, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } else { + log_debug(LD_CIRC, + "Freeing cmux at %p with no queued destroys, the cmux destroy " + "balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } + + cell_queue_clear(&cmux->destroy_cell_queue); + tor_free(cmux); } @@ -816,7 +901,7 @@ circuitmux_num_cells(circuitmux_t *cmux) { tor_assert(cmux); - return cmux->n_cells; + return cmux->n_cells + cmux->destroy_cell_queue.n; } /** @@ -851,9 +936,9 @@ circuitmux_num_circuits(circuitmux_t *cmux) * Attach a circuit to a circuitmux, for the specified direction. */ -void -circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +MOCK_IMPL(void, +circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction)) { channel_t *chan = NULL; uint64_t channel_id; @@ -1000,8 +1085,8 @@ circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, * no-op if not attached. */ -void -circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ) +MOCK_IMPL(void, +circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) { chanid_circid_muxinfo_t search, *hashent = NULL; /* @@ -1368,16 +1453,36 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, /** * Pick a circuit to send from, using the active circuits list or a * circuitmux policy if one is available. This is called from channel.c. + * + * If we would rather send a destroy cell, return NULL and set + * *<b>destroy_queue_out</b> to the destroy queue. + * + * If we have nothing to send, set *<b>destroy_queue_out</b> to NULL and + * return NULL. */ circuit_t * -circuitmux_get_first_active_circuit(circuitmux_t *cmux) +circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out) { circuit_t *circ = NULL; tor_assert(cmux); + tor_assert(destroy_queue_out); + + *destroy_queue_out = NULL; - if (cmux->n_active_circuits > 0) { + if (cmux->destroy_cell_queue.n && + (!cmux->last_cell_was_destroy || cmux->n_active_circuits == 0)) { + /* We have destroy cells to send, and either we just sent a relay cell, + * or we have no relay cells to send. */ + + /* XXXX We should let the cmux policy have some say in this eventually. */ + /* XXXX Alternating is not a terribly brilliant approach here. */ + *destroy_queue_out = &cmux->destroy_cell_queue; + + cmux->last_cell_was_destroy = 1; + } else if (cmux->n_active_circuits > 0) { /* We also must have a cell available for this to be the case */ tor_assert(cmux->n_cells > 0); /* Do we have a policy-provided circuit selector? */ @@ -1389,7 +1494,11 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux) tor_assert(cmux->active_circuits_head); circ = cmux->active_circuits_head; } - } else tor_assert(cmux->n_cells == 0); + cmux->last_cell_was_destroy = 0; + } else { + tor_assert(cmux->n_cells == 0); + tor_assert(cmux->destroy_cell_queue.n == 0); + } return circ; } @@ -1463,6 +1572,26 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, circuitmux_assert_okay_paranoid(cmux); } +/** + * Notify the circuitmux that a destroy was sent, so we can update + * the counter. + */ + +void +circuitmux_notify_xmit_destroy(circuitmux_t *cmux) +{ + tor_assert(cmux); + + --(cmux->destroy_ctr); + --(global_destroy_ctr); + log_debug(LD_CIRC, + "Cmux at %p sent a destroy, cmux counter is now "I64_FORMAT", " + "global counter is now "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); +} + /* * Circuitmux consistency checking assertions */ @@ -1743,3 +1872,40 @@ circuitmux_assert_okay_pass_three(circuitmux_t *cmux) } } +/*DOCDOC */ +void +circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, + circid_t circ_id, + uint8_t reason) +{ + cell_t cell; + memset(&cell, 0, sizeof(cell_t)); + cell.circ_id = circ_id; + cell.command = CELL_DESTROY; + cell.payload[0] = (uint8_t) reason; + + cell_queue_append_packed_copy(NULL, &cmux->destroy_cell_queue, 0, &cell, + chan->wide_circ_ids, 0); + + /* Destroy entering the queue, update counters */ + ++(cmux->destroy_ctr); + ++global_destroy_ctr; + log_debug(LD_CIRC, + "Cmux at %p queued a destroy for circ %u, cmux counter is now " + I64_FORMAT", global counter is now "I64_FORMAT, + cmux, circ_id, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + + /* XXXX Duplicate code from append_cell_to_circuit_queue */ + if (!channel_has_queued_writes(chan)) { + /* There is no data at all waiting to be sent on the outbuf. Add a + * cell, so that we can notice when it gets flushed, flushed_some can + * get called, and we can start putting more data onto the buffer then. + */ + log_debug(LD_GENERAL, "Primed a buffer."); + channel_flush_from_first_active_circuit(chan, 1); + } +} + diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 25644ffab..c4c0649c6 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -10,6 +10,7 @@ #define TOR_CIRCUITMUX_H #include "or.h" +#include "testsupport.h" typedef struct circuitmux_policy_s circuitmux_policy_t; typedef struct circuitmux_policy_data_s circuitmux_policy_data_t; @@ -98,7 +99,8 @@ void circuitmux_assert_okay(circuitmux_t *cmux); /* Create/destroy */ circuitmux_t * circuitmux_alloc(void); -void circuitmux_detach_all_circuits(circuitmux_t *cmux); +void circuitmux_detach_all_circuits(circuitmux_t *cmux, + smartlist_t *detached_out); void circuitmux_free(circuitmux_t *cmux); /* Policy control */ @@ -120,17 +122,27 @@ unsigned int circuitmux_num_circuits(circuitmux_t *cmux); unsigned int circuitmux_num_active_circuits(circuitmux_t *cmux); /* Channel interface */ -circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux); +circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out); void circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_notify_xmit_destroy(circuitmux_t *cmux); /* Circuit interface */ -void circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -void circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ); +MOCK_DECL(void, circuitmux_attach_circuit, (circuitmux_t *cmux, + circuit_t *circ, + cell_direction_t direction)); +MOCK_DECL(void, circuitmux_detach_circuit, + (circuitmux_t *cmux, circuit_t *circ)); void circuitmux_clear_num_cells(circuitmux_t *cmux, circuit_t *circ); void circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, circid_t circ_id, + uint8_t reason); +void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, + channel_t *chan); + #endif /* TOR_CIRCUITMUX_H */ diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index 1d7812bf2..e362b1b49 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -12,12 +12,17 @@ #include "config.h" #include "confparse.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "statefile.h" #undef log #include <math.h> +static void cbt_control_event_buildtimeout_set( + const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + #define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /** Global list of circuit build times */ @@ -26,12 +31,46 @@ // vary in their own latency. The downside of this is that guards // can change frequently, so we'd be building a lot more circuits // most likely. -/* XXXX024 Make this static; add accessor functions. */ -circuit_build_times_t circ_times; +static circuit_build_times_t circ_times; +#ifdef TOR_UNIT_TESTS /** If set, we're running the unit tests: we should avoid clobbering * our state file or accessing get_options() or get_or_state() */ static int unit_tests = 0; +#else +#define unit_tests 0 +#endif + +/** Return a pointer to the data structure describing our current circuit + * build time history and computations. */ +const circuit_build_times_t * +get_circuit_build_times(void) +{ + return &circ_times; +} + +/** As get_circuit_build_times, but return a mutable pointer. */ +circuit_build_times_t * +get_circuit_build_times_mutable(void) +{ + return &circ_times; +} + +/** Return the time to wait before actually closing an under-construction, in + * milliseconds. */ +double +get_circuit_build_close_time_ms(void) +{ + return circ_times.close_ms; +} + +/** Return the time to wait before giving up on an under-construction circuit, + * in milliseconds. */ +double +get_circuit_build_timeout_ms(void) +{ + return circ_times.timeout_ms; +} /** * This function decides if CBT learning should be disabled. It returns @@ -56,18 +95,22 @@ circuit_build_times_disabled(void) if (consensus_disabled || config_disabled || dirauth_disabled || state_disabled) { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 1; } else { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is not disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 0; } } @@ -154,7 +197,7 @@ circuit_build_times_min_circs_to_observe(void) /** Return true iff <b>cbt</b> has recorded enough build times that we * want to start acting on the timeout it implies. */ int -circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt) { return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); } @@ -438,7 +481,7 @@ circuit_build_times_get_initial_timeout(void) * Leave estimated parameters, timeout and network liveness intact * for future use. */ -void +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt) { memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); @@ -471,7 +514,7 @@ circuit_build_times_init(circuit_build_times_t *cbt) cbt->liveness.timeouts_after_firsthop = NULL; } cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); } /** @@ -557,7 +600,7 @@ circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) * Return maximum circuit build time */ static build_time_t -circuit_build_times_max(circuit_build_times_t *cbt) +circuit_build_times_max(const circuit_build_times_t *cbt) { int i = 0; build_time_t max_build_time = 0; @@ -598,7 +641,7 @@ circuit_build_times_min(circuit_build_times_t *cbt) * The return value must be freed by the caller. */ static uint32_t * -circuit_build_times_create_histogram(circuit_build_times_t *cbt, +circuit_build_times_create_histogram(const circuit_build_times_t *cbt, build_time_t *nbins) { uint32_t *histogram; @@ -688,7 +731,7 @@ circuit_build_times_get_xm(circuit_build_times_t *cbt) * the or_state_t state structure. */ void -circuit_build_times_update_state(circuit_build_times_t *cbt, +circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state) { uint32_t *histogram; @@ -949,7 +992,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, * an acceptable approximation because we are only concerned with the * accuracy of the CDF of the tail. */ -int +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt) { build_time_t *x=cbt->circuit_build_times; @@ -1033,7 +1076,7 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt) * * Return value is in milliseconds. */ -double +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile) { @@ -1050,6 +1093,7 @@ circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, return ret; } +#ifdef TOR_UNIT_TESTS /** Pareto CDF */ double circuit_build_times_cdf(circuit_build_times_t *cbt, double x) @@ -1060,7 +1104,9 @@ circuit_build_times_cdf(circuit_build_times_t *cbt, double x) tor_assert(0 <= ret && ret <= 1.0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Generate a synthetic time using our distribution parameters. * @@ -1093,7 +1139,9 @@ circuit_build_times_generate_sample(circuit_build_times_t *cbt, tor_assert(ret > 0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Estimate an initial alpha parameter by solving the quantile * function with a quantile point and a specific timeout value. @@ -1114,12 +1162,13 @@ circuit_build_times_initial_alpha(circuit_build_times_t *cbt, (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); tor_assert(cbt->alpha > 0); } +#endif /** * Returns true if we need circuits to be built */ int -circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits(const circuit_build_times_t *cbt) { /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ return !circuit_build_times_enough_to_compute(cbt); @@ -1130,13 +1179,19 @@ circuit_build_times_needs_circuits(circuit_build_times_t *cbt) * right now. */ int -circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt) { return circuit_build_times_needs_circuits(cbt) && approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); } /** + * How long should we be unreachable before we think we need to check if + * our published IP address has changed. + */ +#define CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP (60*3) + +/** * Called to indicate that the network showed some signs of liveness, * i.e. we received a cell. * @@ -1151,12 +1206,15 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt) { time_t now = approx_time(); if (cbt->liveness.nonlive_timeouts > 0) { + time_t time_since_live = now - cbt->liveness.network_last_live; log_notice(LD_CIRC, "Tor now sees network activity. Restoring circuit build " "timeout recording. Network was down for %d seconds " "during %d circuit attempts.", - (int)(now - cbt->liveness.network_last_live), + (int)time_since_live, cbt->liveness.nonlive_timeouts); + if (time_since_live > CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP) + reschedule_descriptor_update_check(); } cbt->liveness.network_last_live = now; cbt->liveness.nonlive_timeouts = 0; @@ -1263,7 +1321,7 @@ circuit_build_times_network_close(circuit_build_times_t *cbt, * in the case of recent liveness changes. */ int -circuit_build_times_network_check_live(circuit_build_times_t *cbt) +circuit_build_times_network_check_live(const circuit_build_times_t *cbt) { if (cbt->liveness.nonlive_timeouts > 0) { return 0; @@ -1282,7 +1340,7 @@ circuit_build_times_network_check_live(circuit_build_times_t *cbt) * to restart the process of building test circuits and estimating a * new timeout. */ -int +STATIC int circuit_build_times_network_check_changed(circuit_build_times_t *cbt) { int total_build_times = cbt->total_build_times; @@ -1329,7 +1387,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) = circuit_build_times_get_initial_timeout(); } - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); log_notice(LD_CIRC, "Your network connection speed appears to have changed. Resetting " @@ -1511,7 +1569,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) } } - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); timeout_rate = circuit_build_times_timeout_rate(cbt); @@ -1546,6 +1604,8 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) cbt->total_build_times); } } + +#ifdef TOR_UNIT_TESTS /** Make a note that we're running unit tests (rather than running Tor * itself), so we avoid clobbering our state file. */ void @@ -1553,4 +1613,46 @@ circuitbuild_running_unit_tests(void) { unit_tests = 1; } +#endif + +void +circuit_build_times_update_last_circ(circuit_build_times_t *cbt) +{ + cbt->last_circ_at = approx_time(); +} + +static void +cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + char *args = NULL; + double qnt; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_RESET: + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + case BUILDTIMEOUT_SET_EVENT_DISCARD: + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + case BUILDTIMEOUT_SET_EVENT_RESUME: + default: + qnt = circuit_build_times_quantile_cutoff(); + break; + } + + tor_asprintf(&args, "TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " + "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f", + (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + control_event_buildtimeout_set(type, args); + + tor_free(args); +} diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 87dce99f4..3343310b8 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -12,11 +12,14 @@ #ifndef TOR_CIRCUITSTATS_H #define TOR_CIRCUITSTATS_H -extern circuit_build_times_t circ_times; +const circuit_build_times_t *get_circuit_build_times(void); +circuit_build_times_t *get_circuit_build_times_mutable(void); +double get_circuit_build_close_time_ms(void); +double get_circuit_build_timeout_ms(void); int circuit_build_times_disabled(void); -int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); -void circuit_build_times_update_state(circuit_build_times_t *cbt, +int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt); +void circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state); int circuit_build_times_parse_state(circuit_build_times_t *cbt, or_state_t *state); @@ -27,9 +30,9 @@ int circuit_build_times_count_close(circuit_build_times_t *cbt, void circuit_build_times_set_timeout(circuit_build_times_t *cbt); int circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time); -int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt); -int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt); void circuit_build_times_init(circuit_build_times_t *cbt); void circuit_build_times_free_timeouts(circuit_build_times_t *cbt); void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, @@ -37,29 +40,59 @@ void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); double circuit_build_times_close_rate(const circuit_build_times_t *cbt); +void circuit_build_times_update_last_circ(circuit_build_times_t *cbt); + #ifdef CIRCUITSTATS_PRIVATE -double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile); +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +STATIC int circuit_build_times_network_check_changed( + circuit_build_times_t *cbt); +#endif + +#ifdef TOR_UNIT_TESTS build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, double q_lo, double q_hi); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, double quantile, double time_ms); -int circuit_build_times_update_alpha(circuit_build_times_t *cbt); -double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuitbuild_running_unit_tests(void); -void circuit_build_times_reset(circuit_build_times_t *cbt); - -/* Network liveness functions */ -int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); #endif /* Network liveness functions */ void circuit_build_times_network_is_live(circuit_build_times_t *cbt); -int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(const circuit_build_times_t *cbt); void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); -/* DOCDOC circuit_build_times_get_bw_scale */ -int circuit_build_times_get_bw_scale(networkstatus_t *ns); +#ifdef CIRCUITSTATS_PRIVATE +/** Structure for circuit build times history */ +struct circuit_build_times_s { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +}; +#endif #endif diff --git a/src/or/circuituse.c b/src/or/circuituse.c index c2d2b2e87..467bef652 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -12,6 +12,7 @@ #include "or.h" #include "addressmap.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -31,12 +32,6 @@ #include "router.h" #include "routerlist.h" -/********* START VARIABLES **********/ - -extern circuit_t *global_circuitlist; /* from circuitlist.c */ - -/********* END VARIABLES ************/ - static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); @@ -286,7 +281,7 @@ circuit_get_best(const entry_connection_t *conn, tor_gettimeofday(&now); - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) continue; @@ -301,7 +296,7 @@ circuit_get_best(const entry_connection_t *conn, } if (!circuit_is_acceptable(origin_circ,conn,must_be_open,purpose, - need_uptime,need_internal,now.tv_sec)) + need_uptime,need_internal, (time_t)now.tv_sec)) continue; /* now this is an acceptable circ to hand back. but that doesn't @@ -327,7 +322,7 @@ count_pending_general_client_circuits(void) int count = 0; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || circ->state == CIRCUIT_STATE_OPEN || circ->purpose != CIRCUIT_PURPOSE_C_GENERAL || @@ -375,7 +370,7 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(void) { - circuit_t *victim, *next_circ = global_circuitlist; + circuit_t *victim, *next_circ; /* circ_times.timeout_ms and circ_times.close_ms are from * circuit_build_times_get_initial_timeout() if we haven't computed * custom timeouts yet */ @@ -393,10 +388,9 @@ circuit_expire_building(void) * we want to be more lenient with timeouts, in case the * user has relocated and/or changed network connections. * See bug #3443. */ - while (next_circ) { + TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */ next_circ->marked_for_close) { /* don't mess with marked circs */ - next_circ = next_circ->next; continue; } @@ -408,9 +402,7 @@ circuit_expire_building(void) any_opened_circs = 1; break; } - next_circ = next_circ->next; } - next_circ = global_circuitlist; #define SET_CUTOFF(target, msec) do { \ long ms = tor_lround(msec); \ @@ -451,12 +443,12 @@ circuit_expire_building(void) * RTTs = 4a + 3b + 2c * RTTs = 9h */ - SET_CUTOFF(general_cutoff, circ_times.timeout_ms); - SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms); + SET_CUTOFF(general_cutoff, get_circuit_build_timeout_ms()); + SET_CUTOFF(begindir_cutoff, get_circuit_build_timeout_ms()); /* > 3hop circs seem to have a 1.0 second delay on their cannibalized * 4th hop. */ - SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (10/6.0) + 1000); + SET_CUTOFF(fourhop_cutoff, get_circuit_build_timeout_ms() * (10/6.0) + 1000); /* CIRCUIT_PURPOSE_C_ESTABLISH_REND behaves more like a RELAY cell. * Use the stream cutoff (more or less). */ @@ -465,26 +457,25 @@ circuit_expire_building(void) /* Be lenient with cannibalized circs. They already survived the official * CBT, and they're usually not performance-critical. */ SET_CUTOFF(cannibalized_cutoff, - MAX(circ_times.close_ms*(4/6.0), + MAX(get_circuit_build_close_time_ms()*(4/6.0), options->CircuitStreamTimeout * 1000) + 1000); /* Intro circs have an extra round trip (and are also 4 hops long) */ - SET_CUTOFF(c_intro_cutoff, circ_times.timeout_ms * (14/6.0) + 1000); + SET_CUTOFF(c_intro_cutoff, get_circuit_build_timeout_ms() * (14/6.0) + 1000); /* Server intro circs have an extra round trip */ - SET_CUTOFF(s_intro_cutoff, circ_times.timeout_ms * (9/6.0) + 1000); + SET_CUTOFF(s_intro_cutoff, get_circuit_build_timeout_ms() * (9/6.0) + 1000); - SET_CUTOFF(close_cutoff, circ_times.close_ms); - SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + SET_CUTOFF(close_cutoff, get_circuit_build_close_time_ms()); + SET_CUTOFF(extremely_old_cutoff, get_circuit_build_close_time_ms()*2 + 1000); SET_CUTOFF(hs_extremely_old_cutoff, - MAX(circ_times.close_ms*2 + 1000, + MAX(get_circuit_build_close_time_ms()*2 + 1000, options->SocksTimeout * 1000)); - while (next_circ) { + TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { struct timeval cutoff; victim = next_circ; - next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -546,7 +537,9 @@ circuit_expire_building(void) "%d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), num_live_entry_guards(0)); @@ -555,12 +548,14 @@ circuit_expire_building(void) * was a timeout, and the timeout value needs to reset if we * see enough of them. Note this means we also need to avoid * double-counting below, too. */ - circuit_build_times_count_timeout(&circ_times, first_hop_succeeded); + circuit_build_times_count_timeout(get_circuit_build_times_mutable(), + first_hop_succeeded); TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout = 1; } continue; } else { static ratelim_t relax_timeout_limit = RATELIM_INIT(3600); + const double build_close_ms = get_circuit_build_close_time_ms(); log_fn_ratelim(&relax_timeout_limit, LOG_NOTICE, LD_CIRC, "No circuits are opened. Relaxed timeout for circuit %d " "(a %s %d-hop circuit in state %s with channel state %s) to " @@ -568,10 +563,13 @@ circuit_expire_building(void) "anyway. %d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), - (long)circ_times.close_ms, num_live_entry_guards(0)); + (long)build_close_ms, + num_live_entry_guards(0)); } } @@ -651,7 +649,7 @@ circuit_expire_building(void) } if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && - circuit_build_times_enough_to_compute(&circ_times)) { + circuit_build_times_enough_to_compute(get_circuit_build_times())) { /* Circuits are allowed to last longer for measurement. * Switch their purpose and wait. */ if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { @@ -665,8 +663,9 @@ circuit_expire_building(void) * have a timeout. We also want to avoid double-counting * already "relaxed" circuits, which are counted above. */ if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) { - circuit_build_times_count_timeout(&circ_times, - first_hop_succeeded); + circuit_build_times_count_timeout( + get_circuit_build_times_mutable(), + first_hop_succeeded); } continue; } @@ -683,10 +682,11 @@ circuit_expire_building(void) (long)(now.tv_sec - victim->timestamp_began.tv_sec), victim->purpose, circuit_purpose_to_string(victim->purpose)); - } else if (circuit_build_times_count_close(&circ_times, - first_hop_succeeded, - victim->timestamp_created.tv_sec)) { - circuit_build_times_set_timeout(&circ_times); + } else if (circuit_build_times_count_close( + get_circuit_build_times_mutable(), + first_hop_succeeded, + (time_t)victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } } } @@ -711,7 +711,8 @@ circuit_expire_building(void) * and we have tried to send an INTRODUCE1 cell specifying it. * Thus, if the pending_final_cpath field *is* NULL, then we * want to not spare it. */ - if (TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == + if (TO_ORIGIN_CIRCUIT(victim)->build_state && + TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == NULL) break; /* fallthrough! */ @@ -750,23 +751,27 @@ circuit_expire_building(void) if (victim->n_chan) log_info(LD_CIRC, - "Abandoning circ %u %s:%d (state %d,%d:%s, purpose %d, " + "Abandoning circ %u %s:%u (state %d,%d:%s, purpose %d, " "len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, channel_get_canonical_remote_descr(victim->n_chan), (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); else log_info(LD_CIRC, - "Abandoning circ %u %d (state %d,%d:%s, purpose %d, len %d)", + "Abandoning circ %u %u (state %d,%d:%s, purpose %d, len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) @@ -778,6 +783,64 @@ circuit_expire_building(void) } } +/** + * As a diagnostic for bug 8387, log information about how many one-hop + * circuits we have around that have been there for at least <b>age</b> + * seconds. Log a few of them. + */ +void +circuit_log_ancient_one_hop_circuits(int age) +{ +#define MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG 10 + time_t cutoff = time(NULL) - age; + int n_found = 0; + smartlist_t *log_these = smartlist_new(); + const circuit_t *circ; + + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + const origin_circuit_t *ocirc; + if (! CIRCUIT_IS_ORIGIN(circ)) + continue; + if (circ->timestamp_created.tv_sec >= cutoff) + continue; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + + if (ocirc->build_state && ocirc->build_state->onehop_tunnel) { + ++n_found; + + if (smartlist_len(log_these) < MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG) + smartlist_add(log_these, (origin_circuit_t*) ocirc); + } + } + + if (n_found == 0) + goto done; + + log_notice(LD_HEARTBEAT, + "Diagnostic for issue 8387: Found %d one-hop circuits more " + "than %d seconds old! Logging %d...", + n_found, age, smartlist_len(log_these)); + + SMARTLIST_FOREACH_BEGIN(log_these, const origin_circuit_t *, ocirc) { + char created[ISO_TIME_LEN+1]; + circ = TO_CIRCUIT(ocirc); + format_local_iso_time(created, + (time_t)circ->timestamp_created.tv_sec); + + log_notice(LD_HEARTBEAT, " #%d created at %s. %s, %s. %s for close. " + "%s for new conns.", + ocirc_sl_idx, + created, + circuit_state_to_string(circ->state), + circuit_purpose_to_string(circ->purpose), + circ->marked_for_close ? "Marked" : "Not marked", + ocirc->unusable_for_new_conns ? "Not usable" : "usable"); + } SMARTLIST_FOREACH_END(ocirc); + + done: + smartlist_free(log_these); +} + /** Remove any elements in <b>needed_ports</b> that are handled by an * open or in-progress circuit. */ @@ -818,7 +881,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, get_options()->LongLivedPorts, conn ? conn->socks_request->port : port); - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_GENERAL && @@ -869,7 +932,7 @@ circuit_predict_and_launch_new(void) int flags = 0; /* First, count how many of each type of circuit we have already. */ - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { cpath_build_state_t *build_state; origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) @@ -949,7 +1012,7 @@ circuit_predict_and_launch_new(void) * we can still build circuits preemptively as needed. */ if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && ! circuit_build_times_disabled() && - circuit_build_times_needs_circuits_now(&circ_times)) { + circuit_build_times_needs_circuits_now(get_circuit_build_times())) { flags = CIRCLAUNCH_NEED_CAPACITY; log_info(LD_CIRC, "Have %d clean circs need another buildtime test circ.", num); @@ -1085,7 +1148,7 @@ circuit_expire_old_circuits_clientside(void) cutoff = now; if (! circuit_build_times_disabled() && - circuit_build_times_needs_circuits(&circ_times)) { + circuit_build_times_needs_circuits(get_circuit_build_times())) { /* Circuits should be shorter lived if we need more of them * for learning a good build timeout */ cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; @@ -1093,7 +1156,7 @@ circuit_expire_old_circuits_clientside(void) cutoff.tv_sec -= get_options()->CircuitIdleTimeout; } - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams @@ -1176,7 +1239,7 @@ circuit_expire_old_circuits_serverside(time_t now) or_circuit_t *or_circ; time_t cutoff = now - IDLE_ONE_HOP_CIRC_TIMEOUT; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || CIRCUIT_IS_ORIGIN(circ)) continue; or_circ = TO_OR_CIRCUIT(circ); @@ -1223,7 +1286,7 @@ circuit_enough_testing_circs(void) if (have_performed_bandwidth_test) return 1; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && CIRCUIT_IS_ORIGIN(circ) && circ->purpose == CIRCUIT_PURPOSE_TESTING && circ->state == CIRCUIT_STATE_OPEN) @@ -1382,10 +1445,11 @@ circuit_build_failed(origin_circuit_t *circ) failed_at_last_hop = 1; } if (circ->cpath && - circ->cpath->state != CPATH_STATE_OPEN) { - /* We failed at the first hop. If there's an OR connection - * to blame, blame it. Also, avoid this relay for a while, and - * fail any one-hop directory fetches destined for it. */ + circ->cpath->state != CPATH_STATE_OPEN && + ! circ->base_.received_destroy) { + /* We failed at the first hop for some reason other than a DESTROY cell. + * If there's an OR connection to blame, blame it. Also, avoid this relay + * for a while, and fail any one-hop directory fetches destined for it. */ const char *n_chan_id = circ->cpath->extend_info->identity_digest; int already_marked = 0; if (circ->base_.n_chan) { diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 11e5a6416..f228a6758 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -16,6 +16,7 @@ void circuit_expire_building(void); void circuit_remove_handled_ports(smartlist_t *needed_ports); int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port, int min); +void circuit_log_ancient_one_hop_circuits(int age); #if 0 int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options); diff --git a/src/or/command.c b/src/or/command.c index f638fad41..105bdc637 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -53,6 +53,33 @@ static void command_process_created_cell(cell_t *cell, channel_t *chan); static void command_process_relay_cell(cell_t *cell, channel_t *chan); static void command_process_destroy_cell(cell_t *cell, channel_t *chan); +/** Convert the cell <b>command</b> into a lower-case, human-readable + * string. */ +const char * +cell_command_to_string(uint8_t command) +{ + switch (command) { + case CELL_PADDING: return "padding"; + case CELL_CREATE: return "create"; + case CELL_CREATED: return "created"; + case CELL_RELAY: return "relay"; + case CELL_DESTROY: return "destroy"; + case CELL_CREATE_FAST: return "create_fast"; + case CELL_CREATED_FAST: return "created_fast"; + case CELL_VERSIONS: return "versions"; + case CELL_NETINFO: return "netinfo"; + case CELL_RELAY_EARLY: return "relay_early"; + case CELL_CREATE2: return "create2"; + case CELL_CREATED2: return "created2"; + case CELL_VPADDING: return "vpadding"; + case CELL_CERTS: return "certs"; + case CELL_AUTH_CHALLENGE: return "auth_challenge"; + case CELL_AUTHENTICATE: return "authenticate"; + case CELL_AUTHORIZE: return "authorize"; + default: return "unrecognized"; + } +} + #ifdef KEEP_TIMING_STATS /** This is a wrapper function around the actual function that processes the * <b>cell</b> that just arrived on <b>conn</b>. Increment <b>*time</b> @@ -500,6 +527,7 @@ command_process_destroy_cell(cell_t *cell, channel_t *chan) log_debug(LD_OR,"Received for circID %u.",(unsigned)cell->circ_id); reason = (uint8_t)cell->payload[0]; + circ->received_destroy = 1; if (!CIRCUIT_IS_ORIGIN(circ) && chan == TO_OR_CIRCUIT(circ)->p_chan && diff --git a/src/or/command.h b/src/or/command.h index 913f46a5c..adea6adea 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -19,6 +19,8 @@ void command_process_var_cell(channel_t *chan, var_cell_t *cell); void command_setup_channel(channel_t *chan); void command_setup_listener(channel_listener_t *chan_l); +const char *cell_command_to_string(uint8_t command); + extern uint64_t stats_n_padding_cells_processed; extern uint64_t stats_n_create_cells_processed; extern uint64_t stats_n_created_cells_processed; diff --git a/src/or/config.c b/src/or/config.c index 09fdc0c49..10df83975 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2001 Matej Pfajfar. +/* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2013, The Tor Project, Inc. */ @@ -10,7 +10,6 @@ **/ #define CONFIG_PRIVATE - #include "or.h" #include "addressmap.h" #include "channel.h" @@ -40,11 +39,14 @@ #include "rendservice.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "util.h" #include "routerlist.h" #include "routerset.h" #include "statefile.h" #include "transports.h" +#include "ext_orport.h" +#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif @@ -83,6 +85,7 @@ static config_abbrev_t option_abbrevs_[] = { { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -135,7 +138,7 @@ static config_var_t option_vars_[] = { V(AllowSingleHopExits, BOOL, "0"), V(AlternateBridgeAuthority, LINELIST, NULL), V(AlternateDirAuthority, LINELIST, NULL), - V(AlternateHSAuthority, LINELIST, NULL), + OBSOLETE("AlternateHSAuthority"), V(AssumeReachable, BOOL, "0"), V(AuthDirBadDir, LINELIST, NULL), V(AuthDirBadDirCCs, CSV, ""), @@ -213,11 +216,14 @@ static config_var_t option_vars_[] = { V(DisableAllSwap, BOOL, "0"), V(DisableDebuggerAttachment, BOOL, "1"), V(DisableIOCP, BOOL, "1"), - V(DisableV2DirectoryInfo_, BOOL, "0"), + OBSOLETE("DisableV2DirectoryInfo_"), V(DynamicDHGroups, BOOL, "0"), VPORT(DNSPort, LINELIST, NULL), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), + V(TestingEnableConnBwEvent, BOOL, "0"), + V(TestingEnableCellStatsEvent, BOOL, "0"), + V(TestingEnableTbEmptyEvent, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), @@ -230,19 +236,21 @@ static config_var_t option_vars_[] = { V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), V(ExtendAllowPrivateAddresses, BOOL, "0"), + VPORT(ExtORPort, LINELIST, NULL), + V(ExtORPortCookieAuthFile, STRING, NULL), V(ExtraInfoStatistics, BOOL, "1"), V(FallbackDir, LINELIST, NULL), OBSOLETE("FallbackNetworkstatusFile"), V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), - V(FastFirstHopPK, BOOL, "1"), + V(FastFirstHopPK, AUTOBOOL, "auto"), V(FetchDirInfoEarly, BOOL, "0"), V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), - V(FetchV2Networkstatus, BOOL, "0"), + OBSOLETE("FetchV2Networkstatus"), V(GeoIPExcludeUnknown, AUTOBOOL, "auto"), #ifdef _WIN32 V(GeoIPFile, FILENAME, "<default>"), @@ -270,7 +278,7 @@ static config_var_t option_vars_[] = { VAR("HiddenServiceVersion",LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), - V(HSAuthoritativeDir, BOOL, "0"), + OBSOLETE("HSAuthoritativeDir"), OBSOLETE("HSAuthorityRecordStats"), V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), @@ -281,6 +289,7 @@ static config_var_t option_vars_[] = { V(IPv6Exit, BOOL, "0"), VAR("ServerTransportPlugin", LINELIST, ServerTransportPlugin, NULL), V(ServerTransportListenAddr, LINELIST, NULL), + V(ServerTransportOptions, LINELIST, NULL), V(Socks4Proxy, STRING, NULL), V(Socks5Proxy, STRING, NULL), V(Socks5ProxyUsername, STRING, NULL), @@ -299,7 +308,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -310,6 +319,7 @@ static config_var_t option_vars_[] = { V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), + V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), @@ -347,7 +357,7 @@ static config_var_t option_vars_[] = { V(OptimisticData, AUTOBOOL, "auto"), V(PortForwarding, BOOL, "0"), V(PortForwardingHelper, FILENAME, "tor-fw-helper"), - V(PreferTunneledDirConns, BOOL, "1"), + OBSOLETE("PreferTunneledDirConns"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), V(PublishHidServDescriptors, BOOL, "1"), @@ -370,6 +380,7 @@ static config_var_t option_vars_[] = { V(RunAsDaemon, BOOL, "0"), // V(RunTesting, BOOL, "0"), OBSOLETE("RunTesting"), // currently unused + V(Sandbox, BOOL, "0"), V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), @@ -400,21 +411,23 @@ static config_var_t option_vars_[] = { OBSOLETE("TrafficShaping"), V(TransListenAddress, LINELIST, NULL), VPORT(TransPort, LINELIST, NULL), - V(TunnelDirConns, BOOL, "1"), + V(TransProxyType, STRING, "default"), + OBSOLETE("TunnelDirConns"), V(UpdateBridgesFromAuthority, BOOL, "0"), V(UseBridges, BOOL, "0"), V(UseEntryGuards, BOOL, "1"), V(UseEntryGuardsAsDirGuards, BOOL, "1"), V(UseMicrodescriptors, AUTOBOOL, "auto"), - V(UseNTorHandshake, AUTOBOOL, "auto"), + V(UseNTorHandshake, AUTOBOOL, "1"), V(User, STRING, NULL), V(UserspaceIOCPBuffers, BOOL, "0"), - VAR("V1AuthoritativeDirectory",BOOL, V1AuthoritativeDir, "0"), - VAR("V2AuthoritativeDirectory",BOOL, V2AuthoritativeDir, "0"), + OBSOLETE("V1AuthoritativeDirectory"), + OBSOLETE("V2AuthoritativeDirectory"), VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir, "0"), V(TestingV3AuthInitialVotingInterval, INTERVAL, "30 minutes"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "5 minutes"), V(TestingV3AuthInitialDistDelay, INTERVAL, "5 minutes"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(V3AuthVotingInterval, INTERVAL, "1 hour"), V(V3AuthVoteDelay, INTERVAL, "5 minutes"), V(V3AuthDistDelay, INTERVAL, "5 minutes"), @@ -435,6 +448,24 @@ static config_var_t option_vars_[] = { VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "25 hours"), V(VoteOnHidServDirectoriesV2, BOOL, "1"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " + "300, 900, 2147483647"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, " + "2147483647"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 1800, 1800, 1800, " + "1800, 3600, 7200"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 3600, 3600, 3600, " + "10800, 21600, 43200"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "3600, 900, 900, 3600"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"), + V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"), + V(TestingConsensusMaxDownloadTries, UINT, "8"), + V(TestingDescriptorMaxDownloadTries, UINT, "8"), + V(TestingMicrodescMaxDownloadTries, UINT, "8"), + V(TestingCertMaxDownloadTries, UINT, "8"), + V(TestingDirAuthVoteGuard, ROUTERSET, NULL), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -460,9 +491,28 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, " + "20, 30, 60"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, " + "30, 60"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "60, 30, 30, 60"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), + V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), + V(TestingConsensusMaxDownloadTries, UINT, "80"), + V(TestingDescriptorMaxDownloadTries, UINT, "80"), + V(TestingMicrodescMaxDownloadTries, UINT, "80"), + V(TestingCertMaxDownloadTries, UINT, "80"), + V(TestingEnableConnBwEvent, BOOL, "1"), + V(TestingEnableCellStatsEvent, BOOL, "1"), + V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -475,9 +525,6 @@ static const config_var_t testing_tor_network_defaults[] = { #ifdef _WIN32 static char *get_windows_conf_root(void); #endif -static int options_validate(or_options_t *old_options, - or_options_t *options, - int from_setconf, char **msg); static int options_act_reversible(const or_options_t *old_options, char **msg); static int options_act(const or_options_t *old_options); static int options_transition_allowed(const or_options_t *old, @@ -487,12 +534,13 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); -static int check_nickname_list(const char *lst, const char *name, char **msg); +static int check_nickname_list(char **lst, const char *name, char **msg); -static int parse_bridge_line(const char *line, int validate_only); -static int parse_client_transport_line(const char *line, int validate_only); +static int parse_client_transport_line(const or_options_t *options, + const char *line, int validate_only); -static int parse_server_transport_line(const char *line, int validate_only); +static int parse_server_transport_line(const or_options_t *options, + const char *line, int validate_only); static char *get_bindaddr_from_transport_listen_line(const char *line, const char *transport); static int parse_dir_authority_line(const char *line, @@ -517,18 +565,23 @@ static int parse_outbound_addresses(or_options_t *options, int validate_only, char **msg); static void config_maybe_load_geoip_files_(const or_options_t *options, const or_options_t *old_options); +static int options_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); +static uint64_t compute_real_max_mem_in_queues(const uint64_t val, + int log_guess); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 /** Configuration format for or_options_t. */ -static config_format_t options_format = { +STATIC config_format_t options_format = { sizeof(or_options_t), OR_OPTIONS_MAGIC, STRUCT_OFFSET(or_options_t, magic_), option_abbrevs_, option_vars_, - (validate_fn_t)options_validate, + options_validate_cb, NULL }; @@ -545,8 +598,12 @@ static or_options_t *global_default_options = NULL; static char *torrc_fname = NULL; /** Name of the most recently read torrc-defaults file.*/ static char *torrc_defaults_fname; -/** Configuration Options set by command line. */ +/** Configuration options set by command line. */ static config_line_t *global_cmdline_options = NULL; +/** Non-configuration options set by the command line */ +static config_line_t *global_cmdline_only_options = NULL; +/** Boolean: Have we parsed the command line? */ +static int have_parsed_cmdline = 0; /** Contents of most recently read DirPortFrontPage file. */ static char *global_dirfrontpagecontents = NULL; /** List of port_cfg_t for all configured ports. */ @@ -568,8 +625,8 @@ get_options_mutable(void) } /** Returns the currently configured options */ -const or_options_t * -get_options(void) +MOCK_IMPL(const or_options_t *, +get_options,(void)) { return get_options_mutable(); } @@ -678,7 +735,7 @@ get_short_version(void) /** Release additional memory allocated in options */ -static void +STATIC void or_options_free(or_options_t *options) { if (!options) @@ -691,6 +748,7 @@ or_options_free(or_options_t *options) smartlist_free(options->NodeFamilySets); } tor_free(options->BridgePassword_AuthDigest_); + tor_free(options->command_arg); config_free(&options_format, options); } @@ -707,6 +765,9 @@ config_free_all(void) config_free_lines(global_cmdline_options); global_cmdline_options = NULL; + config_free_lines(global_cmdline_only_options); + global_cmdline_only_options = NULL; + if (configured_ports) { SMARTLIST_FOREACH(configured_ports, port_cfg_t *, p, port_cfg_free(p)); @@ -787,30 +848,30 @@ add_default_trusted_dir_authorities(dirinfo_type_t type) { int i; const char *authorities[] = { - "moria1 orport=9101 no-v2 " + "moria1 orport=9101 " "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - "tor26 v1 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " + "tor26 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", "dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", - "Tonga orport=443 bridge no-v2 82.94.251.203:80 " + "Tonga orport=443 bridge 82.94.251.203:80 " "4A0C CD2D DC79 9508 3D73 F5D6 6710 0C8A 5831 F16D", - "turtles orport=9090 no-v2 " + "turtles orport=9090 " "v3ident=27B6B5996C426270A5C95488AA5BCEB6BCC86956 " "76.73.17.194:9030 F397 038A DC51 3361 35E7 B80B D99C A384 4360 292B", - "gabelmoo orport=443 no-v2 " + "gabelmoo orport=443 " "v3ident=ED03BB616EB2F60BEC80151114BB25CEF515B226 " "212.112.245.170:80 F204 4413 DAC2 E02E 3D6B CF47 35A1 9BCA 1DE9 7281", - "dannenberg orport=443 no-v2 " + "dannenberg orport=443 " "v3ident=585769C78764D58426B8B52B6651A5A71137189A " "193.23.244.244:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123", - "urras orport=80 no-v2 v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C " + "urras orport=80 v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C " "208.83.223.34:443 0AD3 FA88 4D18 F89E EA2D 89C0 1937 9E0E 7FD9 4417", - "maatuska orport=80 no-v2 " + "maatuska orport=80 " "v3ident=49015F787433103580E3B66A1707A00E60F2D15B " "171.25.193.9:443 BD6A 8292 55CB 08E6 6FBE 7D37 4836 3586 E46B 3810", - "Faravahar orport=443 no-v2 " + "Faravahar orport=443 " "v3ident=EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 " "154.35.32.5:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC", NULL @@ -850,8 +911,7 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) config_line_t *cl; if (options->DirAuthorities && - (options->AlternateDirAuthority || options->AlternateBridgeAuthority || - options->AlternateHSAuthority)) { + (options->AlternateDirAuthority || options->AlternateBridgeAuthority)) { log_warn(LD_CONFIG, "You cannot set both DirAuthority and Alternate*Authority."); return -1; @@ -887,9 +947,6 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 1)<0) return -1; @@ -912,9 +969,7 @@ consider_adding_dir_servers(const or_options_t *options, !config_lines_eq(options->AlternateBridgeAuthority, old_options->AlternateBridgeAuthority) || !config_lines_eq(options->AlternateDirAuthority, - old_options->AlternateDirAuthority) || - !config_lines_eq(options->AlternateHSAuthority, - old_options->AlternateHSAuthority); + old_options->AlternateDirAuthority); if (!need_to_update) return 0; /* all done */ @@ -928,10 +983,7 @@ consider_adding_dir_servers(const or_options_t *options, if (!options->AlternateBridgeAuthority) type |= BRIDGE_DIRINFO; if (!options->AlternateDirAuthority) - type |= V1_DIRINFO | V2_DIRINFO | V3_DIRINFO | EXTRAINFO_DIRINFO | - MICRODESC_DIRINFO; - if (!options->AlternateHSAuthority) - type |= HIDSERV_DIRINFO; + type |= V3_DIRINFO | EXTRAINFO_DIRINFO | MICRODESC_DIRINFO; add_default_trusted_dir_authorities(type); } if (!options->FallbackDir) @@ -946,9 +998,6 @@ consider_adding_dir_servers(const or_options_t *options, for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 0)<0) return -1; @@ -972,6 +1021,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) int set_conn_limit = 0; int r = -1; int logs_marked = 0; + int old_min_log_level = get_min_log_level(); /* Daemonize _first_, since we only want to open most of this stuff in * the subprocess. Libevent bases can't be reliably inherited across @@ -998,12 +1048,18 @@ options_act_reversible(const or_options_t *old_options, char **msg) if (running_tor) { int n_ports=0; /* We need to set the connection limit before we can open the listeners. */ - if (set_max_file_descriptors((unsigned)options->ConnLimit, - &options->ConnLimit_) < 0) { - *msg = tor_strdup("Problem with ConnLimit value. See logs for details."); - goto rollback; + if (! sandbox_is_active()) { + if (set_max_file_descriptors((unsigned)options->ConnLimit, + &options->ConnLimit_) < 0) { + *msg = tor_strdup("Problem with ConnLimit value. " + "See logs for details."); + goto rollback; + } + set_conn_limit = 1; + } else { + tor_assert(old_options); + options->ConnLimit_ = old_options->ConnLimit_; } - set_conn_limit = 1; /* Set up libevent. (We need to do this before we can register the * listeners as listeners.) */ @@ -1044,7 +1100,8 @@ options_act_reversible(const or_options_t *old_options, char **msg) #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) /* Open /dev/pf before dropping privileges. */ - if (options->TransPort_set) { + if (options->TransPort_set && + options->TransProxyType_parsed == TPT_DEFAULT) { if (get_pf_socket() < 0) { *msg = tor_strdup("Unable to open /dev/pf for transparent proxy."); goto rollback; @@ -1081,23 +1138,6 @@ options_act_reversible(const or_options_t *old_options, char **msg) /* No need to roll back, since you can't change the value. */ } - /* Write control ports to disk as appropriate */ - control_ports_write_to_file(); - - if (directory_caches_v2_dir_info(options)) { - char *fn = NULL; - tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-status", - options->DataDirectory); - if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK, - options->User) < 0) { - tor_asprintf(msg, - "Couldn't access/create private data directory \"%s\"", fn); - tor_free(fn); - goto done; - } - tor_free(fn); - } - /* Bail out at this point if we're not going to be a client or server: * we don't run Tor itself. */ if (!running_tor) @@ -1119,13 +1159,44 @@ options_act_reversible(const or_options_t *old_options, char **msg) add_callback_log(severity, control_event_logmsg); control_adjust_event_log_severity(); tor_free(severity); + tor_log_update_sigsafe_err_fds(); + } + + { + const char *badness = NULL; + int bad_safelog = 0, bad_severity = 0, new_badness = 0; + if (options->SafeLogging_ != SAFELOG_SCRUB_ALL) { + bad_safelog = 1; + if (!old_options || old_options->SafeLogging_ != options->SafeLogging_) + new_badness = 1; + } + if (get_min_log_level() >= LOG_INFO) { + bad_severity = 1; + if (get_min_log_level() != old_min_log_level) + new_badness = 1; + } + if (bad_safelog && bad_severity) + badness = "you disabled SafeLogging, and " + "you're logging more than \"notice\""; + else if (bad_safelog) + badness = "you disabled SafeLogging"; + else + badness = "you're logging more than \"notice\""; + if (new_badness) + log_warn(LD_GENERAL, "Your log may contain sensitive information - %s. " + "Don't log unless it serves an important reason. " + "Overwrite the log afterwards.", badness); } + SMARTLIST_FOREACH(replaced_listeners, connection_t *, conn, { + int marked = conn->marked_for_close; log_notice(LD_NET, "Closing old %s on %s:%d", conn_type_to_string(conn->type), conn->address, conn->port); connection_close_immediate(conn); - connection_mark_for_close(conn); + if (!marked) { + connection_mark_for_close(conn); + } }); goto done; @@ -1272,6 +1343,9 @@ options_act(const or_options_t *old_options) } } + /* Write control ports to disk as appropriate */ + control_ports_write_to_file(); + if (running_tor && !have_lockfile()) { if (try_locking(options, 1) < 0) return -1; @@ -1302,14 +1376,29 @@ options_act(const or_options_t *old_options) } #endif + /* If we are a bridge with a pluggable transport proxy but no + Extended ORPort, inform the user that she is missing out. */ + if (server_mode(options) && options->ServerTransportPlugin && + !options->ExtORPort_lines) { + log_notice(LD_CONFIG, "We use pluggable transports but the Extended " + "ORPort is disabled. Tor and your pluggable transports proxy " + "communicate with each other via the Extended ORPort so it " + "is suggested you enable it: it will also allow your Bridge " + "to collect statistics about its clients that use pluggable " + "transports. Please enable it using the ExtORPort torrc option " + "(e.g. set 'ExtORPort auto')."); + } + if (options->Bridges) { mark_bridge_list(); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 0)<0) { + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) { log_warn(LD_BUG, "Previously validated Bridge line could not be added!"); return -1; } + bridge_add_from_config(bridge_line); } sweep_bridge_list(); } @@ -1337,7 +1426,7 @@ options_act(const or_options_t *old_options) pt_prepare_proxy_list_for_config_read(); if (options->ClientTransportPlugin) { for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 0)<0) { + if (parse_client_transport_line(options, cl->value, 0)<0) { log_warn(LD_BUG, "Previously validated ClientTransportPlugin line " "could not be added!"); @@ -1348,7 +1437,7 @@ options_act(const or_options_t *old_options) if (options->ServerTransportPlugin && server_mode(options)) { for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 0)<0) { + if (parse_server_transport_line(options, cl->value, 0)<0) { log_warn(LD_BUG, "Previously validated ServerTransportPlugin line " "could not be added!"); @@ -1359,6 +1448,12 @@ options_act(const or_options_t *old_options) sweep_transport_list(); sweep_proxy_list(); + /* Start the PT proxy configuration. By doing this configuration + here, we also figure out which proxies need to be restarted and + which not. */ + if (pt_proxies_configuration_pending() && !net_is_disabled()) + pt_configure_remaining_proxies(); + /* Bail out at this point if we're not going to be a client or server: * we want to not fork, and to log stuff to stderr. */ if (!running_tor) @@ -1408,8 +1503,9 @@ options_act(const or_options_t *old_options) /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (options->PidFile) + if (options->PidFile && !sandbox_is_active()) { write_pidfile(options->PidFile); + } /* Register addressmap directives */ config_register_addressmaps(options); @@ -1423,8 +1519,14 @@ options_act(const or_options_t *old_options) return -1; } - if (init_cookie_authentication(options->CookieAuthentication) < 0) { - log_warn(LD_CONFIG,"Error creating cookie authentication file."); + if (init_control_cookie_authentication(options->CookieAuthentication) < 0) { + log_warn(LD_CONFIG,"Error creating control cookie authentication file."); + return -1; + } + + /* If we have an ExtORPort, initialize its auth cookie. */ + if (init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) { + log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file."); return -1; } @@ -1606,10 +1708,14 @@ options_act(const or_options_t *old_options) time_t now = time(NULL); int print_notice = 0; - /* If we aren't acting as a server, we can't collect stats anyway. */ + /* Only collect directory-request statistics on relays and bridges. */ if (!server_mode(options)) { - options->CellStatistics = 0; options->DirReqStatistics = 0; + } + + /* Only collect other relay-only statistics on relays. */ + if (!public_server_mode(options)) { + options->CellStatistics = 0; options->EntryStatistics = 0; options->ExitPortStatistics = 0; } @@ -1732,40 +1838,66 @@ options_act(const or_options_t *old_options) return 0; } -/** Helper: Read a list of configuration options from the command line. - * If successful, put them in *<b>result</b> and return 0, and return - * -1 and leave *<b>result</b> alone. */ -static int -config_get_commandlines(int argc, char **argv, config_line_t **result) +static const struct { + const char *name; + int takes_argument; +} CMDLINE_ONLY_OPTIONS[] = { + { "-f", 1 }, + { "--allow-missing-torrc", 0 }, + { "--defaults-torrc", 1 }, + { "--hash-password", 1 }, + { "--dump-config", 1 }, + { "--list-fingerprint", 0 }, + { "--verify-config", 0 }, + { "--ignore-missing-torrc", 0 }, + { "--quiet", 0 }, + { "--hush", 0 }, + { "--version", 0 }, + { "--library-versions", 0 }, + { "-h", 0 }, + { "--help", 0 }, + { "--list-torrc-options", 0 }, + { "--digests", 0 }, + { "--nt-service", 0 }, + { "-nt-service", 0 }, + { NULL, 0 }, +}; + +/** Helper: Read a list of configuration options from the command line. If + * successful, or if ignore_errors is set, put them in *<b>result</b>, put the + * commandline-only options in *<b>cmdline_result</b>, and return 0; + * otherwise, return -1 and leave *<b>result</b> and <b>cmdline_result</b> + * alone. */ +int +config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result) { + config_line_t *param = NULL; + config_line_t *front = NULL; config_line_t **new = &front; - char *s; + + config_line_t *front_cmdline = NULL; + config_line_t **new_cmdline = &front_cmdline; + + char *s, *arg; int i = 1; while (i < argc) { unsigned command = CONFIG_LINE_NORMAL; int want_arg = 1; + int is_cmdline = 0; + int j; - if (!strcmp(argv[i],"-f") || - !strcmp(argv[i],"--defaults-torrc") || - !strcmp(argv[i],"--hash-password")) { - i += 2; /* command-line option with argument. ignore them. */ - continue; - } else if (!strcmp(argv[i],"--list-fingerprint") || - !strcmp(argv[i],"--verify-config") || - !strcmp(argv[i],"--ignore-missing-torrc") || - !strcmp(argv[i],"--quiet") || - !strcmp(argv[i],"--hush")) { - i += 1; /* command-line option. ignore it. */ - continue; - } else if (!strcmp(argv[i],"--nt-service") || - !strcmp(argv[i],"-nt-service")) { - i += 1; - continue; + for (j = 0; CMDLINE_ONLY_OPTIONS[j].name != NULL; ++j) { + if (!strcmp(argv[i], CMDLINE_ONLY_OPTIONS[j].name)) { + is_cmdline = 1; + want_arg = CMDLINE_ONLY_OPTIONS[j].takes_argument; + break; + } } - *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; /* Each keyword may be prefixed with one or two dashes. */ @@ -1785,22 +1917,38 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) } if (want_arg && i == argc-1) { - log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", - argv[i]); - config_free_lines(front); - return -1; + if (ignore_errors) { + arg = strdup(""); + } else { + log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", + argv[i]); + config_free_lines(front); + config_free_lines(front_cmdline); + return -1; + } + } else { + arg = want_arg ? tor_strdup(argv[i+1]) : strdup(""); } - (*new)->key = tor_strdup(config_expand_abbrev(&options_format, s, 1, 1)); - (*new)->value = want_arg ? tor_strdup(argv[i+1]) : tor_strdup(""); - (*new)->command = command; - (*new)->next = NULL; + param = tor_malloc_zero(sizeof(config_line_t)); + param->key = is_cmdline ? tor_strdup(argv[i]) : tor_strdup(s); + param->value = arg; + param->command = command; + param->next = NULL; log_debug(LD_CONFIG, "command line: parsed keyword '%s', value '%s'", - (*new)->key, (*new)->value); + param->key, param->value); + + if (is_cmdline) { + *new_cmdline = param; + new_cmdline = &((*new_cmdline)->next); + } else { + *new = param; + new = &((*new)->next); + } - new = &((*new)->next); i += want_arg ? 2 : 1; } + *cmdline_result = front_cmdline; *result = front; return 0; } @@ -1852,7 +2000,8 @@ options_trial_assign(config_line_t *list, int use_defaults, return r; } - if (options_validate(get_options_mutable(), trial_options, 1, msg) < 0) { + if (options_validate(get_options_mutable(), trial_options, + global_default_options, 1, msg) < 0) { config_free(&options_format, trial_options); return SETOPT_ERR_PARSE; /*XXX make this a separate return value. */ } @@ -1945,6 +2094,7 @@ resolve_my_address(int warn_severity, const or_options_t *options, int notice_severity = warn_severity <= LOG_NOTICE ? LOG_NOTICE : warn_severity; + tor_addr_t myaddr; tor_assert(addr_out); /* @@ -1995,24 +2145,26 @@ resolve_my_address(int warn_severity, const or_options_t *options, "local interface. Using that.", fmt_addr32(addr)); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); } else { /* resolved hostname into addr */ + tor_addr_from_ipv4h(&myaddr, addr); + if (!explicit_hostname && - is_internal_IP(addr, 0)) { - uint32_t interface_ip; + tor_addr_is_internal(&myaddr, 0)) { + tor_addr_t interface_ip; log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' " "resolves to a private IP address (%s). Trying something " "else.", hostname, fmt_addr32(addr)); - if (get_interface_address(warn_severity, &interface_ip)) { + if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) { log_fn(warn_severity, LD_CONFIG, "Could not get local interface IP address. Too bad."); - } else if (is_internal_IP(interface_ip, 0)) { + } else if (tor_addr_is_internal(&interface_ip, 0)) { log_fn(notice_severity, LD_CONFIG, "Interface IP address '%s' is a private address too. " - "Ignoring.", fmt_addr32(interface_ip)); + "Ignoring.", fmt_addr(&interface_ip)); } else { from_interface = 1; - addr = interface_ip; + addr = tor_addr_to_ipv4h(&interface_ip); log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for local interface." " Using that.", fmt_addr32(addr)); @@ -2030,8 +2182,10 @@ resolve_my_address(int warn_severity, const or_options_t *options, * out if it is and we don't want that. */ + tor_addr_from_ipv4h(&myaddr,addr); + addr_string = tor_dup_ip(addr); - if (is_internal_IP(addr, 0)) { + if (tor_addr_is_internal(&myaddr, 0)) { /* make sure we're ok with publishing an internal IP */ if (!options->DirAuthorities && !options->AlternateDirAuthority) { /* if they are using the default authorities, disallow internal IPs @@ -2137,7 +2291,7 @@ is_local_addr(const tor_addr_t *addr) * resolve_my_address will never be called at all). In those cases, * last_resolved_addr will be 0, and so checking to see whether ip is on * the same /24 as last_resolved_addr will be the same as checking whether - * it was on net 0, which is already done by is_internal_IP. + * it was on net 0, which is already done by tor_addr_is_internal. */ if ((last_resolved_addr & (uint32_t)0xffffff00ul) == (ip & (uint32_t)0xffffff00ul)) @@ -2166,10 +2320,29 @@ options_init(or_options_t *options) * include options that are the same as Tor's defaults. */ char * -options_dump(const or_options_t *options, int minimal) +options_dump(const or_options_t *options, int how_to_dump) { - return config_dump(&options_format, global_default_options, - options, minimal, 0); + const or_options_t *use_defaults; + int minimal; + switch (how_to_dump) { + case OPTIONS_DUMP_MINIMAL: + use_defaults = global_default_options; + minimal = 1; + break; + case OPTIONS_DUMP_DEFAULTS: + use_defaults = NULL; + minimal = 1; + break; + case OPTIONS_DUMP_ALL: + use_defaults = NULL; + minimal = 0; + break; + default: + log_warn(LD_BUG, "Bogus value for how_to_dump==%d", how_to_dump); + return NULL; + } + + return config_dump(&options_format, use_defaults, options, minimal, 0); } /** Return 0 if every element of sl is a string holding a decimal @@ -2218,7 +2391,7 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) /** Parse an authority type from <b>options</b>-\>PublishServerDescriptor * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1" - * as "v2,v3" unless BridgeRelay is 1, in which case treat it as "bridge". + * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge". * Treat "0" as "". * Return 0 on success or -1 if not a recognized authority type (in which * case the value of PublishServerDescriptor_ is undefined). */ @@ -2232,14 +2405,16 @@ compute_publishserverdescriptor(or_options_t *options) return 0; SMARTLIST_FOREACH_BEGIN(list, const char *, string) { if (!strcasecmp(string, "v1")) - *auth |= V1_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because " + "there are no v1 directory authorities anymore."); else if (!strcmp(string, "1")) if (options->BridgeRelay) *auth |= BRIDGE_DIRINFO; else - *auth |= V2_DIRINFO | V3_DIRINFO; + *auth |= V3_DIRINFO; else if (!strcasecmp(string, "v2")) - *auth |= V2_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because " + "there are no v2 directory authorities anymore."); else if (!strcasecmp(string, "v3")) *auth |= V3_DIRINFO; else if (!strcasecmp(string, "bridge")) @@ -2260,6 +2435,11 @@ compute_publishserverdescriptor(or_options_t *options) * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +/** Higest allowable value for PredictedPortsRelevanceTime; if this is + * too high, our selection of exits will decrease for an extended + * period of time to an uncomfortable level .*/ +#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) + /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) @@ -2286,10 +2466,19 @@ compute_publishserverdescriptor(or_options_t *options) * */ #define RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT (10) -/** Return 0 if every setting in <b>options</b> is reasonable, and a - * permissible transition from <b>old_options</b>. Else return -1. - * Should have no side effects, except for normalizing the contents of - * <b>options</b>. +static int +options_validate_cb(void *old_options, void *options, void *default_options, + int from_setconf, char **msg) +{ + return options_validate(old_options, options, default_options, + from_setconf, msg); +} + +/** Return 0 if every setting in <b>options</b> is reasonable, is a + * permissible transition from <b>old_options</b>, and none of the + * testing-only settings differ from <b>default_options</b> unless in + * testing mode. Else return -1. Should have no side effects, except for + * normalizing the contents of <b>options</b>. * * On error, tor_strdup an error explanation into *<b>msg</b>. * @@ -2298,9 +2487,9 @@ compute_publishserverdescriptor(or_options_t *options) * Log line should stay empty. If it's 0, then give us a default log * if there are no logs defined. */ -static int +STATIC int options_validate(or_options_t *old_options, or_options_t *options, - int from_setconf, char **msg) + or_options_t *default_options, int from_setconf, char **msg) { int i; config_line_t *cl; @@ -2381,10 +2570,43 @@ options_validate(or_options_t *old_options, or_options_t *options, "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); -#ifndef USE_TRANSPARENT - /* XXXX024 I think we can remove this TransListenAddress */ - if (options->TransPort_set || options->TransListenAddress) - REJECT("TransPort and TransListenAddress are disabled in this build."); + options->TransProxyType_parsed = TPT_DEFAULT; +#ifdef USE_TRANSPARENT + if (options->TransProxyType) { + if (!strcasecmp(options->TransProxyType, "default")) { + options->TransProxyType_parsed = TPT_DEFAULT; + } else if (!strcasecmp(options->TransProxyType, "pf-divert")) { +#ifndef __OpenBSD__ + REJECT("pf-divert is a OpenBSD-specific feature."); +#else + options->TransProxyType_parsed = TPT_PF_DIVERT; +#endif + } else if (!strcasecmp(options->TransProxyType, "tproxy")) { +#ifndef __linux__ + REJECT("TPROXY is a Linux-specific feature."); +#else + options->TransProxyType_parsed = TPT_TPROXY; +#endif + } else if (!strcasecmp(options->TransProxyType, "ipfw")) { +#ifndef __FreeBSD__ + REJECT("ipfw is a FreeBSD-specific feature."); +#else + options->TransProxyType_parsed = TPT_IPFW; +#endif + } else { + REJECT("Unrecognized value for TransProxyType"); + } + + if (strcasecmp(options->TransProxyType, "default") && + !options->TransPort_set) { + REJECT("Cannot use TransProxyType without any valid TransPort or " + "TransListenAddress."); + } + } +#else + if (options->TransPort_set) + REJECT("TransPort and TransListenAddress are disabled " + "in this build."); #endif if (options->TokenBucketRefillInterval <= 0 @@ -2392,10 +2614,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("TokenBucketRefillInterval must be between 1 and 1000 inclusive."); } - if (options->DisableV2DirectoryInfo_ && ! authdir_mode(options)) { - REJECT("DisableV2DirectoryInfo_ set, but we aren't an authority."); - } - if (options->ExcludeExitNodes || options->ExcludeNodes) { options->ExcludeExitNodesUnion_ = routerset_new(); routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeExitNodes); @@ -2429,8 +2647,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); - if (options->V1AuthoritativeDir && !options->RecommendedVersions) - REJECT("V1 authoritative dir servers must set RecommendedVersions."); if (!options->RecommendedClientVersions) options->RecommendedClientVersions = config_lines_dup(options->RecommendedVersions); @@ -2452,11 +2668,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "extra-info documents. Setting DownloadExtraInfo."); options->DownloadExtraInfo = 1; } - if (!(options->BridgeAuthoritativeDir || options->HSAuthoritativeDir || - options->V1AuthoritativeDir || options->V2AuthoritativeDir || + if (!(options->BridgeAuthoritativeDir || options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " - "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); @@ -2476,10 +2691,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("FetchDirInfoExtraEarly requires that you also set " "FetchDirInfoEarly"); - if (options->HSAuthoritativeDir && proxy_mode(options)) - REJECT("Running as authoritative v0 HS directory, but also configured " - "as a client."); - if (options->ConnLimit <= 0) { tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", @@ -2616,11 +2827,9 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInCellQueues < (256 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 256 MB for now. " - "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (256 << 20); - } + options->MaxMemInQueues = + compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, + server_mode(options)); options->AllowInvalid_ = 0; @@ -2665,8 +2874,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if ((options->BridgeRelay || options->PublishServerDescriptor_ & BRIDGE_DIRINFO) - && (options->PublishServerDescriptor_ - & (V1_DIRINFO|V2_DIRINFO|V3_DIRINFO))) { + && (options->PublishServerDescriptor_ & V3_DIRINFO)) { REJECT("Bridges are not supposed to publish router descriptors to the " "directory authorities. Please correct your " "PublishServerDescriptor line."); @@ -2698,6 +2906,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } + if (options->PredictedPortsRelevanceTime > + MAX_PREDICTED_CIRCS_RELEVANCE) { + log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " + "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); + options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; + } + if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: @@ -2814,6 +3029,11 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); + if (options->PortForwarding && options->Sandbox) { + REJECT("PortForwarding is not compatible with Sandbox; at most one can " + "be set"); + } + if (ensure_bandwidth_cap(&options->BandwidthRate, "BandwidthRate", msg) < 0) return -1; @@ -2973,14 +3193,14 @@ options_validate(or_options_t *old_options, or_options_t *options, size_t len; len = strlen(options->Socks5ProxyUsername); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); if (!options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); len = strlen(options->Socks5ProxyPassword); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); } else if (options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); @@ -3040,7 +3260,7 @@ options_validate(or_options_t *old_options, or_options_t *options, "You should also make sure you aren't listing this bridge's " "fingerprint in any other MyFamily."); } - if (check_nickname_list(options->MyFamily, "MyFamily", msg)) + if (check_nickname_list(&options->MyFamily, "MyFamily", msg)) return -1; for (cl = options->NodeFamilies; cl; cl = cl->next) { routerset_t *rs = routerset_new(); @@ -3060,26 +3280,22 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->UseBridges && !options->Bridges) REJECT("If you set UseBridges, you must specify at least one bridge."); - if (options->UseBridges && !options->TunnelDirConns) - REJECT("If you set UseBridges, you must set TunnelDirConns."); - if (options->RendConfigLines && - (!options->TunnelDirConns || !options->PreferTunneledDirConns)) - REJECT("If you are running a hidden service, you must set TunnelDirConns " - "and PreferTunneledDirConns"); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 1)<0) - REJECT("Bridge line did not parse. See logs for details."); + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) + REJECT("Bridge line did not parse. See logs for details."); + bridge_line_free(bridge_line); } for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 1)<0) - REJECT("Transport line did not parse. See logs for details."); + if (parse_client_transport_line(options, cl->value, 1)<0) + REJECT("Invalid client transport line. See logs for details."); } for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 1)<0) - REJECT("Server transport line did not parse. See logs for details."); + if (parse_server_transport_line(options, cl->value, 1)<0) + REJECT("Invalid server transport line. See logs for details."); } if (options->ServerTransportPlugin && !server_mode(options)) { @@ -3105,6 +3321,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "ServerTransportListenAddr line will be ignored."); } + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + /** If get_options_from_transport_options_line() fails with + 'transport' being NULL, it means that something went wrong + while parsing the ServerTransportOptions line. */ + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, NULL); + if (!options_sl) + REJECT("ServerTransportOptions did not parse. See logs for details."); + + SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp)); + smartlist_free(options_sl); + } + if (options->ConstrainedSockets) { /* If the user wants to constrain socket buffer use, make sure the desired * limit is between MIN|MAX_TCPSOCK_BUFFER in k increments. */ @@ -3163,15 +3392,6 @@ options_validate(or_options_t *old_options, or_options_t *options, AF_INET6, 1, msg)<0) return -1; - if (options->PreferTunneledDirConns && !options->TunnelDirConns) - REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); - - if ((options->Socks4Proxy || options->Socks5Proxy) && - !options->HTTPProxy && !options->PreferTunneledDirConns) - REJECT("When Socks4Proxy or Socks5Proxy is configured, " - "PreferTunneledDirConns and TunnelDirConns must both be " - "set to 1, or HTTPProxy must be configured."); - if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3197,35 +3417,46 @@ options_validate(or_options_t *old_options, or_options_t *options, "ignore you."); } - /*XXXX checking for defaults manually like this is a bit fragile.*/ - - /* Keep changes to hard-coded values synchronous to man page and default - * values table. */ - if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { +#define CHECK_DEFAULT(arg) \ + STMT_BEGIN \ + if (!options->TestingTorNetwork && \ + !options->UsingTestNetworkDefaults_ && \ + !config_is_same(&options_format,options, \ + default_options,#arg)) { \ + REJECT(#arg " may only be changed in testing Tor " \ + "networks!"); \ + } STMT_END + CHECK_DEFAULT(TestingV3AuthInitialVotingInterval); + CHECK_DEFAULT(TestingV3AuthInitialVoteDelay); + CHECK_DEFAULT(TestingV3AuthInitialDistDelay); + CHECK_DEFAULT(TestingV3AuthVotingStartOffset); + CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability); + CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime); + CHECK_DEFAULT(TestingServerDownloadSchedule); + CHECK_DEFAULT(TestingClientDownloadSchedule); + CHECK_DEFAULT(TestingServerConsensusDownloadSchedule); + CHECK_DEFAULT(TestingClientConsensusDownloadSchedule); + CHECK_DEFAULT(TestingBridgeDownloadSchedule); + CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest); + CHECK_DEFAULT(TestingDirConnectionMaxStall); + CHECK_DEFAULT(TestingConsensusMaxDownloadTries); + CHECK_DEFAULT(TestingDescriptorMaxDownloadTries); + CHECK_DEFAULT(TestingMicrodescMaxDownloadTries); + CHECK_DEFAULT(TestingCertMaxDownloadTries); +#undef CHECK_DEFAULT + + if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { REJECT("TestingV3AuthInitialVotingInterval is insanely low."); } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) { REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into " "30 minutes."); } - if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - - REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { REJECT("TestingV3AuthInitialVoteDelay is way too low."); } - if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { + if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { REJECT("TestingV3AuthInitialDistDelay is way too low."); } @@ -3236,26 +3467,79 @@ options_validate(or_options_t *old_options, or_options_t *options, "must be less than half TestingV3AuthInitialVotingInterval"); } - if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " - "testing Tor networks!"); - } else if (options->TestingAuthDirTimeToLearnReachability < 0) { + if (options->TestingV3AuthVotingStartOffset > + MIN(options->TestingV3AuthInitialVotingInterval, + options->V3AuthVotingInterval)) { + REJECT("TestingV3AuthVotingStartOffset is higher than the voting " + "interval."); + } + + if (options->TestingAuthDirTimeToLearnReachability < 0) { REJECT("TestingAuthDirTimeToLearnReachability must be non-negative."); } else if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) { COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high."); } - if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " - "testing Tor networks!"); - } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { + if (options->TestingEstimatedDescriptorPropagationTime < 0) { REJECT("TestingEstimatedDescriptorPropagationTime must be non-negative."); } else if (options->TestingEstimatedDescriptorPropagationTime > 60*60) { COMPLAIN("TestingEstimatedDescriptorPropagationTime is insanely high."); } + if (options->TestingClientMaxIntervalWithoutRequest < 1) { + REJECT("TestingClientMaxIntervalWithoutRequest is way too low."); + } else if (options->TestingClientMaxIntervalWithoutRequest > 3600) { + COMPLAIN("TestingClientMaxIntervalWithoutRequest is insanely high."); + } + + if (options->TestingDirConnectionMaxStall < 5) { + REJECT("TestingDirConnectionMaxStall is way too low."); + } else if (options->TestingDirConnectionMaxStall > 3600) { + COMPLAIN("TestingDirConnectionMaxStall is insanely high."); + } + + if (options->TestingConsensusMaxDownloadTries < 2) { + REJECT("TestingConsensusMaxDownloadTries must be greater than 1."); + } else if (options->TestingConsensusMaxDownloadTries > 800) { + COMPLAIN("TestingConsensusMaxDownloadTries is insanely high."); + } + + if (options->TestingDescriptorMaxDownloadTries < 2) { + REJECT("TestingDescriptorMaxDownloadTries must be greater than 1."); + } else if (options->TestingDescriptorMaxDownloadTries > 800) { + COMPLAIN("TestingDescriptorMaxDownloadTries is insanely high."); + } + + if (options->TestingMicrodescMaxDownloadTries < 2) { + REJECT("TestingMicrodescMaxDownloadTries must be greater than 1."); + } else if (options->TestingMicrodescMaxDownloadTries > 800) { + COMPLAIN("TestingMicrodescMaxDownloadTries is insanely high."); + } + + if (options->TestingCertMaxDownloadTries < 2) { + REJECT("TestingCertMaxDownloadTries must be greater than 1."); + } else if (options->TestingCertMaxDownloadTries > 800) { + COMPLAIN("TestingCertMaxDownloadTries is insanely high."); + } + + if (options->TestingEnableConnBwEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableConnBwEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableCellStatsEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableCellStatsEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableTbEmptyEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableTbEmptyEvent may only be changed in testing " + "Tor networks!"); + } + if (options->TestingTorNetwork) { log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node " "almost unusable in the public Tor network, and is " @@ -3289,6 +3573,68 @@ options_validate(or_options_t *old_options, or_options_t *options, #undef COMPLAIN } +/* Given the value that the user has set for MaxMemInQueues, compute the + * actual maximum value. We clip this value if it's too low, and autodetect + * it if it's set to 0. */ +static uint64_t +compute_real_max_mem_in_queues(const uint64_t val, int log_guess) +{ + uint64_t result; + + if (val == 0) { +#define ONE_GIGABYTE (U64_LITERAL(1) << 30) +#define ONE_MEGABYTE (U64_LITERAL(1) << 20) +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) +#else +#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) +#endif + /* The user didn't pick a memory limit. Choose a very large one + * that is still smaller than the system memory */ + static int notice_sent = 0; + size_t ram = 0; + if (get_total_system_memory(&ram) < 0) { + /* We couldn't determine our total system memory! */ +#if SIZEOF_VOID_P >= 8 + /* 64-bit system. Let's hope for 8 GB. */ + result = 8 * ONE_GIGABYTE; +#else + /* (presumably) 32-bit system. Let's hope for 1 GB. */ + result = ONE_GIGABYTE; +#endif + } else { + /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ + const uint64_t avail = (ram / 4) * 3; + + /* Make sure it's in range from 0.25 GB to 8 GB. */ + if (avail > MAX_DEFAULT_MAXMEM) { + /* If you want to use more than this much RAM, you need to configure + it yourself */ + result = MAX_DEFAULT_MAXMEM; + } else if (avail < ONE_GIGABYTE / 4) { + result = ONE_GIGABYTE / 4; + } else { + result = avail; + } + } + if (log_guess && ! notice_sent) { + log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. " + "You can override this by setting MaxMemInQueues by hand.", + ram ? "Based on detected system memory, " : "", + U64_PRINTF_ARG(result / ONE_MEGABYTE)); + notice_sent = 1; + } + return result; + } else if (val < ONE_GIGABYTE / 4) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " + "Ideally, have it as large as you can afford."); + return ONE_GIGABYTE / 4; + } else { + /* The value was fine all along */ + return val; + } +} + /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ static int @@ -3317,6 +3663,12 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (old->Sandbox != new_val->Sandbox) { + *msg = tor_strdup("While Tor is running, changing Sandbox " + "is not allowed."); + return -1; + } + if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { tor_asprintf(msg, "While Tor is running, changing DataDirectory " @@ -3369,6 +3721,38 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (sandbox_is_active()) { +#define SB_NOCHANGE_STR(opt) \ + do { \ + if (! opt_streq(old->opt, new_val->opt)) { \ + *msg = tor_strdup("Can't change " #opt " while Sandbox is active"); \ + return -1; \ + } \ + } while (0) + + SB_NOCHANGE_STR(PidFile); + SB_NOCHANGE_STR(ServerDNSResolvConfFile); + SB_NOCHANGE_STR(DirPortFrontPage); + SB_NOCHANGE_STR(CookieAuthFile); + SB_NOCHANGE_STR(ExtORPortCookieAuthFile); + +#undef SB_NOCHANGE_STR + + if (! config_lines_eq(old->Logs, new_val->Logs)) { + *msg = tor_strdup("Can't change Logs while Sandbox is active"); + return -1; + } + if (old->ConnLimit != new_val->ConnLimit) { + *msg = tor_strdup("Can't change ConnLimit while Sandbox is active"); + return -1; + } + if (server_mode(old) != server_mode(new_val)) { + *msg = tor_strdup("Can't start/stop being a server while " + "Sandbox is active"); + return -1; + } + } + return 0; } @@ -3514,31 +3898,63 @@ get_default_conf_file(int defaults_file) } /** Verify whether lst is a string containing valid-looking comma-separated - * nicknames, or NULL. Return 0 on success. Warn and return -1 on failure. + * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname + * or fingerprint that needs it. Return 0 on success. + * Warn and return -1 on failure. */ static int -check_nickname_list(const char *lst, const char *name, char **msg) +check_nickname_list(char **lst, const char *name, char **msg) { int r = 0; smartlist_t *sl; + int changes = 0; - if (!lst) + if (!*lst) return 0; sl = smartlist_new(); - smartlist_split_string(sl, lst, ",", + smartlist_split_string(sl, *lst, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); - SMARTLIST_FOREACH(sl, const char *, s, + SMARTLIST_FOREACH_BEGIN(sl, char *, s) { if (!is_legal_nickname_or_hexdigest(s)) { + // check if first char is dollar + if (s[0] != '$') { + // Try again but with a dollar symbol prepended + char *prepended; + tor_asprintf(&prepended, "$%s", s); + + if (is_legal_nickname_or_hexdigest(prepended)) { + // The nickname is valid when it's prepended, swap the current + // version with a prepended one + tor_free(s); + SMARTLIST_REPLACE_CURRENT(sl, s, prepended); + changes = 1; + continue; + } + + // Still not valid, free and fallback to error message + tor_free(prepended); + } + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } - }); + } + SMARTLIST_FOREACH_END(s); + + // Replace the caller's nickname list with a fixed one + if (changes && r == 0) { + char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL); + tor_free(*lst); + *lst = newNicknames; + } + SMARTLIST_FOREACH(sl, char *, s, tor_free(s)); smartlist_free(sl); + return r; } @@ -3554,26 +3970,26 @@ check_nickname_list(const char *lst, const char *name, char **msg) * filename if it doesn't exist. */ static char * -find_torrc_filename(int argc, char **argv, +find_torrc_filename(config_line_t *cmd_arg, int defaults_file, int *using_default_fname, int *ignore_missing_torrc) { char *fname=NULL; - int i; + config_line_t *p_index; const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f"; const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc"; if (defaults_file) *ignore_missing_torrc = 1; - for (i = 1; i < argc; ++i) { - if (i < argc-1 && !strcmp(argv[i],fname_opt)) { + for (p_index = cmd_arg; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key, fname_opt)) { if (fname) { log_warn(LD_CONFIG, "Duplicate %s options on command line.", fname_opt); tor_free(fname); } - fname = expand_filename(argv[i+1]); + fname = expand_filename(p_index->value); { char *absfname; @@ -3583,8 +3999,7 @@ find_torrc_filename(int argc, char **argv, } *using_default_fname = 0; - ++i; - } else if (ignore_opt && !strcmp(argv[i],ignore_opt)) { + } else if (ignore_opt && !strcmp(p_index->key,ignore_opt)) { *ignore_missing_torrc = 1; } } @@ -3621,7 +4036,7 @@ find_torrc_filename(int argc, char **argv, * Return the contents of the file on success, and NULL on failure. */ static char * -load_torrc_from_disk(int argc, char **argv, int defaults_file) +load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file) { char *fname=NULL; char *cf = NULL; @@ -3629,7 +4044,7 @@ load_torrc_from_disk(int argc, char **argv, int defaults_file) int ignore_missing_torrc = 0; char **fname_var = defaults_file ? &torrc_defaults_fname : &torrc_fname; - fname = find_torrc_filename(argc, argv, defaults_file, + fname = find_torrc_filename(cmd_arg, defaults_file, &using_default_torrc, &ignore_missing_torrc); tor_assert(fname); log_debug(LD_CONFIG, "Opening config file \"%s\"", fname); @@ -3671,59 +4086,75 @@ int options_init_from_torrc(int argc, char **argv) { char *cf=NULL, *cf_defaults=NULL; - int i, command; + int command; int retval = -1; - static char **backup_argv; - static int backup_argc; char *command_arg = NULL; char *errmsg=NULL; + config_line_t *p_index = NULL; + config_line_t *cmdline_only_options = NULL; - if (argv) { /* first time we're called. save command line args */ - backup_argv = argv; - backup_argc = argc; - } else { /* we're reloading. need to clean up old options first. */ - argv = backup_argv; - argc = backup_argc; + /* Go through command-line variables */ + if (! have_parsed_cmdline) { + /* Or we could redo the list every time we pass this place. + * It does not really matter */ + if (config_parse_commandline(argc, argv, 0, &global_cmdline_options, + &global_cmdline_only_options) < 0) { + goto err; + } + have_parsed_cmdline = 1; } - if (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1],"--help"))) { + cmdline_only_options = global_cmdline_only_options; + + if (config_line_find(cmdline_only_options, "-h") || + config_line_find(cmdline_only_options, "--help")) { print_usage(); exit(0); } - if (argc > 1 && !strcmp(argv[1], "--list-torrc-options")) { + if (config_line_find(cmdline_only_options, "--list-torrc-options")) { /* For documenting validating whether we've documented everything. */ list_torrc_options(); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--version"))) { + if (config_line_find(cmdline_only_options, "--version")) { printf("Tor version %s.\n",get_version()); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + + if (config_line_find(cmdline_only_options, "--digests")) { printf("Tor version %s.\n",get_version()); printf("%s", libor_get_digests()); printf("%s", tor_get_digests()); exit(0); } - /* Go through command-line variables */ - if (!global_cmdline_options) { - /* Or we could redo the list every time we pass this place. - * It does not really matter */ - if (config_get_commandlines(argc, argv, &global_cmdline_options) < 0) { - goto err; - } + if (config_line_find(cmdline_only_options, "--library-versions")) { + printf("Tor version %s. \n", get_version()); + printf("Library versions\tCompiled\t\tRuntime\n"); + printf("Libevent\t\t%-15s\t\t%s\n", + tor_libevent_get_header_version_str(), + tor_libevent_get_version_str()); + printf("OpenSSL \t\t%-15s\t\t%s\n", + crypto_openssl_get_header_version_str(), + crypto_openssl_get_version_str()); + printf("Zlib \t\t%-15s\t\t%s\n", + tor_zlib_get_header_version_str(), + tor_zlib_get_version_str()); + //TODO: Hex versions? + exit(0); } command = CMD_RUN_TOR; - for (i = 1; i < argc; ++i) { - if (!strcmp(argv[i],"--list-fingerprint")) { + for (p_index = cmdline_only_options; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key,"--list-fingerprint")) { command = CMD_LIST_FINGERPRINT; - } else if (!strcmp(argv[i],"--hash-password")) { + } else if (!strcmp(p_index->key, "--hash-password")) { command = CMD_HASH_PASSWORD; - command_arg = tor_strdup( (i < argc-1) ? argv[i+1] : ""); - ++i; - } else if (!strcmp(argv[i],"--verify-config")) { + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--dump-config")) { + command = CMD_DUMP_CONFIG; + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--verify-config")) { command = CMD_VERIFY_CONFIG; } } @@ -3732,10 +4163,15 @@ options_init_from_torrc(int argc, char **argv) cf_defaults = tor_strdup(""); cf = tor_strdup(""); } else { - cf_defaults = load_torrc_from_disk(argc, argv, 1); - cf = load_torrc_from_disk(argc, argv, 0); - if (!cf) - goto err; + cf_defaults = load_torrc_from_disk(cmdline_only_options, 1); + cf = load_torrc_from_disk(cmdline_only_options, 0); + if (!cf) { + if (config_line_find(cmdline_only_options, "--allow-missing-torrc")) { + cf = tor_strdup(""); + } else { + goto err; + } + } } retval = options_init_from_string(cf_defaults, cf, command, command_arg, @@ -3779,7 +4215,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; for (i = 0; i < 2; ++i) { const char *body = i==0 ? cf_defaults : cf; @@ -3843,7 +4279,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; /* Assign all options a second time. */ for (i = 0; i < 2; ++i) { @@ -3875,7 +4311,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, } /* Validate newoptions */ - if (options_validate(oldoptions, newoptions, 0, msg) < 0) { + if (options_validate(oldoptions, newoptions, newdefaultoptions, + 0, msg) < 0) { err = SETOPT_ERR_PARSE; /*XXX make this a separate return value.*/ goto err; } @@ -4132,21 +4569,72 @@ options_init_logs(or_options_t *options, int validate_only) return ok?0:-1; } +/** Given a smartlist of SOCKS arguments to be passed to a transport + * proxy in <b>args</b>, validate them and return -1 if they are + * corrupted. Return 0 if they seem OK. */ +static int +validate_transport_socks_arguments(const smartlist_t *args) +{ + char *socks_string = NULL; + size_t socks_string_len; + + tor_assert(args); + tor_assert(smartlist_len(args) > 0); + + SMARTLIST_FOREACH_BEGIN(args, const char *, s) { + if (!string_is_key_value(LOG_WARN, s)) { /* items should be k=v items */ + log_warn(LD_CONFIG, "'%s' is not a k=v item.", s); + return -1; + } + } SMARTLIST_FOREACH_END(s); + + socks_string = pt_stringify_socks_args(args); + if (!socks_string) + return -1; + + socks_string_len = strlen(socks_string); + tor_free(socks_string); + + if (socks_string_len > MAX_SOCKS5_AUTH_SIZE_TOTAL) { + log_warn(LD_CONFIG, "SOCKS arguments can't be more than %u bytes (%lu).", + MAX_SOCKS5_AUTH_SIZE_TOTAL, + (unsigned long) socks_string_len); + return -1; + } + + return 0; +} + +/** Deallocate a bridge_line_t structure. */ +/* private */ void +bridge_line_free(bridge_line_t *bridge_line) +{ + if (!bridge_line) + return; + + if (bridge_line->socks_args) { + SMARTLIST_FOREACH(bridge_line->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge_line->socks_args); + } + tor_free(bridge_line->transport_name); + tor_free(bridge_line); +} + /** Read the contents of a Bridge line from <b>line</b>. Return 0 * if the line is well-formed, and -1 if it isn't. If * <b>validate_only</b> is 0, and the line is well-formed, then add - * the bridge described in the line to our internal bridge list. */ -static int -parse_bridge_line(const char *line, int validate_only) + * the bridge described in the line to our internal bridge list. + * + * Bridge line format: + * Bridge [transport] IP:PORT [id-fingerprint] [k=v] [k=v] ... + */ +/* private */ bridge_line_t * +parse_bridge_line(const char *line) { smartlist_t *items = NULL; - int r; char *addrport=NULL, *fingerprint=NULL; - char *transport_name=NULL; - char *field1=NULL; - tor_addr_t addr; - uint16_t port = 0; - char digest[DIGEST_LEN]; + char *field=NULL; + bridge_line_t *bridge_line = tor_malloc_zero(sizeof(bridge_line_t)); items = smartlist_new(); smartlist_split_string(items, line, NULL, @@ -4156,80 +4644,109 @@ parse_bridge_line(const char *line, int validate_only) goto err; } - /* field1 is either a transport name or addrport */ - field1 = smartlist_get(items, 0); + /* first field is either a transport name or addrport */ + field = smartlist_get(items, 0); smartlist_del_keeporder(items, 0); - if (!(strstr(field1, ".") || strstr(field1, ":"))) { - /* new-style bridge line */ - transport_name = field1; + if (string_is_C_identifier(field)) { + /* It's a transport name. */ + bridge_line->transport_name = field; if (smartlist_len(items) < 1) { log_warn(LD_CONFIG, "Too few items to Bridge line."); goto err; } - addrport = smartlist_get(items, 0); + addrport = smartlist_get(items, 0); /* Next field is addrport then. */ smartlist_del_keeporder(items, 0); } else { - addrport = field1; + addrport = field; } - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_INFO, addrport, + &bridge_line->addr, &bridge_line->port, 443)<0) { log_warn(LD_CONFIG, "Error parsing Bridge address '%s'", addrport); goto err; } - if (!port) { - log_info(LD_CONFIG, - "Bridge address '%s' has no port; using default port 443.", - addrport); - port = 443; - } + /* If transports are enabled, next field could be a fingerprint or a + socks argument. If transports are disabled, next field must be + a fingerprint. */ if (smartlist_len(items)) { - fingerprint = smartlist_join_strings(items, "", 0, NULL); + if (bridge_line->transport_name) { /* transports enabled: */ + field = smartlist_get(items, 0); + smartlist_del_keeporder(items, 0); + + /* If it's a key=value pair, then it's a SOCKS argument for the + transport proxy... */ + if (string_is_key_value(LOG_DEBUG, field)) { + bridge_line->socks_args = smartlist_new(); + smartlist_add(bridge_line->socks_args, field); + } else { /* ...otherwise, it's the bridge fingerprint. */ + fingerprint = field; + } + + } else { /* transports disabled: */ + fingerprint = smartlist_join_strings(items, "", 0, NULL); + } + } + + /* Handle fingerprint, if it was provided. */ + if (fingerprint) { if (strlen(fingerprint) != HEX_DIGEST_LEN) { log_warn(LD_CONFIG, "Key digest for Bridge is wrong length."); goto err; } - if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { + if (base16_decode(bridge_line->digest, DIGEST_LEN, + fingerprint, HEX_DIGEST_LEN)<0) { log_warn(LD_CONFIG, "Unable to decode Bridge key digest."); goto err; } } - if (!validate_only) { - log_debug(LD_DIR, "Bridge at %s (transport: %s) (%s)", - fmt_addrport(&addr, port), - transport_name ? transport_name : "no transport", - fingerprint ? fingerprint : "no key listed"); - bridge_add_from_config(&addr, port, - fingerprint ? digest : NULL, transport_name); + /* If we are using transports, any remaining items in the smartlist + should be k=v values. */ + if (bridge_line->transport_name && smartlist_len(items)) { + if (!bridge_line->socks_args) + bridge_line->socks_args = smartlist_new(); + + /* append remaining items of 'items' to 'socks_args' */ + smartlist_add_all(bridge_line->socks_args, items); + smartlist_clear(items); + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + } + + if (bridge_line->socks_args) { + if (validate_transport_socks_arguments(bridge_line->socks_args) < 0) + goto err; } - r = 0; goto done; err: - r = -1; + bridge_line_free(bridge_line); + bridge_line = NULL; done: SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); - tor_free(transport_name); tor_free(fingerprint); - return r; + + return bridge_line; } /** Read the contents of a ClientTransportPlugin line from * <b>line</b>. Return 0 if the line is well-formed, and -1 if it * isn't. * - * If <b>validate_only</b> is 0, and the line is well-formed: + * If <b>validate_only</b> is 0, the line is well-formed, and the + * transport is needed by some bridge: * - If it's an external proxy line, add the transport described in the line to * our internal transport list. * - If it's a managed proxy line, launch the managed proxy. */ static int -parse_client_transport_line(const char *line, int validate_only) +parse_client_transport_line(const or_options_t *options, + const char *line, int validate_only) { smartlist_t *items = NULL; int r; @@ -4246,7 +4763,8 @@ parse_client_transport_line(const char *line, int validate_only) int is_managed=0; char **proxy_argv=NULL; char **tmp=NULL; - int proxy_argc,i; + int proxy_argc, i; + int is_useless_proxy=1; int line_length; @@ -4268,11 +4786,16 @@ parse_client_transport_line(const char *line, int validate_only) smartlist_split_string(transport_list, transports, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport_name) { + /* validate transport names */ if (!string_is_C_identifier(transport_name)) { log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", transport_name); goto err; } + + /* see if we actually need the transports provided by this proxy */ + if (!validate_only && transport_is_needed(transport_name)) + is_useless_proxy = 0; } SMARTLIST_FOREACH_END(transport_name); /* field2 is either a SOCKS version or "exec" */ @@ -4290,10 +4813,22 @@ parse_client_transport_line(const char *line, int validate_only) goto err; } + if (is_managed && options->Sandbox) { + log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." + "(ClientTransportPlugin line was %s)", escaped(line)); + goto err; + } + if (is_managed) { /* managed */ - if (!validate_only) { /* if we are not just validating, use the - rest of the line as the argv of the proxy - to be launched */ + if (!validate_only && is_useless_proxy) { + log_notice(LD_GENERAL, "Pluggable transport proxy (%s) does not provide " + "any needed transports and will not be launched.", line); + } + + /* If we are not just validating, use the rest of the line as the + argv of the proxy to be launched. Also, make sure that we are + only launching proxies that contribute useful transports. */ + if (!validate_only && !is_useless_proxy) { proxy_argc = line_length-2; tor_assert(proxy_argc > 0); proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); @@ -4388,7 +4923,7 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) goto err; /* Validate addrport */ - if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) { log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr " "address '%s'", addrport); goto err; @@ -4407,6 +4942,63 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) return addrport; } +/** Given a ServerTransportOptions <b>line</b>, return a smartlist + * with the options. Return NULL if the line was not well-formed. + * + * If <b>transport</b> is set, return NULL if the line is not + * referring to <b>transport</b>. + * + * The returned smartlist and its strings are allocated on the heap + * and it's the responsibility of the caller to free it. */ +smartlist_t * +get_options_from_transport_options_line(const char *line,const char *transport) +{ + smartlist_t *items = smartlist_new(); + smartlist_t *options = smartlist_new(); + const char *parsed_transport = NULL; + + smartlist_split_string(items, line, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + + if (smartlist_len(items) < 2) { + log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line."); + goto err; + } + + parsed_transport = smartlist_get(items, 0); + /* If 'transport' is given, check if it matches the one on the line */ + if (transport && strcmp(transport, parsed_transport)) + goto err; + + SMARTLIST_FOREACH_BEGIN(items, const char *, option) { + if (option_sl_idx == 0) /* skip the transport field (first field)*/ + continue; + + /* validate that it's a k=v value */ + if (!string_is_key_value(LOG_WARN, option)) { + log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option)); + goto err; + } + + /* add it to the options smartlist */ + smartlist_add(options, tor_strdup(option)); + log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option)); + } SMARTLIST_FOREACH_END(option); + + goto done; + + err: + SMARTLIST_FOREACH(options, char*, s, tor_free(s)); + smartlist_free(options); + options = NULL; + + done: + SMARTLIST_FOREACH(items, char*, s, tor_free(s)); + smartlist_free(items); + + return options; +} + /** Given the name of a pluggable transport in <b>transport</b>, check * the configuration file to see if the user has explicitly asked for * it to listen on a specific port. Return a <address:port> string if @@ -4427,13 +5019,34 @@ get_transport_bindaddr_from_config(const char *transport) return NULL; } +/** Given the name of a pluggable transport in <b>transport</b>, check + * the configuration file to see if the user has asked us to pass any + * parameters to the pluggable transport. Return a smartlist + * containing the parameters, otherwise NULL. */ +smartlist_t * +get_options_for_server_transport(const char *transport) +{ + config_line_t *cl; + const or_options_t *options = get_options(); + + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, transport); + if (options_sl) + return options_sl; + } + + return NULL; +} + /** Read the contents of a ServerTransportPlugin line from * <b>line</b>. Return 0 if the line is well-formed, and -1 if it * isn't. * If <b>validate_only</b> is 0, the line is well-formed, and it's a * managed proxy line, launch the managed proxy. */ static int -parse_server_transport_line(const char *line, int validate_only) +parse_server_transport_line(const or_options_t *options, + const char *line, int validate_only) { smartlist_t *items = NULL; int r; @@ -4488,6 +5101,12 @@ parse_server_transport_line(const char *line, int validate_only) goto err; } + if (is_managed && options->Sandbox) { + log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." + "(ServerTransportPlugin line was %s)", escaped(line)); + goto err; + } + if (is_managed) { /* managed */ if (!validate_only) { proxy_argc = line_length-2; @@ -4563,8 +5182,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, uint16_t dir_port = 0, or_port = 0; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - dirinfo_type_t type = V2_DIRINFO; - int is_not_hidserv_authority = 0, is_not_v2_authority = 0; + dirinfo_type_t type = 0; double weight = 1.0; items = smartlist_new(); @@ -4584,16 +5202,15 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char *flag = smartlist_get(items, 0); if (TOR_ISDIGIT(flag[0])) break; - if (!strcasecmp(flag, "v1")) { - type |= (V1_DIRINFO | HIDSERV_DIRINFO); - } else if (!strcasecmp(flag, "hs")) { - type |= HIDSERV_DIRINFO; - } else if (!strcasecmp(flag, "no-hs")) { - is_not_hidserv_authority = 1; + if (!strcasecmp(flag, "hs") || + !strcasecmp(flag, "no-hs")) { + log_warn(LD_CONFIG, "The DirAuthority options 'hs' and 'no-hs' are " + "obsolete; you don't need them any more."); } else if (!strcasecmp(flag, "bridge")) { type |= BRIDGE_DIRINFO; } else if (!strcasecmp(flag, "no-v2")) { - is_not_v2_authority = 1; + /* obsolete, but may still be contained in DirAuthority lines generated + by various tools */; } else if (!strcasecmpstart(flag, "orport=")) { int ok; char *portstring = flag + strlen("orport="); @@ -4625,10 +5242,6 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, tor_free(flag); smartlist_del_keeporder(items, 0); } - if (is_not_hidserv_authority) - type &= ~HIDSERV_DIRINFO; - if (is_not_v2_authority) - type &= ~V2_DIRINFO; if (smartlist_len(items) < 2) { log_warn(LD_CONFIG, "Too few arguments to DirAuthority line."); @@ -4831,6 +5444,27 @@ warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname, } SMARTLIST_FOREACH_END(port); } +/** Warn for every Extended ORPort port in <b>ports</b> that is on a + * publicly routable address. */ +static void +warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname) +{ + SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { + if (port->type != CONN_TYPE_EXT_OR_LISTENER) + continue; + if (port->is_unix_addr) + continue; + /* XXX maybe warn even if address is RFC1918? */ + if (!tor_addr_is_internal(&port->addr, 1)) { + log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. " + "This is not advised; this address is supposed to only be " + "exposed on localhost so that your pluggable transport " + "proxies can connect to it.", + fmt_addrport(&port->addr, port->port), portname); + } + } SMARTLIST_FOREACH_END(port); +} + /** Given a list of port_cfg_t in <b>ports</b>, warn any controller port there * is listening on any non-loopback address. If <b>forbid</b> is true, * then emit a stronger warning and remove the port from the list. @@ -4931,6 +5565,7 @@ parse_port_config(smartlist_t *out, smartlist_t *elts; int retval = -1; const unsigned is_control = (listener_type == CONN_TYPE_CONTROL_LISTENER); + const unsigned is_ext_orport = (listener_type == CONN_TYPE_EXT_OR_LISTENER); const unsigned allow_no_options = flags & CL_PORT_NO_OPTIONS; const unsigned use_server_options = flags & CL_PORT_SERVER_OPTIONS; const unsigned warn_nonlocal = flags & CL_PORT_WARN_NONLOCAL; @@ -5008,6 +5643,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5281,6 +5918,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5427,6 +6066,14 @@ parse_ports(or_options_t *options, int validate_only, goto err; } if (parse_port_config(ports, + options->ExtORPort_lines, NULL, + "ExtOR", CONN_TYPE_EXT_OR_LISTENER, + "127.0.0.1", 0, + CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) { + *msg = tor_strdup("Invalid ExtORPort configuration"); + goto err; + } + if (parse_port_config(ports, options->DirPort_lines, options->DirListenAddress, "Dir", CONN_TYPE_DIR_LISTENER, "0.0.0.0", 0, @@ -5461,6 +6108,8 @@ parse_ports(or_options_t *options, int validate_only, !! count_real_listeners(ports, CONN_TYPE_DIR_LISTENER); options->DNSPort_set = !! count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER); + options->ExtORPort_set = + !! count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER); if (!validate_only) { if (configured_ports) { @@ -5748,7 +6397,7 @@ write_configuration_file(const char *fname, const or_options_t *options) return -1; } - if (!(new_conf = options_dump(options, 1))) { + if (!(new_conf = options_dump(options, OPTIONS_DUMP_MINIMAL))) { log_warn(LD_BUG, "Couldn't get configuration string"); goto err; } @@ -5767,7 +6416,7 @@ write_configuration_file(const char *fname, const or_options_t *options) ++i; } log_notice(LD_CONFIG, "Renaming old configuration file to \"%s\"", fn_tmp); - if (rename(fname, fn_tmp) < 0) { + if (tor_rename(fname, fn_tmp) < 0) {//XXXX sandbox doesn't allow log_warn(LD_FS, "Couldn't rename configuration file \"%s\" to \"%s\": %s", fname, fn_tmp, strerror(errno)); @@ -5908,6 +6557,43 @@ options_get_datadir_fname2_suffix(const or_options_t *options, return fname; } +/** Check wether the data directory has a private subdirectory + * <b>subdir</b>. If not, try to create it. Return 0 on success, + * -1 otherwise. */ +int +check_or_create_data_subdir(const char *subdir) +{ + char *statsdir = get_datadir_fname(subdir); + int return_val = 0; + + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create %s/ directory!", subdir); + return_val = -1; + } + tor_free(statsdir); + return return_val; +} + +/** Create a file named <b>fname</b> with contents <b>str</b> in the + * subdirectory <b>subdir</b> of the data directory. <b>descr</b> + * should be a short description of the file's content and will be + * used for the warning message, if it's present and the write process + * fails. Return 0 on success, -1 otherwise.*/ +int +write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr) +{ + char *filename = get_datadir_fname2(subdir, fname); + int return_val = 0; + + if (write_str_to_file(filename, str, 0) < 0) { + log_warn(LD_HIST, "Unable to write %s to disk!", descr ? descr : fname); + return_val = -1; + } + tor_free(filename); + return return_val; +} + /** Given a file name check to see whether the file exists but has not been * modified for a very long time. If so, remove it. */ void @@ -5916,12 +6602,17 @@ remove_file_if_very_old(const char *fname, time_t now) #define VERY_OLD_FILE_AGE (28*24*60*60) struct stat st; - if (stat(fname, &st)==0 && st.st_mtime < now-VERY_OLD_FILE_AGE) { + log_debug(LD_FS, "stat()ing %s", fname); + if (stat(sandbox_intern_string(fname), &st)==0 && + st.st_mtime < now-VERY_OLD_FILE_AGE) { char buf[ISO_TIME_LEN+1]; format_local_iso_time(buf, st.st_mtime); log_notice(LD_GENERAL, "Obsolete file %s hasn't been modified since %s. " "Removing it.", fname, buf); - unlink(fname); + if (unlink(fname) != 0) { + log_warn(LD_FS, "Failed to unlink %s: %s", + fname, strerror(errno)); + } } } @@ -5997,6 +6688,7 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_ISOTIME: type = "Time"; break; case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; case CONFIG_TYPE_CSV: type = "CommaList"; break; + case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break; case CONFIG_TYPE_LINELIST: type = "LineList"; break; case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break; case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; @@ -6128,3 +6820,58 @@ config_maybe_load_geoip_files_(const or_options_t *options, config_load_geoip_file_(AF_INET6, options->GeoIPv6File, "geoip6"); } +/** Initialize cookie authentication (used so far by the ControlPort + * and Extended ORPort). + * + * Allocate memory and create a cookie (of length <b>cookie_len</b>) + * in <b>cookie_out</b>. + * Then write it down to <b>fname</b> and prepend it with <b>header</b>. + * + * If the whole procedure was successful, set + * <b>cookie_is_set_out</b> to True. */ +int +init_cookie_authentication(const char *fname, const char *header, + int cookie_len, + uint8_t **cookie_out, int *cookie_is_set_out) +{ + char cookie_file_str_len = strlen(header) + cookie_len; + char *cookie_file_str = tor_malloc(cookie_file_str_len); + int retval = -1; + + /* We don't want to generate a new cookie every time we call + * options_act(). One should be enough. */ + if (*cookie_is_set_out) { + retval = 0; /* we are all set */ + goto done; + } + + /* If we've already set the cookie, free it before re-setting + it. This can happen if we previously generated a cookie, but + couldn't write it to a disk. */ + if (*cookie_out) + tor_free(*cookie_out); + + /* Generate the cookie */ + *cookie_out = tor_malloc(cookie_len); + if (crypto_rand((char *)*cookie_out, cookie_len) < 0) + goto done; + + /* Create the string that should be written on the file. */ + memcpy(cookie_file_str, header, strlen(header)); + memcpy(cookie_file_str+strlen(header), *cookie_out, cookie_len); + if (write_bytes_to_file(fname, cookie_file_str, cookie_file_str_len, 1)) { + log_warn(LD_FS,"Error writing auth cookie to %s.", escaped(fname)); + goto done; + } + + /* Success! */ + log_info(LD_GENERAL, "Generated auth cookie file in '%s'.", escaped(fname)); + *cookie_is_set_out = 1; + retval = 0; + + done: + memwipe(cookie_file_str, 0, cookie_file_str_len); + tor_free(cookie_file_str); + return retval; +} + diff --git a/src/or/config.h b/src/or/config.h index ef4acac51..bf386134b 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -12,8 +12,10 @@ #ifndef TOR_CONFIG_H #define TOR_CONFIG_H +#include "testsupport.h" + const char *get_dirportfrontpage(void); -const or_options_t *get_options(void); +MOCK_DECL(const or_options_t *,get_options,(void)); or_options_t *get_options_mutable(void); int set_options(or_options_t *new_val, char **msg); void config_free_all(void); @@ -32,7 +34,11 @@ int resolve_my_address(int warn_severity, const or_options_t *options, const char **method_out, char **hostname_out); int is_local_addr(const tor_addr_t *addr); void options_init(or_options_t *options); -char *options_dump(const or_options_t *options, int minimal); + +#define OPTIONS_DUMP_MINIMAL 1 +#define OPTIONS_DUMP_DEFAULTS 2 +#define OPTIONS_DUMP_ALL 3 +char *options_dump(const or_options_t *options, int how_to_dump); int options_init_from_torrc(int argc, char **argv); setopt_err_t options_init_from_string(const char *cf_defaults, const char *cf, int command, const char *command_arg, char **msg); @@ -59,6 +65,10 @@ char *options_get_datadir_fname2_suffix(const or_options_t *options, #define get_datadir_fname_suffix(sub1, suffix) \ get_datadir_fname2_suffix((sub1), NULL, (suffix)) +int check_or_create_data_subdir(const char *subdir); +int write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr); + int get_num_cpus(const or_options_t *options); const smartlist_t *get_configured_ports(void); @@ -86,10 +96,15 @@ uint32_t get_effective_bwburst(const or_options_t *options); char *get_transport_bindaddr_from_config(const char *transport); -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ +int init_cookie_authentication(const char *fname, const char *header, + int cookie_len, + uint8_t **cookie_out, int *cookie_is_set_out); + or_options_t *options_new(void); -#endif + +int config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result); void config_register_addressmaps(const or_options_t *options); /* XXXX024 move to connection_edge.h */ @@ -98,5 +113,34 @@ int addressmap_register_auto(const char *from, const char *to, addressmap_entry_source_t addrmap_source, const char **msg); +/** Represents the information stored in a torrc Bridge line. */ +typedef struct bridge_line_t { + tor_addr_t addr; /* The IP address of the bridge. */ + uint16_t port; /* The TCP port of the bridge. */ + char *transport_name; /* The name of the pluggable transport that + should be used to connect to the bridge. */ + char digest[DIGEST_LEN]; /* The bridge's identity key digest. */ + smartlist_t *socks_args; /* SOCKS arguments for the pluggable + transport proxy. */ +} bridge_line_t; + +void bridge_line_free(bridge_line_t *bridge_line); +bridge_line_t *parse_bridge_line(const char *line); +smartlist_t *get_options_from_transport_options_line(const char *line, + const char *transport); +smartlist_t *get_options_for_server_transport(const char *transport); + +#ifdef CONFIG_PRIVATE +#ifdef TOR_UNIT_TESTS +extern struct config_format_t options_format; +#endif + +STATIC void or_options_free(or_options_t *options); +STATIC int options_validate(or_options_t *old_options, + or_options_t *options, + or_options_t *default_options, + int from_setconf, char **msg); +#endif + #endif diff --git a/src/or/confparse.c b/src/or/confparse.c index 8863d9240..c5400a651 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -79,6 +79,21 @@ config_line_append(config_line_t **lst, (*lst) = newline; } +/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL + * if no such key exists. For handling commandline-only options only; other + * options should be looked up in the appropriate data structure. */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Helper: parse the config string and strdup into key/value * strings. Set *result to the list, or NULL if parsing the string * failed. Return 0 on success, -1 on failure. Warn and ignore any @@ -223,6 +238,8 @@ config_assign_value(const config_format_t *fmt, void *options, int i, ok; const config_var_t *var; void *lvalue; + int *csv_int; + smartlist_t *csv_str; CONFIG_CHECK(fmt, options); @@ -357,6 +374,36 @@ config_assign_value(const config_format_t *fmt, void *options, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp)); + smartlist_clear(*(smartlist_t**)lvalue); + } else { + *(smartlist_t**)lvalue = smartlist_new(); + } + csv_str = smartlist_new(); + smartlist_split_string(csv_str, c->value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(csv_str, char *, str) + { + i = config_parse_interval(str, &ok); + if (!ok) { + tor_asprintf(msg, + "Interval in '%s %s' is malformed or out of bounds.", + c->key, c->value); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + return -1; + } + csv_int = tor_malloc_zero(sizeof(int)); + *csv_int = i; + smartlist_add(*(smartlist_t**)lvalue, csv_int); + } + SMARTLIST_FOREACH_END(str); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + break; + case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: { @@ -555,6 +602,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const config_var_t *var; const void *value; config_line_t *result; + smartlist_t *csv_str; tor_assert(options && key); CONFIG_CHECK(fmt, options); @@ -637,6 +685,20 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, else result->value = tor_strdup(""); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)value) { + csv_str = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i) + { + smartlist_add_asprintf(csv_str, "%d", *i); + } + SMARTLIST_FOREACH_END(i); + result->value = smartlist_join_strings(csv_str, ",", 0, NULL); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + } else + result->value = tor_strdup(""); + break; case CONFIG_TYPE_OBSOLETE: log_fn(LOG_INFO, LD_CONFIG, "You asked me for the value of an obsolete config option '%s'.", @@ -826,6 +888,13 @@ config_clear(const config_format_t *fmt, void *options, *(smartlist_t **)lvalue = NULL; } break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp)); + smartlist_free(*(smartlist_t **)lvalue); + *(smartlist_t **)lvalue = NULL; + } + break; case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: config_free_lines(*(config_line_t **)lvalue); @@ -1005,8 +1074,8 @@ config_dump(const config_format_t *fmt, const void *default_options, /* XXX use a 1 here so we don't add a new log line while dumping */ if (default_options == NULL) { - if (fmt->validate_fn(NULL, defaults_tmp, 1, &msg) < 0) { - log_err(LD_BUG, "Failed to validate default config."); + if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + log_err(LD_BUG, "Failed to validate default config: %s", msg); tor_free(msg); tor_assert(0); } @@ -1072,20 +1141,36 @@ static struct unit_table_t memory_units[] = { { "kbytes", 1<<10 }, { "kilobyte", 1<<10 }, { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, { "m", 1<<20 }, { "mb", 1<<20 }, { "mbyte", 1<<20 }, { "mbytes", 1<<20 }, { "megabyte", 1<<20 }, { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, { "gb", 1<<30 }, { "gbyte", 1<<30 }, { "gbytes", 1<<30 }, { "gigabyte", 1<<30 }, { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, { "tb", U64_LITERAL(1)<<40 }, { "terabyte", U64_LITERAL(1)<<40 }, { "terabytes", U64_LITERAL(1)<<40 }, + { "terabits", U64_LITERAL(1)<<37 }, + { "terabit", U64_LITERAL(1)<<37 }, + { "tbits", U64_LITERAL(1)<<37 }, + { "tbit", U64_LITERAL(1)<<37 }, { NULL, 0 }, }; diff --git a/src/or/confparse.h b/src/or/confparse.h index 1b987f3bf..2cd6c49a2 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -26,6 +26,9 @@ typedef enum config_type_t { CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units */ CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, * mixed with other keywords. */ @@ -68,12 +71,12 @@ typedef struct config_var_description_t { /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ -typedef int (*validate_fn_t)(void*,void*,int,char**); +typedef int (*validate_fn_t)(void*,void*,void*,int,char**); /** Information on the keys, value types, key-to-struct-member mappings, * variable descriptions, validation functions, and abbreviations for a * configuration or storage format. */ -typedef struct { +typedef struct config_format_t { size_t size; /**< Size of the struct that everything gets parsed into. */ uint32_t magic; /**< Required 'magic value' to make sure we have a struct * of the right type. */ @@ -100,6 +103,8 @@ void *config_new(const config_format_t *fmt); void config_line_append(config_line_t **lst, const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); void config_free(const config_format_t *fmt, void *options); int config_lines_eq(config_line_t *a, config_line_t *b); int config_count_key(const config_line_t *a, const char *key); diff --git a/src/or/connection.c b/src/or/connection.c index 4f74a1d04..cef9172ff 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -10,6 +10,7 @@ * on connections. **/ +#define CONNECTION_PRIVATE #include "or.h" #include "buffers.h" /* @@ -17,6 +18,7 @@ * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ +#define CONNECTION_PRIVATE #include "channel.h" #include "channeltls.h" #include "circuitbuild.h" @@ -33,6 +35,7 @@ #include "dns.h" #include "dnsserv.h" #include "entrynodes.h" +#include "ext_orport.h" #include "geoip.h" #include "main.h" #include "policies.h" @@ -44,6 +47,7 @@ #include "router.h" #include "transports.h" #include "routerparse.h" +#include "transports.h" #ifdef USE_BUFFEREVENTS #include <event2/event.h> @@ -97,6 +101,7 @@ static smartlist_t *outgoing_addrs = NULL; #define CASE_ANY_LISTENER_TYPE \ case CONN_TYPE_OR_LISTENER: \ + case CONN_TYPE_EXT_OR_LISTENER: \ case CONN_TYPE_AP_LISTENER: \ case CONN_TYPE_DIR_LISTENER: \ case CONN_TYPE_CONTROL_LISTENER: \ @@ -128,6 +133,8 @@ conn_type_to_string(int type) case CONN_TYPE_CPUWORKER: return "CPU worker"; case CONN_TYPE_CONTROL_LISTENER: return "Control listener"; case CONN_TYPE_CONTROL: return "Control"; + case CONN_TYPE_EXT_OR: return "Extended OR"; + case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener"; default: log_warn(LD_BUG, "unknown connection type %d", type); tor_snprintf(buf, sizeof(buf), "unknown [%d]", type); @@ -164,6 +171,18 @@ conn_state_to_string(int type, int state) case OR_CONN_STATE_OPEN: return "open"; } break; + case CONN_TYPE_EXT_OR: + switch (state) { + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return "waiting for authentication type"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return "waiting for client nonce"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return "waiting for client hash"; + case EXT_OR_CONN_STATE_OPEN: return "open"; + case EXT_OR_CONN_STATE_FLUSHING: return "flushing final OKAY"; + } + break; case CONN_TYPE_EXIT: switch (state) { case EXIT_CONN_STATE_RESOLVING: return "waiting for dest info"; @@ -228,6 +247,7 @@ connection_type_uses_bufferevent(connection_t *conn) case CONN_TYPE_DIR: case CONN_TYPE_CONTROL: case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: case CONN_TYPE_CPUWORKER: return 1; default: @@ -251,21 +271,23 @@ dir_connection_new(int socket_family) * * Set timestamp_last_added_nonpadding to now. * - * Assign a pseudorandom next_circ_id between 0 and 2**15. - * * Initialize active_circuit_pqueue. * * Set active_circuit_pqueue_last_recalibrated to current cell_ewma tick. */ or_connection_t * -or_connection_new(int socket_family) +or_connection_new(int type, int socket_family) { or_connection_t *or_conn = tor_malloc_zero(sizeof(or_connection_t)); time_t now = time(NULL); - connection_init(now, TO_CONN(or_conn), CONN_TYPE_OR, socket_family); + tor_assert(type == CONN_TYPE_OR || type == CONN_TYPE_EXT_OR); + connection_init(now, TO_CONN(or_conn), type, socket_family); or_conn->timestamp_last_added_nonpadding = time(NULL); + if (type == CONN_TYPE_EXT_OR) + connection_or_set_ext_or_identifier(or_conn); + return or_conn; } @@ -311,7 +333,6 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); - log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -334,7 +355,8 @@ connection_new(int type, int socket_family) { switch (type) { case CONN_TYPE_OR: - return TO_CONN(or_connection_new(socket_family)); + case CONN_TYPE_EXT_OR: + return TO_CONN(or_connection_new(type, socket_family)); case CONN_TYPE_EXIT: return TO_CONN(edge_connection_new(type, socket_family)); @@ -376,6 +398,7 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) switch (type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: conn->magic = OR_CONNECTION_MAGIC; break; case CONN_TYPE_EXIT: @@ -434,7 +457,7 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. */ -static void +STATIC void connection_free_(connection_t *conn) { void *mem; @@ -444,6 +467,7 @@ connection_free_(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); mem = TO_OR_CONN(conn); memlen = sizeof(or_connection_t); @@ -590,6 +614,13 @@ connection_free_(connection_t *conn) log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest"); connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } + if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) { + connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn)); + tor_free(TO_OR_CONN(conn)->ext_or_conn_id); + tor_free(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash); + tor_free(TO_OR_CONN(conn)->ext_or_transport); + } + #ifdef USE_BUFFEREVENTS if (conn->type == CONN_TYPE_OR && TO_OR_CONN(conn)->bucket_cfg) { ev_token_bucket_cfg_free(TO_OR_CONN(conn)->bucket_cfg); @@ -653,6 +684,7 @@ connection_about_to_close_connection(connection_t *conn) connection_dir_about_to_close(TO_DIR_CONN(conn)); break; case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: connection_or_about_to_close(TO_OR_CONN(conn)); break; case CONN_TYPE_AP: @@ -892,8 +924,11 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) int r = -1; char *p = tor_strdup(path); cpd_check_t flags = CPD_CHECK_MODE_ONLY; - if (get_parent_directory(p)<0) + if (get_parent_directory(p)<0 || p[0] != '/') { + log_warn(LD_GENERAL, "Bad unix socket address '%s'. Tor does not support " + "relative paths for unix sockets.", path); goto done; + } if (options->ControlSocketsGroupWritable) flags |= CPD_GROUP_OK; @@ -921,12 +956,14 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) #endif /** Tell the TCP stack that it shouldn't wait for a long time after - * <b>sock</b> has closed before reusing its port. */ -static void + * <b>sock</b> has closed before reusing its port. Return 0 on success, + * -1 on failure. */ +static int make_socket_reuseable(tor_socket_t sock) { #ifdef _WIN32 (void) sock; + return 0; #else int one=1; @@ -936,9 +973,9 @@ make_socket_reuseable(tor_socket_t sock) * already has it bound_. So, don't do that on Win32. */ if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, (socklen_t)sizeof(one)) == -1) { - log_warn(LD_NET, "Error setting SO_REUSEADDR flag: %s", - tor_socket_strerror(errno)); + return -1; } + return 0; #endif } @@ -971,16 +1008,16 @@ tor_listen(tor_socket_t fd) */ static connection_t * connection_listener_new(const struct sockaddr *listensockaddr, - socklen_t socklen, - int type, const char *address, - const port_cfg_t *port_cfg) + socklen_t socklen, + int type, const char *address, + const port_cfg_t *port_cfg) { listener_connection_t *lis_conn; - connection_t *conn; - tor_socket_t s; /* the socket we're going to make */ + connection_t *conn = NULL; + tor_socket_t s = TOR_INVALID_SOCKET; /* the socket we're going to make */ or_options_t const *options = get_options(); #if defined(HAVE_PWD_H) && defined(HAVE_SYS_UN_H) - struct passwd *pw = NULL; + const struct passwd *pw = NULL; #endif uint16_t usePort = 0, gotPort = 0; int start_reading = 0; @@ -1003,7 +1040,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), fmt_addrport(&addr, usePort)); - s = tor_open_socket(tor_addr_family(&addr), + s = tor_open_socket_nonblocking(tor_addr_family(&addr), is_tcp ? SOCK_STREAM : SOCK_DGRAM, is_tcp ? IPPROTO_TCP: IPPROTO_UDP); if (!SOCKET_OK(s)) { @@ -1012,7 +1049,27 @@ connection_listener_new(const struct sockaddr *listensockaddr, goto err; } - make_socket_reuseable(s); + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(type), + tor_socket_strerror(errno)); + } + +#if defined USE_TRANSPARENT && defined(IP_TRANSPARENT) + if (options->TransProxyType_parsed == TPT_TPROXY && + type == CONN_TYPE_AP_TRANS_LISTENER) { + int one = 1; + if (setsockopt(s, SOL_IP, IP_TRANSPARENT, &one, sizeof(one)) < 0) { + const char *extra = ""; + int e = tor_socket_errno(s); + if (e == EPERM) + extra = "TransTPROXY requires root privileges or similar" + " capabilities."; + log_warn(LD_NET, "Error setting IP_TRANSPARENT flag: %s.%s", + tor_socket_strerror(e), extra); + } + } +#endif #ifdef IPV6_V6ONLY if (listensockaddr->sa_family == AF_INET6) { @@ -1025,7 +1082,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, /* We need to set IPV6_V6ONLY so that this socket can't get used for * IPv4 connections. */ if (setsockopt(s,IPPROTO_IPV6, IPV6_V6ONLY, - (void*)&one, sizeof(one))<0) { + (void*)&one, sizeof(one)) < 0) { int e = tor_socket_errno(s); log_warn(LD_NET, "Error setting IPV6_V6ONLY flag: %s", tor_socket_strerror(e)); @@ -1041,7 +1098,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, helpfulhint = ". Is Tor already running?"; log_warn(LD_NET, "Could not bind to %s:%u: %s%s", address, usePort, tor_socket_strerror(e), helpfulhint); - tor_close_socket(s); goto err; } @@ -1049,7 +1105,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (tor_listen(s) < 0) { log_warn(LD_NET, "Could not listen on %s:%u: %s", address, usePort, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } } @@ -1089,7 +1144,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, strerror(errno)); goto err; } - s = tor_open_socket(AF_UNIX, SOCK_STREAM, 0); + s = tor_open_socket_nonblocking(AF_UNIX, SOCK_STREAM, 0); if (! SOCKET_OK(s)) { log_warn(LD_NET,"Socket creation failed: %s.", strerror(errno)); goto err; @@ -1098,21 +1153,18 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (bind(s, listensockaddr, (socklen_t)sizeof(struct sockaddr_un)) == -1) { log_warn(LD_NET,"Bind to %s failed: %s.", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } #ifdef HAVE_PWD_H if (options->User) { - pw = getpwnam(options->User); + pw = tor_getpwnam(options->User); if (pw == NULL) { log_warn(LD_NET,"Unable to chown() %s socket: user %s not found.", address, options->User); - tor_close_socket(s); goto err; } else if (chown(address, pw->pw_uid, pw->pw_gid) < 0) { log_warn(LD_NET,"Unable to chown() %s socket: %s.", address, strerror(errno)); - tor_close_socket(s); goto err; } } @@ -1122,35 +1174,29 @@ connection_listener_new(const struct sockaddr *listensockaddr, * platforms. */ if (chmod(address, 0660) < 0) { log_warn(LD_FS,"Unable to make %s group-writable.", address); - tor_close_socket(s); goto err; } } - if (listen(s,SOMAXCONN) < 0) { + if (listen(s, SOMAXCONN) < 0) { log_warn(LD_NET, "Could not listen on %s: %s", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } #else (void)options; #endif /* HAVE_SYS_UN_H */ } else { - log_err(LD_BUG,"Got unexpected address family %d.", - listensockaddr->sa_family); - tor_assert(0); - } - - if (set_socket_nonblocking(s) == -1) { - tor_close_socket(s); - goto err; + log_err(LD_BUG, "Got unexpected address family %d.", + listensockaddr->sa_family); + tor_assert(0); } lis_conn = listener_connection_new(type, listensockaddr->sa_family); conn = TO_CONN(lis_conn); conn->socket_family = listensockaddr->sa_family; conn->s = s; + s = TOR_INVALID_SOCKET; /* Prevent double-close */ conn->address = tor_strdup(address); conn->port = gotPort; tor_addr_copy(&conn->addr, &addr); @@ -1186,7 +1232,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); - connection_free(conn); goto err; } @@ -1205,6 +1250,11 @@ connection_listener_new(const struct sockaddr *listensockaddr, return conn; err: + if (SOCKET_OK(s)) + tor_close_socket(s); + if (conn) + connection_free(conn); + return NULL; } @@ -1289,7 +1339,7 @@ connection_handle_listener_read(connection_t *conn, int new_type) tor_assert((size_t)remotelen >= sizeof(struct sockaddr_in)); memset(&addrbuf, 0, sizeof(addrbuf)); - news = tor_accept_socket(conn->s,remote,&remotelen); + news = tor_accept_socket_nonblocking(conn->s,remote,&remotelen); if (!SOCKET_OK(news)) { /* accept() error */ int e = tor_socket_errno(conn->s); if (ERRNO_IS_ACCEPT_EAGAIN(e)) { @@ -1308,8 +1358,15 @@ connection_handle_listener_read(connection_t *conn, int new_type) "Connection accepted on socket %d (child of fd %d).", (int)news,(int)conn->s); - make_socket_reuseable(news); - if (set_socket_nonblocking(news) == -1) { + if (make_socket_reuseable(news) < 0) { + if (tor_socket_errno(news) == EINVAL) { + /* This can happen on OSX if we get a badly timed shutdown. */ + log_debug(LD_NET, "make_socket_reuseable returned EINVAL"); + } else { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(new_type), + tor_socket_strerror(errno)); + } tor_close_socket(news); return 0; } @@ -1367,11 +1424,17 @@ connection_handle_listener_read(connection_t *conn, int new_type) TO_ENTRY_CONN(newconn)->socks_request->socks_prefer_no_auth = TO_LISTENER_CONN(conn)->socks_prefer_no_auth; } + if (new_type == CONN_TYPE_CONTROL) { + log_notice(LD_CONTROL, "New control connection opened from %s.", + fmt_and_decorate_addr(&addr)); + } } else if (conn->socket_family == AF_UNIX) { /* For now only control ports can be Unix domain sockets * and listeners at the same time */ tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(new_type == CONN_TYPE_CONTROL); + log_notice(LD_CONTROL, "New control connection opened."); newconn = connection_new(new_type, conn->socket_family); newconn->s = news; @@ -1411,6 +1474,9 @@ connection_init_accepted_conn(connection_t *conn, connection_start_reading(conn); switch (conn->type) { + case CONN_TYPE_EXT_OR: + /* Initiate Extended ORPort authentication. */ + return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); @@ -1504,7 +1570,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - s = tor_open_socket(protocol_family,SOCK_STREAM,IPPROTO_TCP); + s = tor_open_socket_nonblocking(protocol_family,SOCK_STREAM,IPPROTO_TCP); if (! SOCKET_OK(s)) { *socket_error = tor_socket_errno(-1); log_warn(LD_NET,"Error creating network socket: %s", @@ -1512,7 +1578,10 @@ connection_connect(connection_t *conn, const char *address, return -1; } - make_socket_reuseable(s); + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on new connection: %s", + tor_socket_strerror(errno)); + } if (!tor_addr_is_loopback(addr)) { const tor_addr_t *ext_addr = NULL; @@ -1546,12 +1615,6 @@ connection_connect(connection_t *conn, const char *address, } } - if (set_socket_nonblocking(s) == -1) { - *socket_error = tor_socket_errno(s); - tor_close_socket(s); - return -1; - } - if (options->ConstrainedSockets) set_constrained_socket_buffers(s, (int)options->ConstrainedSockSize); @@ -1617,6 +1680,32 @@ connection_proxy_state_to_string(int state) return states[state]; } +/** Returns the global proxy type used by tor. Use this function for + * logging or high-level purposes, don't use it to fill the + * <b>proxy_type</b> field of or_connection_t; use the actual proxy + * protocol instead.*/ +static int +get_proxy_type(void) +{ + const or_options_t *options = get_options(); + + if (options->HTTPSProxy) + return PROXY_CONNECT; + else if (options->Socks4Proxy) + return PROXY_SOCKS4; + else if (options->Socks5Proxy) + return PROXY_SOCKS5; + else if (options->ClientTransportPlugin) + return PROXY_PLUGGABLE; + else + return PROXY_NONE; +} + +/* One byte for the version, one for the command, two for the + port, and four for the addr... and, one more for the + username NUL: */ +#define SOCKS4_STANDARD_BUFFER_SIZE (1 + 1 + 2 + 4 + 1) + /** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn * for conn->addr:conn->port, authenticating with the auth details given * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies @@ -1671,17 +1760,45 @@ connection_proxy_connect(connection_t *conn, int type) } case PROXY_SOCKS4: { - unsigned char buf[9]; + unsigned char *buf; uint16_t portn; uint32_t ip4addr; + size_t buf_size = 0; + char *socks_args_string = NULL; - /* Send a SOCKS4 connect request with empty user id */ + /* Send a SOCKS4 connect request */ if (tor_addr_family(&conn->addr) != AF_INET) { log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); return -1; } + { /* If we are here because we are trying to connect to a + pluggable transport proxy, check if we have any SOCKS + arguments to transmit. If we do, compress all arguments to + a single string in 'socks_args_string': */ + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (socks_args_string) + log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.", + socks_args_string); + } + } + + { /* Figure out the buffer size we need for the SOCKS message: */ + + buf_size = SOCKS4_STANDARD_BUFFER_SIZE; + + /* If we have a SOCKS argument string, consider its size when + calculating the buffer size: */ + if (socks_args_string) + buf_size += strlen(socks_args_string); + } + + buf = tor_malloc_zero(buf_size); + ip4addr = tor_addr_to_ipv4n(&conn->addr); portn = htons(conn->port); @@ -1689,9 +1806,23 @@ connection_proxy_connect(connection_t *conn, int type) buf[1] = SOCKS_COMMAND_CONNECT; /* command */ memcpy(buf + 2, &portn, 2); /* port */ memcpy(buf + 4, &ip4addr, 4); /* addr */ - buf[8] = 0; /* userid (empty) */ - connection_write_to_buf((char *)buf, sizeof(buf), conn); + /* Next packet field is the userid. If we have pluggable + transport SOCKS arguments, we have to embed them + there. Otherwise, we use an empty userid. */ + if (socks_args_string) { /* place the SOCKS args string: */ + tor_assert(strlen(socks_args_string) > 0); + tor_assert(buf_size >= + SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string)); + strlcpy((char *)buf + 8, socks_args_string, buf_size - 8); + tor_free(socks_args_string); + } else { + buf[8] = 0; /* no userid */ + } + + connection_write_to_buf((char *)buf, buf_size, conn); + tor_free(buf); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; break; } @@ -1703,8 +1834,13 @@ connection_proxy_connect(connection_t *conn, int type) buf[0] = 5; /* version */ + /* We have to use SOCKS5 authentication, if we have a + Socks5ProxyUsername or if we want to pass arguments to our + pluggable transport proxy: */ + if ((options->Socks5ProxyUsername) || + (get_proxy_type() == PROXY_PLUGGABLE && + (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) { /* number of auth methods */ - if (options->Socks5ProxyUsername) { buf[1] = 2; buf[2] = 0x00; /* no authentication */ buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ @@ -1898,15 +2034,49 @@ connection_read_proxy_handshake(connection_t *conn) unsigned char buf[1024]; size_t reqsize, usize, psize; const char *user, *pass; + char *socks_args_string = NULL; + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (!socks_args_string) { + log_warn(LD_NET, "Could not create SOCKS args string."); + ret = -1; + break; + } + + log_debug(LD_NET, "SOCKS5 arguments: %s", socks_args_string); + tor_assert(strlen(socks_args_string) > 0); + tor_assert(strlen(socks_args_string) <= MAX_SOCKS5_AUTH_SIZE_TOTAL); + + if (strlen(socks_args_string) > MAX_SOCKS5_AUTH_FIELD_SIZE) { + user = socks_args_string; + usize = MAX_SOCKS5_AUTH_FIELD_SIZE; + pass = socks_args_string + MAX_SOCKS5_AUTH_FIELD_SIZE; + psize = strlen(socks_args_string) - MAX_SOCKS5_AUTH_FIELD_SIZE; + } else { + user = socks_args_string; + usize = strlen(socks_args_string); + pass = "\0"; + psize = 1; + } + } else if (get_options()->Socks5ProxyUsername) { + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + usize = strlen(user); + psize = strlen(pass); + } else { + log_err(LD_BUG, "We entered %s for no reason!", __func__); + tor_fragile_assert(); + ret = -1; + break; + } - user = get_options()->Socks5ProxyUsername; - pass = get_options()->Socks5ProxyPassword; - tor_assert(user && pass); - - /* XXX len of user and pass must be <= 255 !!! */ - usize = strlen(user); - psize = strlen(pass); - tor_assert(usize <= 255 && psize <= 255); + /* Username and password lengths should have been checked + above and during torrc parsing. */ + tor_assert(usize <= MAX_SOCKS5_AUTH_FIELD_SIZE && + psize <= MAX_SOCKS5_AUTH_FIELD_SIZE); reqsize = 3 + usize + psize; buf[0] = 1; /* negotiation version */ @@ -1915,6 +2085,9 @@ connection_read_proxy_handshake(connection_t *conn) buf[2 + usize] = psize; memcpy(buf + 3 + usize, pass, psize); + if (socks_args_string) + tor_free(socks_args_string); + connection_write_to_buf((char *)buf, reqsize, conn); conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; @@ -2072,7 +2245,7 @@ retry_listener_ports(smartlist_t *old_conns, if (listensockaddr) { conn = connection_listener_new(listensockaddr, listensocklen, - port->type, address, port); + port->type, address, port); tor_free(listensockaddr); tor_free(address); } else { @@ -2184,6 +2357,20 @@ connection_mark_all_noncontrol_connections(void) connection_mark_unattached_ap(TO_ENTRY_CONN(conn), END_STREAM_REASON_HIBERNATING); break; + case CONN_TYPE_OR: + { + or_connection_t *orconn = TO_OR_CONN(conn); + if (orconn->chan) { + connection_or_close_normally(orconn, 0); + } else { + /* + * There should have been one, but mark for close and hope + * for the best.. + */ + connection_mark_for_close(conn); + } + } + break; default: connection_mark_for_close(conn); break; @@ -2358,9 +2545,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now) * shouldn't send <b>attempt</b> bytes of low-priority directory stuff * out to <b>conn</b>. Else return 0. - * Priority is 1 for v1 requests (directories and running-routers), - * and 2 for v2 requests (statuses and descriptors). But see FFFF in - * directory_handle_command_get() for why we don't use priority 2 yet. + * Priority was 1 for v1 requests (directories and running-routers), + * and 2 for v2 requests and later (statuses and descriptors). * * There are a lot of parameters we could use here: * - global_relayed_write_bucket. Low is bad. @@ -2467,6 +2653,35 @@ record_num_bytes_transferred(connection_t *conn, #endif #ifndef USE_BUFFEREVENTS +/** Last time at which the global or relay buckets were emptied in msec + * since midnight. */ +static uint32_t global_relayed_read_emptied = 0, + global_relayed_write_emptied = 0, + global_read_emptied = 0, + global_write_emptied = 0; + +/** Helper: convert given <b>tvnow</b> time value to milliseconds since + * midnight. */ +static uint32_t +msec_since_midnight(const struct timeval *tvnow) +{ + return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) + + ((uint32_t)tvnow->tv_usec / (uint32_t)1000L)); +} + +/** Check if a bucket which had <b>tokens_before</b> tokens and which got + * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run + * out of tokens, and if so, note the milliseconds since midnight in + * <b>timestamp_var</b> for the next TB_EMPTY event. */ +void +connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, size_t tokens_removed, + const struct timeval *tvnow) +{ + if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed) + *timestamp_var = msec_since_midnight(tvnow); +} + /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -2489,6 +2704,30 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ + /* If one or more of our token buckets ran dry just now, note the + * timestamp for TB_EMPTY events. */ + if (get_options()->TestingEnableTbEmptyEvent) { + struct timeval tvnow; + tor_gettimeofday_cached(&tvnow); + if (connection_counts_as_relayed_traffic(conn, now)) { + connection_buckets_note_empty_ts(&global_relayed_read_emptied, + global_relayed_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_relayed_write_emptied, + global_relayed_write_bucket, num_written, &tvnow); + } + connection_buckets_note_empty_ts(&global_read_emptied, + global_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_write_emptied, + global_write_bucket, num_written, &tvnow); + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { + or_connection_t *or_conn = TO_OR_CONN(conn); + connection_buckets_note_empty_ts(&or_conn->read_emptied_time, + or_conn->read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&or_conn->write_emptied_time, + or_conn->write_bucket, num_written, &tvnow); + } + } + if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; global_relayed_write_bucket -= (int)num_written; @@ -2508,6 +2747,9 @@ connection_consider_empty_read_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_read_bucket <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2520,9 +2762,6 @@ connection_consider_empty_read_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->read_blocked_on_bw = 1; connection_stop_reading(conn); @@ -2535,6 +2774,9 @@ connection_consider_empty_write_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_write_bucket <= 0) { reason = "global write bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2547,9 +2789,6 @@ connection_consider_empty_write_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->write_blocked_on_bw = 1; connection_stop_writing(conn); @@ -2601,6 +2840,28 @@ connection_bucket_refill_helper(int *bucket, int rate, int burst, } } +/** Helper: return the time in milliseconds since <b>last_empty_time</b> + * when a bucket ran empty that previously had <b>tokens_before</b> tokens + * now has <b>tokens_after</b> tokens after refilling at timestamp + * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since + * last refilling that bucket. Return 0 if the bucket has not been empty + * since the last refill or has not been refilled. */ +uint32_t +bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow) +{ + uint32_t result = 0, refilled; + if (tokens_before <= 0 && tokens_after > tokens_before) { + refilled = msec_since_midnight(tvnow); + result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) % + (86400L * 1000L)); + if (result > (uint32_t)milliseconds_elapsed) + result = (uint32_t)milliseconds_elapsed; + } + return result; +} + /** Time has passed; increment buckets appropriately. */ void connection_bucket_refill(int milliseconds_elapsed, time_t now) @@ -2609,6 +2870,12 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) smartlist_t *conns = get_connection_array(); int bandwidthrate, bandwidthburst, relayrate, relayburst; + int prev_global_read = global_read_bucket; + int prev_global_write = global_write_bucket; + int prev_relay_read = global_relayed_read_bucket; + int prev_relay_write = global_relayed_write_bucket; + struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */ + bandwidthrate = (int)options->BandwidthRate; bandwidthburst = (int)options->BandwidthBurst; @@ -2643,12 +2910,42 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "global_relayed_write_bucket"); + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + uint32_t global_read_empty_time, global_write_empty_time, + relay_read_empty_time, relay_write_empty_time; + tor_gettimeofday_cached(&tvnow); + global_read_empty_time = bucket_millis_empty(prev_global_read, + global_read_emptied, global_read_bucket, + milliseconds_elapsed, &tvnow); + global_write_empty_time = bucket_millis_empty(prev_global_write, + global_write_emptied, global_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("GLOBAL", global_read_empty_time, + global_write_empty_time, milliseconds_elapsed); + relay_read_empty_time = bucket_millis_empty(prev_relay_read, + global_relayed_read_emptied, + global_relayed_read_bucket, + milliseconds_elapsed, &tvnow); + relay_write_empty_time = bucket_millis_empty(prev_relay_write, + global_relayed_write_emptied, + global_relayed_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("RELAY", relay_read_empty_time, + relay_write_empty_time, milliseconds_elapsed); + } + /* refill the per-connection buckets */ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); int orbandwidthrate = or_conn->bandwidthrate; int orbandwidthburst = or_conn->bandwidthburst; + + int prev_conn_read = or_conn->read_bucket; + int prev_conn_write = or_conn->write_bucket; + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, orbandwidthrate, @@ -2663,6 +2960,27 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "or_conn->write_bucket"); } + + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + char *bucket; + uint32_t conn_read_empty_time, conn_write_empty_time; + tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT, + U64_PRINTF_ARG(or_conn->base_.global_identifier)); + conn_read_empty_time = bucket_millis_empty(prev_conn_read, + or_conn->read_emptied_time, + or_conn->read_bucket, + milliseconds_elapsed, &tvnow); + conn_write_empty_time = bucket_millis_empty(prev_conn_write, + or_conn->write_emptied_time, + or_conn->write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty(bucket, conn_read_empty_time, + conn_write_empty_time, + milliseconds_elapsed); + tor_free(bucket); + } } if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */ @@ -2819,6 +3137,8 @@ connection_handle_read_impl(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR_LISTENER: return connection_handle_listener_read(conn, CONN_TYPE_OR); + case CONN_TYPE_EXT_OR_LISTENER: + return connection_handle_listener_read(conn, CONN_TYPE_EXT_OR); case CONN_TYPE_AP_LISTENER: case CONN_TYPE_AP_TRANS_LISTENER: case CONN_TYPE_AP_NATD_LISTENER: @@ -3071,14 +3391,37 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; - /* Update edge_conn->n_read */ + /* Update edge_conn->n_read and ocirc->n_read_circ_bw */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; + /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) edge_conn->n_read += (int)n_read; else edge_conn->n_read = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read)) + ocirc->n_read_circ_bw += (int)n_read; + else + ocirc->n_read_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_read_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_read_conn_bw > n_read)) + conn->n_read_conn_bw += (int)n_read; + else + conn->n_read_conn_bw = UINT32_MAX; } } @@ -3331,8 +3674,8 @@ connection_outbuf_too_full(connection_t *conn) /** Try to flush more bytes onto <b>conn</b>-\>s. * - * This function gets called either from conn_write() in main.c - * when poll() has declared that conn wants to write, or below + * This function gets called either from conn_write_callback() in main.c + * when libevent tells us that conn wants to write, or below * from connection_write_to_buf() when an entire TLS record is ready. * * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf @@ -3518,12 +3861,34 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) edge_conn->n_written += (int)n_written; else edge_conn->n_written = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written)) + ocirc->n_written_circ_bw += (int)n_written; + else + ocirc->n_written_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_written_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (n_written && get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_written_conn_bw > n_written)) + conn->n_written_conn_bw += (int)n_written; + else + conn->n_written_conn_bw = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -3609,9 +3974,9 @@ connection_flush(connection_t *conn) * it all, so we don't end up with many megabytes of controller info queued at * once. */ -void -connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib) +MOCK_IMPL(void, +connection_write_to_buf_impl_,(const char *string, size_t len, + connection_t *conn, int zlib)) { /* XXXX This function really needs to return -1 on failure. */ int r; @@ -3656,6 +4021,12 @@ connection_write_to_buf_impl_(const char *string, size_t len, "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s); circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)), END_CIRC_REASON_INTERNAL); + } else if (conn->type == CONN_TYPE_OR) { + or_connection_t *orconn = TO_OR_CONN(conn); + log_warn(LD_NET, + "write_to_buf failed on an orconn; notifying of error " + "(fd %d)", (int)(conn->s)); + connection_or_close_for_error(orconn, 0); } else { log_warn(LD_NET, "write_to_buf failed. Closing connection (fd %d).", @@ -3830,20 +4201,29 @@ connection_dir_get_by_purpose_and_resource(int purpose, return NULL; } -/** Return an open, non-marked connection of a given type and purpose, or NULL - * if no such connection exists. */ -connection_t * -connection_get_by_type_purpose(int type, int purpose) +/** Return 1 if there are any active OR connections apart from + * <b>this_conn</b>. + * + * We use this to guess if we should tell the controller that we + * didn't manage to connect to any of our bridges. */ +int +any_other_active_or_conns(const or_connection_t *this_conn) { smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && - !conn->marked_for_close && - (purpose == conn->purpose)) - return conn; - }); - return NULL; + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn == TO_CONN(this_conn)) { /* don't consider this conn */ + continue; + } + + if (conn->type == CONN_TYPE_OR && + !conn->marked_for_close) { + log_debug(LD_DIR, "%s: Found an OR connection: %s", + __func__, conn->address); + return 1; + } + } SMARTLIST_FOREACH_END(conn); + + return 0; } /** Return 1 if <b>conn</b> is a listener conn, else return 0. */ @@ -3851,6 +4231,7 @@ int connection_is_listener(connection_t *conn) { if (conn->type == CONN_TYPE_OR_LISTENER || + conn->type == CONN_TYPE_EXT_OR_LISTENER || conn->type == CONN_TYPE_AP_LISTENER || conn->type == CONN_TYPE_AP_TRANS_LISTENER || conn->type == CONN_TYPE_AP_DNS_LISTENER || @@ -3873,6 +4254,7 @@ connection_state_is_open(connection_t *conn) return 0; if ((conn->type == CONN_TYPE_OR && conn->state == OR_CONN_STATE_OPEN) || + (conn->type == CONN_TYPE_EXT_OR) || (conn->type == CONN_TYPE_AP && conn->state == AP_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_EXIT && conn->state == EXIT_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_CONTROL && @@ -4042,6 +4424,8 @@ connection_process_inbuf(connection_t *conn, int package_partial) switch (conn->type) { case CONN_TYPE_OR: return connection_or_process_inbuf(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_process_inbuf(TO_OR_CONN(conn)); case CONN_TYPE_EXIT: case CONN_TYPE_AP: return connection_edge_process_inbuf(TO_EDGE_CONN(conn), @@ -4102,6 +4486,8 @@ connection_finished_flushing(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: return connection_or_finished_flushing(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_finished_flushing(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: return connection_edge_finished_flushing(TO_EDGE_CONN(conn)); @@ -4157,6 +4543,7 @@ connection_reached_eof(connection_t *conn) { switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: return connection_or_reached_eof(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: @@ -4243,6 +4630,7 @@ assert_connection_ok(connection_t *conn, time_t now) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); break; case CONN_TYPE_AP: @@ -4348,6 +4736,10 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn->state >= OR_CONN_STATE_MIN_); tor_assert(conn->state <= OR_CONN_STATE_MAX_); break; + case CONN_TYPE_EXT_OR: + tor_assert(conn->state >= EXT_OR_CONN_STATE_MIN_); + tor_assert(conn->state <= EXT_OR_CONN_STATE_MAX_); + break; case CONN_TYPE_EXIT: tor_assert(conn->state >= EXIT_CONN_STATE_MIN_); tor_assert(conn->state <= EXIT_CONN_STATE_MAX_); @@ -4409,7 +4801,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, options->Bridges) { const transport_t *transport = NULL; int r; - r = find_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); + r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); if (r<0) return -1; if (transport) { /* transport found */ @@ -4420,28 +4812,12 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, } } + tor_addr_make_unspec(addr); + *port = 0; *proxy_type = PROXY_NONE; return 0; } -/** Returns the global proxy type used by tor. */ -static int -get_proxy_type(void) -{ - const or_options_t *options = get_options(); - - if (options->HTTPSProxy) - return PROXY_CONNECT; - else if (options->Socks4Proxy) - return PROXY_SOCKS4; - else if (options->Socks5Proxy) - return PROXY_SOCKS5; - else if (options->ClientTransportPlugin) - return PROXY_PLUGGABLE; - else - return PROXY_NONE; -} - /** Log a failed connection to a proxy server. * <b>conn</b> is the connection we use the proxy server for. */ void @@ -4498,6 +4874,7 @@ connection_free_all(void) /* Unlink everything from the identity map. */ connection_or_clear_identity_map(); + connection_or_clear_ext_or_id_map(); /* Clear out our list of broken connections */ clear_broken_connection_map(0); diff --git a/src/or/connection.h b/src/or/connection.h index c78fe6e65..13dcbcd91 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -19,7 +19,7 @@ const char *conn_type_to_string(int type); const char *conn_state_to_string(int type, int state); dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); +or_connection_t *or_connection_new(int type, int socket_family); edge_connection_t *edge_connection_new(int type, int socket_family); entry_connection_t *entry_connection_new(int type, int socket_family); control_connection_t *control_connection_new(int socket_family); @@ -89,6 +89,14 @@ int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); +/** Maximum size of information that we can fit into SOCKS5 username + or password fields. */ +#define MAX_SOCKS5_AUTH_FIELD_SIZE 255 + +/** Total maximum size of information that we can fit into SOCKS5 + username and password fields. */ +#define MAX_SOCKS5_AUTH_SIZE_TOTAL 2*MAX_SOCKS5_AUTH_FIELD_SIZE + int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); @@ -122,8 +130,8 @@ int connection_outbuf_too_full(connection_t *conn); int connection_handle_write(connection_t *conn, int force); int connection_flush(connection_t *conn); -void connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib); +MOCK_DECL(void, connection_write_to_buf_impl_, + (const char *string, size_t len, connection_t *conn, int zlib)); /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); @@ -170,7 +178,6 @@ connection_get_outbuf_len(connection_t *conn) connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); connection_t *connection_get_by_type_addr_port_purpose(int type, const tor_addr_t *addr, uint16_t port, int purpose); @@ -180,6 +187,8 @@ connection_t *connection_get_by_type_state_rendquery(int type, int state, dir_connection_t *connection_dir_get_by_purpose_and_resource( int state, const char *resource); +int any_other_active_or_conns(const or_connection_t *this_conn); + #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); int connection_state_is_open(connection_t *conn); @@ -206,5 +215,18 @@ void connection_enable_rate_limiting(connection_t *conn); #define connection_type_uses_bufferevent(c) (0) #endif +#ifdef CONNECTION_PRIVATE +STATIC void connection_free_(connection_t *conn); + +/* Used only by connection.c and test*.c */ +uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow); +void connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, + size_t tokens_removed, + const struct timeval *tvnow); +#endif + #endif diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 895c0f7f0..a8ad9ec2e 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -14,6 +14,7 @@ #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" @@ -61,19 +62,14 @@ static int connection_ap_process_natd(entry_connection_t *conn); static int connection_exit_connect_dir(edge_connection_t *exitconn); static int consider_plaintext_ports(entry_connection_t *conn, uint16_t port); static int connection_ap_supports_optimistic_data(const entry_connection_t *); -static void connection_ap_handshake_socks_resolved_addr( - entry_connection_t *conn, - const tor_addr_t *answer, - int ttl, - time_t expires); /** An AP stream has failed/finished. If it hasn't already sent back * a socks reply, send one now (based on endreason). Also set * has_sent_end to 1, and mark the conn. */ -void -connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file) +MOCK_IMPL(void, +connection_mark_unattached_ap_,(entry_connection_t *conn, int endreason, + int line, const char *file)) { connection_t *base_conn = ENTRY_TO_CONN(conn); edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn); @@ -412,7 +408,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) * that the name resolution that led us to <b>addr</b> will be valid for * <b>ttl</b> seconds. Return -1 on error, or the number of bytes used on * success. */ -/* private */int +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl) @@ -1395,35 +1391,48 @@ get_pf_socket(void) } #endif -/** Fetch the original destination address and port from a - * system-specific interface and put them into a - * socks_request_t as if they came from a socks request. - * - * Return -1 if an error prevents fetching the destination, - * else return 0. - */ +#if defined(TRANS_NETFILTER) || defined(TRANS_PF) +/** Try fill in the address of <b>req</b> from the socket configured + * with <b>conn</b>. */ static int -connection_ap_get_original_destination(entry_connection_t *conn, - socks_request_t *req) +destination_from_socket(entry_connection_t *conn, socks_request_t *req) { -#ifdef TRANS_NETFILTER - /* Linux 2.4+ */ struct sockaddr_storage orig_dst; socklen_t orig_dst_len = sizeof(orig_dst); tor_addr_t addr; +#ifdef TRANS_NETFILTER if (getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); log_warn(LD_NET, "getsockopt() failed: %s", tor_socket_strerror(e)); return -1; } +#elif defined(TRANS_PF) + if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, + &orig_dst_len) < 0) { + int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); + log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); + return -1; + } +#else + (void)conn; + (void)req; + log_warn(LD_BUG, "Unable to determine destination from socket."); + return -1; +#endif tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port); tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); return 0; -#elif defined(TRANS_PF) +} +#endif + +#ifdef TRANS_PF +static int +destination_from_pf(entry_connection_t *conn, socks_request_t *req) +{ struct sockaddr_storage proxy_addr; socklen_t proxy_addr_len = sizeof(proxy_addr); struct sockaddr *proxy_sa = (struct sockaddr*) &proxy_addr; @@ -1439,6 +1448,21 @@ connection_ap_get_original_destination(entry_connection_t *conn, return -1; } +#ifdef __FreeBSD__ + if (get_options()->TransProxyType_parsed == TPT_IPFW) { + /* ipfw(8) is used and in this case getsockname returned the original + destination */ + if (tor_addr_from_sockaddr(&addr, proxy_sa, &req->port) < 0) { + tor_fragile_assert(); + return -1; + } + + tor_addr_to_str(req->address, &addr, sizeof(req->address), 0); + + return 0; + } +#endif + memset(&pnl, 0, sizeof(pnl)); pnl.proto = IPPROTO_TCP; pnl.direction = PF_OUT; @@ -1485,6 +1509,36 @@ connection_ap_get_original_destination(entry_connection_t *conn, req->port = ntohs(pnl.rdport); return 0; +} +#endif + +/** Fetch the original destination address and port from a + * system-specific interface and put them into a + * socks_request_t as if they came from a socks request. + * + * Return -1 if an error prevents fetching the destination, + * else return 0. + */ +static int +connection_ap_get_original_destination(entry_connection_t *conn, + socks_request_t *req) +{ +#ifdef TRANS_NETFILTER + return destination_from_socket(conn, req); +#elif defined(TRANS_PF) + const or_options_t *options = get_options(); + + if (options->TransProxyType_parsed == TPT_PF_DIVERT) + return destination_from_socket(conn, req); + + if (options->TransProxyType_parsed == TPT_DEFAULT) + return destination_from_pf(conn, req); + + (void)conn; + (void)req; + log_warn(LD_BUG, "Proxy destination determination mechanism %s unknown.", + options->TransProxyType); + return -1; #else (void)conn; (void)req; @@ -2064,7 +2118,7 @@ tell_controller_about_resolved_result(entry_connection_t *conn, * As connection_ap_handshake_socks_resolved, but take a tor_addr_t to send * as the answer. */ -static void +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, const tor_addr_t *answer, int ttl, @@ -2097,13 +2151,13 @@ connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, **/ /* XXXX the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ -void -connection_ap_handshake_socks_resolved(entry_connection_t *conn, +MOCK_IMPL(void, +connection_ap_handshake_socks_resolved,(entry_connection_t *conn, int answer_type, size_t answer_len, const uint8_t *answer, int ttl, - time_t expires) + time_t expires)) { char buf[384]; size_t replylen; @@ -2272,13 +2326,24 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, /* leave version, destport, destip zero */ connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn)); } else if (conn->socks_request->socks_version == 5) { - buf[0] = 5; /* version 5 */ - buf[1] = (char)status; - buf[2] = 0; - buf[3] = 1; /* ipv4 addr */ - memset(buf+4,0,6); /* Set external addr/port to 0. - The spec doesn't seem to say what to do here. -RD */ - connection_write_to_buf(buf,10,ENTRY_TO_CONN(conn)); + size_t buf_len; + memset(buf,0,sizeof(buf)); + if (tor_addr_family(&conn->edge_.base_.addr) == AF_INET) { + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 1; /* ipv4 addr */ + /* 4 bytes for the header, 2 bytes for the port, 4 for the address. */ + buf_len = 10; + } else { /* AF_INET6. */ + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 4; /* ipv6 addr */ + /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */ + buf_len = 22; + } + connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn)); } /* If socks_version isn't 4 or 5, don't send anything. * This can happen in the case of AP bridges. */ @@ -2294,7 +2359,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, * Return -1 in the case where want to send a RELAY_END cell, and < -1 when * we don't. **/ -/* static */ int +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out) { diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index ea284cbcf..3c0e30a97 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -12,11 +12,14 @@ #ifndef TOR_CONNECTION_EDGE_H #define TOR_CONNECTION_EDGE_H +#include "testsupport.h" + #define connection_mark_unattached_ap(conn, endreason) \ connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__) -void connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file); +MOCK_DECL(void,connection_mark_unattached_ap_, + (entry_connection_t *conn, int endreason, + int line, const char *file)); int connection_edge_reached_eof(edge_connection_t *conn); int connection_edge_process_inbuf(edge_connection_t *conn, int package_partial); @@ -42,12 +45,17 @@ entry_connection_t *connection_ap_make_link(connection_t *partner, void connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, size_t replylen, int endreason); -void connection_ap_handshake_socks_resolved(entry_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); +MOCK_DECL(void,connection_ap_handshake_socks_resolved, + (entry_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires)); +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, + const tor_addr_t *answer, + int ttl, + time_t expires); int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); @@ -130,9 +138,9 @@ typedef struct begin_cell_t { unsigned is_begindir : 1; } begin_cell_t; -int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out); -int connected_cell_format_payload(uint8_t *payload_out, +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl); #endif diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 8e7cd9ea5..6572a918e 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -37,7 +37,7 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" - +#include "ext_orport.h" #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> #endif @@ -75,6 +75,10 @@ static void connection_or_handle_event_cb(struct bufferevent *bufev, * they form a linked list, with next_with_same_id as the next pointer. */ static digestmap_t *orconn_identity_map = NULL; +/** Global map between Extended ORPort identifiers and OR + * connections. */ +static digestmap_t *orconn_ext_or_id_map = NULL; + /** If conn is listed in orconn_identity_map, remove it, and clear * conn->identity_digest. Otherwise do nothing. */ void @@ -174,6 +178,71 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest) #endif } +/** Remove the Extended ORPort identifier of <b>conn</b> from the + * global identifier list. Also, clear the identifier from the + * connection itself. */ +void +connection_or_remove_from_ext_or_id_map(or_connection_t *conn) +{ + or_connection_t *tmp; + if (!orconn_ext_or_id_map) + return; + if (!conn->ext_or_conn_id) + return; + + tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id); + if (!tor_digest_is_zero(conn->ext_or_conn_id)) + tor_assert(tmp == conn); + + memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN); +} + +/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such + * connection is found. */ +or_connection_t * +connection_or_get_by_ext_or_id(const char *id) +{ + if (!orconn_ext_or_id_map) + return NULL; + return digestmap_get(orconn_ext_or_id_map, id); +} + +/** Deallocate the global Extended ORPort identifier list */ +void +connection_or_clear_ext_or_id_map(void) +{ + digestmap_free(orconn_ext_or_id_map, NULL); + orconn_ext_or_id_map = NULL; +} + +/** Creates an Extended ORPort identifier for <b>conn</b> and deposits + * it into the global list of identifiers. */ +void +connection_or_set_ext_or_identifier(or_connection_t *conn) +{ + char random_id[EXT_OR_CONN_ID_LEN]; + or_connection_t *tmp; + + if (!orconn_ext_or_id_map) + orconn_ext_or_id_map = digestmap_new(); + + /* Remove any previous identifiers: */ + if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id)) + connection_or_remove_from_ext_or_id_map(conn); + + do { + crypto_rand(random_id, sizeof(random_id)); + } while (digestmap_get(orconn_ext_or_id_map, random_id)); + + if (!conn->ext_or_conn_id) + conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN); + + memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN); + + tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn); + tor_assert(!tmp); +} + /**************************************************************/ /** Map from a string describing what a non-open OR connection was doing when @@ -228,7 +297,7 @@ connection_or_get_state_description(or_connection_t *orconn, const char *conn_state; char tls_state[256]; - tor_assert(conn->type == CONN_TYPE_OR); + tor_assert(conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR); conn_state = conn_state_to_string(conn->type, conn->state); tor_tls_get_state_description(orconn->tls, tls_state, sizeof(tls_state)); @@ -645,7 +714,8 @@ connection_or_about_to_close(or_connection_t *or_conn) reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( - orconn_end_reason_to_control_string(reason), reason); + orconn_end_reason_to_control_string(reason), + reason, or_conn); } } } else if (conn->hold_open_until_flushed) { @@ -1008,7 +1078,7 @@ connection_or_connect_failed(or_connection_t *conn, { control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) - control_event_bootstrap_problem(msg, reason); + control_event_bootstrap_problem(msg, reason, conn); } /** <b>conn</b> got an error in connection_handle_read_impl() or @@ -1082,7 +1152,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, return NULL; } - conn = or_connection_new(tor_addr_family(&addr)); + conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr)); /* * Set up conn so it's got all the data we need to remember for channels @@ -1125,6 +1195,12 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, "your pluggable transport proxy stopped running.", fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port), transport_name, transport_name); + + control_event_bootstrap_problem( + "Can't connect to bridge", + END_OR_CONN_REASON_PT_MISSING, + conn); + } else { log_warn(LD_GENERAL, "Tried to connect to '%s' through a proxy, but " "the proxy address could not be found.", @@ -1227,8 +1303,8 @@ connection_or_close_for_error(or_connection_t *orconn, int flush) * * Return -1 if <b>conn</b> is broken, else return 0. */ -int -connection_tls_start_handshake(or_connection_t *conn, int receiving) +MOCK_IMPL(int, +connection_tls_start_handshake,(or_connection_t *conn, int receiving)) { channel_listener_t *chan_listener; channel_t *chan; @@ -1485,7 +1561,8 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event, int connection_or_nonopen_was_started_here(or_connection_t *conn) { - tor_assert(conn->base_.type == CONN_TYPE_OR); + tor_assert(conn->base_.type == CONN_TYPE_OR || + conn->base_.type == CONN_TYPE_EXT_OR); if (!conn->tls) return 1; /* it's still in proxy states or something */ if (conn->handshake_state) @@ -1638,7 +1715,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( "Unexpected identity in router certificate", - END_OR_CONN_REASON_OR_IDENTITY); + END_OR_CONN_REASON_OR_IDENTITY, + conn); return -1; } if (authdir_mode_tests_reachability(options)) { @@ -1688,13 +1766,11 @@ connection_tls_finish_handshake(or_connection_t *conn) safe_str_client(conn->base_.address), tor_tls_get_ciphersuite_name(conn->tls)); - directory_set_dirty(); - if (connection_or_check_valid_tls_handshake(conn, started_here, digest_rcvd) < 0) return -1; - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; @@ -1728,7 +1804,7 @@ connection_or_launch_v3_or_handshake(or_connection_t *conn) tor_assert(connection_or_nonopen_was_started_here(conn)); tor_assert(tor_tls_received_v3_certificate(conn->tls)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V3); if (connection_init_or_handshake_state(conn, 1) < 0) @@ -1961,7 +2037,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); channel_tls_handle_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1977,7 +2053,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 85e68f1a3..8d9302893 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -45,7 +45,8 @@ void connection_or_close_for_error(or_connection_t *orconn, int flush); void connection_or_report_broken_states(int severity, int domain); -int connection_tls_start_handshake(or_connection_t *conn, int receiving); +MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, + int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); int connection_init_or_handshake_state(or_connection_t *conn, diff --git a/src/or/control.c b/src/or/control.c index a88de12d6..21504e685 100644..100755 --- a/src/or/control.c +++ b/src/or/control.c @@ -19,6 +19,7 @@ #include "circuitlist.h" #include "circuitstats.h" #include "circuituse.h" +#include "command.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -52,46 +53,13 @@ * finished authentication and is accepting commands. */ #define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) -/* Recognized asynchronous event types. It's okay to expand this list - * because it is used both as a list of v0 event types, and as indices - * into the bitfield to determine which controllers want which events. - */ -#define EVENT_MIN_ 0x0001 -#define EVENT_CIRCUIT_STATUS 0x0001 -#define EVENT_STREAM_STATUS 0x0002 -#define EVENT_OR_CONN_STATUS 0x0003 -#define EVENT_BANDWIDTH_USED 0x0004 -#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 -#define EVENT_NEW_DESC 0x0006 -#define EVENT_DEBUG_MSG 0x0007 -#define EVENT_INFO_MSG 0x0008 -#define EVENT_NOTICE_MSG 0x0009 -#define EVENT_WARN_MSG 0x000A -#define EVENT_ERR_MSG 0x000B -#define EVENT_ADDRMAP 0x000C -// #define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_DESCCHANGED 0x000E -// #define EVENT_NS 0x000F -#define EVENT_STATUS_CLIENT 0x0010 -#define EVENT_STATUS_SERVER 0x0011 -#define EVENT_STATUS_GENERAL 0x0012 -#define EVENT_GUARD 0x0013 -#define EVENT_STREAM_BANDWIDTH_USED 0x0014 -#define EVENT_CLIENTS_SEEN 0x0015 -#define EVENT_NEWCONSENSUS 0x0016 -#define EVENT_BUILDTIMEOUT_SET 0x0017 -#define EVENT_SIGNAL 0x0018 -#define EVENT_CONF_CHANGED 0x0019 -#define EVENT_MAX_ 0x0019 -/* If EVENT_MAX_ ever hits 0x0020, we need to make the mask wider. */ - /** Bitfield: The bit 1<<e is set if <b>any</b> open control * connection is interested in events of type <b>e</b>. We use this * so that we can decide to skip generating event messages that nobody * has interest in without having to walk over the global connection * list to find out. **/ -typedef uint32_t event_mask_t; +typedef uint64_t event_mask_t; /** An event mask of all the events that any controller is interested in * receiving. */ @@ -103,7 +71,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - (global_event_mask & (1<<(e))) + (!! (global_event_mask & (((uint64_t)1)<<(e)))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -115,7 +83,7 @@ static int authentication_cookie_is_set = 0; /** If authentication_cookie_is_set, a secret cookie that we've stored to disk * and which we're using to authenticate controllers. (If the controller can * read it off disk, it has permission to connect.) */ -static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; +static uint8_t *authentication_cookie = NULL; #define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \ "Tor safe cookie authentication server-to-controller hash" @@ -130,15 +98,6 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the one standard - format. - */ -#define ALL_FORMATS 1 - -/** Bit field of flags to select how to format a controller event. Recognized - * flag is ALL_FORMATS. */ -typedef int event_format_t; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -201,7 +160,6 @@ static int write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len); static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); -static char *get_cookie_file(void); /** Given a control event code for a message event, return the corresponding * log severity. */ @@ -232,6 +190,20 @@ log_severity_to_event(int severity) } } +/** Helper: clear bandwidth counters of all origin circuits. */ +static void +clear_circ_bw_fields(void) +{ + circuit_t *circ; + origin_circuit_t *ocirc; + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } +} + /** Set <b>global_event_mask*</b> to the bitwise OR of each live control * connection's event_mask field. */ void @@ -257,8 +229,8 @@ control_update_global_event_mask(void) * we want to hear...*/ control_adjust_event_log_severity(); - /* ...then, if we've started logging stream bw, clear the appropriate - * fields. */ + /* ...then, if we've started logging stream or circ bw, clear the + * appropriate fields. */ if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) && (new_mask & EVENT_STREAM_BANDWIDTH_USED)) { SMARTLIST_FOREACH(conns, connection_t *, conn, @@ -269,6 +241,10 @@ control_update_global_event_mask(void) } }); } + if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) && + (new_mask & EVENT_CIRC_BANDWIDTH_USED)) { + clear_circ_bw_fields(); + } } /** Adjust the log severities that result in control_event_logmsg being called @@ -334,7 +310,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t write_escaped_data(const char *data, size_t len, char **out) { size_t sz_out = len+8; @@ -382,7 +358,7 @@ write_escaped_data(const char *data, size_t len, char **out) * that appears at the start of a line, and replacing all CRLF sequences * with LF. Return the number of * bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t read_escaped_data(const char *data, size_t len, char **out) { char *outp; @@ -592,9 +568,9 @@ send_control_done(control_connection_t *conn) * * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with * respect to the EXTENDED_EVENTS feature. */ -static void -send_control_event_string(uint16_t event, event_format_t which, - const char *msg) +MOCK_IMPL(STATIC void, +send_control_event_string,(uint16_t event, event_format_t which, + const char *msg)) { smartlist_t *conns = get_connection_array(); (void)which; @@ -958,6 +934,12 @@ static const struct control_event_t control_event_table[] = { { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" }, { EVENT_SIGNAL, "SIGNAL" }, { EVENT_CONF_CHANGED, "CONF_CHANGED"}, + { EVENT_CONN_BW, "CONN_BW" }, + { EVENT_CELL_STATS, "CELL_STATS" }, + { EVENT_TB_EMPTY, "TB_EMPTY" }, + { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, + { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, + { EVENT_HS_DESC, "HS_DESC" }, { 0, NULL }, }; @@ -1447,7 +1429,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "config-defaults-file")) { *answer = tor_strdup(get_torrc_fname(1)); } else if (!strcmp(question, "config-text")) { - *answer = options_dump(get_options(), 1); + *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "dormant")) { @@ -1509,7 +1491,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(""); #else int myUid = geteuid(); - struct passwd *myPwEntry = getpwuid(myUid); + const struct passwd *myPwEntry = tor_getpwuid(myUid); if (myPwEntry) { *answer = tor_strdup(myPwEntry->pw_name); @@ -1521,6 +1503,9 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, int max_fds=-1; set_max_file_descriptors(0, &max_fds); tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, U64_FORMAT, + U64_PRINTF_ARG(get_options()->MaxMemInQueues)); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { @@ -1567,12 +1552,13 @@ munge_extrainfo_into_routerinfo(const char *ri_body, outp += router_sig-ri_body; for (i=0; i < 2; ++i) { - const char *kwd = i?"\nwrite-history ":"\nread-history "; + const char *kwd = i ? "\nwrite-history " : "\nread-history "; const char *cp, *eol; if (!(cp = tor_memstr(ei_body, ei_len, kwd))) continue; ++cp; - eol = memchr(cp, '\n', ei_len - (cp-ei_body)); + if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body)))) + continue; memcpy(outp, cp, eol-cp+1); outp += eol-cp+1; } @@ -1764,39 +1750,7 @@ getinfo_helper_dir(control_connection_t *control_conn, tor_free(url); smartlist_free(descs); } else if (!strcmpstart(question, "dir/status/")) { - if (directory_permits_controller_requests(get_options())) { - size_t len=0; - char *cp; - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2(status_list, - question+strlen("dir/status/")); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, len += d->dir_len); - cp = *answer = tor_malloc(len+1); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, { - memcpy(cp, d->dir, d->dir_len); - cp += d->dir_len; - }); - *cp = '\0'; - smartlist_free(status_list); - } else { - smartlist_t *fp_list = smartlist_new(); - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2_fingerprints( - fp_list, question+strlen("dir/status/")); - SMARTLIST_FOREACH(fp_list, const char *, fp, { - char *s; - char *fname = networkstatus_get_cache_filename(fp); - s = read_file_to_str(fname, 0, NULL); - if (s) - smartlist_add(status_list, s); - tor_free(fname); - }); - SMARTLIST_FOREACH(fp_list, char *, fp, tor_free(fp)); - smartlist_free(fp_list); - *answer = smartlist_join_strings(status_list, "", 0, NULL); - SMARTLIST_FOREACH(status_list, char *, s, tor_free(s)); - smartlist_free(status_list); - } + *answer = tor_strdup(""); } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { const cached_dir_t *consensus = dirserv_get_consensus("ns"); @@ -1927,7 +1881,7 @@ getinfo_helper_events(control_connection_t *control_conn, if (!strcmp(question, "circuit-status")) { circuit_t *circ_; smartlist_t *status = smartlist_new(); - for (circ_ = circuit_get_global_list_(); circ_; circ_ = circ_->next) { + TOR_LIST_FOREACH(circ_, circuit_get_global_list(), head) { origin_circuit_t *circ; char *circdesc; const char *state; @@ -2232,6 +2186,7 @@ static const getinfo_item_t getinfo_items[] = { ITEM("process/user", misc, "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), @@ -2241,6 +2196,9 @@ static const getinfo_item_t getinfo_items[] = { "v3 Networkstatus consensus as retrieved from a DirPort."), ITEM("exit-policy/default", policies, "The default value appended to the configured exit policy."), + ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), + ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), + ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), { NULL, NULL, NULL, 0 } }; @@ -2985,7 +2943,7 @@ handle_control_protocolinfo(control_connection_t *conn, uint32_t len, } else { const or_options_t *options = get_options(); int cookies = options->CookieAuthentication; - char *cfile = get_cookie_file(); + char *cfile = get_controller_cookie_file_name(); char *abs_cfile; char *esc_cfile; char *methods; @@ -3181,6 +3139,30 @@ handle_control_usefeature(control_connection_t *conn, return 0; } +/** Implementation for the DROPGUARDS command. */ +static int +handle_control_dropguards(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + if (smartlist_len(args)) { + connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); + } else { + remove_all_entry_guards(); + send_control_done(conn); + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + /** Called when <b>conn</b> has no more bytes left on its outbuf. */ int connection_control_finished_flushing(control_connection_t *conn) @@ -3200,27 +3182,22 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } +static void lost_owning_controller(const char *owner_type, + const char *loss_manner) + ATTR_NORETURN; + /** Shut down this Tor instance in the same way that SIGINT would, but * with a log message appropriate for the loss of an owning controller. */ static void lost_owning_controller(const char *owner_type, const char *loss_manner) { - int shutdown_slowly = server_mode(get_options()); - - log_notice(LD_CONTROL, "Owning controller %s has %s -- %s.", - owner_type, loss_manner, - shutdown_slowly ? "shutting down" : "exiting now"); + log_notice(LD_CONTROL, "Owning controller %s has %s -- exiting now.", + owner_type, loss_manner); /* XXXX Perhaps this chunk of code should be a separate function, * called here and by process_signal(SIGINT). */ - - if (!shutdown_slowly) { - tor_cleanup(); - exit(0); - } - /* XXXX This will close all listening sockets except control-port - * listeners. Perhaps we should close those too. */ - hibernate_begin_shutdown(); + tor_cleanup(); + exit(0); } /** Called when <b>conn</b> is being freed. */ @@ -3480,6 +3457,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { if (handle_control_authchallenge(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { + if (handle_control_dropguards(conn, cmd_data_len, args)) + return -1; } else { connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", conn->incoming_cmd); @@ -3847,17 +3827,17 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, } ncircs += connection_or_get_num_circuits(conn); if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) { - tor_snprintf(ncircs_buf, sizeof(ncircs_buf), "%sNCIRCS=%d", - reason ? " " : "", ncircs); + tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs); } orconn_target_get_name(name, sizeof(name), conn); send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, - "650 ORCONN %s %s %s%s%s\r\n", + "650 ORCONN %s %s%s%s%s ID="U64_FORMAT"\r\n", name, status, - reason ? "REASON=" : "", + reason ? " REASON=" : "", orconn_end_reason_to_control_string(reason), - ncircs_buf); + ncircs_buf, + U64_PRINTF_ARG(conn->base_.global_identifier)); return 0; } @@ -3868,6 +3848,8 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, int control_event_stream_bandwidth(edge_connection_t *edge_conn) { + circuit_t *circ; + origin_circuit_t *ocirc; if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { if (!edge_conn->n_read && !edge_conn->n_written) return 0; @@ -3878,6 +3860,12 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) (unsigned long)edge_conn->n_read, (unsigned long)edge_conn->n_written); + circ = circuit_get_by_edge_conn(edge_conn); + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_read_circ_bw += edge_conn->n_read; + ocirc->n_written_circ_bw += edge_conn->n_written; + } edge_conn->n_written = edge_conn->n_read = 0; } @@ -3915,6 +3903,235 @@ control_event_stream_bandwidth_used(void) return 0; } +/** A second or more has elapsed: tell any interested control connections + * how much bandwidth origin circuits have used. */ +int +control_event_circ_bandwidth_used(void) +{ + circuit_t *circ; + origin_circuit_t *ocirc; + if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) + return 0; + + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw) + continue; + send_control_event(EVENT_CIRC_BANDWIDTH_USED, ALL_FORMATS, + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu\r\n", + ocirc->global_identifier, + (unsigned long)ocirc->n_read_circ_bw, + (unsigned long)ocirc->n_written_circ_bw); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } + + return 0; +} + +/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset + * bandwidth counters. */ +int +control_event_conn_bandwidth(connection_t *conn) +{ + const char *conn_type_str; + if (!get_options()->TestingEnableConnBwEvent || + !EVENT_IS_INTERESTING(EVENT_CONN_BW)) + return 0; + if (!conn->n_read_conn_bw && !conn->n_written_conn_bw) + return 0; + switch (conn->type) { + case CONN_TYPE_OR: + conn_type_str = "OR"; + break; + case CONN_TYPE_DIR: + conn_type_str = "DIR"; + break; + case CONN_TYPE_EXIT: + conn_type_str = "EXIT"; + break; + default: + return 0; + } + send_control_event(EVENT_CONN_BW, ALL_FORMATS, + "650 CONN_BW ID="U64_FORMAT" TYPE=%s " + "READ=%lu WRITTEN=%lu\r\n", + U64_PRINTF_ARG(conn->global_identifier), + conn_type_str, + (unsigned long)conn->n_read_conn_bw, + (unsigned long)conn->n_written_conn_bw); + conn->n_written_conn_bw = conn->n_read_conn_bw = 0; + return 0; +} + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth connections have used. */ +int +control_event_conn_bandwidth_used(void) +{ + if (get_options()->TestingEnableConnBwEvent && + EVENT_IS_INTERESTING(EVENT_CONN_BW)) { + SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn, + control_event_conn_bandwidth(conn)); + } + return 0; +} + +/** Helper: iterate over cell statistics of <b>circ</b> and sum up added + * cells, removed cells, and waiting times by cell command and direction. + * Store results in <b>cell_stats</b>. Free cell statistics of the + * circuit afterwards. */ +void +sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats) +{ + memset(cell_stats, 0, sizeof(cell_stats_t)); + SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats, + testing_cell_stats_entry_t *, ent) { + tor_assert(ent->command <= CELL_COMMAND_MAX_); + if (!ent->removed && !ent->exitward) { + cell_stats->added_cells_appward[ent->command] += 1; + } else if (!ent->removed && ent->exitward) { + cell_stats->added_cells_exitward[ent->command] += 1; + } else if (!ent->exitward) { + cell_stats->removed_cells_appward[ent->command] += 1; + cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10; + } else { + cell_stats->removed_cells_exitward[ent->command] += 1; + cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10; + } + tor_free(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(circ->testing_cell_stats); + circ->testing_cell_stats = NULL; +} + +/** Helper: append a cell statistics string to <code>event_parts</code>, + * prefixed with <code>key</code>=. Statistics consist of comma-separated + * key:value pairs with lower-case command strings as keys and cell + * numbers or total waiting times as values. A key:value pair is included + * if the entry in <code>include_if_non_zero</code> is not zero, but with + * the (possibly zero) entry from <code>number_to_include</code>. Both + * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1. If no + * entry in <code>include_if_non_zero</code> is positive, no string will + * be added to <code>event_parts</code>. */ +void +append_cell_stats_by_command(smartlist_t *event_parts, const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include) +{ + smartlist_t *key_value_strings = smartlist_new(); + int i; + for (i = 0; i <= CELL_COMMAND_MAX_; i++) { + if (include_if_non_zero[i] > 0) { + smartlist_add_asprintf(key_value_strings, "%s:"U64_FORMAT, + cell_command_to_string(i), + U64_PRINTF_ARG(number_to_include[i])); + } + } + if (smartlist_len(key_value_strings) > 0) { + char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL); + smartlist_add_asprintf(event_parts, "%s=%s", key, joined); + SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp)); + tor_free(joined); + } + smartlist_free(key_value_strings); +} + +/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a + * CELL_STATS event and write result string to <b>event_string</b>. */ +void +format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats) +{ + smartlist_t *event_parts = smartlist_new(); + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "ID=%lu", + (unsigned long)ocirc->global_identifier); + } else if (TO_OR_CIRCUIT(circ)->p_chan) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "InboundQueue=%lu", + (unsigned long)or_circ->p_circ_id); + smartlist_add_asprintf(event_parts, "InboundConn="U64_FORMAT, + U64_PRINTF_ARG(or_circ->p_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "InboundAdded", + cell_stats->added_cells_appward, + cell_stats->added_cells_appward); + append_cell_stats_by_command(event_parts, "InboundRemoved", + cell_stats->removed_cells_appward, + cell_stats->removed_cells_appward); + append_cell_stats_by_command(event_parts, "InboundTime", + cell_stats->removed_cells_appward, + cell_stats->total_time_appward); + } + if (circ->n_chan) { + smartlist_add_asprintf(event_parts, "OutboundQueue=%lu", + (unsigned long)circ->n_circ_id); + smartlist_add_asprintf(event_parts, "OutboundConn="U64_FORMAT, + U64_PRINTF_ARG(circ->n_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "OutboundAdded", + cell_stats->added_cells_exitward, + cell_stats->added_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundRemoved", + cell_stats->removed_cells_exitward, + cell_stats->removed_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundTime", + cell_stats->removed_cells_exitward, + cell_stats->total_time_exitward); + } + *event_string = smartlist_join_strings(event_parts, " ", 0, NULL); + SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp)); + smartlist_free(event_parts); +} + +/** A second or more has elapsed: tell any interested control connection + * how many cells have been processed for a given circuit. */ +int +control_event_circuit_cell_stats(void) +{ + circuit_t *circ; + cell_stats_t *cell_stats; + char *event_string; + if (!get_options()->TestingEnableCellStatsEvent || + !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) + return 0; + cell_stats = tor_malloc(sizeof(cell_stats_t));; + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!circ->testing_cell_stats) + continue; + sum_up_cell_stats_by_command(circ, cell_stats); + format_cell_stats(&event_string, circ, cell_stats); + send_control_event(EVENT_CELL_STATS, ALL_FORMATS, + "650 CELL_STATS %s\r\n", event_string); + tor_free(event_string); + } + tor_free(cell_stats); + return 0; +} + +/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty + * for <b>read_empty_time</b> millis, the write bucket was empty for + * <b>write_empty_time</b> millis, and buckets were last refilled + * <b>milliseconds_elapsed</b> millis ago. Only emit TB_EMPTY event if + * either read or write bucket have been empty before. */ +int +control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed) +{ + if (get_options()->TestingEnableTbEmptyEvent && + EVENT_IS_INTERESTING(EVENT_TB_EMPTY) && + (read_empty_time > 0 || write_empty_time > 0)) { + send_control_event(EVENT_TB_EMPTY, ALL_FORMATS, + "650 TB_EMPTY %s READ=%d WRITTEN=%d " + "LAST=%d\r\n", + bucket, read_empty_time, write_empty_time, + milliseconds_elapsed); + } + return 0; +} + /** A second or more has elapsed: tell any interested control * connections how much bandwidth we used. */ int @@ -4162,32 +4379,26 @@ control_event_newconsensus(const networkstatus_t *consensus) /** Called when we compute a new circuitbuildtimeout */ int -control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type) +control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args) { const char *type_string = NULL; - double qnt; if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) return 0; - qnt = circuit_build_times_quantile_cutoff(); - switch (type) { case BUILDTIMEOUT_SET_EVENT_COMPUTED: type_string = "COMPUTED"; break; case BUILDTIMEOUT_SET_EVENT_RESET: type_string = "RESET"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_SUSPENDED: type_string = "SUSPENDED"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_DISCARD: type_string = "DISCARD"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_RESUME: type_string = "RESUME"; @@ -4198,15 +4409,8 @@ control_event_buildtimeout_set(const circuit_build_times_t *cbt, } send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, - "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " - "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " - "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f\r\n", - type_string, (unsigned long)cbt->total_build_times, - (unsigned long)cbt->timeout_ms, - (unsigned long)cbt->Xm, cbt->alpha, qnt, - circuit_build_times_timeout_rate(cbt), - (unsigned long)cbt->close_ms, - circuit_build_times_close_rate(cbt)); + "650 BUILDTIMEOUT_SET %s %s\r\n", + type_string, args); return 0; } @@ -4434,8 +4638,8 @@ control_event_conf_changed(const smartlist_t *elements) /** Helper: Return a newly allocated string containing a path to the * file where we store our authentication cookie. */ -static char * -get_cookie_file(void) +char * +get_controller_cookie_file_name(void) { const or_options_t *options = get_options(); if (options->CookieAuthFile && strlen(options->CookieAuthFile)) { @@ -4445,44 +4649,27 @@ get_cookie_file(void) } } -/** Choose a random authentication cookie and write it to disk. - * Anybody who can read the cookie from disk will be considered - * authorized to use the control connection. Return -1 if we can't - * write the file, or 0 on success. */ +/* Initialize the cookie-based authentication system of the + * ControlPort. If <b>enabled</b> is 0, then disable the cookie + * authentication system. */ int -init_cookie_authentication(int enabled) +init_control_cookie_authentication(int enabled) { - char *fname; + char *fname = NULL; + int retval; + if (!enabled) { authentication_cookie_is_set = 0; return 0; } - /* We don't want to generate a new cookie every time we call - * options_act(). One should be enough. */ - if (authentication_cookie_is_set) - return 0; /* all set */ - - fname = get_cookie_file(); - crypto_rand(authentication_cookie, AUTHENTICATION_COOKIE_LEN); - authentication_cookie_is_set = 1; - if (write_bytes_to_file(fname, authentication_cookie, - AUTHENTICATION_COOKIE_LEN, 1)) { - log_warn(LD_FS,"Error writing authentication cookie to %s.", - escaped(fname)); - tor_free(fname); - return -1; - } -#ifndef _WIN32 - if (get_options()->CookieAuthFileGroupReadable) { - if (chmod(fname, 0640)) { - log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); - } - } -#endif - + fname = get_controller_cookie_file_name(); + retval = init_cookie_authentication(fname, "", /* no header */ + AUTHENTICATION_COOKIE_LEN, + &authentication_cookie, + &authentication_cookie_is_set); tor_free(fname); - return 0; + return retval; } /** A copy of the process specifier of Tor's owning controller, or @@ -4493,6 +4680,8 @@ static char *owning_controller_process_spec = NULL; * if this Tor instance is not currently owned by a process. */ static tor_process_monitor_t *owning_controller_process_monitor = NULL; +static void owning_controller_procmon_cb(void *unused) ATTR_NORETURN; + /** Process-termination monitor callback for Tor's owning controller * process. */ static void @@ -4636,16 +4825,28 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag, * Tor initializes. */ static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + /** How many problems have we had getting to the next bootstrapping phase? * These include failure to establish a connection to a Tor relay, * failures to finish the TLS handshake, failures to validate the * consensus document, etc. */ static int bootstrap_problems = 0; -/* We only tell the controller once we've hit a threshold of problems +/** We only tell the controller once we've hit a threshold of problems * for the current phase. */ #define BOOTSTRAP_PROBLEM_THRESHOLD 10 +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + /** Called when Tor has made progress at bootstrapping its directory * information and initial circuits. * @@ -4665,7 +4866,7 @@ control_event_bootstrap(bootstrap_status_t status, int progress) * can't distinguish what the connection is going to be for. */ if (status == BOOTSTRAP_STATUS_HANDSHAKE) { if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; + status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; } else { status = BOOTSTRAP_STATUS_HANDSHAKE_OR; } @@ -4673,9 +4874,19 @@ control_event_bootstrap(bootstrap_status_t status, int progress) if (status > bootstrap_percent || (progress && progress > bootstrap_percent)) { + int loglevel = LOG_NOTICE; bootstrap_status_to_string(status, &tag, &summary); - tor_log(status ? LOG_NOTICE : LOG_INFO, LD_CONTROL, - "Bootstrapped %d%%: %s.", progress ? progress : status, summary); + + if (status <= bootstrap_percent && + (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)) { + /* We log the message at info if the status hasn't advanced, and if less + * than BOOTSTRAP_PCT_INCREMENT progress has been made. + */ + loglevel = LOG_INFO; + } + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%%: %s", progress ? progress : status, summary); tor_snprintf(buf, sizeof(buf), "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", progress ? progress : status, tag, summary); @@ -4691,15 +4902,22 @@ control_event_bootstrap(bootstrap_status_t status, int progress) bootstrap_percent = progress; bootstrap_problems = 0; /* Progress! Reset our problem counter. */ } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } } } /** Called when Tor has failed to make bootstrapping progress in a way * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. */ -void -control_event_bootstrap_problem(const char *warn, int reason) +MOCK_IMPL(void, + control_event_bootstrap_problem, (const char *warn, int reason, + or_connection_t *or_conn)) { int status = bootstrap_percent; const char *tag, *summary; @@ -4710,6 +4928,11 @@ control_event_bootstrap_problem(const char *warn, int reason) /* bootstrap_percent must not be in "undefined" state here. */ tor_assert(status >= 0); + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + if (bootstrap_percent == 100) return; /* already bootstrapped; nothing to be done here. */ @@ -4721,9 +4944,10 @@ control_event_bootstrap_problem(const char *warn, int reason) if (reason == END_OR_CONN_REASON_NO_ROUTE) recommendation = "warn"; - if (get_options()->UseBridges && - !any_bridge_descriptors_known() && - !any_pending_bridge_descriptor_fetches()) + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) recommendation = "warn"; if (we_are_hibernating()) @@ -4766,3 +4990,159 @@ control_event_clients_seen(const char *controller_str) "650 CLIENTS_SEEN %s\r\n", controller_str); } +/** A new pluggable transport called <b>transport_name</b> was + * launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either + * "server" or "client" depending on the mode of the pluggable + * transport. + * "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port + */ +void +control_event_transport_launched(const char *mode, const char *transport_name, + tor_addr_t *addr, uint16_t port) +{ + send_control_event(EVENT_TRANSPORT_LAUNCHED, ALL_FORMATS, + "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n", + mode, transport_name, fmt_addr(addr), port); +} + +/** Convert rendezvous auth type to string for HS_DESC control events + */ +const char * +rend_auth_type_to_string(rend_auth_type_t auth_type) +{ + const char *str; + + switch (auth_type) { + case REND_NO_AUTH: + str = "NO_AUTH"; + break; + case REND_BASIC_AUTH: + str = "BASIC_AUTH"; + break; + case REND_STEALTH_AUTH: + str = "STEALTH_AUTH"; + break; + default: + str = "UNKNOWN"; + } + + return str; +} + +/** Return a longname the node whose identity is <b>id_digest</b>. If + * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is + * returned instead. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +MOCK_IMPL(const char *, +node_describe_longname_by_id,(const char *id_digest)) +{ + static char longname[MAX_VERBOSE_NICKNAME_LEN+1]; + node_get_verbose_nickname_by_id(id_digest, longname); + return longname; +} + +/** send HS_DESC requested event. + * + * <b>rend_query</b> is used to fetch requested onion address and auth type. + * <b>hs_dir</b> is the description of contacting hs directory. + * <b>desc_id_base32</b> is the ID of requested hs descriptor. + */ +void +control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *id_digest, + const char *desc_id_base32) +{ + if (!id_digest || !rend_query || !desc_id_base32) { + log_warn(LD_BUG, "Called with rend_query==%p, " + "id_digest==%p, desc_id_base32==%p", + rend_query, id_digest, desc_id_base32); + return; + } + + send_control_event(EVENT_HS_DESC, ALL_FORMATS, + "650 HS_DESC REQUESTED %s %s %s %s\r\n", + rend_query->onion_address, + rend_auth_type_to_string(rend_query->auth_type), + node_describe_longname_by_id(id_digest), + desc_id_base32); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hs_descriptor_received + * control_event_hs_descriptor_failed + * + * So do not call this function directly. + */ +void +control_event_hs_descriptor_receive_end(const char *action, + const rend_data_t *rend_query, + const char *id_digest) +{ + if (!action || !rend_query || !id_digest) { + log_warn(LD_BUG, "Called with action==%p, rend_query==%p, " + "id_digest==%p", action, rend_query, id_digest); + return; + } + + send_control_event(EVENT_HS_DESC, ALL_FORMATS, + "650 HS_DESC %s %s %s %s\r\n", + action, + rend_query->onion_address, + rend_auth_type_to_string(rend_query->auth_type), + node_describe_longname_by_id(id_digest)); +} + +/** send HS_DESC RECEIVED event + * + * called when a we successfully received a hidden service descriptor. + */ +void +control_event_hs_descriptor_received(const rend_data_t *rend_query, + const char *id_digest) +{ + if (!rend_query || !id_digest) { + log_warn(LD_BUG, "Called with rend_query==%p, id_digest==%p", + rend_query, id_digest); + return; + } + control_event_hs_descriptor_receive_end("RECEIVED", rend_query, id_digest); +} + +/** send HS_DESC FAILED event + * + * called when request for hidden service descriptor returned failure. + */ +void +control_event_hs_descriptor_failed(const rend_data_t *rend_query, + const char *id_digest) +{ + if (!rend_query || !id_digest) { + log_warn(LD_BUG, "Called with rend_query==%p, id_digest==%p", + rend_query, id_digest); + return; + } + control_event_hs_descriptor_receive_end("FAILED", rend_query, id_digest); +} + +/** Free any leftover allocated memory of the control.c subsystem. */ +void +control_free_all(void) +{ + if (authentication_cookie) /* Free the auth cookie */ + tor_free(authentication_cookie); +} + +#ifdef TOR_UNIT_TESTS +/* For testing: change the value of global_event_mask */ +void +control_testing_set_global_event_mask(uint64_t mask) +{ + global_event_mask = mask; +} +#endif + diff --git a/src/or/control.h b/src/or/control.h index 61062da2c..68a6c244d 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -50,6 +50,13 @@ int control_event_or_conn_status(or_connection_t *conn, int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); int control_event_stream_bandwidth(edge_connection_t *edge_conn); int control_event_stream_bandwidth_used(void); +int control_event_circ_bandwidth_used(void); +int control_event_conn_bandwidth(connection_t *conn); +int control_event_conn_bandwidth_used(void); +int control_event_circuit_cell_stats(void); +int control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed); void control_event_logmsg(int severity, uint32_t domain, const char *msg); int control_event_descriptors_changed(smartlist_t *routers); int control_event_address_mapped(const char *from, const char *to, @@ -73,11 +80,12 @@ int control_event_server_status(int severity, const char *format, ...) int control_event_guard(const char *nickname, const char *digest, const char *status); int control_event_conf_changed(const smartlist_t *elements); -int control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type); +int control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args); int control_event_signal(uintptr_t signal); -int init_cookie_authentication(int enabled); +int init_control_cookie_authentication(int enabled); +char *get_controller_cookie_file_name(void); smartlist_t *decode_hashed_passwords(config_line_t *passwords); void disable_control_logging(void); void enable_control_logging(void); @@ -85,14 +93,114 @@ void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); +MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn, + int reason, + or_connection_t *or_conn)); void control_event_clients_seen(const char *controller_str); +void control_event_transport_launched(const char *mode, + const char *transport_name, + tor_addr_t *addr, uint16_t port); +const char *rend_auth_type_to_string(rend_auth_type_t auth_type); +MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); +void control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *desc_id_base32, + const char *hs_dir); +void control_event_hs_descriptor_receive_end(const char *action, + const rend_data_t *rend_query, + const char *hs_dir); +void control_event_hs_descriptor_received(const rend_data_t *rend_query, + const char *hs_dir); +void control_event_hs_descriptor_failed(const rend_data_t *rend_query, + const char *hs_dir); + +void control_free_all(void); #ifdef CONTROL_PRIVATE +/* Recognized asynchronous event types. It's okay to expand this list + * because it is used both as a list of v0 event types, and as indices + * into the bitfield to determine which controllers want which events. + */ +#define EVENT_MIN_ 0x0001 +#define EVENT_CIRCUIT_STATUS 0x0001 +#define EVENT_STREAM_STATUS 0x0002 +#define EVENT_OR_CONN_STATUS 0x0003 +#define EVENT_BANDWIDTH_USED 0x0004 +#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 +#define EVENT_NEW_DESC 0x0006 +#define EVENT_DEBUG_MSG 0x0007 +#define EVENT_INFO_MSG 0x0008 +#define EVENT_NOTICE_MSG 0x0009 +#define EVENT_WARN_MSG 0x000A +#define EVENT_ERR_MSG 0x000B +#define EVENT_ADDRMAP 0x000C +/* Exposed above */ +// #define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_DESCCHANGED 0x000E +/* Exposed above */ +// #define EVENT_NS 0x000F +#define EVENT_STATUS_CLIENT 0x0010 +#define EVENT_STATUS_SERVER 0x0011 +#define EVENT_STATUS_GENERAL 0x0012 +#define EVENT_GUARD 0x0013 +#define EVENT_STREAM_BANDWIDTH_USED 0x0014 +#define EVENT_CLIENTS_SEEN 0x0015 +#define EVENT_NEWCONSENSUS 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define EVENT_SIGNAL 0x0018 +#define EVENT_CONF_CHANGED 0x0019 +#define EVENT_CONN_BW 0x001A +#define EVENT_CELL_STATS 0x001B +#define EVENT_TB_EMPTY 0x001C +#define EVENT_CIRC_BANDWIDTH_USED 0x001D +#define EVENT_TRANSPORT_LAUNCHED 0x0020 +#define EVENT_HS_DESC 0x0021 +#define EVENT_MAX_ 0x0021 +/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a + * different structure. */ + /* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); +STATIC size_t write_escaped_data(const char *data, size_t len, char **out); +STATIC size_t read_escaped_data(const char *data, size_t len, char **out); +/** Flag for event_format_t. Indicates that we should use the one standard + format. (Other formats previous existed, and are now deprecated) + */ +#define ALL_FORMATS 1 +/** Bit field of flags to select how to format a controller event. Recognized + * flag is ALL_FORMATS. */ +typedef int event_format_t; + +#ifdef TOR_UNIT_TESTS +MOCK_DECL(STATIC void, +send_control_event_string,(uint16_t event, event_format_t which, + const char *msg)); + +void control_testing_set_global_event_mask(uint64_t mask); +#endif + +/** Helper structure: temporarily stores cell statistics for a circuit. */ +typedef struct cell_stats_t { + /** Number of cells added in app-ward direction by command. */ + uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells added in exit-ward direction by command. */ + uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in app-ward direction by command. */ + uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in exit-ward direction by command. */ + uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in app-ward direction by command. */ + uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in exit-ward direction by command. */ + uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1]; +} cell_stats_t; +void sum_up_cell_stats_by_command(circuit_t *circ, + cell_stats_t *cell_stats); +void append_cell_stats_by_command(smartlist_t *event_parts, + const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include); +void format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats); #endif #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index ecf0d2035..61b2c29b3 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -436,7 +436,7 @@ cpuworker_main(void *data) if (req.task == CPUWORKER_TASK_ONION) { const create_cell_t *cc = &req.create_cell; created_cell_t *cell_out = &rpl.created_cell; - struct timeval tv_start, tv_end; + struct timeval tv_start = {0,0}, tv_end; int n; rpl.timed = req.timed; rpl.started_at = req.started_at; @@ -528,7 +528,12 @@ spawn_cpuworker(void) tor_assert(SOCKET_OK(fdarray[1])); fd = fdarray[0]; - spawn_func(cpuworker_main, (void*)fdarray); + if (spawn_func(cpuworker_main, (void*)fdarray) < 0) { + tor_close_socket(fdarray[0]); + tor_close_socket(fdarray[1]); + tor_free(fdarray); + return -1; + } log_debug(LD_OR,"just spawned a cpu worker."); #ifndef TOR_IS_MULTITHREADED tor_close_socket(fdarray[1]); /* don't need the worker's side of the pipe */ @@ -686,7 +691,7 @@ assign_onionskin_to_cpuworker(connection_t *cpuworker, } if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) - rep_hist_note_circuit_handshake_completed(onionskin->handshake_type); + rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); should_time = should_time_request(onionskin->handshake_type); memset(&req, 0, sizeof(req)); diff --git a/src/or/directory.c b/src/or/directory.c index 3752367c4..22ba056ee 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -67,15 +67,11 @@ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); -static void connection_dir_download_v2_networkstatus_failed( - dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); static void connection_dir_retry_bridges(smartlist_t *descs); -static void dir_networkstatus_download_failed(smartlist_t *failed, - int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int router_purpose, @@ -86,8 +82,7 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const char *address, - const tor_addr_t *addr, +static void directory_initiate_command_rend(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, @@ -135,7 +130,6 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES || - dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS || dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE || dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES || dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS || @@ -154,16 +148,10 @@ authdir_type_to_string(dirinfo_type_t auth) { char *result; smartlist_t *lst = smartlist_new(); - if (auth & V1_DIRINFO) - smartlist_add(lst, (void*)"V1"); - if (auth & V2_DIRINFO) - smartlist_add(lst, (void*)"V2"); if (auth & V3_DIRINFO) smartlist_add(lst, (void*)"V3"); if (auth & BRIDGE_DIRINFO) smartlist_add(lst, (void*)"Bridge"); - if (auth & HIDSERV_DIRINFO) - smartlist_add(lst, (void*)"Hidden service"); if (smartlist_len(lst)) { result = smartlist_join_strings(lst, ", ", 0, NULL); } else { @@ -179,18 +167,12 @@ dir_conn_purpose_to_string(int purpose) { switch (purpose) { - case DIR_PURPOSE_FETCH_RENDDESC: - return "hidden-service descriptor fetch"; case DIR_PURPOSE_UPLOAD_DIR: return "server descriptor upload"; - case DIR_PURPOSE_UPLOAD_RENDDESC: - return "hidden-service descriptor upload"; case DIR_PURPOSE_UPLOAD_VOTE: return "server vote upload"; case DIR_PURPOSE_UPLOAD_SIGNATURES: return "consensus signature upload"; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - return "network-status fetch"; case DIR_PURPOSE_FETCH_SERVERDESC: return "server descriptor fetch"; case DIR_PURPOSE_FETCH_EXTRAINFO: @@ -258,13 +240,13 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> * (of length <b>payload_len</b>). The dir_purpose should be one of - * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. + * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'. * * <b>router_purpose</b> describes the type of descriptor we're * publishing, if we're publishing a descriptor -- e.g. general or bridge. * - * <b>type</b> specifies what sort of dir authorities (V1, V2, - * HIDSERV, BRIDGE) we should upload to. + * <b>type</b> specifies what sort of dir authorities (V3, + * BRIDGE, etc) we should upload to. * * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b> @@ -279,7 +261,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, size_t payload_len, size_t extrainfo_len) { const or_options_t *options = get_options(); - int post_via_tor; + dir_indirection_t indirection; const smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -296,8 +278,12 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, if ((type & ds->type) == 0) continue; - if (exclude_self && router_digest_is_me(ds->digest)) + if (exclude_self && router_digest_is_me(ds->digest)) { + /* we don't upload to ourselves, but at least there's now at least + * one authority of this type that has what we wanted to upload. */ + found = 1; continue; + } if (options->StrictNodes && routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) { @@ -319,11 +305,19 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, (int) extrainfo_len); } tor_addr_from_ipv4h(&ds_addr, ds->addr); - post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) || - !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port); + if (purpose_needs_anonymity(dir_purpose, router_purpose)) { + indirection = DIRIND_ANONYMOUS; + } else if (!fascist_firewall_allows_address_dir(&ds_addr,ds->dir_port)) { + if (fascist_firewall_allows_address_or(&ds_addr,ds->or_port)) + indirection = DIRIND_ONEHOP; + else + indirection = DIRIND_ANONYMOUS; + } else { + indirection = DIRIND_DIRECT_CONN; + } directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - post_via_tor, + indirection, NULL, payload, upload_len, 0); } SMARTLIST_FOREACH_END(ds); if (!found) { @@ -350,10 +344,7 @@ should_use_directory_guards(const or_options_t *options) /* If we're configured to fetch directory info aggressively or of a * nonstandard type, don't use directory guards. */ if (options->DownloadExtraInfo || options->FetchDirInfoEarly || - options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors || - options->FetchV2Networkstatus) - return 0; - if (! options->PreferTunneledDirConns) + options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors) return 0; return 1; } @@ -414,18 +405,10 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : V3_DIRINFO); break; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - type = V2_DIRINFO; - prefer_authority = 1; /* Only v2 authorities have these anyway. */ - require_authority = 1; /* Don't fallback to asking a non-authority */ - break; case DIR_PURPOSE_FETCH_SERVERDESC: type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : V3_DIRINFO); break; - case DIR_PURPOSE_FETCH_RENDDESC: - type = HIDSERV_DIRINFO; - break; case DIR_PURPOSE_FETCH_STATUS_VOTE: case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: case DIR_PURPOSE_FETCH_CERTIFICATE: @@ -465,7 +448,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, } } - if (!options->FetchServerDescriptors && type != HIDSERV_DIRINFO) + if (!options->FetchServerDescriptors) return; if (!get_via_tor) { @@ -484,7 +467,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, tor_addr_t addr; routerinfo_t *ri = node->ri; node_get_addr(node, &addr); - directory_initiate_command(ri->address, &addr, + directory_initiate_command(&addr, ri->or_port, 0/*no dirport*/, ri->cache_info.identity_digest, dir_purpose, @@ -536,11 +519,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, } } else { /* get_via_tor */ /* Never use fascistfirewall; we're going via Tor. */ - if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) { - /* only ask hidserv authorities, any of them will do */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF; - rs = router_pick_trusteddirserver(HIDSERV_DIRINFO, pds_flags); - } else { + if (1) { /* anybody with a non-zero dirport will do. Disregard firewalls. */ pds_flags |= PDS_IGNORE_FASCISTFIREWALL; rs = router_pick_directory_server(type, pds_flags); @@ -617,9 +596,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - char address_buf[INET_NTOA_BUF_LEN+1]; - struct in_addr in; - const char *address; tor_addr_t addr; const int anonymized_connection = dirind_is_anon(indirection); node = node_get_by_id(status->identity_digest); @@ -629,13 +605,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, "don't have its router descriptor.", routerstatus_describe(status)); return; - } else if (node) { - node_get_address_string(node, address_buf, sizeof(address_buf)); - address = address_buf; - } else { - in.s_addr = htonl(status->addr); - tor_inet_ntoa(&in, address_buf, sizeof(address_buf)); - address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); @@ -649,7 +618,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(address, &addr, + directory_initiate_command_rend(&addr, status->or_port, status->dir_port, status->identity_digest, dir_purpose, router_purpose, @@ -662,7 +631,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, * upload or download a server or rendezvous * descriptor. <b>dir_purpose</b> determines what * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b> + * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> * specifies the descriptor purposes we have in mind (currently only * used for FETCH_DIR). * @@ -719,11 +688,7 @@ connection_dir_request_failed(dir_connection_t *conn) } if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", - conn->base_.address); - connection_dir_download_v2_networkstatus_failed(conn, -1); - } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " "directory server at '%s'; retrying", @@ -747,48 +712,11 @@ connection_dir_request_failed(dir_connection_t *conn) conn->base_.address); } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { log_info(LD_DIR, "Giving up on downloading microdescriptors from " - " directory server at '%s'; will retry", conn->base_.address); + "directory server at '%s'; will retry", conn->base_.address); connection_dir_download_routerdesc_failed(conn); } } -/** Called when an attempt to download one or more network status - * documents on connection <b>conn</b> failed. Decide whether to - * retry the fetch now, later, or never. - */ -static void -connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn, - int status_code) -{ - if (!conn->requested_resource) { - /* We never reached directory_send_command, which means that we never - * opened a network connection. Either we're out of sockets, or the - * network is down. Either way, retrying would be pointless. */ - return; - } - if (!strcmpstart(conn->requested_resource, "all")) { - /* We're a non-authoritative directory cache; try again. Ignore status - * code, since we don't want to keep trying forever in a tight loop - * if all the authorities are shutting us out. */ - const smartlist_t *trusted_dirs = router_get_trusted_dir_servers(); - SMARTLIST_FOREACH(trusted_dirs, dir_server_t *, ds, - download_status_failed(&ds->v2_ns_dl_status, 0)); - directory_get_from_dirserver(conn->base_.purpose, conn->router_purpose, - "all.z", 0 /* don't retry_if_no_servers */); - } else if (!strcmpstart(conn->requested_resource, "fp/")) { - /* We were trying to download by fingerprint; mark them all as having - * failed, and possibly retry them later.*/ - smartlist_t *failed = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0); - if (smartlist_len(failed)) { - dir_networkstatus_download_failed(failed, status_code); - SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); - } - smartlist_free(failed); - } -} - /** Helper: Attempt to fetch directly the descriptors of each bridge * listed in <b>failed</b>. */ @@ -912,6 +840,7 @@ directory_command_should_use_begindir(const or_options_t *options, int or_port, uint8_t router_purpose, dir_indirection_t indirection) { + (void) router_purpose; if (!or_port) return 0; /* We don't know an ORPort -- no chance. */ if (indirection == DIRIND_DIRECT_CONN || indirection == DIRIND_ANON_DIRPORT) @@ -920,9 +849,6 @@ directory_command_should_use_begindir(const or_options_t *options, if (!fascist_firewall_allows_address_or(addr, or_port) || directory_fetches_from_authorities(options)) return 0; /* We're firewalled or are acting like a relay -- also no. */ - if (!options->TunnelDirConns && - router_purpose != ROUTER_PURPOSE_BRIDGE) - return 0; /* We prefer to avoid using begindir conns. Fine. */ return 1; } @@ -932,7 +858,7 @@ directory_command_should_use_begindir(const or_options_t *options, * <b>supports_begindir</b>, and whose identity key digest is * <b>digest</b>. */ void -directory_initiate_command(const char *address, const tor_addr_t *_addr, +directory_initiate_command(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -940,7 +866,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(address, _addr, or_port, dir_port, + directory_initiate_command_rend(_addr, or_port, dir_port, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -954,9 +880,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, static int is_sensitive_dir_purpose(uint8_t dir_purpose) { - return ((dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) || - (dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC) || - (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC) || + return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)); } @@ -964,7 +888,7 @@ is_sensitive_dir_purpose(uint8_t dir_purpose) /** Same as directory_initiate_command(), but accepts rendezvous data to * fetch a hidden service descriptor. */ static void -directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, +directory_initiate_command_rend(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -982,7 +906,6 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, const int anonymized_connection = dirind_is_anon(indirection); tor_addr_t addr; - tor_assert(address); tor_assert(_addr); tor_assert(or_port || dir_port); tor_assert(digest); @@ -1015,7 +938,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = use_begindir ? or_port : dir_port; - conn->base_.address = tor_strdup(address); + conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; @@ -1250,11 +1173,6 @@ directory_send_command(dir_connection_t *conn, } switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - tor_assert(resource); - httpcommand = "GET"; - tor_asprintf(&url, "/tor/status/%s", resource); - break; case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ tor_assert(!payload); @@ -1326,12 +1244,6 @@ directory_send_command(dir_connection_t *conn, httpcommand = "GET"; tor_asprintf(&url, "/tor/rendezvous2/%s", resource); break; - case DIR_PURPOSE_UPLOAD_RENDDESC: - tor_assert(!resource); - tor_assert(payload); - httpcommand = "POST"; - url = tor_strdup("/tor/rendezvous/publish"); - break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: tor_assert(!resource); tor_assert(payload); @@ -1387,7 +1299,7 @@ directory_send_command(dir_connection_t *conn, * so it does. Return 0. * Otherwise, return -1. */ -static int +STATIC int parse_http_url(const char *headers, char **url) { char *s, *start, *tmp; @@ -1416,6 +1328,19 @@ parse_http_url(const char *headers, char **url) } } + /* Check if the header is well formed (next sequence + * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */ + { + unsigned minor_ver; + char ch; + char *e = (char *)eat_whitespace_no_nl(s); + if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) { + return -1; + } + if (ch != '\r') + return -1; + } + if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */ *url = tor_malloc(s - start + 5); strlcpy(*url,"/tor", s-start+5); @@ -1462,13 +1387,14 @@ http_set_address_origin(const char *headers, connection_t *conn) if (!fwd) fwd = http_get_header(headers, "X-Forwarded-For: "); if (fwd) { - struct in_addr in; - if (!tor_inet_aton(fwd, &in) || is_internal_IP(ntohl(in.s_addr), 0)) { - log_debug(LD_DIR, "Ignoring unrecognized or internal IP %s", - escaped(fwd)); + tor_addr_t toraddr; + if (tor_addr_parse(&toraddr,fwd) == -1 || + tor_addr_is_internal(&toraddr,0)) { + log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd)); tor_free(fwd); return; } + tor_free(conn->address); conn->address = tor_strdup(fwd); tor_free(fwd); @@ -1565,8 +1491,8 @@ parse_http_response(const char *headers, int *code, time_t *date, } /** Return true iff <b>body</b> doesn't start with a plausible router or - * running-list or directory opening. This is a sign of possible compression. - **/ + * network-status or microdescriptor opening. This is a sign of possible + * compression. */ static int body_is_plausible(const char *body, size_t len, int purpose) { @@ -1578,20 +1504,16 @@ body_is_plausible(const char *body, size_t len, int purpose) if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { return (!strcmpstart(body,"onion-key")); } - if (purpose != DIR_PURPOSE_FETCH_RENDDESC) { + if (1) { if (!strcmpstart(body,"router") || - !strcmpstart(body,"signed-directory") || - !strcmpstart(body,"network-status") || - !strcmpstart(body,"running-routers")) - return 1; + !strcmpstart(body,"network-status")) + return 1; for (i=0;i<32;++i) { if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) return 0; } - return 1; - } else { - return 1; } + return 1; } /** Called when we've just fetched a bunch of router descriptors in @@ -1626,8 +1548,9 @@ load_downloaded_routers(const char *body, smartlist_t *which, added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, descriptor_digests, buf); - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (general) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); return added; } @@ -1646,17 +1569,17 @@ connection_dir_client_reached_eof(dir_connection_t *conn) char *body; char *headers; char *reason = NULL; - size_t body_len=0, orig_len=0; + size_t body_len = 0, orig_len = 0; int status_code; - time_t date_header=0; + time_t date_header = 0; long delta; compress_method_t compression; int plausible; - int skewed=0; + int skewed = 0; int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - int was_compressed=0; + int was_compressed = 0; time_t now = time(NULL); int src_code; @@ -1810,77 +1733,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - smartlist_t *which = NULL; - v2_networkstatus_source_t source; - char *cp; - log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - static ratelim_t warning_limit = RATELIM_INIT(3600); - char *m; - if ((m = rate_limit_log(&warning_limit, now))) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status/%s\". " - "I'll try again soon.%s", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource, m); - tor_free(m); - } - tor_free(body); tor_free(headers); tor_free(reason); - connection_dir_download_v2_networkstatus_failed(conn, status_code); - return -1; - } - if (conn->requested_resource && - !strcmpstart(conn->requested_resource,"fp/")) { - source = NS_FROM_DIR_BY_FP; - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0); - } else if (conn->requested_resource && - !strcmpstart(conn->requested_resource, "all")) { - source = NS_FROM_DIR_ALL; - which = smartlist_new(); - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - { - char *hex = tor_malloc(HEX_DIGEST_LEN+1); - base16_encode(hex, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN); - smartlist_add(which, hex); - }); - } else { - /* XXXX Can we even end up here? -- weasel*/ - source = NS_FROM_DIR_BY_FP; - log_warn(LD_BUG, "We received a networkstatus but we didn't ask " - "for it by fp, nor did we ask for all."); - } - cp = body; - while (*cp) { - char *next = strstr(cp, "\nnetwork-status-version"); - if (next) - next[1] = '\0'; - /* learn from it, and then remove it from 'which' */ - if (router_set_networkstatus_v2(cp, now, source, which)<0) - break; - if (next) { - next[1] = 'n'; - cp = next+1; - } else - break; - } - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 2); - directory_info_has_arrived(now, 0); - if (which) { - if (smartlist_len(which)) { - dir_networkstatus_download_failed(which, status_code); - } - SMARTLIST_FOREACH(which, char *, s, tor_free(s)); - smartlist_free(which); - } - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int r; const char *flavname = conn->requested_resource; @@ -2220,47 +2072,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * dirservers down just because they don't like us. */ } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC) { - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - if (rend_cache_store(body, body_len, 0, - conn->rend_data->onion_address) < -1) { - log_warn(LD_REND,"Failed to parse rendezvous descriptor."); - /* Any pending rendezvous attempts will notice when - * connection_about_to_close_connection() - * cleans this dir conn up. */ - /* We could retry. But since v0 descriptors are going out of - * style, it isn't worth the hassle. We'll do better in v2. */ - } else { - /* Success, or at least there's a v2 descriptor already - * present. Notify pending connections about this. */ - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address); - } - break; - case 404: - /* Not there. Pending connections will be notified when - * connection_about_to_close_connection() cleans this conn up. */ - break; - case 400: - log_warn(LD_REND, - "http status 400 (%s). Dirserver didn't like our " - "rendezvous query?", escaped(reason)); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected while " - "fetching hidden service descriptor (server '%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - break; - } - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { + #define SEND_HS_DESC_FAILED_EVENT() ( \ + control_event_hs_descriptor_failed(conn->rend_data, \ + conn->identity_digest) ) tor_assert(conn->rend_data); log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " "(%s))", @@ -2268,24 +2083,22 @@ connection_dir_client_reached_eof(dir_connection_t *conn) switch (status_code) { case 200: switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) { - case -2: + case RCS_BADDESC: + case RCS_NOTDIR: /* Impossible */ log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " "Retrying at another directory."); /* We'll retry when connection_about_to_close_connection() * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT(); break; - case -1: - /* We already have a v0 descriptor here. Ignoring this one - * and _not_ performing another request. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor, but we already have a v0 descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - break; + case RCS_OKAY: default: /* success. notify pending connections about this. */ log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; + control_event_hs_descriptor_received(conn->rend_data, + conn->identity_digest); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; rend_client_desc_trynow(conn->rend_data->onion_address); break; } @@ -2295,12 +2108,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * connection_about_to_close_connection() cleans this conn up. */ log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT(); break; case 400: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " "http status 400 (%s). Dirserver didn't like our " "v2 rendezvous query? Retrying at another directory.", escaped(reason)); + SEND_HS_DESC_FAILED_EVENT(); break; default: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " @@ -2309,12 +2124,12 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "Retrying at another directory.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_FAILED_EVENT(); break; } } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC || - conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { + if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " "(%s))", status_code, escaped(reason)); @@ -2418,7 +2233,7 @@ connection_dir_about_to_close(dir_connection_t *dir_conn) } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state - * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that + * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that * refetching is unnecessary.) */ if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && dir_conn->rend_data && @@ -2492,7 +2307,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, } if (cache_lifetime > 0) { char expbuf[RFC1123_TIME_LEN+1]; - format_rfc1123_time(expbuf, now + cache_lifetime); + format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); /* We could say 'Cache-control: max-age=%d' here if we start doing * http/1.1 */ tor_snprintf(cp, sizeof(tmp)-(cp-tmp), @@ -2549,7 +2364,6 @@ note_client_request(int purpose, int compressed, size_t bytes) char *key; const char *kind = NULL; switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: kind = "dl/status"; break; case DIR_PURPOSE_FETCH_CONSENSUS: kind = "dl/consensus"; break; case DIR_PURPOSE_FETCH_CERTIFICATE: kind = "dl/cert"; break; case DIR_PURPOSE_FETCH_STATUS_VOTE: kind = "dl/vote"; break; @@ -2560,9 +2374,7 @@ note_client_request(int purpose, int compressed, size_t bytes) case DIR_PURPOSE_UPLOAD_DIR: kind = "dl/ul-dir"; break; case DIR_PURPOSE_UPLOAD_VOTE: kind = "dl/ul-vote"; break; case DIR_PURPOSE_UPLOAD_SIGNATURES: kind = "dl/ul-sig"; break; - case DIR_PURPOSE_FETCH_RENDDESC: kind = "dl/rend"; break; case DIR_PURPOSE_FETCH_RENDDESC_V2: kind = "dl/rend2"; break; - case DIR_PURPOSE_UPLOAD_RENDDESC: kind = "dl/ul-rend"; break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: kind = "dl/ul-rend2"; break; } if (kind) { @@ -2774,109 +2586,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* if no disclaimer file, fall through and continue */ } - if (!strcmp(url,"/tor/") || !strcmp(url,"/tor/dir")) { /* v1 dir fetch */ - cached_dir_t *d = dirserv_get_directory(); - - if (!d) { - log_info(LD_DIRSERV,"Client asked for the mirrored directory, but we " - "don't have a good one yet. Sending 503 Dir not available."); - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_debug(LD_DIRSERV, - "Client asked for the mirrored directory, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - - note_request(url, dlen); - - log_debug(LD_DIRSERV,"Dumping %sdirectory to client.", - compressed?"compressed ":""); - write_http_response_header(conn, dlen, compressed, - FULL_DIR_CACHE_LIFETIME); - conn->cached_dir = d; - conn->cached_dir_offset = 0; - if (!compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); - ++d->refcnt; - - /* Prime the connection with some data. */ - conn->dir_spool_src = DIR_SPOOL_CACHED_DIR; - connection_dirserv_flushed_some(conn); - goto done; - } - - if (!strcmp(url,"/tor/running-routers")) { /* running-routers fetch */ - cached_dir_t *d = dirserv_get_runningrouters(); - if (!d) { - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_info(LD_DIRSERV, - "Client asked for running-routers, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - note_request(url, dlen); - write_http_response_header(conn, dlen, compressed, - RUNNINGROUTERS_CACHE_LIFETIME); - connection_write_to_buf(compressed ? d->dir_z : d->dir, dlen, - TO_CONN(conn)); - goto done; - } - - if (!strcmpstart(url,"/tor/status/") - || !strcmpstart(url, "/tor/status-vote/current/consensus")) { - /* v2 or v3 network status fetch. */ + if (!strcmpstart(url, "/tor/status-vote/current/consensus")) { + /* v3 network status fetch. */ smartlist_t *dir_fps = smartlist_new(); - int is_v3 = !strcmpstart(url, "/tor/status-vote"); const char *request_type = NULL; - const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - if (options->DisableV2DirectoryInfo_ && !is_v3) { - static ratelim_t reject_v2_ratelim = RATELIM_INIT(1800); - char *m; - write_http_status_line(conn, 404, "Not found"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - if ((m = rate_limit_log(&reject_v2_ratelim, approx_time()))) { - log_notice(LD_DIR, "Rejected a v2 networkstatus request.%s", m); - tor_free(m); - } - goto done; - } - - if (!is_v3) { - dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); - if (!strcmpstart(key, "fp/")) - request_type = compressed?"/tor/status/fp.z":"/tor/status/fp"; - else if (!strcmpstart(key, "authority")) - request_type = compressed?"/tor/status/authority.z": - "/tor/status/authority"; - else if (!strcmpstart(key, "all")) - request_type = compressed?"/tor/status/all.z":"/tor/status/all"; - else - request_type = "/tor/status/?"; - } else { + if (1) { networkstatus_t *v; time_t now = time(NULL); const char *want_fps = NULL; @@ -2929,8 +2645,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); + geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2938,15 +2653,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2958,17 +2671,19 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_BUSY); + + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } - if (is_v3) { + if (1) { struct in_addr in; tor_addr_t addr; if (tor_inet_aton((TO_CONN(conn))->address, &in)) { tor_addr_from_ipv4h(&addr, ntohl(in.s_addr)); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, time(NULL)); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); geoip_note_ns_response(GEOIP_SUCCESS); /* Note that a request for a network status has started, so that we * can measure the download time later on. */ @@ -3310,32 +3025,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && !strcmpstart(url,"/tor/rendezvous/")) { - /* rendezvous descriptor fetch */ - const char *descp; - size_t desc_len; - const char *query = url+strlen("/tor/rendezvous/"); - - log_info(LD_REND, "Handling rendezvous descriptor get"); - switch (rend_cache_lookup_desc(query, 0, &descp, &desc_len)) { - case 1: /* valid */ - write_http_response_header_impl(conn, desc_len, - "application/octet-stream", - NULL, NULL, 0); - note_request("/tor/rendezvous?/", desc_len); - /* need to send descp separately, because it may include NULs */ - connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - break; - case 0: /* well-formed but not present */ - write_http_status_line(conn, 404, "Not found"); - break; - case -1: /* not well-formed */ - write_http_status_line(conn, 400, "Bad request"); - break; - } - goto done; - } - if (options->BridgeAuthoritativeDir && options->BridgePassword_AuthDigest_ && connection_dir_is_encrypted(conn) && @@ -3384,22 +3073,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (!strcmp(url,"/tor/dbg-stability.txt")) { - const char *stability; - size_t len; - if (options->BridgeAuthoritativeDir || - ! authdir_mode_tests_reachability(options) || - ! (stability = rep_hist_get_router_stability_doc(time(NULL)))) { - write_http_status_line(conn, 404, "Not found."); - goto done; - } - - len = strlen(stability); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(stability, len, TO_CONN(conn)); - goto done; - } - #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #define ADD_MALLINFO_LINE(x) do { \ smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \ @@ -3474,19 +3147,20 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { switch (rend_cache_store_v2_desc_as_dir(body)) { - case -2: + case RCS_NOTDIR: log_info(LD_REND, "Rejected v2 rend descriptor (length %d) from %s " "since we're not currently a hidden service directory.", (int)body_len, conn->base_.address); write_http_status_line(conn, 503, "Currently not acting as v2 " "hidden service directory"); break; - case -1: + case RCS_BADDESC: log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", (int)body_len, conn->base_.address); write_http_status_line(conn, 400, "Invalid v2 service descriptor rejected"); break; + case RCS_OKAY: default: write_http_status_line(conn, 200, "Service descriptor (v2) stored"); log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); @@ -3510,8 +3184,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, conn->base_.address, &msg); tor_assert(msg); - if (WRA_WAS_ADDED(r)) - dirserv_get_directory(); /* rebuild and write to disk */ if (r == ROUTER_ADDED_NOTIFY_GENERATOR) { /* Accepted with a message. */ @@ -3535,22 +3207,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && - !strcmpstart(url,"/tor/rendezvous/publish")) { - /* rendezvous descriptor post */ - log_info(LD_REND, "Handling rendezvous descriptor post."); - if (rend_cache_store(body, body_len, 1, NULL) < 0) { - log_fn(LOG_PROTOCOL_WARN, LD_DIRSERV, - "Rejected rend descriptor (length %d) from %s.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, - "Invalid v0 service descriptor rejected"); - } else { - write_http_status_line(conn, 200, "Service descriptor (v0) stored"); - } - goto done; - } - if (authdir_mode_v3(options) && !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */ const char *msg = "OK"; @@ -3702,80 +3358,27 @@ connection_dir_finished_connecting(dir_connection_t *conn) return 0; } -/** Called when one or more networkstatus fetches have failed (with uppercase - * fingerprints listed in <b>failed</b>). Mark those fingerprints as having - * failed once, unless they failed with status code 503. */ -static void -dir_networkstatus_download_failed(smartlist_t *failed, int status_code) -{ - if (status_code == 503) - return; - SMARTLIST_FOREACH_BEGIN(failed, const char *, fp) { - char digest[DIGEST_LEN]; - dir_server_t *dir; - if (base16_decode(digest, DIGEST_LEN, fp, strlen(fp))<0) { - log_warn(LD_BUG, "Called with bad fingerprint in list: %s", - escaped(fp)); - continue; - } - dir = router_get_fallback_dirserver_by_digest(digest); - - if (dir) - download_status_failed(&dir->v2_ns_dl_status, status_code); - } SMARTLIST_FOREACH_END(fp); -} - -/** Schedule for when servers should download things in general. */ -static const int server_dl_schedule[] = { - 0, 0, 0, 60, 60, 60*2, 60*5, 60*15, INT_MAX -}; -/** Schedule for when clients should download things in general. */ -static const int client_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, INT_MAX -}; -/** Schedule for when servers should download consensuses. */ -static const int server_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*30, 60*30, 60*30, 60*30, 60*60, 60*60*2 -}; -/** Schedule for when clients should download consensuses. */ -static const int client_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*60, 60*60, 60*60, 60*60*3, 60*60*6, 60*60*12 -}; -/** Schedule for when clients should download bridge descriptors. */ -static const int bridge_dl_schedule[] = { - 60*60, 15*60, 15*60, 60*60 -}; - -/** Decide which download schedule we want to use, and then return a - * pointer to it along with a pointer to its length. Helper function for - * download_status_increment_failure() and download_status_reset(). */ -static void -find_dl_schedule_and_len(download_status_t *dls, int server, - const int **schedule, size_t *schedule_len) +/** Decide which download schedule we want to use based on descriptor type + * in <b>dls</b> and whether we are acting as directory <b>server</b>, and + * then return a list of int pointers defining download delays in seconds. + * Helper function for download_status_increment_failure() and + * download_status_reset(). */ +static const smartlist_t * +find_dl_schedule_and_len(download_status_t *dls, int server) { switch (dls->schedule) { case DL_SCHED_GENERIC: - if (server) { - *schedule = server_dl_schedule; - *schedule_len = sizeof(server_dl_schedule)/sizeof(int); - } else { - *schedule = client_dl_schedule; - *schedule_len = sizeof(client_dl_schedule)/sizeof(int); - } - break; + if (server) + return get_options()->TestingServerDownloadSchedule; + else + return get_options()->TestingClientDownloadSchedule; case DL_SCHED_CONSENSUS: - if (server) { - *schedule = server_consensus_dl_schedule; - *schedule_len = sizeof(server_consensus_dl_schedule)/sizeof(int); - } else { - *schedule = client_consensus_dl_schedule; - *schedule_len = sizeof(client_consensus_dl_schedule)/sizeof(int); - } - break; + if (server) + return get_options()->TestingServerConsensusDownloadSchedule; + else + return get_options()->TestingClientConsensusDownloadSchedule; case DL_SCHED_BRIDGE: - *schedule = bridge_dl_schedule; - *schedule_len = sizeof(bridge_dl_schedule)/sizeof(int); - break; + return get_options()->TestingBridgeDownloadSchedule; default: tor_assert(0); } @@ -3789,8 +3392,7 @@ time_t download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { - const int *schedule; - size_t schedule_len; + const smartlist_t *schedule; int increment; tor_assert(dls); if (status_code != 503 || server) { @@ -3798,14 +3400,14 @@ download_status_increment_failure(download_status_t *dls, int status_code, ++dls->n_download_failures; } - find_dl_schedule_and_len(dls, server, &schedule, &schedule_len); + schedule = find_dl_schedule_and_len(dls, server); - if (dls->n_download_failures < schedule_len) - increment = schedule[dls->n_download_failures]; + if (dls->n_download_failures < smartlist_len(schedule)) + increment = *(int *)smartlist_get(schedule, dls->n_download_failures); else if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD) increment = INT_MAX; else - increment = schedule[schedule_len-1]; + increment = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); if (increment < INT_MAX) dls->next_attempt_at = now+increment; @@ -3838,14 +3440,11 @@ download_status_increment_failure(download_status_t *dls, int status_code, void download_status_reset(download_status_t *dls) { - const int *schedule; - size_t schedule_len; - - find_dl_schedule_and_len(dls, get_options()->DirPort_set, - &schedule, &schedule_len); + const smartlist_t *schedule = find_dl_schedule_and_len( + dls, get_options()->DirPort_set); dls->n_download_failures = 0; - dls->next_attempt_at = time(NULL) + schedule[0]; + dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); } /** Return the number of failures on <b>dls</b> since the last success (if @@ -3890,7 +3489,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } else { dls = router_get_dl_status_by_descriptor_digest(digest); } - if (!dls || dls->n_download_failures >= MAX_ROUTERDESC_DOWNLOAD_FAILURES) + if (!dls || dls->n_download_failures >= + get_options()->TestingDescriptorMaxDownloadTries) continue; download_status_increment_failure(dls, status_code, cp, server, now); } SMARTLIST_FOREACH_END(cp); @@ -3921,7 +3521,8 @@ dir_microdesc_download_failed(smartlist_t *failed, if (!rs) continue; dls = &rs->dl_status; - if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES) + if (dls->n_download_failures >= + get_options()->TestingMicrodescMaxDownloadTries) continue; { char buf[BASE64_DIGEST256_LEN+1]; diff --git a/src/or/directory.h b/src/or/directory.h index 41f18a172..bc200797d 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -30,7 +30,7 @@ typedef enum { DIRIND_ONEHOP=0, /** Connect over a multi-hop anonymizing Tor circuit */ DIRIND_ANONYMOUS=1, - /** Conncet to the DirPort directly */ + /** Connect to the DirPort directly */ DIRIND_DIRECT_CONN, /** Connect over a multi-hop anonymizing Tor circuit to our dirport */ DIRIND_ANON_DIRPORT, @@ -63,7 +63,7 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, +void directory_initiate_command(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -118,5 +118,10 @@ download_status_mark_impossible(download_status_t *dl) int download_status_get_n_failures(const download_status_t *dls); +#ifdef TOR_UNIT_TESTS +/* Used only by directory.c and test_dir.c */ +STATIC int parse_http_url(const char *headers, char **url); +#endif + #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 3e46153a5..aedd09252 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -26,6 +26,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "routerset.h" /** * \file dirserv.c @@ -41,31 +42,10 @@ * directory authorities. */ #define MAX_UNTRUSTED_NETWORKSTATUSES 16 -/** If a v1 directory is older than this, discard it. */ -#define MAX_V1_DIRECTORY_AGE (30*24*60*60) -/** If a v1 running-routers is older than this, discard it. */ -#define MAX_V1_RR_AGE (7*24*60*60) - extern time_t time_of_process_start; /* from main.c */ extern long stats_n_seconds_working; /* from main.c */ -/** Do we need to regenerate the v1 directory when someone asks for it? */ -static time_t the_directory_is_dirty = 1; -/** Do we need to regenerate the v1 runningrouters document when somebody - * asks for it? */ -static time_t runningrouters_is_dirty = 1; -/** Do we need to regenerate our v2 networkstatus document when somebody asks - * for it? */ -static time_t the_v2_networkstatus_is_dirty = 1; - -/** Most recently generated encoded signed v1 directory. (v1 auth dirservers - * only.) */ -static cached_dir_t *the_directory = NULL; - -/** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters; - /** Total number of routers with measured bandwidth; this is set by * dirserv_count_measured_bws() before the loop in * dirserv_generate_networkstatus_vote_obj() and checked by @@ -74,14 +54,12 @@ static cached_dir_t the_runningrouters; static int routers_with_measured_bw = 0; static void directory_remove_invalid(void); -static cached_dir_t *dirserv_regenerate_directory(void); static char *format_versions_list(config_line_t *ln); struct authdir_config_t; static int add_fingerprint_to_dir(const char *nickname, const char *fp, struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log); @@ -329,7 +307,6 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) } return dirserv_get_status_impl(d, router->nickname, - router->address, router->addr, router->or_port, router->platform, router->contact_info, msg, 1); @@ -343,7 +320,6 @@ dirserv_would_reject_router(const routerstatus_t *rs) uint32_t res; res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, - "", /* address is only used in logs */ rs->addr, rs->or_port, NULL, NULL, NULL, 0); @@ -382,7 +358,6 @@ dirserv_get_name_status(const char *id_digest, const char *nickname) */ static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log) @@ -399,13 +374,15 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Versions before Tor 0.2.2.35 have known security issues that - * make them unsuitable for the current network. */ - if (platform && !tor_version_as_new_as(platform,"0.2.2.35")) { + /* Versions before Tor 0.2.3.16-alpha are too old to support, and are + * missing some important security fixes too. Disable them. */ + if (platform && !tor_version_as_new_as(platform,"0.2.3.16-alpha")) { if (msg) *msg = "Tor version is insecure or unsupported. Please upgrade!"; return FP_REJECT; - } else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { + } +#if 0 + else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { /* Versions from 0.2.3-alpha...0.2.3.9-alpha have known security * issues that make them unusable for the current network */ if (!tor_version_as_new_as(platform, "0.2.3.10-alpha")) { @@ -414,6 +391,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, return FP_REJECT; } } +#endif result = dirserv_get_name_status(id_digest, nickname); if (result & FP_NAMED) { @@ -454,14 +432,14 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad directory because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADDIR; } if (authdir_policy_badexit_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADEXIT; } @@ -469,7 +447,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_permits_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); if (msg) *msg = "Authdir is rejecting routers in this range."; return FP_REJECT; @@ -477,7 +455,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_valid_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_INVALID; } if (reject_unlisted) { @@ -526,19 +504,15 @@ dirserv_free_fingerprint_list(void) static int dirserv_router_has_valid_address(routerinfo_t *ri) { - struct in_addr iaddr; + tor_addr_t addr; if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ - if (!tor_inet_aton(ri->address, &iaddr)) { - log_info(LD_DIRSERV,"Router %s published non-IP address '%s'. Refusing.", - router_describe(ri), - ri->address); - return -1; - } - if (is_internal_IP(ntohl(iaddr.s_addr), 0)) { + tor_addr_from_ipv4h(&addr, ri->addr); + + if (tor_addr_is_internal(&addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address '%s'. Refusing.", - router_describe(ri), ri->address); + "Router %s published internal IP address. Refusing.", + router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } return 0; @@ -590,12 +564,10 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, } if (dirserv_router_has_valid_address(ri) < 0) { log_fn(severity, LD_DIRSERV, - "Router %s has invalid address '%s'. " - "Not adding (%s).", + "Router %s has invalid address. Not adding (%s).", router_describe(ri), - ri->address, esc_router_info(ri)); - *msg = "Rejected: Address is not an IP, or IP is a private address."; + *msg = "Rejected: Address is a private address."; return -1; } @@ -839,7 +811,6 @@ dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) static void directory_remove_invalid(void) { - int changed = 0; routerlist_t *rl = router_get_routerlist(); smartlist_t *nodes = smartlist_new(); smartlist_add_all(nodes, nodelist_get_list()); @@ -857,7 +828,6 @@ directory_remove_invalid(void) log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); routerlist_remove(rl, ent, 0, time(NULL)); - changed = 1; continue; } #if 0 @@ -866,72 +836,35 @@ directory_remove_invalid(void) "Router %s is now %snamed.", description, (r&FP_NAMED)?"":"un"); ent->is_named = (r&FP_NAMED)?1:0; - changed = 1; } if (bool_neq((r & FP_UNNAMED), ent->auth_says_is_unnamed)) { log_info(LD_DIRSERV, "Router '%s' is now %snamed. (FP_UNNAMED)", description, (r&FP_NAMED)?"":"un"); ent->is_named = (r&FP_NUNAMED)?0:1; - changed = 1; } #endif if (bool_neq((r & FP_INVALID), !node->is_valid)) { log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description, (r&FP_INVALID) ? "in" : ""); node->is_valid = (r&FP_INVALID)?0:1; - changed = 1; } if (bool_neq((r & FP_BADDIR), node->is_bad_directory)) { log_info(LD_DIRSERV, "Router '%s' is now a %s directory", description, (r & FP_BADDIR) ? "bad" : "good"); node->is_bad_directory = (r&FP_BADDIR) ? 1: 0; - changed = 1; } if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) { log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description, (r & FP_BADEXIT) ? "bad" : "good"); node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0; - changed = 1; } } SMARTLIST_FOREACH_END(node); - if (changed) - directory_set_dirty(); routerlist_assert_ok(rl); smartlist_free(nodes); } -/** Mark the directory as <b>dirty</b> -- when we're next asked for a - * directory, we will rebuild it instead of reusing the most recently - * generated one. - */ -void -directory_set_dirty(void) -{ - time_t now = time(NULL); - int set_v1_dirty=0; - - /* Regenerate stubs only every 8 hours. - * XXXX It would be nice to generate less often, but these are just - * stubs: it doesn't matter. */ -#define STUB_REGENERATE_INTERVAL (8*60*60) - if (!the_directory || !the_runningrouters.dir) - set_v1_dirty = 1; - else if (the_directory->published < now - STUB_REGENERATE_INTERVAL || - the_runningrouters.published < now - STUB_REGENERATE_INTERVAL) - set_v1_dirty = 1; - - if (set_v1_dirty) { - if (!the_directory_is_dirty) - the_directory_is_dirty = now; - if (!runningrouters_is_dirty) - runningrouters_is_dirty = now; - } - if (!the_v2_networkstatus_is_dirty) - the_v2_networkstatus_is_dirty = now; -} - /** * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed @@ -1271,14 +1204,6 @@ directory_fetches_dir_info_later(const or_options_t *options) return options->UseBridges != 0; } -/** Return 1 if we want to cache v2 dir info (each status file). - */ -int -directory_caches_v2_dir_info(const or_options_t *options) -{ - return options->DirPort_set; -} - /** Return true iff we want to fetch and keep certificates for authorities * that we don't acknowledge as aurthorities ourself. */ @@ -1313,15 +1238,6 @@ directory_permits_begindir_requests(const or_options_t *options) return options->BridgeRelay != 0 || options->DirPort_set; } -/** Return 1 if we want to allow controllers to ask us directory - * requests via the controller interface, which doesn't require - * having any separate port open. */ -int -directory_permits_controller_requests(const or_options_t *options) -{ - return options->DirPort_set; -} - /** Return 1 if we have no need to fetch new descriptors. This generally * happens when we're not a dir cache and we haven't built any circuits * lately. @@ -1337,55 +1253,10 @@ directory_too_idle_to_fetch_descriptors(const or_options_t *options, /********************************************************************/ -/* Used only by non-v1-auth dirservers: The v1 directory and - * runningrouters we'll serve when requested. */ - -/** The v1 directory we'll serve (as a cache or as an authority) if - * requested. */ -static cached_dir_t *cached_directory = NULL; -/** The v1 runningrouters document we'll serve (as a cache or as an authority) - * if requested. */ -static cached_dir_t cached_runningrouters; - -/** Used for other dirservers' v2 network statuses. Map from hexdigest to - * cached_dir_t. */ -static digestmap_t *cached_v2_networkstatus = NULL; - /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're * currently serving. */ static strmap_t *cached_consensuses = NULL; -/** Possibly replace the contents of <b>d</b> with the value of - * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than - * the last value, or too far in the future. - * - * Does not copy <b>directory</b>; frees it if it isn't used. - */ -static void -set_cached_dir(cached_dir_t *d, char *directory, time_t when) -{ - time_t now = time(NULL); - if (when<=d->published) { - log_info(LD_DIRSERV, "Ignoring old directory; not caching."); - tor_free(directory); - } else if (when>=now+ROUTER_MAX_AGE_TO_PUBLISH) { - log_info(LD_DIRSERV, "Ignoring future directory; not caching."); - tor_free(directory); - } else { - /* if (when>d->published && when<now+ROUTER_MAX_AGE) */ - log_debug(LD_DIRSERV, "Caching directory."); - tor_free(d->dir); - d->dir = directory; - d->dir_len = strlen(directory); - tor_free(d->dir_z); - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { - log_warn(LD_BUG,"Error compressing cached directory"); - } - d->published = when; - } -} - /** Decrement the reference count on <b>d</b>, and free it if it no longer has * any references. */ void @@ -1435,86 +1306,6 @@ free_cached_dir_(void *_d) cached_dir_decref(d); } -/** If we have no cached v1 directory, or it is older than <b>published</b>, - * then replace it with <b>directory</b>, published at <b>published</b>. - * - * If <b>published</b> is too old, do nothing. - * - * If <b>is_running_routers</b>, this is really a v1 running_routers - * document rather than a v1 directory. - */ -static void -dirserv_set_cached_directory(const char *directory, time_t published) -{ - - cached_dir_decref(cached_directory); - cached_directory = new_cached_dir(tor_strdup(directory), published); -} - -/** If <b>networkstatus</b> is non-NULL, we've just received a v2 - * network-status for an authoritative directory with identity digest - * <b>identity</b> published at <b>published</b> -- store it so we can - * serve it to others. - * - * If <b>networkstatus</b> is NULL, remove the entry with the given - * identity fingerprint from the v2 cache. - */ -void -dirserv_set_cached_networkstatus_v2(const char *networkstatus, - const char *identity, - time_t published) -{ - cached_dir_t *d, *old_d; - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - old_d = digestmap_get(cached_v2_networkstatus, identity); - if (!old_d && !networkstatus) - return; - - if (networkstatus) { - if (!old_d || published > old_d->published) { - d = new_cached_dir(tor_strdup(networkstatus), published); - digestmap_set(cached_v2_networkstatus, identity, d); - if (old_d) - cached_dir_decref(old_d); - } - } else { - if (old_d) { - digestmap_remove(cached_v2_networkstatus, identity); - cached_dir_decref(old_d); - } - } - - /* Now purge old entries. */ - - if (digestmap_size(cached_v2_networkstatus) > - get_n_authorities(V2_DIRINFO) + MAX_UNTRUSTED_NETWORKSTATUSES) { - /* We need to remove the oldest untrusted networkstatus. */ - const char *oldest = NULL; - time_t oldest_published = TIME_MAX; - digestmap_iter_t *iter; - - for (iter = digestmap_iter_init(cached_v2_networkstatus); - !digestmap_iter_done(iter); - iter = digestmap_iter_next(cached_v2_networkstatus, iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - d = val; - if (d->published < oldest_published && - !router_digest_is_trusted_dir(ident)) { - oldest = ident; - oldest_published = d->published; - } - } - tor_assert(oldest); - d = digestmap_remove(cached_v2_networkstatus, oldest); - if (d) - cached_dir_decref(d); - } -} - /** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that * we're serving with <b>networkstatus</b>, published at <b>published</b>. No * validation is performed. */ @@ -1537,186 +1328,6 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, cached_dir_decref(old_networkstatus); } -/** Remove any v2 networkstatus from the directory cache that was published - * before <b>cutoff</b>. */ -void -dirserv_clear_old_networkstatuses(time_t cutoff) -{ - if (!cached_v2_networkstatus) - return; - - DIGESTMAP_FOREACH_MODIFY(cached_v2_networkstatus, id, cached_dir_t *, dir) { - if (dir->published < cutoff) { - char *fname; - fname = networkstatus_get_cache_filename(id); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old untrusted networkstatus in %s", - fname); - unlink(fname); - } - tor_free(fname); - cached_dir_decref(dir); - MAP_DEL_CURRENT(id); - } - } DIGESTMAP_FOREACH_END -} - -/** Remove any v1 info from the directory cache that was published - * too long ago. */ -void -dirserv_clear_old_v1_info(time_t now) -{ - if (cached_directory && - cached_directory->published < (now - MAX_V1_DIRECTORY_AGE)) { - cached_dir_decref(cached_directory); - cached_directory = NULL; - } - if (cached_runningrouters.published < (now - MAX_V1_RR_AGE)) { - clear_cached_dir(&cached_runningrouters); - } -} - -/** Helper: If we're an authority for the right directory version (v1 or v2) - * (based on <b>auth_type</b>), try to regenerate - * auth_src as appropriate and return it, falling back to cache_src on - * failure. If we're a cache, simply return cache_src. - */ -static cached_dir_t * -dirserv_pick_cached_dir_obj(cached_dir_t *cache_src, - cached_dir_t *auth_src, - time_t dirty, cached_dir_t *(*regenerate)(void), - const char *name, - dirinfo_type_t auth_type) -{ - const or_options_t *options = get_options(); - int authority = (auth_type == V1_DIRINFO && authdir_mode_v1(options)) || - (auth_type == V2_DIRINFO && authdir_mode_v2(options)); - - if (!authority || authdir_mode_bridge(options)) { - return cache_src; - } else { - /* We're authoritative. */ - if (regenerate != NULL) { - if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) { - if (!(auth_src = regenerate())) { - log_err(LD_BUG, "Couldn't generate %s?", name); - exit(1); - } - } else { - log_info(LD_DIRSERV, "The %s is still clean; reusing.", name); - } - } - return auth_src ? auth_src : cache_src; - } -} - -/** Return the most recently generated encoded signed v1 directory, - * generating a new one as necessary. If not a v1 authoritative directory - * may return NULL if no directory is yet cached. */ -cached_dir_t * -dirserv_get_directory(void) -{ - return dirserv_pick_cached_dir_obj(cached_directory, the_directory, - the_directory_is_dirty, - dirserv_regenerate_directory, - "v1 server directory", V1_DIRINFO); -} - -/** Only called by v1 auth dirservers. - * Generate a fresh v1 directory; set the_directory and return a pointer - * to the new value. - */ -static cached_dir_t * -dirserv_regenerate_directory(void) -{ - /* XXXX 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *new_directory=NULL; - - if (dirserv_dump_directory_to_string(&new_directory, - get_server_identity_key())) { - log_warn(LD_BUG, "Error creating directory."); - tor_free(new_directory); - return NULL; - } - cached_dir_decref(the_directory); - the_directory = new_cached_dir(new_directory, time(NULL)); - log_info(LD_DIRSERV,"New directory (size %d) has been built.", - (int)the_directory->dir_len); - log_debug(LD_DIRSERV,"New directory (size %d):\n%s", - (int)the_directory->dir_len, the_directory->dir); - - the_directory_is_dirty = 0; - - /* Save the directory to disk so we re-load it quickly on startup. - */ - dirserv_set_cached_directory(the_directory->dir, time(NULL)); - - return the_directory; -} - -/** Only called by v1 auth dirservers. - * Replace the current running-routers list with a newly generated one. */ -static cached_dir_t * -generate_runningrouters(void) -{ - char *s=NULL; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - size_t len; - crypto_pk_t *private_key = get_server_identity_key(); - char *identity_pkey; /* Identity key, DER64-encoded. */ - size_t identity_pkey_len; - - if (crypto_pk_write_public_key_to_string(private_key,&identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"write identity_pkey to string failed!"); - goto err; - } - format_iso_time(published, time(NULL)); - - len = 2048; - s = tor_malloc_zero(len); - tor_snprintf(s, len, - "network-status\n" - "published %s\n" - "router-status %s\n" - "dir-signing-key\n%s" - "directory-signature %s\n", - published, "", identity_pkey, - get_options()->Nickname); - tor_free(identity_pkey); - if (router_get_runningrouters_hash(s,digest)) { - log_warn(LD_BUG,"couldn't compute digest"); - goto err; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, - private_key)<0) - goto err; - - set_cached_dir(&the_runningrouters, s, time(NULL)); - runningrouters_is_dirty = 0; - - return &the_runningrouters; - err: - tor_free(s); - return NULL; -} - -/** Set *<b>rr</b> to the most recently generated encoded signed - * running-routers list, generating a new one as necessary. Return the - * size of the directory on success, and 0 on failure. */ -cached_dir_t * -dirserv_get_runningrouters(void) -{ - return dirserv_pick_cached_dir_obj( - &cached_runningrouters, &the_runningrouters, - runningrouters_is_dirty, - generate_runningrouters, - "v1 network status list", V1_DIRINFO); -} - /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * @@ -1727,19 +1338,6 @@ dirserv_get_consensus(const char *flavor_name) return strmap_get(cached_consensuses, flavor_name); } -/** For authoritative directories: the current (v2) network status. */ -static cached_dir_t *the_v2_networkstatus = NULL; - -/** Return true iff our opinion of the routers has been stale for long - * enough that we should generate a new v2 network status doc. */ -static int -should_generate_v2_networkstatus(void) -{ - return authdir_mode_v2(get_options()) && - the_v2_networkstatus_is_dirty && - the_v2_networkstatus_is_dirty + DIR_REGEN_SLACK_TIME < time(NULL); -} - /** If a router's uptime is at least this value, then it is always * considered stable, regardless of the rest of the network. This * way we resist attacks where an attacker doubles the size of the @@ -1907,7 +1505,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth, - * guard_bandwidh_including_exits, guard_bandwidth_excluding_exits, + * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ static void @@ -1956,6 +1554,10 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, /* Now, fill in the arrays. */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + if (options->BridgeAuthoritativeDir && + node->ri && + node->ri->purpose != ROUTER_PURPOSE_BRIDGE) + continue; if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { routerinfo_t *ri = node->ri; @@ -2070,6 +1672,21 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, tor_free(wfus); } +/* Use dirserv_compute_performance_thresholds() to compute the thresholds + * for the status flags, specifically for bridges. + * + * This is only called by a Bridge Authority from + * networkstatus_getinfo_by_purpose(). + */ +void +dirserv_compute_bridge_flag_thresholds(routerlist_t *rl) +{ + + digestmap_t *omit_as_sybil = digestmap_new(); + dirserv_compute_performance_thresholds(rl, omit_as_sybil); + digestmap_free(omit_as_sybil, NULL); +} + /** Measured bandwidth cache entry */ typedef struct mbw_cache_entry_s { long mbw_kb; @@ -2082,7 +1699,7 @@ static digestmap_t *mbw_cache = NULL; /** Store a measured bandwidth cache entry when reading the measured * bandwidths file. */ -void +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of) { @@ -2112,7 +1729,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -void +STATIC void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -2123,7 +1740,7 @@ dirserv_clear_measured_bw_cache(void) } /** Scan the measured bandwidth cache and remove expired entries */ -void +STATIC void dirserv_expire_measured_bw_cache(time_t now) { @@ -2145,7 +1762,7 @@ dirserv_expire_measured_bw_cache(time_t now) } /** Get the current size of the measured bandwidth cache */ -int +STATIC int dirserv_get_measured_bw_cache_size(void) { if (mbw_cache) return digestmap_size(mbw_cache); @@ -2155,7 +1772,7 @@ dirserv_get_measured_bw_cache_size(void) /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -int +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -2176,7 +1793,7 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -int +STATIC int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); @@ -2391,7 +2008,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", rs->is_unnamed?" Unnamed":"", - rs->is_v2_dir?" V2Dir":"", + (rs->dir_port!=0)?" V2Dir":"", rs->is_valid?" Valid":""); /* length of "opt v \n" */ @@ -2705,12 +2322,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, } else { rs->is_possible_guard = 0; } + if (options->TestingTorNetwork && + routerset_contains_routerstatus(options->TestingDirAuthVoteGuard, + rs, 0)) { + rs->is_possible_guard = 1; + } rs->is_bad_directory = listbaddirs && node->is_bad_directory; rs->is_bad_exit = listbadexits && node->is_bad_exit; node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); rs->is_hs_dir = vote_on_hsdirs && node->is_hs_dir; - rs->is_v2_dir = ri->dir_port != 0; if (!strcasecmp(ri->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = rs->is_unnamed = 0; @@ -2741,7 +2362,7 @@ static void clear_status_flags_on_sybil(routerstatus_t *rs) { rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_v2_dir = + rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = rs->is_bad_directory = 0; /* FFFF we might want some mechanism to check later on if we @@ -2754,7 +2375,7 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * into a measured_bw_line_t output structure. Returns -1 on failure * or 0 on success. */ -int +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) { char *line = tor_strdup(orig_line); @@ -2835,7 +2456,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) * of bandwidth statuses. Returns true if a line is found, * false otherwise. */ -int +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses) { @@ -2886,7 +2507,7 @@ dirserv_read_measured_bandwidths(const char *from_file, } line[strlen(line)-1] = '\0'; - file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); @@ -2957,14 +2578,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, tor_assert(private_key); tor_assert(cert); - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { log_err(LD_BUG, "Error computing signing key digest"); return NULL; @@ -2973,6 +2586,14 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, log_err(LD_BUG, "Error computing identity key digest"); return NULL; } + if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { + log_warn(LD_NET, "Couldn't resolve my hostname"); + return NULL; + } + if (!hostname || !strchr(hostname, '.')) { + tor_free(hostname); + hostname = tor_dup_ip(addr); + } if (options->VersioningAuthoritativeDir) { client_versions = format_versions_list(options->RecommendedClientVersions); @@ -3093,7 +2714,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, else last_consensus_interval = options->TestingV3AuthInitialVotingInterval; v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval); + dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, + options->TestingV3AuthVotingStartOffset); format_iso_time(tbuf, v3_out->valid_after); log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " "consensus_set=%d, last_interval=%d", @@ -3164,270 +2786,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, return v3_out; } -/** For v2 authoritative directories only: Replace the contents of - * <b>the_v2_networkstatus</b> with a newly generated network status - * object. */ -cached_dir_t * -generate_v2_networkstatus_opinion(void) -{ - cached_dir_t *r = NULL; - size_t identity_pkey_len; - char *status = NULL, *client_versions = NULL, *server_versions = NULL, - *identity_pkey = NULL, *hostname = NULL; - const or_options_t *options = get_options(); - char fingerprint[FINGERPRINT_LEN+1]; - char published[ISO_TIME_LEN+1]; - char digest[DIGEST_LEN]; - uint32_t addr; - crypto_pk_t *private_key; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - int naming = options->NamingAuthoritativeDir; - int versioning = options->VersioningAuthoritativeDir; - int listbaddirs = options->AuthDirListBadDirs; - int listbadexits = options->AuthDirListBadExits; - int vote_on_hsdirs = options->VoteOnHidServDirectoriesV2; - const char *contact; - char *version_lines = NULL; - smartlist_t *routers = NULL; - digestmap_t *omit_as_sybil = NULL; - smartlist_t *chunks = NULL; - - private_key = get_server_identity_key(); - - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - goto done; - } - if (!hostname) - hostname = tor_dup_ip(addr); - - format_iso_time(published, now); - - client_versions = format_versions_list(options->RecommendedClientVersions); - server_versions = format_versions_list(options->RecommendedServerVersions); - - if (crypto_pk_write_public_key_to_string(private_key, &identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"Writing public key to string failed."); - goto done; - } - - if (crypto_pk_get_fingerprint(private_key, fingerprint, 0)<0) { - log_err(LD_BUG, "Error computing fingerprint"); - goto done; - } - - contact = options->ContactInfo; - if (!contact) - contact = "(none)"; - - if (versioning) { - tor_asprintf(&version_lines, - "client-versions %s\nserver-versions %s\n", - client_versions, server_versions); - } else { - version_lines = tor_strdup(""); - } - - chunks = smartlist_new(); - smartlist_add_asprintf(chunks, - "network-status-version 2\n" - "dir-source %s %s %d\n" - "fingerprint %s\n" - "contact %s\n" - "published %s\n" - "dir-options%s%s%s%s\n" - "%s" /* client version line, server version line. */ - "dir-signing-key\n%s", - hostname, fmt_addr32(addr), - (int)router_get_advertised_dir_port(options, 0), - fingerprint, - contact, - published, - naming ? " Names" : "", - listbaddirs ? " BadDirectories" : "", - listbadexits ? " BadExits" : "", - versioning ? " Versions" : "", - version_lines, - identity_pkey); - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - dirserv_compute_performance_thresholds(rl, omit_as_sybil); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t rs; - char *version = version_from_platform(ri->platform); - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) { - tor_free(version); - continue; - } - set_routerstatus_from_routerinfo(&rs, node, ri, now, - naming, listbadexits, listbaddirs, - vote_on_hsdirs); - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(&rs); - - { - char *rsf = routerstatus_format_entry(&rs, version, NS_V2, NULL); - if (rsf) - smartlist_add(chunks, rsf); - } - tor_free(version); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_add_asprintf(chunks, "directory-signature %s\n", - options->Nickname); - - crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1); - - note_crypto_pk_op(SIGN_DIR); - { - char *sig; - if (!(sig = router_get_dirobj_signature(digest,DIGEST_LEN, - private_key))) { - log_warn(LD_BUG, "Unable to sign router status."); - goto done; - } - smartlist_add(chunks, sig); - } - - status = smartlist_join_strings(chunks, "", 0, NULL); - - { - networkstatus_v2_t *ns; - if (!(ns = networkstatus_v2_parse_from_string(status))) { - log_err(LD_BUG,"Generated a networkstatus we couldn't parse."); - goto done; - } - networkstatus_v2_free(ns); - } - - { - cached_dir_t **ns_ptr = &the_v2_networkstatus; - if (*ns_ptr) - cached_dir_decref(*ns_ptr); - *ns_ptr = new_cached_dir(status, now); - status = NULL; /* So it doesn't get double-freed. */ - the_v2_networkstatus_is_dirty = 0; - router_set_networkstatus_v2((*ns_ptr)->dir, now, NS_GENERATED, NULL); - r = *ns_ptr; - } - - done: - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } - tor_free(client_versions); - tor_free(server_versions); - tor_free(version_lines); - tor_free(status); - tor_free(hostname); - tor_free(identity_pkey); - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - return r; -} - -/** Given the portion of a networkstatus request URL after "tor/status/" in - * <b>key</b>, append to <b>result</b> the digests of the identity keys of the - * networkstatus objects that the client has requested. */ -void -dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key) -{ - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - if (should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - - if (!strcmp(key,"authority")) { - if (authdir_mode_v2(get_options())) { - const routerinfo_t *me = router_get_my_routerinfo(); - if (me) - smartlist_add(result, - tor_memdup(me->cache_info.identity_digest, DIGEST_LEN)); - } - } else if (!strcmp(key, "all")) { - if (digestmap_size(cached_v2_networkstatus)) { - digestmap_iter_t *iter; - iter = digestmap_iter_init(cached_v2_networkstatus); - while (!digestmap_iter_done(iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - smartlist_add(result, tor_memdup(ident, DIGEST_LEN)); - iter = digestmap_iter_next(cached_v2_networkstatus, iter); - } - } else { - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - if (ds->type & V2_DIRINFO) - smartlist_add(result, tor_memdup(ds->digest, DIGEST_LEN))); - } - smartlist_sort_digests(result); - if (smartlist_len(result) == 0) - log_info(LD_DIRSERV, - "Client requested 'all' network status objects; we have none."); - } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, - DSR_HEX|DSR_SORT_UNIQ); - } -} - -/** Look for a network status object as specified by <b>key</b>, which should - * be either "authority" (to find a network status generated by us), a hex - * identity digest (to find a network status generated by given directory), or - * "all" (to return all the v2 network status objects we have). - */ -void -dirserv_get_networkstatus_v2(smartlist_t *result, - const char *key) -{ - cached_dir_t *cached; - smartlist_t *fingerprints = smartlist_new(); - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - dirserv_get_networkstatus_v2_fingerprints(fingerprints, key); - SMARTLIST_FOREACH_BEGIN(fingerprints, const char *, fp) { - if (router_digest_is_me(fp) && should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - cached = digestmap_get(cached_v2_networkstatus, fp); - if (cached) { - smartlist_add(result, cached); - } else { - char hexbuf[HEX_DIGEST_LEN+1]; - base16_encode(hexbuf, sizeof(hexbuf), fp, DIGEST_LEN); - log_info(LD_DIRSERV, "Don't know about any network status with " - "fingerprint '%s'", hexbuf); - } - } SMARTLIST_FOREACH_END(fp); - SMARTLIST_FOREACH(fingerprints, char *, cp, tor_free(cp)); - smartlist_free(fingerprints); -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other @@ -3661,7 +3019,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); + router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, router->cache_info.identity_digest); @@ -3727,15 +3085,12 @@ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_consensuses) + if (tor_digest_is_zero(fp) && cached_consensuses) { d = strmap_get(cached_consensuses, "ns"); - else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && (d = strmap_get(cached_consensuses, fp))) { /* this here interface is a nasty hack XXXX024 */; - } else if (router_digest_is_me(fp) && the_v2_networkstatus) - d = the_v2_networkstatus; - else if (cached_v2_networkstatus) - d = digestmap_get(cached_v2_networkstatus, fp); + } return d; } @@ -3941,8 +3296,6 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) } body = signed_descriptor_get_body(sd); if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3959,6 +3312,11 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) if (!smartlist_len(conn->fingerprint_stack)) { /* We just wrote the last one; finish up. */ + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -3984,8 +3342,6 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) if (!md || !md->body) continue; if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = !smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); if (last) { @@ -3997,6 +3353,11 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) } } if (!smartlist_len(conn->fingerprint_stack)) { + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -4128,14 +3489,6 @@ dirserv_free_all(void) { dirserv_free_fingerprint_list(); - cached_dir_decref(the_directory); - clear_cached_dir(&the_runningrouters); - cached_dir_decref(the_v2_networkstatus); - cached_dir_decref(cached_directory); - clear_cached_dir(&cached_runningrouters); - - digestmap_free(cached_v2_networkstatus, free_cached_dir_); - cached_v2_networkstatus = NULL; strmap_free(cached_consensuses, free_cached_dir_); cached_consensuses = NULL; diff --git a/src/or/dirserv.h b/src/or/dirserv.h index f9d36d760..858e6e3a0 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -12,6 +12,8 @@ #ifndef TOR_DIRSERV_H #define TOR_DIRSERV_H +#include "testsupport.h" + /** What fraction (1 over this number) of the relay ID space do we * (as a directory authority) launch connections to at each reachability * test? */ @@ -49,34 +51,23 @@ int list_server_status_v1(smartlist_t *routers, char **router_status_out, int dirserv_dump_directory_to_string(char **dir_out, crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); +void dirserv_compute_bridge_flag_thresholds(routerlist_t *rl); int directory_fetches_from_authorities(const or_options_t *options); int directory_fetches_dir_info_early(const or_options_t *options); int directory_fetches_dir_info_later(const or_options_t *options); -int directory_caches_v2_dir_info(const or_options_t *options); int directory_caches_unknown_auth_certs(const or_options_t *options); int directory_caches_dir_info(const or_options_t *options); int directory_permits_begindir_requests(const or_options_t *options); -int directory_permits_controller_requests(const or_options_t *options); int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now); -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); cached_dir_t *dirserv_get_consensus(const char *flavor_name); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); void dirserv_set_cached_consensus_networkstatus(const char *consensus, const char *flavor_name, const digests_t *digests, time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, const char **msg, int for_unencrypted_conn, @@ -119,20 +110,20 @@ cached_dir_t *new_cached_dir(char *s, time_t published); /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ -int measured_bw_line_parse(measured_bw_line_t *out, const char *line); +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line); -int measured_bw_line_apply(measured_bw_line_t *parsed_line, +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses); -void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of); -void dirserv_clear_measured_bw_cache(void); -void dirserv_expire_measured_bw_cache(time_t now); -int dirserv_get_measured_bw_cache_size(void); -int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, - time_t *as_of_out); -int dirserv_has_measured_bw(const char *node_id); -cached_dir_t *generate_v2_networkstatus_opinion(void); +STATIC void dirserv_clear_measured_bw_cache(void); +STATIC void dirserv_expire_measured_bw_cache(time_t now); +STATIC int dirserv_get_measured_bw_cache_size(void); +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, + long *bw_out, + time_t *as_of_out); +STATIC int dirserv_has_measured_bw(const char *node_id); #endif int dirserv_read_measured_bandwidths(const char *from_file, diff --git a/src/or/dirvote.c b/src/or/dirvote.c index c6d124490..c7be343ca 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -60,7 +60,7 @@ static char *make_consensus_method_list(int low, int high, const char *sep); /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ -char * +STATIC char * format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { @@ -587,7 +587,7 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) /** Helper: given a list of valid networkstatus_t, return a new string * containing the contents of the consensus network parameter set. */ -/* private */ char * +STATIC char * dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) { int i; @@ -2533,12 +2533,13 @@ dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) timing_out->dist_delay = options->V3AuthDistDelay; } -/** Return the start of the next interval of size <b>interval</b> (in seconds) - * after <b>now</b>. Midnight always starts a fresh interval, and if the last - * interval of a day would be truncated to less than half its size, it is - * rolled into the previous interval. */ +/** Return the start of the next interval of size <b>interval</b> (in + * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always + * starts a fresh interval, and if the last interval of a day would be + * truncated to less than half its size, it is rolled into the + * previous interval. */ time_t -dirvote_get_start_of_next_interval(time_t now, int interval) +dirvote_get_start_of_next_interval(time_t now, int interval, int offset) { struct tm tm; time_t midnight_today=0; @@ -2566,6 +2567,10 @@ dirvote_get_start_of_next_interval(time_t now, int interval) if (next + interval/2 > midnight_tomorrow) next = midnight_tomorrow; + next += offset; + if (next - interval > now) + next -= interval; + return next; } @@ -2629,8 +2634,10 @@ dirvote_recalculate_timing(const or_options_t *options, time_t now) vote_delay = dist_delay = interval / 4; start = voting_schedule.interval_starts = - dirvote_get_start_of_next_interval(now,interval); - end = dirvote_get_start_of_next_interval(start+1, interval); + dirvote_get_start_of_next_interval(now,interval, + options->TestingV3AuthVotingStartOffset); + end = dirvote_get_start_of_next_interval(start+1, interval, + options->TestingV3AuthVotingStartOffset); tor_assert(end > start); @@ -3136,7 +3143,7 @@ dirvote_compute_consensuses(void) }); votefile = get_datadir_fname("v3-status-votes"); - write_chunks_to_file(votefile, votestrings, 0); + write_chunks_to_file(votefile, votestrings, 0, 0); tor_free(votefile); SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); smartlist_free(votestrings); @@ -3581,6 +3588,12 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) tor_free(p6); } + if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) { + char idbuf[BASE64_DIGEST_LEN+1]; + digest_to_base64(idbuf, ri->cache_info.identity_digest); + smartlist_add_asprintf(chunks, "id rsa1024 %s\n", idbuf); + } + output = smartlist_join_strings(chunks, "", 0, NULL); { @@ -3650,7 +3663,8 @@ static const struct consensus_method_range_t { {MIN_METHOD_FOR_MICRODESC, MIN_METHOD_FOR_A_LINES - 1}, {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, - {MIN_METHOD_FOR_NTOR_KEY, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, + {MIN_METHOD_FOR_ID_HASH_IN_MD, MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; diff --git a/src/or/dirvote.h b/src/or/dirvote.h index b23645212..4c57e4366 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -12,15 +12,17 @@ #ifndef TOR_DIRVOTE_H #define TOR_DIRVOTE_H +#include "testsupport.h" + /** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 +#define MIN_VOTE_SECONDS 2 /** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 +#define MIN_DIST_SECONDS 2 /** Smallest allowable voting interval. */ #define MIN_VOTE_INTERVAL 300 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 17 +#define MAX_SUPPORTED_CONSENSUS_METHOD 18 /** Lowest consensus method that contains a 'directory-footer' marker */ #define MIN_METHOD_FOR_FOOTER 9 @@ -59,6 +61,10 @@ * Unmeasured=1 flag for unmeasured bandwidths */ #define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17 +/** Lowest consensus method where authorities may include an "id" line in + * microdescriptors. */ +#define MIN_METHOD_FOR_ID_HASH_IN_MD 18 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 @@ -86,7 +92,9 @@ authority_cert_t *authority_cert_dup(authority_cert_t *cert); /* vote scheduling */ void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); +time_t dirvote_get_start_of_next_interval(time_t now, + int interval, + int offset); void dirvote_recalculate_timing(const or_options_t *options, time_t now); void dirvote_act(const or_options_t *options, time_t now); @@ -134,9 +142,9 @@ document_signature_t *voter_get_sig_by_algorithm( digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_t *private_key, +STATIC char *format_networkstatus_vote(crypto_pk_t *private_key, networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes, int method, +STATIC char *dirvote_compute_params(smartlist_t *votes, int method, int total_authorities); #endif diff --git a/src/or/dns.c b/src/or/dns.c index fb1b10d82..a9c431865 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -24,6 +24,7 @@ #include "relay.h" #include "router.h" #include "ht.h" +#include "../common/sandbox.h" #ifdef HAVE_EVENT2_DNS_H #include <event2/event.h> #include <event2/dns.h> @@ -238,7 +239,7 @@ cached_resolves_eq(cached_resolve_t *a, cached_resolve_t *b) static INLINE unsigned int cached_resolve_hash(cached_resolve_t *a) { - return ht_string_hash(a->address); + return (unsigned) siphash24g((const uint8_t*)a->address, strlen(a->address)); } HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash, @@ -1444,13 +1445,14 @@ configure_nameservers(int force) const or_options_t *options; const char *conf_fname; struct stat st; - int r; + int r, flags; options = get_options(); conf_fname = options->ServerDNSResolvConfFile; #ifndef _WIN32 if (!conf_fname) conf_fname = "/etc/resolv.conf"; #endif + flags = DNS_OPTIONS_ALL; if (!the_evdns_base) { if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { @@ -1478,7 +1480,8 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { - if (stat(conf_fname, &st)) { + log_debug(LD_FS, "stat()ing %s", conf_fname); + if (stat(sandbox_intern_string(conf_fname), &st)) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); goto err; @@ -1492,9 +1495,17 @@ configure_nameservers(int force) evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); } +#if defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) + if (flags & DNS_OPTION_HOSTSFILE) { + flags ^= DNS_OPTION_HOSTSFILE; + log_debug(LD_FS, "Loading /etc/hosts"); + evdns_base_load_hosts(the_evdns_base, + sandbox_intern_string("/etc/hosts")); + } +#endif log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, - DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; @@ -2163,7 +2174,7 @@ static void assert_cache_ok_(void) { cached_resolve_t **resolve; - int bad_rep = _cache_map_HT_REP_IS_BAD(&cache_root); + int bad_rep = HT_REP_IS_BAD_(cache_map, &cache_root); if (bad_rep) { log_err(LD_BUG, "Bad rep type %d on dns cache hash table", bad_rep); tor_assert(!bad_rep); diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index ebff7b524..ecd45be77 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -35,7 +35,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) entry_connection_t *entry_conn; edge_connection_t *conn; int i = 0; - struct evdns_server_question *q = NULL; + struct evdns_server_question *q = NULL, *supported_q = NULL; struct sockaddr_storage addr; struct sockaddr *sa; int addrlen; @@ -87,31 +87,37 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) for (i = 0; i < req->nquestions; ++i) { if (req->questions[i]->dns_question_class != EVDNS_CLASS_INET) continue; + if (! q) + q = req->questions[i]; switch (req->questions[i]->type) { case EVDNS_TYPE_A: case EVDNS_TYPE_AAAA: case EVDNS_TYPE_PTR: - q = req->questions[i]; + /* We always pick the first one of these questions, if there is + one. */ + if (! supported_q) + supported_q = q; + break; default: break; } } + if (supported_q) + q = supported_q; if (!q) { log_info(LD_APP, "None of the questions we got were ones we're willing " "to support. Sending NOTIMPL."); evdns_server_request_respond(req, DNS_ERR_NOTIMPL); return; } - if (q->type != EVDNS_TYPE_A && q->type != EVDNS_TYPE_AAAA) { - tor_assert(q->type == EVDNS_TYPE_PTR); - } /* Make sure the name isn't too long: This should be impossible, I think. */ if (err == DNS_ERR_NONE && strlen(q->name) > MAX_SOCKS_ADDR_LEN-1) err = DNS_ERR_FORMAT; - if (err != DNS_ERR_NONE) { - /* We got an error? Then send back an answer immediately; we're done. */ + if (err != DNS_ERR_NONE || !supported_q) { + /* We got an error? There's no question we're willing to answer? Then + * send back an answer immediately; we're done. */ evdns_server_request_respond(req, err); return; } @@ -126,10 +132,23 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) TO_CONN(conn)->port = port; TO_CONN(conn)->address = tor_dup_addr(&tor_addr); - if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA) + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || + q->type == EVDNS_QTYPE_ALL) { entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE; - else + } else { + tor_assert(q->type == EVDNS_TYPE_PTR); entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR; + } + + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) { + entry_conn->ipv4_traffic_ok = 1; + entry_conn->ipv6_traffic_ok = 0; + entry_conn->prefer_ipv6_traffic = 0; + } else if (q->type == EVDNS_TYPE_AAAA) { + entry_conn->ipv4_traffic_ok = 0; + entry_conn->ipv6_traffic_ok = 1; + entry_conn->prefer_ipv6_traffic = 1; + } strlcpy(entry_conn->socks_request->address, q->name, sizeof(entry_conn->socks_request->address)); diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 59770fa65..957217ac6 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -13,6 +13,7 @@ **/ #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitstats.h" #include "config.h" @@ -54,6 +55,10 @@ typedef struct { /** When should we next try to fetch a descriptor for this bridge? */ download_status_t fetch_status; + + /** A smartlist of k=v values to be passed to the SOCKS proxy, if + transports are used for this bridge. */ + smartlist_t *socks_args; } bridge_info_t; /** A list of our chosen entry guards. */ @@ -65,7 +70,9 @@ static int entry_guards_dirty = 0; static void bridge_free(bridge_info_t *bridge); static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirtype); + dirinfo_type_t dirtype, + int *n_options_out); +static int num_bridges_usable(void); /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * @@ -359,7 +366,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, entry->can_retry = 1; } entry->is_dir_cache = node->rs && - node->rs->version_supports_microdesc_cache; + node->rs->version_supports_microdesc_cache; if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; return NULL; @@ -371,7 +378,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, } else { const routerstatus_t *rs; rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO, - PDS_PREFER_TUNNELED_DIR_CONNS_|PDS_FOR_GUARD); + PDS_FOR_GUARD); if (!rs) return NULL; node = node_get_by_id(rs->identity_digest); @@ -392,8 +399,8 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, node_describe(node)); strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname)); memcpy(entry->identity, node->identity, DIGEST_LEN); - entry->is_dir_cache = node_is_dir(node) && - node->rs && node->rs->version_supports_microdesc_cache; + entry->is_dir_cache = node_is_dir(node) && node->rs && + node->rs->version_supports_microdesc_cache; if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; @@ -594,6 +601,25 @@ remove_dead_entry_guards(time_t now) return changed ? 1 : 0; } +/** Remove all currently listed entry guards. So new ones will be chosen. */ +void +remove_all_entry_guards(void) +{ + char dbuf[HEX_DIGEST_LEN+1]; + + while (smartlist_len(entry_guards)) { + entry_guard_t *entry = smartlist_get(entry_guards, 0); + base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN); + log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.", + entry->nickname, dbuf); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); + entry_guard_free(entry); + smartlist_del(entry_guards, 0); + } + log_entry_guards(LOG_INFO); + entry_guards_changed(); +} + /** A new directory or router-status has arrived; update the down/listed * status of the entry guards. * @@ -958,7 +984,7 @@ node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) const node_t * choose_random_entry(cpath_build_state_t *state) { - return choose_random_entry_impl(state, 0, 0); + return choose_random_entry_impl(state, 0, 0, NULL); } /** Pick a live (up and listed) directory guard from entry_guards for @@ -966,13 +992,13 @@ choose_random_entry(cpath_build_state_t *state) const node_t * choose_random_dirguard(dirinfo_type_t type) { - return choose_random_entry_impl(NULL, 1, type); + return choose_random_entry_impl(NULL, 1, type, NULL); } /** Helper for choose_random{entry,dirguard}. */ static const node_t * choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirinfo_type) + dirinfo_type_t dirinfo_type, int *n_options_out) { const or_options_t *options = get_options(); smartlist_t *live_entry_guards = smartlist_new(); @@ -986,6 +1012,9 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, int need_descriptor = !for_directory; const int num_needed = decide_num_guards(options, for_directory); + if (n_options_out) + *n_options_out = 0; + if (chosen_exit) { nodelist_add_node_and_family(exit_family, chosen_exit); consider_exit_family = 1; @@ -1112,6 +1141,8 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, * *double*-weight our guard selection. */ node = smartlist_choose(live_entry_guards); } + if (n_options_out) + *n_options_out = smartlist_len(live_entry_guards); smartlist_free(live_entry_guards); smartlist_free(exit_family); return node; @@ -1595,6 +1626,11 @@ bridge_free(bridge_info_t *bridge) return; tor_free(bridge->transport_name); + if (bridge->socks_args) { + SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge->socks_args); + } + tor_free(bridge); } @@ -1628,7 +1664,8 @@ get_configured_bridge_by_orports_digest(const char *digest, /** If we have a bridge configured whose digest matches <b>digest</b>, or a * bridge with no known digest whose address matches <b>addr</b>:<b>/port</b>, - * return that bridge. Else return NULL. */ + * return that bridge. Else return NULL. If <b>digest</b> is NULL, check for + * address/port matches only. */ static bridge_info_t * get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, uint16_t port, @@ -1638,7 +1675,7 @@ get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { - if (tor_digest_is_zero(bridge->identity) && + if ((tor_digest_is_zero(bridge->identity) || digest == NULL) && !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && bridge->port == port) return bridge; @@ -1773,30 +1810,68 @@ bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, } SMARTLIST_FOREACH_END(bridge); } -/** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> - * is set, it tells us the identity key too. If we already had the - * bridge in our list, unmark it, and don't actually add anything new. - * If <b>transport_name</b> is non-NULL - the bridge is associated with a - * pluggable transport - we assign the transport to the bridge. */ +/** Return True if we have a bridge that uses a transport with name + * <b>transport_name</b>. */ +int +transport_is_needed(const char *transport_name) +{ + if (!bridge_list) + return 0; + + SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) { + if (bridge->transport_name && + !strcmp(bridge->transport_name, transport_name)) + return 1; + } SMARTLIST_FOREACH_END(bridge); + + return 0; +} + +/** Register the bridge information in <b>bridge_line</b> to the + * bridge subsystem. Steals reference of <b>bridge_line</b>. */ void -bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, const char *transport_name) +bridge_add_from_config(bridge_line_t *bridge_line) { bridge_info_t *b; - bridge_resolve_conflicts(addr, port, digest, transport_name); + { /* Log the bridge we are about to register: */ + log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)", + fmt_addrport(&bridge_line->addr, bridge_line->port), + bridge_line->transport_name ? + bridge_line->transport_name : "no transport", + tor_digest_is_zero(bridge_line->digest) ? + "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN)); + + if (bridge_line->socks_args) { /* print socks arguments */ + int i = 0; + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + + log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:", + smartlist_len(bridge_line->socks_args)); + SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg, + log_debug(LD_CONFIG, "%d: %s", ++i, arg)); + } + } + + bridge_resolve_conflicts(&bridge_line->addr, + bridge_line->port, + bridge_line->digest, + bridge_line->transport_name); b = tor_malloc_zero(sizeof(bridge_info_t)); - tor_addr_copy(&b->addr, addr); - b->port = port; - if (digest) - memcpy(b->identity, digest, DIGEST_LEN); - if (transport_name) - b->transport_name = tor_strdup(transport_name); + tor_addr_copy(&b->addr, &bridge_line->addr); + b->port = bridge_line->port; + memcpy(b->identity, bridge_line->digest, DIGEST_LEN); + if (bridge_line->transport_name) + b->transport_name = bridge_line->transport_name; b->fetch_status.schedule = DL_SCHED_BRIDGE; + b->socks_args = bridge_line->socks_args; if (!bridge_list) bridge_list = smartlist_new(); + tor_free(bridge_line); /* Deallocate bridge_line now. */ + smartlist_add(bridge_list, b); } @@ -1857,7 +1932,7 @@ find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) * transport, but the transport could not be found. */ int -find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const transport_t **transport) { *transport = NULL; @@ -1884,11 +1959,21 @@ find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, return 0; } +/** Return a smartlist containing all the SOCKS arguments that we + * should pass to the SOCKS proxy. */ +const smartlist_t * +get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) +{ + bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr, + port, + NULL); + return bridge ? bridge->socks_args : NULL; +} + /** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { - char *address; const or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( @@ -1903,15 +1988,12 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - address = tor_dup_addr(&bridge->addr); - - directory_initiate_command(address, &bridge->addr, + directory_initiate_command(&bridge->addr, bridge->port, 0/*no dirport*/, bridge->identity, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, DIRIND_ONEHOP, "authority.z", NULL, 0, 0); - tor_free(address); } /** Fetching the bridge descriptor from the bridge authority returned a @@ -2029,13 +2111,11 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } else { if (tor_addr_family(&bridge->addr) == AF_INET) { ri->addr = tor_addr_to_ipv4h(&bridge->addr); - tor_free(ri->address); - ri->address = tor_dup_ip(ri->addr); ri->or_port = bridge->port; log_info(LD_DIR, "Adjusted bridge routerinfo for '%s' to match configured " "address %s:%d.", - ri->nickname, ri->address, ri->or_port); + ri->nickname, fmt_addr32(ri->addr), ri->or_port); } else if (tor_addr_family(&bridge->addr) == AF_INET6) { tor_addr_copy(&ri->ipv6_addr, &bridge->addr); ri->ipv6_orport = bridge->port; @@ -2093,7 +2173,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { - int first = !any_bridge_descriptors_known(); + int first = num_bridges_usable() <= 1; bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); router_set_status(ri->cache_info.identity_digest, 1); @@ -2116,17 +2196,14 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) entry_guard_register_connect_status(ri->cache_info.identity_digest, 1, 0, now); if (first) { - /* XXXX apparently, this is never called. See bug #9229. */ routerlist_retry_directory_downloads(now); } - - update_networkstatus_downloads(now); } } } -/** Return 1 if any of our entry guards have descriptors that - * are marked with purpose 'bridge' and are running. Else return 0. +/** Return the number of bridges that have descriptors that + * are marked with purpose 'bridge' and are running. * * We use this function to decide if we're ready to start building * circuits through our bridges, or if we need to wait until the @@ -2138,25 +2215,16 @@ any_bridge_descriptors_known(void) return choose_random_entry(NULL) != NULL; } -/** Return 1 if there are any directory conns fetching bridge descriptors - * that aren't marked for close. We use this to guess if we should tell - * the controller that we have a problem. */ -int -any_pending_bridge_descriptor_fetches(void) +/** Return the number of bridges that have descriptors that are marked with + * purpose 'bridge' and are running. + */ +static int +num_bridges_usable(void) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (conn->type == CONN_TYPE_DIR && - conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC && - TO_DIR_CONN(conn)->router_purpose == ROUTER_PURPOSE_BRIDGE && - !conn->marked_for_close && - conn->linked && - conn->linked_conn && !conn->linked_conn->marked_for_close) { - log_debug(LD_DIR, "found one: %s", conn->address); - return 1; - } - } SMARTLIST_FOREACH_END(conn); - return 0; + int n_options = 0; + tor_assert(get_options()->UseBridges); + (void) choose_random_entry_impl(NULL, 0, 0, &n_options); + return n_options; } /** Return 1 if we have at least one descriptor for an entry guard @@ -2254,6 +2322,6 @@ entry_guards_free_all(void) clear_bridge_list(); smartlist_free(bridge_list); bridge_list = NULL; - circuit_build_times_free_timeouts(&circ_times); + circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); } diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index 52b8dc00e..e229f3b79 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file guardnodes.h + * \file entrynodes.h * \brief Header file for circuitbuild.c. **/ @@ -77,6 +77,8 @@ int num_live_entry_guards(int for_directory); #endif +void remove_all_entry_guards(void); + void entry_guards_compute_status(const or_options_t *options, time_t now); int entry_guard_register_connect_status(const char *digest, int succeeded, int mark_relay_status, time_t now); @@ -97,27 +99,30 @@ int routerinfo_is_a_configured_bridge(const routerinfo_t *ri); int node_is_a_configured_bridge(const node_t *node); void learned_router_identity(const tor_addr_t *addr, uint16_t port, const char *digest); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, - const char *transport_name); +struct bridge_line_t; +void bridge_add_from_config(struct bridge_line_t *bridge_line); void retry_bridge_descriptor_fetch_directly(const char *digest); void fetch_bridge_descriptors(const or_options_t *options, time_t now); void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); int entries_known_but_down(const or_options_t *options); void entries_retry_all(const or_options_t *options); int any_bridge_supports_microdescriptors(void); +const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr, + uint16_t port); + +int any_bridges_dont_support_microdescriptors(void); void entry_guards_free_all(void); const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port); struct transport_t; -int find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const struct transport_t **transport); +int transport_is_needed(const char *transport_name); int validate_pluggable_transports_config(void); double pathbias_get_close_success_count(entry_guard_t *guard); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c new file mode 100644 index 000000000..0d28a9199 --- /dev/null +++ b/src/or/ext_orport.c @@ -0,0 +1,648 @@ +/* Copyright (c) 2012, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ext_orport.c + * \brief Code implementing the Extended ORPort. +*/ + +#define EXT_ORPORT_PRIVATE +#include "or.h" +#include "connection.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" +#include "config.h" +#include "util.h" +#include "main.h" + +/** Allocate and return a structure capable of holding an Extended + * ORPort message of body length <b>len</b>. */ +ext_or_cmd_t * +ext_or_cmd_new(uint16_t len) +{ + size_t size = STRUCT_OFFSET(ext_or_cmd_t, body) + len; + ext_or_cmd_t *cmd = tor_malloc(size); + cmd->len = len; + return cmd; +} + +/** Deallocate the Extended ORPort message in <b>cmd</b>. */ +void +ext_or_cmd_free(ext_or_cmd_t *cmd) +{ + tor_free(cmd); +} + +/** Get an Extended ORPort message from <b>conn</b>, and place it in + * <b>out</b>. Return -1 on fail, 0 if we need more data, and 1 if we + * successfully extracted an Extended ORPort command from the + * buffer. */ +static int +connection_fetch_ext_or_cmd_from_buf(connection_t *conn, ext_or_cmd_t **out) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_ext_or_command_from_evbuffer(input, out); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_ext_or_command_from_buf(conn->inbuf, out); + } +} + +/** Write an Extended ORPort message to <b>conn</b>. Use + * <b>command</b> as the command type, <b>bodylen</b> as the body + * length, and <b>body</b>, if it's present, as the body of the + * message. */ +STATIC int +connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen) +{ + char header[4]; + if (bodylen > UINT16_MAX) + return -1; + set_uint16(header, htons(command)); + set_uint16(header+2, htons(bodylen)); + connection_write_to_buf(header, 4, conn); + if (bodylen) { + tor_assert(body); + connection_write_to_buf(body, bodylen, conn); + } + return 0; +} + +/** Transition from an Extended ORPort which accepts Extended ORPort + * messages, to an Extended ORport which accepts OR traffic. */ +static void +connection_ext_or_transition(or_connection_t *conn) +{ + tor_assert(conn->base_.type == CONN_TYPE_EXT_OR); + + conn->base_.type = CONN_TYPE_OR; + TO_CONN(conn)->state = 0; // set the state to a neutral value + control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_tls_start_handshake(conn, 1); +} + +/** Length of authentication cookie. */ +#define EXT_OR_PORT_AUTH_COOKIE_LEN 32 +/** Length of the header of the cookie file. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN 32 +/** Static cookie file header. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER "! Extended ORPort Auth Cookie !\x0a" +/** Length of safe-cookie protocol hashes. */ +#define EXT_OR_PORT_AUTH_HASH_LEN DIGEST256_LEN +/** Length of safe-cookie protocol nonces. */ +#define EXT_OR_PORT_AUTH_NONCE_LEN 32 +/** Safe-cookie protocol constants. */ +#define EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST \ + "ExtORPort authentication server-to-client hash" +#define EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST \ + "ExtORPort authentication client-to-server hash" + +/* Code to indicate cookie authentication */ +#define EXT_OR_AUTHTYPE_SAFECOOKIE 0x01 + +/** If true, we've set ext_or_auth_cookie to a secret code and stored + * it to disk. */ +STATIC int ext_or_auth_cookie_is_set = 0; +/** If ext_or_auth_cookie_is_set, a secret cookie that we've stored to disk + * and which we're using to authenticate controllers. (If the controller can + * read it off disk, it has permission to connect.) */ +STATIC uint8_t *ext_or_auth_cookie = NULL; + +/** Helper: Return a newly allocated string containing a path to the + * file where we store our authentication cookie. */ +char * +get_ext_or_auth_cookie_file_name(void) +{ + const or_options_t *options = get_options(); + if (options->ExtORPortCookieAuthFile && + strlen(options->ExtORPortCookieAuthFile)) { + return tor_strdup(options->ExtORPortCookieAuthFile); + } else { + return get_datadir_fname("extended_orport_auth_cookie"); + } +} + +/* Initialize the cookie-based authentication system of the + * Extended ORPort. If <b>is_enabled</b> is 0, then disable the cookie + * authentication system. */ +int +init_ext_or_cookie_authentication(int is_enabled) +{ + char *fname = NULL; + int retval; + + if (!is_enabled) { + ext_or_auth_cookie_is_set = 0; + return 0; + } + + fname = get_ext_or_auth_cookie_file_name(); + retval = init_cookie_authentication(fname, EXT_OR_PORT_AUTH_COOKIE_HEADER, + EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN, + &ext_or_auth_cookie, + &ext_or_auth_cookie_is_set); + tor_free(fname); + return retval; +} + +/** Read data from <b>conn</b> and see if the client sent us the + * authentication type that she prefers to use in this session. + * + * Return -1 if we received corrupted data or if we don't support the + * authentication type. Return 0 if we need more data in + * <b>conn</b>. Return 1 if the authentication type negotiation was + * successful. */ +static int +connection_ext_or_auth_neg_auth_type(connection_t *conn) +{ + char authtype[1] = {0}; + + if (connection_get_inbuf_len(conn) < 1) + return 0; + + if (connection_fetch_from_buf(authtype, 1, conn) < 0) + return -1; + + log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]); + if (authtype[0] != EXT_OR_AUTHTYPE_SAFECOOKIE) { + /* '1' is the only auth type supported atm */ + return -1; + } + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE; + return 1; +} + +/** DOCDOC */ +STATIC int +handle_client_auth_nonce(const char *client_nonce, size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out) +{ + char server_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + char server_nonce[EXT_OR_PORT_AUTH_NONCE_LEN] = {0}; + char *reply; + size_t reply_len; + + if (client_nonce_len != EXT_OR_PORT_AUTH_NONCE_LEN) + return -1; + + /* Get our nonce */ + if (crypto_rand(server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN) < 0) + return -1; + + { /* set up macs */ + size_t hmac_s_msg_len = strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + size_t hmac_c_msg_len = strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + + char *hmac_s_msg = tor_malloc_zero(hmac_s_msg_len); + char *hmac_c_msg = tor_malloc_zero(hmac_c_msg_len); + char *correct_client_hash = tor_malloc_zero(EXT_OR_PORT_AUTH_HASH_LEN); + + memcpy(hmac_s_msg, + EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST, + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST)); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + memcpy(hmac_c_msg, + EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST, + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST)); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + crypto_hmac_sha256(server_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_s_msg, + hmac_s_msg_len); + + crypto_hmac_sha256(correct_client_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_c_msg, + hmac_c_msg_len); + + /* Store the client hash we generated. We will need to compare it + with the hash sent by the client. */ + *client_hash_out = correct_client_hash; + + memwipe(hmac_s_msg, 0, hmac_s_msg_len); + memwipe(hmac_c_msg, 0, hmac_c_msg_len); + + tor_free(hmac_s_msg); + tor_free(hmac_c_msg); + } + + { /* debug logging */ /* XXX disable this codepath if not logging on debug?*/ + char server_hash_encoded[(2*EXT_OR_PORT_AUTH_HASH_LEN) + 1]; + char server_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + char client_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + + base16_encode(server_hash_encoded, sizeof(server_hash_encoded), + server_hash, sizeof(server_hash)); + base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), + server_nonce, sizeof(server_nonce)); + base16_encode(client_nonce_encoded, sizeof(client_nonce_encoded), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + log_debug(LD_GENERAL, + "server_hash: '%s'\nserver_nonce: '%s'\nclient_nonce: '%s'", + server_hash_encoded, server_nonce_encoded, client_nonce_encoded); + + memwipe(server_hash_encoded, 0, sizeof(server_hash_encoded)); + memwipe(server_nonce_encoded, 0, sizeof(server_nonce_encoded)); + memwipe(client_nonce_encoded, 0, sizeof(client_nonce_encoded)); + } + + { /* write reply: (server_hash, server_nonce) */ + + reply_len = EXT_OR_PORT_AUTH_COOKIE_LEN+EXT_OR_PORT_AUTH_NONCE_LEN; + reply = tor_malloc_zero(reply_len); + memcpy(reply, server_hash, EXT_OR_PORT_AUTH_HASH_LEN); + memcpy(reply + EXT_OR_PORT_AUTH_HASH_LEN, server_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN); + } + + *reply_out = reply; + *reply_len_out = reply_len; + + return 0; +} + +/** Read the client's nonce out of <b>conn</b>, setup the safe-cookie + * crypto, and then send our own hash and nonce to the client + * + * Return -1 if there was an error; return 0 if we need more data in + * <b>conn</b>, and return 1 if we successfully retrieved the + * client's nonce and sent our own. */ +static int +connection_ext_or_auth_handle_client_nonce(connection_t *conn) +{ + char client_nonce[EXT_OR_PORT_AUTH_NONCE_LEN]; + char *reply=NULL; + size_t reply_len=0; + + if (!ext_or_auth_cookie_is_set) { /* this should not happen */ + log_warn(LD_BUG, "Extended ORPort authentication cookie was not set. " + "That's weird since we should have done that on startup. " + "This might be a Tor bug, please file a bug report. "); + return -1; + } + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN) + return 0; + + if (connection_fetch_from_buf(client_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0) + return -1; + + /* We extract the ClientNonce from the received data, and use it to + calculate ServerHash and ServerNonce according to proposal 217. + + We also calculate our own ClientHash value and save it in the + connection state. We validate it later against the ClientHash + sent by the client. */ + if (handle_client_auth_nonce(client_nonce, sizeof(client_nonce), + &TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + &reply, &reply_len) < 0) + return -1; + + connection_write_to_buf(reply, reply_len, conn); + + memwipe(reply, 0, reply_len); + tor_free(reply); + + log_debug(LD_GENERAL, "Got client nonce, and sent our own nonce and hash."); + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH; + return 1; +} + +#define connection_ext_or_auth_send_result_success(c) \ + connection_ext_or_auth_send_result(c, 1) +#define connection_ext_or_auth_send_result_fail(c) \ + connection_ext_or_auth_send_result(c, 0) + +/** Send authentication results to <b>conn</b>. Successful results if + * <b>success</b> is set; failure results otherwise. */ +static void +connection_ext_or_auth_send_result(connection_t *conn, int success) +{ + if (success) + connection_write_to_buf("\x01", 1, conn); + else + connection_write_to_buf("\x00", 1, conn); +} + +/** Receive the client's hash from <b>conn</b>, validate that it's + * correct, and then send the authentication results to the client. + * + * Return -1 if there was an error during validation; return 0 if we + * need more data in <b>conn</b>, and return 1 if we successfully + * validated the client's hash and sent a happy authentication + * result. */ +static int +connection_ext_or_auth_handle_client_hash(connection_t *conn) +{ + char provided_client_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN) + return 0; + + if (connection_fetch_from_buf(provided_client_hash, + EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0) + return -1; + + if (tor_memneq(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + provided_client_hash, EXT_OR_PORT_AUTH_HASH_LEN)) { + log_warn(LD_GENERAL, "Incorrect client hash. Authentication failed."); + connection_ext_or_auth_send_result_fail(conn); + return -1; + } + + log_debug(LD_GENERAL, "Got client's hash and it was legit."); + + /* send positive auth result */ + connection_ext_or_auth_send_result_success(conn); + conn->state = EXT_OR_CONN_STATE_OPEN; + return 1; +} + +/** Handle data from <b>or_conn</b> received on Extended ORPort. + * Return -1 on error. 0 on unsufficient data. 1 on correct. */ +static int +connection_ext_or_auth_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + + /* State transitions of the Extended ORPort authentication protocol: + + EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE (start state) -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH -> + EXT_OR_CONN_STATE_OPEN + + During EXT_OR_CONN_STATE_OPEN, data is handled by + connection_ext_or_process_inbuf(). + */ + + switch (conn->state) { /* Functionify */ + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return connection_ext_or_auth_neg_auth_type(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return connection_ext_or_auth_handle_client_nonce(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return connection_ext_or_auth_handle_client_hash(conn); + + default: + log_warn(LD_BUG, "Encountered unexpected connection state %d while trying " + "to process Extended ORPort authentication data.", conn->state); + return -1; + } +} + +/** Extended ORPort commands (Transport-to-Bridge) */ +#define EXT_OR_CMD_TB_DONE 0x0000 +#define EXT_OR_CMD_TB_USERADDR 0x0001 +#define EXT_OR_CMD_TB_TRANSPORT 0x0002 + +/** Extended ORPort commands (Bridge-to-Transport) */ +#define EXT_OR_CMD_BT_OKAY 0x1000 +#define EXT_OR_CMD_BT_DENY 0x1001 +#define EXT_OR_CMD_BT_CONTROL 0x1002 + +/** Process a USERADDR command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the USERADDR command was well formed, change the address of + * <b>conn</b> to the address on the USERADDR command. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_useraddr(connection_t *conn, + const char *payload, uint16_t len) +{ + /* Copy address string. */ + tor_addr_t addr; + uint16_t port; + char *addr_str; + char *address_part=NULL; + int res; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort UserAddr"); + return -1; + } + + addr_str = tor_memdup_nulterm(payload, len); + + res = tor_addr_port_split(LOG_INFO, addr_str, &address_part, &port); + tor_free(addr_str); + if (res<0) + return -1; + + res = tor_addr_parse(&addr, address_part); + tor_free(address_part); + if (res<0) + return -1; + + { /* do some logging */ + char *old_address = tor_dup_addr(&conn->addr); + char *new_address = tor_dup_addr(&addr); + + log_debug(LD_NET, "Received USERADDR." + "We rewrite our address from '%s:%u' to '%s:%u'.", + safe_str(old_address), conn->port, safe_str(new_address), port); + + tor_free(old_address); + tor_free(new_address); + } + + /* record the address */ + tor_addr_copy(&conn->addr, &addr); + conn->port = port; + if (conn->address) { + tor_free(conn->address); + } + conn->address = tor_dup_addr(&addr); + + return 0; +} + +/** Process a TRANSPORT command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the TRANSPORT command was well formed, register the name of the + * transport on <b>conn</b>. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_transport(or_connection_t *conn, + const char *payload, uint16_t len) +{ + char *transport_str; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort Transport"); + return -1; + } + + transport_str = tor_memdup_nulterm(payload, len); + + /* Transport names MUST be C-identifiers. */ + if (!string_is_C_identifier(transport_str)) { + tor_free(transport_str); + return -1; + } + + /* If ext_or_transport is already occupied (because the PT sent two + * TRANSPORT commands), deallocate the old name and keep the new + * one */ + if (conn->ext_or_transport) + tor_free(conn->ext_or_transport); + + conn->ext_or_transport = transport_str; + return 0; +} + +#define EXT_OR_CONN_STATE_IS_AUTHENTICATING(st) \ + ((st) <= EXT_OR_CONN_STATE_AUTH_MAX) + +/** Process Extended ORPort messages from <b>or_conn</b>. */ +int +connection_ext_or_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + ext_or_cmd_t *command; + int r; + + /* DOCDOC Document the state machine and transitions in this function */ + + /* If we are still in the authentication stage, process traffic as + authentication data: */ + while (EXT_OR_CONN_STATE_IS_AUTHENTICATING(conn->state)) { + log_debug(LD_GENERAL, "Got Extended ORPort authentication data (%u).", + (unsigned int) connection_get_inbuf_len(conn)); + r = connection_ext_or_auth_process_inbuf(or_conn); + if (r < 0) { + connection_mark_for_close(conn); + return -1; + } else if (r == 0) { + return 0; + } + /* if r > 0, loop and process more data (if any). */ + } + + while (1) { + log_debug(LD_GENERAL, "Got Extended ORPort data."); + command = NULL; + r = connection_fetch_ext_or_cmd_from_buf(conn, &command); + if (r < 0) + goto err; + else if (r == 0) + return 0; /* need to wait for more data */ + + /* Got a command! */ + tor_assert(command); + + if (command->cmd == EXT_OR_CMD_TB_DONE) { + if (connection_get_inbuf_len(conn)) { + /* The inbuf isn't empty; the client is misbehaving. */ + goto err; + } + + log_debug(LD_NET, "Received DONE."); + + /* If the transport proxy did not use the TRANSPORT command to + * specify the transport name, mark this as unknown transport. */ + if (!or_conn->ext_or_transport) { + /* We write this string this way to avoid ??>, which is a C + * trigraph. */ + or_conn->ext_or_transport = tor_strdup("<?" "?>"); + } + + connection_write_ext_or_command(conn, EXT_OR_CMD_BT_OKAY, NULL, 0); + + /* can't transition immediately; need to flush first. */ + conn->state = EXT_OR_CONN_STATE_FLUSHING; + connection_stop_reading(conn); + } else if (command->cmd == EXT_OR_CMD_TB_USERADDR) { + if (connection_ext_or_handle_cmd_useraddr(conn, + command->body, command->len) < 0) + goto err; + } else if (command->cmd == EXT_OR_CMD_TB_TRANSPORT) { + if (connection_ext_or_handle_cmd_transport(or_conn, + command->body, command->len) < 0) + goto err; + } else { + log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).", + command->cmd); + } + + ext_or_cmd_free(command); + } + + return 0; + + err: + ext_or_cmd_free(command); + connection_mark_for_close(conn); + return -1; +} + +/** <b>conn</b> finished flushing Extended ORPort messages to the + * network, and is now ready to accept OR traffic. This function + * does the transition. */ +int +connection_ext_or_finished_flushing(or_connection_t *conn) +{ + if (conn->base_.state == EXT_OR_CONN_STATE_FLUSHING) { + connection_start_reading(TO_CONN(conn)); + connection_ext_or_transition(conn); + } + return 0; +} + +/** Initiate Extended ORPort authentication, by sending the list of + * supported authentication types to the client. */ +int +connection_ext_or_start_auth(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + const uint8_t authtypes[] = { + /* We only support authtype '1' for now. */ + EXT_OR_AUTHTYPE_SAFECOOKIE, + /* Marks the end of the list. */ + 0 + }; + + log_debug(LD_GENERAL, + "ExtORPort authentication: Sending supported authentication types"); + + connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn); + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE; + + return 0; +} + +/** Free any leftover allocated memory of the ext_orport.c subsystem. */ +void +ext_orport_free_all(void) +{ + if (ext_or_auth_cookie) /* Free the auth cookie */ + tor_free(ext_or_auth_cookie); +} + diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h new file mode 100644 index 000000000..ce45e5f41 --- /dev/null +++ b/src/or/ext_orport.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef EXT_ORPORT_H +#define EXT_ORPORT_H + +int connection_ext_or_start_auth(or_connection_t *or_conn); + +ext_or_cmd_t *ext_or_cmd_new(uint16_t len); +void ext_or_cmd_free(ext_or_cmd_t *cmd); +void connection_or_set_ext_or_identifier(or_connection_t *conn); +void connection_or_remove_from_ext_or_id_map(or_connection_t *conn); +void connection_or_clear_ext_or_id_map(void); +or_connection_t *connection_or_get_by_ext_or_id(const char *id); + +int connection_ext_or_finished_flushing(or_connection_t *conn); +int connection_ext_or_process_inbuf(or_connection_t *or_conn); + +int init_ext_or_cookie_authentication(int is_enabled); +char *get_ext_or_auth_cookie_file_name(void); +void ext_orport_free_all(void); + +#ifdef EXT_ORPORT_PRIVATE +STATIC int connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen); +STATIC int handle_client_auth_nonce(const char *client_nonce, + size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out); +#ifdef TOR_UNIT_TESTS +extern uint8_t *ext_or_auth_cookie; +extern int ext_or_auth_cookie_is_set; +#endif +#endif + +#endif + diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index 4d8a835c8..55e4c89a4 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -32,17 +32,8 @@ fp_pair_map_entries_eq(const fp_pair_map_entry_t *a, static INLINE unsigned int fp_pair_map_entry_hash(const fp_pair_map_entry_t *a) { - const uint32_t *p; - unsigned int hash; - - p = (const uint32_t *)(a->key.first); - /* Hashes are 20 bytes long, so 5 times uint32_t */ - hash = p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - /* Now XOR in the second fingerprint */ - p = (const uint32_t *)(a->key.second); - hash ^= p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - - return hash; + tor_assert(sizeof(a->key) == DIGEST_LEN*2); + return (unsigned) siphash24g(&a->key, DIGEST_LEN*2); } /* diff --git a/src/or/geoip.c b/src/or/geoip.c index e2e98e8ec..f722bac46 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -120,7 +120,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, /** Add an entry to the GeoIP table indicated by <b>family</b>, * parsing it from <b>line</b>. The format is as for geoip_load_file(). */ -/*private*/ int +STATIC int geoip_parse_entry(const char *line, sa_family_t family) { tor_addr_t low_addr, high_addr; @@ -363,7 +363,7 @@ geoip_load_file(sa_family_t family, const char *filename) * be less than geoip_get_n_countries(). To decode it, call * geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr) { geoip_ipv4_entry_t *ent; @@ -379,7 +379,7 @@ geoip_get_country_by_ipv4(uint32_t ipaddr) * 0 for the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr) { geoip_ipv6_entry_t *ent; @@ -461,6 +461,10 @@ geoip_db_digest(sa_family_t family) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; tor_addr_t addr; + /* Name of pluggable transport used by this client. NULL if no + pluggable transport was used. */ + char *transport_name; + /** Time when we last saw this IP address, in MINUTES since the epoch. * * (This will run out of space around 4011 CE. If Tor is still in use around @@ -482,12 +486,20 @@ static HT_HEAD(clientmap, clientmap_entry_t) client_history = static INLINE unsigned clientmap_entry_hash(const clientmap_entry_t *a) { - return ht_improve_hash(tor_addr_hash(&a->addr)); + unsigned h = (unsigned) tor_addr_hash(&a->addr); + + if (a->transport_name) + h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); + + return h; } /** Hashtable helper: compare two clientmap_entry_t values for equality. */ static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { + if (strcmp_opt(a->transport_name, b->transport_name)) + return 0; + return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && a->action == b->action; } @@ -497,6 +509,17 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Free all storage held by <b>ent</b>. */ +static void +clientmap_entry_free(clientmap_entry_t *ent) +{ + if (!ent) + return; + + tor_free(ent->transport_name); + tor_free(ent); +} + /** Clear history of connecting clients used by entry and bridge stats. */ static void client_history_clear(void) @@ -507,7 +530,7 @@ client_history_clear(void) if ((*ent)->action == GEOIP_CLIENT_CONNECT) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -519,27 +542,40 @@ client_history_clear(void) * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now) + const tor_addr_t *addr, + const char *transport_name, + time_t now) { const or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; + memset(&lookup, 0, sizeof(clientmap_entry_t)); + if (action == GEOIP_CLIENT_CONNECT) { /* Only remember statistics as entry guard or as bridge. */ if (!options->EntryStatistics && (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { - if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirReqStatistics) + /* Only gather directory-request statistics if configured, and + * forcibly disable them on bridge authorities. */ + if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) return; } + log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", + safe_str_client(fmt_addr((addr))), + transport_name ? transport_name : "<no transport>"); + tor_addr_copy(&lookup.addr, addr); lookup.action = (int)action; + lookup.transport_name = (char*) transport_name; ent = HT_FIND(clientmap, &client_history, &lookup); + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); tor_addr_copy(&ent->addr, addr); + if (transport_name) + ent->transport_name = tor_strdup(transport_name); ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } @@ -566,7 +602,7 @@ remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) { time_t cutoff = *(time_t*)_cutoff / 60; if (ent->last_seen_in_minutes < cutoff) { - tor_free(ent); + clientmap_entry_free(ent); return 1; } else { return 0; @@ -769,6 +805,106 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, } } +/** Return the bridge-ip-transports string that should be inserted in + * our extra-info descriptor. Return NULL if the bridge-ip-transports + * line should be empty. */ +char * +geoip_get_transport_history(void) +{ + unsigned granularity = IP_GRANULARITY; + /** String hash table (name of transport) -> (number of users). */ + strmap_t *transport_counts = strmap_new(); + + /** Smartlist that contains copies of the names of the transports + that have been used. */ + smartlist_t *transports_used = smartlist_new(); + + /* Special string to signify that no transport was used for this + connection. Pluggable transport names can't have symbols in their + names, so this string will never collide with a real transport. */ + static const char* no_transport_str = "<OR>"; + + clientmap_entry_t **ent; + const char *transport_name = NULL; + smartlist_t *string_chunks = smartlist_new(); + char *the_string = NULL; + + /* If we haven't seen any clients yet, return NULL. */ + if (HT_EMPTY(&client_history)) + goto done; + + /** We do the following steps to form the transport history string: + * a) Foreach client that uses a pluggable transport, we increase the + * times that transport was used by one. If the client did not use + * a transport, we increase the number of times someone connected + * without obfuscation. + * b) Foreach transport we observed, we write its transport history + * string and push it to string_chunks. So, for example, if we've + * seen 665 obfs2 clients, we write "obfs2=665". + * c) We concatenate string_chunks to form the final string. + */ + + log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", + HT_SIZE(&client_history)); + + /* Loop through all clients. */ + HT_FOREACH(ent, clientmap, &client_history) { + uintptr_t val; + void *ptr; + transport_name = (*ent)->transport_name; + if (!transport_name) + transport_name = no_transport_str; + + /* Increase the count for this transport name. */ + ptr = strmap_get(transport_counts, transport_name); + val = (uintptr_t)ptr; + val++; + ptr = (void*)val; + strmap_set(transport_counts, transport_name, ptr); + + /* If it's the first time we see this transport, note it. */ + if (val == 1) + smartlist_add(transports_used, tor_strdup(transport_name)); + + log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " + "I've now seen %d clients.", + safe_str_client(fmt_addr(&(*ent)->addr)), + transport_name ? transport_name : "<no transport>", + (int)val); + } + + /* Sort the transport names (helps with unit testing). */ + smartlist_sort_strings(transports_used); + + /* Loop through all seen transports. */ + SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { + void *transport_count_ptr = strmap_get(transport_counts, transport_name); + uintptr_t transport_count = (uintptr_t) transport_count_ptr; + + log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.", + U64_PRINTF_ARG((uint64_t)transport_count), transport_name); + + smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT, + transport_name, + U64_PRINTF_ARG(round_uint64_to_next_multiple_of( + (uint64_t)transport_count, + granularity))); + } SMARTLIST_FOREACH_END(transport_name); + + the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); + + log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); + + done: + strmap_free(transport_counts, NULL); + SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); + smartlist_free(transports_used); + SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); + smartlist_free(string_chunks); + + return the_string; +} + /** Return a newly allocated comma-separated string containing statistics * on network status downloads. The string contains the number of completed * requests, timeouts, and still running requests as well as the download @@ -1037,7 +1173,7 @@ geoip_reset_dirreq_stats(time_t now) if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -1132,7 +1268,7 @@ geoip_format_dirreq_stats(time_t now) time_t geoip_dirreq_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_dirreq_stats_interval) return 0; /* Not initialized. */ @@ -1146,21 +1282,13 @@ geoip_dirreq_stats_write(time_t now) str = geoip_format_dirreq_stats(now); /* Write dirreq-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); + /* Reset measurement interval start. */ + geoip_reset_dirreq_stats(now); } - filename = get_datadir_fname2("stats", "dirreq-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write dirreq statistics to disk!"); - - /* Reset measurement interval start. */ - geoip_reset_dirreq_stats(now); done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; } @@ -1197,6 +1325,8 @@ validate_bridge_stats(const char *stats_str, time_t now) const char *BRIDGE_STATS_END = "bridge-stats-end "; const char *BRIDGE_IPS = "bridge-ips "; const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; + const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; const char *tmp; time_t stats_end_time; int seconds; @@ -1231,6 +1361,15 @@ validate_bridge_stats(const char *stats_str, time_t now) return 0; } + /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); + if (!tmp) { + /* Look if there is an empty "bridge-ip-transports" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); + if (!tmp) + return 0; + } + return 1; } @@ -1244,7 +1383,8 @@ static char *bridge_stats_extrainfo = NULL; char * geoip_format_bridge_stats(time_t now) { - char *out = NULL, *country_data = NULL, *ipver_data = NULL; + char *out = NULL; + char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; long duration = now - start_of_bridge_stats_interval; char written[ISO_TIME_LEN+1]; @@ -1255,16 +1395,20 @@ geoip_format_bridge_stats(time_t now) format_iso_time(written, now); geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); + transport_data = geoip_get_transport_history(); tor_asprintf(&out, "bridge-stats-end %s (%ld s)\n" "bridge-ips %s\n" - "bridge-ip-versions %s\n", + "bridge-ip-versions %s\n" + "bridge-ip-transports %s\n", written, duration, country_data ? country_data : "", - ipver_data ? ipver_data : ""); + ipver_data ? ipver_data : "", + transport_data ? transport_data : ""); tor_free(country_data); tor_free(ipver_data); + tor_free(transport_data); return out; } @@ -1297,7 +1441,7 @@ format_bridge_stats_controller(time_t now) time_t geoip_bridge_stats_write(time_t now) { - char *filename = NULL, *val = NULL, *statsdir = NULL; + char *val = NULL; /* Check if 24 hours have passed since starting measurements. */ if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) @@ -1317,24 +1461,20 @@ geoip_bridge_stats_write(time_t now) start_of_bridge_stats_interval = now; /* Write it to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) - goto done; - filename = get_datadir_fname2("stats", "bridge-stats"); - - write_str_to_file(filename, bridge_stats_extrainfo, 0); - - /* Tell the controller, "hey, there are clients!" */ - { - char *controller_str = format_bridge_stats_controller(now); - if (controller_str) - control_event_clients_seen(controller_str); - tor_free(controller_str); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "bridge-stats", + bridge_stats_extrainfo, "bridge statistics"); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } } - done: - tor_free(filename); - tor_free(statsdir); + done: return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; } @@ -1436,7 +1576,7 @@ geoip_format_entry_stats(time_t now) time_t geoip_entry_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_entry_stats_interval) return 0; /* Not initialized. */ @@ -1450,21 +1590,14 @@ geoip_entry_stats_write(time_t now) str = geoip_format_entry_stats(now); /* Write entry-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; - } - filename = get_datadir_fname2("stats", "entry-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write entry statistics to disk!"); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); - /* Reset measurement interval start. */ - geoip_reset_entry_stats(now); + /* Reset measurement interval start. */ + geoip_reset_entry_stats(now); + } done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } @@ -1534,7 +1667,7 @@ geoip_free_all(void) for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } HT_CLEAR(clientmap, &client_history); } @@ -1549,5 +1682,6 @@ geoip_free_all(void) } clear_geoip_db(); + tor_free(bridge_stats_extrainfo); } diff --git a/src/or/geoip.h b/src/or/geoip.h index ebefee5f4..b9b53c300 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -12,10 +12,12 @@ #ifndef TOR_GEOIP_H #define TOR_GEOIP_H +#include "testsupport.h" + #ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line, sa_family_t family); -int geoip_get_country_by_ipv4(uint32_t ipaddr); -int geoip_get_country_by_ipv6(const struct in6_addr *addr); +STATIC int geoip_parse_entry(const char *line, sa_family_t family); +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); #endif int should_record_bridge_info(const or_options_t *options); int geoip_load_file(sa_family_t family, const char *filename); @@ -27,10 +29,12 @@ const char *geoip_db_digest(sa_family_t family); country_t geoip_get_country(const char *countrycode); void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now); + const tor_addr_t *addr, const char *transport_name, + time_t now); void geoip_remove_old_clients(time_t cutoff); void geoip_note_ns_response(geoip_ns_response_t response); +char *geoip_get_transport_history(void); int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str); char *geoip_get_request_history(void); diff --git a/src/or/hibernate.c b/src/or/hibernate.c index a41257133..c433ac1be 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -239,8 +239,8 @@ accounting_parse_options(const or_options_t *options, int validate_only) /** If we want to manage the accounting system and potentially * hibernate, return 1, else return 0. */ -int -accounting_is_enabled(const or_options_t *options) +MOCK_IMPL(int, +accounting_is_enabled,(const or_options_t *options)) { if (options->AccountingMax) return 1; @@ -255,6 +255,13 @@ accounting_get_interval_length(void) return (int)(interval_end_time - interval_start_time); } +/** Return the time at which the current accounting interval will end. */ +MOCK_IMPL(time_t, +accounting_get_end_time,(void)) +{ + return interval_end_time; +} + /** Called from main.c to tell us that <b>seconds</b> seconds have * passed, <b>n_read</b> bytes have been read, and <b>n_written</b> * bytes have been written. */ @@ -641,7 +648,15 @@ read_bandwidth_usage(void) { char *fname = get_datadir_fname("bw_accounting"); - unlink(fname); + int res; + + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + fname, strerror(errno)); + } + tor_free(fname); } @@ -808,8 +823,8 @@ hibernate_begin_shutdown(void) } /** Return true iff we are currently hibernating. */ -int -we_are_hibernating(void) +MOCK_IMPL(int, +we_are_hibernating,(void)) { return hibernate_state != HIBERNATE_STATE_LIVE; } @@ -1010,6 +1025,7 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +#ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit * tests. @@ -1019,4 +1035,5 @@ hibernate_set_state_for_testing_(hibernate_state_t newstate) { hibernate_state = newstate; } +#endif diff --git a/src/or/hibernate.h b/src/or/hibernate.h index d2d6989e1..38ecb7512 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -12,15 +12,18 @@ #ifndef TOR_HIBERNATE_H #define TOR_HIBERNATE_H +#include "testsupport.h" + int accounting_parse_options(const or_options_t *options, int validate_only); -int accounting_is_enabled(const or_options_t *options); +MOCK_DECL(int, accounting_is_enabled, (const or_options_t *options)); int accounting_get_interval_length(void); +MOCK_DECL(time_t, accounting_get_end_time, (void)); void configure_accounting(time_t now); void accounting_run_housekeeping(time_t now); void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); int accounting_record_bandwidth_usage(time_t now, or_state_t *state); void hibernate_begin_shutdown(void); -int we_are_hibernating(void); +MOCK_DECL(int, we_are_hibernating, (void)); void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, @@ -45,8 +48,10 @@ typedef enum { HIBERNATE_STATE_INITIAL=5 } hibernate_state_t; +#ifdef TOR_UNIT_TESTS void hibernate_set_state_for_testing_(hibernate_state_t newstate); #endif +#endif #endif diff --git a/src/or/include.am b/src/or/include.am index 65dbeff53..47bdd0990 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -1,5 +1,13 @@ bin_PROGRAMS+= src/or/tor -noinst_LIBRARIES+= src/or/libtor.a +noinst_LIBRARIES += \ + src/or/libtor.a +if UNITTESTS_ENABLED +noinst_LIBRARIES += \ + src/or/libtor-testing.a +endif +if COVERAGE_ENABLED +noinst_PROGRAMS+= src/or/tor-cov +endif if BUILD_NT_SERVICES tor_platform_source=src/or/ntmain.c @@ -21,11 +29,12 @@ else onion_ntor_source= endif -src_or_libtor_a_SOURCES = \ +LIBTOR_A_SOURCES = \ src/or/addressmap.c \ src/or/buffers.c \ src/or/channel.c \ src/or/channeltls.c \ + src/or/circpathbias.c \ src/or/circuitbuild.c \ src/or/circuitlist.c \ src/or/circuitmux.c \ @@ -48,6 +57,7 @@ src_or_libtor_a_SOURCES = \ src/or/fp_pair.c \ src/or/geoip.c \ src/or/entrynodes.c \ + src/or/ext_orport.c \ src/or/hibernate.c \ src/or/main.c \ src/or/microdesc.c \ @@ -77,6 +87,9 @@ src_or_libtor_a_SOURCES = \ $(onion_ntor_source) \ src/or/config_codedigest.c +src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) +src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES) + #libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ # ../common/libor-event.a @@ -90,6 +103,9 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ -DBINDIR="\"$(bindir)\"" +src_or_libtor_testing_a_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + # -L flags need to go in LDFLAGS. -l flags need to go in LDADD. # This seems to matter nowhere but on windows, but I assure you that it # matters a lot there, and is quite hard to debug if you forget to do it. @@ -102,11 +118,24 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +if COVERAGE_ENABLED +src_or_tor_cov_SOURCES = src/or/tor_main.c +src_or_tor_cov_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ +src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-crypto-testing.a $(LIBDONNA) \ + src/common/libor-event-testing.a \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +endif + ORHEADERS = \ src/or/addressmap.h \ src/or/buffers.h \ src/or/channel.h \ src/or/channeltls.h \ + src/or/circpathbias.h \ src/or/circuitbuild.h \ src/or/circuitlist.h \ src/or/circuitmux.h \ @@ -127,6 +156,7 @@ ORHEADERS = \ src/or/dns.h \ src/or/dnsserv.h \ src/or/eventdns_tor.h \ + src/or/ext_orport.h \ src/or/fp_pair.h \ src/or/geoip.h \ src/or/entrynodes.h \ diff --git a/src/or/main.c b/src/or/main.c index bd23141b9..3c661cd12 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -13,6 +13,7 @@ #define MAIN_PRIVATE #include "or.h" #include "addressmap.h" +#include "backtrace.h" #include "buffers.h" #include "channel.h" #include "channeltls.h" @@ -21,6 +22,7 @@ #include "circuituse.h" #include "command.h" #include "config.h" +#include "confparse.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" @@ -52,11 +54,13 @@ #include "routerparse.h" #include "statefile.h" #include "status.h" +#include "ext_orport.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#include "../common/sandbox.h" #ifdef HAVE_EVENT2_EVENT_H #include <event2/event.h> @@ -155,8 +159,6 @@ int can_complete_circuit=0; /** How often do we 'forgive' undownloadable router descriptors and attempt * to download them again? */ #define DESCRIPTOR_FAILURE_RESET_INTERVAL (60*60) -/** How long do we let a directory connection stall before expiring it? */ -#define DIR_CONN_MAX_STALL (5*60) /** Decides our behavior when no logs are configured/before any * logs have been configured. For 0, we log notice to stdout as normal. @@ -351,6 +353,8 @@ connection_remove(connection_t *conn) (int)conn->s, conn_type_to_string(conn->type), smartlist_len(connection_array)); + control_event_conn_bandwidth(conn); + tor_assert(conn->conn_array_index >= 0); current_index = conn->conn_array_index; connection_unregister_events(conn); /* This is redundant, but cheap. */ @@ -414,6 +418,19 @@ connection_unlink(connection_t *conn) connection_free(conn); } +/** Initialize the global connection list, closeable connection list, + * and active connection list. */ +STATIC void +init_connection_lists(void) +{ + if (!connection_array) + connection_array = smartlist_new(); + if (!closeable_connection_lst) + closeable_connection_lst = smartlist_new(); + if (!active_linked_connection_lst) + active_linked_connection_lst = smartlist_new(); +} + /** Schedule <b>conn</b> to be closed. **/ void add_connection_to_closeable_list(connection_t *conn) @@ -452,15 +469,15 @@ get_connection_array(void) /** Provides the traffic read and written over the life of the process. */ -uint64_t -get_bytes_read(void) +MOCK_IMPL(uint64_t, +get_bytes_read,(void)) { return stats_n_bytes_read; } /* DOCDOC get_bytes_written */ -uint64_t -get_bytes_written(void) +MOCK_IMPL(uint64_t, +get_bytes_written,(void)) { return stats_n_bytes_written; } @@ -507,8 +524,8 @@ connection_is_reading(connection_t *conn) } /** Tell the main loop to stop notifying <b>conn</b> of any read events. */ -void -connection_stop_reading(connection_t *conn) +MOCK_IMPL(void, +connection_stop_reading,(connection_t *conn)) { tor_assert(conn); @@ -532,8 +549,8 @@ connection_stop_reading(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any read events. */ -void -connection_start_reading(connection_t *conn) +MOCK_IMPL(void, +connection_start_reading,(connection_t *conn)) { tor_assert(conn); @@ -572,8 +589,8 @@ connection_is_writing(connection_t *conn) } /** Tell the main loop to stop notifying <b>conn</b> of any write events. */ -void -connection_stop_writing(connection_t *conn) +MOCK_IMPL(void, +connection_stop_writing,(connection_t *conn)) { tor_assert(conn); @@ -598,8 +615,8 @@ connection_stop_writing(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any write events. */ -void -connection_start_writing(connection_t *conn) +MOCK_IMPL(void, +connection_start_writing,(connection_t *conn)) { tor_assert(conn); @@ -687,7 +704,7 @@ connection_stop_reading_from_linked_conn(connection_t *conn) } /** Close all connections that have been scheduled to get closed. */ -static void +STATIC void close_closeable_connections(void) { int i; @@ -902,16 +919,7 @@ conn_close_if_marked(int i) return 0; } if (connection_wants_to_flush(conn)) { - int severity; - if (conn->type == CONN_TYPE_EXIT || - (conn->type == CONN_TYPE_OR && server_mode(get_options())) || - (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER)) - severity = LOG_INFO; - else - severity = LOG_NOTICE; - /* XXXX Maybe allow this to happen a certain amount per hour; it usually - * is meaningless. */ - log_fn(severity, LD_NET, "We stalled too much while trying to write %d " + log_fn(LOG_INFO, LD_NET, "We stalled too much while trying to write %d " "bytes to address %s. If this happens a lot, either " "something is wrong with your network connection, or " "something is wrong with theirs. " @@ -1028,9 +1036,11 @@ run_connection_housekeeping(int i, time_t now) * if a server or received if a client) for 5 min */ if (conn->type == CONN_TYPE_DIR && ((DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastwritten + DIR_CONN_MAX_STALL < now) || + conn->timestamp_lastwritten + + options->TestingDirConnectionMaxStall < now) || (!DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastread + DIR_CONN_MAX_STALL < now))) { + conn->timestamp_lastread + + options->TestingDirConnectionMaxStall < now))) { log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)", (int)conn->s, conn->purpose); /* This check is temporary; it's to let us know whether we should consider @@ -1143,6 +1153,18 @@ get_signewnym_epoch(void) return newnym_epoch; } +static time_t time_to_check_descriptor = 0; +/** + * Update our schedule so that we'll check whether we need to update our + * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL + * seconds. + */ +void +reschedule_descriptor_update_check(void) +{ + time_to_check_descriptor = 0; +} + /** Perform regular maintenance tasks. This function gets run once per * second by second_elapsed_callback(). */ @@ -1152,7 +1174,7 @@ run_scheduled_events(time_t now) static time_t last_rotated_x509_certificate = 0; static time_t time_to_check_v3_certificate = 0; static time_t time_to_check_listeners = 0; - static time_t time_to_check_descriptor = 0; + static time_t time_to_download_networkstatus = 0; static time_t time_to_shrink_memory = 0; static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; @@ -1176,22 +1198,12 @@ run_scheduled_events(time_t now) int i; int have_dir_info; - /** 0. See if we've been asked to shut down and our timeout has + /* 0. See if we've been asked to shut down and our timeout has * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. */ consider_hibernation(now); -#if 0 - { - static time_t nl_check_time = 0; - if (nl_check_time <= now) { - nodelist_assert_ok(); - nl_check_time = now + 30; - } - } -#endif - /* 0b. If we've deferred a signewnym, make sure it gets handled * eventually. */ if (signewnym_is_pending && @@ -1203,7 +1215,7 @@ run_scheduled_events(time_t now) /* 0c. If we've deferred log messages for the controller, handle them now */ flush_pending_log_callbacks(); - /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, + /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ @@ -1219,7 +1231,8 @@ run_scheduled_events(time_t now) router_upload_dir_desc_to_dirservers(0); } - if (!options->DisableNetwork && time_to_try_getting_descriptors < now) { + if (!should_delay_dir_fetches(options, NULL) && + time_to_try_getting_descriptors < now) { update_all_descriptor_downloads(now); update_extrainfo_downloads(now); if (router_have_minimum_dir_info()) @@ -1234,10 +1247,10 @@ run_scheduled_events(time_t now) now + DESCRIPTOR_FAILURE_RESET_INTERVAL; } - if (options->UseBridges) + if (options->UseBridges && !options->DisableNetwork) fetch_bridge_descriptors(options, now); - /** 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our + /* 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our * TLS context. */ if (!last_rotated_x509_certificate) last_rotated_x509_certificate = now; @@ -1263,7 +1276,7 @@ run_scheduled_events(time_t now) time_to_add_entropy = now + ENTROPY_INTERVAL; } - /** 1c. If we have to change the accounting interval or record + /* 1c. If we have to change the accounting interval or record * bandwidth used in this accounting interval, do so. */ if (accounting_is_enabled(options)) accounting_run_housekeeping(now); @@ -1276,7 +1289,7 @@ run_scheduled_events(time_t now) dirserv_test_reachability(now); } - /** 1d. Periodically, we discount older stability information so that new + /* 1d. Periodically, we discount older stability information so that new * stability info counts more, and save the stability information to disk as * appropriate. */ if (time_to_downrate_stability < now) @@ -1395,7 +1408,7 @@ run_scheduled_events(time_t now) dns_init(); } - /** 2. Periodically, we consider force-uploading our descriptor + /* 2. Periodically, we consider force-uploading our descriptor * (if we've passed our internal checks). */ /** How often do we check whether part of our router info has changed in a @@ -1438,22 +1451,29 @@ run_scheduled_events(time_t now) /* If any networkstatus documents are no longer recent, we need to * update all the descriptors' running status. */ - /* purge obsolete entries */ - networkstatus_v2_list_clean(now); /* Remove dead routers. */ routerlist_remove_old_routers(); + } - /* Also, once per minute, check whether we want to download any - * networkstatus documents. - */ + /* 2c. Every minute (or every second if TestingTorNetwork), check + * whether we want to download any networkstatus documents. */ + +/* How often do we check whether we should download network status + * documents? */ +#define networkstatus_dl_check_interval(o) ((o)->TestingTorNetwork ? 1 : 60) + + if (!should_delay_dir_fetches(options, NULL) && + time_to_download_networkstatus < now) { + time_to_download_networkstatus = + now + networkstatus_dl_check_interval(options); update_networkstatus_downloads(now); } - /** 2c. Let directory voting happen. */ + /* 2c. Let directory voting happen. */ if (authdir_mode_v3(options)) dirvote_act(options, now); - /** 3a. Every second, we examine pending circuits and prune the + /* 3a. Every second, we examine pending circuits and prune the * ones which have been pending for more than a few seconds. * We do this before step 4, so it can try building more if * it's not comfortable with the number of available circuits. @@ -1462,24 +1482,24 @@ run_scheduled_events(time_t now) * it can't, currently), we should do this more often.) */ circuit_expire_building(); - /** 3b. Also look at pending streams and prune the ones that 'began' + /* 3b. Also look at pending streams and prune the ones that 'began' * a long time ago but haven't gotten a 'connected' yet. * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ connection_ap_expire_beginning(); - /** 3c. And expire connections that we've held open for too long. + /* 3c. And expire connections that we've held open for too long. */ connection_expire_held_open(); - /** 3d. And every 60 seconds, we relaunch listeners if any died. */ + /* 3d. And every 60 seconds, we relaunch listeners if any died. */ if (!net_is_disabled() && time_to_check_listeners < now) { retry_all_listeners(NULL, NULL, 0); time_to_check_listeners = now+60; } - /** 4. Every second, we try a new circuit if there are no valid + /* 4. Every second, we try a new circuit if there are no valid * circuits. Every NewCircuitPeriod seconds, we expire circuits * that became dirty more than MaxCircuitDirtiness seconds ago, * and we make a new circ if there are no clean circuits. @@ -1492,7 +1512,7 @@ run_scheduled_events(time_t now) if (now % 10 == 5) circuit_expire_old_circuits_serverside(now); - /** 5. We do housekeeping for each connection... */ + /* 5. We do housekeeping for each connection... */ connection_or_set_bad_connections(NULL, 0); for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); @@ -1504,7 +1524,9 @@ run_scheduled_events(time_t now) if (conn->inbuf) buf_shrink(conn->inbuf); }); +#ifdef ENABLE_MEMPOOL clean_cell_pool(); +#endif /* ENABLE_MEMPOOL */ buf_shrink_freelists(0); /** How often do we check buffers and pools for empty space that can be * deallocated? */ @@ -1512,33 +1534,35 @@ run_scheduled_events(time_t now) time_to_shrink_memory = now + MEM_SHRINK_INTERVAL; } - /** 6. And remove any marked circuits... */ + /* 6. And remove any marked circuits... */ circuit_close_all_marked(); - /** 7. And upload service descriptors if necessary. */ + /* 7. And upload service descriptors if necessary. */ if (can_complete_circuit && !net_is_disabled()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } - /** 8. and blow away any connections that need to die. have to do this now, + /* 8. and blow away any connections that need to die. have to do this now, * because if we marked a conn for close and left its socket -1, then * we'll pass it to poll/select and bad things will happen. */ close_closeable_connections(); - /** 8b. And if anything in our state is ready to get flushed to disk, we + /* 8b. And if anything in our state is ready to get flushed to disk, we * flush it. */ or_state_save(now); - /** 8c. Do channel cleanup just like for connections */ + /* 8c. Do channel cleanup just like for connections */ channel_run_cleanup(); channel_listener_run_cleanup(); - /** 9. and if we're a server, check whether our DNS is telling stories to - * us. */ + /* 9. and if we're an exit node, check whether our DNS is telling stories + * to us. */ if (!net_is_disabled() && - public_server_mode(options) && time_to_check_for_correct_dns < now) { + public_server_mode(options) && + time_to_check_for_correct_dns < now && + ! router_my_exit_policy_is_reject_star()) { if (!time_to_check_for_correct_dns) { time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); } else { @@ -1548,7 +1572,7 @@ run_scheduled_events(time_t now) } } - /** 10. write bridge networkstatus file to disk */ + /* 10. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { networkstatus_dump_bridge_status_to_file(now); @@ -1556,7 +1580,7 @@ run_scheduled_events(time_t now) time_to_write_bridge_status_file = now+BRIDGE_STATUSFILE_INTERVAL; } - /** 11. check the port forwarding app */ + /* 11. check the port forwarding app */ if (!net_is_disabled() && time_to_check_port_forwarding < now && options->PortForwarding && @@ -1574,11 +1598,11 @@ run_scheduled_events(time_t now) time_to_check_port_forwarding = now+PORT_FORWARDING_CHECK_INTERVAL; } - /** 11b. check pending unconfigured managed proxies */ + /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); - /** 12. write the heartbeat message */ + /* 12. write the heartbeat message */ if (options->HeartbeatPeriod && time_to_next_heartbeat <= now) { if (time_to_next_heartbeat) /* don't log the first heartbeat */ @@ -1638,6 +1662,9 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); if (server_mode(options) && !net_is_disabled() && @@ -1649,24 +1676,28 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* every 20 minutes, check and complain if necessary */ const routerinfo_t *me = router_get_my_routerinfo(); if (me && !check_whether_orport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " "its ORPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->or_port); + address, me->or_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } if (me && !check_whether_dirport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG, "Your server (%s:%d) has not managed to confirm that its " "DirPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1728,7 +1759,7 @@ refill_callback(periodic_timer_t *timer, void *arg) accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over); if (milliseconds_elapsed > 0) - connection_bucket_refill(milliseconds_elapsed, now.tv_sec); + connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec); stats_prev_global_read_bucket = global_read_bucket; stats_prev_global_write_bucket = global_write_bucket; @@ -1865,7 +1896,7 @@ do_hup(void) } /** Tor main loop. */ -/* static */ int +int do_main_loop(void) { int loop_result; @@ -1900,8 +1931,10 @@ do_main_loop(void) } } +#ifdef ENABLE_MEMPOOLS /* Set up the packed_cell_t memory pool. */ init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ /* Set up our buckets */ connection_bucket_init(); @@ -1917,9 +1950,6 @@ do_main_loop(void) log_warn(LD_DIR, "Couldn't load all cached v3 certificates. Starting anyway."); } - if (router_reload_v2_networkstatus()) { - return -1; - } if (router_reload_consensus_networkstatus()) { return -1; } @@ -2097,8 +2127,8 @@ process_signal(uintptr_t sig) } /** Returns Tor's uptime. */ -long -get_uptime(void) +MOCK_IMPL(long, +get_uptime,(void)) { return stats_n_seconds_working; } @@ -2292,21 +2322,24 @@ handle_signals(int is_parent) /** Main entry point for the Tor command-line client. */ -/* static */ int +int tor_init(int argc, char *argv[]) { - char buf[256]; - int i, quiet = 0; + char progname[256]; + int quiet = 0; + time_of_process_start = time(NULL); - if (!connection_array) - connection_array = smartlist_new(); - if (!closeable_connection_lst) - closeable_connection_lst = smartlist_new(); - if (!active_linked_connection_lst) - active_linked_connection_lst = smartlist_new(); + init_connection_lists(); /* Have the log set up with our application name. */ - tor_snprintf(buf, sizeof(buf), "Tor %s", get_version()); - log_set_application_name(buf); + tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); + log_set_application_name(progname); + + /* Set up the crypto nice and early */ + if (crypto_early_init() < 0) { + log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); + return 1; + } + /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ @@ -2314,17 +2347,31 @@ tor_init(int argc, char *argv[]) addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + { /* We search for the "quiet" option first, since it decides whether we * will log anything at all to the command line. */ - for (i=1;i<argc;++i) { - if (!strcmp(argv[i], "--hush")) - quiet = 1; - if (!strcmp(argv[i], "--quiet")) - quiet = 2; - /* --version implies --quiet */ - if (!strcmp(argv[i], "--version")) - quiet = 2; + config_line_t *opts = NULL, *cmdline_opts = NULL; + const config_line_t *cl; + (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts); + for (cl = cmdline_opts; cl; cl = cl->next) { + if (!strcmp(cl->key, "--hush")) + quiet = 1; + if (!strcmp(cl->key, "--quiet") || + !strcmp(cl->key, "--dump-config")) + quiet = 2; + /* --version, --digests, and --help imply --hush */ + if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") || + !strcmp(cl->key, "--list-torrc-options") || + !strcmp(cl->key, "--library-versions") || + !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) { + if (quiet < 1) + quiet = 1; + } + } + config_free_lines(opts); + config_free_lines(cmdline_opts); } + /* give it somewhere to log to initially */ switch (quiet) { case 2: @@ -2346,11 +2393,12 @@ tor_init(int argc, char *argv[]) #else ""; #endif - log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s " - "and OpenSSL %s.", version, bev_str, + log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s, " + "OpenSSL %s and Zlib %s.", version, bev_str, get_uname(), tor_libevent_get_version_str(), - crypto_openssl_get_version_str()); + crypto_openssl_get_version_str(), + tor_zlib_get_version_str()); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -2390,6 +2438,9 @@ tor_init(int argc, char *argv[]) return -1; } stream_choice_seed_weak_rng(); + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + } return 0; } @@ -2492,15 +2543,22 @@ tor_free_all(int postfork) memarea_clear_freelist(); nodelist_free_all(); microdesc_free_all(); + ext_orport_free_all(); + control_free_all(); if (!postfork) { config_free_all(); or_state_free_all(); router_free_all(); policies_free_all(); } +#ifdef ENABLE_MEMPOOLS free_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ if (!postfork) { tor_tls_free_all(); +#ifndef _WIN32 + tor_getpwnam(NULL); +#endif } /* stuff in main.c */ @@ -2532,10 +2590,19 @@ tor_cleanup(void) time_t now = time(NULL); /* Remove our pid file. We don't care if there was an error when we * unlink, nothing we could do about it anyways. */ - if (options->PidFile) - unlink(options->PidFile); - if (options->ControlPortWriteToFile) - unlink(options->ControlPortWriteToFile); + if (options->PidFile) { + if (unlink(options->PidFile) != 0) { + log_warn(LD_FS, "Couldn't unlink pid file %s: %s", + options->PidFile, strerror(errno)); + } + } + if (options->ControlPortWriteToFile) { + if (unlink(options->ControlPortWriteToFile) != 0) { + log_warn(LD_FS, "Couldn't unlink control port file %s: %s", + options->ControlPortWriteToFile, + strerror(errno)); + } + } if (accounting_is_enabled(options)) accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ @@ -2558,7 +2625,7 @@ tor_cleanup(void) } /** Read/create keys as needed, and echo our fingerprint to stdout. */ -/* static */ int +static int do_list_fingerprint(void) { char buf[FINGERPRINT_LEN+1]; @@ -2566,7 +2633,7 @@ do_list_fingerprint(void) const char *nickname = get_options()->Nickname; if (!server_mode(get_options())) { log_err(LD_GENERAL, - "Clients don't have long-term identity keys. Exiting.\n"); + "Clients don't have long-term identity keys. Exiting."); return -1; } tor_assert(nickname); @@ -2588,7 +2655,7 @@ do_list_fingerprint(void) /** Entry point for password hashing: take the desired password from * the command line, and print its salted hash to stdout. **/ -/* static */ void +static void do_hash_password(void) { @@ -2604,6 +2671,34 @@ do_hash_password(void) printf("16:%s\n",output); } +/** Entry point for configuration dumping: write the configuration to + * stdout. */ +static int +do_dump_config(void) +{ + const or_options_t *options = get_options(); + const char *arg = options->command_arg; + int how; + char *opts; + if (!strcmp(arg, "short")) { + how = OPTIONS_DUMP_MINIMAL; + } else if (!strcmp(arg, "non-builtin")) { + how = OPTIONS_DUMP_DEFAULTS; + } else if (!strcmp(arg, "full")) { + how = OPTIONS_DUMP_ALL; + } else { + printf("%s is not a recognized argument to --dump-config. " + "Please select 'short', 'non-builtin', or 'full'", arg); + return -1; + } + + opts = options_dump(options, how); + printf("%s", opts); + tor_free(opts); + + return 0; +} + #if defined (WINCE) int find_flashcard_path(PWCHAR path, size_t size) @@ -2629,6 +2724,226 @@ find_flashcard_path(PWCHAR path, size_t size) } #endif +static void +init_addrinfo(void) +{ + char hname[256]; + + // host name to sandbox + gethostname(hname, sizeof(hname)); + sandbox_add_addrinfo(hname); +} + +static sandbox_cfg_t* +sandbox_init_filter(void) +{ + const or_options_t *options = get_options(); + sandbox_cfg_t *cfg = sandbox_cfg_new(); + int i; + + sandbox_cfg_allow_openat_filename(&cfg, + get_datadir_fname("cached-status")); + + sandbox_cfg_allow_open_filename_array(&cfg, + get_datadir_fname("cached-certs"), + get_datadir_fname("cached-certs.tmp"), + get_datadir_fname("cached-consensus"), + get_datadir_fname("cached-consensus.tmp"), + get_datadir_fname("unverified-consensus"), + get_datadir_fname("unverified-consensus.tmp"), + get_datadir_fname("unverified-microdesc-consensus"), + get_datadir_fname("unverified-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdesc-consensus"), + get_datadir_fname("cached-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdescs"), + get_datadir_fname("cached-microdescs.tmp"), + get_datadir_fname("cached-microdescs.new"), + get_datadir_fname("cached-microdescs.new.tmp"), + get_datadir_fname("cached-descriptors"), + get_datadir_fname("cached-descriptors.new"), + get_datadir_fname("cached-descriptors.tmp"), + get_datadir_fname("cached-descriptors.new.tmp"), + get_datadir_fname("cached-descriptors.tmp.tmp"), + get_datadir_fname("cached-extrainfo"), + get_datadir_fname("cached-extrainfo.new"), + get_datadir_fname("cached-extrainfo.tmp"), + get_datadir_fname("cached-extrainfo.new.tmp"), + get_datadir_fname("cached-extrainfo.tmp.tmp"), + get_datadir_fname("state.tmp"), + get_datadir_fname("unparseable-desc.tmp"), + get_datadir_fname("unparseable-desc"), + get_datadir_fname("v3-status-votes"), + get_datadir_fname("v3-status-votes.tmp"), + tor_strdup("/dev/srandom"), + tor_strdup("/dev/urandom"), + tor_strdup("/dev/random"), + tor_strdup("/etc/hosts"), + tor_strdup("/proc/meminfo"), + NULL, 0 + ); + if (options->ServerDNSResolvConfFile) + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->ServerDNSResolvConfFile)); + else + sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf")); + + for (i = 0; i < 2; ++i) { + if (get_torrc_fname(i)) { + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i))); + } + } + +#define RENAME_SUFFIX(name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname(name suffix), \ + get_datadir_fname(name)) + +#define RENAME_SUFFIX2(prefix, name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname2(prefix, name suffix), \ + get_datadir_fname2(prefix, name)) + + RENAME_SUFFIX("cached-certs", ".tmp"); + RENAME_SUFFIX("cached-consensus", ".tmp"); + RENAME_SUFFIX("unverified-consensus", ".tmp"); + RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".new"); + RENAME_SUFFIX("cached-microdescs.new", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".new"); + RENAME_SUFFIX("cached-descriptors.new", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".new"); + RENAME_SUFFIX("cached-extrainfo.new", ".tmp"); + RENAME_SUFFIX("state", ".tmp"); + RENAME_SUFFIX("unparseable-desc", ".tmp"); + RENAME_SUFFIX("v3-status-votes", ".tmp"); + + sandbox_cfg_allow_stat_filename_array(&cfg, + get_datadir_fname(NULL), + get_datadir_fname("lock"), + get_datadir_fname("state"), + get_datadir_fname("router-stability"), + get_datadir_fname("cached-extrainfo.new"), + NULL, 0 + ); + + { + smartlist_t *files = smartlist_new(); + tor_log_get_logfile_names(files); + SMARTLIST_FOREACH(files, char *, file_name, { + /* steals reference */ + sandbox_cfg_allow_open_filename(&cfg, file_name); + }); + smartlist_free(files); + } + + { + smartlist_t *files = smartlist_new(); + smartlist_t *dirs = smartlist_new(); + rend_services_add_filenames_to_lists(files, dirs); + SMARTLIST_FOREACH(files, char *, file_name, { + char *tmp_name = NULL; + tor_asprintf(&tmp_name, "%s.tmp", file_name); + sandbox_cfg_allow_rename(&cfg, + tor_strdup(tmp_name), tor_strdup(file_name)); + /* steals references */ + sandbox_cfg_allow_open_filename_array(&cfg, file_name, tmp_name, NULL); + }); + SMARTLIST_FOREACH(dirs, char *, dir, { + /* steals reference */ + sandbox_cfg_allow_stat_filename(&cfg, dir); + }); + smartlist_free(files); + smartlist_free(dirs); + } + + { + char *fname; + if ((fname = get_controller_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + if ((fname = get_ext_or_auth_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + } + + if (options->DirPortFrontPage) { + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->DirPortFrontPage)); + } + + // orport + if (server_mode(get_options())) { + sandbox_cfg_allow_open_filename_array(&cfg, + get_datadir_fname2("keys", "secret_id_key"), + get_datadir_fname2("keys", "secret_onion_key"), + get_datadir_fname2("keys", "secret_onion_key_ntor"), + get_datadir_fname2("keys", "secret_onion_key_ntor.tmp"), + get_datadir_fname2("keys", "secret_id_key.old"), + get_datadir_fname2("keys", "secret_onion_key.old"), + get_datadir_fname2("keys", "secret_onion_key_ntor.old"), + get_datadir_fname2("keys", "secret_onion_key.tmp"), + get_datadir_fname2("keys", "secret_id_key.tmp"), + get_datadir_fname2("stats", "bridge-stats"), + get_datadir_fname2("stats", "bridge-stats.tmp"), + get_datadir_fname2("stats", "dirreq-stats"), + get_datadir_fname2("stats", "dirreq-stats.tmp"), + get_datadir_fname2("stats", "entry-stats"), + get_datadir_fname2("stats", "entry-stats.tmp"), + get_datadir_fname2("stats", "exit-stats"), + get_datadir_fname2("stats", "exit-stats.tmp"), + get_datadir_fname2("stats", "buffer-stats"), + get_datadir_fname2("stats", "buffer-stats.tmp"), + get_datadir_fname2("stats", "conn-stats"), + get_datadir_fname2("stats", "conn-stats.tmp"), + get_datadir_fname("approved-routers"), + get_datadir_fname("fingerprint"), + get_datadir_fname("fingerprint.tmp"), + get_datadir_fname("hashed-fingerprint"), + get_datadir_fname("hashed-fingerprint.tmp"), + get_datadir_fname("router-stability"), + get_datadir_fname("router-stability.tmp"), + tor_strdup("/etc/resolv.conf"), + NULL, 0 + ); + + RENAME_SUFFIX("fingerprint", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key.old", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key.old", ".tmp"); + RENAME_SUFFIX2("stats", "bridge-stats", ".tmp"); + RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp"); + RENAME_SUFFIX2("stats", "entry-stats", ".tmp"); + RENAME_SUFFIX2("stats", "exit-stats", ".tmp"); + RENAME_SUFFIX2("stats", "buffer-stats", ".tmp"); + RENAME_SUFFIX2("stats", "conn-stats", ".tmp"); + RENAME_SUFFIX("hashed-fingerprint", ".tmp"); + RENAME_SUFFIX("router-stability", ".tmp"); + + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key"), + get_datadir_fname2("keys", "secret_onion_key.old")); + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key_ntor"), + get_datadir_fname2("keys", "secret_onion_key_ntor.old")); + + sandbox_cfg_allow_stat_filename_array(&cfg, + get_datadir_fname("keys"), + get_datadir_fname2("stats", "dirreq-stats"), + NULL, 0 + ); + } + + init_addrinfo(); + + return cfg; +} + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2675,6 +2990,8 @@ tor_main(int argc, char *argv[]) } #endif + configure_backtrace_handler(get_version()); + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); @@ -2695,6 +3012,22 @@ tor_main(int argc, char *argv[]) #endif if (tor_init(argc, argv)<0) return -1; + + if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { + sandbox_cfg_t* cfg = sandbox_init_filter(); + + if (sandbox_init(cfg)) { + log_err(LD_BUG,"Failed to create syscall sandbox filter"); + return -1; + } + + // registering libevent rng +#ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE + evutil_secure_rng_set_urandom_device_file( + (char*) sandbox_intern_string("/dev/urandom")); +#endif + } + switch (get_options()->command) { case CMD_RUN_TOR: #ifdef NT_SERVICE @@ -2713,6 +3046,9 @@ tor_main(int argc, char *argv[]) printf("Configuration was valid\n"); result = 0; break; + case CMD_DUMP_CONFIG: + result = do_dump_config(); + break; case CMD_RUN_UNITTESTS: /* only set by test.c */ default: log_warn(LD_BUG,"Illegal command number %d: internal error.", diff --git a/src/or/main.h b/src/or/main.h index 338449b6a..a3bce3486 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -24,8 +24,8 @@ void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -uint64_t get_bytes_read(void); -uint64_t get_bytes_written(void); +MOCK_DECL(uint64_t,get_bytes_read,(void)); +MOCK_DECL(uint64_t,get_bytes_written,(void)); /** Bitmask for events that we can turn on and off with * connection_watch_events. */ @@ -36,12 +36,12 @@ typedef enum watchable_events { } watchable_events_t; void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); +MOCK_DECL(void,connection_stop_reading,(connection_t *conn)); +MOCK_DECL(void,connection_start_reading,(connection_t *conn)); int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); +MOCK_DECL(void,connection_stop_writing,(connection_t *conn)); +MOCK_DECL(void,connection_start_writing,(connection_t *conn)); void connection_stop_reading_from_linked_conn(connection_t *conn); @@ -50,8 +50,10 @@ void directory_info_has_arrived(time_t now, int from_cache); void ip_address_changed(int at_interface); void dns_servers_relaunch_checks(void); +void reschedule_descriptor_update_check(void); + +MOCK_DECL(long,get_uptime,(void)); -long get_uptime(void); unsigned get_signewnym_epoch(void); void handle_signals(int is_parent); @@ -66,11 +68,12 @@ void tor_free_all(int postfork); int tor_main(int argc, char *argv[]); -#ifdef MAIN_PRIVATE int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); int tor_init(int argc, char **argv); + +#ifdef MAIN_PRIVATE +STATIC void init_connection_lists(void); +STATIC void close_closeable_connections(void); #endif #endif diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0e72c0b89..fdb549a9a 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -45,12 +45,7 @@ struct microdesc_cache_t { static INLINE unsigned int microdesc_hash_(microdesc_t *md) { - unsigned *d = (unsigned*)md->digest; -#if SIZEOF_INT == 4 - return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; -#else - return d[0] ^ d[1] ^ d[2] ^ d[3]; -#endif + return (unsigned) siphash24g(md->digest, sizeof(md->digest)); } /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ @@ -139,7 +134,7 @@ get_microdesc_cache(void) * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>, * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, * leave their bodies as pointers to the mmap'd cache. If where is - * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive, + * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1, * set the last_listed field of every microdesc to listed_at. If * requested_digests is non-null, then it contains a list of digests we mean * to allow, so we should reject any non-requested microdesc with a different @@ -158,7 +153,7 @@ microdescs_add_to_cache(microdesc_cache_t *cache, descriptors = microdescs_parse_from_string(s, eos, allow_annotations, where); - if (listed_at > 0) { + if (listed_at != (time_t)-1) { SMARTLIST_FOREACH(descriptors, microdesc_t *, md, md->last_listed = listed_at); } @@ -280,6 +275,7 @@ void microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); @@ -288,7 +284,13 @@ microdesc_cache_clear(microdesc_cache_t *cache) } HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { - tor_munmap_file(cache->cache_content); + int res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "tor_munmap_file() failed clearing microdesc cache; " + "we are probably about to leak memory."); + /* TODO something smarter? */ + } cache->cache_content = NULL; } cache->total_len_seen = 0; @@ -368,7 +370,9 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) cutoff = now - TOLERATE_MICRODESC_AGE; for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { - if ((*mdp)->last_listed < cutoff) { + const int is_old = (*mdp)->last_listed < cutoff; + const unsigned held_by_nodes = (*mdp)->held_by_nodes; + if (is_old && !held_by_nodes) { ++dropped; victim = *mdp; mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); @@ -376,6 +380,57 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) bytes_dropped += victim->bodylen; microdesc_free(victim); } else { + if (is_old) { + /* It's old, but it has held_by_nodes set. That's not okay. */ + /* Let's try to diagnose and fix #7164 . */ + smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp); + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + long networkstatus_age = -1; + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); + if (ns) { + networkstatus_age = now - ns->valid_after; + } + log_warn(LD_BUG, "Microdescriptor seemed very old " + "(last listed %d hours ago vs %d hour cutoff), but is still " + "marked as being held by %d node(s). I found %d node(s) " + "holding it. Current networkstatus is %ld hours old. " + "Hashtable badness is %d.", + (int)((now - (*mdp)->last_listed) / 3600), + (int)((now - cutoff) / 3600), + held_by_nodes, + smartlist_len(nodes), + networkstatus_age / 3600, + ht_badness); + + SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) { + const char *rs_match = "No RS"; + const char *rs_present = ""; + if (node->rs) { + if (tor_memeq(node->rs->descriptor_digest, + (*mdp)->digest, DIGEST256_LEN)) { + rs_match = "Microdesc digest in RS matches"; + } else { + rs_match = "Microdesc digest in RS does match"; + } + if (ns) { + /* This should be impossible, but let's see! */ + rs_present = " RS not present in networkstatus."; + SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, { + if (rs == node->rs) { + rs_present = " RS okay in networkstatus."; + } + }); + } + } + log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s", + node_sl_idx, + hex_str(node->identity, DIGEST_LEN), + node->md, node->rs, node->ri, rs_match, rs_present); + } SMARTLIST_FOREACH_END(node); + smartlist_free(nodes); + (*mdp)->last_listed = now; + } + ++kept; mdp = HT_NEXT(microdesc_map, &cache->map, mdp); } @@ -434,7 +489,7 @@ int microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; - int fd = -1; + int fd = -1, res; microdesc_t **mdp; smartlist_t *wrote; ssize_t size; @@ -489,7 +544,8 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) "By my count, I'm at "I64_FORMAT ", but I should be at "I64_FORMAT, I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real)); - off = off_real; + if (off_real >= 0) + off = off_real; } if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); @@ -500,8 +556,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) /* We must do this unmap _before_ we call finish_writing_to_file(), or * windows will not actually replace the file. */ - if (cache->cache_content) - tor_munmap_file(cache->cache_content); + if (cache->cache_content) { + res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "Failed to unmap old microdescriptor cache while rebuilding"); + } + cache->cache_content = NULL; + } if (finish_writing_to_file(open_file) < 0) { log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s", @@ -605,8 +667,10 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) tor_fragile_assert(); } if (md->held_by_nodes) { + microdesc_cache_t *cache = get_microdesc_cache(); int found=0; const smartlist_t *nodes = nodelist_get_list(); + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); SMARTLIST_FOREACH(nodes, node_t *, node, { if (node->md == md) { ++found; @@ -614,13 +678,14 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) } }); if (found) { - log_info(LD_BUG, "microdesc_free() called from %s:%d, but md was still " - "referenced %d node(s); held_by_nodes == %u", - fname, lineno, found, md->held_by_nodes); + log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still " + "referenced %d node(s); held_by_nodes == %u, ht_badness == %d", + fname, lineno, found, md->held_by_nodes, ht_badness); } else { log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes " - "set to %u, but md was not referenced by any nodes", - fname, lineno, md->held_by_nodes); + "set to %u, but md was not referenced by any nodes. " + "ht_badness == %d", + fname, lineno, md->held_by_nodes, ht_badness); } tor_fragile_assert(); } @@ -696,7 +761,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, continue; if (downloadable_only && !download_status_is_ready(&rs->dl_status, now, - MAX_MICRODESC_DOWNLOAD_FAILURES)) + get_options()->TestingMicrodescMaxDownloadTries)) continue; if (skip && digestmap_get(skip, rs->descriptor_digest)) continue; @@ -725,7 +790,7 @@ update_microdesc_downloads(time_t now) smartlist_t *missing; digestmap_t *pending; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (directory_too_idle_to_fetch_descriptors(options, now)) return; diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 23b7304b3..890da0ad1 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -10,6 +10,7 @@ * client or cache. */ +#define NETWORKSTATUS_PRIVATE #include "or.h" #include "channel.h" #include "circuitmux.h" @@ -31,18 +32,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" - -/* For tracking v2 networkstatus documents. Only caches do this now. */ - -/** Map from descriptor digest of routers listed in the v2 networkstatus - * documents to download_status_t* */ -static digestmap_t *v2_download_status_map = NULL; -/** Global list of all of the current v2 network_status documents that we know - * about. This list is kept sorted by published_on. */ -static smartlist_t *networkstatus_v2_list = NULL; -/** True iff any member of networkstatus_v2_list has changed since the last - * time we called download_status_map_update_from_v2_networkstatus() */ -static int networkstatus_v2_list_has_changed = 0; +#include "transports.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -88,11 +78,6 @@ typedef struct consensus_waiting_for_certs_t { static consensus_waiting_for_certs_t consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; -/** The last time we tried to download a networkstatus, or 0 for "never". We - * use this to rate-limit download attempts for directory caches (including - * mirrors). Clients don't use this now. */ -static time_t last_networkstatus_download_attempted = 0; - /** A time before which we shouldn't try to replace the current consensus: * this will be at some point after the next consensus becomes valid, but * before the current consensus becomes invalid. */ @@ -107,7 +92,6 @@ static int have_warned_about_old_version = 0; * listed by the authorities. */ static int have_warned_about_new_version = 0; -static void download_status_map_update_from_v2_networkstatus(void); static void routerstatus_list_update_named_server_map(void); /** Forget that we've warned about anything networkstatus-related, so we will @@ -131,86 +115,9 @@ void networkstatus_reset_download_failures(void) { int i; - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (!router_get_by_descriptor_digest(rs->descriptor_digest)) - rs->need_to_mirror = 1; - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); for (i=0; i < N_CONSENSUS_FLAVORS; ++i) download_status_reset(&consensus_dl_status[i]); - if (v2_download_status_map) { - digestmap_iter_t *iter; - digestmap_t *map = v2_download_status_map; - const char *key; - void *val; - download_status_t *dls; - for (iter = digestmap_iter_init(map); !digestmap_iter_done(iter); - iter = digestmap_iter_next(map, iter) ) { - digestmap_iter_get(iter, &key, &val); - dls = val; - download_status_reset(dls); - } - } -} - -/** Repopulate our list of network_status_t objects from the list cached on - * disk. Return 0 on success, -1 on failure. */ -int -router_reload_v2_networkstatus(void) -{ - smartlist_t *entries; - struct stat st; - char *s; - char *filename = get_datadir_fname("cached-status"); - int maybe_delete = !directory_caches_v2_dir_info(get_options()); - time_t now = time(NULL); - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - - entries = tor_listdir(filename); - if (!entries) { /* dir doesn't exist */ - tor_free(filename); - return 0; - } else if (!smartlist_len(entries) && maybe_delete) { - rmdir(filename); - tor_free(filename); - smartlist_free(entries); - return 0; - } - tor_free(filename); - SMARTLIST_FOREACH_BEGIN(entries, const char *, fn) { - char buf[DIGEST_LEN]; - if (maybe_delete) { - filename = get_datadir_fname2("cached-status", fn); - remove_file_if_very_old(filename, now); - tor_free(filename); - continue; - } - if (strlen(fn) != HEX_DIGEST_LEN || - base16_decode(buf, sizeof(buf), fn, strlen(fn))) { - log_info(LD_DIR, - "Skipping cached-status file with unexpected name \"%s\"",fn); - continue; - } - filename = get_datadir_fname2("cached-status", fn); - s = read_file_to_str(filename, 0, &st); - if (s) { - if (router_set_networkstatus_v2(s, st.st_mtime, NS_FROM_CACHE, - NULL)<0) { - log_warn(LD_FS, "Couldn't load networkstatus from \"%s\"",filename); - } - tor_free(s); - } - tor_free(filename); - } SMARTLIST_FOREACH_END(fn); - SMARTLIST_FOREACH(entries, char *, fn, tor_free(fn)); - smartlist_free(entries); - networkstatus_v2_list_clean(time(NULL)); - routers_update_all_from_networkstatus(time(NULL), 2); - return 0; } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -277,7 +184,7 @@ router_reload_consensus_networkstatus(void) } /** Free all storage held by the vote_routerstatus object <b>rs</b>. */ -static void +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs) { vote_microdesc_hash_t *h, *next; @@ -303,26 +210,6 @@ routerstatus_free(routerstatus_t *rs) tor_free(rs); } -/** Free all storage held by the networkstatus object <b>ns</b>. */ -void -networkstatus_v2_free(networkstatus_v2_t *ns) -{ - if (!ns) - return; - tor_free(ns->source_address); - tor_free(ns->contact); - if (ns->signing_key) - crypto_pk_free(ns->signing_key); - tor_free(ns->client_versions); - tor_free(ns->server_versions); - if (ns->entries) { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - routerstatus_free(rs)); - smartlist_free(ns->entries); - } - tor_free(ns); -} - /** Free all storage held in <b>sig</b> */ void document_signature_free(document_signature_t *sig) @@ -648,296 +535,10 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, return -2; } -/** Helper: return a newly allocated string containing the name of the filename - * where we plan to cache the network status with the given identity digest. */ -char * -networkstatus_get_cache_filename(const char *identity_digest) -{ - char fp[HEX_DIGEST_LEN+1]; - base16_encode(fp, HEX_DIGEST_LEN+1, identity_digest, DIGEST_LEN); - return get_datadir_fname2("cached-status", fp); -} - -/** Helper for smartlist_sort: Compare two networkstatus objects by - * publication date. */ -static int -compare_networkstatus_v2_published_on_(const void **_a, const void **_b) -{ - const networkstatus_v2_t *a = *_a, *b = *_b; - if (a->published_on < b->published_on) - return -1; - else if (a->published_on > b->published_on) - return 1; - else - return 0; -} - -/** Add the parsed v2 networkstatus in <b>ns</b> (with original document in - * <b>s</b>) to the disk cache (and the in-memory directory server cache) as - * appropriate. */ -static int -add_networkstatus_to_cache(const char *s, - v2_networkstatus_source_t source, - networkstatus_v2_t *ns) -{ - if (source != NS_FROM_CACHE) { - char *fn = networkstatus_get_cache_filename(ns->identity_digest); - if (write_str_to_file(fn, s, 0)<0) { - log_notice(LD_FS, "Couldn't write cached network status to \"%s\"", fn); - } - tor_free(fn); - } - - if (directory_caches_v2_dir_info(get_options())) - dirserv_set_cached_networkstatus_v2(s, - ns->identity_digest, - ns->published_on); - - return 0; -} - /** How far in the future do we allow a network-status to get before removing * it? (seconds) */ #define NETWORKSTATUS_ALLOW_SKEW (24*60*60) -/** Given a string <b>s</b> containing a network status that we received at - * <b>arrived_at</b> from <b>source</b>, try to parse it, see if we want to - * store it, and put it into our cache as necessary. - * - * If <b>source</b> is NS_FROM_DIR or NS_FROM_CACHE, do not replace our - * own networkstatus_t (if we're an authoritative directory server). - * - * If <b>source</b> is NS_FROM_CACHE, do not write our networkstatus_t to the - * cache. - * - * If <b>requested_fingerprints</b> is provided, it must contain a list of - * uppercased identity fingerprints. Do not update any networkstatus whose - * fingerprint is not on the list; after updating a networkstatus, remove its - * fingerprint from the list. - * - * Return 0 on success, -1 on failure. - * - * Callers should make sure that routers_update_all_from_networkstatus() is - * invoked after this function succeeds. - */ -int -router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints) -{ - networkstatus_v2_t *ns; - int i, found; - time_t now; - int skewed = 0; - dir_server_t *trusted_dir = NULL; - const char *source_desc = NULL; - char fp[HEX_DIGEST_LEN+1]; - char published[ISO_TIME_LEN+1]; - - if (!directory_caches_v2_dir_info(get_options())) - return 0; /* Don't bother storing it. */ - - ns = networkstatus_v2_parse_from_string(s); - if (!ns) { - log_warn(LD_DIR, "Couldn't parse network status."); - return -1; - } - base16_encode(fp, HEX_DIGEST_LEN+1, ns->identity_digest, DIGEST_LEN); - if (!(trusted_dir = - router_get_trusteddirserver_by_digest(ns->identity_digest)) || - !(trusted_dir->type & V2_DIRINFO)) { - log_info(LD_DIR, "Network status was signed, but not by an authoritative " - "directory we recognize."); - source_desc = fp; - } else { - source_desc = trusted_dir->description; - } - now = time(NULL); - if (arrived_at > now) - arrived_at = now; - - ns->received_on = arrived_at; - - format_iso_time(published, ns->published_on); - - if (ns->published_on > now + NETWORKSTATUS_ALLOW_SKEW) { - char dbuf[64]; - long delta = now - ns->published_on; - format_time_interval(dbuf, sizeof(dbuf), delta); - log_warn(LD_GENERAL, "Network status from %s was published %s in the " - "future (%s UTC). Check your time and date settings! " - "Not caching.", - source_desc, dbuf, published); - control_event_general_status(LOG_WARN, - "CLOCK_SKEW MIN_SKEW=%ld SOURCE=NETWORKSTATUS:%s:%d", - delta, ns->source_address, ns->source_dirport); - skewed = 1; - } - - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - - if ( (source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) && - router_digest_is_me(ns->identity_digest)) { - /* Don't replace our own networkstatus when we get it from somebody else.*/ - networkstatus_v2_free(ns); - return 0; - } - - if (requested_fingerprints) { - if (smartlist_contains_string(requested_fingerprints, fp)) { - smartlist_string_remove(requested_fingerprints, fp); - } else { - if (source != NS_FROM_DIR_ALL) { - char *requested = - smartlist_join_strings(requested_fingerprints," ",0,NULL); - log_warn(LD_DIR, - "We received a network status with a fingerprint (%s) that we " - "never requested. (We asked for: %s.) Dropping.", - fp, requested); - tor_free(requested); - return 0; - } - } - } - - if (!trusted_dir) { - if (!skewed) { - /* We got a non-trusted networkstatus, and we're a directory cache. - * This means that we asked an authority, and it told us about another - * authority we didn't recognize. */ - log_info(LD_DIR, - "We do not recognize authority (%s) but we are willing " - "to cache it.", fp); - add_networkstatus_to_cache(s, source, ns); - networkstatus_v2_free(ns); - } - return 0; - } - - found = 0; - for (i=0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *old_ns = smartlist_get(networkstatus_v2_list, i); - - if (tor_memeq(old_ns->identity_digest, ns->identity_digest, DIGEST_LEN)) { - if (tor_memeq(old_ns->networkstatus_digest, - ns->networkstatus_digest, DIGEST_LEN)) { - /* Same one we had before. */ - networkstatus_v2_free(ns); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s); " - "we already have it.", - trusted_dir->description, published); - if (old_ns->received_on < arrived_at) { - if (source != NS_FROM_CACHE) { - char *fn; - fn = networkstatus_get_cache_filename(old_ns->identity_digest); - /* We use mtime to tell when it arrived, so update that. */ - touch_file(fn); - tor_free(fn); - } - old_ns->received_on = arrived_at; - } - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else if (old_ns->published_on >= ns->published_on) { - char old_published[ISO_TIME_LEN+1]; - format_iso_time(old_published, old_ns->published_on); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s);" - " we have a newer one (published %s) for this authority.", - trusted_dir->description, published, - old_published); - networkstatus_v2_free(ns); - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else { - networkstatus_v2_free(old_ns); - smartlist_set(networkstatus_v2_list, i, ns); - found = 1; - break; - } - } - } - - if (source != NS_FROM_CACHE && trusted_dir) { - download_status_reset(&trusted_dir->v2_ns_dl_status); - } - - if (!found) - smartlist_add(networkstatus_v2_list, ns); - -/** Retain any routerinfo mentioned in a V2 networkstatus for at least this - * long. */ -#define V2_NETWORKSTATUS_ROUTER_LIFETIME (3*60*60) - - { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - signed_descriptor_t *sd = - router_get_by_descriptor_digest(rs->descriptor_digest); - if (sd) { - if (sd->last_listed_as_valid_until < live_until) - sd->last_listed_as_valid_until = live_until; - } else { - rs->need_to_mirror = 1; - } - } SMARTLIST_FOREACH_END(rs); - } - - log_info(LD_DIR, "Setting networkstatus %s %s (published %s)", - source == NS_FROM_CACHE?"cached from": - ((source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) ? - "downloaded from":"generated for"), - trusted_dir->description, published); - networkstatus_v2_list_has_changed = 1; - - smartlist_sort(networkstatus_v2_list, - compare_networkstatus_v2_published_on_); - - if (!skewed) - add_networkstatus_to_cache(s, source, ns); - - return 0; -} - -/** Remove all very-old network_status_t objects from memory and from the - * disk cache. */ -void -networkstatus_v2_list_clean(time_t now) -{ - int i; - if (!networkstatus_v2_list) - return; - - for (i = 0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - char *fname = NULL; - if (ns->published_on + MAX_NETWORKSTATUS_AGE > now) - continue; - /* Okay, this one is too old. Remove it from the list, and delete it - * from the cache. */ - smartlist_del(networkstatus_v2_list, i--); - fname = networkstatus_get_cache_filename(ns->identity_digest); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old networkstatus in %s", fname); - unlink(fname); - } - tor_free(fname); - if (directory_caches_v2_dir_info(get_options())) { - dirserv_set_cached_networkstatus_v2(NULL, ns->identity_digest, 0); - } - networkstatus_v2_free(ns); - } - - /* And now go through the directory cache for any cached untrusted - * networkstatuses and other network info. */ - dirserv_clear_old_networkstatuses(now - MAX_NETWORKSTATUS_AGE); - dirserv_clear_old_v1_info(now); -} - /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ int @@ -959,22 +560,6 @@ compare_digest_to_vote_routerstatus_entry(const void *_key, return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN); } -/** As networkstatus_v2_find_entry, but do not return a const pointer */ -routerstatus_t * -networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, const char *digest) -{ - return smartlist_bsearch(ns->entries, digest, - compare_digest_to_routerstatus_entry); -} - -/** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or - * NULL if none was found. */ -const routerstatus_t * -networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) -{ - return networkstatus_v2_find_mutable_entry(ns, digest); -} - /** As networkstatus_find_entry, but do not return a const pointer */ routerstatus_t * networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) @@ -1004,15 +589,6 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, found_out); } -/** Return a list of the v2 networkstatus documents. */ -const smartlist_t * -networkstatus_get_v2_list(void) -{ - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - return networkstatus_v2_list; -} - /** As router_get_consensus_status_by_descriptor_digest, but does not return * a const pointer. */ routerstatus_t * @@ -1057,8 +633,6 @@ router_get_dl_status_by_descriptor_digest(const char *d) if ((rs = router_get_mutable_consensus_status_by_descriptor_digest( current_ns_consensus, d))) return &rs->dl_status; - if (v2_download_status_map) - return digestmap_get(v2_download_status_map, d); return NULL; } @@ -1124,72 +698,6 @@ networkstatus_nickname_is_unnamed(const char *nickname) * networkstatus documents? */ #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60) -/** We are a directory server, and so cache network_status documents. - * Initiate downloads as needed to update them. For v2 authorities, - * this means asking each trusted directory for its network-status. - * For caches, this means asking a random v2 authority for all - * network-statuses. - */ -static void -update_v2_networkstatus_cache_downloads(time_t now) -{ - int authority = authdir_mode_v2(get_options()); - int interval = - authority ? AUTHORITY_NS_CACHE_INTERVAL : NONAUTHORITY_NS_CACHE_INTERVAL; - const smartlist_t *trusted_dir_servers = router_get_trusted_dir_servers(); - - if (last_networkstatus_download_attempted + interval >= now) - return; - - last_networkstatus_download_attempted = now; - - if (authority) { - /* An authority launches a separate connection for everybody. */ - SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) - { - char resource[HEX_DIGEST_LEN+6]; /* fp/hexdigit.z\0 */ - tor_addr_t addr; - if (!(ds->type & V2_DIRINFO)) - continue; - if (router_digest_is_me(ds->digest)) - continue; - tor_addr_from_ipv4h(&addr, ds->addr); - /* Is this quite sensible with IPv6 or multiple addresses? */ - if (connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, ds->dir_port, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) { - /* XXX the above dir_port won't be accurate if we're - * doing a tunneled conn. In that case it should be or_port. - * How to guess from here? Maybe make the function less general - * and have it know that it's looking for dir conns. -RD */ - /* Only directory caches download v2 networkstatuses, and they - * don't use tunneled connections. I think it's okay to ignore - * this. */ - continue; - } - strlcpy(resource, "fp/", sizeof(resource)); - base16_encode(resource+3, sizeof(resource)-3, ds->digest, DIGEST_LEN); - strlcat(resource, ".z", sizeof(resource)); - directory_initiate_command_routerstatus( - &ds->fake_status, DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, - DIRIND_ONEHOP, - resource, - NULL, 0 /* No payload. */, - 0 /* No I-M-S. */); - } - SMARTLIST_FOREACH_END(ds); - } else { - /* A non-authority cache launches one connection to a random authority. */ - /* (Check whether we're currently fetching network-status objects.) */ - if (!connection_get_by_type_purpose(CONN_TYPE_DIR, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) - directory_get_from_dirserver(DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, "all.z", - PDS_RETRY_IF_NO_SERVERS); - } -} - /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b> * is the flavor of a consensus networkstatus that we would like to fetch. */ static int @@ -1214,8 +722,6 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor) return flavor == usable_consensus_flavor(); } -/** How many times will we try to fetch a consensus before we give up? */ -#define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8 /** How long will we hang onto a possibly live consensus for which we're * fetching certs before we check whether there is a better one? */ #define DELAY_WHILE_FETCHING_CERTS (20*60) @@ -1249,7 +755,7 @@ update_consensus_networkstatus_downloads(time_t now) resource = networkstatus_get_flavor_name(i); if (!download_status_is_ready(&consensus_dl_status[i], now, - CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) + options->TestingConsensusMaxDownloadTries)) continue; /* We failed downloading a consensus too recently. */ if (connection_dir_get_by_purpose_and_resource( DIR_PURPOSE_FETCH_CONSENSUS, resource)) @@ -1324,7 +830,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) if (directory_fetches_dir_info_early(options)) { /* We want to cache the next one at some point after this one * is no longer fresh... */ - start = c->fresh_until + min_sec_before_caching; + start = (time_t)(c->fresh_until + min_sec_before_caching); /* Some clients may need the consensus sooner than others. */ if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { dl_interval = 60; @@ -1337,7 +843,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ - start = c->fresh_until + (interval*3)/4; + start = (time_t)(c->fresh_until + (interval*3)/4); /* But download the next one well before this one is expired. */ dl_interval = ((c->valid_until - start) * 7 )/ 8; @@ -1345,7 +851,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) * to choose the rest of the interval *after* them. */ if (directory_fetches_dir_info_later(options)) { /* Give all the *clients* enough time to download the consensus. */ - start = start + dl_interval + min_sec_before_caching; + start = (time_t)(start + dl_interval + min_sec_before_caching); /* But try to get it before ours actually expires. */ dl_interval = (c->valid_until - start) - min_sec_before_caching; } @@ -1391,14 +897,45 @@ update_consensus_networkstatus_fetch_time(time_t now) /** Return 1 if there's a reason we shouldn't try any directory * fetches yet (e.g. we demand bridges and none are yet known). - * Else return 0. */ + * Else return 0. + + * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b> + * to an explanation of why directory fetches are delayed. (If we + * return 0, we set msg_out to NULL.) + */ int -should_delay_dir_fetches(const or_options_t *options) +should_delay_dir_fetches(const or_options_t *options, const char **msg_out) { - if (options->UseBridges && !any_bridge_descriptors_known()) { - log_info(LD_DIR, "delaying dir fetches (no running bridges known)"); + if (msg_out) { + *msg_out = NULL; + } + + if (options->DisableNetwork) { + if (msg_out) { + *msg_out = "DisableNetwork is set."; + } + log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)"); return 1; } + + if (options->UseBridges) { + if (!any_bridge_descriptors_known()) { + if (msg_out) { + *msg_out = "No running bridges"; + } + log_info(LD_DIR, "Delaying dir fetches (no running bridges known)"); + return 1; + } + + if (pt_proxies_configuration_pending()) { + if (msg_out) { + *msg_out = "Pluggable transport proxies still configuring"; + } + log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)"); + return 1; + } + } + return 0; } @@ -1408,10 +945,8 @@ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; - if (authdir_mode_any_main(options) || options->FetchV2Networkstatus) - update_v2_networkstatus_cache_downloads(now); update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); } @@ -1518,7 +1053,6 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b) a->is_named != b->is_named || a->is_unnamed != b->is_unnamed || a->is_valid != b->is_valid || - a->is_v2_dir != b->is_v2_dir || a->is_possible_guard != b->is_possible_guard || a->is_bad_exit != b->is_bad_exit || a->is_bad_directory != b->is_bad_directory || @@ -1740,7 +1274,11 @@ networkstatus_set_current_consensus(const char *consensus, /* Even if we had enough signatures, we'd never use this as the * latest consensus. */ if (was_waiting_for_certs && from_cache) - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } goto done; } else { @@ -1750,8 +1288,13 @@ networkstatus_set_current_consensus(const char *consensus, "consensus"); result = -2; } - if (was_waiting_for_certs && (r < -1) && from_cache) - unlink(unverified_fname); + if (was_waiting_for_certs && (r < -1) && from_cache) { + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } + } goto done; } } @@ -1799,7 +1342,11 @@ networkstatus_set_current_consensus(const char *consensus, waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } /* Reset the failure count only if this consensus is actually valid. */ @@ -1835,7 +1382,8 @@ networkstatus_set_current_consensus(const char *consensus, * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); - circuit_build_times_new_consensus_params(&circ_times, current_consensus); + circuit_build_times_new_consensus_params(get_circuit_build_times_mutable(), + current_consensus); } if (directory_caches_dir_info(options)) { @@ -1912,9 +1460,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_NS); - if (networkstatus_v2_list_has_changed) - download_status_map_update_from_v2_networkstatus(); - if (!consensus || dir_version < 3) /* nothing more we should do */ return; @@ -1969,35 +1514,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) } } -/** Update v2_download_status_map to contain an entry for every router - * descriptor listed in the v2 networkstatuses. */ -static void -download_status_map_update_from_v2_networkstatus(void) -{ - digestmap_t *dl_status; - if (!networkstatus_v2_list) - return; - if (!v2_download_status_map) - v2_download_status_map = digestmap_new(); - - dl_status = digestmap_new(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, const routerstatus_t *, rs) { - const char *d = rs->descriptor_digest; - download_status_t *s; - if (digestmap_get(dl_status, d)) - continue; - if (!(s = digestmap_remove(v2_download_status_map, d))) { - s = tor_malloc_zero(sizeof(download_status_t)); - } - digestmap_set(dl_status, d, s); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = dl_status; - networkstatus_v2_list_has_changed = 0; -} - /** Update our view of the list of named servers from the most recently * retrieved networkstatus consensus. */ static void @@ -2029,14 +1545,11 @@ void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures) { - dir_server_t *ds; const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + int authdir = authdir_mode_v3(options); networkstatus_t *ns = current_consensus; if (!ns || !smartlist_len(ns->routerstatus_list)) return; - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); routers_sort_by_identity(routers); @@ -2046,11 +1559,6 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->cache_info.identity_digest, DIGEST_LEN), { }) { - /* We have a routerstatus for this router. */ - const char *digest = router->cache_info.identity_digest; - - ds = router_get_fallback_dirserver_by_digest(digest); - /* Is it the same descriptor, or only the same identity? */ if (tor_memeq(router->cache_info.signed_descriptor_digest, rs->descriptor_digest, DIGEST_LEN)) { @@ -2068,30 +1576,11 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, dirserv_should_launch_reachability_test(router, old_router); } } - if (rs->is_flagged_running && ds) { - download_status_reset(&ds->v2_ns_dl_status); - } if (reset_failures) { download_status_reset(&rs->dl_status); } } SMARTLIST_FOREACH_JOIN_END(rs, router); - /* Now update last_listed_as_valid_until from v2 networkstatuses. */ - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_JOIN(ns->entries, const routerstatus_t *, rs, - routers, routerinfo_t *, ri, - tor_memcmp(rs->identity_digest, - ri->cache_info.identity_digest, DIGEST_LEN), - STMT_NIL) { - if (tor_memeq(ri->cache_info.signed_descriptor_digest, - rs->descriptor_digest, DIGEST_LEN)) { - if (live_until > ri->cache_info.last_listed_as_valid_until) - ri->cache_info.last_listed_as_valid_until = live_until; - } - } SMARTLIST_FOREACH_JOIN_END(rs, ri); - } SMARTLIST_FOREACH_END(ns); - router_dir_info_changed(); } @@ -2183,9 +1672,17 @@ networkstatus_dump_bridge_status_to_file(time_t now) char *status = networkstatus_getinfo_by_purpose("bridge", now); const or_options_t *options = get_options(); char *fname = NULL; + char *thresholds = NULL, *thresholds_and_status = NULL; + routerlist_t *rl = router_get_routerlist(); + dirserv_compute_bridge_flag_thresholds(rl); + thresholds = dirserv_get_flag_thresholds_line(); + tor_asprintf(&thresholds_and_status, "flag-thresholds %s\n%s", + thresholds, status); tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges", options->DataDirectory); - write_str_to_file(fname,status,0); + write_str_to_file(fname,thresholds_and_status,0); + tor_free(thresholds); + tor_free(thresholds_and_status); tor_free(fname); tor_free(status); } @@ -2402,15 +1899,6 @@ void networkstatus_free_all(void) { int i; - if (networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - networkstatus_v2_free(ns)); - smartlist_free(networkstatus_v2_list); - networkstatus_v2_list = NULL; - } - - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = NULL; networkstatus_vote_free(current_ns_consensus); networkstatus_vote_free(current_md_consensus); current_md_consensus = current_ns_consensus = NULL; diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 761f8e7f0..be0a86cdd 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -12,16 +12,10 @@ #ifndef TOR_NETWORKSTATUS_H #define TOR_NETWORKSTATUS_H -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) - void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); void networkstatus_vote_free(networkstatus_t *ns); networkstatus_voter_info_t *networkstatus_get_voter_by_id( networkstatus_t *vote, @@ -31,26 +25,16 @@ int networkstatus_check_consensus_signature(networkstatus_t *consensus, int networkstatus_check_document_signature(const networkstatus_t *consensus, document_signature_t *sig, const authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest); -routerstatus_t *networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, - const char *digest); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); const routerstatus_t *router_get_consensus_status_by_id(const char *digest); routerstatus_t *router_get_mutable_consensus_status_by_id( @@ -69,7 +53,7 @@ int networkstatus_nickname_is_unnamed(const char *nickname); void networkstatus_consensus_download_failed(int status_code, const char *flavname); void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(const or_options_t *options); +int should_delay_dir_fetches(const or_options_t *options,const char **msg_out); void update_networkstatus_downloads(time_t now); void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); @@ -115,5 +99,9 @@ document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); int networkstatus_get_weight_scale_param(networkstatus_t *ns); +#ifdef NETWORKSTATUS_PRIVATE +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs); +#endif + #endif diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 178f084b6..8f870816d 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -25,6 +25,8 @@ static void nodelist_drop_node(node_t *node, int remove_from_ht); static void node_free(node_t *node); static void update_router_have_minimum_dir_info(void); +static double get_frac_paths_needed_for_circs(const or_options_t *options, + const networkstatus_t *ns); /** A nodelist_t holds a node_t object for every router we're "willing to use * for something". Specifically, it should hold a node_t for every node that @@ -41,14 +43,7 @@ typedef struct nodelist_t { static INLINE unsigned int node_id_hash(const node_t *node) { -#if SIZEOF_INT == 4 - const uint32_t *p = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; -#elif SIZEOF_INT == 8 - const uint64_t *p = (const uint32_t*)node->identity; - const uint32_t *p32 = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p32[4]; -#endif + return (unsigned) siphash24g(node->identity, DIGEST_LEN); } static INLINE unsigned int @@ -90,8 +85,8 @@ node_get_mutable_by_id(const char *identity_digest) /** Return the node_t whose identity is <b>identity_digest</b>, or NULL * if no such node exists. */ -const node_t * -node_get_by_id(const char *identity_digest) +MOCK_IMPL(const node_t *, +node_get_by_id,(const char *identity_digest)) { return node_get_mutable_by_id(identity_digest); } @@ -211,7 +206,7 @@ void nodelist_set_consensus(networkstatus_t *ns) { const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + int authdir = authdir_mode_v3(options); int client = !server_mode(options); init_nodelist(); @@ -337,6 +332,25 @@ nodelist_drop_node(node_t *node, int remove_from_ht) node->nodelist_idx = -1; } +/** Return a newly allocated smartlist of the nodes that have <b>md</b> as + * their microdescriptor. */ +smartlist_t * +nodelist_find_nodes_with_microdesc(const microdesc_t *md) +{ + smartlist_t *result = smartlist_new(); + + if (the_nodelist == NULL) + return result; + + SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) { + if (node->md == md) { + smartlist_add(result, node); + } + } SMARTLIST_FOREACH_END(node); + + return result; +} + /** Release storage held by <b>node</b> */ static void node_free(node_t *node) @@ -644,7 +658,7 @@ node_get_purpose(const node_t *node) /** Compute the verbose ("extended") nickname of <b>node</b> and store it * into the MAX_VERBOSE_NICKNAME_LEN+1 character buffer at - * <b>verbose_nickname_out</b> */ + * <b>verbose_name_out</b> */ void node_get_verbose_nickname(const node_t *node, char *verbose_name_out) @@ -660,6 +674,25 @@ node_get_verbose_nickname(const node_t *node, strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1); } +/** Compute the verbose ("extended") nickname of node with + * given <b>id_digest</b> and store it into the MAX_VERBOSE_NICKNAME_LEN+1 + * character buffer at <b>verbose_name_out</b> + * + * If node_get_by_id() returns NULL, base 16 encoding of + * <b>id_digest</b> is returned instead. */ +void +node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out) +{ + const node_t *node = node_get_by_id(id_digest); + if (!node) { + verbose_name_out[0] = '$'; + base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN); + } else { + node_get_verbose_nickname(node, verbose_name_out); + } +} + /** Return true iff it seems that <b>node</b> allows circuits to exit * through it directlry from the client. */ int @@ -771,7 +804,7 @@ void node_get_address_string(const node_t *node, char *buf, size_t len) { if (node->ri) { - strlcpy(buf, node->ri->address, len); + strlcpy(buf, fmt_addr32(node->ri->addr), len); } else if (node->rs) { tor_addr_t addr; tor_addr_from_ipv4h(&addr, node->rs->addr); @@ -1216,10 +1249,12 @@ router_set_status(const char *digest, int up) if (!up && node_is_me(node) && !net_is_disabled()) log_warn(LD_NET, "We just marked ourself as down. Are your external " "addresses reachable?"); + + if (bool_neq(node->is_running, up)) + router_dir_info_changed(); + node->is_running = up; } - - router_dir_info_changed(); } /** True iff, the last time we checked whether we had enough directory info @@ -1317,7 +1352,7 @@ count_usable_descriptors(int *num_present, int *num_usable, md ? "microdesc" : "desc", exit_only ? " exits" : "s"); } -/** Return an extimate of which fraction of usable paths through the Tor +/** Return an estimate of which fraction of usable paths through the Tor * network we have available for use. */ static double compute_frac_paths_available(const networkstatus_t *consensus, @@ -1329,9 +1364,10 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *mid = smartlist_new(); smartlist_t *exits = smartlist_new(); smartlist_t *myexits= smartlist_new(); - double f_guard, f_mid, f_exit, f_myexit; + smartlist_t *myexits_unflagged = smartlist_new(); + double f_guard, f_mid, f_exit, f_myexit, f_myexit_unflagged; int np, nu; /* Ignored */ - const int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, mid, consensus, options, now, NULL, 0); @@ -1350,20 +1386,42 @@ compute_frac_paths_available(const networkstatus_t *consensus, }); } + /* All nodes with exit flag */ count_usable_descriptors(&np, &nu, exits, consensus, options, now, NULL, 1); + /* All nodes with exit flag in ExitNodes option */ count_usable_descriptors(&np, &nu, myexits, consensus, options, now, options->ExitNodes, 1); + /* Now compute the nodes in the ExitNodes option where which we don't know + * what their exit policy is, or we know it permits something. */ + count_usable_descriptors(&np, &nu, myexits_unflagged, + consensus, options, now, + options->ExitNodes, 0); + SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { + if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) + SMARTLIST_DEL_CURRENT(myexits_unflagged, node); + } SMARTLIST_FOREACH_END(node); f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); f_mid = frac_nodes_with_descriptors(mid, WEIGHT_FOR_MID); f_exit = frac_nodes_with_descriptors(exits, WEIGHT_FOR_EXIT); f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); + f_myexit_unflagged= + frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); + + /* If our ExitNodes list has eliminated every possible Exit node, and there + * were some possible Exit nodes, then instead consider nodes that permit + * exiting to some ports. */ + if (smartlist_len(myexits) == 0 && + smartlist_len(myexits_unflagged)) { + f_myexit = f_myexit_unflagged; + } smartlist_free(guards); smartlist_free(mid); smartlist_free(exits); smartlist_free(myexits); + smartlist_free(myexits_unflagged); /* This is a tricky point here: we don't want to make it easy for a * directory to trickle exits to us until it learns which exits we have @@ -1372,13 +1430,14 @@ compute_frac_paths_available(const networkstatus_t *consensus, if (f_myexit < f_exit) f_exit = f_myexit; - tor_asprintf(status_out, - "%d%% of guards bw, " - "%d%% of midpoint bw, and " - "%d%% of exit bw", - (int)(f_guard*100), - (int)(f_mid*100), - (int)(f_exit*100)); + if (status_out) + tor_asprintf(status_out, + "%d%% of guards bw, " + "%d%% of midpoint bw, and " + "%d%% of exit bw", + (int)(f_guard*100), + (int)(f_mid*100), + (int)(f_exit*100)); return f_guard * f_mid * f_exit; } @@ -1391,19 +1450,19 @@ count_loading_descriptors_progress(void) { int num_present = 0, num_usable=0; time_t now = time(NULL); + const or_options_t *options = get_options(); const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); - double fraction; + double paths, fraction; if (!consensus) return 0; /* can't count descriptors if we have no list of them */ - count_usable_descriptors(&num_present, &num_usable, NULL, - consensus, get_options(), now, NULL, 0); + paths = compute_frac_paths_available(consensus, options, now, + &num_present, &num_usable, + NULL); - if (num_usable == 0) - return 0; /* don't div by 0 */ - fraction = num_present / (num_usable/4.); + fraction = paths / get_frac_paths_needed_for_circs(options,consensus); if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) @@ -1439,6 +1498,7 @@ update_router_have_minimum_dir_info(void) const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); int using_md; + const char *delay_fetches_msg = NULL; if (!consensus) { if (!networkstatus_get_latest_consensus()) @@ -1451,10 +1511,9 @@ update_router_have_minimum_dir_info(void) goto done; } - if (should_delay_dir_fetches(get_options())) { - log_notice(LD_DIR, "no known bridge descriptors running yet; stalling"); - strlcpy(dir_info_status, "No live bridge descriptors.", - sizeof(dir_info_status)); + if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) { + log_notice(LD_DIR, "Delaying directory fetches: %s", delay_fetches_msg); + strlcpy(dir_info_status, delay_fetches_msg, sizeof(dir_info_status)); res = 0; goto done; } diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8a4665a8b..8e719e012 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -17,7 +17,7 @@ } STMT_END node_t *node_get_mutable_by_id(const char *identity_digest); -const node_t *node_get_by_id(const char *identity_digest); +MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest)); const node_t *node_get_by_hex_id(const char *identity_digest); node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out); node_t *nodelist_add_microdesc(microdesc_t *md); @@ -26,6 +26,7 @@ void nodelist_set_consensus(networkstatus_t *ns); void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md); void nodelist_remove_routerinfo(routerinfo_t *ri); void nodelist_purge(void); +smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md); void nodelist_free_all(void); void nodelist_assert_ok(void); @@ -33,6 +34,8 @@ void nodelist_assert_ok(void); const node_t *node_get_by_nickname(const char *nickname, int warn_if_unnamed); void node_get_verbose_nickname(const node_t *node, char *verbose_name_out); +void node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out); int node_is_named(const node_t *node); int node_is_dir(const node_t *node); int node_has_descriptor(const node_t *node); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 8b67b8682..e84831404 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -3,7 +3,6 @@ * Copyright (c) 2007-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#define MAIN_PRIVATE #include "or.h" #include "config.h" #include "main.h" @@ -315,6 +314,7 @@ nt_service_main(void) case CMD_LIST_FINGERPRINT: case CMD_HASH_PASSWORD: case CMD_VERIFY_CONFIG: + case CMD_DUMP_CONFIG: log_err(LD_CONFIG, "Unsupported command (--list-fingerprint, " "--hash-password, or --verify-config) in NT service."); break; diff --git a/src/or/onion.c b/src/or/onion.c index 1a0bcf106..ae39f451f 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -22,7 +22,6 @@ #include "relay.h" #include "rephist.h" #include "router.h" -#include "tor_queue.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -59,7 +58,7 @@ static void onion_queue_entry_remove(onion_queue_t *victim); * MAX_ONIONSKIN_CHALLENGE/REPLY_LEN." Also, make sure that we can pass * over-large values via EXTEND2/EXTENDED2, for future-compatibility.*/ -/** Return true iff we have room to queue another oninoskin of type +/** Return true iff we have room to queue another onionskin of type * <b>type</b>. */ static int have_room_for_onionskin(uint16_t type) @@ -330,12 +329,14 @@ onion_queue_entry_remove(onion_queue_t *victim) void clear_pending_onions(void) { - onion_queue_t *victim; + onion_queue_t *victim, *next; int i; for (i=0; i<=MAX_ONION_HANDSHAKE_TYPE; i++) { - while ((victim = TOR_TAILQ_FIRST(&ol_list[i]))) { + for (victim = TOR_TAILQ_FIRST(&ol_list[i]); victim; victim = next) { + next = TOR_TAILQ_NEXT(victim,next); onion_queue_entry_remove(victim); } + tor_assert(TOR_TAILQ_EMPTY(&ol_list[i])); } memset(ol_entries, 0, sizeof(ol_entries)); } @@ -553,8 +554,10 @@ onion_skin_client_handshake(int type, switch (type) { case ONION_HANDSHAKE_TYPE_TAP: - if (reply_len != TAP_ONIONSKIN_REPLY_LEN) + if (reply_len != TAP_ONIONSKIN_REPLY_LEN) { + log_warn(LD_CIRC, "TAP reply was not of the correct length."); return -1; + } if (onion_skin_TAP_client_handshake(handshake_state->u.tap, (const char*)reply, (char *)keys_out, keys_out_len) < 0) @@ -564,8 +567,10 @@ onion_skin_client_handshake(int type, return 0; case ONION_HANDSHAKE_TYPE_FAST: - if (reply_len != CREATED_FAST_LEN) + if (reply_len != CREATED_FAST_LEN) { + log_warn(LD_CIRC, "CREATED_FAST reply was not of the correct length."); return -1; + } if (fast_client_handshake(handshake_state->u.fast, reply, keys_out, keys_out_len) < 0) return -1; @@ -574,8 +579,10 @@ onion_skin_client_handshake(int type, return 0; #ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: - if (reply_len < NTOR_REPLY_LEN) + if (reply_len < NTOR_REPLY_LEN) { + log_warn(LD_CIRC, "ntor reply was not of the correct length."); return -1; + } { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; uint8_t *keys_tmp = tor_malloc(keys_tmp_len); @@ -861,16 +868,19 @@ extend_cell_parse(extend_cell_t *cell_out, const uint8_t command, } case RELAY_COMMAND_EXTEND2: { - uint8_t n_specs = *payload, spectype, speclen; + uint8_t n_specs, spectype, speclen; int i; int found_ipv4 = 0, found_ipv6 = 0, found_id = 0; tor_addr_make_unspec(&cell_out->orport_ipv4.addr); tor_addr_make_unspec(&cell_out->orport_ipv6.addr); + if (payload_length == 0) + return -1; + cell_out->cell_type = RELAY_COMMAND_EXTEND2; - ++payload; + n_specs = *payload++; /* Parse the specifiers. We'll only take the first IPv4 and first IPv6 - * addres, and the node ID, and ignore everything else */ + * address, and the node ID, and ignore everything else */ for (i = 0; i < n_specs; ++i) { if (eop - payload < 2) return -1; diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index aa034a8bd..38b62decc 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -22,7 +22,7 @@ fast_handshake_state_free(fast_handshake_state_t *victim) tor_free(victim); } -/** Create the state needed to perform a CREATE_FAST hasnshake. Return 0 +/** Create the state needed to perform a CREATE_FAST handshake. Return 0 * on success, -1 on failure. */ int fast_onionskin_create(fast_handshake_state_t **handshake_state_out, @@ -104,6 +104,7 @@ fast_client_handshake(const fast_handshake_state_t *handshake_state, out_len = key_out_len+DIGEST_LEN; out = tor_malloc(out_len); if (crypto_expand_key_material_TAP(tmp, sizeof(tmp), out, out_len)) { + log_warn(LD_CIRC, "Failed to expand key material"); goto done; } if (tor_memneq(out, handshake_reply_out+DIGEST_LEN, DIGEST_LEN)) { diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index 9cf7d5dd6..ef501f69d 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -256,7 +256,7 @@ onion_skin_ntor_client_handshake( si += CURVE25519_OUTPUT_LEN; curve25519_handshake(si, &handshake_state->seckey_x, &handshake_state->pubkey_B); - bad |= safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN); + bad |= (safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN) << 1); si += CURVE25519_OUTPUT_LEN; APPEND(si, handshake_state->router_id, DIGEST_LEN); APPEND(si, handshake_state->pubkey_B.public_key, CURVE25519_PUBKEY_LEN); @@ -281,7 +281,7 @@ onion_skin_ntor_client_handshake( /* Compute auth */ h_tweak(s.auth, s.auth_input, sizeof(s.auth_input), T->t_mac); - bad |= tor_memneq(s.auth, auth_candidate, DIGEST256_LEN); + bad |= (tor_memneq(s.auth, auth_candidate, DIGEST256_LEN) << 2); crypto_expand_key_material_rfc5869_sha256( s.secret_input, sizeof(s.secret_input), @@ -290,6 +290,11 @@ onion_skin_ntor_client_handshake( key_out, key_out_len); memwipe(&s, 0, sizeof(s)); + + if (bad) { + log_warn(LD_PROTOCOL, "Invalid result from curve25519 handshake: %d", bad); + } + return bad ? -1 : 0; } diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 3782e75ab..9a9f374b9 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -194,8 +194,10 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, handshake_reply, DH_KEY_LEN, key_material, key_material_len); - if (len < 0) + if (len < 0) { + log_warn(LD_PROTOCOL,"DH computation failed."); goto err; + } if (tor_memneq(key_material, handshake_reply+DH_KEY_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ diff --git a/src/or/or.h b/src/or/or.h index 3eaf3447d..6aa6b59e8 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -99,6 +99,7 @@ #include "ht.h" #include "replaycache.h" #include "crypto_curve25519.h" +#include "tor_queue.h" /* These signals are defined to help handle_control_signal work. */ @@ -195,6 +196,7 @@ typedef enum { * and let it use any circuit ID it wants. */ CIRC_ID_TYPE_NEITHER=2 } circ_id_type_t; +#define circ_id_type_bitfield_t ENUM_BF(circ_id_type_t) #define CONN_TYPE_MIN_ 3 /** Type for sockets listening for OR connections. */ @@ -227,8 +229,14 @@ typedef enum { #define CONN_TYPE_AP_NATD_LISTENER 14 /** Type for sockets listening for DNS requests. */ #define CONN_TYPE_AP_DNS_LISTENER 15 -#define CONN_TYPE_MAX_ 15 -/* !!!! If CONN_TYPE_MAX_ is ever over 15, we must grow the type field in + +/** Type for connections from the Extended ORPort. */ +#define CONN_TYPE_EXT_OR 16 +/** Type for sockets listening for Extended ORPort connections. */ +#define CONN_TYPE_EXT_OR_LISTENER 17 + +#define CONN_TYPE_MAX_ 17 +/* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in * connection_t. */ /* Proxy client types */ @@ -238,7 +246,9 @@ typedef enum { #define PROXY_SOCKS5 3 /* !!!! If there is ever a PROXY_* type over 2, we must grow the proxy_type * field in or_connection_t */ -/* pluggable transports proxy type */ + +/* Pluggable transport proxy type. Don't use this in or_connection_t, + * instead use the actual underlying proxy type (see above). */ #define PROXY_PLUGGABLE 4 /* Proxy client handshake states */ @@ -306,6 +316,25 @@ typedef enum { #define OR_CONN_STATE_OPEN 8 #define OR_CONN_STATE_MAX_ 8 +/** States of the Extended ORPort protocol. Be careful before changing + * the numbers: they matter. */ +#define EXT_OR_CONN_STATE_MIN_ 1 +/** Extended ORPort authentication is waiting for the authentication + * type selected by the client. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1 +/** Extended ORPort authentication is waiting for the client nonce. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2 +/** Extended ORPort authentication is waiting for the client hash. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3 +#define EXT_OR_CONN_STATE_AUTH_MAX 3 +/** Authentication finished and the Extended ORPort is now accepting + * traffic. */ +#define EXT_OR_CONN_STATE_OPEN 4 +/** Extended ORPort is flushing its last messages and preparing to + * start accepting OR connections. */ +#define EXT_OR_CONN_STATE_FLUSHING 5 +#define EXT_OR_CONN_STATE_MAX_ 5 + #define EXIT_CONN_STATE_MIN_ 1 /** State for an exit connection: waiting for response from DNS farm. */ #define EXIT_CONN_STATE_RESOLVING 1 @@ -372,16 +401,10 @@ typedef enum { #define CONTROL_CONN_STATE_NEEDAUTH 2 #define CONTROL_CONN_STATE_MAX_ 2 -#define DIR_PURPOSE_MIN_ 3 -/** A connection to a directory server: download a rendezvous - * descriptor. */ -#define DIR_PURPOSE_FETCH_RENDDESC 3 -/** A connection to a directory server: set after a rendezvous +#define DIR_PURPOSE_MIN_ 4 +/** A connection to a directory server: set after a v2 rendezvous * descriptor is downloaded. */ -#define DIR_PURPOSE_HAS_FETCHED_RENDDESC 4 -/** A connection to a directory server: download one or more v2 - * network-status objects */ -#define DIR_PURPOSE_FETCH_V2_NETWORKSTATUS 5 +#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4 /** A connection to a directory server: download one or more server * descriptors. */ #define DIR_PURPOSE_FETCH_SERVERDESC 6 @@ -390,9 +413,6 @@ typedef enum { #define DIR_PURPOSE_FETCH_EXTRAINFO 7 /** A connection to a directory server: upload a server descriptor. */ #define DIR_PURPOSE_UPLOAD_DIR 8 -/** A connection to a directory server: upload a rendezvous - * descriptor. */ -#define DIR_PURPOSE_UPLOAD_RENDDESC 9 /** A connection to a directory server: upload a v3 networkstatus vote. */ #define DIR_PURPOSE_UPLOAD_VOTE 10 /** A connection to a directory server: upload a v3 consensus signature */ @@ -426,7 +446,6 @@ typedef enum { * directory server. */ #define DIR_PURPOSE_IS_UPLOAD(p) \ ((p)==DIR_PURPOSE_UPLOAD_DIR || \ - (p)==DIR_PURPOSE_UPLOAD_RENDDESC || \ (p)==DIR_PURPOSE_UPLOAD_VOTE || \ (p)==DIR_PURPOSE_UPLOAD_SIGNATURES) @@ -585,7 +604,8 @@ typedef enum { #define END_OR_CONN_REASON_NO_ROUTE 6 /* no route to host/net */ #define END_OR_CONN_REASON_IO_ERROR 7 /* read/write error */ #define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */ -#define END_OR_CONN_REASON_MISC 9 +#define END_OR_CONN_REASON_PT_MISSING 9 /* PT failed or not available */ +#define END_OR_CONN_REASON_MISC 10 /* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for * documentation of these. The values must match. */ @@ -823,9 +843,15 @@ typedef enum { /** Maximum number of queued cells on a circuit for which we are the * midpoint before we give up and kill it. This must be >= circwindow * to avoid killing innocent circuits, and >= circwindow*2 to give - * leaky-pipe a chance for being useful someday. + * leaky-pipe a chance of working someday. The ORCIRC_MAX_MIDDLE_KILL_THRESH + * ratio controls the margin of error between emitting a warning and + * killing the circuit. */ -#define ORCIRC_MAX_MIDDLE_CELLS (21*(CIRCWINDOW_START_MAX)/10) +#define ORCIRC_MAX_MIDDLE_CELLS (CIRCWINDOW_START_MAX*2) +/** Ratio of hard (circuit kill) to soft (warning) thresholds for the + * ORCIRC_MAX_MIDDLE_CELLS tests. + */ +#define ORCIRC_MAX_MIDDLE_KILL_THRESH (1.1f) /* Cell commands. These values are defined in tor-spec.txt. */ #define CELL_PADDING 0 @@ -846,6 +872,7 @@ typedef enum { #define CELL_AUTH_CHALLENGE 130 #define CELL_AUTHENTICATE 131 #define CELL_AUTHORIZE 132 +#define CELL_COMMAND_MAX_ 132 /** How long to test reachability before complaining to the user. */ #define TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT (20*60) @@ -1073,9 +1100,17 @@ typedef struct var_cell_t { uint8_t payload[FLEXIBLE_ARRAY_MEMBER]; } var_cell_t; +/** A parsed Extended ORPort message. */ +typedef struct ext_or_cmd_t { + uint16_t cmd; /** Command type */ + uint16_t len; /** Body length */ + char body[FLEXIBLE_ARRAY_MEMBER]; /** Message body */ +} ext_or_cmd_t; + /** A cell as packed for writing to the network. */ typedef struct packed_cell_t { - struct packed_cell_t *next; /**< Next cell queued on this circuit. */ + /** Next cell queued on this circuit. */ + TOR_SIMPLEQ_ENTRY(packed_cell_t) next; char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */ uint32_t inserted_time; /**< Time (in milliseconds since epoch, with high * bits truncated) when this cell was inserted. */ @@ -1084,8 +1119,8 @@ typedef struct packed_cell_t { /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { - packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ - packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ + /** Linked list of packed_cell_t*/ + TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head; int n; /**< The number of cells in the queue. */ } cell_queue_t; @@ -1139,7 +1174,7 @@ typedef struct connection_t { * *_CONNECTION_MAGIC. */ uint8_t state; /**< Current state of this connection. */ - unsigned int type:4; /**< What kind of connection is this? */ + unsigned int type:5; /**< What kind of connection is this? */ unsigned int purpose:5; /**< Only used for DIR and EXIT types currently. */ /* The next fields are all one-bit booleans. Some are only applicable to @@ -1223,6 +1258,14 @@ typedef struct connection_t { /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; + + /** Bytes read since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_read_conn_bw; + + /** Bytes written since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_written_conn_bw; } connection_t; /** Subtype of connection_t; used for a listener socket. */ @@ -1384,6 +1427,9 @@ typedef struct or_handshake_state_t { /**@}*/ } or_handshake_state_t; +/** Length of Extended ORPort connection identifier. */ +#define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ + /** Subtype of connection_t for an "OR connection" -- that is, one that speaks * cells over TLS. */ typedef struct or_connection_t { @@ -1392,6 +1438,20 @@ typedef struct or_connection_t { /** Hash of the public RSA key for the other side's identity key, or zeroes * if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; + + /** Extended ORPort connection identifier. */ + char *ext_or_conn_id; + /** This is the ClientHash value we expect to receive from the + * client during the Extended ORPort authentication protocol. We + * compute it upon receiving the ClientNoce from the client, and we + * compare it with the acual ClientHash value sent by the + * client. */ + char *ext_or_auth_correct_client_hash; + /** String carrying the name of the pluggable transport + * (e.g. "obfs2") that is obfuscating this connection. If no + * pluggable transports are used, it's NULL. */ + char *ext_or_transport; + char *nickname; /**< Nickname of OR on other side (if any). */ tor_tls_t *tls; /**< TLS connection state. */ @@ -1422,6 +1482,10 @@ typedef struct or_connection_t { unsigned int is_outgoing:1; unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */ unsigned int wide_circ_ids:1; + /** True iff this connection has had its bootstrap failure logged with + * control_event_bootstrap_problem. */ + unsigned int have_noted_bootstrap_problem:1; + uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ @@ -1449,6 +1513,12 @@ typedef struct or_connection_t { struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ + /** Last emptied read token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t read_emptied_time; + /** Last emptied write token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t write_emptied_time; } or_connection_t; /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) @@ -1619,6 +1689,7 @@ typedef enum { DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_source_t; +#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) /** Subtype of connection_t for an "directory connection" -- that is, an HTTP * connection to retrieve or serve directory material. */ @@ -1638,7 +1709,7 @@ typedef struct dir_connection_t { * "spooling" of directory material to the outbuf. Otherwise, we'd have * to append everything to the outbuf in one enormous chunk. */ /** What exactly are we spooling right now? */ - ENUM_BF(dir_spool_source_t) dir_spool_src : 3; + dir_spool_source_bitfield_t dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1811,12 +1882,13 @@ typedef enum { ADDR_POLICY_ACCEPT=1, ADDR_POLICY_REJECT=2, } addr_policy_action_t; +#define addr_policy_action_bitfield_t ENUM_BF(addr_policy_action_t) /** A reference-counted address policy rule. */ typedef struct addr_policy_t { int refcnt; /**< Reference count */ /** What to do when the policy matches.*/ - ENUM_BF(addr_policy_action_t) policy_type:2; + addr_policy_action_bitfield_t policy_type:2; unsigned int is_private:1; /**< True iff this is the pseudo-address, * "private". */ unsigned int is_canonical:1; /**< True iff this policy is the canonical @@ -1868,6 +1940,7 @@ typedef enum { */ SAVED_IN_JOURNAL } saved_location_t; +#define saved_location_bitfield_t ENUM_BF(saved_location_t) /** Enumeration: what kind of download schedule are we using for a given * object? */ @@ -1876,6 +1949,7 @@ typedef enum { DL_SCHED_CONSENSUS = 1, DL_SCHED_BRIDGE = 2, } download_schedule_t; +#define download_schedule_bitfield_t ENUM_BF(download_schedule_t) /** Information about our plans for retrying downloads for a downloadable * object. */ @@ -1884,7 +1958,7 @@ typedef struct download_status_t { * again? */ uint8_t n_download_failures; /**< Number of failures trying to download the * most recent descriptor. */ - ENUM_BF(download_schedule_t) schedule : 8; + download_schedule_bitfield_t schedule : 8; } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -1926,9 +2000,7 @@ typedef struct signed_descriptor_t { * routerlist->old_routers? -1 for none. */ int routerlist_index; /** The valid-until time of the most recent consensus that listed this - * descriptor, or a bit after the publication time of the most recent v2 - * networkstatus that listed it. 0 for "never listed in a consensus or - * status, so far as we know." */ + * descriptor. 0 for "never listed in a consensus, so far as we know." */ time_t last_listed_as_valid_until; /* If true, we do not ever try to save this object in the cache. */ unsigned int do_not_cache : 1; @@ -1947,7 +2019,6 @@ typedef int16_t country_t; /** Information about another onion router in the network. */ typedef struct { signed_descriptor_t cache_info; - char *address; /**< Location of OR: either a hostname or an IP address. */ char *nickname; /**< Human-readable OR name. */ uint32_t addr; /**< IPv4 address of OR, in host order. */ @@ -2065,10 +2136,6 @@ typedef struct routerstatus_t { unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another * router. */ unsigned int is_valid:1; /**< True iff this router isn't invalid. */ - unsigned int is_v2_dir:1; /**< True iff this router can serve directory - * information with v2 of the directory - * protocol. (All directory caches cache v1 - * directories.) */ unsigned int is_possible_guard:1; /**< True iff this router would be a good * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for @@ -2105,12 +2172,6 @@ typedef struct routerstatus_t { /* ---- The fields below aren't derived from the networkstatus; they * hold local information only. */ - /** True if we, as a directory mirror, want to download the corresponding - * routerinfo from the authority who gave us this routerstatus. (That is, - * if we don't have the routerinfo, and if we haven't already tried to get it - * from this authority.) Applies in v2 networkstatus document only. - */ - unsigned int need_to_mirror:1; time_t last_dir_503_at; /**< When did this router last tell us that it * was too busy to serve directory info? */ download_status_t dl_status; @@ -2152,7 +2213,7 @@ typedef struct microdesc_t { */ time_t last_listed; /** Where is this microdescriptor currently stored? */ - ENUM_BF(saved_location_t) saved_location : 3; + saved_location_bitfield_t saved_location : 3; /** If true, do not attempt to cache this microdescriptor on disk. */ unsigned int no_save : 1; /** If true, this microdesc has an entry in the microdesc_map */ @@ -2275,52 +2336,6 @@ typedef struct node_t { } node_t; -/** How many times will we try to download a router's descriptor before giving - * up? */ -#define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 - -/** How many times will we try to download a microdescriptor before giving - * up? */ -#define MAX_MICRODESC_DOWNLOAD_FAILURES 8 - -/** Contents of a v2 (non-consensus, non-vote) network status object. */ -typedef struct networkstatus_v2_t { - /** When did we receive the network-status document? */ - time_t received_on; - - /** What was the digest of the document? */ - char networkstatus_digest[DIGEST_LEN]; - - /* These fields come from the actual network-status document.*/ - time_t published_on; /**< Declared publication date. */ - - char *source_address; /**< Canonical directory server hostname. */ - uint32_t source_addr; /**< Canonical directory server IP. */ - uint16_t source_dirport; /**< Canonical directory server dirport. */ - - unsigned int binds_names:1; /**< True iff this directory server binds - * names. */ - unsigned int recommends_versions:1; /**< True iff this directory server - * recommends client and server software - * versions. */ - unsigned int lists_bad_exits:1; /**< True iff this directory server marks - * malfunctioning exits as bad. */ - /** True iff this directory server marks malfunctioning directories as - * bad. */ - unsigned int lists_bad_directories:1; - - char identity_digest[DIGEST_LEN]; /**< Digest of signing key. */ - char *contact; /**< How to contact directory admin? (may be NULL). */ - crypto_pk_t *signing_key; /**< Key used to sign this directory. */ - char *client_versions; /**< comma-separated list of recommended client - * versions. */ - char *server_versions; /**< comma-separated list of recommended server - * versions. */ - - smartlist_t *entries; /**< List of routerstatus_t*. This list is kept - * sorted by identity_digest. */ -} networkstatus_v2_t; - /** Linked list of microdesc hash lines for a single router in a directory * vote. */ @@ -2408,8 +2423,8 @@ typedef enum { /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - ENUM_BF(networkstatus_type_t) type : 8; /**< Vote, consensus, or opinion? */ - ENUM_BF(consensus_flavor_t) flavor : 8; /**< If a consensus, what kind? */ + networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor; /**< If a consensus, what kind? */ unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains * measured= bandwidth values. */ @@ -2492,10 +2507,6 @@ typedef struct desc_store_t { * filename for a temporary file when rebuilding the store, and .new to this * filename for the journal. */ const char *fname_base; - /** Alternative (obsolete) value for fname_base: if the file named by - * fname_base isn't present, we read from here instead, but we never write - * here. */ - const char *fname_alt_base; /** Human-readable description of what this store contains. */ const char *description; @@ -2572,9 +2583,6 @@ typedef struct authority_cert_t { uint32_t addr; /** This authority's directory port. */ uint16_t dir_port; - /** True iff this certificate was cross-certified by signing the identity - * key with the signing key. */ - uint8_t is_cross_certified; } authority_cert_t; /** Bitfield enum type listing types of information that directory authorities @@ -2588,15 +2596,8 @@ typedef struct authority_cert_t { */ typedef enum { NO_DIRINFO = 0, - /** Serves/signs v1 directory information: Big lists of routers, and short - * routerstatus documents. */ - V1_DIRINFO = 1 << 0, - /** Serves/signs v2 directory information: i.e. v2 networkstatus documents */ - V2_DIRINFO = 1 << 1, /** Serves/signs v3 directory information: votes, consensuses, certs */ V3_DIRINFO = 1 << 2, - /** Serves hidden service descriptors. */ - HIDSERV_DIRINFO = 1 << 3, /** Serves bridge descriptors. */ BRIDGE_DIRINFO = 1 << 4, /** Serves extrainfo documents. */ @@ -2724,6 +2725,19 @@ typedef struct { struct create_cell_t; +/** Entry in the cell stats list of a circuit; used only if CELL_STATS + * events are enabled. */ +typedef struct testing_cell_stats_entry_t { + uint8_t command; /**< cell command number. */ + /** Waiting time in centiseconds if this event is for a removed cell, + * or 0 if this event is for adding a cell to the queue. 22 bits can + * store more than 11 hours, enough to assume that a circuit with this + * delay would long have been closed. */ + unsigned int waiting_time:22; + unsigned int removed:1; /**< 0 for added to, 1 for removed from queue. */ + unsigned int exitward:1; /**< 0 for app-ward, 1 for exit-ward. */ +} testing_cell_stats_entry_t; + /** * A circuit is a path over the onion routing * network. Applications can connect to one end of the circuit, and can @@ -2785,6 +2799,16 @@ typedef struct circuit_t { * allowing n_streams to add any more cells. (OR circuit only.) */ unsigned int streams_blocked_on_p_chan : 1; + /** True iff we have queued a delete backwards on this circuit, but not put + * it on the output buffer. */ + unsigned int p_delete_pending : 1; + /** True iff we have queued a delete forwards on this circuit, but not put + * it on the output buffer. */ + unsigned int n_delete_pending : 1; + + /** True iff this circuit has received a DESTROY cell in either direction */ + unsigned int received_destroy : 1; + uint8_t state; /**< Current status of this circuit. */ uint8_t purpose; /**< Why are we creating this circuit? */ @@ -2798,6 +2822,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ struct create_cell_t *n_chan_create_cell; @@ -2839,7 +2866,8 @@ typedef struct circuit_t { /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; - struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + /** Next circuit in linked list of all circuits (global_circuitlist). */ + TOR_LIST_ENTRY(circuit_t) head; /** Next circuit in the doubly-linked ring of circuits waiting to add * cells to n_conn. NULL if we have no cells pending, or if we're not @@ -2849,6 +2877,11 @@ typedef struct circuit_t { * cells to n_conn. NULL if we have no cells pending, or if we're not * linked to an OR connection. */ struct circuit_t *prev_active_on_n_chan; + + /** Various statistics about cells being added to or removed from this + * circuit's queues; used only if CELL_STATS events are enabled and + * cleared after being sent to control port. */ + smartlist_t *testing_cell_stats; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2910,6 +2943,7 @@ typedef enum { */ PATH_STATE_ALREADY_COUNTED = 6, } path_state_t; +#define path_state_bitfield_t ENUM_BF(path_state_t) /** An origin_circuit_t holds data necessary to build and use a circuit. */ @@ -2919,6 +2953,17 @@ typedef struct origin_circuit_t { /** Linked list of AP streams (or EXIT streams if hidden service) * associated with this circuit. */ edge_connection_t *p_streams; + + /** Bytes read from any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_read_circ_bw; + + /** Bytes written to any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_written_circ_bw; + /** Build state for this circuit. It includes the intended path * length, the chosen exit router, rendezvous information, etc. */ @@ -2949,7 +2994,7 @@ typedef struct origin_circuit_t { * circuit building and usage accounting. See path_state_t * for more details. */ - ENUM_BF(path_state_t) path_state : 3; + path_state_bitfield_t path_state : 3; /* If this flag is set, we should not consider attaching any more * connections to this circuit. */ @@ -3133,20 +3178,8 @@ typedef struct or_circuit_t { * is not marked for close. */ struct or_circuit_t *rend_splice; -#if REND_COOKIE_LEN >= DIGEST_LEN -#define REND_TOKEN_LEN REND_COOKIE_LEN -#else -#define REND_TOKEN_LEN DIGEST_LEN -#endif + struct or_circuit_rendinfo_s *rendinfo; - /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a - * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes - * otherwise. - * ???? move to a subtype or adjunct structure? Wastes 20 bytes. -NM - */ - char rend_token[REND_TOKEN_LEN]; - - /* ???? move to a subtype or adjunct structure? Wastes 20 bytes -NM */ /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ @@ -3165,28 +3198,66 @@ typedef struct or_circuit_t { * exit-ward queues of this circuit; reset every time when writing * buffer stats to disk. */ uint64_t total_cell_waiting_time; + + /** Maximum cell queue size for a middle relay; this is stored per circuit + * so append_cell_to_circuit_queue() can adjust it if it changes. If set + * to zero, it is initialized to the default value. + */ + uint32_t max_middle_cells; } or_circuit_t; +typedef struct or_circuit_rendinfo_s { + +#if REND_COOKIE_LEN != DIGEST_LEN +#error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN" +#endif +#define REND_TOKEN_LEN DIGEST_LEN + + /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a + * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes + * otherwise. + */ + char rend_token[REND_TOKEN_LEN]; + + /** True if this is a rendezvous point circuit; false if this is an + * introduction point. */ + unsigned is_rend_circ; + +} or_circuit_rendinfo_t; + /** Convert a circuit subtype to a circuit_t. */ #define TO_CIRCUIT(x) (&((x)->base_)) /** Convert a circuit_t* to a pointer to the enclosing or_circuit_t. Assert * if the cast is impossible. */ static or_circuit_t *TO_OR_CIRCUIT(circuit_t *); +static const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *); /** Convert a circuit_t* to a pointer to the enclosing origin_circuit_t. * Assert if the cast is impossible. */ static origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *); +static const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *); static INLINE or_circuit_t *TO_OR_CIRCUIT(circuit_t *x) { tor_assert(x->magic == OR_CIRCUIT_MAGIC); return DOWNCAST(or_circuit_t, x); } +static INLINE const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *x) +{ + tor_assert(x->magic == OR_CIRCUIT_MAGIC); + return DOWNCAST(or_circuit_t, x); +} static INLINE origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *x) { tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); return DOWNCAST(origin_circuit_t, x); } +static INLINE const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT( + const circuit_t *x) +{ + tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); + return DOWNCAST(origin_circuit_t, x); +} /** Bitfield type: things that we're willing to use invalid routers for. */ typedef enum invalid_router_usage_t { @@ -3323,9 +3394,9 @@ typedef struct { /** What should the tor process actually do? */ enum { CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD, - CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS + CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG } command; - const char *command_arg; /**< Argument for command-line option. */ + char *command_arg; /**< Argument for command-line option. */ config_line_t *Logs; /**< New-style list of configuration lines * for logs */ @@ -3406,10 +3477,21 @@ typedef struct { char *User; /**< Name of user to run Tor as. */ char *Group; /**< Name of group to run Tor as. */ config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */ + /** Ports to listen on for extended OR connections. */ + config_line_t *ExtORPort_lines; /** Ports to listen on for SOCKS connections. */ config_line_t *SocksPort_lines; /** Ports to listen on for transparent pf/netfilter connections. */ config_line_t *TransPort_lines; + const char *TransProxyType; /**< What kind of transparent proxy + * implementation are we using? */ + /** Parsed value of TransProxyType. */ + enum { + TPT_DEFAULT, + TPT_PF_DIVERT, + TPT_IPFW, + TPT_TPROXY, + } TransProxyType_parsed; config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd * connections. */ config_line_t *ControlPort_lines; /**< Ports to listen on for control @@ -3422,9 +3504,11 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + /* MaxMemInQueues value as input by the user. We clean this up to be + * MaxMemInQueues. */ + uint64_t MaxMemInQueues_raw; + uint64_t MaxMemInQueues;/**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ /** @name port booleans * @@ -3441,18 +3525,13 @@ typedef struct { unsigned int ControlPort_set : 1; unsigned int DirPort_set : 1; unsigned int DNSPort_set : 1; + unsigned int ExtORPort_set : 1; /**@}*/ int AssumeReachable; /**< Whether to publish our descriptor regardless. */ int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */ - int V1AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 1 directories? */ - int V2AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 2 directories? */ int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory * for version 3 directories? */ - int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory - * handle hidden service requests? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -3480,6 +3559,9 @@ typedef struct { /** List of TCP/IP addresses that transports should listen at. */ config_line_t *ServerTransportListenAddr; + /** List of options that must be passed to pluggable transports. */ + config_line_t *ServerTransportOptions; + int BridgeRelay; /**< Boolean: are we acting as a bridge relay? We make * this explicit so we can change how we behave in the * future. */ @@ -3500,8 +3582,6 @@ typedef struct { int PublishHidServDescriptors; int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */ int FetchHidServDescriptors; /**< and hidden service descriptors? */ - int FetchV2Networkstatus; /**< Do we fetch v2 networkstatus documents when - * we don't need to? */ int HidServDirectoryV2; /**< Do we participate in the HS DHT? */ int VoteOnHidServDirectoriesV2; /**< As a directory authority, vote on @@ -3592,6 +3672,10 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ + int PredictedPortsRelevanceTime; /** How long after we've requested a + * connection for a given port, do we want + * to continue to pick exits that support + * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -3655,9 +3739,6 @@ typedef struct { /** If set, use these bridge authorities and not the default one. */ config_line_t *AlternateBridgeAuthority; - /** If set, use these HS authorities and not the default ones. */ - config_line_t *AlternateHSAuthority; - char *MyFamily; /**< Declared family for this OR. */ config_line_t *NodeFamilies; /**< List of config lines for * node families */ @@ -3717,7 +3798,10 @@ typedef struct { int CookieAuthentication; /**< Boolean: do we enable cookie-based auth for * the control system? */ - char *CookieAuthFile; /**< Location of a cookie authentication file. */ + char *CookieAuthFile; /**< Filesystem location of a ControlPort + * authentication cookie. */ + char *ExtORPortCookieAuthFile; /**< Filesystem location of Extended + * ORPort authentication cookie. */ int CookieAuthFileGroupReadable; /**< Boolean: Is the CookieAuthFile g+r? */ int LeaveStreamsUnattached; /**< Boolean: Does Tor attach new streams to * circuits itself (0), or does it expect a controller @@ -3739,6 +3823,7 @@ typedef struct { SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE } SafeLogging_; + int Sandbox; /**< Boolean: should sandboxing be enabled? */ int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -3801,10 +3886,6 @@ typedef struct { * testing our DNS server. */ int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the * same network zone in the same circuit. */ - int TunnelDirConns; /**< If true, use BEGIN_DIR rather than BEGIN when - * possible. */ - int PreferTunneledDirConns; /**< If true, avoid dirservers that don't - * support BEGIN_DIR, when possible. */ int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically * forward the DirPort and ORPort on the NAT device */ char *PortForwardingHelper; /** < Filename or full path of the port @@ -3910,6 +3991,10 @@ typedef struct { * signatures. Only altered on testing networks.*/ int TestingV3AuthInitialDistDelay; + /** Offset in seconds added to the starting time for consensus + voting. Only altered on testing networks. */ + int TestingV3AuthVotingStartOffset; + /** If an authority has been around for less than this amount of time, it * does not believe its reachability information is accurate. Only * altered on testing networks. */ @@ -3920,6 +4005,51 @@ typedef struct { * networks. */ int TestingEstimatedDescriptorPropagationTime; + /** Schedule for when servers should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingServerDownloadSchedule; + + /** Schedule for when clients should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingClientDownloadSchedule; + + /** Schedule for when servers should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingServerConsensusDownloadSchedule; + + /** Schedule for when clients should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingClientConsensusDownloadSchedule; + + /** Schedule for when clients should download bridge descriptors. Only + * altered on testing networks. */ + smartlist_t *TestingBridgeDownloadSchedule; + + /** When directory clients have only a few descriptors to request, they + * batch them until they have more, or until this amount of time has + * passed. Only altered on testing networks. */ + int TestingClientMaxIntervalWithoutRequest; + + /** How long do we let a directory connection stall before expiring + * it? Only altered on testing networks. */ + int TestingDirConnectionMaxStall; + + /** How many times will we try to fetch a consensus before we give + * up? Only altered on testing networks. */ + int TestingConsensusMaxDownloadTries; + + /** How many times will we try to download a router's descriptor before + * giving up? Only altered on testing networks. */ + int TestingDescriptorMaxDownloadTries; + + /** How many times will we try to download a microdescriptor before + * giving up? Only altered on testing networks. */ + int TestingMicrodescMaxDownloadTries; + + /** How many times will we try to fetch a certificate before giving + * up? Only altered on testing networks. */ + int TestingCertMaxDownloadTries; + /** If true, we take part in a testing network. Change the defaults of a * couple of other configuration options and allow to change the values * of certain configuration options. */ @@ -3931,6 +4061,19 @@ typedef struct { /** Minimum value for the Fast flag threshold on testing networks. */ uint64_t TestingMinFastFlagThreshold; + /** Relays in a testing network which should be voted Guard + * regardless of uptime and bandwidth. */ + routerset_t *TestingDirAuthVoteGuard; + + /** Enable CONN_BW events. Only altered on testing networks. */ + int TestingEnableConnBwEvent; + + /** Enable CELL_STATS events. Only altered on testing networks. */ + int TestingEnableCellStatsEvent; + + /** Enable TB_EMPTY events. Only altered on testing networks. */ + int TestingEnableTbEmptyEvent; + /** If true, and we have GeoIP data, and we're a bridge, keep a per-country * count of how many client addresses have contacted us so that we can help * the bridge authority guess which countries have blocked access to us. */ @@ -4069,16 +4212,6 @@ typedef struct { /** Fraction: */ double PathsNeededToBuildCircuits; - /** Do we serve v2 directory info at all? This is a temporary option, since - * we'd like to disable v2 directory serving entirely, but we need a way to - * make it temporarily disableable, in order to do fast testing and be - * able to turn it back on if it turns out to be non-workable. - * - * XXXX025 Make this always-on, or always-off. Right now, it's only - * enableable for authorities. - */ - int DisableV2DirectoryInfo_; - /** What expiry time shall we place on our SSL certs? "0" means we * should guess a suitable value. */ int SSLKeyLifetime; @@ -4343,30 +4476,7 @@ typedef struct { int after_firsthop_idx; } network_liveness_t; -/** Structure for circuit build times history */ -typedef struct { - /** The circular array of recorded build times in milliseconds */ - build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; - /** Current index in the circuit_build_times circular array */ - int build_times_idx; - /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ - int total_build_times; - /** Information about the state of our local network connection */ - network_liveness_t liveness; - /** Last time we built a circuit. Used to decide to build new test circs */ - time_t last_circ_at; - /** "Minimum" value of our pareto distribution (actually mode) */ - build_time_t Xm; - /** alpha exponent for pareto dist. */ - double alpha; - /** Have we computed a timeout? */ - int have_computed_timeout; - /** The exact value for that timeout in milliseconds. Stored as a double - * to maintain precision from calculations to and from quantile value. */ - double timeout_ms; - /** How long we wait before actually closing the circuit. */ - double close_ms; -} circuit_build_times_t; +typedef struct circuit_build_times_s circuit_build_times_t; /********************************* config.c ***************************/ @@ -4403,6 +4513,7 @@ typedef enum { * did this remapping happen." */ ADDRMAPSRC_NONE } addressmap_entry_source_t; +#define addressmap_entry_source_bitfield_t ENUM_BF(addressmap_entry_source_t) /********************************* control.c ***************************/ @@ -4554,8 +4665,6 @@ typedef enum { GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, - /** We've served a v2 networkstatus consensus as a directory server. */ - GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; /** Indicates either a positive reply or a reason for rejectng a network * status request that will be included in geoip statistics. */ @@ -4613,11 +4722,6 @@ typedef struct microdesc_cache_t microdesc_cache_t; /********************************* networkstatus.c *********************/ -/** Location where we found a v2 networkstatus. */ -typedef enum { - NS_FROM_CACHE, NS_FROM_DIR_BY_FP, NS_FROM_DIR_ALL, NS_GENERATED -} v2_networkstatus_source_t; - /** Possible statuses of a version of Tor, given opinions from the directory * servers. */ typedef enum version_status_t { @@ -4768,9 +4872,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes; @@ -4818,8 +4922,6 @@ typedef struct dir_server_t { /** What kind of authority is this? (Bitfield.) */ dirinfo_type_t type; - download_status_t v2_ns_dl_status; /**< Status of downloading this server's - * v2 network status. */ time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ @@ -4868,8 +4970,6 @@ typedef struct dir_server_t { * node that's currently a guard. */ #define PDS_FOR_GUARD (1<<5) -#define PDS_PREFER_TUNNELED_DIR_CONNS_ (1<<16) - /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ typedef enum bandwidth_weight_rule_t { diff --git a/src/or/policies.c b/src/or/policies.c index be4da5506..8a91509a7 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -13,6 +13,7 @@ #include "dirserv.h" #include "nodelist.h" #include "policies.h" +#include "router.h" #include "routerparse.h" #include "geoip.h" #include "ht.h" @@ -438,7 +439,7 @@ validate_addr_policies(const or_options_t *options, char **msg) if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, options->IPv6Exit, - options->ExitPolicyRejectPrivate, NULL, + options->ExitPolicyRejectPrivate, 0, !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); @@ -482,10 +483,12 @@ validate_addr_policies(const or_options_t *options, char **msg) * Ignore port specifiers. */ static int -load_policy_from_option(config_line_t *config, smartlist_t **policy, +load_policy_from_option(config_line_t *config, const char *option_name, + smartlist_t **policy, int assume_action) { int r; + int killed_any_ports = 0; addr_policy_list_free(*policy); *policy = NULL; r = parse_addr_policy(config, policy, assume_action); @@ -504,9 +507,13 @@ load_policy_from_option(config_line_t *config, smartlist_t **policy, c = addr_policy_get_canonical_entry(&newp); SMARTLIST_REPLACE_CURRENT(*policy, n, c); addr_policy_free(n); + killed_any_ports = 1; } } SMARTLIST_FOREACH_END(n); } + if (killed_any_ports) { + log_warn(LD_CONFIG, "Ignoring ports in %s option.", option_name); + } return 0; } @@ -516,20 +523,22 @@ int policies_parse_from_options(const or_options_t *options) { int ret = 0; - if (load_policy_from_option(options->SocksPolicy, &socks_policy, -1) < 0) + if (load_policy_from_option(options->SocksPolicy, "SocksPolicy", + &socks_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->DirPolicy, &dir_policy, -1) < 0) + if (load_policy_from_option(options->DirPolicy, "DirPolicy", + &dir_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirReject, + if (load_policy_from_option(options->AuthDirReject, "AuthDirReject", &authdir_reject_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirInvalid, + if (load_policy_from_option(options->AuthDirInvalid, "AuthDirInvalid", &authdir_invalid_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadDir, + if (load_policy_from_option(options->AuthDirBadDir, "AuthDirBadDir", &authdir_baddir_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadExit, + if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; if (parse_reachable_addresses() < 0) @@ -597,21 +606,25 @@ policy_eq(policy_map_ent_t *a, policy_map_ent_t *b) /** Return a hashcode for <b>ent</b> */ static unsigned int -policy_hash(policy_map_ent_t *ent) +policy_hash(const policy_map_ent_t *ent) { - addr_policy_t *a = ent->policy; - unsigned int r; - if (a->is_private) - r = 0x1234abcd; - else - r = tor_addr_hash(&a->addr); - r += a->prt_min << 8; - r += a->prt_max << 16; - r += a->maskbits; - if (a->policy_type == ADDR_POLICY_REJECT) - r ^= 0xffffffff; + const addr_policy_t *a = ent->policy; + addr_policy_t aa; + memset(&aa, 0, sizeof(aa)); + + aa.prt_min = a->prt_min; + aa.prt_max = a->prt_max; + aa.maskbits = a->maskbits; + aa.policy_type = a->policy_type; + aa.is_private = a->is_private; + + if (a->is_private) { + aa.is_private = 1; + } else { + tor_addr_copy_tight(&aa.addr, &a->addr); + } - return r; + return (unsigned) siphash24g(&aa, sizeof(aa)); } HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash, @@ -958,7 +971,7 @@ exit_policy_remove_redundancies(smartlist_t *dest) int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6_exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy) { if (!ipv6_exit) { @@ -968,7 +981,7 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, append_exit_policy_string(dest, "reject private:*"); if (local_address) { char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject %s:*", local_address); + tor_snprintf(buf, sizeof(buf), "reject %s:*", fmt_addr32(local_address)); append_exit_policy_string(dest, buf); } } @@ -1680,6 +1693,28 @@ getinfo_helper_policies(control_connection_t *conn, (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); + } else if (!strcmpstart(question, "exit-policy/")) { + const routerinfo_t *me = router_get_my_routerinfo(); + + int include_ipv4 = 0; + int include_ipv6 = 0; + + if (!strcmp(question, "exit-policy/ipv4")) { + include_ipv4 = 1; + } else if (!strcmp(question, "exit-policy/ipv6")) { + include_ipv6 = 1; + } else if (!strcmp(question, "exit-policy/full")) { + include_ipv4 = include_ipv6 = 1; + } else { + return 0; /* No such key. */ + } + + if (!me) { + *errmsg = "router_get_my_routerinfo returned NULL"; + return -1; + } + + *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); } return 0; } diff --git a/src/or/policies.h b/src/or/policies.h index facbbb6b5..91ac42749 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -45,7 +45,7 @@ addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, diff --git a/src/or/reasons.c b/src/or/reasons.c index 0674474e7..750e89bbe 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -231,6 +231,8 @@ orconn_end_reason_to_control_string(int r) return "RESOURCELIMIT"; case END_OR_CONN_REASON_MISC: return "MISC"; + case END_OR_CONN_REASON_PT_MISSING: + return "PT_MISSING"; case 0: return ""; default: diff --git a/src/or/relay.c b/src/or/relay.c index 7f06c6e14..5c430a6d7 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -15,6 +15,7 @@ #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,7 +26,9 @@ #include "control.h" #include "geoip.h" #include "main.h" +#ifdef ENABLE_MEMPOOLS #include "mempool.h" +#endif #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -58,6 +61,9 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ, entry_connection_t *conn, node_t *node, const tor_addr_t *addr); +#if 0 +static int get_max_middle_cells(void); +#endif /** Stop reading on edge connections when we have this many cells * waiting on the appropriate queue. */ @@ -968,7 +974,7 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to * the ttl of that address, in seconds, and return 0. On failure, return * -1. */ -int +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out) { @@ -1005,6 +1011,254 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell, return 0; } +/** Drop all storage held by <b>addr</b>. */ +STATIC void +address_ttl_free(address_ttl_t *addr) +{ + if (!addr) + return; + tor_free(addr->hostname); + tor_free(addr); +} + +/** Parse a resolved cell in <b>cell</b>, with parsed header in <b>rh</b>. + * Return -1 on parse error. On success, add one or more newly allocated + * address_ttl_t to <b>addresses_out</b>; set *<b>errcode_out</b> to + * one of 0, RESOLVED_TYPE_ERROR, or RESOLVED_TYPE_ERROR_TRANSIENT, and + * return 0. */ +STATIC int +resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out) +{ + const uint8_t *cp; + uint8_t answer_type; + size_t answer_len; + address_ttl_t *addr; + size_t remaining; + int errcode = 0; + smartlist_t *addrs; + + tor_assert(cell); + tor_assert(rh); + tor_assert(addresses_out); + tor_assert(errcode_out); + + *errcode_out = 0; + + if (rh->length > RELAY_PAYLOAD_SIZE) + return -1; + + addrs = smartlist_new(); + + cp = cell->payload + RELAY_HEADER_SIZE; + + remaining = rh->length; + while (remaining) { + const uint8_t *cp_orig = cp; + if (remaining < 2) + goto err; + answer_type = *cp++; + answer_len = *cp++; + if (remaining < 2 + answer_len + 4) { + goto err; + } + if (answer_type == RESOLVED_TYPE_IPV4) { + if (answer_len != 4) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv4n(&addr->addr, get_uint32(cp)); + cp += 4; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_IPV6) { + if (answer_len != 16) + goto err; + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv6_bytes(&addr->addr, (const char*) cp); + cp += 16; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_HOSTNAME) { + if (answer_len == 0) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + addr->hostname = tor_memdup_nulterm(cp, answer_len); + cp += answer_len; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_ERROR_TRANSIENT || + answer_type == RESOLVED_TYPE_ERROR) { + errcode = answer_type; + /* Ignore the error contents */ + cp += answer_len + 4; + } else { + cp += answer_len + 4; + } + tor_assert(((ssize_t)remaining) >= (cp - cp_orig)); + remaining -= (cp - cp_orig); + } + + if (errcode && smartlist_len(addrs) == 0) { + /* Report an error only if there were no results. */ + *errcode_out = errcode; + } + + smartlist_add_all(addresses_out, addrs); + smartlist_free(addrs); + + return 0; + + err: + /* On parse error, don't report any results */ + SMARTLIST_FOREACH(addrs, address_ttl_t *, a, address_ttl_free(a)); + smartlist_free(addrs); + return -1; +} + +/** Helper for connection_edge_process_resolved_cell: given an error code, + * an entry_connection, and a list of address_ttl_t *, report the best answer + * to the entry_connection. */ +static void +connection_ap_handshake_socks_got_resolved_cell(entry_connection_t *conn, + int error_code, + smartlist_t *results) +{ + address_ttl_t *addr_ipv4 = NULL; + address_ttl_t *addr_ipv6 = NULL; + address_ttl_t *addr_hostname = NULL; + address_ttl_t *addr_best = NULL; + + /* If it's an error code, that's easy. */ + if (error_code) { + tor_assert(error_code == RESOLVED_TYPE_ERROR || + error_code == RESOLVED_TYPE_ERROR_TRANSIENT); + connection_ap_handshake_socks_resolved(conn, + error_code,0,NULL,-1,-1); + return; + } + + /* Get the first answer of each type. */ + SMARTLIST_FOREACH_BEGIN(results, address_ttl_t *, addr) { + if (addr->hostname) { + if (!addr_hostname) { + addr_hostname = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET) { + if (!addr_ipv4 && conn->ipv4_traffic_ok) { + addr_ipv4 = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET6) { + if (!addr_ipv6 && conn->ipv6_traffic_ok) { + addr_ipv6 = addr; + } + } + } SMARTLIST_FOREACH_END(addr); + + /* Now figure out which type we wanted to deliver. */ + if (conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { + if (addr_hostname) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_HOSTNAME, + strlen(addr_hostname->hostname), + (uint8_t*)addr_hostname->hostname, + addr_hostname->ttl,-1); + } else { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + } + return; + } + + if (conn->prefer_ipv6_traffic) { + addr_best = addr_ipv6 ? addr_ipv6 : addr_ipv4; + } else { + addr_best = addr_ipv4 ? addr_ipv4 : addr_ipv6; + } + + /* Now convert it to the ugly old interface */ + if (! addr_best) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + return; + } + + connection_ap_handshake_socks_resolved_addr(conn, + &addr_best->addr, + addr_best->ttl, + -1); + + remap_event_helper(conn, &addr_best->addr); +} + +/** Handle a RELAY_COMMAND_RESOLVED cell that we received on a non-open AP + * stream. */ +STATIC int +connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh) +{ + entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); + smartlist_t *resolved_addresses = NULL; + int errcode = 0; + + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " + "not in state resolve_wait. Dropping."); + return 0; + } + tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); + + resolved_addresses = smartlist_new(); + if (resolved_cell_parse(cell, rh, resolved_addresses, &errcode)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Dropping malformed 'resolved' cell"); + connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); + goto done; + } + + if (get_options()->ClientDNSRejectInternalAddresses) { + int orig_len = smartlist_len(resolved_addresses); + SMARTLIST_FOREACH_BEGIN(resolved_addresses, address_ttl_t *, addr) { + if (addr->hostname == NULL && tor_addr_is_internal(&addr->addr, 0)) { + log_info(LD_APP, "Got a resolved cell with answer %s; dropping that " + "answer.", + safe_str_client(fmt_addr(&addr->addr))); + address_ttl_free(addr); + SMARTLIST_DEL_CURRENT(resolved_addresses, addr); + } + } SMARTLIST_FOREACH_END(addr); + if (orig_len && smartlist_len(resolved_addresses) == 0) { + log_info(LD_APP, "Got a resolved cell with only private addresses; " + "dropping it."); + connection_ap_handshake_socks_resolved(entry_conn, + RESOLVED_TYPE_ERROR_TRANSIENT, + 0, NULL, 0, TIME_MAX); + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_TORPROTOCOL); + goto done; + } + } + + connection_ap_handshake_socks_got_resolved_cell(entry_conn, + errcode, + resolved_addresses); + + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_DONE | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + + done: + SMARTLIST_FOREACH(resolved_addresses, address_ttl_t *, addr, + address_ttl_free(addr)); + smartlist_free(resolved_addresses); + return 0; +} + /** An incoming relay cell has arrived from circuit <b>circ</b> to * stream <b>conn</b>. * @@ -1106,8 +1360,9 @@ connection_edge_process_relay_cell_not_open( break; case DIR_PURPOSE_FETCH_SERVERDESC: case DIR_PURPOSE_FETCH_MICRODESC: - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (TO_DIR_CONN(dirconn)->router_purpose == ROUTER_PURPOSE_GENERAL) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); break; } } @@ -1128,67 +1383,7 @@ connection_edge_process_relay_cell_not_open( } if (conn->base_.type == CONN_TYPE_AP && rh->command == RELAY_COMMAND_RESOLVED) { - int ttl; - int answer_len; - uint8_t answer_type; - entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); - if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { - log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " - "not in state resolve_wait. Dropping."); - return 0; - } - tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); - answer_len = cell->payload[RELAY_HEADER_SIZE+1]; - if (rh->length < 2 || answer_len+2>rh->length) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Dropping malformed 'resolved' cell"); - connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); - return 0; - } - answer_type = cell->payload[RELAY_HEADER_SIZE]; - if (rh->length >= answer_len+6) - ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+ - 2+answer_len)); - else - ttl = -1; - if (answer_type == RESOLVED_TYPE_IPV4 || - answer_type == RESOLVED_TYPE_IPV6) { - tor_addr_t addr; - if (decode_address_from_payload(&addr, cell->payload+RELAY_HEADER_SIZE, - rh->length) && - tor_addr_is_internal(&addr, 0) && - get_options()->ClientDNSRejectInternalAddresses) { - log_info(LD_APP,"Got a resolve with answer %s. Rejecting.", - fmt_addr(&addr)); - connection_ap_handshake_socks_resolved(entry_conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0, NULL, 0, TIME_MAX); - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_TORPROTOCOL); - return 0; - } - } - connection_ap_handshake_socks_resolved(entry_conn, - answer_type, - cell->payload[RELAY_HEADER_SIZE+1], /*answer_len*/ - cell->payload+RELAY_HEADER_SIZE+2, /*answer*/ - ttl, - -1); - if (answer_type == RESOLVED_TYPE_IPV4 && answer_len == 4) { - tor_addr_t addr; - tor_addr_from_ipv4n(&addr, - get_uint32(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } else if (answer_type == RESOLVED_TYPE_IPV6 && answer_len == 16) { - tor_addr_t addr; - tor_addr_from_ipv6_bytes(&addr, - (char*)(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_DONE | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return 0; + return connection_edge_process_resolved_cell(conn, cell, rh); } log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1497,7 +1692,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (layer_hint) { if (layer_hint->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL, "Unexpected sendme cell from exit relay. " "Closing circ."); return -END_CIRC_REASON_TORPROTOCOL; @@ -1509,7 +1705,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } else { if (circ->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&client_warn_ratelim, LOG_WARN, LD_PROTOCOL, "Unexpected sendme cell from client. " "Closing circ (window %d).", circ->package_window); @@ -2036,9 +2233,10 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) #define assert_cmux_ok_paranoid(chan) #endif -/** The total number of cells we have allocated from the memory pool. */ +/** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; +#ifdef ENABLE_MEMPOOLS /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; @@ -2050,8 +2248,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells (and insertion times if we measure - * cell statistics). */ +/** Free all storage used to hold cells (and insertion times/commands if we + * measure cell statistics and/or if CELL_STATS events are enabled). */ void free_cell_pool(void) { @@ -2070,26 +2268,46 @@ clean_cell_pool(void) mp_pool_clean(cell_pool, 0, 1); } +#define relay_alloc_cell() \ + mp_pool_get(cell_pool) +#define relay_free_cell(cell) \ + mp_pool_release(cell) + +#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD) + +#else /* !ENABLE_MEMPOOLS case */ + +#define relay_alloc_cell() \ + tor_malloc_zero(sizeof(packed_cell_t)) +#define relay_free_cell(cell) \ + tor_free(cell) + +#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t)) + +#endif /* ENABLE_MEMPOOLS */ + /** Release storage held by <b>cell</b>. */ static INLINE void packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; - mp_pool_release(cell); + relay_free_cell(cell); } /** Allocate and return a new packed_cell_t. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * packed_cell_new(void) { ++total_cells_allocated; - return mp_pool_get(cell_pool); + return relay_alloc_cell(); } /** Return a packed cell used outside by channel_t lower layer */ void packed_cell_free(packed_cell_t *cell) { + if (!cell) + return; packed_cell_free_unchecked(cell); } @@ -2101,7 +2319,7 @@ dump_cell_pool_usage(int severity) circuit_t *c; int n_circs = 0; int n_cells = 0; - for (c = circuit_get_global_list_(); c; c = c->next) { + TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { n_cells += c->n_chan_cells.n; if (!CIRCUIT_IS_ORIGIN(c)) n_cells += TO_OR_CIRCUIT(c)->p_chan_cells.n; @@ -2110,7 +2328,9 @@ dump_cell_pool_usage(int severity) tor_log(severity, LD_MM, "%d cells allocated on %d circuits. %d cells leaked.", n_cells, n_circs, (int)total_cells_allocated - n_cells); +#ifdef ENABLE_MEMPOOLS mp_pool_log_status(cell_pool, severity); +#endif } /** Allocate a new copy of packed <b>cell</b>. */ @@ -2119,7 +2339,6 @@ packed_cell_copy(const cell_t *cell, int wide_circ_ids) { packed_cell_t *c = packed_cell_new(); cell_pack(c, cell, wide_circ_ids); - c->next = NULL; return c; } @@ -2127,58 +2346,61 @@ packed_cell_copy(const cell_t *cell, int wide_circ_ids) void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) { - if (queue->tail) { - tor_assert(!queue->tail->next); - queue->tail->next = cell; - } else { - queue->head = cell; - } - queue->tail = cell; - cell->next = NULL; + TOR_SIMPLEQ_INSERT_TAIL(&queue->head, cell, next); ++queue->n; } -/** Append a newly allocated copy of <b>cell</b> to the end of <b>queue</b> */ +/** Append a newly allocated copy of <b>cell</b> to the end of the + * <b>exitward</b> (or app-ward) <b>queue</b> of <b>circ</b>. If + * <b>use_stats</b> is true, record statistics about the cell. + */ void -cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids) +cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats) { struct timeval now; packed_cell_t *copy = packed_cell_copy(cell, wide_circ_ids); - tor_gettimeofday_cached(&now); + (void)circ; + (void)exitward; + (void)use_stats; + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); cell_queue_append(queue, copy); } +/** Initialize <b>queue</b> as an empty cell queue. */ +void +cell_queue_init(cell_queue_t *queue) +{ + memset(queue, 0, sizeof(cell_queue_t)); + TOR_SIMPLEQ_INIT(&queue->head); +} + /** Remove and free every cell in <b>queue</b>. */ void cell_queue_clear(cell_queue_t *queue) { - packed_cell_t *cell, *next; - cell = queue->head; - while (cell) { - next = cell->next; + packed_cell_t *cell; + while ((cell = TOR_SIMPLEQ_FIRST(&queue->head))) { + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); packed_cell_free_unchecked(cell); - cell = next; } - queue->head = queue->tail = NULL; + TOR_SIMPLEQ_INIT(&queue->head); queue->n = 0; } /** Extract and return the cell at the head of <b>queue</b>; return NULL if * <b>queue</b> is empty. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * cell_queue_pop(cell_queue_t *queue) { - packed_cell_t *cell = queue->head; + packed_cell_t *cell = TOR_SIMPLEQ_FIRST(&queue->head); if (!cell) return NULL; - queue->head = cell->next; - if (cell == queue->tail) { - tor_assert(!queue->head); - queue->tail = NULL; - } + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); --queue->n; return cell; } @@ -2188,16 +2410,24 @@ cell_queue_pop(cell_queue_t *queue) size_t packed_cell_mem_cost(void) { - return sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD; + return RELAY_CELL_MEM_COST; +} + +/** DOCDOC */ +STATIC size_t +cell_queues_get_total_allocation(void) +{ + return total_cells_allocated * packed_cell_mem_cost(); } /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ -static int +STATIC int cell_queues_check_size(void) { - size_t alloc = total_cells_allocated * packed_cell_mem_cost(); - if (alloc >= get_options()->MaxMemInCellQueues) { + size_t alloc = cell_queues_get_total_allocation(); + alloc += buf_get_total_allocation(); + if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; } @@ -2252,14 +2482,18 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, assert_cmux_ok_paranoid(chan); } -/** Remove all circuits from the cmux on <b>chan</b>. */ +/** Remove all circuits from the cmux on <b>chan</b>. + * + * If <b>circuits_out</b> is non-NULL, add all detached circuits to + * <b>circuits_out</b>. + **/ void -channel_unlink_all_circuits(channel_t *chan) +channel_unlink_all_circuits(channel_t *chan, smartlist_t *circuits_out) { tor_assert(chan); tor_assert(chan->cmux); - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, circuits_out); chan->num_n_circuits = 0; chan->num_p_circuits = 0; } @@ -2318,6 +2552,17 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan, return n; } +/** Extract the command from a packed cell. */ +static uint8_t +packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids) +{ + if (wide_circ_ids) { + return get_uint8(cell->body+4); + } else { + return get_uint8(cell->body+2); + } +} + /** Pull as many cells as possible (but no more than <b>max</b>) from the * queue of the first active circuit on <b>chan</b>, and write them to * <b>chan</b>->outbuf. Return the number of cells written. Advance @@ -2327,7 +2572,7 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) { circuitmux_t *cmux = NULL; int n_flushed = 0; - cell_queue_t *queue; + cell_queue_t *queue, *destroy_queue=NULL; circuit_t *circ; or_circuit_t *or_circ; int streams_blocked; @@ -2340,7 +2585,18 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) /* Main loop: pick a circuit, send a cell, update the cmux */ while (n_flushed < max) { - circ = circuitmux_get_first_active_circuit(cmux); + circ = circuitmux_get_first_active_circuit(cmux, &destroy_queue); + if (destroy_queue) { + /* this code is duplicated from some of the logic below. Ugly! XXXX */ + tor_assert(destroy_queue->n > 0); + cell = cell_queue_pop(destroy_queue); + channel_write_packed_cell(chan, cell); + /* Update the cmux destroy counter */ + circuitmux_notify_xmit_destroy(cmux); + cell = NULL; + ++n_flushed; + continue; + } /* If it returns NULL, no cells left to send */ if (!circ) break; assert_cmux_ok_paranoid(chan); @@ -2366,15 +2622,33 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) cell = cell_queue_pop(queue); /* Calculate the exact time that this cell has spent in the queue. */ - if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + if (get_options()->CellStatistics || + get_options()->TestingEnableCellStatsEvent) { uint32_t msec_waiting; struct timeval tvnow; - or_circ = TO_OR_CIRCUIT(circ); tor_gettimeofday_cached(&tvnow); msec_waiting = ((uint32_t)tv_to_msec(&tvnow)) - cell->inserted_time; - or_circ->total_cell_waiting_time += msec_waiting; - or_circ->processed_cells++; + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + or_circ = TO_OR_CIRCUIT(circ); + or_circ->total_cell_waiting_time += msec_waiting; + or_circ->processed_cells++; + } + + if (get_options()->TestingEnableCellStatsEvent) { + uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids); + + testing_cell_stats_entry_t *ent = + tor_malloc_zero(sizeof(testing_cell_stats_entry_t)); + ent->command = command; + ent->waiting_time = msec_waiting / 10; + ent->removed = 1; + if (circ->n_chan == chan) + ent->exitward = 1; + if (!circ->testing_cell_stats) + circ->testing_cell_stats = smartlist_new(); + smartlist_add(circ->testing_cell_stats, ent); + } } /* If we just flushed our queue and this circuit is used for a @@ -2420,6 +2694,20 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) return n_flushed; } +#if 0 +/** Indicate the current preferred cap for middle circuits; zero disables + * the cap. Right now it's just a constant, ORCIRC_MAX_MIDDLE_CELLS, but + * the logic in append_cell_to_circuit_queue() is written to be correct + * if we want to base it on a consensus param or something that might change + * in the future. + */ +static int +get_max_middle_cells(void) +{ + return ORCIRC_MAX_MIDDLE_CELLS; +} +#endif + /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b> * transmitting in <b>direction</b>. */ void @@ -2430,11 +2718,16 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, or_circuit_t *orcirc = NULL; cell_queue_t *queue; int streams_blocked; +#if 0 + uint32_t tgt_max_middle_cells, p_len, n_len, tmp, hard_max_middle_cells; +#endif + int exitward; if (circ->marked_for_close) return; - if (direction == CELL_DIRECTION_OUT) { + exitward = (direction == CELL_DIRECTION_OUT); + if (exitward) { queue = &circ->n_chan_cells; streams_blocked = circ->streams_blocked_on_n_chan; } else { @@ -2451,28 +2744,82 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, if ((circ->n_chan != NULL) && CIRCUIT_IS_ORCIRC(circ)) { orcirc = TO_OR_CIRCUIT(circ); if (orcirc->p_chan) { - if (queue->n + 1 >= ORCIRC_MAX_MIDDLE_CELLS) { - /* Queueing this cell would put queue over the cap */ - log_warn(LD_CIRC, - "Got a cell exceeding the cap of %u in the %s direction " - "on middle circ ID %u on chan ID " U64_FORMAT - "; killing the circuit.", - ORCIRC_MAX_MIDDLE_CELLS, - (direction == CELL_DIRECTION_OUT) ? "n" : "p", - (direction == CELL_DIRECTION_OUT) ? - circ->n_circ_id : orcirc->p_circ_id, - U64_PRINTF_ARG( + /* We are a middle circuit if we have both n_chan and p_chan */ + /* We'll need to know the current preferred maximum */ + tgt_max_middle_cells = get_max_middle_cells(); + if (tgt_max_middle_cells > 0) { + /* Do we need to initialize middle_max_cells? */ + if (orcirc->max_middle_cells == 0) { + orcirc->max_middle_cells = tgt_max_middle_cells; + } else { + if (tgt_max_middle_cells > orcirc->max_middle_cells) { + /* If we want to increase the cap, we can do so right away */ + orcirc->max_middle_cells = tgt_max_middle_cells; + } else if (tgt_max_middle_cells < orcirc->max_middle_cells) { + /* + * If we're shrinking the cap, we can't shrink past either queue; + * compare tgt_max_middle_cells rather than tgt_max_middle_cells * + * ORCIRC_MAX_MIDDLE_KILL_THRESH so the queues don't shrink enough + * to generate spurious warnings, either. + */ + n_len = circ->n_chan_cells.n; + p_len = orcirc->p_chan_cells.n; + tmp = tgt_max_middle_cells; + if (tmp < n_len) tmp = n_len; + if (tmp < p_len) tmp = p_len; + orcirc->max_middle_cells = tmp; + } + /* else no change */ + } + } else { + /* tgt_max_middle_cells == 0 indicates we should disable the cap */ + orcirc->max_middle_cells = 0; + } + + /* Now we know orcirc->max_middle_cells is set correctly */ + if (orcirc->max_middle_cells > 0) { + hard_max_middle_cells = + (uint32_t)(((double)orcirc->max_middle_cells) * + ORCIRC_MAX_MIDDLE_KILL_THRESH); + + if ((unsigned)queue->n + 1 >= hard_max_middle_cells) { + /* Queueing this cell would put queue over the kill theshold */ + log_warn(LD_CIRC, + "Got a cell exceeding the hard cap of %u in the " + "%s direction on middle circ ID %u on chan ID " + U64_FORMAT "; killing the circuit.", + hard_max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", (direction == CELL_DIRECTION_OUT) ? - circ->n_chan->global_identifier : - orcirc->p_chan->global_identifier)); - circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); - return; + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); + return; + } else if ((unsigned)queue->n + 1 == orcirc->max_middle_cells) { + /* Only use ==, not >= for this test so we don't spam the log */ + log_warn(LD_CIRC, + "While trying to queue a cell, reached the soft cap of %u " + "in the %s direction on middle circ ID %u " + "on chan ID " U64_FORMAT ".", + orcirc->max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", + (direction == CELL_DIRECTION_OUT) ? + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + } } } } #endif - cell_queue_append_packed_copy(queue, cell, chan->wide_circ_ids); + cell_queue_append_packed_copy(circ, queue, exitward, cell, + chan->wide_circ_ids, 1); if (PREDICT_UNLIKELY(cell_queues_check_size())) { /* We ran the OOM handler */ diff --git a/src/or/relay.h b/src/or/relay.h index 1fef10a7d..479d474b3 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -42,24 +42,28 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +#ifdef ENABLE_MEMPOOLS void init_cell_pool(void); void free_cell_pool(void); void clean_cell_pool(void); +#endif /* ENABLE_MEMPOOLS */ void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); /* For channeltls.c */ void packed_cell_free(packed_cell_t *cell); +void cell_queue_init(cell_queue_t *queue); void cell_queue_clear(cell_queue_t *queue); void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids); +void cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats); void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); -void channel_unlink_all_circuits(channel_t *chan); +void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); int channel_flush_from_first_active_circuit(channel_t *chan, int max); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, @@ -75,11 +79,28 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); void stream_choice_seed_weak_rng(void); -#ifdef RELAY_PRIVATE int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t **layer_hint, char *recognized); -int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, + +#ifdef RELAY_PRIVATE +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); +/** An address-and-ttl tuple as yielded by resolved_cell_parse */ +typedef struct address_ttl_s { + tor_addr_t addr; + char *hostname; + int ttl; +} address_ttl_t; +STATIC void address_ttl_free(address_ttl_t *addr); +STATIC int resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out); +STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh); +STATIC packed_cell_t *packed_cell_new(void); +STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); +STATIC size_t cell_queues_get_total_allocation(void); +STATIC int cell_queues_check_size(void); #endif #endif diff --git a/src/or/rendclient.c b/src/or/rendclient.c index bb4bd9bfd..d42024010 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,7 @@ **/ #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,6 +26,7 @@ #include "router.h" #include "routerlist.h" #include "routerset.h" +#include "control.h" static extend_info_t *rend_client_get_random_intro_impl( const rend_cache_entry_t *rend_query, @@ -376,7 +378,7 @@ rend_client_close_other_intros(const char *onion_address) { circuit_t *c; /* abort parallel intro circs, if any */ - for (c = circuit_get_global_list_(); c; c = c->next) { + TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { if ((c->purpose == CIRCUIT_PURPOSE_C_INTRODUCING || c->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) && !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) { @@ -617,11 +619,14 @@ static int directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) { smartlist_t *responsible_dirs = smartlist_new(); + smartlist_t *usable_responsible_dirs = smartlist_new(); + const or_options_t *options = get_options(); routerstatus_t *hs_dir; char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; time_t now = time(NULL); char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; - int tor2web_mode = get_options()->Tor2webMode; + const int tor2web_mode = options->Tor2webMode; + int excluded_some; tor_assert(desc_id); tor_assert(rend_query); /* Determine responsible dirs. Even if we can't get all we want, @@ -642,16 +647,33 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) dir, desc_id_base32, rend_query, 0, 0); const node_t *node = node_get_by_id(dir->identity_digest); if (last + REND_HID_SERV_DIR_REQUERY_PERIOD >= now || - !node || !node_has_descriptor(node)) - SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + !node || !node_has_descriptor(node)) { + SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + continue; + } + if (! routerset_contains_node(options->ExcludeNodes, node)) { + smartlist_add(usable_responsible_dirs, dir); + } }); - hs_dir = smartlist_choose(responsible_dirs); + excluded_some = + smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); + + hs_dir = smartlist_choose(usable_responsible_dirs); + if (! hs_dir && ! options->StrictNodes) + hs_dir = smartlist_choose(responsible_dirs); + smartlist_free(responsible_dirs); + smartlist_free(usable_responsible_dirs); if (!hs_dir) { log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories, because we requested them all " "recently without success."); + if (options->StrictNodes && excluded_some) { + log_warn(LD_REND, "Could not pick a hidden service directory for the " + "requested hidden service: they are all either down or " + "excluded, and StrictNodes is set."); + } return 0; } @@ -693,6 +715,9 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) (rend_query->auth_type == REND_NO_AUTH ? "[none]" : escaped_safe_str_client(descriptor_cookie_base64)), routerstatus_describe(hs_dir)); + control_event_hs_descriptor_requested(rend_query, + hs_dir->identity_digest, + desc_id_base32); return 1; } @@ -772,8 +797,7 @@ rend_client_cancel_descriptor_fetches(void) SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && - (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC || - conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)) { + conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { /* It's a rendezvous descriptor fetch in progress -- cancel it * by marking the connection for close. * diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index d1b49411c..a664b5d50 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -672,79 +672,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On - * success, return a newly alloced service_descriptor_t. On failure, - * return NULL. - */ -rend_service_descriptor_t * -rend_parse_service_descriptor(const char *str, size_t len) -{ - rend_service_descriptor_t *result = NULL; - int i, n_intro_points; - size_t keylen, asn1len; - const char *end, *cp, *eos; - rend_intro_point_t *intro; - - result = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - cp = str; - end = str+len; - if (end-cp<2) goto truncated; - result->version = 0; - if (end-cp < 2) goto truncated; - asn1len = ntohs(get_uint16(cp)); - cp += 2; - if ((size_t)(end-cp) < asn1len) goto truncated; - result->pk = crypto_pk_asn1_decode(cp, asn1len); - if (!result->pk) goto truncated; - cp += asn1len; - if (end-cp < 4) goto truncated; - result->timestamp = (time_t) ntohl(get_uint32(cp)); - cp += 4; - result->protocols = 1<<2; /* always use intro format 2 */ - if (end-cp < 2) goto truncated; - n_intro_points = ntohs(get_uint16(cp)); - cp += 2; - - result->intro_nodes = smartlist_new(); - for (i=0;i<n_intro_points;++i) { - if (end-cp < 2) goto truncated; - eos = (const char *)memchr(cp,'\0',end-cp); - if (!eos) goto truncated; - /* Write nickname to extend info, but postpone the lookup whether - * we know that router. It's not part of the parsing process. */ - intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - strlcpy(intro->extend_info->nickname, cp, - sizeof(intro->extend_info->nickname)); - smartlist_add(result->intro_nodes, intro); - cp = eos+1; - } - keylen = crypto_pk_keysize(result->pk); - tor_assert(end-cp >= 0); - if ((size_t)(end-cp) < keylen) goto truncated; - if ((size_t)(end-cp) > keylen) { - log_warn(LD_PROTOCOL, - "Signature is %d bytes too long on service descriptor.", - (int)((size_t)(end-cp) - keylen)); - goto error; - } - note_crypto_pk_op(REND_CLIENT); - if (crypto_pk_public_checksig_digest(result->pk, - (char*)str,cp-str, /* data */ - (char*)cp,end-cp /* signature*/ - )<0) { - log_warn(LD_PROTOCOL, "Bad signature on service descriptor."); - goto error; - } - - return result; - truncated: - log_warn(LD_PROTOCOL, "Truncated service descriptor."); - error: - rend_service_descriptor_free(result); - return NULL; -} - /** Sets <b>out</b> to the first 10 bytes of the digest of <b>pk</b>, * base32 encoded. NUL-terminates out. (We use this string to * identify services in directory requests and .onion URLs.) @@ -843,7 +770,7 @@ void rend_cache_purge(void) { if (rend_cache) { - log_info(LD_REND, "Purging client/v0-HS-authority HS descriptor cache"); + log_info(LD_REND, "Purging HS descriptor cache"); strmap_free(rend_cache, rend_cache_entry_free_); } rend_cache = strmap_new(); @@ -954,27 +881,6 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) return 1; } -/** <b>query</b> is a base32'ed service id. If it's malformed, return -1. - * Else look it up. - * - If it is found, point *desc to it, and write its length into - * *desc_len, and return 1. - * - If it is not found, return 0. - * Note: calls to rend_cache_clean or rend_cache_store may invalidate - * *desc. - */ -int -rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len) -{ - rend_cache_entry_t *e; - int r; - r = rend_cache_lookup_entry(query,version,&e); - if (r <= 0) return r; - *desc = e->desc; - *desc_len = e->len; - return 1; -} - /** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and * copy the pointer to it to *<b>desc</b>. Return 1 on success, 0 on * well-formed-but-not-found, and -1 on failure. @@ -1006,130 +912,16 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) * descriptor */ #define MAX_INTRO_POINTS 10 -/** Parse *desc, calculate its service id, and store it in the cache. - * If we have a newer v0 descriptor with the same ID, ignore this one. - * If we have an older descriptor with the same ID, replace it. - * If we are acting as client due to the published flag and have any v2 - * descriptor with the same ID, reject this one in order to not get - * confused with having both versions for the same service. - * - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v2 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. - * - * The published flag tells us if we store the descriptor - * in our role as directory (1) or if we cache it as client (0). - * - * If <b>service_id</b> is non-NULL and the descriptor is not for that - * service ID, reject it. <b>service_id</b> must be specified if and - * only if <b>published</b> is 0 (we fetched this descriptor). - */ -int -rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id) -{ - rend_cache_entry_t *e; - rend_service_descriptor_t *parsed; - char query[REND_SERVICE_ID_LEN_BASE32+1]; - char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ - time_t now; - tor_assert(rend_cache); - parsed = rend_parse_service_descriptor(desc,desc_len); - if (!parsed) { - log_warn(LD_PROTOCOL,"Couldn't parse service descriptor."); - return -2; - } - if (rend_get_service_id(parsed->pk, query)<0) { - log_warn(LD_BUG,"Couldn't compute service ID."); - rend_service_descriptor_free(parsed); - return -2; - } - if ((service_id != NULL) && strcmp(query, service_id)) { - log_warn(LD_REND, "Received service descriptor for service ID %s; " - "expected descriptor for service ID %s.", - query, safe_str(service_id)); - rend_service_descriptor_free(parsed); - return -2; - } - now = time(NULL); - if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - /* Do we have a v2 descriptor and fetched this descriptor as a client? */ - tor_snprintf(key, sizeof(key), "2%s", query); - if (!published && strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -1; - } - if (parsed->intro_nodes && - smartlist_len(parsed->intro_nodes) > MAX_INTRO_POINTS) { - log_warn(LD_REND, "Found too many introduction points on a hidden " - "service descriptor for %s. This is probably a (misguided) " - "attempt to improve reliability, but it could also be an " - "attempt to do a guard enumeration attack. Rejecting.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - tor_snprintf(key, sizeof(key), "0%s", query); - e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); - if (e && e->parsed->timestamp > parsed->timestamp) { - log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return 0; - } - if (e && e->len == desc_len && tor_memeq(desc,e->desc,desc_len)) { - log_info(LD_REND,"We already have this service descriptor %s.", - safe_str_client(query)); - e->received = time(NULL); - rend_service_descriptor_free(parsed); - return 0; - } - if (!e) { - e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - strmap_set_lc(rend_cache, key, e); - } else { - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - } - e->received = time(NULL); - e->parsed = parsed; - e->len = desc_len; - e->desc = tor_malloc(desc_len); - memcpy(e->desc, desc, desc_len); - - log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str_client(query), (int)desc_len); - return 1; -} - /** Parse the v2 service descriptor(s) in <b>desc</b> and store it/them to the * local rend cache. Don't attempt to decrypt the included list of introduction * points (as we don't have a descriptor cookie for it). * * If we have a newer descriptor with the same ID, ignore this one. * If we have an older descriptor with the same ID, replace it. - * Return -2 if we are not acting as hidden service directory; - * return -1 if the descriptor(s) were not parsable; return 0 if all - * descriptors are the same or older than those we've already got; - * return a positive number for the number of novel stored descriptors. + * + * Return an appropriate rend_cache_store_status_t. */ -int +rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc) { rend_service_descriptor_t *parsed; @@ -1149,7 +941,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) /* Cannot store descs, because we are (currently) not acting as * hidden service directory. */ log_info(LD_REND, "Cannot store descs: Not acting as hs dir"); - return -2; + return RCS_NOTDIR; } while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, &intro_size, &encoded_size, @@ -1225,11 +1017,11 @@ rend_cache_store_v2_desc_as_dir(const char *desc) } if (!number_parsed) { log_info(LD_REND, "Could not parse any descriptor."); - return -1; + return RCS_BADDESC; } log_info(LD_REND, "Parsed %d and added %d descriptor%s.", number_parsed, number_stored, number_stored != 1 ? "s" : ""); - return number_stored; + return RCS_OKAY; } /** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list @@ -1239,15 +1031,12 @@ rend_cache_store_v2_desc_as_dir(const char *desc) * * If we have a newer v2 descriptor with the same ID, ignore this one. * If we have an older descriptor with the same ID, replace it. - * If we have any v0 descriptor with the same ID, reject this one in order - * to not get confused with having both versions for the same service. * If the descriptor's service ID does not match * <b>rend_query</b>-\>onion_address, reject it. - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v0 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. + * + * Return an appropriate rend_cache_store_status_t. */ -int +rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, const rend_data_t *rend_query) { @@ -1276,7 +1065,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, char key[REND_SERVICE_ID_LEN_BASE32+2]; char service_id[REND_SERVICE_ID_LEN_BASE32+1]; rend_cache_entry_t *e; - int retval; + rend_cache_store_status_t retval = RCS_BADDESC; tor_assert(rend_cache); tor_assert(desc); /* Parse the descriptor. */ @@ -1284,20 +1073,17 @@ rend_cache_store_v2_desc_as_client(const char *desc, &intro_size, &encoded_size, &next_desc, desc) < 0) { log_warn(LD_REND, "Could not parse descriptor."); - retval = -2; goto err; } /* Compute service ID from public key. */ if (rend_get_service_id(parsed->pk, service_id)<0) { log_warn(LD_REND, "Couldn't compute service ID."); - retval = -2; goto err; } if (strcmp(rend_query->onion_address, service_id)) { log_warn(LD_REND, "Received service descriptor for service ID %s; " "expected descriptor for service ID %s.", service_id, safe_str(rend_query->onion_address)); - retval = -2; goto err; } /* Decode/decrypt introduction points. */ @@ -1329,7 +1115,6 @@ rend_cache_store_v2_desc_as_client(const char *desc, log_warn(LD_REND, "Failed to parse introduction points. Either the " "service has published a corrupt descriptor or you have " "provided invalid authorization data."); - retval = -2; goto err; } else if (n_intro_points > MAX_INTRO_POINTS) { log_warn(LD_REND, "Found too many introduction points on a hidden " @@ -1337,7 +1122,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, "attempt to improve reliability, but it could also be an " "attempt to do a guard enumeration attack. Rejecting.", safe_str_client(rend_query->onion_address)); - retval = -2; + goto err; } } else { @@ -1350,22 +1135,12 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", safe_str_client(service_id)); - retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " "the future.", safe_str_client(service_id)); - retval = -2; - goto err; - } - /* Do we have a v0 descriptor? */ - tor_snprintf(key, sizeof(key), "0%s", service_id); - if (strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str_client(service_id)); - retval = -1; goto err; } /* Do we already have a newer descriptor? */ @@ -1375,16 +1150,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", safe_str_client(service_id)); - retval = 0; - goto err; + goto okay; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", safe_str_client(service_id)); e->received = time(NULL); - retval = 0; - goto err; + goto okay; } if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); @@ -1400,7 +1173,10 @@ rend_cache_store_v2_desc_as_client(const char *desc, e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", safe_str_client(service_id), (int)encoded_size); - return 1; + return RCS_OKAY; + + okay: + retval = RCS_OKAY; err: rend_service_descriptor_free(parsed); diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index f476593d2..07a47accf 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -26,8 +26,6 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, const uint8_t *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); int rend_get_service_id(crypto_pk_t *pk, char *out); void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc); @@ -39,16 +37,20 @@ void rend_cache_clean_v2_descs_as_dir(time_t now); void rend_cache_purge(void); void rend_cache_free_all(void); int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); int rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **entry_out); int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id); -int rend_cache_store_v2_desc_as_client(const char *desc, +/** Return value from rend_cache_store_v2_desc_as_{dir,client}. */ +typedef enum { + RCS_NOTDIR = -2, /**< We're not a directory */ + RCS_BADDESC = -1, /**< This descriptor is no good. */ + RCS_OKAY = 0 /**< All worked as expected */ +} rend_cache_store_status_t; + +rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc); +rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); + int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 1bd11f6dc..110381680 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -94,7 +94,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Close any other intro circuits with the same pk. */ c = NULL; - while ((c = circuit_get_intro_point(pk_digest))) { + while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) { log_info(LD_REND, "Replacing old circuit for service %s", safe_str(serviceid)); circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED); @@ -111,7 +111,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Now, set up this circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); - memcpy(circ->rend_token, pk_digest, DIGEST_LEN); + circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest); log_info(LD_REND, "Established introduction point on circuit %u for service %s", @@ -165,7 +165,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, (char*)request, REND_SERVICE_ID_LEN); /* The first 20 bytes are all we look at: they have a hash of Bob's PK. */ - intro_circ = circuit_get_intro_point((char*)request); + intro_circ = circuit_get_intro_point((const uint8_t*)request); if (!intro_circ) { log_info(LD_REND, "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; " @@ -231,11 +231,12 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } if (request_len != REND_COOKIE_LEN) { - log_warn(LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); + log_fn(LOG_PROTOCOL_WARN, + LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); goto err; } - if (circuit_get_rendezvous((char*)request)) { + if (circuit_get_rendezvous(request)) { log_warn(LD_PROTOCOL, "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS."); goto err; @@ -251,7 +252,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING); - memcpy(circ->rend_token, request, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, request); base16_encode(hexid,9,(char*)request,4); @@ -299,7 +300,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, "Got request for rendezvous from circuit %u to cookie %s.", (unsigned)circ->p_circ_id, hexid); - rend_circ = circuit_get_rendezvous((char*)request); + rend_circ = circuit_get_rendezvous(request); if (!rend_circ) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", @@ -327,7 +328,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); circuit_change_purpose(TO_CIRCUIT(rend_circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); - memset(circ->rend_token, 0, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, NULL); rend_circ->rend_splice = circ; circ->rend_splice = rend_circ; diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 8a4a11e47..a7c1e32f1 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -10,6 +10,7 @@ #define RENDSERVICE_PRIVATE #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -81,7 +82,7 @@ typedef struct rend_service_port_config_t { #define MAX_INTRO_CIRCS_PER_PERIOD 10 /** How many times will a hidden service operator attempt to connect to * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 30 +#define MAX_REND_FAILURES 8 /** How many seconds should we spend trying to connect to a requested * rendezvous point before giving up? */ #define MAX_REND_TIMEOUT 30 @@ -543,7 +544,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX it would be nicer if we had a nicer abstraction to use here, * so we could just iterate over the list of services to close, but * once again, this isn't critical-path code. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -655,6 +656,35 @@ rend_service_load_all_keys(void) return 0; } +/** Add to <b>lst</b> every filename used by <b>s</b>. */ +static void +rend_service_add_filenames_to_list(smartlist_t *lst, const rend_service_t *s) +{ + tor_assert(lst); + tor_assert(s); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"private_key", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"hostname", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"client_keys", + s->directory); +} + +/** Add to <b>open_lst</b> every filename used by a configured hidden service, + * and to <b>stat_lst</b> every directory used by a configured hidden + * service */ +void +rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst) +{ + if (!rend_service_list) + return; + SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) { + rend_service_add_filenames_to_list(open_lst, s); + smartlist_add(stat_lst, tor_strdup(s->directory)); + } SMARTLIST_FOREACH_END(s); +} + /** Load and/or generate private keys for the hidden service <b>s</b>, * possibly including keys for client authorization. Return 0 on success, -1 * on failure. */ @@ -1208,7 +1238,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, /* check for replay of PK-encrypted portion. */ replay = replaycache_add_test_and_elapsed( intro_point->accepted_intro_rsa_parts, - parsed_req->ciphertext, (int)parsed_req->ciphertext_len, + parsed_req->ciphertext, parsed_req->ciphertext_len, &elapsed); if (replay) { @@ -1502,27 +1532,6 @@ find_rp_for_intro(const rend_intro_cell_t *intro, return rp; } -/** Remove unnecessary parts from a rend_intro_cell_t - the ciphertext if - * already decrypted, the plaintext too if already parsed - */ - -void -rend_service_compact_intro(rend_intro_cell_t *request) -{ - if (!request) return; - - if ((request->plaintext && request->plaintext_len > 0) || - request->parsed) { - tor_free(request->ciphertext); - request->ciphertext_len = 0; - } - - if (request->parsed) { - tor_free(request->plaintext); - request->plaintext_len = 0; - } -} - /** Free a parsed INTRODUCE1 or INTRODUCE2 cell that was allocated by * rend_service_parse_intro(). */ @@ -2061,7 +2070,7 @@ rend_service_decrypt_intro( if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error decrypting encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } if (status >= 0) status = -1; @@ -2167,7 +2176,7 @@ rend_service_parse_intro_plaintext( if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error parsing encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } if (status >= 0) status = -1; @@ -2376,7 +2385,7 @@ count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || diff --git a/src/or/rendservice.h b/src/or/rendservice.h index caf88a3d6..40198b07e 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -71,6 +71,8 @@ struct rend_intro_cell_s { int num_rend_services(void); int rend_config_services(const or_options_t *options, int validate_only); int rend_service_load_all_keys(void); +void rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst); void rend_services_introduce(void); void rend_consider_services_upload(time_t now); void rend_hsdir_routers_changed(void); @@ -83,7 +85,6 @@ int rend_service_intro_established(origin_circuit_t *circuit, void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, size_t request_len); -void rend_service_compact_intro(rend_intro_cell_t *request); int rend_service_decrypt_intro(rend_intro_cell_t *request, crypto_pk_t *key, char **err_msg_out); diff --git a/src/or/rephist.c b/src/or/rephist.c index 2948bf8f0..5446c25e3 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -879,126 +879,6 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) return -1; } -/** Format the current tracked status of the router in <b>hist</b> at time - * <b>now</b> for analysis; return it in a newly allocated string. */ -static char * -rep_hist_format_router_status(or_history_t *hist, time_t now) -{ - char sor_buf[ISO_TIME_LEN+1]; - char sod_buf[ISO_TIME_LEN+1]; - double wfu; - double mtbf; - int up = 0, down = 0; - char *cp = NULL; - - if (hist->start_of_run) { - format_iso_time(sor_buf, hist->start_of_run); - up = 1; - } - if (hist->start_of_downtime) { - format_iso_time(sod_buf, hist->start_of_downtime); - down = 1; - } - - wfu = get_weighted_fractional_uptime(hist, now); - mtbf = get_stability(hist, now); - tor_asprintf(&cp, - "%s%s%s" - "%s%s%s" - "wfu %0.3f\n" - " weighted-time %lu\n" - " weighted-uptime %lu\n" - "mtbf %0.1f\n" - " weighted-run-length %lu\n" - " total-run-weights %f\n", - up?"uptime-started ":"", up?sor_buf:"", up?" UTC\n":"", - down?"downtime-started ":"", down?sod_buf:"", down?" UTC\n":"", - wfu, - hist->total_weighted_time, - hist->weighted_uptime, - mtbf, - hist->weighted_run_length, - hist->total_run_weights - ); - return cp; -} - -/** The last stability analysis document that we created, or NULL if we never - * have created one. */ -static char *last_stability_doc = NULL; -/** The last time we created a stability analysis document, or 0 if we never - * have created one. */ -static time_t built_last_stability_doc_at = 0; -/** Shortest allowable time between building two stability documents. */ -#define MAX_STABILITY_DOC_BUILD_RATE (3*60) - -/** Return a pointer to a NUL-terminated document describing our view of the - * stability of the routers we've been tracking. Return NULL on failure. */ -const char * -rep_hist_get_router_stability_doc(time_t now) -{ - char *result; - smartlist_t *chunks; - if (built_last_stability_doc_at + MAX_STABILITY_DOC_BUILD_RATE > now) - return last_stability_doc; - - if (!history_map) - return NULL; - - tor_free(last_stability_doc); - chunks = smartlist_new(); - - if (rep_hist_have_measured_enough_stability()) { - smartlist_add(chunks, tor_strdup("we-have-enough-measurements\n")); - } else { - smartlist_add(chunks, tor_strdup("we-do-not-have-enough-measurements\n")); - } - - DIGESTMAP_FOREACH(history_map, id, or_history_t *, hist) { - const node_t *node; - char dbuf[BASE64_DIGEST_LEN+1]; - char *info; - digest_to_base64(dbuf, id); - node = node_get_by_id(id); - if (node) { - char ip[INET_NTOA_BUF_LEN+1]; - char tbuf[ISO_TIME_LEN+1]; - time_t published = node_get_published_on(node); - node_get_address_string(node,ip,sizeof(ip)); - if (published > 0) - format_iso_time(tbuf, published); - else - strlcpy(tbuf, "???", sizeof(tbuf)); - smartlist_add_asprintf(chunks, - "router %s %s %s\n" - "published %s\n" - "relevant-flags %s%s%s\n" - "declared-uptime %ld\n", - dbuf, node_get_nickname(node), ip, - tbuf, - node->is_running ? "Running " : "", - node->is_valid ? "Valid " : "", - node->ri && node->ri->is_hibernating ? "Hibernating " : "", - node_get_declared_uptime(node)); - } else { - smartlist_add_asprintf(chunks, - "router %s {no descriptor}\n", dbuf); - } - info = rep_hist_format_router_status(hist, now); - if (info) - smartlist_add(chunks, info); - - } DIGESTMAP_FOREACH_END; - - result = smartlist_join_strings(chunks, "", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - - last_stability_doc = result; - built_last_stability_doc_at = time(NULL); - return result; -} - /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no * such line exists. */ @@ -1051,7 +931,7 @@ correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring) return 0; else { long run_length = stored_at - t; - t = now - run_length; + t = (time_t)(now - run_length); if (t < started_measuring) t = started_measuring; return t; @@ -1212,7 +1092,7 @@ rep_hist_load_mtbf_data(time_t now) hist->start_of_run = correct_time(start_of_run, now, stored_at, tracked_since); if (hist->start_of_run < latest_possible_start + wrl) - latest_possible_start = hist->start_of_run - wrl; + latest_possible_start = (time_t)(hist->start_of_run - wrl); hist->weighted_run_length = wrl; hist->total_run_weights = trw; @@ -1862,22 +1742,20 @@ rep_hist_note_used_port(time_t now, uint16_t port) add_predicted_port(now, port); } -/** For this long after we've seen a request for a given port, assume that - * we'll want to make connections to the same port in the future. */ -#define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) - /** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { + int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { - if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + if (pp->time + predicted_circs_relevance_time < now) { log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); rephist_total_alloc -= sizeof(predicted_port_t); @@ -1944,14 +1822,17 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; predicted_internal_uptime_time = now; predicted_internal_capacity_time = now; } - if (predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) + if (predicted_internal_time + predicted_circs_relevance_time < now) return 0; /* too long ago */ - if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) *need_uptime = 1; // Always predict that we need capacity. *need_capacity = 1; @@ -1963,8 +1844,11 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + return smartlist_len(predicted_ports_list) || - predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now; + predicted_internal_time + predicted_circs_relevance_time >= now; } /** Return 1 if we have no need for circuits currently, else return 0. */ @@ -2313,7 +2197,7 @@ rep_hist_format_exit_stats(time_t now) time_t rep_hist_exit_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_exit_stats_interval) return 0; /* Not initialized. */ @@ -2329,19 +2213,12 @@ rep_hist_exit_stats_write(time_t now) rep_hist_reset_exit_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "exit-stats", str, "exit port statistics"); } - filename = get_datadir_fname2("stats", "exit-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); done: tor_free(str); - tor_free(statsdir); - tor_free(filename); return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } @@ -2434,7 +2311,7 @@ rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) return; start_of_interval = (circ->timestamp_created.tv_sec > start_of_buffer_stats_interval) ? - circ->timestamp_created.tv_sec : + (time_t)circ->timestamp_created.tv_sec : start_of_buffer_stats_interval; interval_length = (int) (end_of_interval - start_of_interval); if (interval_length <= 0) @@ -2598,7 +2475,7 @@ time_t rep_hist_buffer_stats_write(time_t now) { circuit_t *circ; - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_buffer_stats_interval) return 0; /* Not initialized. */ @@ -2606,7 +2483,7 @@ rep_hist_buffer_stats_write(time_t now) goto done; /* Not ready to write */ /* Add open circuits to the history. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { rep_hist_buffer_stats_add_circ(circ, now); } @@ -2617,19 +2494,12 @@ rep_hist_buffer_stats_write(time_t now) rep_hist_reset_buffer_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics"); } - filename = get_datadir_fname2("stats", "buffer-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write buffer stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } @@ -2741,7 +2611,7 @@ rep_hist_format_desc_stats(time_t now) time_t rep_hist_desc_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *filename = NULL, *str = NULL; if (!start_of_served_descs_stats_interval) return 0; /* We're not collecting stats. */ @@ -2751,10 +2621,8 @@ rep_hist_desc_stats_write(time_t now) str = rep_hist_format_desc_stats(now); tor_assert(str != NULL); - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (check_or_create_data_subdir("stats") < 0) { + goto done; } filename = get_datadir_fname2("stats", "served-desc-stats"); if (append_bytes_to_file(filename, str, strlen(str), 0) < 0) @@ -2763,7 +2631,6 @@ rep_hist_desc_stats_write(time_t now) rep_hist_reset_desc_stats(now); done: - tor_free(statsdir); tor_free(filename); tor_free(str); return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL; @@ -2981,7 +2848,7 @@ rep_hist_format_conn_stats(time_t now) time_t rep_hist_conn_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_conn_stats_interval) return 0; /* Not initialized. */ @@ -2995,28 +2862,21 @@ rep_hist_conn_stats_write(time_t now) rep_hist_reset_conn_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "conn-stats", str, "connection statistics"); } - filename = get_datadir_fname2("stats", "conn-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write conn stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; } /** Internal statistics to track how many requests of each type of - * handshake we've received, and how many we've completed. Useful for - * seeing trends in cpu load. + * handshake we've received, and how many we've assigned to cpuworkers. + * Useful for seeing trends in cpu load. * @{ */ -static int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; -static int onion_handshakes_completed[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ /** A new onionskin (using the <b>type</b> handshake) has arrived. */ @@ -3030,10 +2890,10 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) /** We've sent an onionskin (using the <b>type</b> handshake) to a * cpuworker. */ void -rep_hist_note_circuit_handshake_completed(uint16_t type) +rep_hist_note_circuit_handshake_assigned(uint16_t type) { if (type <= MAX_ONION_HANDSHAKE_TYPE) - onion_handshakes_completed[type]++; + onion_handshakes_assigned[type]++; } /** Log our onionskin statistics since the last time we were called. */ @@ -3041,13 +2901,13 @@ void rep_hist_log_circuit_handshake_stats(time_t now) { (void)now; - log_notice(LD_HIST, "Circuit handshake stats since last time: " + log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: " "%d/%d TAP, %d/%d NTor.", - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_TAP], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP], - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_NTOR], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]); - memset(onion_handshakes_completed, 0, sizeof(onion_handshakes_completed)); + memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned)); memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested)); } @@ -3061,11 +2921,9 @@ rep_hist_free_all(void) tor_free(write_array); tor_free(dir_read_array); tor_free(dir_write_array); - tor_free(last_stability_doc); tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); - built_last_stability_doc_at = 0; predicted_ports_free(); bidi_map_free(); diff --git a/src/or/rephist.h b/src/or/rephist.h index de824749b..cd6231e6e 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -47,7 +47,6 @@ double rep_hist_get_stability(const char *id, time_t when); double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); long rep_hist_get_weighted_time_known(const char *id, time_t when); int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); void rep_hist_note_used_port(time_t now, uint16_t port); smartlist_t *rep_hist_get_predicted_ports(time_t now); @@ -97,7 +96,7 @@ time_t rep_hist_conn_stats_write(time_t now); void rep_hist_conn_stats_term(void); void rep_hist_note_circuit_handshake_requested(uint16_t type); -void rep_hist_note_circuit_handshake_completed(uint16_t type); +void rep_hist_note_circuit_handshake_assigned(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); void rep_hist_free_all(void); diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 59b98489b..90f87c12d 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -63,9 +63,9 @@ replaycache_new(time_t horizon, time_t interval) /** See documentation for replaycache_add_and_test() */ -int +STATIC int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed) { int rv = 0; @@ -73,7 +73,7 @@ replaycache_add_and_test_internal( time_t *access_time; /* sanity check */ - if (present <= 0 || !r || !data || len <= 0) { + if (present <= 0 || !r || !data || len == 0) { log_info(LD_BUG, "replaycache_add_and_test_internal() called with stupid" " parameters; please fix this."); goto done; @@ -127,14 +127,13 @@ replaycache_add_and_test_internal( /** See documentation for replaycache_scrub_if_needed() */ -void +STATIC void replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) { digestmap_iter_t *itr = NULL; const char *digest; void *valp; time_t *access_time; - char scrub_this; /* sanity check */ if (!r || !(r->digests_seen)) { @@ -152,20 +151,10 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) /* okay, scrub time */ itr = digestmap_iter_init(r->digests_seen); while (!digestmap_iter_done(itr)) { - scrub_this = 0; digestmap_iter_get(itr, &digest, &valp); access_time = (time_t *)valp; - if (access_time) { - /* aged out yet? */ - if (*access_time < present - r->horizon) scrub_this = 1; - } else { - /* Buh? Get rid of it, anyway */ - log_info(LD_BUG, "replaycache_scrub_if_needed_internal() saw a NULL" - " entry in the digestmap."); - scrub_this = 1; - } - - if (scrub_this) { + /* aged out yet? */ + if (*access_time < present - r->horizon) { /* Advance the iterator and remove this one */ itr = digestmap_iter_next_rmv(r->digests_seen, itr); /* Free the value removed */ @@ -187,7 +176,7 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) */ int -replaycache_add_and_test(replaycache_t *r, const void *data, int len) +replaycache_add_and_test(replaycache_t *r, const void *data, size_t len) { return replaycache_add_and_test_internal(time(NULL), r, data, len, NULL); } @@ -198,7 +187,7 @@ replaycache_add_and_test(replaycache_t *r, const void *data, int len) int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed) + replaycache_t *r, const void *data, size_t len, time_t *elapsed) { return replaycache_add_and_test_internal(time(NULL), r, data, len, elapsed); } diff --git a/src/or/replaycache.h b/src/or/replaycache.h index de20cab62..cd713fe89 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -45,10 +45,10 @@ replaycache_t * replaycache_new(time_t horizon, time_t interval); * testing. For everything else, use the wrappers below instead. */ -int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, +STATIC int replaycache_add_and_test_internal( + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed); -void replaycache_scrub_if_needed_internal( +STATIC void replaycache_scrub_if_needed_internal( time_t present, replaycache_t *r); #endif /* REPLAYCACHE_PRIVATE */ @@ -57,9 +57,9 @@ void replaycache_scrub_if_needed_internal( * replaycache_t methods */ -int replaycache_add_and_test(replaycache_t *r, const void *data, int len); +int replaycache_add_and_test(replaycache_t *r, const void *data, size_t len); int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed); + replaycache_t *r, const void *data, size_t len, time_t *elapsed); void replaycache_scrub_if_needed(replaycache_t *r); #endif diff --git a/src/or/router.c b/src/or/router.c index eabd9c3f5..2cdbb0c8b 100644..100755 --- a/src/or/router.c +++ b/src/or/router.c @@ -232,7 +232,8 @@ get_server_identity_key(void) return server_identitykey; } -/** Return true iff the server identity key has been set. */ +/** Return true iff we are a server and the server identity key + * has been set. */ int server_identity_key_is_set(void) { @@ -683,6 +684,63 @@ router_initialize_tls_context(void) (unsigned int)lifetime); } +/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write + * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or + * -1 if Tor should die, + */ +STATIC int +router_write_fingerprint(int hashed) +{ + char *keydir = NULL, *cp = NULL; + const char *fname = hashed ? "hashed-fingerprint" : + "fingerprint"; + char fingerprint[FINGERPRINT_LEN+1]; + const or_options_t *options = get_options(); + char *fingerprint_line = NULL; + int result = -1; + + keydir = get_datadir_fname(fname); + log_info(LD_GENERAL,"Dumping %sfingerprint to \"%s\"...", + hashed ? "hashed " : "", keydir); + if (!hashed) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { + log_err(LD_GENERAL,"Error computing fingerprint"); + goto done; + } + } else { + if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(), + fingerprint) < 0) { + log_err(LD_GENERAL,"Error computing hashed fingerprint"); + goto done; + } + } + + tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint); + + /* Check whether we need to write the (hashed-)fingerprint file. */ + + cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL); + if (!cp || strcmp(cp, fingerprint_line)) { + if (write_str_to_file(keydir, fingerprint_line, 0)) { + log_err(LD_FS, "Error writing %sfingerprint line to file", + hashed ? "hashed " : ""); + goto done; + } + } + + log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'", + hashed ? "bridge's hashed" : "server's", options->Nickname, + fingerprint); + + result = 0; + done: + tor_free(cp); + tor_free(keydir); + tor_free(fingerprint_line); + return result; +} + /** Initialize all OR private keys, and the TLS context, as necessary. * On OPs, this only initializes the tls context. Return 0 on success, * or -1 if Tor should die. @@ -691,14 +749,10 @@ int init_keys(void) { char *keydir; - char fingerprint[FINGERPRINT_LEN+1]; - /*nickname<space>fp\n\0 */ - char fingerprint_line[MAX_NICKNAME_LEN+FINGERPRINT_LEN+3]; const char *mydesc; crypto_pk_t *prkey; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - char *cp; const or_options_t *options = get_options(); dirinfo_type_t type; time_t now = time(NULL); @@ -888,40 +942,16 @@ init_keys(void) } } - /* 5. Dump fingerprint to 'fingerprint' */ - keydir = get_datadir_fname("fingerprint"); - log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_server_identity_key(), - fingerprint, 0) < 0) { - log_err(LD_GENERAL,"Error computing fingerprint"); - tor_free(keydir); + /* 5. Dump fingerprint and possibly hashed fingerprint to files. */ + if (router_write_fingerprint(0)) { + log_err(LD_FS, "Error writing fingerprint to file"); return -1; } - tor_assert(strlen(options->Nickname) <= MAX_NICKNAME_LEN); - if (tor_snprintf(fingerprint_line, sizeof(fingerprint_line), - "%s %s\n",options->Nickname, fingerprint) < 0) { - log_err(LD_GENERAL,"Error writing fingerprint line"); - tor_free(keydir); + if (!public_server_mode(options) && router_write_fingerprint(1)) { + log_err(LD_FS, "Error writing hashed fingerprint to file"); return -1; } - /* Check whether we need to write the fingerprint file. */ - cp = NULL; - if (file_status(keydir) == FN_FILE) - cp = read_file_to_str(keydir, 0, NULL); - if (!cp || strcmp(cp, fingerprint_line)) { - if (write_str_to_file(keydir, fingerprint_line, 0)) { - log_err(LD_FS, "Error writing fingerprint line to file"); - tor_free(keydir); - tor_free(cp); - return -1; - } - } - tor_free(cp); - tor_free(keydir); - log_notice(LD_GENERAL, - "Your Tor server's identity key fingerprint is '%s %s'", - options->Nickname, fingerprint); if (!authdir_mode(options)) return 0; /* 6. [authdirserver only] load approved-routers file */ @@ -931,12 +961,9 @@ init_keys(void) } /* 6b. [authdirserver only] add own key to approved directories. */ crypto_pk_get_digest(get_server_identity_key(), digest); - type = ((options->V1AuthoritativeDir ? V1_DIRINFO : NO_DIRINFO) | - (options->V2AuthoritativeDir ? V2_DIRINFO : NO_DIRINFO) | - (options->V3AuthoritativeDir ? + type = ((options->V3AuthoritativeDir ? (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) | - (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO) | - (options->HSAuthoritativeDir ? HIDSERV_DIRINFO : NO_DIRINFO)); + (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO)); ds = router_get_trusteddirserver_by_digest(digest); if (!ds) { @@ -1149,7 +1176,7 @@ consider_testing_reachability(int test_or, int test_dir) /* XXX IPv6 self testing */ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", - me->address, me->or_port); + fmt_addr32(me->addr), me->or_port); circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); extend_info_free(ei); @@ -1161,7 +1188,7 @@ consider_testing_reachability(int test_or, int test_dir) CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(me->address, &addr, + directory_initiate_command(&addr, me->or_port, me->dir_port, me->cache_info.identity_digest, DIR_PURPOSE_FETCH_SERVERDESC, @@ -1176,6 +1203,7 @@ router_orport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", get_options()->PublishServerDescriptor_ != NO_DIRINFO ? @@ -1184,7 +1212,8 @@ router_orport_found_reachable(void) mark_my_descriptor_dirty("ORPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } } @@ -1194,6 +1223,7 @@ router_dirport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; @@ -1201,7 +1231,8 @@ router_dirport_found_reachable(void) mark_my_descriptor_dirty("DirPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1236,7 +1267,8 @@ router_perform_bandwidth_test(int num_circs, time_t now) } /** Return true iff our network is in some sense disabled: either we're - * hibernating, entering hibernation, or */ + * hibernating, entering hibernation, or the network is turned off with + * DisableNetwork. */ int net_is_disabled(void) { @@ -1251,22 +1283,6 @@ authdir_mode(const or_options_t *options) { return options->AuthoritativeDir != 0; } -/** Return true iff we believe ourselves to be a v1 authoritative - * directory server. - */ -int -authdir_mode_v1(const or_options_t *options) -{ - return authdir_mode(options) && options->V1AuthoritativeDir != 0; -} -/** Return true iff we believe ourselves to be a v2 authoritative - * directory server. - */ -int -authdir_mode_v2(const or_options_t *options) -{ - return authdir_mode(options) && options->V2AuthoritativeDir != 0; -} /** Return true iff we believe ourselves to be a v3 authoritative * directory server. */ @@ -1275,13 +1291,11 @@ authdir_mode_v3(const or_options_t *options) { return authdir_mode(options) && options->V3AuthoritativeDir != 0; } -/** Return true iff we are a v1, v2, or v3 directory authority. */ +/** Return true iff we are a v3 directory authority. */ int authdir_mode_any_main(const or_options_t *options) { - return options->V1AuthoritativeDir || - options->V2AuthoritativeDir || - options->V3AuthoritativeDir; + return options->V3AuthoritativeDir; } /** Return true if we believe ourselves to be any kind of * authoritative directory beyond just a hidserv authority. */ @@ -1335,8 +1349,8 @@ authdir_mode_bridge(const or_options_t *options) /** Return true iff we are trying to be a server. */ -int -server_mode(const or_options_t *options) +MOCK_IMPL(int, +server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; /* XXXX024 I believe we can kill off ORListenAddress here.*/ @@ -1345,8 +1359,8 @@ server_mode(const or_options_t *options) /** Return true iff we are trying to be a non-bridge server. */ -int -public_server_mode(const or_options_t *options) +MOCK_IMPL(int, +public_server_mode,(const or_options_t *options)) { if (!server_mode(options)) return 0; return (!options->BridgeRelay); @@ -1674,22 +1688,10 @@ router_is_me(const routerinfo_t *router) return router_digest_is_me(router->cache_info.identity_digest); } -/** Return true iff <b>fp</b> is a hex fingerprint of my identity digest. */ -int -router_fingerprint_is_me(const char *fp) -{ - char digest[DIGEST_LEN]; - if (strlen(fp) == HEX_DIGEST_LEN && - base16_decode(digest, sizeof(digest), fp, HEX_DIGEST_LEN) == 0) - return router_digest_is_me(digest); - - return 0; -} - /** Return a routerinfo for this OR, rebuilding a fresh one if * necessary. Return NULL on error, or if called on an OP. */ -const routerinfo_t * -router_get_my_routerinfo(void) +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo,(void)) { if (!server_mode(get_options())) return NULL; @@ -1793,7 +1795,6 @@ router_rebuild_descriptor(int force) ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; - ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; ri->or_port = router_get_advertised_or_port(options); @@ -1858,7 +1859,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->IPv6Exit, options->ExitPolicyRejectPrivate, - ri->address, !options->BridgeRelay); + ri->addr, !options->BridgeRelay); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -1871,12 +1872,6 @@ router_rebuild_descriptor(int force) tor_free(p_tmp); } -#if 0 - /* XXXX NM NM I belive this is safe to remove */ - if (authdir_mode(options)) - ri->is_valid = ri->is_named = 1; /* believe in yourself */ -#endif - if (options->MyFamily && ! options->BridgeRelay) { smartlist_t *family; if (!warned_nonexistent_family) @@ -2249,7 +2244,7 @@ router_guess_address_from_dir_headers(uint32_t *guess) * string describing the version of Tor and the operating system we're * currently running on. */ -void +STATIC void get_platform_str(char *platform, size_t len) { tor_snprintf(platform, len, "Tor %s on %s", @@ -2270,8 +2265,7 @@ char * router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key) { - /* XXXX025 Make this look entirely at its arguments, and not at globals. - */ + char *address = NULL; char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ char digest[DIGEST_LEN]; @@ -2345,7 +2339,9 @@ router_dump_router_to_string(routerinfo_t *router, } } + address = tor_dup_ip(router->addr); chunks = smartlist_new(); + /* Generate the easy portion of the router descriptor. */ smartlist_add_asprintf(chunks, "router %s %s %d 0 %d\n" @@ -2361,7 +2357,7 @@ router_dump_router_to_string(routerinfo_t *router, "signing-key\n%s" "%s%s%s%s", router->nickname, - router->address, + address, router->or_port, decide_to_advertise_dirport(options, router->dir_port), extra_or_address ? extra_or_address : "", @@ -2403,20 +2399,13 @@ router_dump_router_to_string(routerinfo_t *router, if (!router->exit_policy || !smartlist_len(router->exit_policy)) { smartlist_add(chunks, tor_strdup("reject *:*\n")); } else if (router->exit_policy) { - int i; - for (i = 0; i < smartlist_len(router->exit_policy); ++i) { - char pbuf[POLICY_BUF_LEN]; - addr_policy_t *tmpe = smartlist_get(router->exit_policy, i); - int result; - if (tor_addr_family(&tmpe->addr) == AF_INET6) - continue; /* Don't include IPv6 parts of address policy */ - result = policy_write_item(pbuf, POLICY_BUF_LEN, tmpe, 1); - if (result < 0) { - log_warn(LD_BUG,"descriptor policy_write_item ran out of room!"); - goto err; - } - smartlist_add_asprintf(chunks, "%s\n", pbuf); - } + char *exit_policy = router_dump_exit_policy_to_string(router,1,0); + + if (!exit_policy) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", exit_policy); + tor_free(exit_policy); } if (router->ipv6_exit_policy) { @@ -2475,6 +2464,7 @@ router_dump_router_to_string(routerinfo_t *router, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); } + tor_free(address); tor_free(family_line); tor_free(onion_pkey); tor_free(identity_pkey); @@ -2483,6 +2473,56 @@ router_dump_router_to_string(routerinfo_t *router, return output; } +/** + * OR only: Given <b>router</b>, produce a string with its exit policy. + * If <b>include_ipv4</b> is true, include IPv4 entries. + * If <b>include_ipv6</b> is true, include IPv6 entries. + */ +char * +router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6) +{ + smartlist_t *exit_policy_strings; + char *policy_string = NULL; + + if ((!router->exit_policy) || (router->policy_is_reject_star)) { + return tor_strdup("reject *:*"); + } + + exit_policy_strings = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(router->exit_policy, addr_policy_t *, tmpe) { + char *pbuf; + int bytes_written_to_pbuf; + if ((tor_addr_family(&tmpe->addr) == AF_INET6) && (!include_ipv6)) { + continue; /* Don't include IPv6 parts of address policy */ + } + if ((tor_addr_family(&tmpe->addr) == AF_INET) && (!include_ipv4)) { + continue; /* Don't include IPv4 parts of address policy */ + } + + pbuf = tor_malloc(POLICY_BUF_LEN); + bytes_written_to_pbuf = policy_write_item(pbuf,POLICY_BUF_LEN, tmpe, 1); + + if (bytes_written_to_pbuf < 0) { + log_warn(LD_BUG, "router_dump_exit_policy_to_string ran out of room!"); + tor_free(pbuf); + goto done; + } + + smartlist_add(exit_policy_strings,pbuf); + } SMARTLIST_FOREACH_END(tmpe); + + policy_string = smartlist_join_strings(exit_policy_strings, "\n", 0, NULL); + + done: + SMARTLIST_FOREACH(exit_policy_strings, char *, str, tor_free(str)); + smartlist_free(exit_policy_strings); + + return policy_string; +} + /** Copy the primary (IPv4) OR port (IP address and TCP port) for * <b>router</b> into *<b>ap_out</b>. */ void diff --git a/src/or/router.h b/src/or/router.h index 60095d087..d18ff065e 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -12,6 +12,8 @@ #ifndef TOR_ROUTER_H #define TOR_ROUTER_H +#include "testsupport.h" + crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); @@ -48,8 +50,6 @@ void router_perform_bandwidth_test(int num_circs, time_t now); int net_is_disabled(void); int authdir_mode(const or_options_t *options); -int authdir_mode_v1(const or_options_t *options); -int authdir_mode_v2(const or_options_t *options); int authdir_mode_v3(const or_options_t *options); int authdir_mode_any_main(const or_options_t *options); int authdir_mode_any_nonhidserv(const or_options_t *options); @@ -66,8 +66,8 @@ uint16_t router_get_advertised_or_port_by_af(const or_options_t *options, uint16_t router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport); -int server_mode(const or_options_t *options); -int public_server_mode(const or_options_t *options); +MOCK_DECL(int, server_mode, (const or_options_t *options)); +MOCK_DECL(int, public_server_mode, (const or_options_t *options)); int advertised_server_mode(void); int proxy_mode(const or_options_t *options); void consider_publishable_server(int force); @@ -82,7 +82,7 @@ void router_new_address_suggestion(const char *suggestion, const dir_connection_t *d_conn); int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port); int router_my_exit_policy_is_reject_star(void); -const routerinfo_t *router_get_my_routerinfo(void); +MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); extrainfo_t *router_get_my_extrainfo(void); const char *router_get_my_descriptor(void); const char *router_get_descriptor_gen_reason(void); @@ -90,11 +90,13 @@ int router_digest_is_me(const char *digest); const uint8_t *router_get_my_id_digest(void); int router_extrainfo_digest_is_me(const char *digest); int router_is_me(const routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); int router_pick_published_address(const or_options_t *options, uint32_t *addr); int router_rebuild_descriptor(int force); char *router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key); +char *router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6); void router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *addr_port_out); void router_get_pref_orport(const routerinfo_t *router, @@ -146,7 +148,8 @@ smartlist_t *router_get_all_orports(const routerinfo_t *ri); #ifdef ROUTER_PRIVATE /* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); +STATIC void get_platform_str(char *platform, size_t len); +STATIC int router_write_fingerprint(int hashed); #endif #endif diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 8fe496b51..8f3477a4a 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -37,7 +37,7 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" - +#include "../common/sandbox.h" // #define DEBUG_ROUTERLIST /****************************************************************************/ @@ -98,7 +98,8 @@ static smartlist_t *trusted_dir_servers = NULL; * and all fallback directory servers. */ static smartlist_t *fallback_dir_servers = NULL; -/** List of for a given authority, and download status for latest certificate. +/** List of certificates for a single authority, and download status for + * latest certificate. */ struct cert_list_t { /* @@ -130,16 +131,6 @@ static smartlist_t *warned_nicknames = NULL; * download is low. */ static time_t last_descriptor_download_attempted = 0; -/** When we last computed the weights to use for bandwidths on directory - * requests, what were the total weighted bandwidth, and our share of that - * bandwidth? Used to determine what fraction of directory requests we should - * expect to see. - * - * @{ */ -static uint64_t sl_last_total_weighted_bw = 0, - sl_last_weighted_bw_of_me = 0; -/**@}*/ - /** Return the number of directory authorities whose type matches some bit set * in <b>type</b> */ int @@ -220,8 +211,6 @@ download_status_is_ready_by_sk_in_cl(cert_list_t *cl, return rv; } -#define get_n_v2_authorities() get_n_authorities(V2_DIRINFO) - /** Helper: Return the cert_list_t for an authority whose authority ID is * <b>id_digest</b>, allocating a new list if necessary. */ static cert_list_t * @@ -449,7 +438,7 @@ trusted_dirs_flush_certs_to_disk(void) } DIGESTMAP_FOREACH_END; filename = get_datadir_fname("cached-certs"); - if (write_chunks_to_file(filename, chunks, 0)) { + if (write_chunks_to_file(filename, chunks, 0, 0)) { log_warn(LD_FS, "Error writing certificates to disk."); } tor_free(filename); @@ -681,9 +670,6 @@ authority_cert_dl_looks_uncertain(const char *id_digest) return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; } -/** How many times will we try to fetch a certificate before giving up? */ -#define MAX_CERT_DL_FAILURES 8 - /** Try to download any v3 authority certificates that we may be missing. If * <b>status</b> is provided, try to get all the ones that were used to sign * <b>status</b>. Additionally, try to have a non-expired certificate for @@ -715,7 +701,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) char id_digest_str[2*DIGEST_LEN+1]; char sk_digest_str[2*DIGEST_LEN+1]; - if (should_delay_dir_fetches(get_options())) + if (should_delay_dir_fetches(get_options(), NULL)) return; pending_cert = fp_pair_map_new(); @@ -755,7 +741,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(cert); if (!found && download_status_is_ready(&(cl->dl_status_by_id), now, - MAX_CERT_DL_FAILURES) && + get_options()->TestingCertMaxDownloadTries) && !digestmap_get(pending_id, ds->v3_identity_digest)) { log_info(LD_DIR, "No current certificate known for authority %s " @@ -817,7 +803,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } if (download_status_is_ready_by_sk_in_cl( cl, sig->signing_key_digest, - now, MAX_CERT_DL_FAILURES) && + now, get_options()->TestingCertMaxDownloadTries) && !fp_pair_map_get_by_digests(pending_cert, voter->identity_digest, sig->signing_key_digest)) { @@ -1103,15 +1089,18 @@ router_rebuild_store(int flags, desc_store_t *store) smartlist_add(chunk_list, c); } SMARTLIST_FOREACH_END(sd); - if (write_chunks_to_file(fname_tmp, chunk_list, 1)<0) { + if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) { log_warn(LD_FS, "Error writing router store to disk."); goto done; } /* Our mmap is now invalid. */ if (store->mmap) { - tor_munmap_file(store->mmap); + int res = tor_munmap_file(store->mmap); store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Unable to munmap route store in %s", fname); + } } if (replace_file(fname_tmp, fname)<0) { @@ -1178,32 +1167,25 @@ router_rebuild_store(int flags, desc_store_t *store) static int router_reload_router_list_impl(desc_store_t *store) { - char *fname = NULL, *altname = NULL, *contents = NULL; + char *fname = NULL, *contents = NULL; struct stat st; - int read_from_old_location = 0; int extrainfo = (store->type == EXTRAINFO_STORE); - time_t now = time(NULL); store->journal_len = store->store_len = 0; fname = get_datadir_fname(store->fname_base); - if (store->fname_alt_base) - altname = get_datadir_fname(store->fname_alt_base); - if (store->mmap) /* get rid of it first */ - tor_munmap_file(store->mmap); - store->mmap = NULL; + if (store->mmap) { + /* get rid of it first */ + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Failed to munmap %s", fname); + tor_free(fname); + return -1; + } + } store->mmap = tor_mmap_file(fname); - if (!store->mmap && altname && file_status(altname) == FN_FILE) { - read_from_old_location = 1; - log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old " - "location %s.", fname, altname); - if ((store->mmap = tor_mmap_file(altname))) - read_from_old_location = 1; - } - if (altname && !read_from_old_location) { - remove_file_if_very_old(altname, now); - } if (store->mmap) { store->store_len = store->mmap->size; if (extrainfo) @@ -1220,14 +1202,6 @@ router_reload_router_list_impl(desc_store_t *store) fname = get_datadir_fname_suffix(store->fname_base, ".new"); if (file_status(fname) == FN_FILE) contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - if (read_from_old_location) { - tor_free(altname); - altname = get_datadir_fname_suffix(store->fname_alt_base, ".new"); - if (!contents) - contents = read_file_to_str(altname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - else - remove_file_if_very_old(altname, now); - } if (contents) { if (extrainfo) router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL, @@ -1240,9 +1214,8 @@ router_reload_router_list_impl(desc_store_t *store) } tor_free(fname); - tor_free(altname); - if (store->journal_len || read_from_old_location) { + if (store->journal_len) { /* Always clear the journal on startup.*/ router_rebuild_store(RRS_FORCE, store); } else if (!extrainfo) { @@ -1309,8 +1282,6 @@ const routerstatus_t * router_pick_directory_server(dirinfo_type_t type, int flags) { const routerstatus_t *choice; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; if (!routerlist) return NULL; @@ -1329,47 +1300,6 @@ router_pick_directory_server(dirinfo_type_t type, int flags) return choice; } -/** Try to determine which fraction of v2 and v3 directory requests aimed at - * caches will be sent to us. Set *<b>v2_share_out</b> and - * *<b>v3_share_out</b> to the fractions of v2 and v3 protocol shares we - * expect to see, respectively. Return 0 on success, negative on failure. */ -int -router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out) -{ - const routerinfo_t *me = router_get_my_routerinfo(); - const routerstatus_t *rs; - const int pds_flags = PDS_ALLOW_SELF|PDS_IGNORE_FASCISTFIREWALL; - *v2_share_out = *v3_share_out = 0.0; - if (!me) - return -1; - rs = router_get_consensus_status_by_id(me->cache_info.identity_digest); - if (!rs) - return -1; - - /* Calling for side effect */ - /* XXXX This is a bit of a kludge */ - if (rs->is_v2_dir) { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V2_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v2_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V3_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v3_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - return 0; -} - /** Return the dir_server_t for the directory authority whose identity * key hashes to <b>digest</b>, or NULL if no such authority is known. */ @@ -1453,8 +1383,6 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, { const routerstatus_t *choice; int busy = 0; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy); if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) @@ -1479,10 +1407,7 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, /** Pick a random running valid directory server/mirror from our * routerlist. Arguments are as for router_pick_directory_server(), except - * that RETRY_IF_NO_SERVERS is ignored, and: - * - * If the PDS_PREFER_TUNNELED_DIR_CONNS_ flag is set, prefer directory servers - * that we can use with BEGINDIR. + * that RETRY_IF_NO_SERVERS is ignored. */ static const routerstatus_t * router_pick_directory_server_impl(dirinfo_type_t type, int flags) @@ -1496,7 +1421,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) const networkstatus_t *consensus = networkstatus_get_latest_consensus(); int requireother = ! (flags & PDS_ALLOW_SELF); int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); int for_guard = (flags & PDS_FOR_GUARD); int try_excluding = 1, n_excluded = 0; @@ -1529,8 +1453,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) if (requireother && router_digest_is_me(node->identity)) continue; is_trusted = router_digest_is_trusted_dir(node->identity); - if ((type & V2_DIRINFO) && !(node->rs->is_v2_dir || is_trusted)) - continue; if ((type & EXTRAINFO_DIRINFO) && !router_supports_extrainfo(node->identity, 0)) continue; @@ -1551,8 +1473,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; - if (prefer_tunnel && - (!fascistfirewall || + if ((!fascistfirewall || fascist_firewall_allows_address_or(&addr, status->or_port))) smartlist_add(is_trusted ? trusted_tunnel : is_overloaded ? overloaded_tunnel : tunnel, (void*)node); @@ -1639,7 +1560,6 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, time_t now = time(NULL); const int requireother = ! (flags & PDS_ALLOW_SELF); const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); const double auth_weight = (sourcelist == fallback_dir_servers) ? @@ -1700,8 +1620,7 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, } } - if (prefer_tunnel && - d->or_port && + if (d->or_port && (!fascistfirewall || fascist_firewall_allows_address_or(&addr, d->or_port))) smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d); @@ -1757,7 +1676,6 @@ mark_all_dirservers_up(smartlist_t *server_list) routerstatus_t *rs; node_t *node; dir->is_running = 1; - download_status_reset(&dir->v2_ns_dl_status); node = node_get_mutable_by_id(dir->digest); if (node) node->is_running = 1; @@ -1879,7 +1797,7 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) * doubles, convert them to uint64_t, and try to scale them linearly so as to * much of the range of uint64_t. If <b>total_out</b> is provided, set it to * the sum of all elements in the array _before_ scaling. */ -/* private */ void +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out) { @@ -1922,7 +1840,7 @@ gt_i64_timei(uint64_t a, uint64_t b) * value, and return the index of that element. If all elements are 0, choose * an index at random. Return -1 on error. */ -/* private */ int +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) { int i, i_chosen=-1, n_chosen=0; @@ -2015,8 +1933,7 @@ smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, if (compute_weighted_bandwidths(sl, rule, &bandwidths) < 0) return NULL; - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); + scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); { int idx = choose_array_element_by_weight(bandwidths, @@ -2125,7 +2042,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, // Cycle through smartlist and total the bandwidth. SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; double weight = 1; is_exit = node->is_exit && ! node->is_bad_exit; is_guard = node->is_possible_guard; @@ -2148,7 +2065,6 @@ compute_weighted_bandwidths(const smartlist_t *sl, /* We can't use this one. */ continue; } - is_me = router_digest_is_me(node->identity); if (is_guard && is_exit) { weight = (is_dir ? Wdb*Wd : Wd); @@ -2167,8 +2083,6 @@ compute_weighted_bandwidths(const smartlist_t *sl, weight = 0.0; bandwidths[node_sl_idx].dbl = weight*this_bw + 0.5; - if (is_me) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[node_sl_idx].dbl; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2250,7 +2164,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, bitarray_t *fast_bits; bitarray_t *exit_bits; bitarray_t *guard_bits; - int me_idx = -1; // This function does not support WEIGHT_FOR_DIR // or WEIGHT_FOR_MID @@ -2284,9 +2197,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, uint32_t this_bw = 0; i = node_sl_idx; - if (router_digest_is_me(node->identity)) - me_idx = node_sl_idx; - is_exit = node->is_exit; is_guard = node->is_possible_guard; if (node->rs) { @@ -2390,7 +2300,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, if (guard_weight <= 0.0) guard_weight = 0.0; - sl_last_weighted_bw_of_me = 0; for (i=0; i < (unsigned)smartlist_len(sl); i++) { tor_assert(bandwidths[i].dbl >= 0.0); @@ -2402,9 +2311,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, bandwidths[i].dbl *= guard_weight; else if (is_exit) bandwidths[i].dbl *= exit_weight; - - if (i == (unsigned) me_idx) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[i].dbl; } } @@ -2423,8 +2329,7 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, guard_weight, (int)(rule == WEIGHT_FOR_GUARD)); #endif - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); + scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); { int idx = choose_array_element_by_weight(bandwidths, @@ -2816,7 +2721,6 @@ router_get_routerlist(void) routerlist->extra_info_map = eimap_new(); routerlist->desc_store.fname_base = "cached-descriptors"; - routerlist->desc_store.fname_alt_base = "cached-routers"; routerlist->extrainfo_store.fname_base = "cached-extrainfo"; routerlist->desc_store.type = ROUTER_STORE; @@ -2836,7 +2740,6 @@ routerinfo_free(routerinfo_t *router) return; tor_free(router->cache_info.signed_descriptor_body); - tor_free(router->address); tor_free(router->nickname); tor_free(router->platform); tor_free(router->contact_info); @@ -2922,10 +2825,18 @@ routerlist_free(routerlist_t *rl) signed_descriptor_free(sd)); smartlist_free(rl->routers); smartlist_free(rl->old_routers); - if (routerlist->desc_store.mmap) - tor_munmap_file(routerlist->desc_store.mmap); - if (routerlist->extrainfo_store.mmap) - tor_munmap_file(routerlist->extrainfo_store.mmap); + if (rl->desc_store.mmap) { + int res = tor_munmap_file(routerlist->desc_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap"); + } + } + if (rl->extrainfo_store.mmap) { + int res = tor_munmap_file(routerlist->extrainfo_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap"); + } + } tor_free(rl); router_dir_info_changed(); @@ -3412,7 +3323,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(FLAV_NS); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); int in_consensus = 0; tor_assert(msg); @@ -3483,15 +3393,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } /* We no longer need a router with this descriptor digest. */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - routerstatus_t *rs = - networkstatus_v2_find_mutable_entry(ns, id_digest); - if (rs && tor_memeq(rs->descriptor_digest, - router->cache_info.signed_descriptor_digest, - DIGEST_LEN)) - rs->need_to_mirror = 0; - }); if (consensus) { routerstatus_t *rs = networkstatus_vote_find_mutable_entry( consensus, id_digest); @@ -3499,7 +3400,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { in_consensus = 1; - rs->need_to_mirror = 0; } } @@ -3553,7 +3453,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); *msg = authdir_believes_valid ? "Valid server updated" : ("Invalid server updated. (This dirserver is marking your " "server as unapproved.)"); @@ -3575,7 +3474,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3738,11 +3636,7 @@ routerlist_remove_old_routers(void) routerinfo_t *router; signed_descriptor_t *sd; digestset_t *retain; - int caches = directory_caches_dir_info(get_options()); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - int have_enough_v2; - const or_options_t *options = get_options(); trusted_dirs_remove_old_certs(); @@ -3758,38 +3652,10 @@ routerlist_remove_old_routers(void) { /* We'll probably retain everything in the consensus. */ int n_max_retain = smartlist_len(consensus->routerstatus_list); - if (caches && networkstatus_v2_list) { - /* If we care about v2 statuses, we'll retain at most as many as are - listed any of the v2 statues. This will be at least the length of - the largest v2 networkstatus, and in the worst case, this set will be - equal to the sum of the lengths of all v2 consensuses. Take the - worst case. - */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - n_max_retain += smartlist_len(ns->entries)); - } retain = digestset_new(n_max_retain); } cutoff = now - OLD_ROUTER_DESC_MAX_AGE; - /* Build a list of all the descriptors that _anybody_ lists. */ - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - /* XXXX The inner loop here gets pretty expensive, and actually shows up - * on some profiles. It may be the reason digestmap_set shows up in - * profiles too. If instead we kept a per-descriptor digest count of - * how many networkstatuses recommended each descriptor, and changed - * that only when the networkstatuses changed, that would be a speed - * improvement, possibly 1-4% if it also removes digestmap_set from the - * profile. Not worth it for 0.1.2.x, though. The new directory - * system will obsolete this whole thing in 0.2.0.x. */ - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (rs->published_on >= cutoff) - digestset_add(retain, rs->descriptor_digest); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - } - /* Retain anything listed in the consensus. */ if (consensus) { SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, @@ -3797,18 +3663,11 @@ routerlist_remove_old_routers(void) digestset_add(retain, rs->descriptor_digest)); } - /* If we have a consensus, and nearly as many v2 networkstatuses as we want, - * we should consider pruning current routers that are too old and that - * nobody recommends. (If we don't have a consensus or enough v2 - * networkstatuses, then we should get more before we decide to kill - * routers.) */ - /* we set this to true iff we don't care about v2 info, or we have enough. */ - have_enough_v2 = !caches || - !(authdir_mode_any_main(options) || options->FetchV2Networkstatus) || - (networkstatus_v2_list && - smartlist_len(networkstatus_v2_list) > get_n_v2_authorities() / 2); - - if (have_enough_v2 && consensus) { + /* If we have a consensus, we should consider pruning current routers that + * are too old and that nobody recommends. (If we don't have a consensus, + * then we should get one before we decide to kill routers.) */ + + if (consensus) { cutoff = now - ROUTER_MAX_AGE; /* Remove too-old unrecommended members of routerlist->routers. */ for (i = 0; i < smartlist_len(routerlist->routers); ++i) { @@ -4107,8 +3966,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) { const routerstatus_t *rs; networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int caches = directory_caches_dir_info(get_options()); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); if (consensus) { rs = networkstatus_vote_find_entry(consensus, desc->identity_digest); @@ -4116,16 +3973,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) desc->signed_descriptor_digest, DIGEST_LEN)) return 1; } - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - if (!(rs = networkstatus_v2_find_entry(ns, desc->identity_digest))) - continue; - if (tor_memeq(rs->descriptor_digest, - desc->signed_descriptor_digest, DIGEST_LEN)) - return 1; - }); - } return 0; } @@ -4141,7 +3988,7 @@ update_all_descriptor_downloads(time_t now) launch_dummy_descriptor_download_as_needed(now, get_options()); } -/** Clear all our timeouts for fetching v2 and v3 directory stuff, and then +/** Clear all our timeouts for fetching v3 directory stuff, and then * give it all a try again. */ void routerlist_retry_directory_downloads(time_t now) @@ -4520,12 +4367,8 @@ initiate_descriptor_downloads(const routerstatus_t *source, * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we - * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has - * passed. */ + * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */ #define MAX_DL_TO_DELAY 16 -/** When directory clients have only a few servers to request, they batch - * them until they have more, or until this amount of time has passed. */ -#define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60) /** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of * router descriptor digests or microdescriptor digest256s in @@ -4557,7 +4400,7 @@ launch_descriptor_downloads(int purpose, should_delay = 0; } else { should_delay = (last_descriptor_download_attempted + - MAX_CLIENT_INTERVAL_WITHOUT_REQUEST) > now; + options->TestingClientMaxIntervalWithoutRequest) > now; if (!should_delay && n_downloadable) { if (last_descriptor_download_attempted) { log_info(LD_DIR, @@ -4626,152 +4469,6 @@ launch_descriptor_downloads(int purpose, } } -/** Launch downloads for router status as needed, using the strategy used by - * authorities and caches: based on the v2 networkstatuses we have, download - * every descriptor we don't have but would serve, from a random authority - * that lists it. */ -static void -update_router_descriptor_cache_downloads_v2(time_t now) -{ - smartlist_t **downloadable; /* For each authority, what can we dl from it? */ - smartlist_t **download_from; /* ... and, what will we dl from it? */ - digestmap_t *map; /* Which descs are in progress, or assigned? */ - int i, j, n; - int n_download; - const or_options_t *options = get_options(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - - if (! directory_fetches_dir_info_early(options)) { - log_warn(LD_BUG, "Called update_router_descriptor_cache_downloads_v2() " - "on a non-dir-mirror?"); - } - - if (!networkstatus_v2_list || !smartlist_len(networkstatus_v2_list)) - return; - - map = digestmap_new(); - n = smartlist_len(networkstatus_v2_list); - - downloadable = tor_malloc_zero(sizeof(smartlist_t*) * n); - download_from = tor_malloc_zero(sizeof(smartlist_t*) * n); - - /* Set map[d]=1 for the digest of every descriptor that we are currently - * downloading. */ - list_pending_descriptor_downloads(map, 0); - - /* For the digest of every descriptor that we don't have, and that we aren't - * downloading, add d to downloadable[i] if the i'th networkstatus knows - * about that descriptor, and we haven't already failed to get that - * descriptor from the corresponding authority. - */ - n_download = 0; - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - dir_server_t *ds; - smartlist_t *dl; - dl = downloadable[ns_sl_idx] = smartlist_new(); - download_from[ns_sl_idx] = smartlist_new(); - if (ns->published_on + MAX_NETWORKSTATUS_AGE+10*60 < now) { - /* Don't download if the networkstatus is almost ancient. */ - /* Actually, I suspect what's happening here is that we ask - * for the descriptor when we have a given networkstatus, - * and then we get a newer networkstatus, and then we receive - * the descriptor. Having a networkstatus actually expire is - * probably a rare event, and we'll probably be happiest if - * we take this clause out. -RD */ - continue; - } - - /* Don't try dirservers that we think are down -- we might have - * just tried them and just marked them as down. */ - ds = router_get_trusteddirserver_by_digest(ns->identity_digest); - if (ds && !ds->is_running) - continue; - - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t * , rs) { - if (!rs->need_to_mirror) - continue; - if (router_get_by_descriptor_digest(rs->descriptor_digest)) { - log_warn(LD_BUG, - "We have a router descriptor, but need_to_mirror=1."); - rs->need_to_mirror = 0; - continue; - } - if (authdir_mode(options) && dirserv_would_reject_router(rs)) { - rs->need_to_mirror = 0; - continue; - } - if (digestmap_get(map, rs->descriptor_digest)) { - /* We're downloading it already. */ - continue; - } else { - /* We could download it from this guy. */ - smartlist_add(dl, rs->descriptor_digest); - ++n_download; - } - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - - /* At random, assign descriptors to authorities such that: - * - if d is a member of some downloadable[x], d is a member of some - * download_from[y]. (Everything we want to download, we try to download - * from somebody.) - * - If d is a member of download_from[y], d is a member of downloadable[y]. - * (We only try to download descriptors from authorities who claim to have - * them.) - * - No d is a member of download_from[x] and download_from[y] s.t. x != y. - * (We don't try to download anything from two authorities concurrently.) - */ - while (n_download) { - int which_ns = crypto_rand_int(n); - smartlist_t *dl = downloadable[which_ns]; - int idx; - char *d; - if (!smartlist_len(dl)) - continue; - idx = crypto_rand_int(smartlist_len(dl)); - d = smartlist_get(dl, idx); - if (! digestmap_get(map, d)) { - smartlist_add(download_from[which_ns], d); - digestmap_set(map, d, (void*) 1); - } - smartlist_del(dl, idx); - --n_download; - } - - /* Now, we can actually launch our requests. */ - for (i=0; i<n; ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - dir_server_t *ds = - router_get_trusteddirserver_by_digest(ns->identity_digest); - smartlist_t *dl = download_from[i]; - int pds_flags = PDS_RETRY_IF_NO_SERVERS; - if (! authdir_mode_any_nonhidserv(options)) - pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; /* XXXX ignored*/ - - if (!ds) { - log_info(LD_DIR, "Networkstatus with no corresponding authority!"); - continue; - } - if (! smartlist_len(dl)) - continue; - log_info(LD_DIR, "Requesting %d descriptors from authority \"%s\"", - smartlist_len(dl), ds->nickname); - for (j=0; j < smartlist_len(dl); j += MAX_DL_PER_REQUEST) { - initiate_descriptor_downloads(&(ds->fake_status), - DIR_PURPOSE_FETCH_SERVERDESC, dl, j, - j+MAX_DL_PER_REQUEST, pds_flags); - } - } - - for (i=0; i<n; ++i) { - smartlist_free(download_from[i]); - smartlist_free(downloadable[i]); - } - tor_free(download_from); - tor_free(downloadable); - digestmap_free(map,NULL); -} - /** For any descriptor that we want that's currently listed in * <b>consensus</b>, download it as appropriate. */ void @@ -4830,7 +4527,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, continue; /* We have an in-progress download. */ } if (!download_status_is_ready(&rs->dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delayed; /* Not ready for retry. */ continue; } @@ -4932,13 +4629,10 @@ void update_router_descriptor_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!we_fetch_router_descriptors(options)) return; - if (directory_fetches_dir_info_early(options)) { - update_router_descriptor_cache_downloads_v2(now); - } update_consensus_router_descriptor_downloads(now, 0, networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); @@ -4956,7 +4650,7 @@ update_extrainfo_downloads(time_t now) int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0; if (! options->DownloadExtraInfo) return; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!router_have_minimum_dir_info()) return; @@ -4990,7 +4684,7 @@ update_extrainfo_downloads(time_t now) continue; } if (!download_status_is_ready(&sd->ei_dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delay; continue; } @@ -5062,7 +4756,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) } /* If any key fields differ, they're different. */ - if (strcasecmp(r1->address, r2->address) || + if (r1->addr != r2->addr || strcasecmp(r1->nickname, r2->nickname) || r1->or_port != r2->or_port || !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 505685897..6e2f2eaea 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -11,6 +11,8 @@ #ifndef TOR_ROUTERLIST_H #define TOR_ROUTERLIST_H +#include "testsupport.h" + int get_n_authorities(dirinfo_type_t type); int trusted_dirs_reload_certs(void); @@ -53,8 +55,7 @@ const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, int flags); const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); +int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); const routerinfo_t *routerlist_find_my_routerinfo(void); @@ -207,9 +208,10 @@ typedef union u64_dbl_t { double dbl; } u64_dbl_t; -int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries); -void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, - uint64_t *total_out); +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, + int n_entries); +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, + uint64_t *total_out); #endif #endif diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 01f65f262..5add728d6 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -339,7 +339,7 @@ static token_rule_t extrainfo_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the body part of v2 and v3 networkstatus +/** List of tokens recognized in the body part of v3 networkstatus * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), @@ -353,31 +353,6 @@ static token_rule_t rtrstatus_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the header part of v2 networkstatus documents. - */ -static token_rule_t netstatus_token_table[] = { - T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), - T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), - T1( "contact", K_CONTACT, CONCAT_ARGS, NO_OBJ ), - T1( "dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY_1024 ), - T1( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), - T1_START("network-status-version", K_NETWORK_STATUS_VERSION, - GE(1), NO_OBJ ), - T1( "dir-source", K_DIR_SOURCE, GE(3), NO_OBJ ), - T01("dir-options", K_DIR_OPTIONS, ARGS, NO_OBJ ), - T01("client-versions", K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ), - T01("server-versions", K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ), - - END_OF_TABLE -}; - -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ -static token_rule_t dir_footer_token_table[] = { - T1("directory-signature", K_DIRECTORY_SIGNATURE, EQ(1), NEED_OBJ ), - END_OF_TABLE -}; - /** List of tokens common to V3 authority certificates and V3 consensuses. */ #define CERTIFICATE_MEMBERS \ T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION, \ @@ -386,7 +361,7 @@ static token_rule_t dir_footer_token_table[] = { T1("dir-key-published",K_DIR_KEY_PUBLISHED, CONCAT_ARGS, NO_OBJ), \ T1("dir-key-expires", K_DIR_KEY_EXPIRES, CONCAT_ARGS, NO_OBJ), \ T1("dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY ),\ - T01("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ + T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ T1("dir-key-certification", K_DIR_KEY_CERTIFICATION, \ NO_ARGS, NEED_OBJ), \ T01("dir-address", K_DIR_ADDRESS, GE(1), NO_OBJ), @@ -486,8 +461,7 @@ static token_rule_t networkstatus_consensus_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ +/** List of tokens recognized in the footer of v1 directory footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), @@ -598,7 +572,7 @@ dump_desc(const char *desc, const char *type) char *content = tor_malloc_zero(filelen); tor_snprintf(content, filelen, "Unable to parse descriptor of type " "%s:\n%s", type, desc); - write_str_to_file(debugfile, content, 0); + write_str_to_file(debugfile, content, 1); log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " "unparseable-desc in data directory for details.", type); tor_free(content); @@ -629,28 +603,6 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) DIGEST_SHA1); } -/** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers - * string in <b>s</b>. Return 0 on success, -1 on failure. - */ -int -router_get_runningrouters_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n', - DIGEST_SHA1); -} - -/** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status - * string in <b>s</b>. Return 0 on success, -1 on failure. */ -int -router_get_networkstatus_v2_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status-version","\ndirectory-signature", - '\n', - DIGEST_SHA1); -} - /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int @@ -728,7 +680,7 @@ router_get_dirobj_signature(const char *digest, /** Helper: used to generate signatures for routers, directories and * network-status objects. Given a digest in <b>digest</b> and a secret - * <b>private_key</b>, generate an PKCS1-padded signature, BASE64-encode it, + * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it, * surround it with -----BEGIN/END----- pairs, and write it to the * <b>buf_len</b>-byte buffer at <b>buf</b>. Return 0 on success, -1 on * failure. @@ -751,6 +703,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, return -1; } memcpy(buf+s_len, sig, sig_len+1); + tor_free(sig); return 0; } @@ -1232,8 +1185,7 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR,"Router nickname is invalid"); goto err; } - router->address = tor_strdup(tok->args[1]); - if (!tor_inet_aton(router->address, &in)) { + if (!tor_inet_aton(tok->args[1], &in)) { log_warn(LD_DIR,"Router address is not an IP address."); goto err; } @@ -1728,7 +1680,6 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_debug(LD_DIR, "We already checked the signature on this " "certificate; no need to do so again."); found = 1; - cert->is_cross_certified = old_cert->is_cross_certified; } } if (!found) { @@ -1737,18 +1688,14 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } - if ((tok = find_opt_by_keyword(tokens, K_DIR_KEY_CROSSCERT))) { - /* XXXX Once all authorities generate cross-certified certificates, - * make this field mandatory. */ - if (check_signature_token(cert->cache_info.identity_digest, - DIGEST_LEN, - tok, - cert->signing_key, - CST_NO_CHECK_OBJTYPE, - "key cross-certification")) { - goto err; - } - cert->is_cross_certified = 1; + tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT); + if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, + tok, + cert->signing_key, + CST_NO_CHECK_OBJTYPE, + "key cross-certification")) { + goto err; } } @@ -1948,8 +1895,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_named = 1; else if (!strcmp(tok->args[i], "Valid")) rs->is_valid = 1; - else if (!strcmp(tok->args[i], "V2Dir")) - rs->is_v2_dir = 1; else if (!strcmp(tok->args[i], "Guard")) rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) @@ -2084,14 +2029,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, return rs; } -/** Helper to sort a smartlist of pointers to routerstatus_t */ -int -compare_routerstatus_entries(const void **_a, const void **_b) -{ - const routerstatus_t *a = *_a, *b = *_b; - return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); -} - int compare_vote_routerstatus_entries(const void **_a, const void **_b) { @@ -2100,188 +2037,6 @@ compare_vote_routerstatus_entries(const void **_a, const void **_b) DIGEST_LEN); } -/** Helper: used in call to _smartlist_uniq to clear out duplicate entries. */ -static void -free_duplicate_routerstatus_entry_(void *e) -{ - log_warn(LD_DIR, - "Network-status has two entries for the same router. " - "Dropping one."); - routerstatus_free(e); -} - -/** Given a v2 network-status object in <b>s</b>, try to - * parse it and return the result. Return NULL on failure. Check the - * signature of the network status, but do not (yet) check the signing key for - * authority. - */ -networkstatus_v2_t * -networkstatus_v2_parse_from_string(const char *s) -{ - const char *eos, *s_dup = s; - smartlist_t *tokens = smartlist_new(); - smartlist_t *footer_tokens = smartlist_new(); - networkstatus_v2_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - char tmp_digest[DIGEST_LEN]; - struct in_addr in; - directory_token_t *tok; - int i; - memarea_t *area = NULL; - - if (router_get_networkstatus_v2_hash(s, ns_digest)) { - log_warn(LD_DIR, "Unable to compute digest of network-status"); - goto err; - } - - area = memarea_new(); - eos = find_start_of_next_routerstatus(s); - if (tokenize_string(area, s, eos, tokens, netstatus_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status header."); - goto err; - } - ns = tor_malloc_zero(sizeof(networkstatus_v2_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); - - tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); - tor_assert(tok->n_args >= 1); - if (strcmp(tok->args[0], "2")) { - log_warn(LD_BUG, "Got a non-v2 networkstatus. Version was " - "%s", escaped(tok->args[0])); - goto err; - } - - tok = find_by_keyword(tokens, K_DIR_SOURCE); - tor_assert(tok->n_args >= 3); - ns->source_address = tor_strdup(tok->args[0]); - if (tor_inet_aton(tok->args[1], &in) == 0) { - log_warn(LD_DIR, "Error parsing network-status source address %s", - escaped(tok->args[1])); - goto err; - } - ns->source_addr = ntohl(in.s_addr); - ns->source_dirport = - (uint16_t) tor_parse_long(tok->args[2],10,0,65535,NULL,NULL); - if (ns->source_dirport == 0) { - log_warn(LD_DIR, "Directory source without dirport; skipping."); - goto err; - } - - tok = find_by_keyword(tokens, K_FINGERPRINT); - tor_assert(tok->n_args); - if (base16_decode(ns->identity_digest, DIGEST_LEN, tok->args[0], - strlen(tok->args[0]))) { - log_warn(LD_DIR, "Couldn't decode networkstatus fingerprint %s", - escaped(tok->args[0])); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) { - tor_assert(tok->n_args); - ns->contact = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY); - tor_assert(tok->key); - ns->signing_key = tok->key; - tok->key = NULL; - - if (crypto_pk_get_digest(ns->signing_key, tmp_digest)<0) { - log_warn(LD_DIR, "Couldn't compute signing key digest"); - goto err; - } - if (tor_memneq(tmp_digest, ns->identity_digest, DIGEST_LEN)) { - log_warn(LD_DIR, - "network-status fingerprint did not match dir-signing-key"); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_DIR_OPTIONS))) { - for (i=0; i < tok->n_args; ++i) { - if (!strcmp(tok->args[i], "Names")) - ns->binds_names = 1; - if (!strcmp(tok->args[i], "Versions")) - ns->recommends_versions = 1; - if (!strcmp(tok->args[i], "BadExits")) - ns->lists_bad_exits = 1; - if (!strcmp(tok->args[i], "BadDirectories")) - ns->lists_bad_directories = 1; - } - } - - if (ns->recommends_versions) { - if (!(tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) { - log_warn(LD_DIR, "Missing client-versions on versioning directory"); - goto err; - } - ns->client_versions = tor_strdup(tok->args[0]); - - if (!(tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS)) || - tok->n_args<1) { - log_warn(LD_DIR, "Missing server-versions on versioning directory"); - goto err; - } - ns->server_versions = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_PUBLISHED); - tor_assert(tok->n_args == 1); - if (parse_iso_time(tok->args[0], &ns->published_on) < 0) { - goto err; - } - - ns->entries = smartlist_new(); - s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_clear(tokens); - memarea_clear(area); - while (!strcmpstart(s, "r ")) { - routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0, 0))) - smartlist_add(ns->entries, rs); - } - smartlist_sort(ns->entries, compare_routerstatus_entries); - smartlist_uniq(ns->entries, compare_routerstatus_entries, - free_duplicate_routerstatus_entry_); - - if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status footer."); - goto err; - } - if (smartlist_len(footer_tokens) < 1) { - log_warn(LD_DIR, "Too few items in network-status footer."); - goto err; - } - tok = smartlist_get(footer_tokens, smartlist_len(footer_tokens)-1); - if (tok->tp != K_DIRECTORY_SIGNATURE) { - log_warn(LD_DIR, - "Expected network-status footer to end with a signature."); - goto err; - } - - note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, - "network-status") < 0) - goto err; - - goto done; - err: - dump_desc(s_dup, "v2 networkstatus"); - networkstatus_v2_free(ns); - ns = NULL; - done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(footer_tokens); - if (area) { - DUMP_AREA(area, "v2 networkstatus"); - memarea_drop_all(area); - } - return ns; -} - /** Verify the bandwidth weights of a network status document */ int networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) diff --git a/src/or/routerparse.h b/src/or/routerparse.h index eb2e885cb..5d5d9e59e 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -14,8 +14,6 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 @@ -52,9 +50,7 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); -int compare_routerstatus_entries(const void **_a, const void **_b); int compare_vote_routerstatus_entries(const void **_a, const void **_b); -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, diff --git a/src/or/routerset.c b/src/or/routerset.c index 2e41f7f6c..7aee90d6d 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -358,39 +358,6 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, } } -#if 0 -/** Add to <b>target</b> every node_t from <b>source</b> except: - * - * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in - * <b>include</b>; and - * 2) Don't add it if <b>exclude</b> is non-empty and the relay is - * excluded in a more specific fashion by <b>exclude</b>. - * 3) If <b>running_only</b>, don't add non-running routers. - */ -void -routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only) -{ - SMARTLIST_FOREACH(source, const node_t *, node, { - int include_result; - if (running_only && !node->is_running) - continue; - if (!routerset_is_empty(include)) - include_result = routerset_contains_node(include, node); - else - include_result = 1; - - if (include_result) { - int exclude_result = routerset_contains_node(exclude, node); - if (include_result >= exclude_result) - smartlist_add(target, (void*)node); - } - }); -} -#endif - /** Remove every node_t from <b>lst</b> that is in <b>routerset</b>. */ void routerset_subtract_nodes(smartlist_t *lst, const routerset_t *routerset) diff --git a/src/or/routerset.h b/src/or/routerset.h index bfa0c59ac..8261c7fb0 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -32,12 +32,6 @@ void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, const routerset_t *excludeset, int running_only); int routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set); -#if 0 -void routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -#endif void routerset_subtract_nodes(smartlist_t *out, const routerset_t *routerset); diff --git a/src/or/statefile.c b/src/or/statefile.c index bcb7b0741..7b9998fc1 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -4,6 +4,7 @@ * Copyright (c) 2007-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define STATEFILE_PRIVATE #include "or.h" #include "circuitstats.h" #include "config.h" @@ -12,6 +13,7 @@ #include "hibernate.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "statefile.h" /** A list of state-file "abbreviations," for compatibility. */ @@ -90,8 +92,11 @@ static config_var_t state_vars_[] = { #undef VAR #undef V -static int or_state_validate(or_state_t *old_options, or_state_t *options, - int from_setconf, char **msg); +static int or_state_validate(or_state_t *state, char **msg); + +static int or_state_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); /** Magic value for or_state_t. */ #define OR_STATE_MAGIC 0x57A73f57 @@ -109,7 +114,7 @@ static const config_format_t state_format = { STRUCT_OFFSET(or_state_t, magic_), state_abbrevs_, state_vars_, - (validate_fn_t)or_state_validate, + or_state_validate_cb, &state_extra_var, }; @@ -117,8 +122,8 @@ static const config_format_t state_format = { static or_state_t *global_state = NULL; /** Return the persistent state struct for this Tor. */ -or_state_t * -get_or_state(void) +MOCK_IMPL(or_state_t *, +get_or_state, (void)) { tor_assert(global_state); return global_state; @@ -194,21 +199,27 @@ validate_transports_in_state(or_state_t *state) return 0; } -/** Return 0 if every setting in <b>state</b> is reasonable, and a - * permissible transition from <b>old_state</b>. Else warn and return -1. - * Should have no side effects, except for normalizing the contents of - * <b>state</b>. - */ -/* XXX from_setconf is here because of bug 238 */ static int -or_state_validate(or_state_t *old_state, or_state_t *state, - int from_setconf, char **msg) +or_state_validate_cb(void *old_state, void *state, void *default_state, + int from_setconf, char **msg) { /* We don't use these; only options do. Still, we need to match that * signature. */ (void) from_setconf; + (void) default_state; (void) old_state; + return or_state_validate(state, msg); +} + +/** Return 0 if every setting in <b>state</b> is reasonable, and a + * permissible transition from <b>old_state</b>. Else warn and return -1. + * Should have no side effects, except for normalizing the contents of + * <b>state</b>. + */ +static int +or_state_validate(or_state_t *state, char **msg) +{ if (entry_guards_parse_state(state, 0, msg)<0) return -1; @@ -237,7 +248,8 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + if (circuit_build_times_parse_state( + get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } return ret; @@ -249,7 +261,7 @@ or_state_set(or_state_t *new_state) static void or_state_save_broken(char *fname) { - int i; + int i, res; file_status_t status; char *fname2 = NULL; for (i = 0; i < 100; ++i) { @@ -263,12 +275,18 @@ or_state_save_broken(char *fname) log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " "state files to move aside. Discarding the old state file.", fname); - unlink(fname); + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Also couldn't discard old state file \"%s\" because " + "unlink() failed: %s", + fname, strerror(errno)); + } } else { log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " "to \"%s\". This could be a bug in Tor; please tell " "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { + if (tor_rename(fname, fname2) < 0) {//XXXX sandbox prohibits log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " "OS gave an error of %s", strerror(errno)); } @@ -276,6 +294,16 @@ or_state_save_broken(char *fname) tor_free(fname2); } +STATIC or_state_t * +or_state_new(void) +{ + or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t)); + new_state->magic_ = OR_STATE_MAGIC; + config_init(&state_format, new_state); + + return new_state; +} + /** Reload the persistent state from disk, generating a new state as needed. * Return 0 on success, less than 0 on failure. */ @@ -303,9 +331,7 @@ or_state_load(void) log_warn(LD_GENERAL,"State file \"%s\" is not a file? Failing.", fname); goto done; } - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); if (contents) { config_line_t *lines=NULL; int assign_retval; @@ -322,7 +348,7 @@ or_state_load(void) } } - if (!badstate && or_state_validate(NULL, new_state, 1, &errmsg) < 0) + if (!badstate && or_state_validate(new_state, &errmsg) < 0) badstate = 1; if (errmsg) { @@ -340,9 +366,7 @@ or_state_load(void) tor_free(contents); config_free(&state_format, new_state); - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); } else if (contents) { log_info(LD_GENERAL, "Loaded state from \"%s\"", fname); } else { @@ -404,7 +428,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); - circuit_build_times_update_state(&circ_times, global_state); + circuit_build_times_update_state(get_circuit_build_times(), global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); @@ -449,7 +473,7 @@ or_state_save(time_t now) /** Return the config line for transport <b>transport</b> in the current state. * Return NULL if there is no config line for <b>transport</b>. */ -static config_line_t * +STATIC config_line_t * get_transport_in_state_by_name(const char *transport) { or_state_t *or_state = get_or_state(); @@ -607,10 +631,19 @@ save_transport_to_state(const char *transport, tor_free(transport_addrport); } +STATIC void +or_state_free(or_state_t *state) +{ + if (!state) + return; + + config_free(&state_format, state); +} + void or_state_free_all(void) { - config_free(&state_format, global_state); + or_state_free(global_state); global_state = NULL; } diff --git a/src/or/statefile.h b/src/or/statefile.h index dcdee6c60..15bb0b4aa 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -7,7 +7,7 @@ #ifndef TOR_STATEFILE_H #define TOR_STATEFILE_H -or_state_t *get_or_state(void); +MOCK_DECL(or_state_t *,get_or_state,(void)); int did_last_state_file_write_fail(void); int or_state_save(time_t now); @@ -18,5 +18,11 @@ int or_state_load(void); int or_state_loaded(void); void or_state_free_all(void); +#ifdef STATEFILE_PRIVATE +STATIC config_line_t *get_transport_in_state_by_name(const char *transport); +STATIC void or_state_free(or_state_t *state); +STATIC or_state_t *or_state_new(void); +#endif + #endif diff --git a/src/or/status.c b/src/or/status.c index 69f92ed09..afaa9de84 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -6,7 +6,10 @@ * \brief Keep status information and log the heartbeat messages. **/ +#define STATUS_PRIVATE + #include "or.h" +#include "circuituse.h" #include "config.h" #include "status.h" #include "nodelist.h" @@ -14,17 +17,21 @@ #include "router.h" #include "circuitlist.h" #include "main.h" +#include "rephist.h" #include "hibernate.h" #include "rephist.h" +#include "statefile.h" + +static void log_accounting(const time_t now, const or_options_t *options); /** Return the total number of circuits. */ -static int +STATIC int count_circuits(void) { circuit_t *circ; int nr=0; - for (circ = circuit_get_global_list_(); circ; circ = circ->next) + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) nr++; return nr; @@ -32,7 +39,7 @@ count_circuits(void) /** Take seconds <b>secs</b> and return a newly allocated human-readable * uptime string */ -static char * +STATIC char * secs_to_uptime(long secs) { long int days = secs / 86400; @@ -59,7 +66,7 @@ secs_to_uptime(long secs) /** Take <b>bytes</b> and returns a newly allocated human-readable usage * string. */ -static char * +STATIC char * bytes_to_usage(uint64_t bytes) { char *bw_string = NULL; @@ -112,6 +119,10 @@ log_heartbeat(time_t now) uptime, count_circuits(),bw_sent,bw_rcvd, hibernating?" We are currently hibernating.":""); + if (server_mode(options) && accounting_is_enabled(options) && !hibernating) { + log_accounting(now, options); + } + if (stats_n_data_cells_packaged && !hibernating) log_notice(LD_HEARTBEAT, "Average packaged cell fullness: %2.3f%%", 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / @@ -125,6 +136,8 @@ log_heartbeat(time_t now) if (public_server_mode(options)) rep_hist_log_circuit_handshake_stats(now); + circuit_log_ancient_one_hop_circuits(1800); + tor_free(uptime); tor_free(bw_sent); tor_free(bw_rcvd); @@ -132,3 +145,27 @@ log_heartbeat(time_t now) return 0; } +static void +log_accounting(const time_t now, const or_options_t *options) +{ + or_state_t *state = get_or_state(); + char *acc_rcvd = bytes_to_usage(state->AccountingBytesReadInInterval); + char *acc_sent = bytes_to_usage(state->AccountingBytesWrittenInInterval); + char *acc_max = bytes_to_usage(options->AccountingMax); + time_t interval_end = accounting_get_end_time(); + char end_buf[ISO_TIME_LEN + 1]; + char *remaining = NULL; + format_local_iso_time(end_buf, interval_end); + remaining = secs_to_uptime(interval_end - now); + + log_notice(LD_HEARTBEAT, "Heartbeat: Accounting enabled. " + "Sent: %s / %s, Received: %s / %s. The " + "current accounting interval ends on %s, in %s.", + acc_sent, acc_max, acc_rcvd, acc_max, end_buf, remaining); + + tor_free(acc_rcvd); + tor_free(acc_sent); + tor_free(acc_max); + tor_free(remaining); +} + diff --git a/src/or/status.h b/src/or/status.h index 7c3b969c8..13458ea47 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -4,7 +4,15 @@ #ifndef TOR_STATUS_H #define TOR_STATUS_H +#include "testsupport.h" + int log_heartbeat(time_t now); +#ifdef STATUS_PRIVATE +STATIC int count_circuits(void); +STATIC char *secs_to_uptime(long secs); +STATIC char *bytes_to_usage(uint64_t bytes); +#endif + #endif diff --git a/src/or/transports.c b/src/or/transports.c index 3749d6bb2..dc3075416 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -51,35 +51,37 @@ * logic, because of race conditions that can cause dangling * pointers. ] * - * <b>In even more detail, this is what happens when a SIGHUP - * occurs:</b> + * <b>In even more detail, this is what happens when a config read + * (like a SIGHUP or a SETCONF) occurs:</b> * * We immediately destroy all unconfigured proxies (We shouldn't have - * unconfigured proxies in the first place, except when SIGHUP rings - * immediately after tor is launched.). + * unconfigured proxies in the first place, except when the config + * read happens immediately after tor is launched.). * * We mark all managed proxies and transports to signify that they * must be removed if they don't contribute by the new torrc * (we mark using the <b>marked_for_removal</b> element). * We also mark all managed proxies to signify that they might need to * be restarted so that they end up supporting all the transports the - * new torrc wants them to support (using the <b>got_hup</b> element). + * new torrc wants them to support + * (we mark using the <b>was_around_before_config_read</b> element). * We also clear their <b>transports_to_launch</b> list so that we can * put there the transports we need to launch according to the new * torrc. * * We then start parsing torrc again. * - * Everytime we encounter a transport line using a known pre-SIGHUP - * managed proxy, we cleanse that proxy from the removal mark. - * We also mark it as unconfigured so that on the next scheduled - * events tick, we investigate whether we need to restart the proxy - * so that it also spawns the new transports. - * If the post-SIGHUP <b>transports_to_launch</b> list is identical to - * the pre-SIGHUP one, it means that no changes were introduced to - * this proxy during the SIGHUP and no restart has to take place. + * Everytime we encounter a transport line using a managed proxy that + * was around before the config read, we cleanse that proxy from the + * removal mark. We also toggle the <b>check_if_restarts_needed</b> + * flag, so that on the next <b>pt_configure_remaining_proxies</b> + * tick, we investigate whether we need to restart the proxy so that + * it also spawns the new transports. If the post-config-read + * <b>transports_to_launch</b> list is identical to the pre-config-read + * one, it means that no changes were introduced to this proxy during + * the config read and no restart has to take place. * - * During the post-SIGHUP torrc parsing, we unmark all transports + * During the post-config-read torrc parsing, we unmark all transports * spawned by managed proxies that we find in our torrc. * We do that so that if we don't need to restart a managed proxy, we * can continue using its old transports normally. @@ -95,18 +97,17 @@ #include "util.h" #include "router.h" #include "statefile.h" +#include "entrynodes.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp); static INLINE int proxy_configuration_finished(const managed_proxy_t *mp); -static void managed_proxy_destroy(managed_proxy_t *mp, - int also_terminate_process); - static void handle_finished_proxy(managed_proxy_t *mp); -static int configure_proxy(managed_proxy_t *mp); - static void parse_method_error(const char *line, int is_server_method); #define parse_server_method_error(l) parse_method_error(l, 1) #define parse_client_method_error(l) parse_method_error(l, 0) @@ -136,7 +137,8 @@ static smartlist_t *transport_list = NULL; SOCKS version <b>socks_ver</b>. */ static transport_t * transport_new(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver) + const char *name, int socks_ver, + const char *extra_info_args) { transport_t *t = tor_malloc_zero(sizeof(transport_t)); @@ -144,6 +146,8 @@ transport_new(const tor_addr_t *addr, uint16_t port, t->port = port; t->name = tor_strdup(name); t->socks_version = socks_ver; + if (extra_info_args) + t->extra_info_args = tor_strdup(extra_info_args); return t; } @@ -156,6 +160,7 @@ transport_free(transport_t *transport) return; tor_free(transport->name); + tor_free(transport->extra_info_args); tor_free(transport); } @@ -323,7 +328,7 @@ int transport_add_from_config(const tor_addr_t *addr, uint16_t port, const char *name, int socks_ver) { - transport_t *t = transport_new(addr, port, name, socks_ver); + transport_t *t = transport_new(addr, port, name, socks_ver, NULL); int r = transport_add(t); @@ -531,8 +536,7 @@ launch_managed_proxy(managed_proxy_t *mp) } /** Check if any of the managed proxies we are currently trying to - * configure have anything new to say. This is called from - * run_scheduled_events(). */ + * configure has anything new to say. */ void pt_configure_remaining_proxies(void) { @@ -549,14 +553,15 @@ pt_configure_remaining_proxies(void) assert_unconfigured_count_ok(); SMARTLIST_FOREACH_BEGIN(tmp, managed_proxy_t *, mp) { - tor_assert(mp->conf_state != PT_PROTO_BROKEN || + tor_assert(mp->conf_state != PT_PROTO_BROKEN && mp->conf_state != PT_PROTO_FAILED_LAUNCH); - if (mp->got_hup) { - mp->got_hup = 0; + if (mp->was_around_before_config_read) { + /* This proxy is marked by a config read. Check whether we need + to restart it. */ + + mp->was_around_before_config_read = 0; - /* This proxy is marked by a SIGHUP. Check whether we need to - restart it. */ if (proxy_needs_restart(mp)) { log_info(LD_GENERAL, "Preparing managed proxy '%s' for restart.", mp->argv[0]); @@ -589,7 +594,7 @@ pt_configure_remaining_proxies(void) * Return 1 if the transport configuration finished, and return 0 * otherwise (if we still have more configuring to do for this * proxy). */ -static int +STATIC int configure_proxy(managed_proxy_t *mp) { int configuration_finished = 0; @@ -657,6 +662,7 @@ register_server_proxy(const managed_proxy_t *mp) save_transport_to_state(t->name, &t->addr, t->port); log_notice(LD_GENERAL, "Registered server transport '%s' at '%s'", t->name, fmt_addrport(&t->addr, t->port)); + control_event_transport_launched("server", t->name, &t->addr, t->port); } SMARTLIST_FOREACH_END(t); } @@ -679,9 +685,11 @@ register_client_proxy(const managed_proxy_t *mp) break; case 0: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); break; case 1: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); transport_free(transport_tmp); break; } @@ -699,7 +707,7 @@ register_proxy(const managed_proxy_t *mp) } /** Free memory allocated by managed proxy <b>mp</b>. */ -static void +STATIC void managed_proxy_destroy(managed_proxy_t *mp, int also_terminate_process) { @@ -713,7 +721,8 @@ managed_proxy_destroy(managed_proxy_t *mp, smartlist_free(mp->transports_to_launch); /* remove it from the list of managed proxies */ - smartlist_remove(managed_proxy_list, mp); + if (managed_proxy_list) + smartlist_remove(managed_proxy_list, mp); /* free the argv */ free_execve_args(mp->argv); @@ -750,7 +759,6 @@ handle_finished_proxy(managed_proxy_t *mp) } unconfigured_proxies_n--; - tor_assert(unconfigured_proxies_n >= 0); } /** Return true if the configuration of the managed proxy <b>mp</b> is @@ -781,7 +789,7 @@ handle_methods_done(const managed_proxy_t *mp) /** Handle a configuration protocol <b>line</b> received from a * managed proxy <b>mp</b>. */ -void +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp) { log_info(LD_GENERAL, "Got a line from managed proxy '%s': (%s)", @@ -882,7 +890,7 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) } /** Parses an ENV-ERROR <b>line</b> and warns the user accordingly. */ -void +STATIC void parse_env_error(const char *line) { /* (Length of the protocol string) plus (a space) and (the first char of @@ -898,7 +906,7 @@ parse_env_error(const char *line) /** Handles a VERSION <b>line</b>. Updates the configuration protocol * version in <b>mp</b>. */ -int +STATIC int parse_version(const char *line, managed_proxy_t *mp) { if (strlen(line) < (strlen(PROTO_NEG_SUCCESS) + 2)) { @@ -939,14 +947,14 @@ parse_method_error(const char *line, int is_server) /** Parses an SMETHOD <b>line</b> and if well-formed it registers the * new transport in <b>mp</b>. */ -int +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp) { int r; smartlist_t *items = NULL; char *method_name=NULL; - + char *args_string=NULL; char *addrport=NULL; tor_addr_t tor_addr; char *address=NULL; @@ -963,6 +971,9 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } + /* Example of legit SMETHOD line: + SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ + tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD)); method_name = smartlist_get(items,1); @@ -990,7 +1001,19 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, PROXY_NONE); + if (smartlist_len(items) > 3) { + /* Seems like there are also some [options] in the SMETHOD line. + Let's see if we can parse them. */ + char *options_string = smartlist_get(items, 3); + log_debug(LD_CONFIG, "Got options_string: %s", options_string); + if (!strcmpstart(options_string, "ARGS:")) { + args_string = options_string+strlen("ARGS:"); + log_debug(LD_CONFIG, "Got ARGS: %s", args_string); + } + } + + transport = transport_new(&tor_addr, port, method_name, + PROXY_NONE, args_string); if (!transport) goto err; @@ -1016,7 +1039,7 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) /** Parses a CMETHOD <b>line</b>, and if well-formed it registers * the new transport in <b>mp</b>. */ -int +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp) { int r; @@ -1082,7 +1105,7 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, socks_ver); + transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL); if (!transport) goto err; @@ -1105,6 +1128,50 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Return a newly allocated string that tor should place in + * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server + * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ +STATIC char * +get_transport_options_for_server_proxy(const managed_proxy_t *mp) +{ + char *options_string = NULL; + smartlist_t *string_sl = smartlist_new(); + + tor_assert(mp->is_server); + + /** Loop over the transports of the proxy. If we have options for + any of them, format them appropriately and place them in our + smartlist. Finally, join our smartlist to get the final + string. */ + SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) { + smartlist_t *options_tmp_sl = NULL; + options_tmp_sl = get_options_for_server_transport(transport); + if (!options_tmp_sl) + continue; + + /** Loop over the options of this transport, escape them, and + place them in the smartlist. */ + SMARTLIST_FOREACH_BEGIN(options_tmp_sl, const char *, options) { + char *escaped_opts = tor_escape_str_for_pt_args(options, ":;\\"); + smartlist_add_asprintf(string_sl, "%s:%s", + transport, escaped_opts); + tor_free(escaped_opts); + } SMARTLIST_FOREACH_END(options); + + SMARTLIST_FOREACH(options_tmp_sl, char *, c, tor_free(c)); + smartlist_free(options_tmp_sl); + } SMARTLIST_FOREACH_END(transport); + + if (smartlist_len(string_sl)) { + options_string = smartlist_join_strings(string_sl, ";", 0, NULL); + } + + SMARTLIST_FOREACH(string_sl, char *, t, tor_free(t)); + smartlist_free(string_sl); + + return options_string; +} + /** Return the string that tor should place in TOR_PT_SERVER_BINDADDR * while configuring the server managed proxy in <b>mp</b>. The * string is stored in the heap, and it's the the responsibility of @@ -1139,6 +1206,8 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp) { + const or_options_t *options = get_options(); + /* Environment variables to be added to or set in mp's environment. */ smartlist_t *envs = smartlist_new(); /* XXXX The next time someone touches this code, shorten the name of @@ -1176,8 +1245,10 @@ create_managed_proxy_environment(const managed_proxy_t *mp) { char *orport_tmp = get_first_listener_addrport_string(CONN_TYPE_OR_LISTENER); - smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); - tor_free(orport_tmp); + if (orport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); + tor_free(orport_tmp); + } } { @@ -1186,13 +1257,41 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free(bindaddr_tmp); } + { + char *server_transport_options = + get_transport_options_for_server_proxy(mp); + if (server_transport_options) { + smartlist_add_asprintf(envs, "TOR_PT_SERVER_TRANSPORT_OPTIONS=%s", + server_transport_options); + tor_free(server_transport_options); + } + } + /* XXX024 Remove the '=' here once versions of obfsproxy which * assert that this env var exists are sufficiently dead. * * (If we remove this line entirely, some joker will stick this * variable in Tor's environment and crash PTs that try to parse * it even when not run in server mode.) */ - smartlist_add(envs, tor_strdup("TOR_PT_EXTENDED_SERVER_PORT=")); + + if (options->ExtORPort_lines) { + char *ext_or_addrport_tmp = + get_first_listener_addrport_string(CONN_TYPE_EXT_OR_LISTENER); + char *cookie_file_loc = get_ext_or_auth_cookie_file_name(); + + if (ext_or_addrport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s", + ext_or_addrport_tmp); + } + smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s", + cookie_file_loc); + + tor_free(ext_or_addrport_tmp); + tor_free(cookie_file_loc); + + } else { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT="); + } } SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) { @@ -1216,7 +1315,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp) * <b>proxy_argv</b>. * * Requires that proxy_argv have at least one element. */ -static managed_proxy_t * +STATIC managed_proxy_t * managed_proxy_create(const smartlist_t *transport_list, char **proxy_argv, int is_server) { @@ -1267,19 +1366,20 @@ pt_kickstart_proxy(const smartlist_t *transport_list, managed_proxy_create(transport_list, proxy_argv, is_server); } else { /* known proxy. add its transport to its transport list */ - if (mp->got_hup) { - /* If the managed proxy we found is marked by a SIGHUP, it means - that it's not useless and should be kept. If it's marked for - removal, unmark it and increase the unconfigured proxies so - that we try to restart it if we need to. Afterwards, check if - a transport_t for 'transport' used to exist before the SIGHUP - and make sure it doesn't get deleted because we might reuse - it. */ + if (mp->was_around_before_config_read) { + /* If this managed proxy was around even before we read the + config this time, it means that it was already enabled before + and is not useless and should be kept. If it's marked for + removal, unmark it and make sure that we check whether it + needs to be restarted. */ if (mp->marked_for_removal) { mp->marked_for_removal = 0; check_if_restarts_needed = 1; } + /* For each new transport, check if the managed proxy used to + support it before the SIGHUP. If that was the case, make sure + it doesn't get removed because we might reuse it. */ SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport) { old_transport = transport_get_by_name(transport); if (old_transport) @@ -1328,8 +1428,10 @@ pt_prepare_proxy_list_for_config_read(void) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + /* Mark all proxies for removal, and also note that they have been + here before the config read. */ mp->marked_for_removal = 1; - mp->got_hup = 1; + mp->was_around_before_config_read = 1; SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t)); smartlist_clear(mp->transports_to_launch); } SMARTLIST_FOREACH_END(mp); @@ -1390,6 +1492,8 @@ pt_get_extra_info_descriptor_string(void) tor_assert(mp->transports); SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) { + char *transport_args = NULL; + /* If the transport proxy returned "0.0.0.0" as its address, and * we know our external IP address, use it. Otherwise, use the * returned address. */ @@ -1405,9 +1509,16 @@ pt_get_extra_info_descriptor_string(void) addrport = fmt_addrport(&t->addr, t->port); } + /* If this transport has any arguments with it, prepend a space + to them so that we can add them to the transport line. */ + if (t->extra_info_args) + tor_asprintf(&transport_args, " %s", t->extra_info_args); + smartlist_add_asprintf(string_chunks, - "transport %s %s", - t->name, addrport); + "transport %s %s%s", + t->name, addrport, + transport_args ? transport_args : ""); + tor_free(transport_args); } SMARTLIST_FOREACH_END(t); } SMARTLIST_FOREACH_END(mp); @@ -1426,6 +1537,57 @@ pt_get_extra_info_descriptor_string(void) return the_string; } +/** Stringify the SOCKS arguments in <b>socks_args</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_stringify_socks_args(const smartlist_t *socks_args) +{ + /* tmp place to store escaped socks arguments, so that we can + concatenate them up afterwards */ + smartlist_t *sl_tmp = NULL; + char *escaped_string = NULL; + char *new_string = NULL; + + tor_assert(socks_args); + tor_assert(smartlist_len(socks_args) > 0); + + sl_tmp = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(socks_args, const char *, s) { + /* Escape ';' and '\'. */ + escaped_string = tor_escape_str_for_pt_args(s, ";\\"); + if (!escaped_string) + goto done; + + smartlist_add(sl_tmp, escaped_string); + } SMARTLIST_FOREACH_END(s); + + new_string = smartlist_join_strings(sl_tmp, ";", 0, NULL); + + done: + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + + return new_string; +} + +/** Return a string of the SOCKS arguments that we should pass to the + * pluggable transports proxy in <b>addr</b>:<b>port</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, uint16_t port) +{ + const smartlist_t *socks_args = NULL; + + socks_args = get_socks_args_by_bridge_addrport(addr, port); + if (!socks_args) + return NULL; + + return pt_stringify_socks_args(socks_args); +} + /** The tor config was read. * Destroy all managed proxies that were marked by a previous call to * prepare_proxy_list_for_config_read() and are not used by the new diff --git a/src/or/transports.h b/src/or/transports.h index 6ee82f455..1365ead00 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -25,6 +25,9 @@ typedef struct transport_t { /** Boolean: We are re-parsing our transport list, and we are going to remove * this one if we don't find it in the list of configured transports. */ unsigned marked_for_removal : 1; + /** Arguments for this transport that must be written to the + extra-info descriptor. */ + char *extra_info_args; } transport_t; void mark_transport_list(void); @@ -55,6 +58,10 @@ void pt_prepare_proxy_list_for_config_read(void); void sweep_proxy_list(void); smartlist_t *get_transport_proxy_ports(void); +char *pt_stringify_socks_args(const smartlist_t *socks_args); + +char *pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, + uint16_t port); #ifdef PT_PRIVATE /** State of the managed proxy configuration protocol. */ @@ -90,7 +97,7 @@ typedef struct { * this flag to signify that this proxy might need to be restarted * so that it can listen for other transports according to the new * torrc. */ - unsigned int got_hup : 1; + unsigned int was_around_before_config_read : 1; /* transports to-be-launched by this proxy */ smartlist_t *transports_to_launch; @@ -100,12 +107,21 @@ typedef struct { smartlist_t *transports; } managed_proxy_t; -int parse_cmethod_line(const char *line, managed_proxy_t *mp); -int parse_smethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); + +STATIC int parse_version(const char *line, managed_proxy_t *mp); +STATIC void parse_env_error(const char *line); +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); + +STATIC void managed_proxy_destroy(managed_proxy_t *mp, + int also_terminate_process); + +STATIC managed_proxy_t *managed_proxy_create(const smartlist_t *transport_list, + char **proxy_argv, int is_server); -int parse_version(const char *line, managed_proxy_t *mp); -void parse_env_error(const char *line); -void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC int configure_proxy(managed_proxy_t *mp); #endif |