diff options
| -rw-r--r-- | ChangeLog | 372 | ||||
| -rw-r--r-- | changes/11543 | 8 | ||||
| -rw-r--r-- | changes/bug9963 | 5 | ||||
| -rw-r--r-- | changes/ticket11528 | 6 | ||||
| -rw-r--r-- | configure.ac | 2 | ||||
| -rwxr-xr-x | contrib/format_changelog.py | 141 | ||||
| -rw-r--r-- | doc/tor.1.txt | 3 | ||||
| -rw-r--r-- | src/common/compat.c | 119 | ||||
| -rw-r--r-- | src/common/compat.h | 2 | ||||
| -rw-r--r-- | src/or/channel.c | 4 | ||||
| -rw-r--r-- | src/or/channel.h | 9 | ||||
| -rw-r--r-- | src/or/circuitbuild.c | 74 | ||||
| -rw-r--r-- | src/or/circuitlist.c | 14 | ||||
| -rw-r--r-- | src/or/config.c | 80 | ||||
| -rw-r--r-- | src/or/connection.c | 2 | ||||
| -rwxr-xr-x | src/or/control.c | 4 | ||||
| -rw-r--r-- | src/or/dnsserv.c | 29 | ||||
| -rw-r--r-- | src/or/entrynodes.c | 36 | ||||
| -rw-r--r-- | src/or/main.c | 1 | ||||
| -rw-r--r-- | src/or/networkstatus.c | 8 | ||||
| -rw-r--r-- | src/or/or.h | 5 | ||||
| -rw-r--r-- | src/or/transports.c | 63 | ||||
| -rw-r--r-- | src/or/transports.h | 2 | ||||
| -rw-r--r-- | src/test/test_util.c | 29 |
24 files changed, 749 insertions, 269 deletions
@@ -1,87 +1,168 @@ Changes in version 0.2.5.4-alpha - 2014-04-?? + This release includes several security and performance improvements + for clients and relays, including XXX - o Major features (security): - - Block authority signing keys that were used on an authorities - vulnerable to the "heartbleed" bug in openssl (CVE-2014-0160). - (We don't have any evidence that these keys _were_ compromised; - we're doing this to be prudent.) Resolves ticket 11464. + This release marks end-of-line for Tor 0.2.2.x; those Tor versions + have accumulated many known flaws; everyone should upgrade. - o Deprecated versions: - - Tor 0.2.2.x has reached end-of-life; it has received no patches or - attention for some while. Directory authorities no longer accept - descriptors from Tor relays running any version of Tor prior to Tor - 0.2.3.16-alpha. Resolves ticket 11149. + o Major features (security): + - Block authority signing keys that were used on authorities + vulnerable to the "heartbleed" bug in OpenSSL (CVE-2014-0160). (We + don't have any evidence that these keys _were_ compromised; we're + doing this to be prudent.) Resolves ticket 11464. o Major features (relay performance): - - Faster server-side lookups of rendezvous and introduction point - circuits by using hashtables instead of linear searches over all - the circuits. These functions previously accounted between 3 and - 7% of CPU usage on some busy relays. - - o Major features (seccomp2 sandbox): - - Refinements and improvements to the Linux seccomp2 sandbox code: - the sandbox can now run a test network for multiple hours without - crashing. (Previous crash reasons included: reseeding the OpenSSL PRNG, - seeding the Libevent PRNG, using the wrong combination of CLOEXEC and - NONBLOCK at the same place and time, having server keys, being an - authority, receiving a HUP, or using IPv6.) The sandbox is still - experimental, and more bugs will probably turn up. To try it, - enable "Sandbox 1" on a Linux host. - - Strengthen the Linux seccomp2 sandbox code: the sandbox can now - test the arguments for rename(), and blocks _sysctl() entirely. - - When the Linux syscall sandbox finds an illegal system call, it - now tries to log a stack trace before exiting. Resolves ticket - 11465. + - Speed up server-side lookups of rendezvous and introduction point + circuits by using hashtables instead of linear searches. These + functions previously accounted between 3 and 7% of CPU usage on + some busy relays. Resolves ticket 9841. + - Avoid wasting CPU when extending a circuit over a channel that is + nearly out of circuit IDs. Previously, we would in the worst case + do a linear scan over all possible circuit IDs before deciding + that we had exhausted our possibilities. Now, we try at most 64 + random circuit IDs before deciding that we probably won't succeed. + Fix for a possible root cause of ticket #11553. + + o Major features (seccomp2 sandbox, Linux only): + - The seccomp2 sandbox can now run a test network for multiple hours + without crashing. The sandbox is still experimental, and more bugs + will probably turn up. To try it, enable "Sandbox 1" on a Linux + host. + - Strengthen sandbox code: the sandbox can now test the arguments + for rename(), and blocks _sysctl() entirely. + - When the sandbox blocks a system call, it now tries to log a stack + trace before exiting. Resolves ticket 11465. o Major bugfixes (TLS cipher selection): - - Generate the server's preference list for ciphersuites - automatically based on uniform criteria, and considering all - OpenSSL ciphersuites with acceptable strength and forward - secrecy. (The sort order is: prefer AES to 3DES; break ties by - preferring ECDHE to DHE; break ties by preferring GCM to CBC; - break ties by preferring SHA384 to SHA256 to SHA1; and finally, - break ties by preferring AES256 to AES128.) This resolves bugs - #11513, #11492, #11498, #11499. Bugs reported by 'cypherpunks'. - Bugfix on 0.2.4.8-alpha. + - The relay ciphersuite list is now generated automatically based on + uniform criteria, and includes all OpenSSL ciphersuites with + acceptable strength and forward secrecy. Previously, we had + omitted some perfectly fine ciphersuites. Resolves bugs #11513, + #11492, #11498, #11499. Bugs reported by 'cypherpunks'. Bugfix on + 0.2.4.8-alpha. + - Relays now trust themselves to have a better view than clients of + which TLS ciphersuites are better than others. (Thanks to #11513, + the relay list is now well-considered, whereas the client list has + been chosen mainly for anti-fingerprinting purposes.) Relays + prefer: AES over 3DES; then ECDHE over DHE; then GCM over CBC; + then SHA384 over SHA256 over SHA1; and last, AES256 over AES128. + Resolves ticket 11528. + - Clients now try to advertise the same list of ciphersuites as + Firefox 28. This change enables selection of (fast) GCM + ciphersuites, disables some strange old ciphers, and disables the + ECDH (not to be confused with ECDHE) ciphersuites. Resolves ticket + 11438. o Major bugfixes (undefined behavior): - - Fix two instances of possible undefined behavior in channeltls.c - that could, under unlucky circumstances, have led to a pointer - overflow. Fixes bug #10363; bugfixes on 0.2.0.10-alpha and - 0.2.3.6-alpha. Reported by "bobnomnom". - - Fix another possibly undefined pointer operations in tor_memmem - fallback implementation. Another case of bug #10363; bugfix on - 0.1.1.1-alpha. - - Fix another possibly undefined pointer operations in the eventdns - fallback implementation. Another case of bug #10363; bugfix on - 0.1.2.1-alpha. - - Use AddressSanitizer and Ubsan sanitizers (in clang-3.4) to fix some - miscellaneous errors in our tests and codebase. Fix for bug 11232. - Bugfixes on versions back as far as 0.2.1.11-alpha. + - Fix various instances of undefined behavior in channeltls.c, + tor_memmem(), and eventdns.c, that would cause us to construct + pointers to memory outside an allocated object. (These invalid + pointers were not accessed, but C does not even allow them to + exist.) Fixes bug #10363; bugfixes on 0.2.0.10-alpha, + 0.2.3.6-alpha, 0.1.1.1-alpha, and 0.1.2.1-alpha. Reported by + "bobnomnom". + - Use the AddressSanitizer and Ubsan sanitizers (in clang-3.4) to + fix some miscellaneous errors in our tests and codebase. Fix for + bug 11232. Bugfixes on versions back as far as 0.2.1.11-alpha. + + o Minor features (Transparent proxy, *BSD): + - Support FreeBSD's ipfw firewall interface for TransPort ports. on + FreeBSD. To enable it, set "TransProxyType ipfw". Resolves ticket + 10267; patch from "yurivict". + - Support OpenBSD's divert-to rules with the pf firewall for + transparent proxy ports. To enable it, set "TransProxyType pf- + divert". This allows Tor to run a TransPort transparent proxy port + on OpenBSD 4.4 or later without root privileges. See the + pf.conf(5) manual page for information on configuring pf to use + divert-to rules. Closes ticket 10896; patch from Dana Koch. + + o Minor features (security): + - New --enable-expensive-hardening option to enable security + hardening options that consume nontrivial amounts of CPU and + memory. Right now, this includes AddressSanitizer and UbSan, which + are supported in newer versions of GCC and Clang. Closes ticket + 11477. + - If you don't specify MaxMemInQueues yourself, Tor now tries to + pick a good value based on your total system memory. Previously, + the default was always 8 GB. You can still override the default by + setting MaxMemInQueues yourself. Resolves ticket 11396. + + o Minor features (log verbosity): + - Demote the message that we give when a flushing connection times + out for too long from NOTICE to INFO. It was usually meaningless. + Resolves ticket 5286. + - Don't log so many notice-level bootstrapping messages at startup + about downloading descriptors. Previously, we'd log a notice + whenever we learned about more routers. Now, we only log a notice + at every 5% of progress. Fixes bug 9963. + + o Minor features (relay): + - If a circuit timed out for at least 3 minutes, check if we have a + new external IP address, and publish a new descriptor with the new + IP address if it changed. Resolves ticket 2454. + - Warn less verbosely when receiving a malformed + ESTABLISH_RENDEZVOUS cell. Fixes ticket 11279. + - When we run out of usable circuit IDs on a channel, log only one + warning for the whole channel, and describe how many circuits + there were on the channel. Fix for part of ticket #11553. + + o Minor features (controller): + - Make the entire exit policy available from the control port via + GETINFO exit-policy/*. Implements enhancement #7952. Patch from + "rl1987". + - Because of the fix for ticket 11396, the real limit for memory + usage may no longer match the configured MaxMemInQueues value. The + real limit is now exposed via GETINFO limits/max-mem-in-queues. + + o Minor features (misc): + - Always check return values for unlink, munmap, UnmapViewOfFile; + check strftime return values more often. In some cases all we can + do is report a warning, but this may help prevent deeper bugs from + going unnoticed. Closes ticket 8787. + + o Minor features (bridge client): + - Report a more useful failure message when we can't connect to a + bridge because we don't have the right pluggable transport + configured. Resolves ticket 9665. Patch from Fábio J. Bertinatto. + + o Minor features (diagnostic): + - Add more log messages to diagnose bug 7164, which causes + intermittent "microdesc_free() called but md was still referenced" + warnings. We now include more information, to figure out why we + might be cleaning a microdescriptor for being too old if it's + still referenced by a live node_t object. o Minor bugfixes (logging): - - Log only one message when we start logging in an unsafe - way. Previously, we would log as many messages as we had - problems. Fix for #9870; bugfix on 0.2.5.1-alpha. - - Using the Linux syscall sandbox no longer prevents stack-trace + - Warn only once we start logging in an unsafe way. Previously, we + complain as many times we had problems. Fix for #9870; bugfix on + 0.2.5.1-alpha. + - Using the Linux seccomp2 sandbox no longer prevents stack-trace logging on crashes or errors. Fixes part 11465; bugfix on 0.2.5.1-alpha. - - Only report the first fatal boostrap error on a given OR - connection. This prevents controllers from declaring that a - connection has failed because of "DONE" or other junk reasons. - Fixes bug 10431; bugfix on 0.2.1.1-alpha. + - Only report the first fatal bootstrap error on a given OR + connection. This stops us from telling the controller bogus error + messages like "DONE". Fixes bug 10431; bugfix on 0.2.1.1-alpha. + - Be more helpful when trying to run sandboxed on Linux without + libseccomp. Instead of saying "Sandbox is not implemented on this + platform", we now explain that we to need be built with + libseccomp. Fixes bug 11543; bugfix on 0.2.5.1-alpha. + - Avoid generating spurious warnings when starting with + DisableNetwork enabled. Fixes bug 11200 and bug 10405; bugfix on + 0.2.3.9-alpha. o Minor bugfixes (closing channels): - If write_to_buf() in connection_write_to_buf_impl_() ever fails, check if it's an or_connection_t and correctly call - connection_or_close_for_error() rather than connection_mark_for_close() - directly. Fixes bug #11304; bugfix on 0.2.4.4-alpha. + connection_or_close_for_error() rather than + connection_mark_for_close() directly. Fixes bug #11304; bugfix on + 0.2.4.4-alpha. - When closing all connections on setting DisableNetwork to 1, use - connection_or_close_normally() rather than closing orconns out from - under the channel layer. Fixes bug #11306; bugfix on 0.2.4.4-alpha. + connection_or_close_normally() rather than closing OR connections + out from under the channel layer. Fixes bug #11306; bugfix on + 0.2.4.4-alpha. o Minor bugfixes (controller): - - Avoid sending an garbage value to the controller when a circuit is + - Avoid sending a garbage value to the controller when a circuit is cannibalized. Fixes bug 11519; bugfix on 0.2.3.11-alpha. o Minor bugfixes (tor-fw-helper): @@ -89,142 +170,101 @@ Changes in version 0.2.5.4-alpha - 2014-04-?? CPPFLAGS. Fixes bug 11296; bugfix on 0.2.5.3-alpha. o Minor bugfixes (bridges): - - Avoid potential crashes or bad behavior when launching a - server-side managed proxy with ORPort or ExtORPort temporarily - disabled. Fixes bug 9650; bugfix on 0.2.3.16-alpha. + - Avoid potential crashes or bad behavior when launching a server- + side managed proxy with ORPort or ExtORPort temporarily disabled. + Fixes bug 9650; bugfix on 0.2.3.16-alpha. o Minor bugfixes (misc): - - Don't re-initialize a second set of openssl mutexes when starting - up. Fixes bug 11726; bugfix on 0.2.5.3-alpha. - - o Minor bugfixes (memory leaks): - - Fix a minor memory leak that occurred when signing a directory - object. Fixes bug 11275; bugfix on 0.2.4.13-alpha. + - Don't re-initialize a second set of OpenSSL mutexes when starting + up. Previously, we'd make one set of mutexes, and then immediately + replace them with another. Fixes bug 11726; bugfix on + 0.2.5.3-alpha. o Minor bugfixes (platform-specific): - - Fix compilation on Solaris, which does not have <endian.h>. - Fixes bug 11426; bugfix on 0.2.5.3-alpha. - - When dumping a malformed directory object to disk, save it in binary - mode on windows, not text mode. Fixes bug 11342; bugfix on + - Fix compilation on Solaris, which does not have <endian.h>. Fixes + bug 11426; bugfix on 0.2.5.3-alpha. + - When dumping a malformed directory object to disk, save it in + binary mode on Windows, not text mode. Fixes bug 11342; bugfix on 0.2.2.1-alpha. - - When reporting a failure from make_socket_reuseable(), don't - report a warning when we get a failure from an incoming socket - on OSX. Fix for bug 10081. + - Don't report failures from make_socket_reuseable() on incoming + sockets on OSX: this can happen when incoming connections close + early. Fix for bug 10081. o Minor bugfixes (trivial memory leaks): - - Free placeholder entries in our circuit table at exit; fixes - a harmless memory leak. Fixes bug 11278; bugfix on 0.2.5.1-alpha. - - Resolve some memory leaks found by coverity in the unit tests, - on exit in tor-gencert, and on a failure to compute digests - for our own keys when generating a v3 networkstatus vote. - These leaks should never have affected anyone in practice. + - Fix a small memory leak when signing a directory object. Fixes bug + 11275; bugfix on 0.2.4.13-alpha. + - Free placeholder entries in our circuit table at exit; fixes a + harmless memory leak. Fixes bug 11278; bugfix on 0.2.5.1-alpha. + - Resolve some memory leaks found by coverity in the unit tests, on + exit in tor-gencert, and on a failure to compute digests for our + own keys when generating a v3 networkstatus vote. These leaks + should never have affected anyone in practice. o Minor bugfixes (hidden service): - - Only retry attempts to connect to a chosen rendezvous point 8 times, - not 30. Fixes bug #4241; bugfix on 0.1.0.1-rc. + - Only retry attempts to connect to a chosen rendezvous point 8 + times, not 30. Fixes bug #4241; bugfix on 0.1.0.1-rc. o Minor bugfixes (bridge client): - - Stop accepting bridge lines containing hostnames. Doing so allowed - clients to perform DNS requests on the hostnames, which was not - sensible behavior. Fixes bug 10801; bugfix on 0.2.0.1-alpha. + - Stop accepting bridge lines containing hostnames. Doing so would + cause clients to perform DNS requests on the hostnames, which was + not sensible behavior. Fixes bug 10801; bugfix on 0.2.0.1-alpha. + - Avoid a 60-second delay in the bootstrapping process when a Tor + client with pluggable transports re-reads its configuration at + just the wrong time. Re-fixes bug 11156; bugfix on 0.2.5.3-alpha. + - Avoid 60-second delays in the bootstrapping process when Tor is + launching for a second time while using bridges. Fixes bug 9229; + bugfix on 0.2.0.3-alpha. + + o Minor bugfixes (DNS): + - When receiving a DNS query for an unsupported record type, reply + with no answer rather than with a NOTIMPL error. This behavior + isn't correct either, but it will break fewer client programs, we + hope. Fixes bug 10268; bugfix on 0.2.0.1-alpha. Original patch + from "epoch". o Minor bugfixes (exit): - Stop leaking memory when we successfully resolve a PTR record. Fixes bug 11437; bugfix on 0.2.4.7-alpha. - o Minor features (Transparent proxy): - - Support the ipfw firewall interface for transparent proxy support on - FreeBSD. To enable it, set "TransProxyType ipfw" in your torrc. - Resolves ticket 10267; patch from "yurivict". - - Support OpenBSD's divert-to rules with the pf firewall, when - "TransProxyType pf-divert" is specified. This allows Tor to run a - TransPort transparent proxy port on OpenBSD 4.4 or later without - root privileges. See the pf.conf(5) manual page for information on - configuring pf to use divert-to rules. Closes ticket 10896; patch - from Dana Koch. - - o Minor features (security): - - New --enable-expensive-hardening option to turn on security hardening - options that consume nontrivial amounts of CPU and memory. Right now, - this includes AddressSanitizer and UbSan. Closes ticket 11477. - - o Minor features (usability): - - Demote the message that we give when a flushing connection times - out for too long from NOTICE to INFO. It was usually meaningless. - Resolves ticket 5286. - - o Minor features (performance, compatibility): - - Update the list of TLS cipehrsuites that a client advertises - to match those advertised by Firefox 28. This enables selection of - (fast) GCM ciphersuites, disables some strange old ciphers, and - disables the ECDH (not to be confused with ECDHE) ciphersuites. - Resolves ticket 11438. - - o Minor bugfixes (IPv6): - When using DNSPort and AutomapHostsOnResolve, respond to AAAA - requests with AAAA automapped answers. Fixes bug 10468; bugfix - on 0.2.4.7-alpha. - - o Minor features (relay): - - If a circuit timed out for at least 3 minutes check if we have a new - external IP address the next time we run our routine checks. If our - IP address has changed, then publish a new descriptor with the new - IP address. Resolves ticket 2454. - - Warn less verbosely when receiving a misformed ESTABLISH_RENDEZVOUS - cell. Fixes ticket 11279. - - o Minor features (controller): - - Make the entire exit policy available from the control port via - GETINFO exit-policy/*. Implements enhancement #7952. Patch from - "rl1987". - - o Minor features (misc): - - Always check return values for unlink, munmap, UnmapViewOfFile; - check strftime return values more often. In some cases all we - can do is report a warning, but this may help prevent deeper - bugs from going unnoticed. Closes ticket 8787. - - o Minor features (bridge client): - - Report a failure to connect to a bridge because its transport - type has no configured pluggable transport as a new type of bootstrap - failure. Resolves ticket 9665. Patch from Fábio J. Bertinatto. - - o Minor features (diagnostic): - - Try harder to diagnose a possible cause of bug 7164, which causes - intermittent "microdesc_free() called but md was still referenced" - warnings. We now log more information about the likely error case, - to try to figure out why we might be cleaning a microdescriptor - as old if it's still referenced by a live node. + requests with AAAA automapped answers. Fixes bug 10468; bugfix on + 0.2.4.7-alpha. o Documentation: - - Build the torify.1 manpage again. Previously, we were only - trying to build it when also building tor-fw-helper. That's why - we didn't notice that we'd broken the ability to build it. - Fixes bug 11321; bugfix on 0.2.5.1-alpha. + - Build the torify.1 manpage again. Previously, we were only trying + to build it when also building tor-fw-helper. That's why we didn't + notice that we'd broken the ability to build it. Fixes bug 11321; + bugfix on 0.2.5.1-alpha. - Fix the layout of the SOCKSPort flags in the manpage. Fixes bug 11061; bugfix on 0.2.4.7-alpha. - Correctly document that we search for a system torrc file before - looking in ~/.torrc. Fixes documentation side of 9213; bugfix - on 0.2.3.18-rc. + looking in ~/.torrc. Fixes documentation side of 9213; bugfix on + 0.2.3.18-rc. - Resolve warnings from Doxygen. o Code simplifications and refactoring: - Removing is_internal_IP() function. Resolves ticket 4645. - - Remove unused function circuit_dump_by_chan from circuitlist.c. Closes - issue #9107; patch from "marek". + - Remove unused function circuit_dump_by_chan from circuitlist.c. + Closes issue #9107; patch from "marek". - Change our use of the ENUM_BF macro to avoid declarations that confuse Doxygen. + o Deprecated versions: + - Tor 0.2.2.x has reached end-of-life; it has received no patches or + attention for some while. Directory authorities no longer accept + descriptors from relays running any version of Tor prior to Tor + 0.2.3.16-alpha. Resolves ticket 11149. + o Testing: - - New macros in test.h to simplify writting mock-functions for unit + - New macros in test.h to simplify writing mock-functions for unit tests. Part of ticket 11507. Patch from Dana Koch. - Complete tests for the status.c module. Resolves ticket 11507. Patch from Dana Koch. o Removed code: - - Remove all code for the long unused v1 directory protocol. Resolves - ticket 11070. + - Remove all code for the long unused v1 directory protocol. + Resolves ticket 11070. @@ -1648,6 +1688,10 @@ Changes in version 0.2.4.11-alpha - 2013-03-11 - Randomize the lifetime of our SSL link certificate, so censors can't use the static value for filtering Tor flows. Resolves ticket 8443; related to ticket 4014 which was included in 0.2.2.33. + - Support a new version of the link protocol that allows 4-byte circuit + IDs. Previously, circuit IDs were limited to 2 bytes, which presented + a possible resource exhaustion issue. Closes ticket 7351; implements + proposal 214. o Minor features (portability): - Tweak the curve25519-donna*.c implementations to tolerate systems diff --git a/changes/11543 b/changes/11543 deleted file mode 100644 index ebc22f775..000000000 --- a/changes/11543 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes: - - - Improve the warning message when trying to enable the Linux - sandbox code on a Tor built without libseccomp. Instead of - saying "Sandbox is not implemented on this platform", we now - explain that we to need be built with libseccomp. Fixes bug - 11543; bugfix on 0.2.5.1-alpha. - diff --git a/changes/bug9963 b/changes/bug9963 deleted file mode 100644 index b8b95f2a7..000000000 --- a/changes/bug9963 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features: - - Don't log so many notice-level bootstrapping messages at startup about - downloading descriptors. Previously, we'd log a notice whenever we - learned about more routers. Now, we only log a notice at every 5% of - progress. Fixes bug 9963. diff --git a/changes/ticket11528 b/changes/ticket11528 deleted file mode 100644 index 15daad995..000000000 --- a/changes/ticket11528 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features: - - Servers now trust themselves to have a better view than clients of - which TLS ciphersuites to choose. (Thanks to #11513, the server - list is now well-considered, whereas the client list has been - chosen mainly for anti-fingerprinting purposes.) Resolves ticket - 11528. diff --git a/configure.ac b/configure.ac index 07b23dbcf..4831b4e3e 100644 --- a/configure.ac +++ b/configure.ac @@ -353,6 +353,7 @@ AC_CHECK_FUNCS( strtok_r \ strtoull \ sysconf \ + sysctl \ uname \ vasprintf \ _vscprintf @@ -885,6 +886,7 @@ AC_CHECK_HEADERS( sys/prctl.h \ sys/resource.h \ sys/socket.h \ + sys/sysctl.h \ sys/syslimits.h \ sys/time.h \ sys/types.h \ diff --git a/contrib/format_changelog.py b/contrib/format_changelog.py new file mode 100755 index 000000000..6e3b8045b --- /dev/null +++ b/contrib/format_changelog.py @@ -0,0 +1,141 @@ +#!/usr/bin/python +# Copyright (c) 2014, The Tor Project, Inc. +# See LICENSE for licensing information +# +# This script reformats a section of the changelog to wrap everything to +# the right width and put blank lines in the right places. Eventually, +# it might include a linter. +# +# To run it, pipe a section of the changelog (starting with "Changes +# in Tor 0.x.y.z-alpha" through the script.) + +import re +import sys +import textwrap + +TP_MAINHEAD = 0 +TP_HEADTEXT = 1 +TP_BLANK = 2 +TP_SECHEAD = 3 +TP_ITEMFIRST = 4 +TP_ITEMBODY = 5 + +def head_parser(line): + if re.match(r'^[A-Z]', line): + return TP_MAINHEAD + elif re.match(r'^ o ', line): + return TP_SECHEAD + elif re.match(r'^\s*$', line): + return TP_BLANK + else: + return TP_HEADTEXT + +def body_parser(line): + if re.match(r'^ o ', line): + return TP_SECHEAD + elif re.match(r'^ -',line): + return TP_ITEMFIRST + elif re.match(r'^ \S', line): + return TP_ITEMBODY + elif re.match(r'^\s*$', line): + return TP_BLANK + else: + print "Weird line %r"%line + +class ChangeLog(object): + def __init__(self): + self.mainhead = None + self.headtext = [] + self.curgraf = None + self.sections = [] + self.cursection = None + self.lineno = 0 + + def addLine(self, tp, line): + self.lineno += 1 + + if tp == TP_MAINHEAD: + assert not self.mainhead + self.mainhead = line + + elif tp == TP_HEADTEXT: + if self.curgraf is None: + self.curgraf = [] + self.headtext.append(self.curgraf) + self.curgraf.append(line) + + elif tp == TP_BLANK: + self.curgraf = None + + elif tp == TP_SECHEAD: + self.cursection = [ self.lineno, line, [] ] + self.sections.append(self.cursection) + + elif tp == TP_ITEMFIRST: + item = ( self.lineno, [ [line] ]) + self.curgraf = item[1][0] + self.cursection[2].append(item) + + elif tp == TP_ITEMBODY: + if self.curgraf is None: + self.curgraf = [] + self.cursection[2][1][-1].append(self.curgraf) + self.curgraf.append(line) + + else: + assert "This" is "unreachable" + + def lint_head(self, line, head): + m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head) + if not m: + print >>sys.stderr, "Weird header format on line %s"%line + + def lint_item(self, line, grafs, head_type): + pass + + def lint(self): + self.head_lines = {} + for sec_line, sec_head, items in self.sections: + head_type = self.lint_head(sec_line, sec_head) + for item_line, grafs in items: + self.lint_item(item_line, grafs, head_type) + + def dumpGraf(self,par,indent1,indent2=-1): + if indent2 == -1: + indent2 = indent1 + text = " ".join(re.sub(r'\s+', ' ', line.strip()) for line in par) + print textwrap.fill(text, width=72, + initial_indent=" "*indent1, + subsequent_indent=" "*indent2) + + def dump(self): + print self.mainhead + for par in self.headtext: + self.dumpGraf(par, 2) + print + for _,head,items in self.sections: + if not head.endswith(':'): + print >>sys.stderr, "adding : to %r"%head + head = head + ":" + print head + for _,grafs in items: + self.dumpGraf(grafs[0],4,6) + for par in grafs[1:]: + print + self.dumpGraf(par,6,6) + print + print + +CL = ChangeLog() +parser = head_parser + +for line in sys.stdin: + line = line.rstrip() + tp = parser(line) + + CL.addLine(tp,line) + if tp == TP_SECHEAD: + parser = body_parser + +CL.lint() +CL.dump() diff --git a/doc/tor.1.txt b/doc/tor.1.txt index d1556a554..94c7d40a6 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1761,7 +1761,8 @@ is non-zero): it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only affects some queues, so the actual process size will be larger than - this. (Default: 8GB) + this. If this option is set to 0, Tor will try to pick a reasonable + default based on your system's physical memory. (Default: 0) DIRECTORY SERVER OPTIONS ------------------------ diff --git a/src/common/compat.c b/src/common/compat.c index 8d816b90e..1ba264a0c 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -38,6 +38,9 @@ #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif +#ifdef HAVE_SYS_SYSCTL_H +#include <sys/sysctl.h> +#endif #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif @@ -3331,3 +3334,119 @@ format_win32_error(DWORD err) } #endif +#if defined(HW_PHYSMEM64) +/* This appears to be an OpenBSD thing */ +#define INT64_HW_MEM HW_PHYSMEM64 +#elif defined(HW_MEMSIZE) +/* OSX defines this one */ +#define INT64_HW_MEM HW_MEMSIZE +#endif + +/** + * Helper: try to detect the total system memory, and return it. On failure, + * return 0. + */ +static uint64_t +get_total_system_memory_impl(void) +{ +#if defined(__linux__) + /* On linux, sysctl is deprecated. Because proc is so awesome that you + * shouldn't _want_ to write portable code, I guess? */ + unsigned long long result=0; + int fd = -1; + char *s = NULL; + const char *cp; + size_t file_size=0; + if (-1 == (fd = tor_open_cloexec("/proc/meminfo",O_RDONLY,0))) + return 0; + s = read_file_to_str_until_eof(fd, 65536, &file_size); + if (!s) + goto err; + cp = strstr(s, "MemTotal:"); + if (!cp) + goto err; + /* Use the system sscanf so that space will match a wider number of space */ + if (sscanf(cp, "MemTotal: %llu kB\n", &result) != 1) + goto err; + + close(fd); + tor_free(s); + return result * 1024; + + err: + tor_free(s); + close(fd); + return 0; +#elif defined (_WIN32) + /* Windows has MEMORYSTATUSEX; pretty straightforward. */ + MEMORYSTATUSEX ms; + memset(&ms, 0, sizeof(ms)); + ms.dwLength = sizeof(ms); + if (! GlobalMemoryStatusEx(&ms)) + return 0; + + return ms.ullTotalPhys; + +#elif defined(HAVE_SYSCTL) && defined(INT64_HW_MEM) + /* On many systems, HW_PYHSMEM is clipped to 32 bits; let's use a better + * variant if we know about it. */ + uint64_t memsize = 0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, INT64_HW_MEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return 0; + + return memsize; + +#elif defined(HAVE_SYSCTL) && defined(HW_PHYSMEM) + /* On some systems (like FreeBSD I hope) you can use a size_t with + * HW_PHYSMEM. */ + size_t memsize=0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, HW_USERMEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return -1; + + return memsize; + +#else + /* I have no clue. */ + return 0; +#endif +} + +/** + * Try to find out how much physical memory the system has. On success, + * return 0 and set *<b>mem_out</b> to that value. On failure, return -1. + */ +int +get_total_system_memory(size_t *mem_out) +{ + static size_t mem_cached=0; + uint64_t m = get_total_system_memory_impl(); + if (0 == m) { + /* We couldn't find our memory total */ + if (0 == mem_cached) { + /* We have no cached value either */ + *mem_out = 0; + return -1; + } + + *mem_out = mem_cached; + return 0; + } + +#if SIZE_T_MAX != UINT64_MAX + if (m > SIZE_T_MAX) { + /* I think this could happen if we're a 32-bit Tor running on a 64-bit + * system: we could have more system memory than would fit in a + * size_t. */ + m = SIZE_T_MAX; + } +#endif + + *mem_out = mem_cached = (size_t) m; + + return -1; +} + diff --git a/src/common/compat.h b/src/common/compat.h index 9a381fb97..314b1aa00 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -638,6 +638,8 @@ char *make_path_absolute(char *fname); char **get_environment(void); +int get_total_system_memory(size_t *mem_out); + int spawn_func(void (*func)(void *), void *data); void spawn_exit(void) ATTR_NORETURN; diff --git a/src/or/channel.c b/src/or/channel.c index 32e87c342..63af2f91c 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -728,8 +728,8 @@ channel_init(channel_t *chan) /* Init timestamp */ chan->timestamp_last_added_nonpadding = time(NULL); - /* Init next_circ_id */ - chan->next_circ_id = crypto_rand_int(1 << 15); + /* Warn about exhausted circuit IDs no more than hourly. */ + chan->last_warned_circ_ids_exhausted.rate = 3600; /* Initialize queues. */ TOR_SIMPLEQ_INIT(&chan->incoming_queue); diff --git a/src/or/channel.h b/src/or/channel.h index 7ec222df0..bd9a02f32 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -149,11 +149,6 @@ struct channel_s { circ_id_type_bitfield_t circ_id_type:2; /** DOCDOC*/ unsigned wide_circ_ids:1; - /** - * Which circ_id do we try to use next on this connection? This is - * always in the range 0..1<<15-1. - */ - circid_t next_circ_id; /** For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; @@ -182,6 +177,10 @@ struct channel_s { */ unsigned int is_local:1; + /** Have we logged a warning about circID exhaustion on this channel? + * If so, when? */ + ratelim_t last_warned_circ_ids_exhausted; + /** Channel timestamps for cell channels */ time_t timestamp_client; /* Client used this, according to relay.c */ time_t timestamp_drained; /* Output queue empty */ diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 6bdbead2e..cd92326b3 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -77,18 +77,27 @@ channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, return chan; } -/** Iterate over values of circ_id, starting from conn-\>next_circ_id, - * and with the high bit specified by conn-\>circ_id_type, until we get - * a circ_id that is not in use by any other circuit on that conn. +/** Search for a value for circ_id that we can use on <b>chan</b> for an + * outbound circuit, until we get a circ_id that is not in use by any other + * circuit on that conn. * * Return it, or 0 if can't get a unique circ_id. */ static circid_t get_unique_circ_id_by_chan(channel_t *chan) { +/* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ +#define MAX_CIRCID_ATTEMPTS 64 + int in_use; + unsigned n_with_circ = 0, n_pending_destroy = 0; circid_t test_circ_id; circid_t attempts=0; - circid_t high_bit, max_range; + circid_t high_bit, max_range, mask; tor_assert(chan); @@ -98,25 +107,52 @@ get_unique_circ_id_by_chan(channel_t *chan) "a client with no identity."); return 0; } - max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + mask = max_range - 1; high_bit = (chan->circ_id_type == CIRC_ID_TYPE_HIGHER) ? max_range : 0; do { - /* Sequentially iterate over test_circ_id=1...max_range until we find a - * circID such that (high_bit|test_circ_id) is not already used. */ - test_circ_id = chan->next_circ_id++; - if (test_circ_id == 0 || test_circ_id >= max_range) { - test_circ_id = 1; - chan->next_circ_id = 2; - } - if (++attempts > max_range) { - /* Make sure we don't loop forever if all circ_id's are used. This - * matters because it's an external DoS opportunity. + if (++attempts > MAX_CIRCID_ATTEMPTS) { + /* Make sure we don't loop forever because all circuit IDs are used. + * + * Once, we would try until we had tried every possible circuit ID. But + * that's quite expensive. Instead, we try MAX_CIRCID_ATTEMPTS random + * circuit IDs, and then give up. + * + * This potentially causes us to give up early if our circuit ID space + * is nearly full. If we have N circuit IDs in use, then we will reject + * a new circuit with probability (N / max_range) ^ MAX_CIRCID_ATTEMPTS. + * This means that in practice, a few percent of our circuit ID capacity + * will go unused. + * + * The alternative here, though, is to do a linear search over the + * whole circuit ID space every time we extend a circuit, which is + * not so great either. */ - log_warn(LD_CIRC,"No unused circ IDs. Failing."); + log_fn_ratelim(&chan->last_warned_circ_ids_exhausted, LOG_WARN, + LD_CIRC,"No unused circIDs found on channel %s wide " + "circID support, with %u inbound and %u outbound circuits. " + "Found %u circuit IDs in use by circuits, and %u with " + "pending destroy cells." + "Failing a circuit.", + chan->wide_circ_ids ? "with" : "without", + chan->num_p_circuits, chan->num_n_circuits, + n_with_circ, n_pending_destroy); return 0; } + + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; - } while (circuit_id_in_use_on_channel(test_circ_id, chan)); + + in_use = circuit_id_in_use_on_channel(test_circ_id, chan); + if (in_use == 1) + ++n_with_circ; + else if (in_use == 2) + ++n_pending_destroy; + } while (in_use); return test_circ_id; } @@ -561,7 +597,9 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, id = get_unique_circ_id_by_chan(circ->n_chan); if (!id) { - log_warn(LD_CIRC,"failed to get unique circID."); + static ratelim_t circid_warning_limit = RATELIM_INIT(9600); + log_fn_ratelim(&circid_warning_limit, LOG_WARN, LD_CIRC, + "failed to get unique circID."); return -1; } log_debug(LD_CIRC,"Chosen circID %u.", (unsigned)id); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 24cb9fc8d..c54a95419 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1065,13 +1065,21 @@ circuit_get_by_circid_channel_even_if_marked(circid_t circ_id, } /** Return true iff the circuit ID <b>circ_id</b> is currently used by a - * circuit, marked or not, on <b>chan</b>. */ + * circuit, marked or not, on <b>chan</b>, or if the circ ID is reserved until + * a queued destroy cell can be sent. + * + * (Return 1 if the circuit is present, marked or not; Return 2 + * if the circuit ID is pending a destroy.) + **/ int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan) { int found = 0; - return circuit_get_by_circid_channel_impl(circ_id, chan, &found) != NULL - || found; + if (circuit_get_by_circid_channel_impl(circ_id, chan, &found) != NULL) + return 1; + if (found) + return 2; + return 0; } /** Return the circuit that a given edge connection is using. */ diff --git a/src/or/config.c b/src/or/config.c index a5407dd4e..3954c582e 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -307,7 +307,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInQueues, MEMUNIT, "8 GB"), + VAR("MaxMeminQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -565,6 +565,8 @@ static void config_maybe_load_geoip_files_(const or_options_t *options, static int options_validate_cb(void *old_options, void *options, void *default_options, int from_setconf, char **msg); +static uint64_t compute_real_max_mem_in_queues(const uint64_t val, + int log_guess); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -1445,6 +1447,12 @@ options_act(const or_options_t *old_options) sweep_transport_list(); sweep_proxy_list(); + /* Start the PT proxy configuration. By doing this configuration + here, we also figure out which proxies need to be restarted and + which not. */ + if (pt_proxies_configuration_pending() && !net_is_disabled()) + pt_configure_remaining_proxies(); + /* Bail out at this point if we're not going to be a client or server: * we want to not fork, and to log stuff to stderr. */ if (!running_tor) @@ -2818,11 +2826,9 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInQueues < (256 << 20)) { - log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " - "Ideally, have it as large as you can afford."); - options->MaxMemInQueues = (256 << 20); - } + options->MaxMemInQueues = + compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, + server_mode(options)); options->AllowInvalid_ = 0; @@ -3572,6 +3578,68 @@ options_validate(or_options_t *old_options, or_options_t *options, #undef COMPLAIN } +/* Given the value that the user has set for MaxMemInQueues, compute the + * actual maximum value. We clip this value if it's too low, and autodetect + * it if it's set to 0. */ +static uint64_t +compute_real_max_mem_in_queues(const uint64_t val, int log_guess) +{ + uint64_t result; + + if (val == 0) { +#define ONE_GIGABYTE (U64_LITERAL(1) << 30) +#define ONE_MEGABYTE (U64_LITERAL(1) << 20) +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) +#else +#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) +#endif + /* The user didn't pick a memory limit. Choose a very large one + * that is still smaller than the system memory */ + static int notice_sent = 0; + size_t ram = 0; + if (get_total_system_memory(&ram) < 0) { + /* We couldn't determine our total system memory! */ +#if SIZEOF_VOID_P >= 8 + /* 64-bit system. Let's hope for 8 GB. */ + result = 8 * ONE_GIGABYTE; +#else + /* (presumably) 32-bit system. Let's hope for 1 GB. */ + result = ONE_GIGABYTE; +#endif + } else { + /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ + const uint64_t avail = (ram / 4) * 3; + + /* Make sure it's in range from 0.25 GB to 8 GB. */ + if (avail > MAX_DEFAULT_MAXMEM) { + /* If you want to use more than this much RAM, you need to configure + it yourself */ + result = MAX_DEFAULT_MAXMEM; + } else if (avail < ONE_GIGABYTE / 4) { + result = ONE_GIGABYTE / 4; + } else { + result = avail; + } + } + if (log_guess && ! notice_sent) { + log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. " + "You can override this by setting MaxMemInQueues by hand.", + ram ? "Based on detected system memory, " : "", + U64_PRINTF_ARG(result / ONE_MEGABYTE)); + notice_sent = 1; + } + return result; + } else if (val < ONE_GIGABYTE / 4) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " + "Ideally, have it as large as you can afford."); + return ONE_GIGABYTE / 4; + } else { + /* The value was fine all along */ + return val; + } +} + /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ static int diff --git a/src/or/connection.c b/src/or/connection.c index 5dbc47728..3cc4e09fb 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -271,8 +271,6 @@ dir_connection_new(int socket_family) * * Set timestamp_last_added_nonpadding to now. * - * Assign a pseudorandom next_circ_id between 0 and 2**15. - * * Initialize active_circuit_pqueue. * * Set active_circuit_pqueue_last_recalibrated to current cell_ewma tick. diff --git a/src/or/control.c b/src/or/control.c index 2815b7901..d571900ac 100755 --- a/src/or/control.c +++ b/src/or/control.c @@ -1504,6 +1504,9 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, int max_fds=-1; set_max_file_descriptors(0, &max_fds); tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, U64_FORMAT, + U64_PRINTF_ARG(get_options()->MaxMemInQueues)); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { @@ -2184,6 +2187,7 @@ static const getinfo_item_t getinfo_items[] = { ITEM("process/user", misc, "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 4190e9cc2..ecd45be77 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -35,7 +35,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) entry_connection_t *entry_conn; edge_connection_t *conn; int i = 0; - struct evdns_server_question *q = NULL; + struct evdns_server_question *q = NULL, *supported_q = NULL; struct sockaddr_storage addr; struct sockaddr *sa; int addrlen; @@ -87,31 +87,37 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) for (i = 0; i < req->nquestions; ++i) { if (req->questions[i]->dns_question_class != EVDNS_CLASS_INET) continue; + if (! q) + q = req->questions[i]; switch (req->questions[i]->type) { case EVDNS_TYPE_A: case EVDNS_TYPE_AAAA: case EVDNS_TYPE_PTR: - q = req->questions[i]; + /* We always pick the first one of these questions, if there is + one. */ + if (! supported_q) + supported_q = q; + break; default: break; } } + if (supported_q) + q = supported_q; if (!q) { log_info(LD_APP, "None of the questions we got were ones we're willing " "to support. Sending NOTIMPL."); evdns_server_request_respond(req, DNS_ERR_NOTIMPL); return; } - if (q->type != EVDNS_TYPE_A && q->type != EVDNS_TYPE_AAAA) { - tor_assert(q->type == EVDNS_TYPE_PTR); - } /* Make sure the name isn't too long: This should be impossible, I think. */ if (err == DNS_ERR_NONE && strlen(q->name) > MAX_SOCKS_ADDR_LEN-1) err = DNS_ERR_FORMAT; - if (err != DNS_ERR_NONE) { - /* We got an error? Then send back an answer immediately; we're done. */ + if (err != DNS_ERR_NONE || !supported_q) { + /* We got an error? There's no question we're willing to answer? Then + * send back an answer immediately; we're done. */ evdns_server_request_respond(req, err); return; } @@ -126,12 +132,15 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) TO_CONN(conn)->port = port; TO_CONN(conn)->address = tor_dup_addr(&tor_addr); - if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA) + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || + q->type == EVDNS_QTYPE_ALL) { entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE; - else + } else { + tor_assert(q->type == EVDNS_TYPE_PTR); entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR; + } - if (q->type == EVDNS_TYPE_A) { + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) { entry_conn->ipv4_traffic_ok = 1; entry_conn->ipv6_traffic_ok = 0; entry_conn->prefer_ipv6_traffic = 0; diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index b374ac7a3..70587bd75 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -70,7 +70,9 @@ static int entry_guards_dirty = 0; static void bridge_free(bridge_info_t *bridge); static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirtype); + dirinfo_type_t dirtype, + int *n_options_out); +static int num_bridges_usable(void); /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * @@ -982,7 +984,7 @@ node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) const node_t * choose_random_entry(cpath_build_state_t *state) { - return choose_random_entry_impl(state, 0, 0); + return choose_random_entry_impl(state, 0, 0, NULL); } /** Pick a live (up and listed) directory guard from entry_guards for @@ -990,13 +992,13 @@ choose_random_entry(cpath_build_state_t *state) const node_t * choose_random_dirguard(dirinfo_type_t type) { - return choose_random_entry_impl(NULL, 1, type); + return choose_random_entry_impl(NULL, 1, type, NULL); } /** Helper for choose_random{entry,dirguard}. */ static const node_t * choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirinfo_type) + dirinfo_type_t dirinfo_type, int *n_options_out) { const or_options_t *options = get_options(); smartlist_t *live_entry_guards = smartlist_new(); @@ -1010,6 +1012,9 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, int need_descriptor = !for_directory; const int num_needed = decide_num_guards(options, for_directory); + if (n_options_out) + *n_options_out = 0; + if (chosen_exit) { nodelist_add_node_and_family(exit_family, chosen_exit); consider_exit_family = 1; @@ -1136,6 +1141,8 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, * *double*-weight our guard selection. */ node = smartlist_choose(live_entry_guards); } + if (n_options_out) + *n_options_out = smartlist_len(live_entry_guards); smartlist_free(live_entry_guards); smartlist_free(exit_family); return node; @@ -2166,7 +2173,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { - int first = !any_bridge_descriptors_known(); + int first = num_bridges_usable() <= 1; bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); router_set_status(ri->cache_info.identity_digest, 1); @@ -2188,14 +2195,15 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) * our entry node list */ entry_guard_register_connect_status(ri->cache_info.identity_digest, 1, 0, now); - if (first) + if (first) { routerlist_retry_directory_downloads(now); + } } } } -/** Return 1 if any of our entry guards have descriptors that - * are marked with purpose 'bridge' and are running. Else return 0. +/** Return the number of bridges that have descriptors that + * are marked with purpose 'bridge' and are running. * * We use this function to decide if we're ready to start building * circuits through our bridges, or if we need to wait until the @@ -2207,6 +2215,18 @@ any_bridge_descriptors_known(void) return choose_random_entry(NULL) != NULL; } +/** Return the number of bridges that have descriptors that are marked with + * purpose 'bridge' and are running. + */ +static int +num_bridges_usable(void) +{ + int n_options = 0; + tor_assert(get_options()->UseBridges); + (void) choose_random_entry_impl(NULL, 0, 0, &n_options); + return n_options; +} + /** Return 1 if we have at least one descriptor for an entry guard * (bridge or member of EntryNodes) and all descriptors we know are * down. Else return 0. If <b>act</b> is 1, then mark the down guards diff --git a/src/or/main.c b/src/or/main.c index 4770b7e6d..6713d8036 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2767,6 +2767,7 @@ sandbox_init_filter(void) tor_strdup("/dev/urandom"), tor_strdup("/dev/random"), tor_strdup("/etc/hosts"), + tor_strdup("/proc/meminfo"), NULL, 0 ); if (options->ServerDNSResolvConfFile) diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index b8ac2e05f..ef450073e 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -909,6 +909,14 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out) *msg_out = NULL; } + if (options->DisableNetwork) { + if (msg_out) { + *msg_out = "DisableNetwork is set."; + } + log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)"); + return 1; + } + if (options->UseBridges) { if (!any_bridge_descriptors_known()) { if (msg_out) { diff --git a/src/or/or.h b/src/or/or.h index 4ca7ecc60..701877c64 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3491,7 +3491,10 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInQueues; /**< If we have more memory than this allocated + /* MaxMemInQueues value as input by the user. We clean this up to be + * MaxMemInQueues. */ + uint64_t MaxMemInQueues_raw; + uint64_t MaxMemInQueues;/**< If we have more memory than this allocated * for queues and buffers, run the OOM handler */ /** @name port booleans diff --git a/src/or/transports.c b/src/or/transports.c index 7e496fe21..dc3075416 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -51,35 +51,37 @@ * logic, because of race conditions that can cause dangling * pointers. ] * - * <b>In even more detail, this is what happens when a SIGHUP - * occurs:</b> + * <b>In even more detail, this is what happens when a config read + * (like a SIGHUP or a SETCONF) occurs:</b> * * We immediately destroy all unconfigured proxies (We shouldn't have - * unconfigured proxies in the first place, except when SIGHUP rings - * immediately after tor is launched.). + * unconfigured proxies in the first place, except when the config + * read happens immediately after tor is launched.). * * We mark all managed proxies and transports to signify that they * must be removed if they don't contribute by the new torrc * (we mark using the <b>marked_for_removal</b> element). * We also mark all managed proxies to signify that they might need to * be restarted so that they end up supporting all the transports the - * new torrc wants them to support (using the <b>got_hup</b> element). + * new torrc wants them to support + * (we mark using the <b>was_around_before_config_read</b> element). * We also clear their <b>transports_to_launch</b> list so that we can * put there the transports we need to launch according to the new * torrc. * * We then start parsing torrc again. * - * Everytime we encounter a transport line using a known pre-SIGHUP - * managed proxy, we cleanse that proxy from the removal mark. - * We also mark it as unconfigured so that on the next scheduled - * events tick, we investigate whether we need to restart the proxy - * so that it also spawns the new transports. - * If the post-SIGHUP <b>transports_to_launch</b> list is identical to - * the pre-SIGHUP one, it means that no changes were introduced to - * this proxy during the SIGHUP and no restart has to take place. + * Everytime we encounter a transport line using a managed proxy that + * was around before the config read, we cleanse that proxy from the + * removal mark. We also toggle the <b>check_if_restarts_needed</b> + * flag, so that on the next <b>pt_configure_remaining_proxies</b> + * tick, we investigate whether we need to restart the proxy so that + * it also spawns the new transports. If the post-config-read + * <b>transports_to_launch</b> list is identical to the pre-config-read + * one, it means that no changes were introduced to this proxy during + * the config read and no restart has to take place. * - * During the post-SIGHUP torrc parsing, we unmark all transports + * During the post-config-read torrc parsing, we unmark all transports * spawned by managed proxies that we find in our torrc. * We do that so that if we don't need to restart a managed proxy, we * can continue using its old transports normally. @@ -534,8 +536,7 @@ launch_managed_proxy(managed_proxy_t *mp) } /** Check if any of the managed proxies we are currently trying to - * configure have anything new to say. This is called from - * run_scheduled_events(). */ + * configure has anything new to say. */ void pt_configure_remaining_proxies(void) { @@ -555,11 +556,12 @@ pt_configure_remaining_proxies(void) tor_assert(mp->conf_state != PT_PROTO_BROKEN && mp->conf_state != PT_PROTO_FAILED_LAUNCH); - if (mp->got_hup) { - mp->got_hup = 0; + if (mp->was_around_before_config_read) { + /* This proxy is marked by a config read. Check whether we need + to restart it. */ + + mp->was_around_before_config_read = 0; - /* This proxy is marked by a SIGHUP. Check whether we need to - restart it. */ if (proxy_needs_restart(mp)) { log_info(LD_GENERAL, "Preparing managed proxy '%s' for restart.", mp->argv[0]); @@ -1364,19 +1366,20 @@ pt_kickstart_proxy(const smartlist_t *transport_list, managed_proxy_create(transport_list, proxy_argv, is_server); } else { /* known proxy. add its transport to its transport list */ - if (mp->got_hup) { - /* If the managed proxy we found is marked by a SIGHUP, it means - that it's not useless and should be kept. If it's marked for - removal, unmark it and increase the unconfigured proxies so - that we try to restart it if we need to. Afterwards, check if - a transport_t for 'transport' used to exist before the SIGHUP - and make sure it doesn't get deleted because we might reuse - it. */ + if (mp->was_around_before_config_read) { + /* If this managed proxy was around even before we read the + config this time, it means that it was already enabled before + and is not useless and should be kept. If it's marked for + removal, unmark it and make sure that we check whether it + needs to be restarted. */ if (mp->marked_for_removal) { mp->marked_for_removal = 0; check_if_restarts_needed = 1; } + /* For each new transport, check if the managed proxy used to + support it before the SIGHUP. If that was the case, make sure + it doesn't get removed because we might reuse it. */ SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport) { old_transport = transport_get_by_name(transport); if (old_transport) @@ -1425,8 +1428,10 @@ pt_prepare_proxy_list_for_config_read(void) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + /* Mark all proxies for removal, and also note that they have been + here before the config read. */ mp->marked_for_removal = 1; - mp->got_hup = 1; + mp->was_around_before_config_read = 1; SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t)); smartlist_clear(mp->transports_to_launch); } SMARTLIST_FOREACH_END(mp); diff --git a/src/or/transports.h b/src/or/transports.h index 7b524f207..1365ead00 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -97,7 +97,7 @@ typedef struct { * this flag to signify that this proxy might need to be restarted * so that it can listen for other transports according to the new * torrc. */ - unsigned int got_hup : 1; + unsigned int was_around_before_config_read : 1; /* transports to-be-launched by this proxy */ smartlist_t *transports_to_launch; diff --git a/src/test/test_util.c b/src/test/test_util.c index 08efd453c..6d6b6dbdf 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -3612,6 +3612,34 @@ test_util_socketpair(void *arg) tor_close_socket(fds[1]); } +static void +test_util_max_mem(void *arg) +{ + size_t memory1, memory2; + int r, r2; + (void) arg; + + r = get_total_system_memory(&memory1); + r2 = get_total_system_memory(&memory2); + tt_int_op(r, ==, r2); + tt_int_op(memory2, ==, memory1); + + TT_BLATHER(("System memory: "U64_FORMAT, U64_PRINTF_ARG(memory1))); + + if (r==0) { + /* You have at least a megabyte. */ + tt_int_op(memory1, >, (1<<20)); + } else { + /* You do not have a petabyte. */ +#if SIZEOF_SIZE_T == SIZEOF_UINT64_T + tt_int_op(memory1, <, (U64_LITERAL(1)<<50)); +#endif + } + + done: + ; +} + struct testcase_t util_tests[] = { UTIL_LEGACY(time), UTIL_TEST(parse_http_time, 0), @@ -3675,6 +3703,7 @@ struct testcase_t util_tests[] = { (void*)"0" }, { "socketpair_ersatz", test_util_socketpair, TT_FORK, &socketpair_setup, (void*)"1" }, + UTIL_TEST(max_mem, 0), END_OF_TESTCASES }; |