From ca371c8c818ff4c829337db876fdbe3df325d9a8 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sat, 5 Apr 2014 22:20:01 +0100 Subject: Major changes to chutney --- chutney | 2 +- lib/chutney/Templating.py | 3 +- lib/chutney/Testing.py | 72 +++++++++++++++++- lib/chutney/TorNet.py | 159 ++++++++++++++++++--------------------- networks/hs-intro-fail-2 | 186 ++++------------------------------------------ networks/hs-intro-fail-3 | 173 ++++-------------------------------------- 6 files changed, 179 insertions(+), 416 deletions(-) diff --git a/chutney b/chutney index 344118d..ebb6315 100755 --- a/chutney +++ b/chutney @@ -1,6 +1,6 @@ #!/bin/sh - PYTHONPATH="`dirname $0`/lib":${PYTHONPATH} +PYTHONPATH="`dirname $0`/networks":${PYTHONPATH} export PYTHONPATH ${PYTHON:=python} -m chutney.TorNet $* diff --git a/lib/chutney/Templating.py b/lib/chutney/Templating.py index 8141a85..ddcb850 100644 --- a/lib/chutney/Templating.py +++ b/lib/chutney/Templating.py @@ -114,6 +114,7 @@ class _DictWrapper(object): on other values. """ try: + #print("calling self._getitem with " + key) return self._getitem(key, my) except KeyError: pass @@ -211,11 +212,11 @@ class Environ(_DictWrapper): #print("self") #print(dir(self)) fn = getattr(self, "_get_%s"%key, None) - #print("function " + str(fn)) #if key == "dir": # print("found dir " + self._get_dir(self)) if fn is not None: try: + #print("calling function") rv = fn(my) return rv except _KeyError: diff --git a/lib/chutney/Testing.py b/lib/chutney/Testing.py index ee1e62b..396b949 100644 --- a/lib/chutney/Testing.py +++ b/lib/chutney/Testing.py @@ -2,6 +2,11 @@ import Queue import stem.control import logging +from twisted.web import server, resource +from twisted.internet import reactor + +from chutney.TorNet import * + # Introduction Point Tracking node_intro_circuits = {} @@ -14,8 +19,6 @@ introduction_point_circuits = {} nodes_by_fingerprint = {} -#{ nodenum: { circuit: "fingerprint", circuit: "fingerprint" } } - def get_node_fingerprints(nodes): for n in nodes: @@ -228,3 +231,68 @@ def check_same_intro_points(): node = nodes_by_fingerprint[fingerprint] logging.info(" - " + node._env["nick"] + "(" + fingerprint + ")") return False + +def create_hidden_service(nodes): + hs_nodes = [] + hs_servers = [] + + # Use twisted to create web servers in this script + class Site(resource.Resource): + isLeaf = True + numberRequests = 0 + + def __init__(self, siteNum): + self.siteNum = siteNum + + def render_GET(self, request): + self.numberRequests += 1 + request.setHeader("content-type", "text/plain") + return str(self.siteNum) + + base_port = 10080 + for i in range(2): + + port = base_port + i + + node = Node( + tag="h", + hiddenservice=1, + torrc="hidden-service.tmpl", + hiddenservicetarget="127.0.0.1:%i" % port + ) + hs_nodes.append(node) + + site = server.Site(Site(i)) + + s = reactor.listenTCP(port, site) + hs_servers.append(s) + + port += 1 + + return hs_nodes, hs_servers + +def connection_test(client_nodes, expected_nodes): + logging.info("Connecting to clients") + + responses = {str(i): 0 for i in range(expected_nodes)} + + for c in client_nodes: + result = c.query("http://2oiifbe3wne4iaqb.onion/"); + + if result in responses: + responses[result] += 1 + else: + logging.info("Unknown response: " + str(result)) + + test_pass = True + + for node, responses in sorted(responses.items()): + logging.info(node + ": " + str(responses)) + + if responses == 0: + test_pass = False + + if test_pass: + logging.info("Test PASS") + else: + logging.info("Test FAIL") diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py index 334615f..82a5fc0 100644 --- a/lib/chutney/TorNet.py +++ b/lib/chutney/TorNet.py @@ -35,9 +35,40 @@ import Queue import chutney.Templating import chutney.Traffic +import chutney.Testing import logging +DEFAULTS = { + 'authority' : False, + 'bridgeauthority' : False, + 'hasbridgeauth' : False, + 'relay' : False, + 'bridge' : False, + 'hiddenservice' : False, + 'hiddenserviceport' : 80, + 'hiddenservicetarget' : '127.0.0.1', + 'connlimit' : 60, + 'net_base_dir' : 'net', + 'tor' : 'tor', + 'auth_cert_lifetime' : 12, + 'ip' : '127.0.0.1', + 'ipv6_addr' : None, + 'dirserver_flags' : 'no-v2', + 'chutney_dir' : '.', + 'torrc_fname' : '${dir}/torrc', + 'orport_base' : 5000, + 'dirport_base' : 7000, + 'controlport_base' : 8000, + 'socksport_base' : 9000, + 'authorities' : "AlternateDirAuthority bleargh bad torrc file!", + 'bridges' : "Bridge bleargh bad torrc file!", + 'core' : True, + 'delay' : 0, +} + +_BASE_ENVIRON = chutney.Templating.Environ(**DEFAULTS) + def mkdir_p(d, mode=0777): """Create directory 'd' and all of its parents as needed. Unlike os.makedirs, does not give an error if d already exists. @@ -49,9 +80,6 @@ def mkdir_p(d, mode=0777): return raise -global nodelist -nodelist = [] - class Node(object): """A Node represents a Tor node or a set of Tor nodes. It's created in a network configuration file. @@ -74,7 +102,6 @@ class Node(object): self._builder = None self._controller = None self._stemcontroller = None - nodelist.append(self) @staticmethod def create(number, kwargs): @@ -426,7 +453,6 @@ class LocalNodeBuilder(NodeBuilder): class LocalNodeController(NodeController): def __init__(self, env): NodeController.__init__(self, env) - self._env = env def getPid(self): """Assuming that this node has its pidfile in ${dir}/pid, return @@ -536,35 +562,6 @@ class LocalNodeController(NodeController): - -DEFAULTS = { - 'authority' : False, - 'bridgeauthority' : False, - 'hasbridgeauth' : False, - 'relay' : False, - 'bridge' : False, - 'hiddenservice' : False, - 'hiddenserviceport' : 80, - 'hiddenservicetarget' : '127.0.0.1', - 'connlimit' : 60, - 'net_base_dir' : 'net', - 'tor' : 'tor', - 'auth_cert_lifetime' : 12, - 'ip' : '127.0.0.1', - 'ipv6_addr' : None, - 'dirserver_flags' : 'no-v2', - 'chutney_dir' : '.', - 'torrc_fname' : '${dir}/torrc', - 'orport_base' : 5000, - 'dirport_base' : 7000, - 'controlport_base' : 8000, - 'socksport_base' : 9000, - 'authorities' : "AlternateDirAuthority bleargh bad torrc file!", - 'bridges' : "Bridge bleargh bad torrc file!", - 'core' : True, - 'delay' : 0, -} - class TorEnviron(chutney.Templating.Environ): """Subclass of chutney.Templating.Environ to implement commonly-used substitutions. @@ -603,9 +600,12 @@ class TorEnviron(chutney.Templating.Environ): return my['dirport_base']+my['nodenum'] def _get_dir(self, my): - return os.path.abspath(os.path.join(my['net_base_dir'], + nodenum = my['nodenum'] + net_base_dir = my['net_base_dir'] + tag = my['tag'] + return os.path.abspath(os.path.join(net_base_dir, "nodes", - "%03d%s"%(my['nodenum'], my['tag']))) + "%03d%s"%(nodenum, tag))) def _get_nick(self, my): return "test%03d%s"%(my['nodenum'], my['tag']) @@ -620,28 +620,43 @@ class TorEnviron(chutney.Templating.Environ): return [ os.path.join(my['chutney_dir'], 'torrc_templates') ] -class Network(object): +class Network(list): """A network of Tor nodes, plus functions to manipulate them """ - def __init__(self,defaultEnviron): - self._nodes = [] + def __init__(self, defaultEnviron=_BASE_ENVIRON): self._dfltEnv = defaultEnviron self._nextnodenum = 0 + def get(self, tag): + nodes = [] + + for node in self: + if node._env["tag"] == tag: + nodes.append(node) + + return nodes + + def add(self, nodes): + if nodes is Node: + self._addNode(nodes) + else: + for node in nodes: + self._addNode(node) + def _addNode(self, n): n.setNodenum(self._nextnodenum) self._nextnodenum += 1 - self._nodes.append(n) + self.append(n) def _checkConfig(self): - for n in self._nodes: + for n in self: n.getBuilder().checkConfig(self) def configure(self): network = self altauthlines = [] bridgelines = [] - builders = [ n.getBuilder() for n in self._nodes ] + builders = [ n.getBuilder() for n in self ] self._checkConfig() @@ -664,10 +679,10 @@ class Network(object): b.postConfig(network) def status(self): - statuses = [ n.getController().check() for n in self._nodes] + statuses = [ n.getController().check() for n in self] n_ok = len([x for x in statuses if x]) - print "%d/%d nodes are running"%(n_ok,len(self._nodes)) - if n_ok != len(self._nodes): + print "%d/%d nodes are running"%(n_ok,len(self)) + if n_ok != len(self): return False return True @@ -682,15 +697,15 @@ class Network(object): def hup(self): print "Sending SIGHUP to nodes" - return all([n.getController().hup() for n in self._nodes]) + return all([n.getController().hup() for n in self]) def stop(self): - for n in self._nodes: + for n in self: if n._stemcontroller: n._stemcontroller.close() - controllers = [ n.getController() for n in self._nodes ] + controllers = [ n.getController() for n in self ] for sig, desc in [(signal.SIGINT, "SIGINT"), (signal.SIGINT, "another SIGINT"), (signal.SIGKILL, "SIGKILL")]: @@ -728,7 +743,7 @@ class Network(object): tmpdata = randfp.read(DATALEN) bind_to = ('127.0.0.1', LISTEN_PORT) tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT) - for op in filter(lambda n: n._env['tag'] == 'c', self._nodes): + for op in filter(lambda n: n._env['tag'] == 'c', self): tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, ('localhost', int(op._env['socksport'])))) return tt.run() @@ -739,12 +754,7 @@ def usage(network): " ".join(x for x in dir(network) if not x.startswith("_")))]) def runConfigFile(verb, f): - def stop(exitcode=0): - _THE_NETWORK.stop() - print("Network stoped, exiting") - sys.exit(exitcode) - - _GLOBALS = dict(_BASE_ENVIRON = _BASE_ENVIRON, + _GLOBALS = dict(_BASE_ENVIRON=_BASE_ENVIRON, Node=Node, EventType=stem.control.EventType, time=time, @@ -753,16 +763,14 @@ def runConfigFile(verb, f): random=random, threading=threading, logging=logging, - stop=stop, - _THE_NETWORK=_THE_NETWORK) + testing=chutney.Testing) exec f in _GLOBALS - network = _GLOBALS['_THE_NETWORK'] + network = _GLOBALS['network'] - for n in nodelist: - network._addNode(n) - if n._env['bridgeauthority']: - network._dfltEnv['hasbridgeauth'] = True + if len(network) is 0: + print("ERROR: No nodes") + return False; network._start = _GLOBALS['start'] @@ -773,26 +781,7 @@ def runConfigFile(verb, f): return getattr(network, verb)() -def logold(string, show=True): - output = time.strftime("%b %d %H:%M:%S.000") + " " + str(string) - - f = open('log.log', 'w') - f.write(output) - - if show: - print(output) - -def signal_handler(signal, frame): - _THE_NETWORK.stop() - print("Network stoped, exiting") - sys.exit(0) - def main(): - global _BASE_ENVIRON - global _THE_NETWORK - _BASE_ENVIRON = chutney.Templating.Environ(**DEFAULTS) - _THE_NETWORK = Network(_BASE_ENVIRON) - stem.util.log.get_logger().setLevel(logging.WARN) logger = logging.getLogger() @@ -813,7 +802,7 @@ def main(): logger.addHandler(handler) if len(sys.argv) < 3: - print usage(_THE_NETWORK) + print usage(network) print "Error: Not enough arguments given." sys.exit(1) @@ -823,9 +812,9 @@ def main(): if result is False: sys.exit(-1) - if sys.argv[1] == "start": - signal.signal(signal.SIGINT, signal_handler) - signal.pause() + #if sys.argv[1] == "start": + # signal.signal(signal.SIGINT, signal_handler) + # signal.pause() return 0 diff --git a/networks/hs-intro-fail-2 b/networks/hs-intro-fail-2 index 23d4c6e..20e9e11 100644 --- a/networks/hs-intro-fail-2 +++ b/networks/hs-intro-fail-2 @@ -1,189 +1,35 @@ -from chutney.Testing import * +from hs_intro_fail import * -authority_nodes = Node.create(3, { +network = Network() + +hs_nodes, hs_servers = testing.create_hidden_service(2) + +network.add(hs_nodes); + +network.add(Node.create(3, { "tag": "a", "authority": 1, "relay": 1, "torrc": "authority.tmpl" -}) - -# hidden service (hs) nodes -hs_nodes = [] - -hs_servers = [] - -from twisted.web import server, resource -from twisted.internet import reactor - -# Use twisted to create web servers in this script -class Site(resource.Resource): - isLeaf = True - numberRequests = 0 - - def __init__(self, siteNum): - self.siteNum = siteNum - - def render_GET(self, request): - self.numberRequests += 1 - request.setHeader("content-type", "text/plain") - return str(self.siteNum) - -base_port = 10080 -for i in range(2): - - port = base_port + i - - node = Node( - tag="h", - hiddenservice=1, - torrc="hidden-service.tmpl", - hiddenservicetarget="127.0.0.1:%i" % port - ) - hs_nodes.append(node) - - site = server.Site(Site(i)) - - s = reactor.listenTCP(port, site) - hs_servers.append(s) - - port += 1 +})) -client_nodes = Node.create(12, { +network.add(Node.create(12, { "tag": "c", "torrc": "client.tmpl" -}) +})) -relay_nodes = Node.create(10, { +network.add(Node.create(10, { "tag": "r", "relay": 1, "torrc": "intro.tmpl" -}) - -thread.start_new_thread(reactor.run, (), {"installSignalHandlers": 0}) - -initial_nodes = authority_nodes + relay_nodes + client_nodes + hs_nodes[:1] +})) def start(): - if not all([ n.getController().start() for n in initial_nodes ]): - return False - - logging.info("All initial nodes running") - - nodes_by_fingerprint = get_node_fingerprints(authority_nodes + relay_nodes) - - track_introduction_points(hs_nodes[0]) - - node_0_published_descriptor = threading.Event() - - def hs_node_0_listener(logevent): - if "Successfully uploaded v2 rend descriptors" in logevent.message: - node_0_published_descriptor.set() - - - hs_nodes[0].getStemController().add_event_listener(hs_node_0_listener, EventType.INFO) - - node_0_published_descriptor.wait() - - hs_nodes[0].getStemController().remove_event_listener(hs_node_0_listener) - - # list to cope with scope problems - node_counter = [len(hs_nodes) - 1] - node_counter_lock = threading.RLock() - - nodes_published_descriptors = threading.Event() - - def hs_node_listener(logevent): - if "Successfully uploaded v2 rend descriptors" in logevent.message: - with node_counter_lock: - node_counter[0] -= 1 - - if node_counter[0] == 0: - nodes_published_descriptors.set() - - for node in hs_nodes[1:]: - node.getController().start() - track_introduction_points(node) - - node.getStemController().add_event_listener(hs_node_listener, EventType.INFO) - - nodes_published_descriptors.wait() - - for node in hs_nodes[1:]: - node.getStemController().remove_event_listener(hs_node_listener) - - test_intro_failure(nodes_by_fingerprint) - -def test_intro_failure(nodes_by_fingerprint): - connection_test() - - time.sleep(5) - - # Select a random node that is being used as an introduction point - nodenum = random.choice(node_intro_circuits.keys()) - fingerprint = random.choice(node_intro_circuits[nodenum].values()) - - node = nodes_by_fingerprint[fingerprint] - - logging.info("stopping " + node._env["nick"] + " (" + fingerprint + ")") - - node.getStemController().close() - node.getController().stop() - - logging.info("begining to watch for the establishment of new introduction points") - - changed = [False for n in hs_nodes] - - intro_points_before = [set(node_intro_circuits[n._env["nodenum"]].values()) for n in hs_nodes] - - time.sleep(20) - - intro_points_after = [set(node_intro_circuits[n._env["nodenum"]].values()) for n in hs_nodes] - - for i, node in enumerate(hs_nodes): - before = intro_points_before[i] - after = intro_points_after[i] - - if before != after: - changed[i] = True - - if all(changed): - logging.info("All changed") - else: - logging.info("All did not change") - - check_same_intro_points() - - connection_test() + results = hs_fail_test(network) for server in hs_servers: server.stopListening() - reactor.stop() - - stop() - -def connection_test(): - logging.info("Connecting to clients") - - responses = {str(i): 0 for i in range(len(hs_servers))} - - for c in client_nodes: - result = c.query("http://2oiifbe3wne4iaqb.onion/"); - - if result in responses: - responses[result] += 1 - else: - logging.info("Unknown response: " + str(result)) - - test_pass = True - - for node, responses in sorted(responses.items()): - logging.info(node + ": " + str(responses)) - - if responses == 0: - test_pass = False + network.stop() - if test_pass: - logging.info("Test PASS") - else: - logging.info("Test FAIL") + return results diff --git a/networks/hs-intro-fail-3 b/networks/hs-intro-fail-3 index 0e1b95c..fa4dc7e 100644 --- a/networks/hs-intro-fail-3 +++ b/networks/hs-intro-fail-3 @@ -1,177 +1,36 @@ -from chutney.Testing import * +from hs_intro_fail import * -authority_nodes = Node.create(3, { +network = Network() + +hs_nodes, hs_servers = testing.create_hidden_service(3) + +network.add(hs_nodes); + +network.add(Node.create(3, { "tag": "a", "authority": 1, "relay": 1, "torrc": "authority.tmpl" -}) - -# hidden service (hs) nodes -hs_nodes = [] - -hs_servers = [] - -from twisted.web import server, resource -from twisted.internet import reactor - -# Use twisted to create web servers in this script -class Site(resource.Resource): - isLeaf = True - numberRequests = 0 - - def __init__(self, siteNum): - self.siteNum = siteNum - - def render_GET(self, request): - self.numberRequests += 1 - request.setHeader("content-type", "text/plain") - return str(self.siteNum) - -base_port = 10080 -for i in range(3): - - port = base_port + i - - node = Node( - tag="h", - hiddenservice=1, - torrc="hidden-service.tmpl", - hiddenservicetarget="127.0.0.1:%i" % port - ) - hs_nodes.append(node) - - site = server.Site(Site(i)) - - s = reactor.listenTCP(port, site) - hs_servers.append(s) +})) - port += 1 - -client_nodes = Node.create(12, { +network.add(Node.create(12, { "tag": "c", "torrc": "client.tmpl" -}) +})) -relay_nodes = Node.create(10, { +network.add(Node.create(10, { "tag": "r", "relay": 1, "torrc": "intro.tmpl" -}) - -thread.start_new_thread(reactor.run, (), {"installSignalHandlers": 0}) - -initial_nodes = authority_nodes + relay_nodes + client_nodes + hs_nodes[:1] +})) def start(): - if not all([ n.getController().start() for n in initial_nodes ]): - return False - - logging.info("All initial nodes running") - - nodes_by_fingerprint = get_node_fingerprints(authority_nodes + relay_nodes) - - track_introduction_points(hs_nodes[0]) - - node_0_published_descriptor = threading.Event() - - def hs_node_0_listener(logevent): - if "Successfully uploaded v2 rend descriptors" in logevent.message: - node_0_published_descriptor.set() - - - hs_nodes[0].getStemController().add_event_listener(hs_node_0_listener, EventType.INFO) - - node_0_published_descriptor.wait() - - hs_nodes[0].getStemController().remove_event_listener(hs_node_0_listener) - - # list to cope with scope problems - node_counter = [len(hs_nodes) - 1] - node_counter_lock = threading.RLock() - - nodes_published_descriptors = threading.Event() - - def hs_node_listener(logevent): - if "Successfully uploaded v2 rend descriptors" in logevent.message: - with node_counter_lock: - node_counter[0] -= 1 - - if node_counter[0] == 0: - nodes_published_descriptors.set() - - for node in hs_nodes[1:]: - node.getController().start() - track_introduction_points(node) - - node.getStemController().add_event_listener(hs_node_listener, EventType.INFO) - - nodes_published_descriptors.wait() - - for node in hs_nodes[1:]: - node.getStemController().remove_event_listener(hs_node_listener) - - test_intro_failure(nodes_by_fingerprint) - -def test_intro_failure(nodes_by_fingerprint): - connection_test() - - time.sleep(5) - - # Select a random node that is being used as an introduction point - nodenum = random.choice(node_intro_circuits.keys()) - fingerprint = random.choice(node_intro_circuits[nodenum].values()) - - node = nodes_by_fingerprint[fingerprint] - - logging.info("stopping " + node._env["nick"] + " (" + fingerprint + ")") - - node.getStemController().close() - node.getController().stop() - - logging.info("begining to watch for the establishment of new introduction points") - - changed = [False for n in hs_nodes] - - intro_points_before = [set(node_intro_circuits[n._env["nodenum"]].values()) for n in hs_nodes] - - time.sleep(20) - - intro_points_after = [set(node_intro_circuits[n._env["nodenum"]].values()) for n in hs_nodes] - - for i, node in enumerate(hs_nodes): - before = intro_points_before[i] - after = intro_points_after[i] - - if before != after: - changed[i] = True - - if all(changed): - logging.info("All changed") - else: - logging.info("All did not change") - - check_same_intro_points() - - connection_test() + results = hs_fail_test(network) for server in hs_servers: server.stopListening() - reactor.stop() - - stop() - -def connection_test(): - logging.info("connecting to clients") - responses = {"0": 0, "1": 0, "2": 0} - - for c in client_nodes: - result = c.query("http://2oiifbe3wne4iaqb.onion/"); + network.stop() - if result in responses: - responses[result] += 1 - else: - logging.info("Unknown response: " + str(result)) + return results - logging.info(str(responses)) -- cgit v1.2.3