More tests

4 files changed, 658 insertions, 0 deletions

diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py
index 4ddda49..28629ff 100644
--- a/lib/chutney/TorNet.py
+++ b/lib/chutney/TorNet.py
@@ -20,6 +20,7 @@ import re
 import errno
 import time
 import thread
+import threading
 import random
 
 import stem.control
@@ -756,6 +757,7 @@ def runConfigFile(verb, f):
                     thread=thread,
                     Queue=Queue,
                     random=random,
+                    threading=threading,
                     log=log,
                     stop=stop,
                     _THE_NETWORK=_THE_NETWORK)
diff --git a/networks/hs-intro-fail b/networks/hs-intro-fail
index 3b18d0b..deae478 100644
--- a/networks/hs-intro-fail
+++ b/networks/hs-intro-fail
@@ -48,6 +48,8 @@ node_intro_circuits = {}
 
 node_intro_events = {}
 
+introduction_point_circuits = {}
+
 def track_introduction_points(node):
     nodenum = node._env["nodenum"]
     nodenick = node._env["nick"]
@@ -99,6 +101,55 @@ def track_introduction_points(node):
     node.getStemController().add_event_listener(listener, EventType.INFO)
     log("Tracking introduction points for " + node._env["nick"])
 
+def track_introduction_point(node):
+    nodenum = node._env["nodenum"]
+    nodenick = node._env["nick"]
+
+    introduction_point_circuits[node] = {}
+
+    def listener(logevent):
+        prefix = " "
+        if logevent.message.startswith(prefix):
+            circuit = logevent.message[len(prefix):].split(" ")[0]
+            fingerprint = logevent.message[len(prefix):].split(" ")[1][1:]
+
+            node_intro_circuits[nodenum][circuit] = fingerprint
+
+            added_intro_node = nodes_by_fingerprint[fingerprint]
+
+            log("%s: added %s (%s)" % (nodenick, added_intro_node._env["nick"], fingerprint))
+
+            node_intro_events[nodenum].put(("added", fingerprint))
+
+        prefix = "rend_services_introduce(): Giving up on "
+        if logevent.message.startswith(prefix) or "Giving up on" in logevent.message:
+            log(logevent.message)
+            fingerprint = logevent.message[len(prefix):].split(" ")[0][1:41]
+
+            for_deletion = []
+
+            for circuit, fp in node_intro_circuits[nodenum].iteritems():
+                #log("circuit " + str(circuit) + " fp " + str(fp))
+                if fingerprint == fp:
+                    for_deletion.append(circuit)
+
+            if len(for_deletion) == 0:
+                log("got log message " + logevent.message + " but not deleting an introduction point")
+
+            for circuit in for_deletion:
+                fingerprint = node_intro_circuits[nodenum][circuit]
+
+                removed_intro_node = nodes_by_fingerprint[fingerprint]
+
+                log("%s: removed %s (%s)" % (nodenick, removed_intro_node._env["nick"], fingerprint))
+
+                node_intro_events[nodenum].put(("removed", fingerprint))
+                del node_intro_circuits[nodenum][circuit]
+
+
+    node.getStemController().add_event_listener(listener, EventType.INFO)
+    log("Tracking introduction points for " + node._env["nick"])
+
 # Use twisted to create web servers in this script
 class Site(resource.Resource):
     isLeaf = True
diff --git a/networks/hs-intro-round-robin b/networks/hs-intro-round-robin
new file mode 100644
index 0000000..f210bc7
--- /dev/null
+++ b/networks/hs-intro-round-robin
@@ -0,0 +1,228 @@
+Authority = Node.create(3, {
+    "tag": "a",
+    "authority": 1,
+    "relay": 1,
+    "torrc": "authority.tmpl"
+})
+
+Relay = Node.create(10, {
+    "tag": "r",
+    "relay": 1,
+    "torrc": "intro.tmpl"
+})
+
+Client = Node.create(6, {
+    "tag": "c",
+    "torrc": "client.tmpl"
+})
+
+# hidden service (hs) nodes
+
+hs_node_1 = Node(
+    tag="h",
+    hiddenservice=1,
+    torrc="hidden-service.tmpl",
+    hiddenservicetarget="127.0.0.1:8081"
+)
+
+hs_node_2 = Node(
+    tag="h",
+    hiddenservice=1,
+    torrc="hidden-service.tmpl",
+    hiddenservicetarget="127.0.0.1:8082"
+)
+
+hs_nodes = (hs_node_1, hs_node_2)
+
+initial_nodes = Authority + Relay + Client + [ hs_node_1 ]
+
+nodes_by_fingerprint = {}
+
+#{ nodenum: { circuit: "fingerprint", circuit: "fingerprint" } }
+
+# Introduction Point Tracking
+
+node_intro_circuits = {}
+
+node_intro_events = {}
+
+introduction_point_circuits = {}
+
+def track_hs_node(node):
+    nodenum = node._env["nodenum"]
+    nodenick = node._env["nick"]
+
+    node_intro_circuits[nodenum] = {}
+
+    node_intro_events[nodenum] = Queue.Queue()
+
+    def listener(logevent):
+        prefix = "rend_service_intro_has_opened(): Established circuit "
+        if logevent.message.startswith(prefix):
+            circuit = logevent.message[len(prefix):].split(" ")[0]
+            fingerprint = logevent.message[len(prefix):].split(" ")[1][1:]
+
+            node_intro_circuits[nodenum][circuit] = fingerprint
+
+            added_intro_node = nodes_by_fingerprint[fingerprint]
+
+            log("%s: added %s (%s)" % (nodenick, added_intro_node._env["nick"], fingerprint))
+
+            node_intro_events[nodenum].put(("added", fingerprint))
+
+        prefix = "rend_services_introduce(): Giving up on "
+        if logevent.message.startswith(prefix) or "Giving up on" in logevent.message:
+            log(logevent.message)
+            fingerprint = logevent.message[len(prefix):].split(" ")[0][1:41]
+
+            for_deletion = []
+
+            for circuit, fp in node_intro_circuits[nodenum].iteritems():
+                #log("circuit " + str(circuit) + " fp " + str(fp))
+                if fingerprint == fp:
+                    for_deletion.append(circuit)
+
+            if len(for_deletion) == 0:
+                log("got log message " + logevent.message + " but not deleting an introduction point")
+
+            for circuit in for_deletion:
+                fingerprint = node_intro_circuits[nodenum][circuit]
+
+                removed_intro_node = nodes_by_fingerprint[fingerprint]
+
+                log("%s: removed %s (%s)" % (nodenick, removed_intro_node._env["nick"], fingerprint))
+
+                node_intro_events[nodenum].put(("removed", fingerprint))
+                del node_intro_circuits[nodenum][circuit]
+
+
+    node.getStemController().add_event_listener(listener, EventType.INFO)
+    log("Tracking introduction points for " + node._env["nick"])
+
+def track_introduction_point(node):
+    nodenum = node._env["nodenum"]
+    nodenick = node._env["nick"]
+
+    introduction_point_circuits[node] = {}
+
+    def listener(logevent):
+        prefix = "rend_mid_introduce"
+        if logevent.message.startswith(prefix):
+            print(logevent.message)
+
+    node.getStemController().add_event_listener(listener, EventType.INFO)
+    log("Tracking introduction points for " + node._env["nick"])
+
+# Use twisted to create web servers in this script
+class Site(resource.Resource):
+    isLeaf = True
+    numberRequests = 0
+
+    def __init__(self, siteNum):
+        self.siteNum = siteNum
+
+    def render_GET(self, request):
+        self.numberRequests += 1
+        request.setHeader("content-type", "text/plain")
+        return str(self.siteNum)
+
+def start_test():
+    log("Starting test")
+
+    port8081 = reactor.listenTCP(8081, server.Site(Site(1)))
+    port8082 = reactor.listenTCP(8082, server.Site(Site(2)))
+
+    def cleanup():
+        port8081.stopListening()
+        log("stopped 8081")
+        port8082.stopListening()
+        log("stopped 8082")
+        reactor.stop()
+        log("stopped the reactor")
+
+    def connection_test():
+        log("connecting to clients")
+        responses = {"1": 0, "2": 0}
+
+        for c in Client:
+            result = c.query("http://2oiifbe3wne4iaqb.onion/");
+
+            if result in responses:
+                responses[result] += 1
+            else:
+                log("Unknown response:")
+
+        log(responses)
+
+    def test():
+        connection_test()
+
+        connection_test()
+
+        cleanup()
+
+        stop()
+
+    thread.start_new_thread(test, ())
+
+    log("running reactor")
+
+    reactor.run(installSignalHandlers=0)
+
+def check_same_intro_points():
+    intro_sets = []
+    for introduction_points in node_intro_circuits.values():
+        intro_sets.append(set(introduction_points.values()))
+
+    if intro_sets[0].issubset(intro_sets[1]) and intro_sets[1].issubset(intro_sets[0]):
+        log("Same introduction points")
+        for fingerprint in node_intro_circuits.values()[0].values():
+            node = nodes_by_fingerprint[fingerprint]
+            log(" - " + node._env["nick"] + "(" + fingerprint + ")")
+        return True
+    else:
+        log("Not the same introduction points")
+        for num, circuits in node_intro_circuits.items():
+            log("node %i" % num)
+            for fingerprint in circuits.values():
+                node = nodes_by_fingerprint[fingerprint]
+                log(" - " + node._env["nick"] + "(" + fingerprint + ")")
+        return False
+
+def start():
+    log("Starting round robin test")
+
+    if not all([ n.getController().start() for n in initial_nodes ]):
+        return False
+
+    log("All initial nodes running")
+
+    for node in Authority + Relay:
+        fingerprint = node.getStemController().get_info("fingerprint")
+        nodes_by_fingerprint[fingerprint] = node
+
+        track_introduction_point(node)
+
+    track_hs_node(hs_node_1)
+
+    def log_listener(logevent):
+        if "Successfully uploaded v2 rend descriptors" in logevent.message:
+            hs_node_1.getStemController().remove_event_listener(log_listener)
+
+            def start_listener(logevent):
+
+                # Probably after what needs waiting for
+                if "Successfully uploaded v2 rend descriptors" in logevent.message:
+                    hs_node_2.getStemController().remove_event_listener(start_listener)
+
+                    if check_same_intro_points():
+                        thread.start_new_thread(start_test, ())
+                        log("finished running start_test in another thread")
+
+            hs_node_2.getController().start()
+
+            track_hs_node(hs_node_2)
+
+            hs_node_2.getStemController().add_event_listener(start_listener, EventType.INFO)
+
+    hs_node_1.getStemController().add_event_listener(log_listener, EventType.INFO)
diff --git a/networks/hs-intro-scale b/networks/hs-intro-scale
new file mode 100644
index 0000000..7645520
--- /dev/null
+++ b/networks/hs-intro-scale
@@ -0,0 +1,377 @@
+authority_nodes = Node.create(3, {
+    "tag": "a",
+    "authority": 1,
+    "relay": 1,
+    "torrc": "authority.tmpl"
+})
+
+relay_nodes = Node.create(100, {
+    "tag": "r",
+    "relay": 1,
+    "torrc": "intro.tmpl"
+})
+
+client_nodes = Node.create(50, {
+    "tag": "c",
+    "torrc": "client.tmpl"
+})
+
+# hidden service (hs) nodes
+hs_nodes = []
+
+hs_servers = []
+
+port = 8081
+
+# Use twisted to create web servers in this script
+class Site(resource.Resource):
+    isLeaf = True
+    numberRequests = 0
+
+    def __init__(self, siteNum):
+        self.siteNum = siteNum
+
+    def render_GET(self, request):
+        self.numberRequests += 1
+        request.setHeader("content-type", "text/plain")
+        return str(self.siteNum)
+
+base_port = 9080
+for i in range(1, 3):
+
+    port = base_port + i
+
+    node = Node(
+        tag="h",
+        hiddenservice=1,
+        torrc="hidden-service.tmpl",
+        hiddenservicetarget="127.0.0.1:%i" % port
+    )
+    hs_nodes.append(node)
+
+    s = reactor.listenTCP(port, server.Site(Site(i)))
+    hs_servers.append(s)
+
+    port += 1
+
+
+
+initial_nodes = authority_nodes + relay_nodes + client_nodes + hs_nodes[:1]
+
+def start():
+    if not all([ n.getController().start() for n in initial_nodes ]):
+        return False
+
+    log("All initial nodes running")
+
+    for n in authority_nodes + relay_nodes:
+        fingerprint = n.getStemController().get_info("fingerprint")
+        nodes_by_fingerprint[fingerprint] = n
+
+    track_introduction_points(hs_nodes[0])
+
+    def hs_node_0_listener(logevent):
+        if "Successfully uploaded v2 rend descriptors" in logevent.message:
+            hs_nodes[0].getStemController().remove_event_listener(hs_node_0_listener)
+
+            thread.start_new_thread(test_load, ())
+
+    hs_nodes[0].getStemController().add_event_listener(hs_node_0_listener, EventType.INFO)
+
+def test_load():
+    thread.start_new_thread(reactor.run, kwargs={ installSignalHandlers: 0 })
+
+    # Run initial bandwidth test
+    max_load = determine_max_load("http://2oiifbe3wne4iaqb.onion/", client_nodes)
+
+    print("max_load: " + str(max_load))
+
+    for server in hs_servers:
+        server.stopListening()
+
+    reactor.stop()
+    log("stopped the reactor")
+
+    """def start_listener(logevent):
+
+        # Probably after what needs waiting for
+        if "Successfully uploaded v2 rend descriptors" in logevent.message:
+            hs_node_2.getStemController().remove_event_listener(start_listener)
+
+            if check_same_intro_points():
+                thread.start_new_thread(start_test, ())
+                log("finished running start_test in another thread")
+
+    hs_node_2.getController().start()
+
+    track_introduction_points(hs_node_2)
+
+    hs_node_2.getStemController().add_event_listener(start_listener, EventType.INFO)"""
+
+## Utility Code
+
+# Introduction Point Tracking
+
+node_intro_circuits = {}
+
+node_intro_events = {}
+
+introduction_point_circuits = {}
+
+nodes_by_fingerprint = {}
+
+#{ nodenum: { circuit: "fingerprint", circuit: "fingerprint" } }
+
+def track_introduction_points(node):
+    nodenum = node._env["nodenum"]
+    nodenick = node._env["nick"]
+
+    node_intro_circuits[nodenum] = {}
+
+    node_intro_events[nodenum] = Queue.Queue()
+
+    def listener(logevent):
+        prefix = "rend_service_intro_has_opened(): Established circuit "
+        if logevent.message.startswith(prefix):
+            circuit = logevent.message[len(prefix):].split(" ")[0]
+            fingerprint = logevent.message[len(prefix):].split(" ")[1][1:]
+
+            node_intro_circuits[nodenum][circuit] = fingerprint
+
+            added_intro_node = nodes_by_fingerprint[fingerprint]
+
+            log("%s: added %s (%s)" % (nodenick, added_intro_node._env["nick"], fingerprint))
+
+            node_intro_events[nodenum].put(("added", fingerprint))
+
+        prefix = "rend_services_introduce(): Giving up on "
+        if logevent.message.startswith(prefix) or "Giving up on" in logevent.message:
+            log(logevent.message)
+            fingerprint = logevent.message[len(prefix):].split(" ")[0][1:41]
+
+            for_deletion = []
+
+            for circuit, fp in node_intro_circuits[nodenum].iteritems():
+                #log("circuit " + str(circuit) + " fp " + str(fp))
+                if fingerprint == fp:
+                    for_deletion.append(circuit)
+
+            if len(for_deletion) == 0:
+                log("got log message " + logevent.message + " but not deleting an introduction point")
+
+            for circuit in for_deletion:
+                fingerprint = node_intro_circuits[nodenum][circuit]
+
+                removed_intro_node = nodes_by_fingerprint[fingerprint]
+
+                log("%s: removed %s (%s)" % (nodenick, removed_intro_node._env["nick"], fingerprint))
+
+                node_intro_events[nodenum].put(("removed", fingerprint))
+                del node_intro_circuits[nodenum][circuit]
+
+
+    node.getStemController().add_event_listener(listener, EventType.INFO)
+    log("Tracking introduction points for " + node._env["nick"])
+
+def track_introduction_point(node):
+    nodenum = node._env["nodenum"]
+    nodenick = node._env["nick"]
+
+    introduction_point_circuits[node] = {}
+
+    def listener(logevent):
+        prefix = " "
+        if logevent.message.startswith(prefix):
+            circuit = logevent.message[len(prefix):].split(" ")[0]
+            fingerprint = logevent.message[len(prefix):].split(" ")[1][1:]
+
+            node_intro_circuits[nodenum][circuit] = fingerprint
+
+            added_intro_node = nodes_by_fingerprint[fingerprint]
+
+            log("%s: added %s (%s)" % (nodenick, added_intro_node._env["nick"], fingerprint))
+
+            node_intro_events[nodenum].put(("added", fingerprint))
+
+        prefix = "rend_services_introduce(): Giving up on "
+        if logevent.message.startswith(prefix) or "Giving up on" in logevent.message:
+            log(logevent.message)
+            fingerprint = logevent.message[len(prefix):].split(" ")[0][1:41]
+
+            for_deletion = []
+
+            for circuit, fp in node_intro_circuits[nodenum].iteritems():
+                #log("circuit " + str(circuit) + " fp " + str(fp))
+                if fingerprint == fp:
+                    for_deletion.append(circuit)
+
+            if len(for_deletion) == 0:
+                log("got log message " + logevent.message + " but not deleting an introduction point")
+
+            for circuit in for_deletion:
+                fingerprint = node_intro_circuits[nodenum][circuit]
+
+                removed_intro_node = nodes_by_fingerprint[fingerprint]
+
+                log("%s: removed %s (%s)" % (nodenick, removed_intro_node._env["nick"], fingerprint))
+
+                node_intro_events[nodenum].put(("removed", fingerprint))
+                del node_intro_circuits[nodenum][circuit]
+
+
+    node.getStemController().add_event_listener(listener, EventType.INFO)
+    log("Tracking introduction points for " + node._env["nick"])
+
+
+def determine_max_load(service_url, clients):
+    current_load = 0 # requests per second
+
+    request_queue = Queue.Queue(1000)
+    singlelock = threading.Lock()
+
+    responses = []
+
+    running = True
+
+    class worker(threading.Thread):
+        def __init__ (self, client):
+            super(worker, self).__init__()
+            self.client = client
+
+        def run(self):
+            while running:
+                try:
+                    request = request_queue.get(True, 1)
+
+                    result = self.client.query("http://2oiifbe3wne4iaqb.onion/");
+
+                    singlelock.acquire()
+                    responses.append((result, time.time()))
+                    singlelock.release()
+
+                    # Let the queue know the job is finished.
+                    request_queue.task_done()
+                except:
+                    pass
+
+    # Create worker threads
+    for client in clients:
+        worker(client).start()
+
+    # Add elements to queue at rate current_load
+    # Check size of queue, if above limit, return current_load
+
+    while True:
+        # Determine Average Load
+        time_barrier = time.time() - 10.0
+
+        last_responses = []
+
+        singlelock.acquire()
+        for response, response_time in reversed(responses):
+            if response_time < time_barrier:
+                break
+            last_responses.append(response)
+        singlelock.release()
+
+        current_average_load = len(last_responses) / 10.0
+        print("current average load %f" % current_average_load)
+
+        if current_average_load > current_load:
+            running = False
+            return current_average_load
+
+        current_load += 1
+
+        time_per_item = 1.0 / current_load
+
+        for i in range(current_load):
+            request_queue.put(1)
+            time.sleep(time_per_item)
+
+
+def start_test():
+    log("Starting test")
+
+    port8081 = reactor.listenTCP(8081, server.Site(Site(1)))
+    port8082 = reactor.listenTCP(8082, server.Site(Site(2)))
+
+    def cleanup():
+        port8081.stopListening()
+        log("stopped 8081")
+        port8082.stopListening()
+        log("stopped 8082")
+        reactor.stop()
+        log("stopped the reactor")
+
+    def connection_test():
+        log("connecting to clients")
+        responses = {"1": 0, "2": 0}
+
+        for c in Client:
+            result = c.query("http://2oiifbe3wne4iaqb.onion/");
+
+            if result in responses:
+                responses[result] += 1
+            else:
+                log("Unknown response:")
+
+        log(responses)
+
+    def test():
+        connection_test()
+
+        time.sleep(5)
+
+        # Clear out the event queues such that the events are relevant
+        for node in hs_nodes:
+            nodenum = node._env["nodenum"]
+
+            log("replaced queue")
+            node_intro_events[nodenum] = Queue.Queue()
+
+        # Select a random node that is being used as an introduction point
+        nodenum = random.choice(node_intro_circuits.keys())
+        fingerprint = random.choice(node_intro_circuits[nodenum].values())
+
+        node = nodes_by_fingerprint[fingerprint]
+
+        log("stopping " + node._env["nick"] + " (" + fingerprint + ")")
+
+        node.getController().stop()
+
+        watch_for_new_intro_points()
+
+        check_same_intro_points()
+
+        connection_test()
+
+        cleanup()
+
+        stop()
+
+    thread.start_new_thread(test, ())
+
+    log("running reactor")
+
+    reactor.run(installSignalHandlers=0)
+
+def check_same_intro_points():
+    intro_sets = []
+    for introduction_points in node_intro_circuits.values():
+        intro_sets.append(set(introduction_points.values()))
+
+    if intro_sets[0].issubset(intro_sets[1]) and intro_sets[1].issubset(intro_sets[0]):
+        log("Same introduction points")
+        for fingerprint in node_intro_circuits.values()[0].values():
+            node = nodes_by_fingerprint[fingerprint]
+            log(" - " + node._env["nick"] + "(" + fingerprint + ")")
+        return True
+    else:
+        log("Not the same introduction points")
+        for num, circuits in node_intro_circuits.items():
+            log("node %i" % num)
+            for fingerprint in circuits.values():
+                node = nodes_by_fingerprint[fingerprint]
+                log(" - " + node._env["nick"] + "(" + fingerprint + ")")
+        return False