Get SSL connections and linked connections working with bufferevents.

Clients are now verified to work and build circuits correctly.  There
are still a few warnings given here and there that I need to look into.

9 files changed, 241 insertions, 52 deletions

diff --git a/src/common/tortls.c b/src/common/tortls.c
index 7dfdca6af..47f6adb9c 100644
--- a/src/common/tortls.c
+++ b/src/common/tortls.c
@@ -44,7 +44,14 @@
 #error "We require OpenSSL >= 0.9.7"
 #endif
 
+#ifdef USE_BUFFEREVENTS
+#include <event2/bufferevent_ssl.h>
+#include <event2/buffer.h>
+#include "compat_libevent.h"
+#endif
+
 #define CRYPTO_PRIVATE /* to import prototypes from crypto.h */
+#define TORTLS_PRIVATE
 
 #include "crypto.h"
 #include "tortls.h"
@@ -107,6 +114,7 @@ struct tor_tls_t {
   enum {
     TOR_TLS_ST_HANDSHAKE, TOR_TLS_ST_OPEN, TOR_TLS_ST_GOTCLOSE,
     TOR_TLS_ST_SENTCLOSE, TOR_TLS_ST_CLOSED, TOR_TLS_ST_RENEGOTIATE,
+    TOR_TLS_ST_BUFFEREVENT
   } state : 3; /**< The current SSL state, depending on which operations have
                 * completed successfully. */
   unsigned int isServer:1; /**< True iff this is a server-side connection */
@@ -1192,56 +1200,76 @@ tor_tls_handshake(tor_tls_t *tls)
   }
   if (r == TOR_TLS_DONE) {
     tls->state = TOR_TLS_ST_OPEN;
-    if (tls->isServer) {
-      SSL_set_info_callback(tls->ssl, NULL);
-      SSL_set_verify(tls->ssl, SSL_VERIFY_PEER, always_accept_verify_cb);
-      /* There doesn't seem to be a clear OpenSSL API to clear mode flags. */
-      tls->ssl->mode &= ~SSL_MODE_NO_AUTO_CHAIN;
+    return tor_tls_finish_handshake(tls);
+  }
+  return r;
+}
+
+int
+tor_tls_finish_handshake(tor_tls_t *tls)
+{
+  int r = TOR_TLS_DONE;
+  if (tls->isServer) {
+    SSL_set_info_callback(tls->ssl, NULL);
+    SSL_set_verify(tls->ssl, SSL_VERIFY_PEER, always_accept_verify_cb);
+    /* There doesn't seem to be a clear OpenSSL API to clear mode flags. */
+    tls->ssl->mode &= ~SSL_MODE_NO_AUTO_CHAIN;
 #ifdef V2_HANDSHAKE_SERVER
-      if (tor_tls_client_is_using_v2_ciphers(tls->ssl, ADDR(tls))) {
-        /* This check is redundant, but back when we did it in the callback,
-         * we might have not been able to look up the tor_tls_t if the code
-         * was buggy.  Fixing that. */
-        if (!tls->wasV2Handshake) {
-          log_warn(LD_BUG, "For some reason, wasV2Handshake didn't"
-                   " get set. Fixing that.");
-        }
-        tls->wasV2Handshake = 1;
-        log_debug(LD_HANDSHAKE,
-                  "Completed V2 TLS handshake with client; waiting "
-                  "for renegotiation.");
-      } else {
-        tls->wasV2Handshake = 0;
+    if (tor_tls_client_is_using_v2_ciphers(tls->ssl, ADDR(tls))) {
+      /* This check is redundant, but back when we did it in the callback,
+       * we might have not been able to look up the tor_tls_t if the code
+       * was buggy.  Fixing that. */
+      if (!tls->wasV2Handshake) {
+        log_warn(LD_BUG, "For some reason, wasV2Handshake didn't"
+                 " get set. Fixing that.");
       }
-#endif
+      tls->wasV2Handshake = 1;
+      log_debug(LD_HANDSHAKE, "Completed V2 TLS handshake with client; waiting "
+                "for renegotiation.");
     } else {
+      tls->wasV2Handshake = 0;
+    }
+#endif
+  } else {
 #ifdef V2_HANDSHAKE_CLIENT
-      /* If we got no ID cert, we're a v2 handshake. */
-      X509 *cert = SSL_get_peer_certificate(tls->ssl);
-      STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl);
-      int n_certs = sk_X509_num(chain);
-      if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) {
-        log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it "
-                  "looks like a v1 handshake on %p", tls);
-        tls->wasV2Handshake = 0;
-      } else {
-        log_debug(LD_HANDSHAKE,
-                  "Server sent back a single certificate; looks like "
-                  "a v2 handshake on %p.", tls);
-        tls->wasV2Handshake = 1;
-      }
-      if (cert)
-        X509_free(cert);
+    /* If we got no ID cert, we're a v2 handshake. */
+    X509 *cert = SSL_get_peer_certificate(tls->ssl);
+    STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl);
+    int n_certs = sk_X509_num(chain);
+    if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) {
+      log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it "
+                "looks like a v1 handshake on %p", tls);
+      tls->wasV2Handshake = 0;
+    } else {
+      log_debug(LD_HANDSHAKE,
+                "Server sent back a single certificate; looks like "
+                "a v2 handshake on %p.", tls);
+      tls->wasV2Handshake = 1;
+    }
+    if (cert)
+      X509_free(cert);
 #endif
-      if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) {
-        tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers");
-        r = TOR_TLS_ERROR_MISC;
-      }
+    if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) {
+      tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers");
+      r = TOR_TLS_ERROR_MISC;
     }
   }
   return r;
 }
 
+#ifdef USE_BUFFEREVENTS
+/** DOCDOC */
+int
+tor_tls_start_renegotiating(tor_tls_t *tls)
+{
+  int r = SSL_renegotiate(tls->ssl);
+  if (r <= 0) {
+    return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, LD_HANDSHAKE);
+  }
+  return 0;
+}
+#endif
+
 /** Client only: Renegotiate a TLS session.  When finished, returns
  * TOR_TLS_DONE.  On failure, returns TOR_TLS_ERROR, TOR_TLS_WANTREAD, or
  * TOR_TLS_WANTWRITE.
@@ -1458,6 +1486,8 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key)
     log_fn(severity,LD_PROTOCOL,"No distinct identity certificate found");
     goto done;
   }
+  tls_log_errors(tls, severity, LD_HANDSHAKE, "before verifying certificate");
+
   if (!(id_pkey = X509_get_pubkey(id_cert)) ||
       X509_verify(cert, id_pkey) <= 0) {
     log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0");
@@ -1629,3 +1659,43 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls,
   *wbuf_bytes = tls->ssl->s3->wbuf.left;
 }
 
+#ifdef USE_BUFFEREVENTS
+/** DOCDOC may free bufev_in */
+struct bufferevent *
+tor_tls_init_bufferevent(tor_tls_t *tls, struct bufferevent *bufev_in,
+                         evutil_socket_t socket, int receiving)
+{
+  struct bufferevent *out;
+  const enum bufferevent_ssl_state state = receiving ?
+    BUFFEREVENT_SSL_ACCEPTING : BUFFEREVENT_SSL_CONNECTING;
+
+#if 0
+  (void) socket;
+  out = bufferevent_openssl_filter_new(tor_libevent_get_base(),
+                                       bufev_in,
+                                       tls->ssl,
+                                       state,
+                                       BEV_OPT_DEFER_CALLBACKS);
+#else
+  /* Disabled: just use filter for now. */
+  if (bufev_in) {
+    evutil_socket_t s = bufferevent_getfd(bufev_in);
+    tor_assert(s == -1 || s == socket);
+    tor_assert(evbuffer_get_length(bufferevent_get_input(bufev_in)) == 0);
+    tor_assert(evbuffer_get_length(bufferevent_get_output(bufev_in)) == 0);
+    tor_assert(BIO_number_read(SSL_get_rbio(tls->ssl)) == 0);
+    tor_assert(BIO_number_written(SSL_get_rbio(tls->ssl)) == 0);
+  }
+  tls->state = TOR_TLS_ST_BUFFEREVENT;
+  bufferevent_free(bufev_in);
+  out = bufferevent_openssl_socket_new(tor_libevent_get_base(),
+                                       socket,
+                                       tls->ssl,
+                                       state,
+                                       0);
+                                      //BEV_OPT_DEFER_CALLBACKS);
+#endif
+  return out;
+}
+#endif
+
diff --git a/src/common/tortls.h b/src/common/tortls.h
index e4b1ad65f..64ce5c634 100644
--- a/src/common/tortls.h
+++ b/src/common/tortls.h
@@ -64,6 +64,9 @@ int tor_tls_check_lifetime(tor_tls_t *tls, int tolerance);
 int tor_tls_read(tor_tls_t *tls, char *cp, size_t len);
 int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n);
 int tor_tls_handshake(tor_tls_t *tls);
+#if defined(USE_BUFFEREVENTS) || defined(TORTLS_PRIVATE)
+int tor_tls_finish_handshake(tor_tls_t *tls);
+#endif
 int tor_tls_renegotiate(tor_tls_t *tls);
 void tor_tls_block_renegotiation(tor_tls_t *tls);
 int tor_tls_shutdown(tor_tls_t *tls);
@@ -85,5 +88,12 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls);
 
 void _check_no_tls_errors(const char *fname, int line);
 
+#ifdef USE_BUFFEREVENTS
+int tor_tls_start_renegotiating(tor_tls_t *tls);
+struct bufferevent *tor_tls_init_bufferevent(tor_tls_t *tls,
+                                     struct bufferevent *bufev_in,
+                                     evutil_socket_t socket, int receiving);
+#endif
+
 #endif
 
diff --git a/src/or/Makefile.am b/src/or/Makefile.am
index a9ac3cdee..b1add7cb6 100644
--- a/src/or/Makefile.am
+++ b/src/or/Makefile.am
@@ -45,6 +45,10 @@ tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \
 	../common/libor-event.a \
 	@TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@
 
+if USE_BUFFEREVENTS
+tor_LDADD += -levent_openssl
+endif
+
 noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \
 	command.h config.h connection_edge.h connection.h connection_or.h \
 	control.h cpuworker.h directory.h dirserv.h dirvote.h dns.h \
diff --git a/src/or/buffers.c b/src/or/buffers.c
index b750a0540..ffdbcd3b4 100644
--- a/src/or/buffers.c
+++ b/src/or/buffers.c
@@ -1083,7 +1083,7 @@ fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out,
     goto done;
   }
   n = inspect_evbuffer(buf, &hdr, VAR_CELL_HEADER_SIZE, &free_hdr, NULL);
-  tor_assert(n == VAR_CELL_HEADER_SIZE);
+  tor_assert(n >= VAR_CELL_HEADER_SIZE);
 
   command = get_uint8(hdr+2);
   if (!(CELL_COMMAND_IS_VAR_LENGTH(command))) {
diff --git a/src/or/connection.c b/src/or/connection.c
index 570eceeef..b39763055 100644
--- a/src/or/connection.c
+++ b/src/or/connection.c
@@ -36,6 +36,10 @@
 #include "router.h"
 #include "routerparse.h"
 
+#ifdef USE_BUFFEREVENTS
+#include <event2/event.h>
+#endif
+
 static connection_t *connection_create_listener(
                                struct sockaddr *listensockaddr,
                                socklen_t listensocklen, int type,
@@ -194,6 +198,7 @@ connection_type_uses_bufferevent(connection_t *conn)
     case CONN_TYPE_EXIT:
     case CONN_TYPE_DIR:
     case CONN_TYPE_CONTROL:
+    case CONN_TYPE_OR:
       return 1;
     default:
       return 0;
@@ -2648,6 +2653,7 @@ evbuffer_inbuf_callback(struct evbuffer *buf,
   }
 }
 
+/** DOCDOC */
 static void
 evbuffer_outbuf_callback(struct evbuffer *buf,
                          const struct evbuffer_cb_info *info, void *arg)
@@ -2667,7 +2673,8 @@ evbuffer_outbuf_callback(struct evbuffer *buf,
   }
 }
 
-static void
+/** DOCDOC */
+void
 connection_handle_read_cb(struct bufferevent *bufev, void *arg)
 {
   connection_t *conn = arg;
@@ -2677,7 +2684,8 @@ connection_handle_read_cb(struct bufferevent *bufev, void *arg)
       connection_mark_for_close(conn);
 }
 
-static void
+/** DOCDOC */
+void
 connection_handle_write_cb(struct bufferevent *bufev, void *arg)
 {
   connection_t *conn = arg;
@@ -2690,12 +2698,18 @@ connection_handle_write_cb(struct bufferevent *bufev, void *arg)
   output = bufferevent_get_output(bufev);
   if (!evbuffer_get_length(output)) {
     connection_finished_flushing(conn);
-    if (conn->marked_for_close && conn->hold_open_until_flushed)
+    if (conn->marked_for_close && conn->hold_open_until_flushed) {
       conn->hold_open_until_flushed = 0;
+      if (conn->linked) {
+        /* send eof */
+        bufferevent_flush(conn->bufev, EV_WRITE, BEV_FINISHED);
+      }
+    }
   }
 }
 
-static void
+/** DOCDOC */
+void
 connection_handle_event_cb(struct bufferevent *bufev, short event, void *arg)
 {
   connection_t *conn = arg;
diff --git a/src/or/connection.h b/src/or/connection.h
index adf79f139..83aec0ef0 100644
--- a/src/or/connection.h
+++ b/src/or/connection.h
@@ -140,6 +140,10 @@ void remove_file_if_very_old(const char *fname, time_t now);
 #ifdef USE_BUFFEREVENTS
 int connection_type_uses_bufferevent(connection_t *conn);
 void connection_configure_bufferevent_callbacks(connection_t *conn);
+void connection_handle_read_cb(struct bufferevent *bufev, void *arg);
+void connection_handle_write_cb(struct bufferevent *bufev, void *arg);
+void connection_handle_event_cb(struct bufferevent *bufev, short event,
+                                 void *arg);
 #else
 #define connection_type_uses_bufferevent(c) (0)
 #endif
diff --git a/src/or/connection_or.c b/src/or/connection_or.c
index a2eb868d9..2fbc230bc 100644
--- a/src/or/connection_or.c
+++ b/src/or/connection_or.c
@@ -28,6 +28,10 @@
 #include "router.h"
 #include "routerlist.h"
 
+#ifdef USE_BUFFEREVENTS
+#include <event2/bufferevent_ssl.h>
+#endif
+
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_process_cells_from_inbuf(or_connection_t *conn);
 static int connection_or_send_versions(or_connection_t *conn);
@@ -37,6 +41,12 @@ static int connection_or_check_valid_tls_handshake(or_connection_t *conn,
                                                    int started_here,
                                                    char *digest_rcvd_out);
 
+#ifdef USE_BUFFEREVENTS
+static void connection_or_handle_event_cb(struct bufferevent *bufev,
+                                          short event, void *arg);
+#include <event2/buffer.h>/*XXXX REMOVE */
+#endif
+
 /**************************************************************/
 
 /** Map from identity digest of connected OR or desired OR to a connection_t
@@ -851,6 +861,7 @@ int
 connection_tls_start_handshake(or_connection_t *conn, int receiving)
 {
   conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING;
+  tor_assert(!conn->tls);
   conn->tls = tor_tls_new(conn->_base.s, receiving);
   tor_tls_set_logged_address(conn->tls, // XXX client and relay?
       escaped_safe_str(conn->_base.address));
@@ -858,12 +869,31 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving)
     log_warn(LD_BUG,"tor_tls_new failed. Closing.");
     return -1;
   }
+#ifdef USE_BUFFEREVENTS
+  if (connection_type_uses_bufferevent(TO_CONN(conn))) {
+    struct bufferevent *b =
+      tor_tls_init_bufferevent(conn->tls, conn->_base.bufev, conn->_base.s,
+                               receiving);
+    if (!b) {
+      log_warn(LD_BUG,"tor_tls_init_bufferevent failed. Closing.");
+      return -1;
+    }
+    conn->_base.bufev = b;
+    bufferevent_setcb(b, connection_handle_read_cb,
+                      connection_handle_write_cb,
+                      connection_or_handle_event_cb,
+                      TO_CONN(conn));
+  }
+#endif
   connection_start_reading(TO_CONN(conn));
   log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s);
   note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C);
 
-  if (connection_tls_continue_handshake(conn) < 0) {
-    return -1;
+  IF_HAS_BUFFEREVENT(TO_CONN(conn), {
+    /* ???? */;
+  }) ELSE_IF_NO_BUFFEREVENT {
+    if (connection_tls_continue_handshake(conn) < 0)
+      return -1;
   }
   return 0;
 }
@@ -948,6 +978,53 @@ connection_tls_continue_handshake(or_connection_t *conn)
   return 0;
 }
 
+#ifdef USE_BUFFEREVENTS
+static void
+connection_or_handle_event_cb(struct bufferevent *bufev, short event,
+                              void *arg)
+{
+  struct or_connection_t *conn = TO_OR_CONN(arg);
+
+  /* XXXX cut-and-paste code; should become a function. */
+  if (event & BEV_EVENT_CONNECTED) {
+    if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) {
+      if (tor_tls_finish_handshake(conn->tls) < 0) {
+        log_warn(LD_OR, "Problem finishing handshake");
+        connection_mark_for_close(TO_CONN(conn));
+        return;
+      }
+    }
+
+    if (! tor_tls_used_v1_handshake(conn->tls)) {
+      if (!tor_tls_is_server(conn->tls)) {
+        if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) {
+          conn->_base.state = OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING;
+          if (bufferevent_ssl_renegotiate(conn->_base.bufev)<0) {
+            log_warn(LD_OR, "Start_renegotiating went badly.");
+            connection_mark_for_close(TO_CONN(conn));
+          }
+          return; /* ???? */
+        }
+      } else {
+        /* improved handshake, but not a client. */
+        tor_tls_set_renegotiate_callback(conn->tls,
+                                         connection_or_tls_renegotiated_cb,
+                                         conn);
+        conn->_base.state = OR_CONN_STATE_TLS_SERVER_RENEGOTIATING;
+        /* return 0; */
+        return; /* ???? */
+      }
+    }
+    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
+    if (connection_tls_finish_handshake(conn) < 0)
+      connection_mark_for_close(TO_CONN(conn)); /* ???? */
+    return;
+  }
+
+  connection_handle_event_cb(bufev, event, arg);
+}
+#endif
+
 /** Return 1 if we initiated this connection, or 0 if it started
  * out as an incoming connection.
  */
@@ -1208,8 +1285,12 @@ connection_or_set_state_open(or_connection_t *conn)
 
   or_handshake_state_free(conn->handshake_state);
   conn->handshake_state = NULL;
+  IF_HAS_BUFFEREVENT(TO_CONN(conn), {
+    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
+  }) ELSE_IF_NO_BUFFEREVENT {
+    connection_start_reading(TO_CONN(conn));
+  }
 
-  connection_start_reading(TO_CONN(conn));
   circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */
 
   return 0;
@@ -1260,8 +1341,8 @@ connection_fetch_var_cell_from_buf(or_connection_t *or_conn, var_cell_t **out)
 {
   connection_t *conn = TO_CONN(or_conn);
   IF_HAS_BUFFEREVENT(conn, {
-      struct evbuffer *input = bufferevent_get_input(conn->bufev);
-      return fetch_var_cell_from_evbuffer(input, out, or_conn->link_proto);
+    struct evbuffer *input = bufferevent_get_input(conn->bufev);
+    return fetch_var_cell_from_evbuffer(input, out, or_conn->link_proto);
   }) ELSE_IF_NO_BUFFEREVENT {
     return fetch_var_cell_from_buf(conn->inbuf, out, or_conn->link_proto);
   }
diff --git a/src/or/main.c b/src/or/main.c
index cba98a884..263103662 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -722,9 +722,9 @@ conn_close_if_marked(int i)
       /* We need to do this explicitly so that the linked connection
        * notices that there was an EOF. */
       bufferevent_flush(conn->bufev, EV_WRITE, BEV_FINISHED);
-      /* XXXX Now can we free it? */
     }
-    return 0;
+    if (evbuffer_get_length(bufferevent_get_output(conn->bufev)))
+      return 0;
   }
 #endif
 
diff --git a/src/test/Makefile.am b/src/test/Makefile.am
index 546fa2f4b..d6f217658 100644
--- a/src/test/Makefile.am
+++ b/src/test/Makefile.am
@@ -27,4 +27,10 @@ test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \
 	../common/libor-event.a \
 	@TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@
 
+if USE_BUFFEREVENTS
+test_LDADD += -levent_openssl
+endif
+
 noinst_HEADERS = tinytest.h tinytest_macros.h test.h
+
+