patch: use BIO with tcp_send in TLS code, flush send-queue on tcp socket close

include/re_tcp.h

@@ -102,3 +102,5 @@ int tcp_register_helper(struct tcp_helper **thp, struct tcp_conn *tc,
 			int layer,
 			tcp_helper_estab_h *eh, tcp_helper_send_h *sh,
 			tcp_helper_recv_h *rh, void *arg);
+int tcp_send_helper(struct tcp_conn *tc, struct mbuf *mb,
+		    struct tcp_helper *th);

src/tcp/tcp.c

@@ -247,9 +247,14 @@ static int dequeue(struct tcp_conn *tc)
 
 static void conn_close(struct tcp_conn *tc, int err)
 {
+	list_flush(&tc->sendq);
+
 	/* Stop polling */
-	if (tc->fdc >= 0)
+	if (tc->fdc >= 0) {
 		fd_close(tc->fdc);
+		(void)close(tc->fdc);
+		tc->fdc = -1;
+	}
 
 	if (tc->closeh)
 		tc->closeh(err, tc->arg);
@@ -316,6 +321,8 @@ static void tcp_recv_handler(int flags, void *arg)
 			return;
 		}
 
+		tc->connected = true;
+
 		err = fd_listen(tc->fdc, FD_READ, tcp_recv_handler, tc);
 		if (err) {
 			DEBUG_WARNING("recv handler: fd_listen(): %s\n",
@@ -340,7 +347,6 @@ static void tcp_recv_handler(int flags, void *arg)
 		if (tc->estabh)
 			tc->estabh(tc->arg);
 
-		tc->connected = true;
 		return;
 	}
 
@@ -1012,17 +1018,9 @@ int tcp_conn_connect(struct tcp_conn *tc, const struct sa *peer)
 }
 
 
-/**
- * Send data on a TCP Connection to a remote peer
- *
- * @param tc TCP Connection
- * @param mb Buffer to send
- *
- * @return 0 if success, otherwise errorcode
- */
-int tcp_send(struct tcp_conn *tc, struct mbuf *mb)
+static int tcp_send_internal(struct tcp_conn *tc, struct mbuf *mb,
+			     struct le *le)
 {
-	struct le *le;
 	int err = 0;
 	ssize_t n;
 #ifdef MSG_NOSIGNAL
@@ -1031,8 +1029,8 @@ int tcp_send(struct tcp_conn *tc, struct mbuf *mb)
 	const int flags = 0;
 #endif
 
-	if (!tc || !mb)
-		return EINVAL;
+	if (tc->fdc < 0)
+		return ENOTCONN;
 
 	if (!mbuf_get_left(mb)) {
 		DEBUG_WARNING("send: empty mbuf (pos=%u end=%u)\n",
@@ -1041,7 +1039,6 @@ int tcp_send(struct tcp_conn *tc, struct mbuf *mb)
 	}
 
 	/* call helpers in reverse order */
-	le = tc->helpers.tail;
 	while (le) {
 		struct tcp_helper *th = le->data;
 
@@ -1083,6 +1080,43 @@ int tcp_send(struct tcp_conn *tc, struct mbuf *mb)
 }
 
 
+/**
+ * Send data on a TCP Connection to a remote peer
+ *
+ * @param tc TCP Connection
+ * @param mb Buffer to send
+ *
+ * @return 0 if success, otherwise errorcode
+ */
+int tcp_send(struct tcp_conn *tc, struct mbuf *mb)
+{
+	if (!tc || !mb)
+		return EINVAL;
+
+	return tcp_send_internal(tc, mb, tc->helpers.tail);
+}
+
+
+/**
+ * Send data on a TCP Connection to a remote peer bypassing this
+ * helper and the helpers above it.
+ *
+ * @param tc TCP Connection
+ * @param mb Buffer to send
+ * @param th TCP Helper
+ *
+ * @return 0 if success, otherwise errorcode
+ */
+int tcp_send_helper(struct tcp_conn *tc, struct mbuf *mb,
+		    struct tcp_helper *th)
+{
+	if (!tc || !mb || !th)
+		return EINVAL;
+
+	return tcp_send_internal(tc, mb, th->le.prev);
+}
+
+
 /**
  * Set the send handler on a TCP Connection, which will be called
  * every time it is ready to send data

src/tls/openssl/tls_tcp.c

@@ -28,7 +28,7 @@ struct tls_conn {
 	BIO *sbio_out;
 	BIO *sbio_in;
 	struct tcp_helper *th;
-	struct tcpconn *tcp;
+	struct tcp_conn *tcp;
 	bool active;
 	bool up;
 };
@@ -47,6 +47,77 @@ static void destructor(void *arg)
 }
 
 
+static int bio_create(BIO *b)
+{
+	b->init  = 1;
+	b->num   = 0;
+	b->ptr   = NULL;
+	b->flags = 0;
+
+	return 1;
+}
+
+
+static int bio_destroy(BIO *b)
+{
+	if (!b)
+		return 0;
+
+	b->ptr   = NULL;
+	b->init  = 0;
+	b->flags = 0;
+
+	return 1;
+}
+
+
+static int bio_write(BIO *b, const char *buf, int len)
+{
+	struct tls_conn *tc = b->ptr;
+	struct mbuf mb;
+	int err;
+
+	mb.buf = (void *)buf;
+	mb.pos = 0;
+	mb.end = mb.size = len;
+
+	err = tcp_send_helper(tc->tcp, &mb, tc->th);
+	if (err)
+		return -1;
+
+	return len;
+}
+
+
+static long bio_ctrl(BIO *b, int cmd, long num, void *ptr)
+{
+	(void)b;
+	(void)num;
+	(void)ptr;
+
+	if (cmd == BIO_CTRL_FLUSH) {
+		/* The OpenSSL library needs this */
+		return 1;
+	}
+
+	return 0;
+}
+
+
+static struct bio_method_st bio_tcp_send = {
+	BIO_TYPE_SOURCE_SINK,
+	"tcp_send",
+	bio_write,
+	0,
+	0,
+	0,
+	bio_ctrl,
+	bio_create,
+	bio_destroy,
+	0
+};
+
+
 static int tls_connect(struct tls_conn *tc)
 {
 	int err = 0, r;
@@ -245,13 +316,15 @@ int tls_start_tcp(struct tls_conn **ptc, struct tls *tls, struct tcp_conn *tcp)
 		goto out;
 	}
 
-	tc->sbio_out = BIO_new_socket(tcp_conn_fd(tcp), BIO_NOCLOSE);
+	tc->sbio_out = BIO_new(&bio_tcp_send);
 	if (!tc->sbio_out) {
 		DEBUG_WARNING("alloc: BIO_new_socket() failed\n");
 		BIO_free(tc->sbio_in);
 		goto out;
 	}
 
+	tc->sbio_out->ptr = tc;
+
 	SSL_set_bio(tc->ssl, tc->sbio_in, tc->sbio_out);
 
 	err = 0;