From 770c944599597e68f0e0494b5dcc53d342ab5611 Mon Sep 17 00:00:00 2001
From: Annika Wierichs <annika.wierichs@rwth-aachen.de>
Date: Wed, 13 Dec 2017 17:18:31 +0100
Subject: [PATCH] added more conversions for host ptr version.

---
 kernel/ibv.c               | 21 ++++++++++-----------
 kernel/ibv_guest_host.c    | 31 +++++++++++++++++++++++++++++++
 usr/tests/ib/pingpong-ud.c | 19 ++++++++++++-------
 3 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/kernel/ibv.c b/kernel/ibv.c
index 049736651..9c00dd8a0 100644
--- a/kernel/ibv.c
+++ b/kernel/ibv.c
@@ -150,8 +150,9 @@ const char * ibv_wc_status_str(enum ibv_wc_status status) {
 	uhyve_ibv_wc_status_str_t uhyve_args;
 	uhyve_args.status = status;
 
-	uhyve_args.ret = kmalloc(MAX_CHAR_ARR_LENGTH); // !
+	static const char ret[MAX_CHAR_ARR_LENGTH];
 	uhyve_send(UHYVE_PORT_IBV_WC_STATUS_STR, (unsigned) virt_to_phys((size_t) &uhyve_args));
+	// memcpy here TODO
 
 	return uhyve_args.ret;
 }
@@ -936,7 +937,7 @@ int ibv_poll_cq(struct ibv_cq * cq, int num_entries, struct ibv_wc * wc) {
 	uhyve_ibv_poll_cq_t uhyve_args;
 	uhyve_args.cq = cq;
 	uhyve_args.num_entries = num_entries;
-	uhyve_args.wc = wc;
+	uhyve_args.wc = (struct ibv_wc *) guest_to_host((size_t) wc); // !
 
 	uhyve_send(UHYVE_PORT_IBV_POLL_CQ, (unsigned) virt_to_phys((size_t) &uhyve_args));
 
@@ -1313,11 +1314,13 @@ typedef struct {
 int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) {
 	uhyve_ibv_post_send_t uhyve_args;
 	uhyve_args.qp = qp;
-	uhyve_args.wr = wr;
-	// TODO: Take care of ** parameter.
+	uhyve_args.wr = guest_to_host_ibv_send_wr(wr); // !
+	uhyve_args.bad_wr = (struct ibv_send_wr **) guest_to_host((size_t) bad_wr); // will be written to
 
 	uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args));
 
+	// TODO: Do we want to convert bad_wr's content back to guest memory?
+
 	return uhyve_args.ret;
 }
 
@@ -1340,14 +1343,10 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w
 
 	uhyve_ibv_post_recv_t uhyve_args;
 	uhyve_args.qp     = qp;
-	/* LOG_INFO("KERNEL: ibv_post_recv() - guest_to_host_ibv_recv_wr\n"); */
 	uhyve_args.wr     = guest_to_host_ibv_recv_wr(wr);
-	/* LOG_INFO("KERNEL: ibv_post_recv() - \n"); */
-	uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr);
+	uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); // will be written to
 
-	/* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
 	uhyve_send(UHYVE_PORT_IBV_POST_RECV, (unsigned) virt_to_phys((size_t) &uhyve_args));
-	/* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
 
 	/* host_to_guest_ibv_recv_wr(wr, GUEST); */ // TODO: add this back in
 	/* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
@@ -1372,8 +1371,8 @@ typedef struct {
 
 struct ibv_ah * ibv_create_ah(struct ibv_pd * pd, struct ibv_ah_attr * attr) {
 	uhyve_ibv_create_ah_t uhyve_args;
-	uhyve_args.pd = pd;
-	uhyve_args.attr = attr;
+	uhyve_args.pd   = pd;
+	uhyve_args.attr = (struct ibv_ah_attr *) guest_to_host((size_t) attr); // !
 
 	uhyve_send(UHYVE_PORT_IBV_CREATE_AH, (unsigned) virt_to_phys((size_t) &uhyve_args));
 
diff --git a/kernel/ibv_guest_host.c b/kernel/ibv_guest_host.c
index 4f31930c5..de0dfa3b8 100644
--- a/kernel/ibv_guest_host.c
+++ b/kernel/ibv_guest_host.c
@@ -158,6 +158,37 @@ struct ibv_recv_wr * host_to_guest_ibv_recv_wr(struct ibv_recv_wr * wr, addr_typ
 		LOG_INFO("\twr->next is not NULL\n");
 		vaddr->next = host_to_guest_ibv_recv_wr(vaddr->next, HOST); // linked list
 	}
+	// TODO: make this work with an actual list > 1 element.
+	vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
+
+	return vaddr;
+}
+
+
+struct ibv_send_wr * guest_to_host_ibv_send_wr(struct ibv_send_wr * wr) {
+	LOG_INFO("Entered guest_to_host_ibv_send_wr()\n");
+	if (wr->next) {
+		LOG_INFO("\twr->next is not NULL\n");
+		wr->next = guest_to_host_ibv_send_wr(wr->next); // linked list
+	}
+	// TODO: make this work with an actual list > 1 element. (recv_wr as well)
+	/* for (int i = 0; i < wr->num_sge; i++) { */
+		/* todo */
+	/* } */
+	wr->sg_list = guest_to_host_ibv_sge(wr->sg_list);
+
+	return (struct ibv_send_wr *) guest_to_host((size_t) wr);
+}
+
+struct ibv_send_wr * host_to_guest_ibv_send_wr(struct ibv_send_wr * wr, addr_type type) {
+	LOG_INFO("Entered host_to_guest_ibv_send_wr()\n");
+	struct ibv_send_wr * vaddr = (type == GUEST) ? wr
+		: (struct ibv_send_wr *) host_to_guest((size_t) wr);
+
+	if (wr->next) {
+		LOG_INFO("\twr->next is not NULL\n");
+		vaddr->next = host_to_guest_ibv_send_wr(vaddr->next, HOST); // linked list
+	}
 	vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
 
 	return vaddr;
diff --git a/usr/tests/ib/pingpong-ud.c b/usr/tests/ib/pingpong-ud.c
index bf1a6ed63..996e244ac 100644
--- a/usr/tests/ib/pingpong-ud.c
+++ b/usr/tests/ib/pingpong-ud.c
@@ -185,7 +185,7 @@ static struct pingpong_dest *pp_client_exch_dest(const char *servername, int por
 
 	gid_to_wire_gid(&my_dest->gid, gid);
 	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
-							my_dest->psn, gid);
+			my_dest->psn, gid);
 	if (write(sockfd, msg, sizeof msg) != sizeof msg) {
 		fprintf(stderr, "Couldn't send local address\n");
 		goto out;
@@ -291,8 +291,7 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
 	wire_gid_to_gid(gid, &rem_dest->gid);
 
 	printf("\tBefore pp_connect_ctx().\n");
-	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
-								sgid_idx)) {
+	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
 		fprintf(stderr, "Couldn't connect to remote QP\n");
 		free(rem_dest);
 		rem_dest = NULL;
@@ -540,7 +539,7 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
 	struct ibv_sge list = {
 		.addr	= (uintptr_t) ctx->buf + 40,
 		.length = ctx->size,
-		.lkey	= ctx->mr->lkey
+		.lkey	= ibv_get_mr_lkey(ctx->mr)
 	};
 	struct ibv_send_wr wr = {
 		.wr_id	    = PINGPONG_SEND_WRID,
@@ -603,6 +602,8 @@ int main(int argc, char *argv[])
 	int			 gidx = -1;
 	char			 gid[33];
 
+	printf("optint: %d / argc: %d\n", optind, argc);
+
 	srand48(getpid() * time(NULL));
 
 	while (1) {
@@ -621,8 +622,7 @@ int main(int argc, char *argv[])
 			{}
 		};
 
-		c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
-							long_options, NULL);
+		c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL);
 		if (c == -1)
 			break;
 
@@ -677,6 +677,9 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* servername = "137.226.133.156"; // ! */
+	/* printf("Servername manually set.\n"); */
+
 	if (optind == argc - 1) {
 		servername = strdupa(argv[optind]);
 		printf("Servername set.\n");
@@ -753,6 +756,7 @@ int main(int argc, char *argv[])
 	/* printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", */
 				 /* my_dest.lid, my_dest.qpn, my_dest.psn, gid); */
 
+	printf("if servername() 1\n");
 	if (servername)
 		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 	else
@@ -761,6 +765,7 @@ int main(int argc, char *argv[])
 	if (!rem_dest)
 		return 1;
 
+	// TODO: fix me
 	/* inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); */
 	/* printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", */
 				 /* rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); */
@@ -825,7 +830,7 @@ int main(int argc, char *argv[])
 			for (i = 0; i < ne; ++i) {
 				if (wc[i].status != IBV_WC_SUCCESS) {
 					fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
-						ibv_wc_status_str(wc[i].status),
+						ibv_wc_status_str(wc[i].status), // todo here
 						wc[i].status, (int) wc[i].wr_id);
 					return 1;
 				}