From 770c944599597e68f0e0494b5dcc53d342ab5611 Mon Sep 17 00:00:00 2001
From: Annika Wierichs <annika.wierichs@rwth-aachen.de>
Date: Wed, 13 Dec 2017 17:18:31 +0100
Subject: [PATCH] added more conversions for host ptr version.
---
kernel/ibv.c | 21 ++++++++++-----------
kernel/ibv_guest_host.c | 31 +++++++++++++++++++++++++++++++
usr/tests/ib/pingpong-ud.c | 19 ++++++++++++-------
3 files changed, 53 insertions(+), 18 deletions(-)
diff --git a/kernel/ibv.c b/kernel/ibv.c
index 049736651..9c00dd8a0 100644
--- a/kernel/ibv.c
+++ b/kernel/ibv.c
@@ -150,8 +150,9 @@ const char * ibv_wc_status_str(enum ibv_wc_status status) {
uhyve_ibv_wc_status_str_t uhyve_args;
uhyve_args.status = status;
- uhyve_args.ret = kmalloc(MAX_CHAR_ARR_LENGTH); // !
+ static const char ret[MAX_CHAR_ARR_LENGTH];
uhyve_send(UHYVE_PORT_IBV_WC_STATUS_STR, (unsigned) virt_to_phys((size_t) &uhyve_args));
+ // memcpy here TODO
return uhyve_args.ret;
}
@@ -936,7 +937,7 @@ int ibv_poll_cq(struct ibv_cq * cq, int num_entries, struct ibv_wc * wc) {
uhyve_ibv_poll_cq_t uhyve_args;
uhyve_args.cq = cq;
uhyve_args.num_entries = num_entries;
- uhyve_args.wc = wc;
+ uhyve_args.wc = (struct ibv_wc *) guest_to_host((size_t) wc); // !
uhyve_send(UHYVE_PORT_IBV_POLL_CQ, (unsigned) virt_to_phys((size_t) &uhyve_args));
@@ -1313,11 +1314,13 @@ typedef struct {
int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) {
uhyve_ibv_post_send_t uhyve_args;
uhyve_args.qp = qp;
- uhyve_args.wr = wr;
- // TODO: Take care of ** parameter.
+ uhyve_args.wr = guest_to_host_ibv_send_wr(wr); // !
+ uhyve_args.bad_wr = (struct ibv_send_wr **) guest_to_host((size_t) bad_wr); // will be written to
uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args));
+ // TODO: Do we want to convert bad_wr's content back to guest memory?
+
return uhyve_args.ret;
}
@@ -1340,14 +1343,10 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w
uhyve_ibv_post_recv_t uhyve_args;
uhyve_args.qp = qp;
- /* LOG_INFO("KERNEL: ibv_post_recv() - guest_to_host_ibv_recv_wr\n"); */
uhyve_args.wr = guest_to_host_ibv_recv_wr(wr);
- /* LOG_INFO("KERNEL: ibv_post_recv() - \n"); */
- uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr);
+ uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); // will be written to
- /* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
uhyve_send(UHYVE_PORT_IBV_POST_RECV, (unsigned) virt_to_phys((size_t) &uhyve_args));
- /* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
/* host_to_guest_ibv_recv_wr(wr, GUEST); */ // TODO: add this back in
/* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
@@ -1372,8 +1371,8 @@ typedef struct {
struct ibv_ah * ibv_create_ah(struct ibv_pd * pd, struct ibv_ah_attr * attr) {
uhyve_ibv_create_ah_t uhyve_args;
- uhyve_args.pd = pd;
- uhyve_args.attr = attr;
+ uhyve_args.pd = pd;
+ uhyve_args.attr = (struct ibv_ah_attr *) guest_to_host((size_t) attr); // !
uhyve_send(UHYVE_PORT_IBV_CREATE_AH, (unsigned) virt_to_phys((size_t) &uhyve_args));
diff --git a/kernel/ibv_guest_host.c b/kernel/ibv_guest_host.c
index 4f31930c5..de0dfa3b8 100644
--- a/kernel/ibv_guest_host.c
+++ b/kernel/ibv_guest_host.c
@@ -158,6 +158,37 @@ struct ibv_recv_wr * host_to_guest_ibv_recv_wr(struct ibv_recv_wr * wr, addr_typ
LOG_INFO("\twr->next is not NULL\n");
vaddr->next = host_to_guest_ibv_recv_wr(vaddr->next, HOST); // linked list
}
+ // TODO: make this work with an actual list > 1 element.
+ vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
+
+ return vaddr;
+}
+
+
+struct ibv_send_wr * guest_to_host_ibv_send_wr(struct ibv_send_wr * wr) {
+ LOG_INFO("Entered guest_to_host_ibv_send_wr()\n");
+ if (wr->next) {
+ LOG_INFO("\twr->next is not NULL\n");
+ wr->next = guest_to_host_ibv_send_wr(wr->next); // linked list
+ }
+ // TODO: make this work with an actual list > 1 element. (recv_wr as well)
+ /* for (int i = 0; i < wr->num_sge; i++) { */
+ /* todo */
+ /* } */
+ wr->sg_list = guest_to_host_ibv_sge(wr->sg_list);
+
+ return (struct ibv_send_wr *) guest_to_host((size_t) wr);
+}
+
+struct ibv_send_wr * host_to_guest_ibv_send_wr(struct ibv_send_wr * wr, addr_type type) {
+ LOG_INFO("Entered host_to_guest_ibv_send_wr()\n");
+ struct ibv_send_wr * vaddr = (type == GUEST) ? wr
+ : (struct ibv_send_wr *) host_to_guest((size_t) wr);
+
+ if (wr->next) {
+ LOG_INFO("\twr->next is not NULL\n");
+ vaddr->next = host_to_guest_ibv_send_wr(vaddr->next, HOST); // linked list
+ }
vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
return vaddr;
diff --git a/usr/tests/ib/pingpong-ud.c b/usr/tests/ib/pingpong-ud.c
index bf1a6ed63..996e244ac 100644
--- a/usr/tests/ib/pingpong-ud.c
+++ b/usr/tests/ib/pingpong-ud.c
@@ -185,7 +185,7 @@ static struct pingpong_dest *pp_client_exch_dest(const char *servername, int por
gid_to_wire_gid(&my_dest->gid, gid);
sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
- my_dest->psn, gid);
+ my_dest->psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
@@ -291,8 +291,7 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
wire_gid_to_gid(gid, &rem_dest->gid);
printf("\tBefore pp_connect_ctx().\n");
- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
- sgid_idx)) {
+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
@@ -540,7 +539,7 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
struct ibv_sge list = {
.addr = (uintptr_t) ctx->buf + 40,
.length = ctx->size,
- .lkey = ctx->mr->lkey
+ .lkey = ibv_get_mr_lkey(ctx->mr)
};
struct ibv_send_wr wr = {
.wr_id = PINGPONG_SEND_WRID,
@@ -603,6 +602,8 @@ int main(int argc, char *argv[])
int gidx = -1;
char gid[33];
+ printf("optint: %d / argc: %d\n", optind, argc);
+
srand48(getpid() * time(NULL));
while (1) {
@@ -621,8 +622,7 @@ int main(int argc, char *argv[])
{}
};
- c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
- long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL);
if (c == -1)
break;
@@ -677,6 +677,9 @@ int main(int argc, char *argv[])
}
}
+ /* servername = "137.226.133.156"; // ! */
+ /* printf("Servername manually set.\n"); */
+
if (optind == argc - 1) {
servername = strdupa(argv[optind]);
printf("Servername set.\n");
@@ -753,6 +756,7 @@ int main(int argc, char *argv[])
/* printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", */
/* my_dest.lid, my_dest.qpn, my_dest.psn, gid); */
+ printf("if servername() 1\n");
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
@@ -761,6 +765,7 @@ int main(int argc, char *argv[])
if (!rem_dest)
return 1;
+ // TODO: fix me
/* inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); */
/* printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", */
/* rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); */
@@ -825,7 +830,7 @@ int main(int argc, char *argv[])
for (i = 0; i < ne; ++i) {
if (wc[i].status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
- ibv_wc_status_str(wc[i].status),
+ ibv_wc_status_str(wc[i].status), // todo here
wc[i].status, (int) wc[i].wr_id);
return 1;
}