From 770c944599597e68f0e0494b5dcc53d342ab5611 Mon Sep 17 00:00:00 2001 From: Annika Wierichs Date: Wed, 13 Dec 2017 17:18:31 +0100 Subject: [PATCH] added more conversions for host ptr version. --- kernel/ibv.c | 21 ++++++++++----------- kernel/ibv_guest_host.c | 31 +++++++++++++++++++++++++++++++ usr/tests/ib/pingpong-ud.c | 19 ++++++++++++------- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/kernel/ibv.c b/kernel/ibv.c index 049736651..9c00dd8a0 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -150,8 +150,9 @@ const char * ibv_wc_status_str(enum ibv_wc_status status) { uhyve_ibv_wc_status_str_t uhyve_args; uhyve_args.status = status; - uhyve_args.ret = kmalloc(MAX_CHAR_ARR_LENGTH); // ! + static const char ret[MAX_CHAR_ARR_LENGTH]; uhyve_send(UHYVE_PORT_IBV_WC_STATUS_STR, (unsigned) virt_to_phys((size_t) &uhyve_args)); + // memcpy here TODO return uhyve_args.ret; } @@ -936,7 +937,7 @@ int ibv_poll_cq(struct ibv_cq * cq, int num_entries, struct ibv_wc * wc) { uhyve_ibv_poll_cq_t uhyve_args; uhyve_args.cq = cq; uhyve_args.num_entries = num_entries; - uhyve_args.wc = wc; + uhyve_args.wc = (struct ibv_wc *) guest_to_host((size_t) wc); // ! uhyve_send(UHYVE_PORT_IBV_POLL_CQ, (unsigned) virt_to_phys((size_t) &uhyve_args)); @@ -1313,11 +1314,13 @@ typedef struct { int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) { uhyve_ibv_post_send_t uhyve_args; uhyve_args.qp = qp; - uhyve_args.wr = wr; - // TODO: Take care of ** parameter. + uhyve_args.wr = guest_to_host_ibv_send_wr(wr); // ! + uhyve_args.bad_wr = (struct ibv_send_wr **) guest_to_host((size_t) bad_wr); // will be written to uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args)); + // TODO: Do we want to convert bad_wr's content back to guest memory? + return uhyve_args.ret; } @@ -1340,14 +1343,10 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w uhyve_ibv_post_recv_t uhyve_args; uhyve_args.qp = qp; - /* LOG_INFO("KERNEL: ibv_post_recv() - guest_to_host_ibv_recv_wr\n"); */ uhyve_args.wr = guest_to_host_ibv_recv_wr(wr); - /* LOG_INFO("KERNEL: ibv_post_recv() - \n"); */ - uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); + uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); // will be written to - /* LOG_INFO("KERNEL: ibv_post_recv()\n"); */ uhyve_send(UHYVE_PORT_IBV_POST_RECV, (unsigned) virt_to_phys((size_t) &uhyve_args)); - /* LOG_INFO("KERNEL: ibv_post_recv()\n"); */ /* host_to_guest_ibv_recv_wr(wr, GUEST); */ // TODO: add this back in /* LOG_INFO("KERNEL: ibv_post_recv()\n"); */ @@ -1372,8 +1371,8 @@ typedef struct { struct ibv_ah * ibv_create_ah(struct ibv_pd * pd, struct ibv_ah_attr * attr) { uhyve_ibv_create_ah_t uhyve_args; - uhyve_args.pd = pd; - uhyve_args.attr = attr; + uhyve_args.pd = pd; + uhyve_args.attr = (struct ibv_ah_attr *) guest_to_host((size_t) attr); // ! uhyve_send(UHYVE_PORT_IBV_CREATE_AH, (unsigned) virt_to_phys((size_t) &uhyve_args)); diff --git a/kernel/ibv_guest_host.c b/kernel/ibv_guest_host.c index 4f31930c5..de0dfa3b8 100644 --- a/kernel/ibv_guest_host.c +++ b/kernel/ibv_guest_host.c @@ -158,6 +158,37 @@ struct ibv_recv_wr * host_to_guest_ibv_recv_wr(struct ibv_recv_wr * wr, addr_typ LOG_INFO("\twr->next is not NULL\n"); vaddr->next = host_to_guest_ibv_recv_wr(vaddr->next, HOST); // linked list } + // TODO: make this work with an actual list > 1 element. + vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST); + + return vaddr; +} + + +struct ibv_send_wr * guest_to_host_ibv_send_wr(struct ibv_send_wr * wr) { + LOG_INFO("Entered guest_to_host_ibv_send_wr()\n"); + if (wr->next) { + LOG_INFO("\twr->next is not NULL\n"); + wr->next = guest_to_host_ibv_send_wr(wr->next); // linked list + } + // TODO: make this work with an actual list > 1 element. (recv_wr as well) + /* for (int i = 0; i < wr->num_sge; i++) { */ + /* todo */ + /* } */ + wr->sg_list = guest_to_host_ibv_sge(wr->sg_list); + + return (struct ibv_send_wr *) guest_to_host((size_t) wr); +} + +struct ibv_send_wr * host_to_guest_ibv_send_wr(struct ibv_send_wr * wr, addr_type type) { + LOG_INFO("Entered host_to_guest_ibv_send_wr()\n"); + struct ibv_send_wr * vaddr = (type == GUEST) ? wr + : (struct ibv_send_wr *) host_to_guest((size_t) wr); + + if (wr->next) { + LOG_INFO("\twr->next is not NULL\n"); + vaddr->next = host_to_guest_ibv_send_wr(vaddr->next, HOST); // linked list + } vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST); return vaddr; diff --git a/usr/tests/ib/pingpong-ud.c b/usr/tests/ib/pingpong-ud.c index bf1a6ed63..996e244ac 100644 --- a/usr/tests/ib/pingpong-ud.c +++ b/usr/tests/ib/pingpong-ud.c @@ -185,7 +185,7 @@ static struct pingpong_dest *pp_client_exch_dest(const char *servername, int por gid_to_wire_gid(&my_dest->gid, gid); sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); + my_dest->psn, gid); if (write(sockfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); goto out; @@ -291,8 +291,7 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, wire_gid_to_gid(gid, &rem_dest->gid); printf("\tBefore pp_connect_ctx().\n"); - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, - sgid_idx)) { + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { fprintf(stderr, "Couldn't connect to remote QP\n"); free(rem_dest); rem_dest = NULL; @@ -540,7 +539,7 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) struct ibv_sge list = { .addr = (uintptr_t) ctx->buf + 40, .length = ctx->size, - .lkey = ctx->mr->lkey + .lkey = ibv_get_mr_lkey(ctx->mr) }; struct ibv_send_wr wr = { .wr_id = PINGPONG_SEND_WRID, @@ -603,6 +602,8 @@ int main(int argc, char *argv[]) int gidx = -1; char gid[33]; + printf("optint: %d / argc: %d\n", optind, argc); + srand48(getpid() * time(NULL)); while (1) { @@ -621,8 +622,7 @@ int main(int argc, char *argv[]) {} }; - c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", - long_options, NULL); + c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL); if (c == -1) break; @@ -677,6 +677,9 @@ int main(int argc, char *argv[]) } } + /* servername = "137.226.133.156"; // ! */ + /* printf("Servername manually set.\n"); */ + if (optind == argc - 1) { servername = strdupa(argv[optind]); printf("Servername set.\n"); @@ -753,6 +756,7 @@ int main(int argc, char *argv[]) /* printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", */ /* my_dest.lid, my_dest.qpn, my_dest.psn, gid); */ + printf("if servername() 1\n"); if (servername) rem_dest = pp_client_exch_dest(servername, port, &my_dest); else @@ -761,6 +765,7 @@ int main(int argc, char *argv[]) if (!rem_dest) return 1; + // TODO: fix me /* inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); */ /* printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", */ /* rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); */ @@ -825,7 +830,7 @@ int main(int argc, char *argv[]) for (i = 0; i < ne; ++i) { if (wc[i].status != IBV_WC_SUCCESS) { fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), + ibv_wc_status_str(wc[i].status), // todo here wc[i].status, (int) wc[i].wr_id); return 1; }