1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00

local pingpong working, need to fix for rx_depth > 1

This commit is contained in:
Annika Wierichs 2017-12-19 23:00:03 +01:00
parent c86a846fdd
commit f929f91a6f
4 changed files with 73 additions and 70 deletions

View file

@ -64,6 +64,7 @@ struct ibv_device ** ibv_get_device_list(int * num_devices) { // !
uhyve_args.ret = (struct ibv_device **) guest_to_host((size_t) list);
uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) virt_to_phys((size_t) &uhyve_args));
return list;
}
@ -401,7 +402,7 @@ typedef struct {
int ibv_query_device(struct ibv_context * context, struct ibv_device_attr * device_attr) {
uhyve_ibv_query_device_t uhyve_args;
uhyve_args.context = context;
uhyve_args.device_attr = device_attr;
uhyve_args.device_attr = (struct ibv_device_attr *) guest_to_host((size_t) device_attr); // !
uhyve_send(UHYVE_PORT_IBV_QUERY_DEVICE, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -428,7 +429,7 @@ int ibv_query_gid(struct ibv_context * context, uint8_t port_num, int index, uni
uhyve_args.context = context;
uhyve_args.port_num = port_num;
uhyve_args.index = index;
uhyve_args.gid = gid;
uhyve_args.gid = (union ibv_gid *) guest_to_host((size_t) gid); // !
uhyve_send(UHYVE_PORT_IBV_QUERY_GID, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -455,7 +456,7 @@ int ibv_query_pkey(struct ibv_context * context, uint8_t port_num, int index, __
uhyve_args.context = context;
uhyve_args.port_num = port_num;
uhyve_args.index = index;
uhyve_args.pkey = pkey;
uhyve_args.pkey = (__be16 *) guest_to_host((size_t) pkey); // !
uhyve_send(UHYVE_PORT_IBV_QUERY_PKEY, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -821,10 +822,11 @@ typedef struct {
struct ibv_cq * ibv_create_cq(struct ibv_context * context, int cqe, void * cq_context, struct ibv_comp_channel * channel, int comp_vector) {
uhyve_ibv_create_cq_t uhyve_args;
uhyve_args.context = context;
uhyve_args.cqe = cqe;
uhyve_args.cq_context = cq_context;
uhyve_args.channel = channel;
uhyve_args.context = context;
uhyve_args.cqe = cqe;
uhyve_args.cq_context = cq_context;
uhyve_args.channel = channel;
uhyve_args.comp_vector = comp_vector;
uhyve_send(UHYVE_PORT_IBV_CREATE_CQ, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -936,9 +938,10 @@ typedef struct {
int ibv_poll_cq(struct ibv_cq * cq, int num_entries, struct ibv_wc * wc) {
uhyve_ibv_poll_cq_t uhyve_args;
uhyve_args.cq = cq;
uhyve_args.cq = cq;
uhyve_args.num_entries = num_entries;
uhyve_args.wc = (struct ibv_wc *) guest_to_host((size_t) wc); // !
uhyve_args.wc = (struct ibv_wc *) guest_to_host((size_t) wc); // !
uhyve_send(UHYVE_PORT_IBV_POLL_CQ, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -1267,6 +1270,7 @@ typedef struct {
int ibv_query_qp(struct ibv_qp * qp, struct ibv_qp_attr * attr, int attr_mask, struct ibv_qp_init_attr * init_attr) {
uhyve_ibv_query_qp_t uhyve_args;
uhyve_args.qp = qp;
uhyve_args.attr = (struct ibv_qp_attr *) guest_to_host((size_t) attr); // !
uhyve_args.attr_mask = attr_mask;
@ -1341,8 +1345,8 @@ typedef struct {
int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_wr ** bad_wr) {
/* LOG_INFO("KERNEL: ibv_post_recv()\n"); */
uhyve_ibv_post_recv_t uhyve_args;
uhyve_args.qp = qp;
uhyve_args.wr = guest_to_host_ibv_recv_wr(wr);
uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); // will be written to

View file

@ -29,6 +29,7 @@
*/
#include <asm/page.h>
#include <hermit/ibv_guest_host.h>
@ -102,16 +103,16 @@ struct ibv_port_attr * host_to_guest_ibv_port_attr(struct ibv_port_attr * port_a
*/
struct ibv_sge * guest_to_host_ibv_sge(struct ibv_sge * sg) {
LOG_INFO("Entered guest_to_host_ibv_recv_wr()\n");
LOG_INFO("\tsg->addr before conversion: %lu\n", sg->addr);
/* LOG_INFO("Entered guest_to_host_ibv_recv_wr()\n"); */
/* LOG_INFO("\tsg->addr before conversion: %lu\n", sg->addr); */
sg->addr = (uint64_t) guest_to_host((size_t) sg->addr);
LOG_INFO("\tsg->addr after conversion: %lu\n", sg->addr);
/* LOG_INFO("\tsg->addr after conversion: %lu\n", sg->addr); */
return (struct ibv_sge *) guest_to_host((size_t) sg);
}
struct ibv_sge * host_to_guest_ibv_sge(struct ibv_sge * sg, addr_type type) {
LOG_INFO("Entered host_to_guest_ibv_recv_wr()\n");
/* LOG_INFO("Entered host_to_guest_ibv_recv_wr()\n"); */
struct ibv_sge * vaddr = (type == GUEST) ? sg
: (struct ibv_sge *) host_to_guest((size_t) sg);
@ -120,7 +121,6 @@ struct ibv_sge * host_to_guest_ibv_sge(struct ibv_sge * sg, addr_type type) {
return vaddr;
}
// struct ibv_recv_wr {
// uint64_t wr_id;
// struct ibv_recv_wr *next;
@ -134,30 +134,41 @@ struct ibv_sge * host_to_guest_ibv_sge(struct ibv_sge * sg, addr_type type) {
// uint32_t lkey;
// };
/*
* struct ibv_recv_wr
*/
struct ibv_recv_wr * guest_to_host_ibv_recv_wr(struct ibv_recv_wr * wr) {
LOG_INFO("Entered guest_to_host_ibv_recv_wr()\n");
/* LOG_INFO("Entered guest_to_host_ibv_recv_wr()\n"); */
if (wr->next) {
LOG_INFO("\twr->next is not NULL\n");
wr->next = guest_to_host_ibv_recv_wr(wr->next); // linked list
/* LOG_INFO("\twr->next is not NULL\n"); */
wr->next = guest_to_host_ibv_recv_wr(wr->next); // linked list
}
wr->sg_list = guest_to_host_ibv_sge(wr->sg_list);
for (int i = 0; i < wr->num_sge; i++) {
guest_to_host_ibv_sge(wr->sg_list + i);
}
wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) wr->sg_list);
return (struct ibv_recv_wr *) guest_to_host((size_t) wr);
}
struct ibv_recv_wr * host_to_guest_ibv_recv_wr(struct ibv_recv_wr * wr, addr_type type) {
LOG_INFO("Entered host_to_guest_ibv_recv_wr()\n");
/* LOG_INFO("Entered host_to_guest_ibv_recv_wr()\n"); */
struct ibv_recv_wr * vaddr = (type == GUEST) ? wr
: (struct ibv_recv_wr *) host_to_guest((size_t) wr);
if (wr->next) {
LOG_INFO("\twr->next is not NULL\n");
if (vaddr->next) {
/* LOG_INFO("\twr->next is not NULL\n"); */
vaddr->next = host_to_guest_ibv_recv_wr(vaddr->next, HOST); // linked list
}
vaddr->sg_list = (struct ibv_sge *) host_to_guest((size_t) vaddr->sg_list);
for (int i = 0; i < vaddr->num_sge; i++) {
host_to_guest_ibv_sge(vaddr->sg_list + i, GUEST);
}
// TODO: make this work with an actual list > 1 element.
vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
@ -165,31 +176,39 @@ struct ibv_recv_wr * host_to_guest_ibv_recv_wr(struct ibv_recv_wr * wr, addr_typ
}
/*
* struct ibv_send_wr
*/
struct ibv_send_wr * guest_to_host_ibv_send_wr(struct ibv_send_wr * wr) {
LOG_INFO("Entered guest_to_host_ibv_send_wr()\n");
/* LOG_INFO("Entered guest_to_host_ibv_send_wr()\n"); */
if (wr->next) {
LOG_INFO("\twr->next is not NULL\n");
/* LOG_INFO("\twr->next is not NULL\n"); */
wr->next = guest_to_host_ibv_send_wr(wr->next); // linked list
}
// TODO: make this work with an actual list > 1 element. (recv_wr as well)
/* for (int i = 0; i < wr->num_sge; i++) { */
/* todo */
/* } */
wr->sg_list = guest_to_host_ibv_sge(wr->sg_list);
for (int i = 0; i < wr->num_sge; i++) {
guest_to_host_ibv_sge(wr->sg_list + i);
}
wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) wr->sg_list);
return (struct ibv_send_wr *) guest_to_host((size_t) wr);
}
struct ibv_send_wr * host_to_guest_ibv_send_wr(struct ibv_send_wr * wr, addr_type type) {
LOG_INFO("Entered host_to_guest_ibv_send_wr()\n");
/* LOG_INFO("Entered host_to_guest_ibv_send_wr()\n"); */
struct ibv_send_wr * vaddr = (type == GUEST) ? wr
: (struct ibv_send_wr *) host_to_guest((size_t) wr);
if (wr->next) {
LOG_INFO("\twr->next is not NULL\n");
/* LOG_INFO("\twr->next is not NULL\n"); */
vaddr->next = host_to_guest_ibv_send_wr(vaddr->next, HOST); // linked list
}
vaddr->sg_list = host_to_guest_ibv_sge(vaddr->sg_list, HOST);
vaddr->sg_list = (struct ibv_sge *) host_to_guest((size_t) vaddr->sg_list);
for (int i = 0; i < vaddr->num_sge; i++) {
host_to_guest_ibv_sge(vaddr->sg_list + i, GUEST);
}
return vaddr;
}

View file

@ -354,7 +354,7 @@ void call_ibv_ack_cq_events(struct kvm_run * run, uint8_t * guest_mem) {
}
void call_ibv_poll_cq(struct kvm_run * run, uint8_t * guest_mem) {
printf("UHYVE call: call_ibv_poll_cq\n");
/* printf("UHYVE call: call_ibv_poll_cq\n"); */
unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));
uhyve_ibv_poll_cq_t * args = (uhyve_ibv_poll_cq_t *) (guest_mem + data);
@ -482,7 +482,7 @@ void call_ibv_destroy_qp(struct kvm_run * run, uint8_t * guest_mem) {
}
void call_ibv_post_send(struct kvm_run * run, uint8_t * guest_mem) {
printf("UHYVE call: call_ibv_post_send\n");
/* printf("UHYVE call: call_ibv_post_send\n"); */
unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));
uhyve_ibv_post_send_t * args = (uhyve_ibv_post_send_t *) (guest_mem + data);
@ -490,7 +490,7 @@ void call_ibv_post_send(struct kvm_run * run, uint8_t * guest_mem) {
}
void call_ibv_post_recv(struct kvm_run * run, uint8_t * guest_mem) {
printf("UHYVE call: call_ibv_post_recv\n");
/* printf("UHYVE call: call_ibv_post_recv\n"); */
unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));
uhyve_ibv_post_recv_t * args = (uhyve_ibv_post_recv_t *) (guest_mem + data);
@ -590,7 +590,7 @@ void call_ibv_is_qpt_supported(struct kvm_run * run, uint8_t * guest_mem) {
}
void call_ibv_get_mr_lkey(struct kvm_run * run, uint8_t * guest_mem) {
printf("UHYVE call: call_ibv_get_mr_lkey\n");
/* printf("UHYVE call: call_ibv_get_mr_lkey\n"); */
unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));
uhyve_ibv_get_mr_lkey_t * args = (uhyve_ibv_get_mr_lkey_t *) (guest_mem + data);

View file

@ -158,41 +158,30 @@ static struct pingpong_dest *pp_client_exch_dest(const char *servername, int por
return NULL;
n = getaddrinfo(servername, service, &hints, &res);
printf(" after getaddrinfo().\n");
printf(" .res->ai_addr->sa_data (the address): %s\n", res->ai_addr->sa_data);
if (n < 0) {
fprintf(stderr, "error for %s:%d\n", servername, port);
free(service);
return NULL;
}
printf(" after if(n < 0) {.\n");
for (t = res; t; t = t->ai_next) {
printf(" \twithin for loop.\n");
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
if (sockfd >= 0) {
printf(" \t\tsockfd >= 0.\n");
if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
printf(" \t\t!connect.\n");
break;
printf(" \t\tbefore close.\n");
close(sockfd);
printf(" \t\tafter close.\n");
sockfd = -1;
}
}
printf(" after for loop.\n");
freeaddrinfo(res);
free(service);
printf(" after free(service).\n");
if (sockfd < 0) {
fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
return NULL;
}
printf(" after if (sockfd < 0).\n");
gid_to_wire_gid(&my_dest->gid, gid);
sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
@ -276,9 +265,9 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
}
listen(sockfd, 1);
printf("\tAfter listen.\n");
printf("\tServer listening.\n");
connfd = accept(sockfd, NULL, NULL);
printf("\tBefore close.\n");
printf("\tAccepted connection.\n");
close(sockfd);
if (connfd < 0) {
fprintf(stderr, "accept() failed\n");
@ -292,7 +281,6 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
goto out;
}
printf("\tBefore malloc.\n");
rem_dest = malloc(sizeof *rem_dest);
if (!rem_dest)
goto out;
@ -301,7 +289,6 @@ static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
&rem_dest->psn, gid);
wire_gid_to_gid(gid, &rem_dest->gid);
printf("\tBefore pp_connect_ctx().\n");
if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
@ -519,7 +506,7 @@ static int pp_close_ctx(struct pingpong_context *ctx)
static int pp_post_recv(struct pingpong_context *ctx, int n)
{
printf("Entered pp_post_recv().\n");
/* printf("Entered pp_post_recv().\n"); */
struct ibv_sge list = {
.addr = (uintptr_t) ctx->buf,
.length = ctx->size + 40,
@ -535,18 +522,16 @@ static int pp_post_recv(struct pingpong_context *ctx, int n)
int i;
for (i = 0; i < n; ++i) {
printf("loop.\n");
if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
break;
}
return i;
printf("Finished pp_post_recv().\n");
}
static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
{
printf("Entered pp_post_send().\n");
/* printf("Entered pp_post_send().\n"); */
struct ibv_sge list = {
.addr = (uintptr_t) ctx->buf + 40,
.length = ctx->size,
@ -569,7 +554,6 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
struct ibv_send_wr *bad_wr;
return ibv_post_send(ctx->qp, &wr, &bad_wr);
printf("Finished pp_post_send().\n");
}
static void usage(const char *argv0)
@ -603,7 +587,7 @@ int main(int argc, char *argv[])
unsigned int port = 18515;
int ib_port = 1;
unsigned int size = 2048;
unsigned int rx_depth = 1; // TODO: set back to 500
unsigned int rx_depth = 500;
unsigned int iters = 1000;
int use_event = 0;
int routs;
@ -613,8 +597,6 @@ int main(int argc, char *argv[])
int gidx = -1;
char gid[33];
printf("optint: %d / argc: %d\n", optind, argc);
srand48(getpid() * time(NULL));
while (1) {
@ -747,27 +729,25 @@ int main(int argc, char *argv[])
fprintf(stderr, "Couldn't get port info\n");
return 1;
}
/* printf("After pp_get_port_info(), containing ibv_query_port()\n"); */
my_dest.lid = ctx->portinfo.lid;
my_dest.qpn = ibv_get_qp_num(ctx->qp);
my_dest.psn = lrand48() & 0xffffff;
my_dest.psn = lrand48() & 0xffffff; // packet sequence number
if (gidx >= 0) {
if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
fprintf(stderr, "Could not get local gid for gid index "
"%d\n", gidx);
fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
return 1;
}
} else
} else {
memset(&my_dest.gid, 0, sizeof my_dest.gid);
}
// TODO: fix me
/* inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); */
/* printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", */
/* my_dest.lid, my_dest.qpn, my_dest.psn, gid); */
printf("if servername() 1\n");
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
@ -781,14 +761,12 @@ int main(int argc, char *argv[])
/* printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", */
/* rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); */
printf("if servername() 2\n");
if (servername)
if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx))
return 1;
ctx->pending = PINGPONG_RECV_WRID;
printf("if servername() 3\n");
if (servername) {
if (pp_post_send(ctx, rem_dest->qpn)) {
fprintf(stderr, "Couldn't post send\n");
@ -802,6 +780,7 @@ int main(int argc, char *argv[])
return 1;
}
printf("Entering while loop of post_send & post_recv.\n");
rcnt = scnt = 0;
while (rcnt < iters || scnt < iters) {
if (use_event) {
@ -815,7 +794,7 @@ int main(int argc, char *argv[])
++num_cq_events;
if (ev_cq != ctx->cq) {
if (ev_cq != ctx->cq) { // both host space pointers
fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
return 1;
}
@ -827,11 +806,12 @@ int main(int argc, char *argv[])
}
{
struct ibv_wc wc[2];
int num_wc = 2;
struct ibv_wc wc[num_wc];
int ne, i;
do {
ne = ibv_poll_cq(ctx->cq, 2, wc);
ne = ibv_poll_cq(ctx->cq, num_wc, wc);
if (ne < 0) {
fprintf(stderr, "poll CQ failed %d\n", ne);
return 1;