Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-16 00:00:02 +01:00
Code Issues Releases Wiki Activity

Performance improvement for UDP

Browse source 
Before this commit, the Infiniband node recreated the address handle for
the remote node during every cycle. Now, it only creates it directly
after it got ah_attr.
This commit is contained in:
Dennis Potter 2018-07-25 18:51:28 +02:00
parent a92491ad30
commit 45ddebf5d1
1 changed files with 12 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
lib/nodes/infiniband.c
View file

@ -547,8 +547,10 @@ void * ib_rdma_cm_event_thread(void *n)
case RDMA_CM_EVENT_ESTABLISHED:
// If the connection is unreliable connectionless, set appropriate variables
if (ib->conn.port_space == RDMA_PS_UDP)
ib->conn.ud = event->param.ud;
if (ib->conn.port_space == RDMA_PS_UDP) {
ib->conn.ud.ud = event->param.ud;
ib->conn.ud.ah = ibv_create_ah(ib->ctx.pd, &ib->conn.ud.ud.ah_attr);
}
node->state = STATE_CONNECTED;
@ -630,8 +632,8 @@ int ib_start(struct node *n)
// Allocate space for 40 Byte GHR. We don't use this.
if (ib->conn.port_space == RDMA_PS_UDP) {
ib->conn.grh_ptr = alloc(40);
ib->conn.grh_mr = ibv_reg_mr(ib->ctx.pd, ib->conn.grh_ptr, 40, IBV_ACCESS_LOCAL_WRITE);
ib->conn.ud.grh_ptr = alloc(40);
ib->conn.ud.grh_mr = ibv_reg_mr(ib->ctx.pd, ib->conn.ud.grh_ptr, 40, IBV_ACCESS_LOCAL_WRITE);
}
// Several events should occur on the event channel, to make
@ -659,7 +661,7 @@ int ib_stop(struct node *n)
// Call RDMA disconnect function
// Will flush all outstanding WRs to the Completion Queue and
// will call RDMA_CM_EVENT_DISCONNECTED if that is done.
if (n->state == STATE_CONNECTED) {
if (n->state == STATE_CONNECTED && ib->conn.port_space == RDMA_PS_TCP) {
ret = rdma_disconnect(ib->ctx.id);
if (ret)
@ -758,9 +760,9 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *relea
// First 40 byte of UD data are GRH and unused in our case
if (ib->conn.port_space == RDMA_PS_UDP) {
sge[i][j].addr = (uint64_t) ib->conn.grh_ptr;
sge[i][j].addr = (uint64_t) ib->conn.ud.grh_ptr;
sge[i][j].length = 40;
sge[i][j].lkey = ib->conn.grh_mr->lkey;
sge[i][j].lkey = ib->conn.ud.grh_mr->lkey;
j++;
}
@ -894,9 +896,9 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele
// Check if connection is connected or unconnected and set appropriate values
if (ib->conn.port_space == RDMA_PS_UDP) {
wr[sent].wr.ud.ah = ibv_create_ah(ib->ctx.pd, &ib->conn.ud.ah_attr);
wr[sent].wr.ud.remote_qkey = ib->conn.ud.qkey;
wr[sent].wr.ud.remote_qpn = ib->conn.ud.qp_num;
wr[sent].wr.ud.ah = ib->conn.ud.ah;
wr[sent].wr.ud.remote_qkey = ib->conn.ud.ud.qkey;
wr[sent].wr.ud.remote_qpn = ib->conn.ud.ud.qp_num;
}
// Check if data can be send inline