Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-16 00:00:02 +01:00
Code Issues Releases Wiki Activity

Started to implement memory managemen and registration. Not yet working and in an early stage. Still some debugging and testing printfs

Browse source
This commit is contained in:
Dennis Potter 2018-06-24 13:02:04 +02:00
parent 4f6c2543b5
commit 077355d8ba
2 changed files with 91 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
include/villas/nodes/infiniband.h
View file

@ -44,6 +44,15 @@ enum poll_mode_e
BUSY
};
struct payload_s {
int data;
};
struct r_addr_key_s {
uint64_t remote_addr;
uint32_t rkey;
};
struct infiniband {
struct rdma_cm_id *id;
struct rdma_event_channel *ec;
@ -65,7 +74,6 @@ struct infiniband {
int timeout;
enum rdma_port_space port_space;
struct ibv_mr *mr_payload;
struct r_addr_key_s *r_addr_key;
} conn;
@ -73,6 +81,14 @@ struct infiniband {
int is_source;
int cq_size;
struct ib_memory {
struct pool p_recv;
struct pool p_send;
struct ibv_mr *mr_recv;
struct ibv_mr *mr_send;
} mem;
};
/** @see node_type::reverse */

74
lib/nodes/infiniband.c
View file

@ -26,6 +26,9 @@
#include <villas/plugin.h>
#include <villas/utils.h>
#include <villas/format_type.h>
#include <villas/memory.h>
#include <villas/pool.h>
#include <rdma/rdma_cma.h>
static void ib_create_busy_poll(struct node *n, struct rdma_cm_id *id)
@ -101,6 +104,75 @@ static void ib_build_ibv(struct node *n, struct rdma_cm_id *id)
error("Failed to create Queue Pair in node %s.", node_name(n));
info("Successfully created Queue Pair.");
// Allocate memory
ib->mem.p_recv.state = STATE_DESTROYED;
ib->mem.p_recv.queue.state = STATE_DESTROYED;
// Set pool size to maximum size of Receive Queue
pool_init(&ib->mem.p_recv,
//ib->qp_init.cap.max_recv_wr,
1,
sizeof(struct payload_s),
&memtype_heap);
if(ret) {
error("Failed to init recv memory pool of node %s: %s",
node_name(n), gai_strerror(ret));
}
//ToDo: initialize r_addr_key struct if mode is RDMA
struct payload_s* test;
test = pool_get(&ib->mem.p_recv);
printf("Address value: %p\n", test);
printf("Address pool: %p\n", &ib->mem.p_recv);
printf("Address calculated: %p\n", &ib->mem.p_recv+ib->mem.p_recv.buffer_off);
printf("Offset: %li\n", ib->mem.p_recv.buffer_off);
printf("Size of struct: %lu\n", sizeof(struct payload_s));
printf("Size of block: %lu\n", ib->mem.p_recv.blocksz);
// Register memory for IB Device. Not necessary if data is send
// exclusively inline
ib->mem.mr_recv = ibv_reg_mr(
ib->ctx.pd,
&ib->mem.p_recv+ib->mem.p_recv.buffer_off,
ib->mem.p_recv.len*ib->mem.p_recv.blocksz,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if(!ib->mem.mr_recv) {
error("Failed to register mr_recv with ibv_reg_mr of node %s.",
node_name(n));
}
if(ib->is_source)
{
ib->mem.p_send.state = STATE_DESTROYED;
ib->mem.p_send.queue.state = STATE_DESTROYED;
// Set pool size to maximum size of Receive Queue
pool_init(&ib->mem.p_send,
ib->qp_init.cap.max_send_wr,
sizeof(struct payload_s),
&memtype_heap);
if(ret) {
error("Failed to init send memory of node %s: %s",
node_name(n), gai_strerror(ret));
}
//ToDo: initialize r_addr_key struct if mode is RDMA
// Register memory for IB Device. Not necessary if data is send
// exclusively inline
ib->mem.mr_send = ibv_reg_mr(
ib->ctx.pd,
&ib->mem.p_send+ib->mem.p_send.buffer_off,
ib->mem.p_send.len*ib->mem.p_send.blocksz,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if(!ib->mem.mr_send) {
error("Failed to register mr_send with ibv_reg_mr of node %s.",
node_name(n));
}
}
}
static int ib_addr_resolved(struct node *n, struct rdma_cm_id *id)
@ -238,6 +310,7 @@ int ib_parse(struct node *n, json_t *cfg)
jerror(&err, "Failed to parse configuration of node %s", node_name(n));
// Translate IP:PORT to a struct addrinfo
//ToDo: Fix fixed port
ret = getaddrinfo(local, (char *)"13337", NULL, &ib->conn.src_addr);
if(ret) {
error("Failed to resolve local address '%s' of node %s: %s",
@ -291,6 +364,7 @@ int ib_parse(struct node *n, json_t *cfg)
ib->is_source = 1;
// Translate address info
//ToDo: Fix fixed port
ret = getaddrinfo(remote, (char *)"13337", NULL, &ib->conn.dst_addr);
if(ret) {
error("Failed to resolve remote address '%s' of node %s: %s",