1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00

Started to implement memory managemen and registration. Not yet working and in an early stage. Still some debugging and testing printfs

This commit is contained in:
Dennis Potter 2018-06-24 13:02:04 +02:00
parent 4f6c2543b5
commit 077355d8ba
2 changed files with 91 additions and 1 deletions

View file

@ -44,6 +44,15 @@ enum poll_mode_e
BUSY
};
struct payload_s {
int data;
};
struct r_addr_key_s {
uint64_t remote_addr;
uint32_t rkey;
};
struct infiniband {
struct rdma_cm_id *id;
struct rdma_event_channel *ec;
@ -65,7 +74,6 @@ struct infiniband {
int timeout;
enum rdma_port_space port_space;
struct ibv_mr *mr_payload;
struct r_addr_key_s *r_addr_key;
} conn;
@ -73,6 +81,14 @@ struct infiniband {
int is_source;
int cq_size;
struct ib_memory {
struct pool p_recv;
struct pool p_send;
struct ibv_mr *mr_recv;
struct ibv_mr *mr_send;
} mem;
};
/** @see node_type::reverse */

View file

@ -26,6 +26,9 @@
#include <villas/plugin.h>
#include <villas/utils.h>
#include <villas/format_type.h>
#include <villas/memory.h>
#include <villas/pool.h>
#include <rdma/rdma_cma.h>
static void ib_create_busy_poll(struct node *n, struct rdma_cm_id *id)
@ -101,6 +104,75 @@ static void ib_build_ibv(struct node *n, struct rdma_cm_id *id)
error("Failed to create Queue Pair in node %s.", node_name(n));
info("Successfully created Queue Pair.");
// Allocate memory
ib->mem.p_recv.state = STATE_DESTROYED;
ib->mem.p_recv.queue.state = STATE_DESTROYED;
// Set pool size to maximum size of Receive Queue
pool_init(&ib->mem.p_recv,
//ib->qp_init.cap.max_recv_wr,
1,
sizeof(struct payload_s),
&memtype_heap);
if(ret) {
error("Failed to init recv memory pool of node %s: %s",
node_name(n), gai_strerror(ret));
}
//ToDo: initialize r_addr_key struct if mode is RDMA
struct payload_s* test;
test = pool_get(&ib->mem.p_recv);
printf("Address value: %p\n", test);
printf("Address pool: %p\n", &ib->mem.p_recv);
printf("Address calculated: %p\n", &ib->mem.p_recv+ib->mem.p_recv.buffer_off);
printf("Offset: %li\n", ib->mem.p_recv.buffer_off);
printf("Size of struct: %lu\n", sizeof(struct payload_s));
printf("Size of block: %lu\n", ib->mem.p_recv.blocksz);
// Register memory for IB Device. Not necessary if data is send
// exclusively inline
ib->mem.mr_recv = ibv_reg_mr(
ib->ctx.pd,
&ib->mem.p_recv+ib->mem.p_recv.buffer_off,
ib->mem.p_recv.len*ib->mem.p_recv.blocksz,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if(!ib->mem.mr_recv) {
error("Failed to register mr_recv with ibv_reg_mr of node %s.",
node_name(n));
}
if(ib->is_source)
{
ib->mem.p_send.state = STATE_DESTROYED;
ib->mem.p_send.queue.state = STATE_DESTROYED;
// Set pool size to maximum size of Receive Queue
pool_init(&ib->mem.p_send,
ib->qp_init.cap.max_send_wr,
sizeof(struct payload_s),
&memtype_heap);
if(ret) {
error("Failed to init send memory of node %s: %s",
node_name(n), gai_strerror(ret));
}
//ToDo: initialize r_addr_key struct if mode is RDMA
// Register memory for IB Device. Not necessary if data is send
// exclusively inline
ib->mem.mr_send = ibv_reg_mr(
ib->ctx.pd,
&ib->mem.p_send+ib->mem.p_send.buffer_off,
ib->mem.p_send.len*ib->mem.p_send.blocksz,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if(!ib->mem.mr_send) {
error("Failed to register mr_send with ibv_reg_mr of node %s.",
node_name(n));
}
}
}
static int ib_addr_resolved(struct node *n, struct rdma_cm_id *id)
@ -238,6 +310,7 @@ int ib_parse(struct node *n, json_t *cfg)
jerror(&err, "Failed to parse configuration of node %s", node_name(n));
// Translate IP:PORT to a struct addrinfo
//ToDo: Fix fixed port
ret = getaddrinfo(local, (char *)"13337", NULL, &ib->conn.src_addr);
if(ret) {
error("Failed to resolve local address '%s' of node %s: %s",
@ -291,6 +364,7 @@ int ib_parse(struct node *n, json_t *cfg)
ib->is_source = 1;
// Translate address info
//ToDo: Fix fixed port
ret = getaddrinfo(remote, (char *)"13337", NULL, &ib->conn.dst_addr);
if(ret) {
error("Failed to resolve remote address '%s' of node %s: %s",