mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
Node can determine if data should be send inline
The user can set the maximum size of the inline data and the node checks if a sample can be send inline. This commit doesn't contain a info message to the user about what the final max inline size will be. (The HCA will probably change the value set by the user.)
This commit is contained in:
parent
b1b778f542
commit
1d6ee5aec8
2 changed files with 27 additions and 9 deletions
|
@ -76,8 +76,9 @@ struct infiniband {
|
|||
|
||||
pthread_t rdma_cm_event_thread;
|
||||
|
||||
int available_recv_wrs;
|
||||
int inline_mode;
|
||||
|
||||
int available_recv_wrs;
|
||||
struct send_wc_stack_s {
|
||||
uint64_t* array;
|
||||
unsigned top;
|
||||
|
|
|
@ -108,8 +108,6 @@ static void ib_build_ibv(struct node *n)
|
|||
ib->qp_init.send_cq = ib->ctx.send_cq;
|
||||
ib->qp_init.recv_cq = ib->ctx.recv_cq;
|
||||
|
||||
//ToDo: Set maximum inline data
|
||||
|
||||
// Create the actual QP
|
||||
ret = rdma_create_qp(ib->ctx.id, ib->ctx.pd, &ib->qp_init);
|
||||
if (ret)
|
||||
|
@ -248,10 +246,13 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
int cq_size = 128;
|
||||
int max_send_wr = 128;
|
||||
int max_recv_wr = 128;
|
||||
int max_inline_data = 0;
|
||||
int inline_mode = 1;
|
||||
|
||||
json_error_t err;
|
||||
ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, \
|
||||
s?: s, s?: i, s?: s, s?: i, s?: i}",
|
||||
s?: s, s?: i, s?: s, s?: i, s?: i, \
|
||||
s?: i, s?: i}",
|
||||
"remote", &remote,
|
||||
"local", &local,
|
||||
"rdma_port_space", &port_space,
|
||||
|
@ -260,7 +261,9 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
"cq_size", &cq_size,
|
||||
"qp_type", &qp_type,
|
||||
"max_send_wr", &max_send_wr,
|
||||
"max_recv_wr", &max_recv_wr
|
||||
"max_recv_wr", &max_recv_wr,
|
||||
"max_inline_data", &max_inline_data,
|
||||
"inline_mode", &inline_mode
|
||||
);
|
||||
if (ret)
|
||||
jerror(&err, "Failed to parse configuration of node %s", node_name(n));
|
||||
|
@ -321,6 +324,11 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
|
||||
debug(LOG_IB | 4, "Set Queue Pair type to %s in node %s", qp_type, node_name(n));
|
||||
|
||||
// Translate inline mode
|
||||
ib->conn.inline_mode = inline_mode;
|
||||
|
||||
debug(LOG_IB | 4, "Set inline_modee to %i in node %s", inline_mode, node_name(n));
|
||||
|
||||
// Set max. send and receive Work Requests
|
||||
ib->qp_init.cap.max_send_wr = max_send_wr;
|
||||
ib->qp_init.cap.max_recv_wr = max_recv_wr;
|
||||
|
@ -335,6 +343,9 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
ib->qp_init.cap.max_send_sge = 1;
|
||||
ib->qp_init.cap.max_recv_sge = 1;
|
||||
|
||||
// Set number of bytes to be send inline
|
||||
ib->qp_init.cap.max_inline_data = max_inline_data;
|
||||
|
||||
//Check if node is a source and connect to target
|
||||
if (remote) {
|
||||
debug(LOG_IB | 3, "Node %s is up as source and target", node_name(n));
|
||||
|
@ -394,6 +405,10 @@ int ib_check(struct node *n)
|
|||
if (ib->qp_init.cap.max_recv_wr > 8192)
|
||||
warn("Max number of receive WRs (%i) is bigger than send queue!", ib->qp_init.cap.max_recv_wr);
|
||||
|
||||
// Warn user if he changed the default inline value
|
||||
if (ib->qp_init.cap.max_inline_data != 0)
|
||||
warn("You changed the default value of max_inline_data. This might influence the maximum number of outstanding Work Requests in the Queue Pair and can be a reason for the Queue Pair creation to fail");
|
||||
|
||||
info("Finished check of node %s", node_name(n));
|
||||
|
||||
return 0;
|
||||
|
@ -738,10 +753,6 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele
|
|||
|
||||
if (n->state == STATE_CONNECTED) {
|
||||
// First, write
|
||||
//ToDo: Place this into configuration and create checks if settings are valid
|
||||
int send_inline = 1;
|
||||
|
||||
debug(LOG_IB | 10, "Data will be send inline [0/1]: %i", send_inline);
|
||||
|
||||
// Get Memory Region
|
||||
mr = memory_ib_get_mr(smps[0]);
|
||||
|
@ -752,6 +763,12 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele
|
|||
sge[sent].length = smps[sent]->length*sizeof(double);
|
||||
sge[sent].lkey = mr->lkey;
|
||||
|
||||
// Check if data can be send inline
|
||||
int send_inline = (sge[sent].length < ib->qp_init.cap.max_inline_data) ?
|
||||
(ib->conn.inline_mode > 0) : 0;
|
||||
|
||||
debug(LOG_IB | 10, "Sample will be send inline [0/1]: %i", send_inline);
|
||||
|
||||
// Set Send Work Request
|
||||
wr[sent].wr_id = send_inline ? 0 : (uintptr_t) smps[sent]; // This way the sample can be release in WC
|
||||
wr[sent].sg_list = &sge[sent];
|
||||
|
|
Loading…
Add table
Reference in a new issue