From 027555c34d6516aecc266e99e7c24cca7cf11929 Mon Sep 17 00:00:00 2001 From: Dennis Potter Date: Thu, 2 Aug 2018 10:41:37 +0200 Subject: [PATCH] Added flag in config to enable/disable connection fallback. Closes #188 --- etc/infiniband.conf | 15 +++++++-------- include/villas/nodes/infiniband.h | 4 ++++ lib/nodes/infiniband.c | 26 +++++++++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/etc/infiniband.conf b/etc/infiniband.conf index 81107bdb3..ac8104158 100644 --- a/etc/infiniband.conf +++ b/etc/infiniband.conf @@ -17,12 +17,13 @@ nodes = { values = 3, frequency = 3, rate = 100000, + limit = 100000, }, ib_node_source = { type = "infiniband", - rdma_port_space = "RDMA_PS_UDP", + rdma_port_space = "RDMA_PS_TCP", in = { address = "10.0.0.2:1337", @@ -34,11 +35,8 @@ nodes = { poll_mode = "BUSY", buffer_subtraction = 128, - - hooks = ( - { type = "stats", verbose = true } - ) }, + out = { address = "10.0.0.1:1337", resolution_timeout = 1000, @@ -48,16 +46,17 @@ nodes = { vectorize = 1, - send_inline = 1, + send_inline = true, max_inline_data = 60, + + use_fallback = true, } - } ib_node_target = { type = "infiniband", - rdma_port_space = "RDMA_PS_UDP", + rdma_port_space = "RDMA_PS_TCP", in = { address = "10.0.0.1:1337", diff --git a/include/villas/nodes/infiniband.h b/include/villas/nodes/infiniband.h index fa5917f91..cb7c92555 100644 --- a/include/villas/nodes/infiniband.h +++ b/include/villas/nodes/infiniband.h @@ -87,6 +87,10 @@ struct infiniband { /* Bool, should data be send inline if possible? */ int send_inline; + /* Bool, should node have a fallback if it can't connect to a remote host? */ + int use_fallback; + + /* Buffer, used to temporarily store Work Completions from send queue */ struct queue send_wc_buffer; /* Counter to keep track of available recv. WRs */ diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c index 85aa02cec..64b6c8985 100644 --- a/lib/nodes/infiniband.c +++ b/lib/nodes/infiniband.c @@ -197,6 +197,7 @@ int ib_parse(struct node *n, json_t *cfg) int vectorize_in = 1; int vectorize_out = 1; int buffer_subtraction = 16; + int use_fallback = 1; // Parse JSON files and copy to local variables json_t *json_in = NULL; @@ -226,14 +227,15 @@ int ib_parse(struct node *n, json_t *cfg) } if (json_out) { - ret = json_unpack_ex(json_out, &err, 0, "{ s?: s, s?: i, s?: i, s?: i, s?: i, s?: i, s?: i,}", + ret = json_unpack_ex(json_out, &err, 0, "{ s?: s, s?: i, s?: i, s?: i, s?: i, s?: b, s?: i, s?: b}", "address", &remote, "resolution_timeout", &timeout, "cq_size", &send_cq_size, "max_wrs", &max_send_wr, "max_inline_data", &max_inline_data, "send_inline", &send_inline, - "vectorize", &vectorize_out + "vectorize", &vectorize_out, + "use_fallback", &use_fallback ); if (ret) jerror(&err, "Failed to parse output configuration of node %s", node_name(n)); @@ -250,6 +252,9 @@ int ib_parse(struct node *n, json_t *cfg) debug(LOG_IB | 3, "Node %s is up as target", node_name(n)); } + // Set fallback mode + ib->conn.use_fallback = use_fallback; + // Set vectorize mode. Do not print, since framework will print this information n->in.vectorize = vectorize_in; n->out.vectorize = vectorize_out; @@ -392,11 +397,8 @@ int ib_check(struct node *n) // Warn user if he changed the default inline value if (ib->qp_init.cap.max_inline_data != 0) - warn("You changed the default value of max_inline_data. This might influence the maximum number of outstanding Work Requests in the Queue Pair and can be a reason for the Queue Pair creation to fail"); - - // Check if inline mode is set to a valid value - if (ib->conn.send_inline != 0 && ib->conn.send_inline != 1) - error("send_inline has to be set to either 0 or 1! %i is not a valid value", ib->conn.send_inline); + warn("You changed the default value of max_inline_data. This might influence the maximum number " + "of outstanding Work Requests in the Queue Pair and can be a reason for the Queue Pair creation to fail"); info("Finished check of node %s", node_name(n)); @@ -444,7 +446,13 @@ static void ib_continue_as_listen(struct node *n, struct rdma_cm_event *event) struct infiniband *ib = (struct infiniband *) n->_vd; int ret; - warn("Trying to continue as listening node"); + if (ib->conn.use_fallback) + warn("Trying to continue as listening node"); + else + error("Cannot establish a connection with remote host! If you want that %s tries to " + "continue as listening node in such cases, set use_fallback = true in the configuration", + node_name(n)); + // Acknowledge event rdma_ack_cm_event(event); @@ -906,7 +914,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele // Check if data can be send inline // 32 byte meta data is always send. int send_inline = ( (sge[sent][j-1].length + META_SIZE) < ib->qp_init.cap.max_inline_data) ? - ib->conn.send_inline : 0; + ib->conn.send_inline : 0; debug(LOG_IB | 10, "Sample will be send inline [0/1]: %i", send_inline);