From 027555c34d6516aecc266e99e7c24cca7cf11929 Mon Sep 17 00:00:00 2001
From: Dennis Potter <dennis@dennispotter.eu>
Date: Thu, 2 Aug 2018 10:41:37 +0200
Subject: [PATCH] Added flag in config to enable/disable connection fallback.
 Closes #188

---
 etc/infiniband.conf               | 15 +++++++--------
 include/villas/nodes/infiniband.h |  4 ++++
 lib/nodes/infiniband.c            | 26 +++++++++++++++++---------
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/etc/infiniband.conf b/etc/infiniband.conf
index 81107bdb3..ac8104158 100644
--- a/etc/infiniband.conf
+++ b/etc/infiniband.conf
@@ -17,12 +17,13 @@ nodes = {
         values = 3,
         frequency = 3,
         rate = 100000,
+        limit = 100000,
     },
 
     ib_node_source = {
         type = "infiniband",
 
-        rdma_port_space = "RDMA_PS_UDP",
+        rdma_port_space = "RDMA_PS_TCP",
         
         in = {
             address = "10.0.0.2:1337",
@@ -34,11 +35,8 @@ nodes = {
 
             poll_mode = "BUSY",
             buffer_subtraction = 128,
-
-            hooks = (
-                { type = "stats", verbose = true }
-            )
         },
+
         out = {
             address = "10.0.0.1:1337",
             resolution_timeout = 1000,
@@ -48,16 +46,17 @@ nodes = {
 
             vectorize = 1,
 
-            send_inline = 1,
+            send_inline = true,
             max_inline_data = 60,
+
+            use_fallback = true,
         }
-    
     }
 
     ib_node_target = {
         type = "infiniband",
 
-        rdma_port_space = "RDMA_PS_UDP",
+        rdma_port_space = "RDMA_PS_TCP",
 
         in = {
             address = "10.0.0.1:1337",
diff --git a/include/villas/nodes/infiniband.h b/include/villas/nodes/infiniband.h
index fa5917f91..cb7c92555 100644
--- a/include/villas/nodes/infiniband.h
+++ b/include/villas/nodes/infiniband.h
@@ -87,6 +87,10 @@ struct infiniband {
 		/* Bool, should data be send inline if possible? */
 		int send_inline;
 
+		/* Bool, should node have a fallback if it can't connect to a remote host? */
+		int use_fallback;
+
+		/* Buffer, used to temporarily store Work Completions from send queue */
 		struct queue send_wc_buffer;
 
 		/* Counter to keep track of available recv. WRs */
diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c
index 85aa02cec..64b6c8985 100644
--- a/lib/nodes/infiniband.c
+++ b/lib/nodes/infiniband.c
@@ -197,6 +197,7 @@ int ib_parse(struct node *n, json_t *cfg)
 	int vectorize_in = 1;
 	int vectorize_out = 1;
 	int buffer_subtraction = 16;
+	int use_fallback = 1;
 
 	// Parse JSON files and copy to local variables
 	json_t *json_in = NULL;
@@ -226,14 +227,15 @@ int ib_parse(struct node *n, json_t *cfg)
 	}
 
 	if (json_out) {
-		ret = json_unpack_ex(json_out, &err, 0, "{ s?: s, s?: i, s?: i, s?: i, s?: i, s?: i, s?: i,}",
+		ret = json_unpack_ex(json_out, &err, 0, "{ s?: s, s?: i, s?: i, s?: i, s?: i, s?: b, s?: i, s?: b}",
 			"address", &remote,
 			"resolution_timeout", &timeout,
 			"cq_size", &send_cq_size,
 			"max_wrs", &max_send_wr,
 			"max_inline_data", &max_inline_data,
 			"send_inline", &send_inline,
-			"vectorize", &vectorize_out
+			"vectorize", &vectorize_out,
+			"use_fallback", &use_fallback
 		);
 		if (ret)
 			jerror(&err, "Failed to parse output configuration of node %s", node_name(n));
@@ -250,6 +252,9 @@ int ib_parse(struct node *n, json_t *cfg)
 		debug(LOG_IB | 3, "Node %s is up as target", node_name(n));
 	}
 
+	// Set fallback mode
+	ib->conn.use_fallback = use_fallback;
+
 	// Set vectorize mode. Do not print, since framework will print this information
 	n->in.vectorize = vectorize_in;
 	n->out.vectorize = vectorize_out;
@@ -392,11 +397,8 @@ int ib_check(struct node *n)
 
 	// Warn user if he changed the default inline value
 	if (ib->qp_init.cap.max_inline_data != 0)
-		warn("You changed the default value of max_inline_data. This might influence the maximum number of outstanding Work Requests in the Queue Pair and can be a reason for the Queue Pair creation to fail");
-
-	// Check if inline mode is set to a valid value
-	if (ib->conn.send_inline != 0 && ib->conn.send_inline != 1)
-		error("send_inline has to be set to either 0 or 1! %i is not a valid value", ib->conn.send_inline);
+		warn("You changed the default value of max_inline_data. This might influence the maximum number "
+			"of outstanding Work Requests in the Queue Pair and can be a reason for the Queue Pair creation to fail");
 
 	info("Finished check of node %s", node_name(n));
 
@@ -444,7 +446,13 @@ static void ib_continue_as_listen(struct node *n, struct rdma_cm_event *event)
 	struct infiniband *ib = (struct infiniband *) n->_vd;
 	int ret;
 
-	warn("Trying to continue as listening node");
+	if (ib->conn.use_fallback)
+		warn("Trying to continue as listening node");
+	else
+		error("Cannot establish a connection with remote host! If you want that %s tries to "
+			"continue as listening node in such cases, set use_fallback = true in the configuration",
+			node_name(n));
+
 
 	// Acknowledge event
 	rdma_ack_cm_event(event);
@@ -906,7 +914,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele
 			// Check if data can be send inline
 			// 32 byte meta data is always send.
 			int send_inline = ( (sge[sent][j-1].length + META_SIZE) < ib->qp_init.cap.max_inline_data) ?
-				ib->conn.send_inline : 0;
+						ib->conn.send_inline : 0;
 
 
 			debug(LOG_IB | 10, "Sample will be send inline [0/1]: %i", send_inline);