From e3ce379c3628615f63fce14d5fa1b4cf4b097610 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sun, 13 Sep 2020 11:01:20 +0200
Subject: [PATCH] if: rewrote libnl / network if code to C++ and fixed if
affinity (closes #233)
---
common | 2 +-
include/villas/kernel/if.h | 133 -----------
include/villas/kernel/if.hpp | 141 ++++++++++++
include/villas/kernel/{nl.h => nl.hpp} | 26 ++-
include/villas/kernel/{tc.h => tc.hpp} | 20 +-
.../kernel/{tc_netem.h => tc_netem.hpp} | 22 +-
include/villas/super_node.hpp | 29 ++-
lib/kernel/if.cpp | 211 +++++++-----------
lib/kernel/nl.cpp | 46 +++-
lib/kernel/tc.cpp | 19 +-
lib/kernel/tc_netem.cpp | 125 ++++++-----
lib/node.cpp | 10 +-
lib/nodes/rtp.cpp | 11 +-
lib/nodes/socket.cpp | 13 +-
lib/signal_list.cpp | 2 +-
lib/socket_addr.cpp | 5 +-
lib/super_node.cpp | 31 +--
17 files changed, 441 insertions(+), 405 deletions(-)
delete mode 100644 include/villas/kernel/if.h
create mode 100644 include/villas/kernel/if.hpp
rename include/villas/kernel/{nl.h => nl.hpp} (69%)
rename include/villas/kernel/{tc.h => tc.hpp} (86%)
rename include/villas/kernel/{tc_netem.h => tc_netem.hpp} (84%)
diff --git a/common b/common
index f0059871a..dfd8bb232 160000
--- a/common
+++ b/common
@@ -1 +1 @@
-Subproject commit f0059871af74e121b2d091cb445ba0b048ba675b
+Subproject commit dfd8bb23202c554b74d622dff40fb1bf16ced6c8
diff --git a/include/villas/kernel/if.h b/include/villas/kernel/if.h
deleted file mode 100644
index f2bfd4610..000000000
--- a/include/villas/kernel/if.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/** Interface related functions
- *
- * These functions are used to manage a network interface.
- * Most of them make use of Linux-specific APIs.
- *
- * @file
- * @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
- * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
- * @license GNU General Public License (version 3)
- *
- * VILLASnode
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *********************************************************************************/
-
-/** @addtogroup kernel Kernel
- * @{
- */
-
-#pragma once
-
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <villas/list.h>
-
-#define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */
-
-#ifndef SO_MARK
- #define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */
-#endif
-
-struct socket;
-struct nl_addr;
-struct rtnl_link;
-
-/** Interface data structure */
-struct interface {
- struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */
- struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */
-
- char irqs[IF_IRQ_MAX]; /**< List of IRQs of the NIC. */
- int affinity; /**< IRQ / Core Affinity of this interface. */
-
- struct vlist nodes; /**< Linked list of nodes which use this interface. */
-};
-
-/** Add a new interface to the global list and lookup name, irqs...
- *
- * @param link The libnl3 link handle
- * @retval >0 Success. A pointer to the new interface.
- * @retval 0 Error. The creation failed.
- */
-int if_init(struct interface * , struct rtnl_link *link) __attribute__ ((warn_unused_result));
-
-/** Get name of interface */
-const char * if_name(struct interface *);
-
-/** Destroy interface by freeing dynamically allocated memory.
- *
- * @param i A pointer to the interface structure.
- */
-int if_destroy(struct interface *i) __attribute__ ((warn_unused_result));
-
-/** Start interface.
- *
- * This setups traffic controls queue discs, network emulation and
- * maps interface IRQs according to affinity.
- *
- * @param i A pointer to the interface structure.
- * @retval 0 Success. Everything went well.
- * @retval <0 Error. Something went wrong.
- */
-int if_start(struct interface *i);
-
-/** Stop interface
- *
- * This resets traffic qdiscs ant network emulation
- * and maps interface IRQs to all CPUs.
- *
- * @param i A pointer to the interface structure.
- * @retval 0 Success. Everything went well.
- * @retval <0 Error. Something went wrong.
- */
-int if_stop(struct interface *i);
-
-/** Find existing or create new interface instance on which packets for a certain destination
- * will leave the system.
- */
-struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces);
-
-/** Lookup routing tables to get the interface on which packets for a certain destination
- * will leave the system.
- *
- * @param[in] sa The destination address for outgoing packets.
- * @param[out] link The egress interface.
- * @retval 0 Success. Everything went well.
- * @retval <0 Error. Something went wrong.
- */
-struct rtnl_link * if_get_egress_link(struct sockaddr *sa);
-
-/** Get all IRQs for this interface.
- *
- * Only MSI IRQs are determined by looking at:
- * /sys/class/net/{ifname}/device/msi_irqs/
- *
- * @param i A pointer to the interface structure
- * @retval 0 Success. Everything went well.
- * @retval <0 Error. Something went wrong.
- */
-int if_get_irqs(struct interface *i);
-
-/** Change the SMP affinity of NIC interrupts.
- *
- * @param i A pointer to the interface structure
- * @param affinity A mask specifying which cores should handle this interrupt.
- * @retval 0 Success. Everything went well.
- * @retval <0 Error. Something went wrong.
- */
-int if_set_affinity(struct interface *i, int affinity);
-
-/** @} */
diff --git a/include/villas/kernel/if.hpp b/include/villas/kernel/if.hpp
new file mode 100644
index 000000000..493f521f9
--- /dev/null
+++ b/include/villas/kernel/if.hpp
@@ -0,0 +1,141 @@
+/** Interface related functions
+ *
+ * These functions are used to manage a network interface.
+ * Most of them make use of Linux-specific APIs.
+ *
+ * @file
+ * @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
+ * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
+ * @license GNU General Public License (version 3)
+ *
+ * VILLASnode
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *********************************************************************************/
+
+/** @addtogroup kernel Kernel
+ * @{
+ */
+
+#pragma once
+
+#include <list>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+
+#include <villas/log.hpp>
+
+#ifndef SO_MARK
+ #define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */
+#endif
+
+/* Forward declarations */
+struct vnode;
+struct nl_addr;
+struct rtnl_link;
+struct rtnl_qdisc;
+
+namespace villas {
+
+namespace node {
+
+/* Forward declarations */
+class SuperNode;
+
+}
+
+namespace kernel {
+
+/** Interface data structure */
+class Interface {
+
+public:
+ struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */
+ struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */
+
+protected:
+ int affinity; /**< IRQ / Core Affinity of this interface. */
+
+ std::list<int> irqs; /**< List of IRQs of the NIC. */
+ std::list<struct vnode *> nodes; /**< List of nodes which use this interface. */
+
+ Logger logger;
+
+public:
+ /** Add a new interface to the global list and lookup name, irqs... */
+ Interface(struct rtnl_link *link, int affinity = 0);
+ ~Interface();
+
+ /** Start interface.
+ *
+ * This setups traffic controls queue discs, network emulation and
+ * maps interface IRQs according to affinity.
+ *
+ * @param i A pointer to the interface structure.
+ * @retval 0 Success. Everything went well.
+ * @retval <0 Error. Something went wrong.
+ */
+ int start();
+
+ /** Stop interface
+ *
+ * This resets traffic qdiscs ant network emulation
+ * and maps interface IRQs to all CPUs.
+ *
+ * @param i A pointer to the interface structure.
+ * @retval 0 Success. Everything went well.
+ * @retval <0 Error. Something went wrong.
+ */
+ int stop();
+
+ /** Find existing or create new interface instance on which packets for a certain destination
+ * will leave the system.
+ */
+ static
+ Interface *
+ getEgress(struct sockaddr *sa, villas::node::SuperNode *sn);
+
+ /** Get all IRQs for this interface.
+ *
+ * Only MSI IRQs are determined by looking at:
+ * /sys/class/net/{ifname}/device/msi_irqs/
+ *
+ * @param i A pointer to the interface structure
+ * @retval 0 Success. Everything went well.
+ * @retval <0 Error. Something went wrong.
+ */
+ int getIRQs();
+
+ /** Change the SMP affinity of NIC interrupts.
+ *
+ * @param i A pointer to the interface structure
+ * @param affinity A mask specifying which cores should handle this interrupt.
+ * @retval 0 Success. Everything went well.
+ * @retval <0 Error. Something went wrong.
+ */
+ int setAffinity(int affinity);
+
+ std::string getName() const;
+
+ void addNode(struct vnode *n)
+ {
+ nodes.push_back(n);
+ }
+};
+
+} /* namespace kernel */
+} /* namespace villas */
+
+/** @} */
diff --git a/include/villas/kernel/nl.h b/include/villas/kernel/nl.hpp
similarity index 69%
rename from include/villas/kernel/nl.h
rename to include/villas/kernel/nl.hpp
index 3bec7cc5c..b33ec1a31 100644
--- a/include/villas/kernel/nl.h
+++ b/include/villas/kernel/nl.hpp
@@ -27,21 +27,41 @@
#pragma once
+#include <sys/socket.h>
+
#include <netlink/netlink.h>
#include <netlink/route/route.h>
#include <netlink/route/link.h>
+namespace villas {
+namespace kernel {
+namespace nl {
+
/** Get index of outgoing interface for given destination address.
*
* @retval >=0 Interface index of outgoing interface.
* @retval <0 Error. Something went wrong.
*/
-int nl_get_egress(struct nl_addr *addr);
+int get_egress(struct nl_addr *addr);
+
+/** Lookup routing tables to get the interface on which packets for a certain destination
+ * will leave the system.
+ *
+ * @param[in] sa The destination address for outgoing packets.
+ * @param[out] link The egress interface.
+ * @retval 0 Success. Everything went well.
+ * @retval <0 Error. Something went wrong.
+ */
+struct rtnl_link * get_egress_link(struct sockaddr *sa);
/** Get or create global netlink socket. */
-struct nl_sock *nl_init();
+struct nl_sock * init();
/** Close and free global netlink socket. */
-void nl_shutdown();
+void shutdown();
+
+} /* namespace nl */
+} /* namespace kernel */
+} /* namespace villas */
/** @} */
diff --git a/include/villas/kernel/tc.h b/include/villas/kernel/tc.hpp
similarity index 86%
rename from include/villas/kernel/tc.h
rename to include/villas/kernel/tc.hpp
index 9fc6dd9b1..c465989bd 100644
--- a/include/villas/kernel/tc.h
+++ b/include/villas/kernel/tc.hpp
@@ -39,9 +39,15 @@
#include <jansson.h>
-typedef uint32_t tc_hdl_t;
+namespace villas {
+namespace kernel {
-struct interface;
+/* Forward declarations */
+class Interface;
+
+namespace tc {
+
+typedef uint32_t tc_hdl_t;
/** Remove all queuing disciplines and filters.
*
@@ -49,7 +55,7 @@ struct interface;
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
-int tc_reset(struct interface *i);
+int reset(Interface *i);
/** Create a priority (prio) queueing discipline.
*
@@ -61,7 +67,7 @@ int tc_reset(struct interface *i);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
-int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands);
+int prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands);
/** Add a new filter based on the netfilter mark.
*
@@ -72,6 +78,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
-int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
+int mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
+
+} /* namespace tc */
+} /* namespace kernel */
+} /* namespace villas */
/** @} */
diff --git a/include/villas/kernel/tc_netem.h b/include/villas/kernel/tc_netem.hpp
similarity index 84%
rename from include/villas/kernel/tc_netem.h
rename to include/villas/kernel/tc_netem.hpp
index 9239ac649..931501c05 100644
--- a/include/villas/kernel/tc_netem.h
+++ b/include/villas/kernel/tc_netem.hpp
@@ -39,9 +39,15 @@
#include <jansson.h>
-typedef uint32_t tc_hdl_t;
+namespace villas {
+namespace kernel {
-struct interface;
+/* Forward declarations */
+class Interface;
+
+namespace tc {
+
+typedef uint32_t tc_hdl_t;
/** Parse network emulator (netem) settings.
*
@@ -50,14 +56,14 @@ struct interface;
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
-int tc_netem_parse(struct rtnl_qdisc **ne, json_t *cfg);
+int netem_parse(struct rtnl_qdisc **ne, json_t *cfg);
/** Print network emulator (netem) setting into buffer.
*
* @param tc A pointer to the libnl3 qdisc object where settings will be read from.
* @return A pointer to a string which must be freed() by the caller.
*/
-char * tc_netem_print(struct rtnl_qdisc *ne);
+char * netem_print(struct rtnl_qdisc *ne);
/** Add a new network emulator (netem) discipline.
*
@@ -68,8 +74,12 @@ char * tc_netem_print(struct rtnl_qdisc *ne);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
-int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
+int netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
-int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json);
+int netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json);
+
+} /* namespace tc */
+} /* namespace kernel */
+} /* namespace villas */
/** @} */
diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp
index e64b855c1..9ccda1c2d 100644
--- a/include/villas/super_node.hpp
+++ b/include/villas/super_node.hpp
@@ -41,6 +41,7 @@ extern "C" {
#include <villas/node.h>
#include <villas/task.hpp>
#include <villas/common.hpp>
+#include <villas/kernel/if.hpp>
/* Forward declarations */
struct vnode;
@@ -60,7 +61,7 @@ protected:
struct vlist nodes;
struct vlist paths;
- struct vlist interfaces;
+ std::list<kernel::Interface *> interfaces;
#ifdef WITH_API
Api api;
@@ -141,26 +142,31 @@ public:
return &nodes;
}
- struct vlist * getPaths() {
+ struct vlist * getPaths()
+ {
return &paths;
}
- struct vlist * getInterfaces() {
- return &interfaces;
+ std::list<kernel::Interface *> & getInterfaces()
+ {
+ return interfaces;
}
- enum State getState() {
+ enum State getState() const
+ {
return state;
}
#ifdef WITH_API
- Api * getApi() {
+ Api * getApi()
+ {
return &api;
}
#endif
#ifdef WITH_WEB
- Web * getWeb() {
+ Web * getWeb()
+ {
return &web;
}
#endif
@@ -170,16 +176,21 @@ public:
return config.root;
}
- std::string getConfigUri()
+ std::string getConfigUri() const
{
return uri;
}
- std::string getName()
+ std::string getName() const
{
return name;
}
+ int getAffinity() const
+ {
+ return affinity;
+ }
+
/** Destroy configuration object. */
~SuperNode();
};
diff --git a/lib/kernel/if.cpp b/lib/kernel/if.cpp
index c024cc1e1..825759601 100644
--- a/lib/kernel/if.cpp
+++ b/lib/kernel/if.cpp
@@ -23,72 +23,59 @@
#include <cstdio>
#include <cstdlib>
#include <dirent.h>
-#include <linux/if_packet.h>
#include <netlink/route/link.h>
#include <villas/node/config.h>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
+#include <villas/super_node.hpp>
+#include <villas/cpuset.hpp>
-#include <villas/kernel/if.h>
-#include <villas/kernel/tc.h>
-#include <villas/kernel/tc_netem.h>
-#include <villas/kernel/nl.h>
+#include <villas/kernel/if.hpp>
+#include <villas/kernel/tc.hpp>
+#include <villas/kernel/tc_netem.hpp>
+#include <villas/kernel/nl.hpp>
#include <villas/kernel/kernel.hpp>
#include <villas/nodes/socket.hpp>
using namespace villas;
+using namespace villas::node;
using namespace villas::utils;
+using namespace villas::kernel;
-int if_init(struct interface *i, struct rtnl_link *link)
+Interface::Interface(struct rtnl_link *link, int aff) :
+ nl_link(link),
+ tc_qdisc(nullptr),
+ affinity(aff)
{
- int ret;
+ logger = logging.get(fmt::format("kernel:if:{}", getName()));
- i->nl_link = link;
+ int n = getIRQs();
+ if (n)
+ logger->warn("Did not found any interrupts");
- debug(LOG_IF | 3, "Created interface '%s'", if_name(i));
-
- int n = if_get_irqs(i);
- if (n > 0)
- debug(6, "Found %u IRQs for interface '%s'", n, if_name(i));
- else
- warning("Did not found any interrupts for interface '%s'", if_name(i));
-
- ret = vlist_init(&i->nodes);
- if (ret)
- return ret;
-
- return 0;
+ logger->debug("Found {} IRQs", irqs.size());
}
-int if_destroy(struct interface *i)
+Interface::~Interface()
{
- int ret;
-
- /* List members are freed by the nodes they belong to. */
- ret = vlist_destroy(&i->nodes);
- if (ret)
- return ret;
-
- rtnl_qdisc_put(i->tc_qdisc);
-
- return 0;
+ if (tc_qdisc)
+ rtnl_qdisc_put(tc_qdisc);
}
-int if_start(struct interface *i)
+int Interface::start()
{
- info("Starting interface '%s' which is used by %zu nodes", if_name(i), vlist_length(&i->nodes));
+ logger->info("Starting interface which is used by {} nodes", nodes.size());
/* Set affinity for network interfaces (skip _loopback_ dev) */
- //if_set_affinity(i, i->affinity);
+ if (affinity)
+ setAffinity(affinity);
/* Assign fwmark's to nodes which have netem options */
int ret, fwmark = 0;
- for (size_t j = 0; j < vlist_length(&i->nodes); j++) {
- struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j);
-
+ for (auto *n : nodes) {
if (n->tc_qdisc && n->fwmark < 0)
n->fwmark = 1 + fwmark++;
}
@@ -98,141 +85,97 @@ int if_start(struct interface *i)
return 0;
if (getuid() != 0)
- error("Network emulation requires super-user privileges!");
+ throw RuntimeError("Network emulation requires super-user privileges!");
/* Replace root qdisc */
- ret = tc_prio(i, &i->tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark);
+ ret = tc::prio(this, &tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark);
if (ret)
- error("Failed to setup priority queuing discipline: %s", nl_geterror(ret));
+ throw RuntimeError("Failed to setup priority queuing discipline: {}", nl_geterror(ret));
/* Create netem qdisks and appropriate filter per netem node */
- for (size_t j = 0; j < vlist_length(&i->nodes); j++) {
- struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j);
-
+ for (auto *n : nodes) {
if (n->tc_qdisc) {
- ret = tc_mark(i, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark);
+ ret = tc::mark(this, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark);
if (ret)
- error("Failed to setup FW mark classifier: %s", nl_geterror(ret));
+ throw RuntimeError("Failed to setup FW mark classifier: {}", nl_geterror(ret));
- char *buf = tc_netem_print(n->tc_qdisc);
- debug(LOG_IF | 5, "Starting network emulation on interface '%s' for FW mark %u: %s",
- if_name(i), n->fwmark, buf);
+ char *buf = tc::netem_print(n->tc_qdisc);
+ logger->debug("Starting network emulation for FW mark {}: {}", n->fwmark, buf);
free(buf);
- ret = tc_netem(i, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark));
+ ret = tc::netem(this, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark));
if (ret)
- error("Failed to setup netem qdisc: %s", nl_geterror(ret));
+ throw RuntimeError("Failed to setup netem qdisc: {}", nl_geterror(ret));
}
}
return 0;
}
-int if_stop(struct interface *i)
+int Interface::stop()
{
- info("Stopping interface '%s'", if_name(i));
+ logger->info("Stopping interface");
- //if_set_affinity(i, -1L);
+ if (affinity)
+ setAffinity(-1L);
- if (i->tc_qdisc)
- tc_reset(i);
+ if (tc_qdisc)
+ tc::reset(this);
return 0;
}
-const char * if_name(struct interface *i)
+std::string Interface::getName() const
{
- return rtnl_link_get_name(i->nl_link);
+ auto str = rtnl_link_get_name(nl_link);
+
+ return std::string(str);
}
-struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces)
+Interface * Interface::getEgress(struct sockaddr *sa, SuperNode *sn)
{
- int ret;
struct rtnl_link *link;
- /* Determine outgoing interface */
- link = if_get_egress_link(sa);
- if (!link) {
- char *buf = socket_print_addr(sa);
- error("Failed to get interface for socket address '%s'", buf);
- free(buf);
+ Logger logger = logging.get("kernel:if");
- return nullptr;
- }
+ auto & interfaces = sn->getInterfaces();
+ auto affinity = sn->getAffinity();
+
+ /* Determine outgoing interface */
+ link = nl::get_egress_link(sa);
+ if (!link)
+ throw RuntimeError("Failed to get interface for socket address '{}'", socket_print_addr(sa));
/* Search of existing interface with correct ifindex */
- struct interface *i;
- for (size_t k = 0; k < vlist_length(interfaces); k++) {
- i = (struct interface *) vlist_at(interfaces, k);
-
+ for (auto *i : interfaces) {
if (rtnl_link_get_ifindex(i->nl_link) == rtnl_link_get_ifindex(link))
return i;
}
/* If not found, create a new interface */
- i = new struct interface;
+ auto *i = new Interface(link, affinity);
if (!i)
throw MemoryAllocationError();
- memset(i, 0, sizeof(struct interface));
-
- ret = if_init(i, link);
- if (ret)
- return nullptr;
-
- vlist_push(interfaces, i);
+ interfaces.push_back(i);
return i;
}
-struct rtnl_link * if_get_egress_link(struct sockaddr *sa)
+int Interface::getIRQs()
{
- int ifindex = -1;
+ int irq;
- switch (sa->sa_family) {
- case AF_INET:
- case AF_INET6: {
- struct sockaddr_in *sin = (struct sockaddr_in *) sa;
- struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
-
- struct nl_addr *addr = (sa->sa_family == AF_INET)
- ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
- : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
-
- ifindex = nl_get_egress(addr); nl_addr_put(addr);
- if (ifindex < 0)
- error("Netlink error: %s", nl_geterror(ifindex));
- break;
- }
-
- case AF_PACKET: {
- struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
-
- ifindex = sll->sll_ifindex;
- break;
- }
- }
-
- struct nl_cache *cache = nl_cache_mngt_require("route/link");
-
- return rtnl_link_get(cache, ifindex);
-}
-
-int if_get_irqs(struct interface *i)
-{
- char dirname[NAME_MAX];
- int irq, n = 0;
-
- snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", if_name(i));
- DIR *dir = opendir(dirname);
+ auto dirname = fmt::format("/sys/class/net/{}/device/msi_irqs/", getName());
+ DIR *dir = opendir(dirname.c_str());
if (dir) {
- memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX);
+ irqs.clear();
struct dirent *entry;
- while ((entry = readdir(dir)) && n < IF_IRQ_MAX) {
+ while ((entry = readdir(dir))) {
irq = atoi(entry->d_name);
if (irq)
- i->irqs[n++] = irq;
+ irqs.push_back(irq);
}
closedir(dir);
@@ -241,24 +184,32 @@ int if_get_irqs(struct interface *i)
return 0;
}
-int if_set_affinity(struct interface *i, int affinity)
+int Interface::setAffinity(int affinity)
{
- char filename[NAME_MAX];
+ assert(affinity != 0);
+
+ if (getuid() != 0) {
+ logger->warn("Failed to tune IRQ affinity. Please run as super-user");
+ return 0;
+ }
+
FILE *file;
- for (int n = 0; n < IF_IRQ_MAX && i->irqs[n]; n++) {
- snprintf(filename, sizeof(filename), "/proc/irq/%d/smp_affinity", (int) i->irqs[n]);
+ CpuSet cset_pin(affinity);
- file = fopen(filename, "w");
+ for (int irq : irqs) {
+ std::string filename = fmt::format("/proc/irq/{}/smp_affinity", irq);
+
+ file = fopen(filename.c_str(), "w");
if (file) {
- if (fprintf(file, "%8x", affinity) < 0)
- error("Failed to set affinity for IRQ %u", i->irqs[n]);
+ if (fprintf(file, "%8lx", (unsigned long) cset_pin) < 0)
+ throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName());
fclose(file);
- debug(LOG_IF | 5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], if_name(i), affinity);
+ logger->debug("Set affinity of IRQ {} to {} {}", irq, cset_pin.count() == 1 ? "core" : "cores", (std::string) cset_pin);
}
else
- error("Failed to set affinity for interface '%s'", if_name(i));
+ throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName());
}
return 0;
diff --git a/lib/kernel/nl.cpp b/lib/kernel/nl.cpp
index a0f42e806..1e8d04afe 100644
--- a/lib/kernel/nl.cpp
+++ b/lib/kernel/nl.cpp
@@ -24,19 +24,22 @@
#include <cstdio>
+#include <linux/if_packet.h>
+
#include <netlink/route/route.h>
#include <netlink/route/link.h>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
-#include <villas/kernel/nl.h>
+#include <villas/kernel/nl.hpp>
/** Singleton for global netlink socket */
static struct nl_sock *sock = nullptr;
using namespace villas;
+using namespace villas::kernel::nl;
-struct nl_sock * nl_init()
+struct nl_sock * villas::kernel::nl::init()
{
int ret;
@@ -62,7 +65,7 @@ struct nl_sock * nl_init()
return sock;
}
-void nl_shutdown()
+void villas::kernel::nl::shutdown()
{
nl_close(sock);
nl_socket_free(sock);
@@ -80,10 +83,10 @@ static int egress_cb(struct nl_msg *msg, void *arg)
return NL_STOP;
}
-int nl_get_egress(struct nl_addr *addr)
+int villas::kernel::nl::get_egress(struct nl_addr *addr)
{
int ret;
- struct nl_sock *sock = nl_init();
+ struct nl_sock *sock = nl::init();
struct nl_cb *cb;
struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0);
struct rtnl_route *route = nullptr;
@@ -130,3 +133,36 @@ out: nlmsg_free(msg);
return ret;
}
+
+struct rtnl_link * villas::kernel::nl::get_egress_link(struct sockaddr *sa)
+{
+ int ifindex = -1;
+
+ switch (sa->sa_family) {
+ case AF_INET:
+ case AF_INET6: {
+ struct sockaddr_in *sin = (struct sockaddr_in *) sa;
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
+
+ struct nl_addr *addr = (sa->sa_family == AF_INET)
+ ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
+ : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
+
+ ifindex = nl::get_egress(addr); nl_addr_put(addr);
+ if (ifindex < 0)
+ error("Netlink error: %s", nl_geterror(ifindex));
+ break;
+ }
+
+ case AF_PACKET: {
+ struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
+
+ ifindex = sll->sll_ifindex;
+ break;
+ }
+ }
+
+ struct nl_cache *cache = nl_cache_mngt_require("route/link");
+
+ return rtnl_link_get(cache, ifindex);
+}
diff --git a/lib/kernel/tc.cpp b/lib/kernel/tc.cpp
index f0e096ade..7e34be0ae 100644
--- a/lib/kernel/tc.cpp
+++ b/lib/kernel/tc.cpp
@@ -30,16 +30,17 @@
#include <villas/utils.hpp>
#include <villas/kernel/kernel.hpp>
-#include <villas/kernel/if.h>
-#include <villas/kernel/tc.h>
-#include <villas/kernel/nl.h>
+#include <villas/kernel/if.hpp>
+#include <villas/kernel/tc.hpp>
+#include <villas/kernel/nl.hpp>
using namespace villas;
+using namespace villas::kernel;
-int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands)
+int villas::kernel::tc::prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands)
{
int ret;
- struct nl_sock *sock = nl_init();
+ struct nl_sock *sock = nl::init();
struct rtnl_qdisc *q = rtnl_qdisc_alloc();
ret = kernel::module_load("sch_prio");
@@ -69,10 +70,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl
return ret;
}
-int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
+int villas::kernel::tc::mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
{
int ret;
- struct nl_sock *sock = nl_init();
+ struct nl_sock *sock = nl::init();
struct rtnl_cls *c = rtnl_cls_alloc();
ret = kernel::module_load("cls_fw");
@@ -97,9 +98,9 @@ int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_
return ret;
}
-int tc_reset(struct interface *i)
+int villas::kernel::tc::reset(Interface *i)
{
- struct nl_sock *sock = nl_init();
+ struct nl_sock *sock = nl::init();
/* We restore the default pfifo_fast qdisc, by deleting ours */
return rtnl_qdisc_delete(sock, i->tc_qdisc);
diff --git a/lib/kernel/tc_netem.cpp b/lib/kernel/tc_netem.cpp
index 15c0be5f1..16df90370 100644
--- a/lib/kernel/tc_netem.cpp
+++ b/lib/kernel/tc_netem.cpp
@@ -27,20 +27,75 @@
#include <netlink/route/qdisc/netem.h>
-#include <villas/kernel/if.h>
-#include <villas/kernel/nl.h>
+#include <villas/kernel/if.hpp>
+#include <villas/kernel/nl.hpp>
#include <villas/kernel/nl-private.h>
-#include <villas/kernel/tc_netem.h>
+#include <villas/kernel/tc_netem.hpp>
#include <villas/kernel/kernel.hpp>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
using namespace villas;
using namespace villas::utils;
+using namespace villas::kernel;
static const double max_percent_value = 0xffffffff;
-int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
+/**
+ * Set the delay distribution. Latency/jitter must be set before applying.
+ * @arg qdisc Netem qdisc.
+ * @return 0 on success, error code on failure.
+ */
+static int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len)
+{
+ struct rtnl_netem *netem;
+
+ if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc))))
+ return -1;
+
+ if (len > MAXDIST)
+ return -NLE_INVAL;
+
+ netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t));
+
+ size_t i;
+ for (i = 0; i < len; i++)
+ netem->qnm_dist.dist_data[i] = data[i];
+
+ netem->qnm_dist.dist_size = len;
+ netem->qnm_mask |= SCH_NETEM_ATTR_DIST;
+
+ return 0;
+}
+
+/** Customized version of rtnl_netem_set_delay_distribution() of libnl */
+static int set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json)
+{
+ if (json_is_string(json))
+ return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json));
+ else if (json_is_array(json)) {
+ json_t *elm;
+ size_t idx;
+ size_t len = json_array_size(json);
+
+ int16_t *data = new int16_t[len];
+ if (!data)
+ throw MemoryAllocationError();
+
+ json_array_foreach(json, idx, elm) {
+ if (!json_is_integer(elm))
+ return -1;
+
+ data[idx] = json_integer_value(elm);;
+ }
+
+ return rtnl_netem_set_delay_distribution_data(qdisc, data, len);
+ }
+
+ return 0;
+}
+
+int villas::kernel::tc::netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
{
int ret, val;
@@ -75,7 +130,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
rtnl_tc_set_kind(TC_CAST(ne), "netem");
if (json_delay_distribution) {
- if (tc_netem_set_delay_distribution(ne, json_delay_distribution))
+ if (set_delay_distribution(ne, json_delay_distribution))
error("Invalid delay distribution in netem config");
}
@@ -163,7 +218,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
return 0;
}
-char * tc_netem_print(struct rtnl_qdisc *ne)
+char * villas::kernel::tc::netem_print(struct rtnl_qdisc *ne)
{
char *buf = nullptr;
@@ -212,10 +267,10 @@ char * tc_netem_print(struct rtnl_qdisc *ne)
return buf;
}
-int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
+int villas::kernel::tc::netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
{
int ret;
- struct nl_sock *sock = nl_init();
+ struct nl_sock *sock = nl::init();
struct rtnl_qdisc *q = *qd;
ret = kernel::module_load("sch_netem");
@@ -235,57 +290,3 @@ int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hd
return ret;
}
-
-/**
- * Set the delay distribution. Latency/jitter must be set before applying.
- * @arg qdisc Netem qdisc.
- * @return 0 on success, error code on failure.
- */
-int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len)
-{
- struct rtnl_netem *netem;
-
- if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc))))
- return -1;
-
- if (len > MAXDIST)
- return -NLE_INVAL;
-
- netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t));
-
- size_t i;
- for (i = 0; i < len; i++)
- netem->qnm_dist.dist_data[i] = data[i];
-
- netem->qnm_dist.dist_size = len;
- netem->qnm_mask |= SCH_NETEM_ATTR_DIST;
-
- return 0;
-}
-
-/** Customized version of rtnl_netem_set_delay_distribution() of libnl */
-int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json)
-{
- if (json_is_string(json))
- return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json));
- else if (json_is_array(json)) {
- json_t *elm;
- size_t idx;
- size_t len = json_array_size(json);
-
- int16_t *data = new int16_t[len];
- if (!data)
- throw MemoryAllocationError();
-
- json_array_foreach(json, idx, elm) {
- if (!json_is_integer(elm))
- return -1;
-
- data[idx] = json_integer_value(elm);;
- }
-
- return rtnl_netem_set_delay_distribution_data(qdisc, data, len);
- }
-
- return 0;
-}
diff --git a/lib/node.cpp b/lib/node.cpp
index 13c6dec0b..230721e2a 100644
--- a/lib/node.cpp
+++ b/lib/node.cpp
@@ -40,10 +40,10 @@
#include <villas/memory.h>
#ifdef WITH_NETEM
- #include <villas/kernel/if.h>
- #include <villas/kernel/nl.h>
- #include <villas/kernel/tc.h>
- #include <villas/kernel/tc_netem.h>
+ #include <villas/kernel/if.hpp>
+ #include <villas/kernel/nl.hpp>
+ #include <villas/kernel/tc.hpp>
+ #include <villas/kernel/tc_netem.hpp>
#endif /* WITH_NETEM */
using namespace villas;
@@ -184,7 +184,7 @@ int node_parse(struct vnode *n, json_t *json, const char *name)
return ret;
if (enabled)
- tc_netem_parse(&n->tc_qdisc, json_netem);
+ kernel::tc::netem_parse(&n->tc_qdisc, json_netem);
else
n->tc_qdisc = nullptr;
#endif /* WITH_NETEM */
diff --git a/lib/nodes/rtp.cpp b/lib/nodes/rtp.cpp
index dec6d596e..b8735b9d0 100644
--- a/lib/nodes/rtp.cpp
+++ b/lib/nodes/rtp.cpp
@@ -48,14 +48,15 @@ extern "C" {
#include <villas/super_node.hpp>
#ifdef WITH_NETEM
- #include <villas/kernel/if.h>
+ #include <villas/kernel/if.hpp>
#endif /* WITH_NETEM */
static pthread_t re_pthread;
using namespace villas;
-using namespace villas::node;
using namespace villas::utils;
+using namespace villas::node;
+using namespace villas::kernel;
static struct plugin p;
@@ -491,20 +492,18 @@ int rtp_type_start(villas::node::SuperNode *sn)
return ret;
#ifdef WITH_NETEM
- struct vlist *interfaces = sn->getInterfaces();
-
/* Gather list of used network interfaces */
for (size_t i = 0; i < vlist_length(&p.node.instances); i++) {
struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i);
struct rtp *r = (struct rtp *) n->_vd;
- struct interface *j = if_get_egress(&r->out.saddr_rtp.u.sa, interfaces);
+ Interface *j = Interface::getEgress(&r->out.saddr_rtp.u.sa, sn);
if (!j) {
r->logger->error("Failed to find egress interface for node: {}", node_name(n));
return -1;
}
- vlist_push(&j->nodes, n);
+ j->addNode(n);
}
#endif /* WITH_NETEM */
diff --git a/lib/nodes/socket.cpp b/lib/nodes/socket.cpp
index 47657acbc..ccfcd5d6a 100644
--- a/lib/nodes/socket.cpp
+++ b/lib/nodes/socket.cpp
@@ -40,22 +40,21 @@
#endif /* WITH_SOCKET_LAYER_ETH */
#ifdef WITH_NETEM
- #include <villas/kernel/if.h>
- #include <villas/kernel/nl.h>
+ #include <villas/kernel/if.hpp>
+ #include <villas/kernel/nl.hpp>
#endif /* WITH_NETEM */
/* Forward declartions */
static struct plugin p;
using namespace villas;
-using namespace villas::node;
using namespace villas::utils;
+using namespace villas::node;
+using namespace villas::kernel;
int socket_type_start(villas::node::SuperNode *sn)
{
#ifdef WITH_NETEM
- struct vlist *interfaces = sn->getInterfaces();
-
/* Gather list of used network interfaces */
for (size_t i = 0; i < vlist_length(&p.node.instances); i++) {
struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i);
@@ -65,9 +64,9 @@ int socket_type_start(villas::node::SuperNode *sn)
continue;
/* Determine outgoing interface */
- struct interface *j = if_get_egress((struct sockaddr *) &s->out.saddr, interfaces);
+ Interface *j = Interface::getEgress((struct sockaddr *) &s->out.saddr, sn);
- vlist_push(&j->nodes, n);
+ j->addNode(n);
}
#endif /* WITH_NETEM */
diff --git a/lib/signal_list.cpp b/lib/signal_list.cpp
index 13577cf41..0cceef672 100644
--- a/lib/signal_list.cpp
+++ b/lib/signal_list.cpp
@@ -151,7 +151,7 @@ void signal_list_dump(const struct vlist *list, const union signal_data *data, u
strcatf(&buf, " = %s", val);
}
- info("%s", buf);
+ debug(5, "%s", buf);
free(buf);
}
}
diff --git a/lib/socket_addr.cpp b/lib/socket_addr.cpp
index 56f60a5d8..b2995dbe3 100644
--- a/lib/socket_addr.cpp
+++ b/lib/socket_addr.cpp
@@ -30,7 +30,7 @@
#include <villas/exceptions.hpp>
#ifdef WITH_SOCKET_LAYER_ETH
- #include <villas/kernel/nl.h>
+ #include <villas/kernel/nl.hpp>
#endif /* WITH_SOCKET_LAYER_ETH */
using namespace villas;
@@ -126,7 +126,8 @@ int socket_parse_address(const char *addr, struct sockaddr *saddr, enum SocketLa
memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, ETHER_ADDR_LEN);
/* Get interface index from name */
- nl_init();
+ kernel::nl::init();
+
struct nl_cache *cache = nl_cache_mngt_require("route/link");
struct rtnl_link *link = rtnl_link_get_by_name(cache, ifname);
if (!link)
diff --git a/lib/super_node.cpp b/lib/super_node.cpp
index b86018f8b..34f2aae13 100644
--- a/lib/super_node.cpp
+++ b/lib/super_node.cpp
@@ -36,10 +36,10 @@
#include <villas/log.hpp>
#include <villas/node/exceptions.hpp>
#include <villas/kernel/rt.hpp>
-#include <villas/kernel/if.h>
+#include <villas/kernel/if.hpp>
#ifdef WITH_NETEM
- #include <villas/kernel/nl.h>
+ #include <villas/kernel/nl.hpp>
#endif
using namespace villas;
@@ -75,12 +75,8 @@ SuperNode::SuperNode() :
if (ret)
throw RuntimeError("Failed to initialize list");
- ret = vlist_init(&interfaces);
- if (ret)
- throw RuntimeError("Failed to initialize list");
-
#ifdef WITH_NETEM
- nl_init(); /* Fill link cache */
+ kernel::nl::init(); /* Fill link cache */
#endif /* WITH_NETEM */
char hname[128];
@@ -270,12 +266,10 @@ void SuperNode::startInterfaces()
#ifdef WITH_NETEM
int ret;
- for (size_t i = 0; i < vlist_length(&interfaces); i++) {
- auto *j = (struct interface *) vlist_at(&interfaces, i);
-
- ret = if_start(j);
+ for (auto *i : interfaces) {
+ ret = i->start();
if (ret)
- throw RuntimeError("Failed to initialize network interface: {}", if_name(j));
+ throw RuntimeError("Failed to start network interface: {}", i->getName());
}
#endif /* WITH_NETEM */
}
@@ -363,7 +357,7 @@ void SuperNode::prepare()
auto *n = (struct vnode *) vlist_at(&nodes, i);
if (vlist_length(&n->sources) == 0 &&
vlist_length(&n->destinations) == 0) {
- logger->info("Node {} is not used by any path. Disabling...");
+ logger->info("Node {} is not used by any path. Disabling...", node_name(n));
n->enabled = false;
}
}
@@ -442,12 +436,10 @@ void SuperNode::stopInterfaces()
#ifdef WITH_NETEM
int ret;
- for (size_t j = 0; j < vlist_length(&interfaces); j++) {
- struct interface *i = (struct interface *) vlist_at(&interfaces, j);
-
- ret = if_stop(i);
+ for (auto *i : interfaces) {
+ ret = i->stop();
if (ret)
- throw RuntimeError("Failed to stop interface: {}", if_name(i));
+ throw RuntimeError("Failed to stop interface: {}", i->getName());
}
#endif /* WITH_NETEM */
}
@@ -491,9 +483,6 @@ SuperNode::~SuperNode()
ret = vlist_destroy(&paths, (dtor_cb_t) path_destroy, true);
ret = vlist_destroy(&nodes, (dtor_cb_t) node_destroy, true);
-#ifdef WITH_NETEM
- ret = vlist_destroy(&interfaces, (dtor_cb_t) if_destroy, true);
-#endif /* WITH_NETEM */
}
int SuperNode::periodic()