From e3ce379c3628615f63fce14d5fa1b4cf4b097610 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 13 Sep 2020 11:01:20 +0200 Subject: [PATCH] if: rewrote libnl / network if code to C++ and fixed if affinity (closes #233) --- common | 2 +- include/villas/kernel/if.h | 133 ----------- include/villas/kernel/if.hpp | 141 ++++++++++++ include/villas/kernel/{nl.h => nl.hpp} | 26 ++- include/villas/kernel/{tc.h => tc.hpp} | 20 +- .../kernel/{tc_netem.h => tc_netem.hpp} | 22 +- include/villas/super_node.hpp | 29 ++- lib/kernel/if.cpp | 211 +++++++----------- lib/kernel/nl.cpp | 46 +++- lib/kernel/tc.cpp | 19 +- lib/kernel/tc_netem.cpp | 125 ++++++----- lib/node.cpp | 10 +- lib/nodes/rtp.cpp | 11 +- lib/nodes/socket.cpp | 13 +- lib/signal_list.cpp | 2 +- lib/socket_addr.cpp | 5 +- lib/super_node.cpp | 31 +-- 17 files changed, 441 insertions(+), 405 deletions(-) delete mode 100644 include/villas/kernel/if.h create mode 100644 include/villas/kernel/if.hpp rename include/villas/kernel/{nl.h => nl.hpp} (69%) rename include/villas/kernel/{tc.h => tc.hpp} (86%) rename include/villas/kernel/{tc_netem.h => tc_netem.hpp} (84%) diff --git a/common b/common index f0059871a..dfd8bb232 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit f0059871af74e121b2d091cb445ba0b048ba675b +Subproject commit dfd8bb23202c554b74d622dff40fb1bf16ced6c8 diff --git a/include/villas/kernel/if.h b/include/villas/kernel/if.h deleted file mode 100644 index f2bfd4610..000000000 --- a/include/villas/kernel/if.h +++ /dev/null @@ -1,133 +0,0 @@ -/** Interface related functions - * - * These functions are used to manage a network interface. - * Most of them make use of Linux-specific APIs. - * - * @file - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup kernel Kernel - * @{ - */ - -#pragma once - -#include -#include - -#include - -#define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */ - -#ifndef SO_MARK - #define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */ -#endif - -struct socket; -struct nl_addr; -struct rtnl_link; - -/** Interface data structure */ -struct interface { - struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */ - struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */ - - char irqs[IF_IRQ_MAX]; /**< List of IRQs of the NIC. */ - int affinity; /**< IRQ / Core Affinity of this interface. */ - - struct vlist nodes; /**< Linked list of nodes which use this interface. */ -}; - -/** Add a new interface to the global list and lookup name, irqs... - * - * @param link The libnl3 link handle - * @retval >0 Success. A pointer to the new interface. - * @retval 0 Error. The creation failed. - */ -int if_init(struct interface * , struct rtnl_link *link) __attribute__ ((warn_unused_result)); - -/** Get name of interface */ -const char * if_name(struct interface *); - -/** Destroy interface by freeing dynamically allocated memory. - * - * @param i A pointer to the interface structure. - */ -int if_destroy(struct interface *i) __attribute__ ((warn_unused_result)); - -/** Start interface. - * - * This setups traffic controls queue discs, network emulation and - * maps interface IRQs according to affinity. - * - * @param i A pointer to the interface structure. - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int if_start(struct interface *i); - -/** Stop interface - * - * This resets traffic qdiscs ant network emulation - * and maps interface IRQs to all CPUs. - * - * @param i A pointer to the interface structure. - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int if_stop(struct interface *i); - -/** Find existing or create new interface instance on which packets for a certain destination - * will leave the system. - */ -struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces); - -/** Lookup routing tables to get the interface on which packets for a certain destination - * will leave the system. - * - * @param[in] sa The destination address for outgoing packets. - * @param[out] link The egress interface. - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -struct rtnl_link * if_get_egress_link(struct sockaddr *sa); - -/** Get all IRQs for this interface. - * - * Only MSI IRQs are determined by looking at: - * /sys/class/net/{ifname}/device/msi_irqs/ - * - * @param i A pointer to the interface structure - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int if_get_irqs(struct interface *i); - -/** Change the SMP affinity of NIC interrupts. - * - * @param i A pointer to the interface structure - * @param affinity A mask specifying which cores should handle this interrupt. - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int if_set_affinity(struct interface *i, int affinity); - -/** @} */ diff --git a/include/villas/kernel/if.hpp b/include/villas/kernel/if.hpp new file mode 100644 index 000000000..493f521f9 --- /dev/null +++ b/include/villas/kernel/if.hpp @@ -0,0 +1,141 @@ +/** Interface related functions + * + * These functions are used to manage a network interface. + * Most of them make use of Linux-specific APIs. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup kernel Kernel + * @{ + */ + +#pragma once + +#include + +#include +#include + +#include + +#ifndef SO_MARK + #define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */ +#endif + +/* Forward declarations */ +struct vnode; +struct nl_addr; +struct rtnl_link; +struct rtnl_qdisc; + +namespace villas { + +namespace node { + +/* Forward declarations */ +class SuperNode; + +} + +namespace kernel { + +/** Interface data structure */ +class Interface { + +public: + struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */ + struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */ + +protected: + int affinity; /**< IRQ / Core Affinity of this interface. */ + + std::list irqs; /**< List of IRQs of the NIC. */ + std::list nodes; /**< List of nodes which use this interface. */ + + Logger logger; + +public: + /** Add a new interface to the global list and lookup name, irqs... */ + Interface(struct rtnl_link *link, int affinity = 0); + ~Interface(); + + /** Start interface. + * + * This setups traffic controls queue discs, network emulation and + * maps interface IRQs according to affinity. + * + * @param i A pointer to the interface structure. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ + int start(); + + /** Stop interface + * + * This resets traffic qdiscs ant network emulation + * and maps interface IRQs to all CPUs. + * + * @param i A pointer to the interface structure. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ + int stop(); + + /** Find existing or create new interface instance on which packets for a certain destination + * will leave the system. + */ + static + Interface * + getEgress(struct sockaddr *sa, villas::node::SuperNode *sn); + + /** Get all IRQs for this interface. + * + * Only MSI IRQs are determined by looking at: + * /sys/class/net/{ifname}/device/msi_irqs/ + * + * @param i A pointer to the interface structure + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ + int getIRQs(); + + /** Change the SMP affinity of NIC interrupts. + * + * @param i A pointer to the interface structure + * @param affinity A mask specifying which cores should handle this interrupt. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ + int setAffinity(int affinity); + + std::string getName() const; + + void addNode(struct vnode *n) + { + nodes.push_back(n); + } +}; + +} /* namespace kernel */ +} /* namespace villas */ + +/** @} */ diff --git a/include/villas/kernel/nl.h b/include/villas/kernel/nl.hpp similarity index 69% rename from include/villas/kernel/nl.h rename to include/villas/kernel/nl.hpp index 3bec7cc5c..b33ec1a31 100644 --- a/include/villas/kernel/nl.h +++ b/include/villas/kernel/nl.hpp @@ -27,21 +27,41 @@ #pragma once +#include + #include #include #include +namespace villas { +namespace kernel { +namespace nl { + /** Get index of outgoing interface for given destination address. * * @retval >=0 Interface index of outgoing interface. * @retval <0 Error. Something went wrong. */ -int nl_get_egress(struct nl_addr *addr); +int get_egress(struct nl_addr *addr); + +/** Lookup routing tables to get the interface on which packets for a certain destination + * will leave the system. + * + * @param[in] sa The destination address for outgoing packets. + * @param[out] link The egress interface. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +struct rtnl_link * get_egress_link(struct sockaddr *sa); /** Get or create global netlink socket. */ -struct nl_sock *nl_init(); +struct nl_sock * init(); /** Close and free global netlink socket. */ -void nl_shutdown(); +void shutdown(); + +} /* namespace nl */ +} /* namespace kernel */ +} /* namespace villas */ /** @} */ diff --git a/include/villas/kernel/tc.h b/include/villas/kernel/tc.hpp similarity index 86% rename from include/villas/kernel/tc.h rename to include/villas/kernel/tc.hpp index 9fc6dd9b1..c465989bd 100644 --- a/include/villas/kernel/tc.h +++ b/include/villas/kernel/tc.hpp @@ -39,9 +39,15 @@ #include -typedef uint32_t tc_hdl_t; +namespace villas { +namespace kernel { -struct interface; +/* Forward declarations */ +class Interface; + +namespace tc { + +typedef uint32_t tc_hdl_t; /** Remove all queuing disciplines and filters. * @@ -49,7 +55,7 @@ struct interface; * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_reset(struct interface *i); +int reset(Interface *i); /** Create a priority (prio) queueing discipline. * @@ -61,7 +67,7 @@ int tc_reset(struct interface *i); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands); +int prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands); /** Add a new filter based on the netfilter mark. * @@ -72,6 +78,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark); +int mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark); + +} /* namespace tc */ +} /* namespace kernel */ +} /* namespace villas */ /** @} */ diff --git a/include/villas/kernel/tc_netem.h b/include/villas/kernel/tc_netem.hpp similarity index 84% rename from include/villas/kernel/tc_netem.h rename to include/villas/kernel/tc_netem.hpp index 9239ac649..931501c05 100644 --- a/include/villas/kernel/tc_netem.h +++ b/include/villas/kernel/tc_netem.hpp @@ -39,9 +39,15 @@ #include -typedef uint32_t tc_hdl_t; +namespace villas { +namespace kernel { -struct interface; +/* Forward declarations */ +class Interface; + +namespace tc { + +typedef uint32_t tc_hdl_t; /** Parse network emulator (netem) settings. * @@ -50,14 +56,14 @@ struct interface; * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_netem_parse(struct rtnl_qdisc **ne, json_t *cfg); +int netem_parse(struct rtnl_qdisc **ne, json_t *cfg); /** Print network emulator (netem) setting into buffer. * * @param tc A pointer to the libnl3 qdisc object where settings will be read from. * @return A pointer to a string which must be freed() by the caller. */ -char * tc_netem_print(struct rtnl_qdisc *ne); +char * netem_print(struct rtnl_qdisc *ne); /** Add a new network emulator (netem) discipline. * @@ -68,8 +74,12 @@ char * tc_netem_print(struct rtnl_qdisc *ne); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); +int netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); -int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json); +int netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json); + +} /* namespace tc */ +} /* namespace kernel */ +} /* namespace villas */ /** @} */ diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index e64b855c1..9ccda1c2d 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -41,6 +41,7 @@ extern "C" { #include #include #include +#include /* Forward declarations */ struct vnode; @@ -60,7 +61,7 @@ protected: struct vlist nodes; struct vlist paths; - struct vlist interfaces; + std::list interfaces; #ifdef WITH_API Api api; @@ -141,26 +142,31 @@ public: return &nodes; } - struct vlist * getPaths() { + struct vlist * getPaths() + { return &paths; } - struct vlist * getInterfaces() { - return &interfaces; + std::list & getInterfaces() + { + return interfaces; } - enum State getState() { + enum State getState() const + { return state; } #ifdef WITH_API - Api * getApi() { + Api * getApi() + { return &api; } #endif #ifdef WITH_WEB - Web * getWeb() { + Web * getWeb() + { return &web; } #endif @@ -170,16 +176,21 @@ public: return config.root; } - std::string getConfigUri() + std::string getConfigUri() const { return uri; } - std::string getName() + std::string getName() const { return name; } + int getAffinity() const + { + return affinity; + } + /** Destroy configuration object. */ ~SuperNode(); }; diff --git a/lib/kernel/if.cpp b/lib/kernel/if.cpp index c024cc1e1..825759601 100644 --- a/lib/kernel/if.cpp +++ b/lib/kernel/if.cpp @@ -23,72 +23,59 @@ #include #include #include -#include #include #include #include #include +#include +#include -#include -#include -#include -#include +#include +#include +#include +#include #include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; +using namespace villas::kernel; -int if_init(struct interface *i, struct rtnl_link *link) +Interface::Interface(struct rtnl_link *link, int aff) : + nl_link(link), + tc_qdisc(nullptr), + affinity(aff) { - int ret; + logger = logging.get(fmt::format("kernel:if:{}", getName())); - i->nl_link = link; + int n = getIRQs(); + if (n) + logger->warn("Did not found any interrupts"); - debug(LOG_IF | 3, "Created interface '%s'", if_name(i)); - - int n = if_get_irqs(i); - if (n > 0) - debug(6, "Found %u IRQs for interface '%s'", n, if_name(i)); - else - warning("Did not found any interrupts for interface '%s'", if_name(i)); - - ret = vlist_init(&i->nodes); - if (ret) - return ret; - - return 0; + logger->debug("Found {} IRQs", irqs.size()); } -int if_destroy(struct interface *i) +Interface::~Interface() { - int ret; - - /* List members are freed by the nodes they belong to. */ - ret = vlist_destroy(&i->nodes); - if (ret) - return ret; - - rtnl_qdisc_put(i->tc_qdisc); - - return 0; + if (tc_qdisc) + rtnl_qdisc_put(tc_qdisc); } -int if_start(struct interface *i) +int Interface::start() { - info("Starting interface '%s' which is used by %zu nodes", if_name(i), vlist_length(&i->nodes)); + logger->info("Starting interface which is used by {} nodes", nodes.size()); /* Set affinity for network interfaces (skip _loopback_ dev) */ - //if_set_affinity(i, i->affinity); + if (affinity) + setAffinity(affinity); /* Assign fwmark's to nodes which have netem options */ int ret, fwmark = 0; - for (size_t j = 0; j < vlist_length(&i->nodes); j++) { - struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j); - + for (auto *n : nodes) { if (n->tc_qdisc && n->fwmark < 0) n->fwmark = 1 + fwmark++; } @@ -98,141 +85,97 @@ int if_start(struct interface *i) return 0; if (getuid() != 0) - error("Network emulation requires super-user privileges!"); + throw RuntimeError("Network emulation requires super-user privileges!"); /* Replace root qdisc */ - ret = tc_prio(i, &i->tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark); + ret = tc::prio(this, &tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark); if (ret) - error("Failed to setup priority queuing discipline: %s", nl_geterror(ret)); + throw RuntimeError("Failed to setup priority queuing discipline: {}", nl_geterror(ret)); /* Create netem qdisks and appropriate filter per netem node */ - for (size_t j = 0; j < vlist_length(&i->nodes); j++) { - struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j); - + for (auto *n : nodes) { if (n->tc_qdisc) { - ret = tc_mark(i, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark); + ret = tc::mark(this, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark); if (ret) - error("Failed to setup FW mark classifier: %s", nl_geterror(ret)); + throw RuntimeError("Failed to setup FW mark classifier: {}", nl_geterror(ret)); - char *buf = tc_netem_print(n->tc_qdisc); - debug(LOG_IF | 5, "Starting network emulation on interface '%s' for FW mark %u: %s", - if_name(i), n->fwmark, buf); + char *buf = tc::netem_print(n->tc_qdisc); + logger->debug("Starting network emulation for FW mark {}: {}", n->fwmark, buf); free(buf); - ret = tc_netem(i, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark)); + ret = tc::netem(this, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark)); if (ret) - error("Failed to setup netem qdisc: %s", nl_geterror(ret)); + throw RuntimeError("Failed to setup netem qdisc: {}", nl_geterror(ret)); } } return 0; } -int if_stop(struct interface *i) +int Interface::stop() { - info("Stopping interface '%s'", if_name(i)); + logger->info("Stopping interface"); - //if_set_affinity(i, -1L); + if (affinity) + setAffinity(-1L); - if (i->tc_qdisc) - tc_reset(i); + if (tc_qdisc) + tc::reset(this); return 0; } -const char * if_name(struct interface *i) +std::string Interface::getName() const { - return rtnl_link_get_name(i->nl_link); + auto str = rtnl_link_get_name(nl_link); + + return std::string(str); } -struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces) +Interface * Interface::getEgress(struct sockaddr *sa, SuperNode *sn) { - int ret; struct rtnl_link *link; - /* Determine outgoing interface */ - link = if_get_egress_link(sa); - if (!link) { - char *buf = socket_print_addr(sa); - error("Failed to get interface for socket address '%s'", buf); - free(buf); + Logger logger = logging.get("kernel:if"); - return nullptr; - } + auto & interfaces = sn->getInterfaces(); + auto affinity = sn->getAffinity(); + + /* Determine outgoing interface */ + link = nl::get_egress_link(sa); + if (!link) + throw RuntimeError("Failed to get interface for socket address '{}'", socket_print_addr(sa)); /* Search of existing interface with correct ifindex */ - struct interface *i; - for (size_t k = 0; k < vlist_length(interfaces); k++) { - i = (struct interface *) vlist_at(interfaces, k); - + for (auto *i : interfaces) { if (rtnl_link_get_ifindex(i->nl_link) == rtnl_link_get_ifindex(link)) return i; } /* If not found, create a new interface */ - i = new struct interface; + auto *i = new Interface(link, affinity); if (!i) throw MemoryAllocationError(); - memset(i, 0, sizeof(struct interface)); - - ret = if_init(i, link); - if (ret) - return nullptr; - - vlist_push(interfaces, i); + interfaces.push_back(i); return i; } -struct rtnl_link * if_get_egress_link(struct sockaddr *sa) +int Interface::getIRQs() { - int ifindex = -1; + int irq; - switch (sa->sa_family) { - case AF_INET: - case AF_INET6: { - struct sockaddr_in *sin = (struct sockaddr_in *) sa; - struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa; - - struct nl_addr *addr = (sa->sa_family == AF_INET) - ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr)) - : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr)); - - ifindex = nl_get_egress(addr); nl_addr_put(addr); - if (ifindex < 0) - error("Netlink error: %s", nl_geterror(ifindex)); - break; - } - - case AF_PACKET: { - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - - ifindex = sll->sll_ifindex; - break; - } - } - - struct nl_cache *cache = nl_cache_mngt_require("route/link"); - - return rtnl_link_get(cache, ifindex); -} - -int if_get_irqs(struct interface *i) -{ - char dirname[NAME_MAX]; - int irq, n = 0; - - snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", if_name(i)); - DIR *dir = opendir(dirname); + auto dirname = fmt::format("/sys/class/net/{}/device/msi_irqs/", getName()); + DIR *dir = opendir(dirname.c_str()); if (dir) { - memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX); + irqs.clear(); struct dirent *entry; - while ((entry = readdir(dir)) && n < IF_IRQ_MAX) { + while ((entry = readdir(dir))) { irq = atoi(entry->d_name); if (irq) - i->irqs[n++] = irq; + irqs.push_back(irq); } closedir(dir); @@ -241,24 +184,32 @@ int if_get_irqs(struct interface *i) return 0; } -int if_set_affinity(struct interface *i, int affinity) +int Interface::setAffinity(int affinity) { - char filename[NAME_MAX]; + assert(affinity != 0); + + if (getuid() != 0) { + logger->warn("Failed to tune IRQ affinity. Please run as super-user"); + return 0; + } + FILE *file; - for (int n = 0; n < IF_IRQ_MAX && i->irqs[n]; n++) { - snprintf(filename, sizeof(filename), "/proc/irq/%d/smp_affinity", (int) i->irqs[n]); + CpuSet cset_pin(affinity); - file = fopen(filename, "w"); + for (int irq : irqs) { + std::string filename = fmt::format("/proc/irq/{}/smp_affinity", irq); + + file = fopen(filename.c_str(), "w"); if (file) { - if (fprintf(file, "%8x", affinity) < 0) - error("Failed to set affinity for IRQ %u", i->irqs[n]); + if (fprintf(file, "%8lx", (unsigned long) cset_pin) < 0) + throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName()); fclose(file); - debug(LOG_IF | 5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], if_name(i), affinity); + logger->debug("Set affinity of IRQ {} to {} {}", irq, cset_pin.count() == 1 ? "core" : "cores", (std::string) cset_pin); } else - error("Failed to set affinity for interface '%s'", if_name(i)); + throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName()); } return 0; diff --git a/lib/kernel/nl.cpp b/lib/kernel/nl.cpp index a0f42e806..1e8d04afe 100644 --- a/lib/kernel/nl.cpp +++ b/lib/kernel/nl.cpp @@ -24,19 +24,22 @@ #include +#include + #include #include #include #include -#include +#include /** Singleton for global netlink socket */ static struct nl_sock *sock = nullptr; using namespace villas; +using namespace villas::kernel::nl; -struct nl_sock * nl_init() +struct nl_sock * villas::kernel::nl::init() { int ret; @@ -62,7 +65,7 @@ struct nl_sock * nl_init() return sock; } -void nl_shutdown() +void villas::kernel::nl::shutdown() { nl_close(sock); nl_socket_free(sock); @@ -80,10 +83,10 @@ static int egress_cb(struct nl_msg *msg, void *arg) return NL_STOP; } -int nl_get_egress(struct nl_addr *addr) +int villas::kernel::nl::get_egress(struct nl_addr *addr) { int ret; - struct nl_sock *sock = nl_init(); + struct nl_sock *sock = nl::init(); struct nl_cb *cb; struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0); struct rtnl_route *route = nullptr; @@ -130,3 +133,36 @@ out: nlmsg_free(msg); return ret; } + +struct rtnl_link * villas::kernel::nl::get_egress_link(struct sockaddr *sa) +{ + int ifindex = -1; + + switch (sa->sa_family) { + case AF_INET: + case AF_INET6: { + struct sockaddr_in *sin = (struct sockaddr_in *) sa; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa; + + struct nl_addr *addr = (sa->sa_family == AF_INET) + ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr)) + : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr)); + + ifindex = nl::get_egress(addr); nl_addr_put(addr); + if (ifindex < 0) + error("Netlink error: %s", nl_geterror(ifindex)); + break; + } + + case AF_PACKET: { + struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; + + ifindex = sll->sll_ifindex; + break; + } + } + + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + + return rtnl_link_get(cache, ifindex); +} diff --git a/lib/kernel/tc.cpp b/lib/kernel/tc.cpp index f0e096ade..7e34be0ae 100644 --- a/lib/kernel/tc.cpp +++ b/lib/kernel/tc.cpp @@ -30,16 +30,17 @@ #include #include -#include -#include -#include +#include +#include +#include using namespace villas; +using namespace villas::kernel; -int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands) +int villas::kernel::tc::prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands) { int ret; - struct nl_sock *sock = nl_init(); + struct nl_sock *sock = nl::init(); struct rtnl_qdisc *q = rtnl_qdisc_alloc(); ret = kernel::module_load("sch_prio"); @@ -69,10 +70,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl return ret; } -int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark) +int villas::kernel::tc::mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark) { int ret; - struct nl_sock *sock = nl_init(); + struct nl_sock *sock = nl::init(); struct rtnl_cls *c = rtnl_cls_alloc(); ret = kernel::module_load("cls_fw"); @@ -97,9 +98,9 @@ int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_ return ret; } -int tc_reset(struct interface *i) +int villas::kernel::tc::reset(Interface *i) { - struct nl_sock *sock = nl_init(); + struct nl_sock *sock = nl::init(); /* We restore the default pfifo_fast qdisc, by deleting ours */ return rtnl_qdisc_delete(sock, i->tc_qdisc); diff --git a/lib/kernel/tc_netem.cpp b/lib/kernel/tc_netem.cpp index 15c0be5f1..16df90370 100644 --- a/lib/kernel/tc_netem.cpp +++ b/lib/kernel/tc_netem.cpp @@ -27,20 +27,75 @@ #include -#include -#include +#include +#include #include -#include +#include #include #include #include using namespace villas; using namespace villas::utils; +using namespace villas::kernel; static const double max_percent_value = 0xffffffff; -int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg) +/** + * Set the delay distribution. Latency/jitter must be set before applying. + * @arg qdisc Netem qdisc. + * @return 0 on success, error code on failure. + */ +static int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len) +{ + struct rtnl_netem *netem; + + if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc)))) + return -1; + + if (len > MAXDIST) + return -NLE_INVAL; + + netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t)); + + size_t i; + for (i = 0; i < len; i++) + netem->qnm_dist.dist_data[i] = data[i]; + + netem->qnm_dist.dist_size = len; + netem->qnm_mask |= SCH_NETEM_ATTR_DIST; + + return 0; +} + +/** Customized version of rtnl_netem_set_delay_distribution() of libnl */ +static int set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json) +{ + if (json_is_string(json)) + return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json)); + else if (json_is_array(json)) { + json_t *elm; + size_t idx; + size_t len = json_array_size(json); + + int16_t *data = new int16_t[len]; + if (!data) + throw MemoryAllocationError(); + + json_array_foreach(json, idx, elm) { + if (!json_is_integer(elm)) + return -1; + + data[idx] = json_integer_value(elm);; + } + + return rtnl_netem_set_delay_distribution_data(qdisc, data, len); + } + + return 0; +} + +int villas::kernel::tc::netem_parse(struct rtnl_qdisc **netem, json_t *cfg) { int ret, val; @@ -75,7 +130,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg) rtnl_tc_set_kind(TC_CAST(ne), "netem"); if (json_delay_distribution) { - if (tc_netem_set_delay_distribution(ne, json_delay_distribution)) + if (set_delay_distribution(ne, json_delay_distribution)) error("Invalid delay distribution in netem config"); } @@ -163,7 +218,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg) return 0; } -char * tc_netem_print(struct rtnl_qdisc *ne) +char * villas::kernel::tc::netem_print(struct rtnl_qdisc *ne) { char *buf = nullptr; @@ -212,10 +267,10 @@ char * tc_netem_print(struct rtnl_qdisc *ne) return buf; } -int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) +int villas::kernel::tc::netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) { int ret; - struct nl_sock *sock = nl_init(); + struct nl_sock *sock = nl::init(); struct rtnl_qdisc *q = *qd; ret = kernel::module_load("sch_netem"); @@ -235,57 +290,3 @@ int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hd return ret; } - -/** - * Set the delay distribution. Latency/jitter must be set before applying. - * @arg qdisc Netem qdisc. - * @return 0 on success, error code on failure. - */ -int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len) -{ - struct rtnl_netem *netem; - - if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc)))) - return -1; - - if (len > MAXDIST) - return -NLE_INVAL; - - netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t)); - - size_t i; - for (i = 0; i < len; i++) - netem->qnm_dist.dist_data[i] = data[i]; - - netem->qnm_dist.dist_size = len; - netem->qnm_mask |= SCH_NETEM_ATTR_DIST; - - return 0; -} - -/** Customized version of rtnl_netem_set_delay_distribution() of libnl */ -int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json) -{ - if (json_is_string(json)) - return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json)); - else if (json_is_array(json)) { - json_t *elm; - size_t idx; - size_t len = json_array_size(json); - - int16_t *data = new int16_t[len]; - if (!data) - throw MemoryAllocationError(); - - json_array_foreach(json, idx, elm) { - if (!json_is_integer(elm)) - return -1; - - data[idx] = json_integer_value(elm);; - } - - return rtnl_netem_set_delay_distribution_data(qdisc, data, len); - } - - return 0; -} diff --git a/lib/node.cpp b/lib/node.cpp index 13c6dec0b..230721e2a 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -40,10 +40,10 @@ #include #ifdef WITH_NETEM - #include - #include - #include - #include + #include + #include + #include + #include #endif /* WITH_NETEM */ using namespace villas; @@ -184,7 +184,7 @@ int node_parse(struct vnode *n, json_t *json, const char *name) return ret; if (enabled) - tc_netem_parse(&n->tc_qdisc, json_netem); + kernel::tc::netem_parse(&n->tc_qdisc, json_netem); else n->tc_qdisc = nullptr; #endif /* WITH_NETEM */ diff --git a/lib/nodes/rtp.cpp b/lib/nodes/rtp.cpp index dec6d596e..b8735b9d0 100644 --- a/lib/nodes/rtp.cpp +++ b/lib/nodes/rtp.cpp @@ -48,14 +48,15 @@ extern "C" { #include #ifdef WITH_NETEM - #include + #include #endif /* WITH_NETEM */ static pthread_t re_pthread; using namespace villas; -using namespace villas::node; using namespace villas::utils; +using namespace villas::node; +using namespace villas::kernel; static struct plugin p; @@ -491,20 +492,18 @@ int rtp_type_start(villas::node::SuperNode *sn) return ret; #ifdef WITH_NETEM - struct vlist *interfaces = sn->getInterfaces(); - /* Gather list of used network interfaces */ for (size_t i = 0; i < vlist_length(&p.node.instances); i++) { struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i); struct rtp *r = (struct rtp *) n->_vd; - struct interface *j = if_get_egress(&r->out.saddr_rtp.u.sa, interfaces); + Interface *j = Interface::getEgress(&r->out.saddr_rtp.u.sa, sn); if (!j) { r->logger->error("Failed to find egress interface for node: {}", node_name(n)); return -1; } - vlist_push(&j->nodes, n); + j->addNode(n); } #endif /* WITH_NETEM */ diff --git a/lib/nodes/socket.cpp b/lib/nodes/socket.cpp index 47657acbc..ccfcd5d6a 100644 --- a/lib/nodes/socket.cpp +++ b/lib/nodes/socket.cpp @@ -40,22 +40,21 @@ #endif /* WITH_SOCKET_LAYER_ETH */ #ifdef WITH_NETEM - #include - #include + #include + #include #endif /* WITH_NETEM */ /* Forward declartions */ static struct plugin p; using namespace villas; -using namespace villas::node; using namespace villas::utils; +using namespace villas::node; +using namespace villas::kernel; int socket_type_start(villas::node::SuperNode *sn) { #ifdef WITH_NETEM - struct vlist *interfaces = sn->getInterfaces(); - /* Gather list of used network interfaces */ for (size_t i = 0; i < vlist_length(&p.node.instances); i++) { struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i); @@ -65,9 +64,9 @@ int socket_type_start(villas::node::SuperNode *sn) continue; /* Determine outgoing interface */ - struct interface *j = if_get_egress((struct sockaddr *) &s->out.saddr, interfaces); + Interface *j = Interface::getEgress((struct sockaddr *) &s->out.saddr, sn); - vlist_push(&j->nodes, n); + j->addNode(n); } #endif /* WITH_NETEM */ diff --git a/lib/signal_list.cpp b/lib/signal_list.cpp index 13577cf41..0cceef672 100644 --- a/lib/signal_list.cpp +++ b/lib/signal_list.cpp @@ -151,7 +151,7 @@ void signal_list_dump(const struct vlist *list, const union signal_data *data, u strcatf(&buf, " = %s", val); } - info("%s", buf); + debug(5, "%s", buf); free(buf); } } diff --git a/lib/socket_addr.cpp b/lib/socket_addr.cpp index 56f60a5d8..b2995dbe3 100644 --- a/lib/socket_addr.cpp +++ b/lib/socket_addr.cpp @@ -30,7 +30,7 @@ #include #ifdef WITH_SOCKET_LAYER_ETH - #include + #include #endif /* WITH_SOCKET_LAYER_ETH */ using namespace villas; @@ -126,7 +126,8 @@ int socket_parse_address(const char *addr, struct sockaddr *saddr, enum SocketLa memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, ETHER_ADDR_LEN); /* Get interface index from name */ - nl_init(); + kernel::nl::init(); + struct nl_cache *cache = nl_cache_mngt_require("route/link"); struct rtnl_link *link = rtnl_link_get_by_name(cache, ifname); if (!link) diff --git a/lib/super_node.cpp b/lib/super_node.cpp index b86018f8b..34f2aae13 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -36,10 +36,10 @@ #include #include #include -#include +#include #ifdef WITH_NETEM - #include + #include #endif using namespace villas; @@ -75,12 +75,8 @@ SuperNode::SuperNode() : if (ret) throw RuntimeError("Failed to initialize list"); - ret = vlist_init(&interfaces); - if (ret) - throw RuntimeError("Failed to initialize list"); - #ifdef WITH_NETEM - nl_init(); /* Fill link cache */ + kernel::nl::init(); /* Fill link cache */ #endif /* WITH_NETEM */ char hname[128]; @@ -270,12 +266,10 @@ void SuperNode::startInterfaces() #ifdef WITH_NETEM int ret; - for (size_t i = 0; i < vlist_length(&interfaces); i++) { - auto *j = (struct interface *) vlist_at(&interfaces, i); - - ret = if_start(j); + for (auto *i : interfaces) { + ret = i->start(); if (ret) - throw RuntimeError("Failed to initialize network interface: {}", if_name(j)); + throw RuntimeError("Failed to start network interface: {}", i->getName()); } #endif /* WITH_NETEM */ } @@ -363,7 +357,7 @@ void SuperNode::prepare() auto *n = (struct vnode *) vlist_at(&nodes, i); if (vlist_length(&n->sources) == 0 && vlist_length(&n->destinations) == 0) { - logger->info("Node {} is not used by any path. Disabling..."); + logger->info("Node {} is not used by any path. Disabling...", node_name(n)); n->enabled = false; } } @@ -442,12 +436,10 @@ void SuperNode::stopInterfaces() #ifdef WITH_NETEM int ret; - for (size_t j = 0; j < vlist_length(&interfaces); j++) { - struct interface *i = (struct interface *) vlist_at(&interfaces, j); - - ret = if_stop(i); + for (auto *i : interfaces) { + ret = i->stop(); if (ret) - throw RuntimeError("Failed to stop interface: {}", if_name(i)); + throw RuntimeError("Failed to stop interface: {}", i->getName()); } #endif /* WITH_NETEM */ } @@ -491,9 +483,6 @@ SuperNode::~SuperNode() ret = vlist_destroy(&paths, (dtor_cb_t) path_destroy, true); ret = vlist_destroy(&nodes, (dtor_cb_t) node_destroy, true); -#ifdef WITH_NETEM - ret = vlist_destroy(&interfaces, (dtor_cb_t) if_destroy, true); -#endif /* WITH_NETEM */ } int SuperNode::periodic()