Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

if: rewrote libnl / network if code to C++ and fixed if affinity (closes #233)

Browse source
This commit is contained in:
Steffen Vogel 2020-09-13 11:01:20 +02:00
parent 6ddc34f028
commit e3ce379c36
17 changed files with 441 additions and 405 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common

@ -1 +1 @@
Subproject commit f0059871af74e121b2d091cb445ba0b048ba675b
Subproject commit dfd8bb23202c554b74d622dff40fb1bf16ced6c8

133
include/villas/kernel/if.h
View file

@ -1,133 +0,0 @@
/** Interface related functions
*
* These functions are used to manage a network interface.
* Most of them make use of Linux-specific APIs.
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
* @license GNU General Public License (version 3)
*
* VILLASnode
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
/** @addtogroup kernel Kernel
* @{
*/
#pragma once
#include <sys/types.h>
#include <sys/socket.h>
#include <villas/list.h>
#define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */
#ifndef SO_MARK
#define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */
#endif
struct socket;
struct nl_addr;
struct rtnl_link;
/** Interface data structure */
struct interface {
struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */
struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */
char irqs[IF_IRQ_MAX]; /**< List of IRQs of the NIC. */
int affinity; /**< IRQ / Core Affinity of this interface. */
struct vlist nodes; /**< Linked list of nodes which use this interface. */
};
/** Add a new interface to the global list and lookup name, irqs...
*
* @param link The libnl3 link handle
* @retval >0 Success. A pointer to the new interface.
* @retval 0 Error. The creation failed.
*/
int if_init(struct interface * , struct rtnl_link *link) __attribute__ ((warn_unused_result));
/** Get name of interface */
const char * if_name(struct interface *);
/** Destroy interface by freeing dynamically allocated memory.
*
* @param i A pointer to the interface structure.
*/
int if_destroy(struct interface *i) __attribute__ ((warn_unused_result));
/** Start interface.
*
* This setups traffic controls queue discs, network emulation and
* maps interface IRQs according to affinity.
*
* @param i A pointer to the interface structure.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_start(struct interface *i);
/** Stop interface
*
* This resets traffic qdiscs ant network emulation
* and maps interface IRQs to all CPUs.
*
* @param i A pointer to the interface structure.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_stop(struct interface *i);
/** Find existing or create new interface instance on which packets for a certain destination
* will leave the system.
*/
struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces);
/** Lookup routing tables to get the interface on which packets for a certain destination
* will leave the system.
*
* @param[in] sa The destination address for outgoing packets.
* @param[out] link The egress interface.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
struct rtnl_link * if_get_egress_link(struct sockaddr *sa);
/** Get all IRQs for this interface.
*
* Only MSI IRQs are determined by looking at:
* /sys/class/net/{ifname}/device/msi_irqs/
*
* @param i A pointer to the interface structure
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_get_irqs(struct interface *i);
/** Change the SMP affinity of NIC interrupts.
*
* @param i A pointer to the interface structure
* @param affinity A mask specifying which cores should handle this interrupt.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_set_affinity(struct interface *i, int affinity);
/** @} */

141
include/villas/kernel/if.hpp Normal file
View file

@ -0,0 +1,141 @@
/** Interface related functions
*
* These functions are used to manage a network interface.
* Most of them make use of Linux-specific APIs.
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
* @license GNU General Public License (version 3)
*
* VILLASnode
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
/** @addtogroup kernel Kernel
* @{
*/
#pragma once
#include <list>
#include <sys/types.h>
#include <sys/socket.h>
#include <villas/log.hpp>
#ifndef SO_MARK
#define SO_MARK 36 /**< Workaround: add missing constant for OPAL-RT Redhawk target */
#endif
/* Forward declarations */
struct vnode;
struct nl_addr;
struct rtnl_link;
struct rtnl_qdisc;
namespace villas {
namespace node {
/* Forward declarations */
class SuperNode;
}
namespace kernel {
/** Interface data structure */
class Interface {
public:
struct rtnl_link *nl_link; /**< libnl3: Handle of interface. */
struct rtnl_qdisc *tc_qdisc; /**< libnl3: Root priority queuing discipline (qdisc). */
protected:
int affinity; /**< IRQ / Core Affinity of this interface. */
std::list<int> irqs; /**< List of IRQs of the NIC. */
std::list<struct vnode *> nodes; /**< List of nodes which use this interface. */
Logger logger;
public:
/** Add a new interface to the global list and lookup name, irqs... */
Interface(struct rtnl_link *link, int affinity = 0);
~Interface();
/** Start interface.
*
* This setups traffic controls queue discs, network emulation and
* maps interface IRQs according to affinity.
*
* @param i A pointer to the interface structure.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int start();
/** Stop interface
*
* This resets traffic qdiscs ant network emulation
* and maps interface IRQs to all CPUs.
*
* @param i A pointer to the interface structure.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int stop();
/** Find existing or create new interface instance on which packets for a certain destination
* will leave the system.
*/
static
Interface *
getEgress(struct sockaddr *sa, villas::node::SuperNode *sn);
/** Get all IRQs for this interface.
*
* Only MSI IRQs are determined by looking at:
* /sys/class/net/{ifname}/device/msi_irqs/
*
* @param i A pointer to the interface structure
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int getIRQs();
/** Change the SMP affinity of NIC interrupts.
*
* @param i A pointer to the interface structure
* @param affinity A mask specifying which cores should handle this interrupt.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int setAffinity(int affinity);
std::string getName() const;
void addNode(struct vnode *n)
{
nodes.push_back(n);
}
};
} /* namespace kernel */
} /* namespace villas */
/** @} */

26
include/villas/kernel/nl.h → include/villas/kernel/nl.hpp
View file

@ -27,21 +27,41 @@
#pragma once
#include <sys/socket.h>
#include <netlink/netlink.h>
#include <netlink/route/route.h>
#include <netlink/route/link.h>
namespace villas {
namespace kernel {
namespace nl {
/** Get index of outgoing interface for given destination address.
*
* @retval >=0 Interface index of outgoing interface.
* @retval <0 Error. Something went wrong.
*/
int nl_get_egress(struct nl_addr *addr);
int get_egress(struct nl_addr *addr);
/** Lookup routing tables to get the interface on which packets for a certain destination
* will leave the system.
*
* @param[in] sa The destination address for outgoing packets.
* @param[out] link The egress interface.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
struct rtnl_link * get_egress_link(struct sockaddr *sa);
/** Get or create global netlink socket. */
struct nl_sock *nl_init();
struct nl_sock * init();
/** Close and free global netlink socket. */
void nl_shutdown();
void shutdown();
} /* namespace nl */
} /* namespace kernel */
} /* namespace villas */
/** @} */

20
include/villas/kernel/tc.h → include/villas/kernel/tc.hpp
View file

@ -39,9 +39,15 @@
#include <jansson.h>
typedef uint32_t tc_hdl_t;
namespace villas {
namespace kernel {
struct interface;
/* Forward declarations */
class Interface;
namespace tc {
typedef uint32_t tc_hdl_t;
/** Remove all queuing disciplines and filters.
*
@ -49,7 +55,7 @@ struct interface;
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_reset(struct interface *i);
int reset(Interface *i);
/** Create a priority (prio) queueing discipline.
*
@ -61,7 +67,7 @@ int tc_reset(struct interface *i);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands);
int prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands);
/** Add a new filter based on the netfilter mark.
*
@ -72,6 +78,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
int mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
} /* namespace tc */
} /* namespace kernel */
} /* namespace villas */
/** @} */

22
include/villas/kernel/tc_netem.h → include/villas/kernel/tc_netem.hpp
View file

@ -39,9 +39,15 @@
#include <jansson.h>
typedef uint32_t tc_hdl_t;
namespace villas {
namespace kernel {
struct interface;
/* Forward declarations */
class Interface;
namespace tc {
typedef uint32_t tc_hdl_t;
/** Parse network emulator (netem) settings.
*
@ -50,14 +56,14 @@ struct interface;
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_netem_parse(struct rtnl_qdisc **ne, json_t *cfg);
int netem_parse(struct rtnl_qdisc **ne, json_t *cfg);
/** Print network emulator (netem) setting into buffer.
*
* @param tc A pointer to the libnl3 qdisc object where settings will be read from.
* @return A pointer to a string which must be freed() by the caller.
*/
char * tc_netem_print(struct rtnl_qdisc *ne);
char * netem_print(struct rtnl_qdisc *ne);
/** Add a new network emulator (netem) discipline.
*
@ -68,8 +74,12 @@ char * tc_netem_print(struct rtnl_qdisc *ne);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
int netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json);
int netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json);
} /* namespace tc */
} /* namespace kernel */
} /* namespace villas */
/** @} */

29
include/villas/super_node.hpp
View file

@ -41,6 +41,7 @@ extern "C" {
#include <villas/node.h>
#include <villas/task.hpp>
#include <villas/common.hpp>
#include <villas/kernel/if.hpp>
/* Forward declarations */
struct vnode;
@ -60,7 +61,7 @@ protected:
struct vlist nodes;
struct vlist paths;
struct vlist interfaces;
std::list<kernel::Interface *> interfaces;
#ifdef WITH_API
Api api;
@ -141,26 +142,31 @@ public:
return &nodes;
}
struct vlist * getPaths() {
struct vlist * getPaths()
{
return &paths;
}
struct vlist * getInterfaces() {
return &interfaces;
std::list<kernel::Interface *> & getInterfaces()
{
return interfaces;
}
enum State getState() {
enum State getState() const
{
return state;
}
#ifdef WITH_API
Api * getApi() {
Api * getApi()
{
return &api;
}
#endif
#ifdef WITH_WEB
Web * getWeb() {
Web * getWeb()
{
return &web;
}
#endif
@ -170,16 +176,21 @@ public:
return config.root;
}
std::string getConfigUri()
std::string getConfigUri() const
{
return uri;
}
std::string getName()
std::string getName() const
{
return name;
}
int getAffinity() const
{
return affinity;
}
/** Destroy configuration object. */
~SuperNode();
};

211
lib/kernel/if.cpp
View file

@ -23,72 +23,59 @@
#include <cstdio>
#include <cstdlib>
#include <dirent.h>
#include <linux/if_packet.h>
#include <netlink/route/link.h>
#include <villas/node/config.h>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
#include <villas/super_node.hpp>
#include <villas/cpuset.hpp>
#include <villas/kernel/if.h>
#include <villas/kernel/tc.h>
#include <villas/kernel/tc_netem.h>
#include <villas/kernel/nl.h>
#include <villas/kernel/if.hpp>
#include <villas/kernel/tc.hpp>
#include <villas/kernel/tc_netem.hpp>
#include <villas/kernel/nl.hpp>
#include <villas/kernel/kernel.hpp>
#include <villas/nodes/socket.hpp>
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
using namespace villas::kernel;
int if_init(struct interface *i, struct rtnl_link *link)
Interface::Interface(struct rtnl_link *link, int aff) :
nl_link(link),
tc_qdisc(nullptr),
affinity(aff)
{
int ret;
logger = logging.get(fmt::format("kernel:if:{}", getName()));
i->nl_link = link;
int n = getIRQs();
if (n)
logger->warn("Did not found any interrupts");
debug(LOG_IF | 3, "Created interface '%s'", if_name(i));
int n = if_get_irqs(i);
if (n > 0)
debug(6, "Found %u IRQs for interface '%s'", n, if_name(i));
else
warning("Did not found any interrupts for interface '%s'", if_name(i));
ret = vlist_init(&i->nodes);
if (ret)
return ret;
return 0;
logger->debug("Found {} IRQs", irqs.size());
}
int if_destroy(struct interface *i)
Interface::~Interface()
{
int ret;
/* List members are freed by the nodes they belong to. */
ret = vlist_destroy(&i->nodes);
if (ret)
return ret;
rtnl_qdisc_put(i->tc_qdisc);
return 0;
if (tc_qdisc)
rtnl_qdisc_put(tc_qdisc);
}
int if_start(struct interface *i)
int Interface::start()
{
info("Starting interface '%s' which is used by %zu nodes", if_name(i), vlist_length(&i->nodes));
logger->info("Starting interface which is used by {} nodes", nodes.size());
/* Set affinity for network interfaces (skip _loopback_ dev) */
//if_set_affinity(i, i->affinity);
if (affinity)
setAffinity(affinity);
/* Assign fwmark's to nodes which have netem options */
int ret, fwmark = 0;
for (size_t j = 0; j < vlist_length(&i->nodes); j++) {
struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j);
for (auto *n : nodes) {
if (n->tc_qdisc && n->fwmark < 0)
n->fwmark = 1 + fwmark++;
}
@ -98,141 +85,97 @@ int if_start(struct interface *i)
return 0;
if (getuid() != 0)
error("Network emulation requires super-user privileges!");
throw RuntimeError("Network emulation requires super-user privileges!");
/* Replace root qdisc */
ret = tc_prio(i, &i->tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark);
ret = tc::prio(this, &tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark);
if (ret)
error("Failed to setup priority queuing discipline: %s", nl_geterror(ret));
throw RuntimeError("Failed to setup priority queuing discipline: {}", nl_geterror(ret));
/* Create netem qdisks and appropriate filter per netem node */
for (size_t j = 0; j < vlist_length(&i->nodes); j++) {
struct vnode *n = (struct vnode *) vlist_at(&i->nodes, j);
for (auto *n : nodes) {
if (n->tc_qdisc) {
ret = tc_mark(i, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark);
ret = tc::mark(this, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark);
if (ret)
error("Failed to setup FW mark classifier: %s", nl_geterror(ret));
throw RuntimeError("Failed to setup FW mark classifier: {}", nl_geterror(ret));
char *buf = tc_netem_print(n->tc_qdisc);
debug(LOG_IF | 5, "Starting network emulation on interface '%s' for FW mark %u: %s",
if_name(i), n->fwmark, buf);
char *buf = tc::netem_print(n->tc_qdisc);
logger->debug("Starting network emulation for FW mark {}: {}", n->fwmark, buf);
free(buf);
ret = tc_netem(i, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark));
ret = tc::netem(this, &n->tc_qdisc, TC_HANDLE(0x1000+n->fwmark, 0), TC_HANDLE(1, n->fwmark));
if (ret)
error("Failed to setup netem qdisc: %s", nl_geterror(ret));
throw RuntimeError("Failed to setup netem qdisc: {}", nl_geterror(ret));
}
}
return 0;
}
int if_stop(struct interface *i)
int Interface::stop()
{
info("Stopping interface '%s'", if_name(i));
logger->info("Stopping interface");
//if_set_affinity(i, -1L);
if (affinity)
setAffinity(-1L);
if (i->tc_qdisc)
tc_reset(i);
if (tc_qdisc)
tc::reset(this);
return 0;
}
const char * if_name(struct interface *i)
std::string Interface::getName() const
{
return rtnl_link_get_name(i->nl_link);
auto str = rtnl_link_get_name(nl_link);
return std::string(str);
}
struct interface * if_get_egress(struct sockaddr *sa, struct vlist *interfaces)
Interface * Interface::getEgress(struct sockaddr *sa, SuperNode *sn)
{
int ret;
struct rtnl_link *link;
/* Determine outgoing interface */
link = if_get_egress_link(sa);
if (!link) {
char *buf = socket_print_addr(sa);
error("Failed to get interface for socket address '%s'", buf);
free(buf);
Logger logger = logging.get("kernel:if");
return nullptr;
}
auto & interfaces = sn->getInterfaces();
auto affinity = sn->getAffinity();
/* Determine outgoing interface */
link = nl::get_egress_link(sa);
if (!link)
throw RuntimeError("Failed to get interface for socket address '{}'", socket_print_addr(sa));
/* Search of existing interface with correct ifindex */
struct interface *i;
for (size_t k = 0; k < vlist_length(interfaces); k++) {
i = (struct interface *) vlist_at(interfaces, k);
for (auto *i : interfaces) {
if (rtnl_link_get_ifindex(i->nl_link) == rtnl_link_get_ifindex(link))
return i;
}
/* If not found, create a new interface */
i = new struct interface;
auto *i = new Interface(link, affinity);
if (!i)
throw MemoryAllocationError();
memset(i, 0, sizeof(struct interface));
ret = if_init(i, link);
if (ret)
return nullptr;
vlist_push(interfaces, i);
interfaces.push_back(i);
return i;
}
struct rtnl_link * if_get_egress_link(struct sockaddr *sa)
int Interface::getIRQs()
{
int ifindex = -1;
int irq;
switch (sa->sa_family) {
case AF_INET:
case AF_INET6: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
struct nl_addr *addr = (sa->sa_family == AF_INET)
? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
: nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
ifindex = nl_get_egress(addr); nl_addr_put(addr);
if (ifindex < 0)
error("Netlink error: %s", nl_geterror(ifindex));
break;
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
ifindex = sll->sll_ifindex;
break;
}
}
struct nl_cache *cache = nl_cache_mngt_require("route/link");
return rtnl_link_get(cache, ifindex);
}
int if_get_irqs(struct interface *i)
{
char dirname[NAME_MAX];
int irq, n = 0;
snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", if_name(i));
DIR *dir = opendir(dirname);
auto dirname = fmt::format("/sys/class/net/{}/device/msi_irqs/", getName());
DIR *dir = opendir(dirname.c_str());
if (dir) {
memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX);
irqs.clear();
struct dirent *entry;
while ((entry = readdir(dir)) && n < IF_IRQ_MAX) {
while ((entry = readdir(dir))) {
irq = atoi(entry->d_name);
if (irq)
i->irqs[n++] = irq;
irqs.push_back(irq);
}
closedir(dir);
@ -241,24 +184,32 @@ int if_get_irqs(struct interface *i)
return 0;
}
int if_set_affinity(struct interface *i, int affinity)
int Interface::setAffinity(int affinity)
{
char filename[NAME_MAX];
assert(affinity != 0);
if (getuid() != 0) {
logger->warn("Failed to tune IRQ affinity. Please run as super-user");
return 0;
}
FILE *file;
for (int n = 0; n < IF_IRQ_MAX && i->irqs[n]; n++) {
snprintf(filename, sizeof(filename), "/proc/irq/%d/smp_affinity", (int) i->irqs[n]);
CpuSet cset_pin(affinity);
file = fopen(filename, "w");
for (int irq : irqs) {
std::string filename = fmt::format("/proc/irq/{}/smp_affinity", irq);
file = fopen(filename.c_str(), "w");
if (file) {
if (fprintf(file, "%8x", affinity) < 0)
error("Failed to set affinity for IRQ %u", i->irqs[n]);
if (fprintf(file, "%8lx", (unsigned long) cset_pin) < 0)
throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName());
fclose(file);
debug(LOG_IF | 5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], if_name(i), affinity);
logger->debug("Set affinity of IRQ {} to {} {}", irq, cset_pin.count() == 1 ? "core" : "cores", (std::string) cset_pin);
}
else
error("Failed to set affinity for interface '%s'", if_name(i));
throw SystemError("Failed to set affinity for for IRQ {} on interface '{}'", irq, getName());
}
return 0;

46
lib/kernel/nl.cpp
View file

@ -24,19 +24,22 @@
#include <cstdio>
#include <linux/if_packet.h>
#include <netlink/route/route.h>
#include <netlink/route/link.h>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
#include <villas/kernel/nl.h>
#include <villas/kernel/nl.hpp>
/** Singleton for global netlink socket */
static struct nl_sock *sock = nullptr;
using namespace villas;
using namespace villas::kernel::nl;
struct nl_sock * nl_init()
struct nl_sock * villas::kernel::nl::init()
{
int ret;
@ -62,7 +65,7 @@ struct nl_sock * nl_init()
return sock;
}
void nl_shutdown()
void villas::kernel::nl::shutdown()
{
nl_close(sock);
nl_socket_free(sock);
@ -80,10 +83,10 @@ static int egress_cb(struct nl_msg *msg, void *arg)
return NL_STOP;
}
int nl_get_egress(struct nl_addr *addr)
int villas::kernel::nl::get_egress(struct nl_addr *addr)
{
int ret;
struct nl_sock *sock = nl_init();
struct nl_sock *sock = nl::init();
struct nl_cb *cb;
struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0);
struct rtnl_route *route = nullptr;
@ -130,3 +133,36 @@ out: nlmsg_free(msg);
return ret;
}
struct rtnl_link * villas::kernel::nl::get_egress_link(struct sockaddr *sa)
{
int ifindex = -1;
switch (sa->sa_family) {
case AF_INET:
case AF_INET6: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
struct nl_addr *addr = (sa->sa_family == AF_INET)
? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
: nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
ifindex = nl::get_egress(addr); nl_addr_put(addr);
if (ifindex < 0)
error("Netlink error: %s", nl_geterror(ifindex));
break;
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
ifindex = sll->sll_ifindex;
break;
}
}
struct nl_cache *cache = nl_cache_mngt_require("route/link");
return rtnl_link_get(cache, ifindex);
}

19
lib/kernel/tc.cpp
View file

@ -30,16 +30,17 @@
#include <villas/utils.hpp>
#include <villas/kernel/kernel.hpp>
#include <villas/kernel/if.h>
#include <villas/kernel/tc.h>
#include <villas/kernel/nl.h>
#include <villas/kernel/if.hpp>
#include <villas/kernel/tc.hpp>
#include <villas/kernel/nl.hpp>
using namespace villas;
using namespace villas::kernel;
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands)
int villas::kernel::tc::prio(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands)
{
int ret;
struct nl_sock *sock = nl_init();
struct nl_sock *sock = nl::init();
struct rtnl_qdisc *q = rtnl_qdisc_alloc();
ret = kernel::module_load("sch_prio");
@ -69,10 +70,10 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl
return ret;
}
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
int villas::kernel::tc::mark(Interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
{
int ret;
struct nl_sock *sock = nl_init();
struct nl_sock *sock = nl::init();
struct rtnl_cls *c = rtnl_cls_alloc();
ret = kernel::module_load("cls_fw");
@ -97,9 +98,9 @@ int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_
return ret;
}
int tc_reset(struct interface *i)
int villas::kernel::tc::reset(Interface *i)
{
struct nl_sock *sock = nl_init();
struct nl_sock *sock = nl::init();
/* We restore the default pfifo_fast qdisc, by deleting ours */
return rtnl_qdisc_delete(sock, i->tc_qdisc);

125
lib/kernel/tc_netem.cpp
View file

@ -27,20 +27,75 @@
#include <netlink/route/qdisc/netem.h>
#include <villas/kernel/if.h>
#include <villas/kernel/nl.h>
#include <villas/kernel/if.hpp>
#include <villas/kernel/nl.hpp>
#include <villas/kernel/nl-private.h>
#include <villas/kernel/tc_netem.h>
#include <villas/kernel/tc_netem.hpp>
#include <villas/kernel/kernel.hpp>
#include <villas/utils.hpp>
#include <villas/exceptions.hpp>
using namespace villas;
using namespace villas::utils;
using namespace villas::kernel;
static const double max_percent_value = 0xffffffff;
int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
/**
* Set the delay distribution. Latency/jitter must be set before applying.
* @arg qdisc Netem qdisc.
* @return 0 on success, error code on failure.
*/
static int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len)
{
struct rtnl_netem *netem;
if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc))))
return -1;
if (len > MAXDIST)
return -NLE_INVAL;
netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t));
size_t i;
for (i = 0; i < len; i++)
netem->qnm_dist.dist_data[i] = data[i];
netem->qnm_dist.dist_size = len;
netem->qnm_mask |= SCH_NETEM_ATTR_DIST;
return 0;
}
/** Customized version of rtnl_netem_set_delay_distribution() of libnl */
static int set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json)
{
if (json_is_string(json))
return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json));
else if (json_is_array(json)) {
json_t *elm;
size_t idx;
size_t len = json_array_size(json);
int16_t *data = new int16_t[len];
if (!data)
throw MemoryAllocationError();
json_array_foreach(json, idx, elm) {
if (!json_is_integer(elm))
return -1;
data[idx] = json_integer_value(elm);;
}
return rtnl_netem_set_delay_distribution_data(qdisc, data, len);
}
return 0;
}
int villas::kernel::tc::netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
{
int ret, val;
@ -75,7 +130,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
rtnl_tc_set_kind(TC_CAST(ne), "netem");
if (json_delay_distribution) {
if (tc_netem_set_delay_distribution(ne, json_delay_distribution))
if (set_delay_distribution(ne, json_delay_distribution))
error("Invalid delay distribution in netem config");
}
@ -163,7 +218,7 @@ int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg)
return 0;
}
char * tc_netem_print(struct rtnl_qdisc *ne)
char * villas::kernel::tc::netem_print(struct rtnl_qdisc *ne)
{
char *buf = nullptr;
@ -212,10 +267,10 @@ char * tc_netem_print(struct rtnl_qdisc *ne)
return buf;
}
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
int villas::kernel::tc::netem(Interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
{
int ret;
struct nl_sock *sock = nl_init();
struct nl_sock *sock = nl::init();
struct rtnl_qdisc *q = *qd;
ret = kernel::module_load("sch_netem");
@ -235,57 +290,3 @@ int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hd
return ret;
}
/**
* Set the delay distribution. Latency/jitter must be set before applying.
* @arg qdisc Netem qdisc.
* @return 0 on success, error code on failure.
*/
int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len)
{
struct rtnl_netem *netem;
if (!(netem = (struct rtnl_netem *) rtnl_tc_data(TC_CAST(qdisc))))
return -1;
if (len > MAXDIST)
return -NLE_INVAL;
netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t));
size_t i;
for (i = 0; i < len; i++)
netem->qnm_dist.dist_data[i] = data[i];
netem->qnm_dist.dist_size = len;
netem->qnm_mask |= SCH_NETEM_ATTR_DIST;
return 0;
}
/** Customized version of rtnl_netem_set_delay_distribution() of libnl */
int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json)
{
if (json_is_string(json))
return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json));
else if (json_is_array(json)) {
json_t *elm;
size_t idx;
size_t len = json_array_size(json);
int16_t *data = new int16_t[len];
if (!data)
throw MemoryAllocationError();
json_array_foreach(json, idx, elm) {
if (!json_is_integer(elm))
return -1;
data[idx] = json_integer_value(elm);;
}
return rtnl_netem_set_delay_distribution_data(qdisc, data, len);
}
return 0;
}

10
lib/node.cpp
View file

@ -40,10 +40,10 @@
#include <villas/memory.h>
#ifdef WITH_NETEM
#include <villas/kernel/if.h>
#include <villas/kernel/nl.h>
#include <villas/kernel/tc.h>
#include <villas/kernel/tc_netem.h>
#include <villas/kernel/if.hpp>
#include <villas/kernel/nl.hpp>
#include <villas/kernel/tc.hpp>
#include <villas/kernel/tc_netem.hpp>
#endif /* WITH_NETEM */
using namespace villas;
@ -184,7 +184,7 @@ int node_parse(struct vnode *n, json_t *json, const char *name)
return ret;
if (enabled)
tc_netem_parse(&n->tc_qdisc, json_netem);
kernel::tc::netem_parse(&n->tc_qdisc, json_netem);
else
n->tc_qdisc = nullptr;
#endif /* WITH_NETEM */

11
lib/nodes/rtp.cpp
View file

@ -48,14 +48,15 @@ extern "C" {
#include <villas/super_node.hpp>
#ifdef WITH_NETEM
#include <villas/kernel/if.h>
#include <villas/kernel/if.hpp>
#endif /* WITH_NETEM */
static pthread_t re_pthread;
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
using namespace villas::node;
using namespace villas::kernel;
static struct plugin p;
@ -491,20 +492,18 @@ int rtp_type_start(villas::node::SuperNode *sn)
return ret;
#ifdef WITH_NETEM
struct vlist *interfaces = sn->getInterfaces();
/* Gather list of used network interfaces */
for (size_t i = 0; i < vlist_length(&p.node.instances); i++) {
struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i);
struct rtp *r = (struct rtp *) n->_vd;
struct interface *j = if_get_egress(&r->out.saddr_rtp.u.sa, interfaces);
Interface *j = Interface::getEgress(&r->out.saddr_rtp.u.sa, sn);
if (!j) {
r->logger->error("Failed to find egress interface for node: {}", node_name(n));
return -1;
}
vlist_push(&j->nodes, n);
j->addNode(n);
}
#endif /* WITH_NETEM */

13
lib/nodes/socket.cpp
View file

@ -40,22 +40,21 @@
#endif /* WITH_SOCKET_LAYER_ETH */
#ifdef WITH_NETEM
#include <villas/kernel/if.h>
#include <villas/kernel/nl.h>
#include <villas/kernel/if.hpp>
#include <villas/kernel/nl.hpp>
#endif /* WITH_NETEM */
/* Forward declartions */
static struct plugin p;
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
using namespace villas::node;
using namespace villas::kernel;
int socket_type_start(villas::node::SuperNode *sn)
{
#ifdef WITH_NETEM
struct vlist *interfaces = sn->getInterfaces();
/* Gather list of used network interfaces */
for (size_t i = 0; i < vlist_length(&p.node.instances); i++) {
struct vnode *n = (struct vnode *) vlist_at(&p.node.instances, i);
@ -65,9 +64,9 @@ int socket_type_start(villas::node::SuperNode *sn)
continue;
/* Determine outgoing interface */
struct interface *j = if_get_egress((struct sockaddr *) &s->out.saddr, interfaces);
Interface *j = Interface::getEgress((struct sockaddr *) &s->out.saddr, sn);
vlist_push(&j->nodes, n);
j->addNode(n);
}
#endif /* WITH_NETEM */

2
lib/signal_list.cpp
View file

@ -151,7 +151,7 @@ void signal_list_dump(const struct vlist *list, const union signal_data *data, u
strcatf(&buf, " = %s", val);
}
info("%s", buf);
debug(5, "%s", buf);
free(buf);
}
}

5
lib/socket_addr.cpp
View file

@ -30,7 +30,7 @@
#include <villas/exceptions.hpp>
#ifdef WITH_SOCKET_LAYER_ETH
#include <villas/kernel/nl.h>
#include <villas/kernel/nl.hpp>
#endif /* WITH_SOCKET_LAYER_ETH */
using namespace villas;
@ -126,7 +126,8 @@ int socket_parse_address(const char *addr, struct sockaddr *saddr, enum SocketLa
memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, ETHER_ADDR_LEN);
/* Get interface index from name */
nl_init();
kernel::nl::init();
struct nl_cache *cache = nl_cache_mngt_require("route/link");
struct rtnl_link *link = rtnl_link_get_by_name(cache, ifname);
if (!link)

31
lib/super_node.cpp
View file

@ -36,10 +36,10 @@
#include <villas/log.hpp>
#include <villas/node/exceptions.hpp>
#include <villas/kernel/rt.hpp>
#include <villas/kernel/if.h>
#include <villas/kernel/if.hpp>
#ifdef WITH_NETEM
#include <villas/kernel/nl.h>
#include <villas/kernel/nl.hpp>
#endif
using namespace villas;
@ -75,12 +75,8 @@ SuperNode::SuperNode() :
if (ret)
throw RuntimeError("Failed to initialize list");
ret = vlist_init(&interfaces);
if (ret)
throw RuntimeError("Failed to initialize list");
#ifdef WITH_NETEM
nl_init(); /* Fill link cache */
kernel::nl::init(); /* Fill link cache */
#endif /* WITH_NETEM */
char hname[128];
@ -270,12 +266,10 @@ void SuperNode::startInterfaces()
#ifdef WITH_NETEM
int ret;
for (size_t i = 0; i < vlist_length(&interfaces); i++) {
auto *j = (struct interface *) vlist_at(&interfaces, i);
ret = if_start(j);
for (auto *i : interfaces) {
ret = i->start();
if (ret)
throw RuntimeError("Failed to initialize network interface: {}", if_name(j));
throw RuntimeError("Failed to start network interface: {}", i->getName());
}
#endif /* WITH_NETEM */
}
@ -363,7 +357,7 @@ void SuperNode::prepare()
auto *n = (struct vnode *) vlist_at(&nodes, i);
if (vlist_length(&n->sources) == 0 &&
vlist_length(&n->destinations) == 0) {
logger->info("Node {} is not used by any path. Disabling...");
logger->info("Node {} is not used by any path. Disabling...", node_name(n));
n->enabled = false;
}
}
@ -442,12 +436,10 @@ void SuperNode::stopInterfaces()
#ifdef WITH_NETEM
int ret;
for (size_t j = 0; j < vlist_length(&interfaces); j++) {
struct interface *i = (struct interface *) vlist_at(&interfaces, j);
ret = if_stop(i);
for (auto *i : interfaces) {
ret = i->stop();
if (ret)
throw RuntimeError("Failed to stop interface: {}", if_name(i));
throw RuntimeError("Failed to stop interface: {}", i->getName());
}
#endif /* WITH_NETEM */
}
@ -491,9 +483,6 @@ SuperNode::~SuperNode()
ret = vlist_destroy(&paths, (dtor_cb_t) path_destroy, true);
ret = vlist_destroy(&nodes, (dtor_cb_t) node_destroy, true);
#ifdef WITH_NETEM
ret = vlist_destroy(&interfaces, (dtor_cb_t) if_destroy, true);
#endif /* WITH_NETEM */
}
int SuperNode::periodic()