diff --git a/include/villas/kernel/nl-private.h b/include/villas/kernel/nl-private.h new file mode 100644 index 000000000..968cdb209 --- /dev/null +++ b/include/villas/kernel/nl-private.h @@ -0,0 +1,45 @@ +#pragma once + +#define SCH_NETEM_ATTR_DIST 0x2000 + +struct rtnl_netem_corr +{ + uint32_t nmc_delay; + uint32_t nmc_loss; + uint32_t nmc_duplicate; +}; + +struct rtnl_netem_reo +{ + uint32_t nmro_probability; + uint32_t nmro_correlation; +}; + +struct rtnl_netem_crpt +{ + uint32_t nmcr_probability; + uint32_t nmcr_correlation; +}; + +struct rtnl_netem_dist +{ + int16_t * dist_data; + size_t dist_size; +}; + +struct rtnl_netem +{ + uint32_t qnm_latency; + uint32_t qnm_limit; + uint32_t qnm_loss; + uint32_t qnm_gap; + uint32_t qnm_duplicate; + uint32_t qnm_jitter; + uint32_t qnm_mask; + struct rtnl_netem_corr qnm_corr; + struct rtnl_netem_reo qnm_ro; + struct rtnl_netem_crpt qnm_crpt; + struct rtnl_netem_dist qnm_dist; +}; + +void *rtnl_tc_data(struct rtnl_tc *tc); diff --git a/include/villas/kernel/tc.h b/include/villas/kernel/tc.h index c6c69f2db..a64439be8 100644 --- a/include/villas/kernel/tc.h +++ b/include/villas/kernel/tc.h @@ -41,22 +41,6 @@ typedef uint32_t tc_hdl_t; struct interface; -/** Parse network emulator (netem) settings. - * - * @param cfg A jansson object containing the settings. - * @param[out] ne A pointer to a libnl3 qdisc object where setting will be written to. - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int tc_parse(struct rtnl_qdisc **ne, json_t *cfg); - -/** Print network emulator (netem) setting into buffer. - * - * @param tc A pointer to the libnl3 qdisc object where settings will be read from. - * @return A pointer to a string which must be freed() by the caller. - */ -char * tc_print(struct rtnl_qdisc *ne); - /** Remove all queuing disciplines and filters. * * @param i The interface @@ -77,17 +61,6 @@ int tc_reset(struct interface *i); */ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands); -/** Add a new network emulator (netem) discipline. - * - * @param i[in] The interface to which this qdisc will be added. - * @param qd[in,out] The libnl3 object of the new prio qdisc. - * @param handle[in] The handle of the new qdisc. - * @param parent[in] Make this qdisc a child of this class - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); - /** Add a new filter based on the netfilter mark. * * @param i The interface to which this classifier is applied to. diff --git a/include/villas/kernel/tc_netem.h b/include/villas/kernel/tc_netem.h new file mode 100644 index 000000000..78dc10951 --- /dev/null +++ b/include/villas/kernel/tc_netem.h @@ -0,0 +1,73 @@ +/** Setup network emulation + * + * We use the firewall mark to apply individual netem qdiscs + * per node. Every node uses an own BSD socket. + * By using so SO_MARK socket option (see socket(7)) + * we can classify traffic originating from a node seperately. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup kernel Kernel @{ */ + +#pragma once + +#include + +#include +#include + +#include + +typedef uint32_t tc_hdl_t; + +struct interface; + +/** Parse network emulator (netem) settings. + * + * @param cfg A jansson object containing the settings. + * @param[out] ne A pointer to a libnl3 qdisc object where setting will be written to. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int tc_netem_parse(struct rtnl_qdisc **ne, json_t *cfg); + +/** Print network emulator (netem) setting into buffer. + * + * @param tc A pointer to the libnl3 qdisc object where settings will be read from. + * @return A pointer to a string which must be freed() by the caller. + */ +char * tc_netem_print(struct rtnl_qdisc *ne); + +/** Add a new network emulator (netem) discipline. + * + * @param i[in] The interface to which this qdisc will be added. + * @param qd[in,out] The libnl3 object of the new prio qdisc. + * @param handle[in] The handle of the new qdisc. + * @param parent[in] Make this qdisc a child of this class + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); + +int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json); + +/** @} */ diff --git a/lib/kernel/if.c b/lib/kernel/if.c index 8e9a33901..2211a2a20 100644 --- a/lib/kernel/if.c +++ b/lib/kernel/if.c @@ -29,6 +29,7 @@ #include "kernel/if.h" #include "kernel/tc.h" +#include "kernel/tc_netem.h" #include "kernel/nl.h" #include "kernel/kernel.h" @@ -103,7 +104,7 @@ int if_start(struct interface *i) if (ret) error("Failed to setup FW mark classifier: %s", nl_geterror(ret)); - char *buf = tc_print(s->tc_qdisc); + char *buf = tc_netem_print(s->tc_qdisc); debug(LOG_IF | 5, "Starting network emulation on interface '%s' for FW mark %u: %s", rtnl_link_get_name(i->nl_link), s->mark, buf); free(buf); diff --git a/lib/kernel/tc.c b/lib/kernel/tc.c index 6f5a37711..531bd551c 100644 --- a/lib/kernel/tc.c +++ b/lib/kernel/tc.c @@ -23,7 +23,6 @@ *********************************************************************************/ #include -#include #include #include @@ -35,158 +34,6 @@ #include "utils.h" -int tc_parse(struct rtnl_qdisc **netem, json_t *cfg) -{ - const char *str; - int ret, val; - - json_t *json_distribution = NULL; - json_t *json_limit = NULL; - json_t *json_delay = NULL; - json_t *json_jitter = NULL; - json_t *json_loss = NULL; - json_t *json_duplicate = NULL; - json_t *json_corruption = NULL; - - json_error_t err; - - ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: o, s?: o, s?: o, s?: o, s?: o }", - "distribution", &json_distribution, - "limit", &json_limit, - "delay", &json_delay, - "jitter", &json_jitter, - "loss", &json_loss, - "duplicate", &json_duplicate, - "corruption", &json_corruption - ); - if (ret) - jerror(&err, "Failed to parse setting network emulation settings"); - - struct rtnl_qdisc *ne = rtnl_qdisc_alloc(); - if (!ne) - error("Failed to allocated memory!"); - - rtnl_tc_set_kind(TC_CAST(ne), "netem"); - - if (json_distribution) { - str = json_string_value(json_distribution); - if (!str) - error("Setting 'distribution' must be a JSON string"); - - if (rtnl_netem_set_delay_distribution(ne, str)) - error("Invalid delay distribution '%s' in netem config", str); - } - - if (json_limit) { - val = json_integer_value(json_limit); - - if (!json_is_integer(json_limit) || val <= 0) - error("Setting 'limit' must be a positive integer"); - - rtnl_netem_set_limit(ne, val); - } - else - rtnl_netem_set_limit(ne, 0); - - if (json_delay) { - val = json_integer_value(json_delay); - - if (!json_is_integer(json_delay) || val <= 0) - error("Setting 'delay' must be a positive integer"); - - rtnl_netem_set_delay(ne, val); - } - - if (json_jitter) { - val = json_integer_value(json_jitter); - - if (!json_is_integer(json_jitter) || val <= 0) - error("Setting 'jitter' must be a positive integer"); - - rtnl_netem_set_jitter(ne, val); - } - - if (json_loss) { - val = json_integer_value(json_loss); - - if (!json_is_integer(json_loss) || val < 0 || val > 100) - error("Setting 'loss' must be a positive integer within the range [ 0, 100 ]"); - - rtnl_netem_set_loss(ne, val); - } - - if (json_duplicate) { - val = json_integer_value(json_duplicate); - - if (!json_is_integer(json_duplicate) || val < 0 || val > 100) - error("Setting 'duplicate' must be a positive integer within the range [ 0, 100 ]"); - - rtnl_netem_set_duplicate(ne, val); - } - - if (json_corruption) { - val = json_integer_value(json_corruption); - - if (!json_is_integer(json_corruption) || val < 0 || val > 100) - error("Setting 'corruption' must be a positive integer within the range [ 0, 100 ]"); - - rtnl_netem_set_corruption_probability(ne, val); - } - - *netem = ne; - - return 0; -} - -char * tc_print(struct rtnl_qdisc *ne) -{ - char *buf = NULL; - - if (rtnl_netem_get_limit(ne) > 0) - strcatf(&buf, "limit %upkts", rtnl_netem_get_limit(ne)); - - if (rtnl_netem_get_delay(ne) > 0) { - strcatf(&buf, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0); - - if (rtnl_netem_get_jitter(ne) > 0) { - strcatf(&buf, "jitter %.2fms ", rtnl_netem_get_jitter(ne) / 1000.0); - - if (rtnl_netem_get_delay_correlation(ne) > 0) - strcatf(&buf, "%u%% ", rtnl_netem_get_delay_correlation(ne)); - } - } - - if (rtnl_netem_get_loss(ne) > 0) { - strcatf(&buf, "loss %u%% ", rtnl_netem_get_loss(ne)); - - if (rtnl_netem_get_loss_correlation(ne) > 0) - strcatf(&buf, "%u%% ", rtnl_netem_get_loss_correlation(ne)); - } - - if (rtnl_netem_get_reorder_probability(ne) > 0) { - strcatf(&buf, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne)); - - if (rtnl_netem_get_reorder_correlation(ne) > 0) - strcatf(&buf, "%u%% ", rtnl_netem_get_reorder_correlation(ne)); - } - - if (rtnl_netem_get_corruption_probability(ne) > 0) { - strcatf(&buf, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne)); - - if (rtnl_netem_get_corruption_correlation(ne) > 0) - strcatf(&buf, "%u%% ", rtnl_netem_get_corruption_correlation(ne)); - } - - if (rtnl_netem_get_duplicate(ne) > 0) { - strcatf(&buf, "duplication %u%% ", rtnl_netem_get_duplicate(ne)); - - if (rtnl_netem_get_duplicate_correlation(ne) > 0) - strcatf(&buf, "%u%% ", rtnl_netem_get_duplicate_correlation(ne)); - } - - return buf; -} - int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands) { int ret; @@ -220,30 +67,6 @@ int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl return ret; } -int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) -{ - int ret; - struct nl_sock *sock = nl_init(); - struct rtnl_qdisc *q = *qd; - - ret = kernel_module_load("sch_netem"); - if (ret) - error("Failed to load kernel module: sch_netem (%d)", ret); - - rtnl_tc_set_link(TC_CAST(q), i->nl_link); - rtnl_tc_set_parent(TC_CAST(q), parent); - rtnl_tc_set_handle(TC_CAST(q), handle); - //rtnl_tc_set_kind(TC_CAST(q), "netem"); - - ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); - - *qd = q; - - debug(LOG_TC | 3, "Added netem qdisc to interface '%s'", rtnl_link_get_name(i->nl_link)); - - return ret; -} - int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark) { int ret; diff --git a/lib/kernel/tc_netem.c b/lib/kernel/tc_netem.c new file mode 100644 index 000000000..5e380d440 --- /dev/null +++ b/lib/kernel/tc_netem.c @@ -0,0 +1,261 @@ +/** Traffic control (tc): setup network emulation qdisc + * + * VILLASnode uses these functions to setup the network emulation feature. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include + +#include "kernel/if.h" +#include "kernel/nl.h" +#include "kernel/nl-private.h" +#include "kernel/tc_netem.h" +#include "kernel/kernel.h" +#include "utils.h" + + +int tc_netem_parse(struct rtnl_qdisc **netem, json_t *cfg) +{ + int ret, val; + + json_t *json_limit = NULL; + json_t *json_delay = NULL; + json_t *json_delay_distribution = NULL; + json_t *json_delay_correlation = NULL; + json_t *json_jitter = NULL; + json_t *json_loss = NULL; + json_t *json_duplicate = NULL; + json_t *json_corruption = NULL; + + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: o, s?: o, s?: o, s?: o, s?: o, s?: o }", + "distribution", &json_delay_distribution, + "correlation", &json_delay_correlation, + "limit", &json_limit, + "delay", &json_delay, + "jitter", &json_jitter, + "loss", &json_loss, + "duplicate", &json_duplicate, + "corruption", &json_corruption + ); + if (ret) + jerror(&err, "Failed to parse setting network emulation settings"); + + struct rtnl_qdisc *ne = rtnl_qdisc_alloc(); + if (!ne) + error("Failed to allocated memory!"); + + rtnl_tc_set_kind(TC_CAST(ne), "netem"); + + if (json_delay_distribution) { + if (tc_netem_set_delay_distribution(ne, json_delay_distribution)) + error("Invalid delay distribution in netem config"); + } + + if (json_limit) { + val = json_integer_value(json_limit); + + if (!json_is_integer(json_limit) || val <= 0) + error("Setting 'limit' must be a positive integer"); + + rtnl_netem_set_limit(ne, val); + } + else + rtnl_netem_set_limit(ne, 0); + + if (json_delay) { + val = json_integer_value(json_delay); + + if (!json_is_integer(json_delay) || val <= 0) + error("Setting 'delay' must be a positive integer"); + + rtnl_netem_set_delay(ne, val); + } + + if (json_jitter) { + val = json_integer_value(json_jitter); + + if (!json_is_integer(json_jitter) || val <= 0) + error("Setting 'jitter' must be a positive integer"); + + rtnl_netem_set_jitter(ne, val); + } + + if (json_loss) { + val = json_integer_value(json_loss); + + if (!json_is_integer(json_loss) || val < 0 || val > 100) + error("Setting 'loss' must be a positive integer within the range [ 0, 100 ]"); + + rtnl_netem_set_loss(ne, val); + } + + if (json_duplicate) { + val = json_integer_value(json_duplicate); + + if (!json_is_integer(json_duplicate) || val < 0 || val > 100) + error("Setting 'duplicate' must be a positive integer within the range [ 0, 100 ]"); + + rtnl_netem_set_duplicate(ne, val); + } + + if (json_corruption) { + val = json_integer_value(json_corruption); + + if (!json_is_integer(json_corruption) || val < 0 || val > 100) + error("Setting 'corruption' must be a positive integer within the range [ 0, 100 ]"); + + rtnl_netem_set_corruption_probability(ne, val); + } + + *netem = ne; + + return 0; +} + +char * tc_netem_print(struct rtnl_qdisc *ne) +{ + char *buf = NULL; + + if (rtnl_netem_get_limit(ne) > 0) + strcatf(&buf, "limit %upkts", rtnl_netem_get_limit(ne)); + + if (rtnl_netem_get_delay(ne) > 0) { + strcatf(&buf, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0); + + if (rtnl_netem_get_jitter(ne) > 0) { + strcatf(&buf, "jitter %.2fms ", rtnl_netem_get_jitter(ne) / 1000.0); + + if (rtnl_netem_get_delay_correlation(ne) > 0) + strcatf(&buf, "%u%% ", rtnl_netem_get_delay_correlation(ne)); + } + } + + if (rtnl_netem_get_loss(ne) > 0) { + strcatf(&buf, "loss %u%% ", rtnl_netem_get_loss(ne)); + + if (rtnl_netem_get_loss_correlation(ne) > 0) + strcatf(&buf, "%u%% ", rtnl_netem_get_loss_correlation(ne)); + } + + if (rtnl_netem_get_reorder_probability(ne) > 0) { + strcatf(&buf, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne)); + + if (rtnl_netem_get_reorder_correlation(ne) > 0) + strcatf(&buf, "%u%% ", rtnl_netem_get_reorder_correlation(ne)); + } + + if (rtnl_netem_get_corruption_probability(ne) > 0) { + strcatf(&buf, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne)); + + if (rtnl_netem_get_corruption_correlation(ne) > 0) + strcatf(&buf, "%u%% ", rtnl_netem_get_corruption_correlation(ne)); + } + + if (rtnl_netem_get_duplicate(ne) > 0) { + strcatf(&buf, "duplication %u%% ", rtnl_netem_get_duplicate(ne)); + + if (rtnl_netem_get_duplicate_correlation(ne) > 0) + strcatf(&buf, "%u%% ", rtnl_netem_get_duplicate_correlation(ne)); + } + + return buf; +} + +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) +{ + int ret; + struct nl_sock *sock = nl_init(); + struct rtnl_qdisc *q = *qd; + + ret = kernel_module_load("sch_netem"); + if (ret) + error("Failed to load kernel module: sch_netem (%d)", ret); + + rtnl_tc_set_link(TC_CAST(q), i->nl_link); + rtnl_tc_set_parent(TC_CAST(q), parent); + rtnl_tc_set_handle(TC_CAST(q), handle); + //rtnl_tc_set_kind(TC_CAST(q), "netem"); + + ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); + + *qd = q; + + debug(LOG_TC | 3, "Added netem qdisc to interface '%s'", rtnl_link_get_name(i->nl_link)); + + return ret; +} + +/** + * Set the delay distribution. Latency/jitter must be set before applying. + * @arg qdisc Netem qdisc. + * @return 0 on success, error code on failure. + */ +int rtnl_netem_set_delay_distribution_data(struct rtnl_qdisc *qdisc, short *data, size_t len) +{ + struct rtnl_netem *netem; + + if (!(netem = rtnl_tc_data(TC_CAST(qdisc)))) + return -1; + + if (len > MAXDIST) + return -NLE_INVAL; + + netem->qnm_dist.dist_data = (int16_t *) calloc(len, sizeof(int16_t)); + + size_t i; + for (i = 0; i < len; i++) + netem->qnm_dist.dist_data[i] = data[i]; + + netem->qnm_dist.dist_size = len; + netem->qnm_mask |= SCH_NETEM_ATTR_DIST; + + return 0; +} + + +/** Customized version of rtnl_netem_set_delay_distribution() of libnl */ +int tc_netem_set_delay_distribution(struct rtnl_qdisc *qdisc, json_t *json) +{ + if (json_is_string(json)) + return rtnl_netem_set_delay_distribution(qdisc, json_string_value(json)); + else if (json_is_array(json)){ + json_t *elm; + size_t idx; + size_t len = json_array_size(json); + + int16_t *data = (int16_t *) alloc(len * sizeof(int16_t)); + + json_array_foreach(json, idx, elm) { + if (!json_is_integer(elm)) + return -1; + + data[idx] = json_integer_value(elm);; + } + + return rtnl_netem_set_delay_distribution_data(qdisc, data, len); + } + + return 0; +} diff --git a/lib/nodes/Makefile.inc b/lib/nodes/Makefile.inc index ba01a07a0..9b3adb842 100644 --- a/lib/nodes/Makefile.inc +++ b/lib/nodes/Makefile.inc @@ -129,7 +129,7 @@ ifeq ($(WITH_SOCKET),1) # libnl3 is optional but required for network emulation and IRQ pinning ifeq ($(shell $(PKGCONFIG) libnl-route-3.0; echo $$?),0) - LIB_SRCS += $(addprefix lib/kernel/, nl.c tc.c if.c) + LIB_SRCS += $(addprefix lib/kernel/, nl.c tc.c tc_netem.c if.c) LIB_PKGS += libnl-route-3.0 endif endif diff --git a/lib/nodes/socket.c b/lib/nodes/socket.c index 7bfa56d73..d148d5b84 100644 --- a/lib/nodes/socket.c +++ b/lib/nodes/socket.c @@ -37,7 +37,7 @@ #ifdef WITH_NETEM #include "kernel/if.h" #include "kernel/nl.h" - #include "kernel/tc.h" + #include "kernel/tc_netem.h" #endif /* WITH_NETEM */ #include "io_format.h" @@ -553,7 +553,7 @@ int socket_parse(struct node *n, json_t *cfg) jerror(&err, "Failed to parse setting 'netem' of node %s", node_name(n)); if (enabled) - tc_parse(&s->tc_qdisc, json_netem); + tc_netem_parse(&s->tc_qdisc, json_netem); else s->tc_qdisc = NULL; #endif /* WITH_NETEM */