diff --git a/documentation/Install.md b/documentation/Install.md index eb7ad7bf2..d70aaf05e 100644 --- a/documentation/Install.md +++ b/documentation/Install.md @@ -7,11 +7,12 @@ Install libraries including developement headers for: - [libconfig](http://www.hyperrealm.com/libconfig/) for parsing the config file - - [libnl3](http://www.infradead.org/~tgr/libnl/) for the network emulation support + - [libnl3](http://www.infradead.org/~tgr/libnl/) for the network communication & emulation support + - libOpal{AsyncApi,Core,Utils} for running the S2SS server as an Asynchronous process inside your RT-LAB model Use the following command to install the dependencies under Debian-based distributions: - $ sudo apt-get install iproute2 libconfig-dev linbl-3-dev + $ sudo apt-get install libconfig-dev libnl-3-dev libnl-route-3-dev or the following line for Fedora / CentOS / Redhat systems: @@ -23,19 +24,12 @@ tools to setup the network emulation and interfaces. ### Compilation -Checkout the `Makefile` and `include/config.h` for some config options which have to be specified by compile time. +Checkout the `Makefile` and `include/config.h` for some options which have to be specified at compile time. Afterwards, start the compilation with: $ make -Add `V=5` for a more verbose debugging output. - -## Installation - -Install the server by executing: - - $ sudo make install - -Add `PREFIX=/usr/local/` to specify a non-standard installation destination. +Append `V=5` to `make` for a more verbose debugging output. +Append `DEBUG=1` to `make` to add debug symbols. diff --git a/documentation/clients/File.md b/documentation/clients/File.md index 5e0e80e23..d77c298cb 100644 --- a/documentation/clients/File.md +++ b/documentation/clients/File.md @@ -1,17 +1,17 @@ # File {#file} -The `file` node-type can be used to log or replay sample values to disk. +The `file` node-type can be used to log or replay samples to / from disk. ## Configuration Every `file` node supports the following settings: -#### `in` +#### `in` *(string: filesystem path)* Specifies the path to a file which contains data for replaying. See below for a description of the file format. -#### `out` +#### `out` *(string: filesystem path)* Specifies the path to a file where samples will be written to. This setting allows to add special paceholders for time and date values. @@ -23,13 +23,15 @@ See [strftime(3)](http://man7.org/linux/man-pages/man3/strftime.3.html) for a li will create a file called: *path_of_working_directory*/logs/measurements_2015-08-09_22-20-50.log -#### `file_mode` +See below for a description of the file format. + +#### `file_mode` *(string)* Specifies the mode which should be used to open the output file. See [open(2)](http://man7.org/linux/man-pages/man2/open.2.html) for an explanation of allowed values. The default value is `w+` which will start writing at the beginning of the file and create it in case it does not exist yet. -#### `epoch_mode` +#### `epoch_mode` *("now"|"relative"|"absolute")* This setting allows to select the behaviour of the following `epoch` setting. It can be used to adjust the point in time when the first value should be read. @@ -40,13 +42,38 @@ The behaviour of `epoch` is depending on the value of `epoch_mode`. - `epoch_mode = relative`: The first value is read at *start* + `epoch` seconds. - `epoch_mode = absolute`: The first value is read at `epoch` seconds after 1970-01-01 00:00:00. -#### `rate` +#### `send_rate` *(float)* -By default `rate` has the value `0`. If the value is non-zero, +By default `send_rate` has the value `0` which means that the time between consecutive samples is the same as in the `in` file based on the timestamps in the first column. + +If this setting has a non-zero value, the default behaviour is overwritten with a fixed rate. ### Example -@todo Add extract of example.conf + file_node = { + type = "file", + + ### The following settings are specific to the file node-type!! ### + mode = "w+", # The mode in which files should be opened (see open(2)) + # You might want to use "a+" to append to a file + + in = "logs/file_input.log", # These options specify the path prefix where the the files are stored + out = "logs/file_output_%F_%T.log" # The output path accepts all format tokens of (see strftime(3)) + + epoch_mode = "now" # One of: + # now (default) + # relative + # absolute + + epoch = 10 # The interpretation of this value depends on epoch_mode (default is 0): + # - epoch_mode = now: The first value is read at: _now_ + epoch seconds. + # - epoch_mode = relative: The first value is read at _start_ + `epoch` seconds. + # - epoch_mode = absolute: The first value is read at epoch seconds after 1970-01-01 00:00:00. + + rate = 2.0 # A constant rate at which the lines of the input files should be read + # A missing or zero value will use the timestamp in the first column + # of the file to determine the pause between consecutive lines. + } ## File Format diff --git a/server/Makefile b/server/Makefile index bc71ff6df..e1e70a444 100644 --- a/server/Makefile +++ b/server/Makefile @@ -5,7 +5,7 @@ OBJS = path.o node.o hooks.o msg.o cfg.o stats.o # Helper libs OBJS += utils.o list.o hist.o log.o timing.o # Node types -OBJS += file.o socket.o if.o tc.o +OBJS += file.o VPATH = src @@ -13,33 +13,45 @@ VPATH = src V ?= 2 # Compiler and linker flags +CC = gcc LDLIBS = -pthread -lrt -lm -lconfig -CFLAGS = -std=gnu99 -Iinclude/ -MMD -Wall -O3 -CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V) +override CFLAGS += -std=gnu99 -Iinclude/ -MMD -Wall -O3 +override CFLAGS += -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE=1 -DV=$(V) + +# Default include paths for third-party libs +PCIDIR ?= /usr/include +NLDIR ?= /usr/include/libnl3 +OPALDIR ?= /usr/opalrt/common +#OPALDIR ?= ../contrib/opal # Add more compiler flags ifdef DEBUG - CFLAGS += -O0 -g + override CFLAGS += -O0 -g endif ifneq (,$(shell which git)) - CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"' + override CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"' endif -# Enabled GTFPGA support when libpci is available -ifneq (,$(wildcard /usr/include/pci/pci.h)) - CFLAGS += -DENABLE_GTFPGA +# Enable Socket node type when libnl3 is available +ifneq (,$(wildcard $(NLDIR)/netlink/netlink.h)) + override CFLAGS += -DENABLE_SOCKET -I$(NLDIR) + LDLIBS += -lnl-3 -lnl-route-3 + OBJS += nl.o tc.o if.o socket.o +endif + +# Enable GTFPGA support when libpci is available +ifneq (,$(wildcard $(PCIDIR)/pci/pci.h)) + override CFLAGS += -DENABLE_GTFPGA -I$(PCIDIR) LDLIBS += -lpci OBJS += gtfpga.o endif -# Enable OPAL-RT Asynchronous Process support -OPALDIR = /usr/opalrt/common -#OPALDIR = ../contrib/opal +# Enable OPAL-RT Asynchronous Process support (will result in 32bit binary!!!) ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h)) - override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target - override LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/ - override LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) - override OBJS += opal.o + override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target + LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/,-L/usr/lib/i386-linux-gnu/ + LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) + OBJS += opal.o endif .PHONY: all clean strip protected diff --git a/server/include/hist.h b/server/include/hist.h index 3b0d33c93..50386a32c 100644 --- a/server/include/hist.h +++ b/server/include/hist.h @@ -35,7 +35,7 @@ struct hist { /** The number of buckets in #data. */ int length; - /** Total number of counted values between #low and #high. */ + /** Total number of counted values. */ hist_cnt_t total; /** The number of values which are higher than #high. */ hist_cnt_t higher; diff --git a/server/include/if.h b/server/include/if.h index b46f15b03..a01ecafa7 100644 --- a/server/include/if.h +++ b/server/include/if.h @@ -14,11 +14,10 @@ #define _IF_H_ #include -#include +#include #include "list.h" -#define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */ #define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */ #ifndef SO_MARK @@ -26,13 +25,16 @@ #endif struct socket; +struct nl_addr; +struct rtnl_link; /** Interface data structure */ struct interface { - /** The index used by the kernel to reference this interface */ - int index; - /** Human readable name of this interface */ - char name[IF_NAME_MAX]; + /** libnl3: Handle of interface */ + struct rtnl_link *nl_link; + /** libnl3: Root prio qdisc */ + struct rtnl_qdisc *tc_qdisc; + /** List of IRQs of the NIC */ char irqs[IF_IRQ_MAX]; @@ -42,11 +44,11 @@ struct interface { /** Add a new interface to the global list and lookup name, irqs... * - * @param index The interface index of the OS + * @param link The libnl3 link handle * @retval >0 Success. A pointer to the new interface. * @retval 0 Error. The creation failed. */ -struct interface * if_create(int index); +struct interface * if_create(struct rtnl_link *link); /** Destroy interface by freeing dynamically allocated memory. @@ -78,20 +80,15 @@ int if_start(struct interface *i, int affinity); */ int if_stop(struct interface *i); -/** Get outgoing interface. +/** Lookup routing tables to get the interface on which packets for a certain destination + * will leave the system. * - * Depending on the address family of the socker address, - * this function tries to determine outgoing interface - * which is used to send packages to a remote host with the specified - * socket address. - * - * For AF_INET the fnuction performs a lookup in the kernel routing table. - * For AF_PACKET the function uses the existing sll_ifindex field of the socket address. - * - * @param sa A destination address for outgoing packets. - * @return The interface index. + * @param[in] sa The destination address for outgoing packets. + * @param[out] link The egress interface. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ -int if_getegress(struct sockaddr *sa); +int if_get_egress(struct sockaddr *sa, struct rtnl_link **link); /** Get all IRQs for this interface. * @@ -102,7 +99,7 @@ int if_getegress(struct sockaddr *sa); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int if_getirqs(struct interface *i); +int if_get_irqs(struct interface *i); /** Change the SMP affinity of NIC interrupts. * @@ -111,7 +108,7 @@ int if_getirqs(struct interface *i); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int if_setaffinity(struct interface *i, int affinity); +int if_set_affinity(struct interface *i, int affinity); /** Search the global list of interfaces for a given index. * diff --git a/server/include/log.h b/server/include/log.h index bde122b48..55b524563 100644 --- a/server/include/log.h +++ b/server/include/log.h @@ -14,9 +14,9 @@ #include #ifdef __GNUC__ - #define INDENT int __attribute__ ((__cleanup__(log_outdent), unused)) _old_indent = log_indent(1); + #define INDENT int __attribute__ ((__cleanup__(log_outdent), unused)) _old_indent = log_indent(1); #else - #define INDENT ; + #define INDENT ; #endif /* The log level which is passed as first argument to print() */ diff --git a/server/include/msg_format.h b/server/include/msg_format.h index d7b41d6cc..86f68d68a 100644 --- a/server/include/msg_format.h +++ b/server/include/msg_format.h @@ -13,10 +13,10 @@ #include #ifdef __linux__ - #define _BSD_SOURCE 1 - #include + #define _BSD_SOURCE 1 + #include #elif defined(__PPC__) /* Xilinx toolchain */ - #include + #include #endif #include "config.h" diff --git a/server/include/nl.h b/server/include/nl.h new file mode 100644 index 000000000..599d08147 --- /dev/null +++ b/server/include/nl.h @@ -0,0 +1,30 @@ +/** Netlink related functions. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC + * This file is part of S2SS. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#ifndef _NL_H_ +#define _NL_H_ + +#include +#include +#include + +/** Get index of outgoing interface for given destination address. + * + * @retval >=0 Interface index of outgoing interface. + * @retval <0 Error. Something went wrong. + */ +int nl_get_egress(struct nl_addr *addr); + +/** Get or create global netlink socket. */ +struct nl_sock * nl_init(); + +/** Close and free global netlink socket. */ +void nl_shutdown(); + +#endif \ No newline at end of file diff --git a/server/include/node.h b/server/include/node.h index 86509322c..7d3b5e584 100644 --- a/server/include/node.h +++ b/server/include/node.h @@ -21,7 +21,6 @@ #include #include "msg.h" -#include "tc.h" #include "list.h" /** Static node initialization */ diff --git a/server/include/socket.h b/server/include/socket.h index 718b9ba69..28946efcf 100644 --- a/server/include/socket.h +++ b/server/include/socket.h @@ -16,6 +16,7 @@ #define _SOCKET_H_ #include +#include #include "node.h" @@ -25,6 +26,13 @@ enum socket_layer { LAYER_UDP }; +union sockaddr_union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + struct sockaddr_ll sll; +}; + struct socket { /** The socket descriptor */ int sd; @@ -35,12 +43,14 @@ struct socket { enum socket_layer layer; /** Local address of the socket */ - struct sockaddr_storage local; + union sockaddr_union local; /** Remote address of the socket */ - struct sockaddr_storage remote; + union sockaddr_union remote; - /** Network emulator settings */ - struct netem *netem; + /** libnl3: Network emulator queuing discipline */ + struct rtnl_qdisc *tc_qdisc; + /** libnl3: Firewall mark classifier */ + struct rtnl_cls *tc_classifier; /* Linked list _per_interface_ */ struct socket *next; diff --git a/server/include/stats.h b/server/include/stats.h index 324476163..992a45a2e 100644 --- a/server/include/stats.h +++ b/server/include/stats.h @@ -13,16 +13,24 @@ /* Forward declarations */ struct path; +/** Print a table header for statistics printed by stats_line() */ +void stats_header(); + +/** Print a single line of stats including received, sent, invalid and dropped packet counters */ int stats_line(struct path *p); int stats_show(struct path *p); +/** Update histograms */ int stats_collect(struct path *p); +/** Create histograms */ int stats_start(struct path *p); +/** Destroy histograms */ int stats_stop(struct path *p); +/** Reset all statistic counters to zero */ int stats_reset(struct path *p); #endif \ No newline at end of file diff --git a/server/include/tc.h b/server/include/tc.h index 7d6f2ddf8..a5b460154 100644 --- a/server/include/tc.h +++ b/server/include/tc.h @@ -16,65 +16,34 @@ #define _TC_H_ #include + +#include +#include + #include -/** A type alias for TC handles. - * - * TC handles are used to construct a tree - * of classes, qdiscs and filters. - */ typedef uint32_t tc_hdl_t; -/** Concatenate 16 bit minor and majar parts to a 32 bit tc handle */ -#define TC_HDL(maj, min) ((maj & 0xFFFF) << 16 | (min & 0xFFFF)) -/** Get the major part of a tc handle */ -#define TC_HDL_MAJ(h) ((h >> 16) & 0xFFFF) -/** Get the minor part of a tc handle */ -#define TC_HDL_MIN(h) ((h >> 0) & 0xFFFF) -/** The root handle */ -#define TC_HDL_ROOT (0xFFFFFFFFU) - -/* Bitfield for valid fields in struct netem */ -#define TC_NETEM_DELAY (1 << 0) /**< netem::delay is valid @see netem::valid */ -#define TC_NETEM_JITTER (1 << 1) /**< netem::jitter is valid @see netem::valid */ -#define TC_NETEM_DISTR (1 << 2) /**< netem::distribution is valid @see netem::valid */ -#define TC_NETEM_LOSS (1 << 3) /**< netem::loss is valid @see netem::valid */ -#define TC_NETEM_CORRUPT (1 << 4) /**< netem::corrupt is valid @see netem::valid */ -#define TC_NETEM_DUPL (1 << 5) /**< netem::duplicate is valid @see netem::valid */ - struct interface; -/** Netem configuration settings. - * - * This struct is used to pass the netem configuration - * from config_parse_netem() to tc_netem() - */ -struct netem { - /** Which fields of this struct contain valid data (TC_NETEM_*). */ - char valid; - - /** Delay distribution: uniform, normal, pareto, paretonormal */ - const char *distribution; - /** Added delay (uS) */ - int delay; - /** Delay jitter (uS) */ - int jitter; - /** Random loss probability (%) */ - int loss; - /** Packet corruption probability (%) */ - int corrupt; - /** Packet duplication probability (%) */ - int duplicate; -}; - /** Parse network emulator (netem) settings. * * @param cfg A libconfig object containing the settings. - * @param em A pointer to the netem settings structure (part of the path structure). + * @param[out] ne A pointer to a libnl3 qdisc object where setting will be written to. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_parse(config_setting_t *cfg, struct netem *em); +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **ne); + +/** Print network emulator (netem) setting into buffer. + * + * @param buf A character buffer to write to. + * @param len The length of the supplied buffer. + * @param tc A pointer to the libnl3 qdisc object where settings will be read from. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne); /** Remove all queuing disciplines and filters. * @@ -86,32 +55,36 @@ int tc_reset(struct interface *i); /** Create a priority (prio) queueing discipline. * - * @param i The interface - * @param handle The handle for the new qdisc - * @param bands The number of classes for this new qdisc + * @param i[in] The interface + * @param qd[in,out] The libnl3 object of the new prio qdisc. + * @param handle[in] The handle for the new qdisc + * @param parent[in] Make this qdisc a child of this class + * @param bands[in] The number of classes for this new qdisc * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_prio(struct interface *i, tc_hdl_t handle, int bands); +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands); /** Add a new network emulator (netem) discipline. * - * @param i The interface - * @param parent Make this qdisc a child of - * @param em The netem settings + * @param i[in] The interface to which this qdisc will be added. + * @param qd[in,out] The libnl3 object of the new prio qdisc. + * @param handle[in] The handle of the new qdisc. + * @param parent[in] Make this qdisc a child of this class * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em); +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); /** Add a new filter based on the netfilter mark. * - * @param i The interface + * @param i The interface to which this classifier is applied to. + * @param cls[in,out] The libnl3 object of the new prio qdisc. * @param flowid The destination class for matched traffic * @param mark The netfilter firewall mark (sometime called 'fwmark') * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_mark(struct interface *i, tc_hdl_t flowid, int mark); +int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark); #endif /* _TC_H_ */ diff --git a/server/include/utils.h b/server/include/utils.h index 142148022..c0bff7f4c 100644 --- a/server/include/utils.h +++ b/server/include/utils.h @@ -16,9 +16,9 @@ #include "log.h" #ifdef __GNUC__ - #define EXPECT(x, v) __builtin_expect(x, v) + #define EXPECT(x, v) __builtin_expect(x, v) #else - #define EXPECT(x, v) (x) + #define EXPECT(x, v) (x) #endif /* Some color escape codes for pretty log messages */ diff --git a/server/src/cfg.c b/server/src/cfg.c index 80a419a45..aed896637 100644 --- a/server/src/cfg.c +++ b/server/src/cfg.c @@ -21,10 +21,10 @@ #include "socket.h" #include "file.h" #ifdef ENABLE_GTFPGA - #include "gtfpga.h" + #include "gtfpga.h" #endif #ifdef ENABLE_OPAL_ASYNC - #include "opal.h" + #include "opal.h" #endif int config_parse(const char *filename, config_t *cfg, struct settings *set, diff --git a/server/src/file.c b/server/src/file.c index 992e92603..1b695eb9f 100644 --- a/server/src/file.c +++ b/server/src/file.c @@ -54,10 +54,10 @@ int file_parse(config_setting_t *cfg, struct node *n) if (config_setting_lookup_string(cfg, "in", &in)) f->path_in = strdup(in); - if (!config_setting_lookup_string(cfg, "mode", &f->file_mode)) + if (!config_setting_lookup_string(cfg, "file_mode", &f->file_mode)) f->file_mode = "w+"; - if (!config_setting_lookup_float(cfg, "rate", &f->rate)) + if (!config_setting_lookup_float(cfg, "send_rate", &f->rate)) f->rate = 0; /* Disable fixed rate sending. Using timestamps of file instead */ if (config_setting_lookup_float(n->cfg, "epoch", &epoch_flt)) @@ -170,14 +170,30 @@ int file_read(struct node *n, struct msg *pool, int poolsize, int first, int cnt if (f->in) { for (i = 0; i < cnt; i++) { struct msg *cur = &pool[(first+i) % poolsize]; - msg_fscan(f->in, cur); - + if (f->rate) { - if (timerfd_wait(f->tfd) < 0) - serror("Failed to wait for timer"); + /* Wait until epoch for the first time only */ + if (ftell(f->in) == 0) { + struct timespec until = time_add(&f->start, &f->offset); + if (timerfd_wait_until(f->tfd, &until)) + serror("Failed to wait for timer"); + } + /* Wait with fixed rate delay */ + else { + if (timerfd_wait(f->tfd) < 0) + serror("Failed to wait for timer"); + } + + msg_fscan(f->in, cur); } else { - struct timespec until = time_add(&MSG_TS(cur), &f->offset); + struct timespec until; + + /* Get message and timestamp */ + msg_fscan(f->in, cur); + + /* Wait for next message / sampe */ + until = time_add(&MSG_TS(cur), &f->offset); if (timerfd_wait_until(f->tfd, &until) < 0) serror("Failed to wait for timer"); } diff --git a/server/src/gtfpga.c b/server/src/gtfpga.c index bfe380d19..1a07e1beb 100644 --- a/server/src/gtfpga.c +++ b/server/src/gtfpga.c @@ -33,6 +33,9 @@ static void gtfpga_debug(char *msg, ...) { int gtfpga_init(int argc, char * argv[], struct settings *set) { pacc = pci_alloc(); /* Get the pci_access structure */ + if (!pacc) + error("Failed to allocate PCI access structure"); + pci_init(pacc); /* Initialize the PCI library */ pacc->error = (log_cb_t) error; /* Replace logging and debug functions */ diff --git a/server/src/if.c b/server/src/if.c index 0170d283b..28125c253 100644 --- a/server/src/if.c +++ b/server/src/if.c @@ -10,27 +10,30 @@ #include #include #include - -#include #include -#include #include +#include +#include + #include "if.h" #include "tc.h" +#include "nl.h" #include "socket.h" #include "utils.h" /** Linked list of interfaces. */ struct list interfaces; -struct interface * if_create(int index) { +struct interface * if_create(struct rtnl_link *link) +{ struct interface *i = alloc(sizeof(struct interface)); + + i->nl_link = link; - i->index = index; - if_indextoname(index, i->name); + debug(3, "Created interface '%s'", rtnl_link_get_name(i->nl_link)); - debug(3, "Created interface '%s' (index=%u)", i->name, i->index); + if_get_irqs(i); list_init(&i->sockets, NULL); list_push(&interfaces, i); @@ -40,22 +43,27 @@ struct interface * if_create(int index) { void if_destroy(struct interface *i) { - /* List members are freed by their belonging nodes. */ + /* List members are freed by the nodes they belong to. */ list_destroy(&i->sockets); + + rtnl_qdisc_put(i->tc_qdisc); free(i); } int if_start(struct interface *i, int affinity) { - info("Starting interface '%s' (index=%u)", i->name, i->index); + info("Starting interface '%s' which is used by %u sockets", rtnl_link_get_name(i->nl_link), list_length(&i->sockets)); { INDENT + /* Set affinity for network interfaces (skip _loopback_ dev) */ + if_set_affinity(i, affinity); + /* Assign fwmark's to socket nodes which have netem options */ - int mark = 0; + int ret, mark = 0; FOREACH(&i->sockets, it) { struct socket *s = it->socket; - if (s->netem) + if (s->tc_qdisc) s->mark = 1 + mark++; } @@ -64,20 +72,25 @@ int if_start(struct interface *i, int affinity) return 0; /* Replace root qdisc */ - tc_prio(i, TC_HDL(4000, 0), mark); + if ((ret = tc_prio(i, &i->tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, mark))) + ;//error("Failed to setup priority queuing discipline: %s", nl_geterror(ret)); /* Create netem qdisks and appropriate filter per netem node */ FOREACH(&i->sockets, it) { struct socket *s = it->socket; - if (s->netem) { - tc_mark(i, TC_HDL(4000, s->mark), s->mark); - tc_netem(i, TC_HDL(4000, s->mark), s->netem); + if (s->tc_qdisc) { + if ((ret = tc_mark(i, &s->tc_classifier, TC_HANDLE(1, s->mark), s->mark))) + error("Failed to setup FW mark classifier: %s", nl_geterror(ret)); + + char buf[256]; + tc_print(buf, sizeof(buf), s->tc_qdisc); + debug(5, "Starting network emulation on interface '%s' for FW mark %u: %s", + rtnl_link_get_name(i->nl_link), s->mark, buf); + + if ((ret = tc_netem(i, &s->tc_qdisc, TC_HANDLE(0x1000+s->mark, 0), TC_HANDLE(1, s->mark)))) + error("Failed to setup netem qdisc: %s", nl_geterror(ret)); } } - - /* Set affinity for network interfaces (skip _loopback_ dev) */ - if_getirqs(i); - if_setaffinity(i, affinity); } return 0; @@ -85,65 +98,59 @@ int if_start(struct interface *i, int affinity) int if_stop(struct interface *i) { - info("Stopping interface '%s' (index=%u)", i->name, i->index); + info("Stopping interface '%s'", rtnl_link_get_name(i->nl_link)); { INDENT - if_setaffinity(i, -1L); + if_set_affinity(i, -1L); - /* Only reset tc if it was initialized before */ - FOREACH(&i->sockets, it) { - if (it->socket->netem) { - tc_reset(i); - break; - } - } + if (i->tc_qdisc) + tc_reset(i); } return 0; } -int if_getegress(struct sockaddr *sa) +int if_get_egress(struct sockaddr *sa, struct rtnl_link **link) { + int ifindex = -1; + switch (sa->sa_family) { - case AF_INET: { + case AF_INET: + case AF_INET6: { struct sockaddr_in *sin = (struct sockaddr_in *) sa; - char cmd[128]; - char token[32]; - - snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sin->sin_addr)); - - debug(8, "System: %s", cmd); - - FILE *p = popen(cmd, "r"); - if (!p) - return -1; - - while (!feof(p) && fscanf(p, "%31s", token)) { - if (!strcmp(token, "dev")) { - fscanf(p, "%31s", token); - break; - } - } - - return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token); + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa; + + struct nl_addr *addr = (sa->sa_family == AF_INET) + ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr)) + : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr)); + + ifindex = nl_get_egress(addr); + if (ifindex < 0) + error("Netlink error: %s", nl_geterror(ifindex)); + break; } - + case AF_PACKET: { struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - return sll->sll_ifindex; + + ifindex = sll->sll_ifindex; + break; } - - default: - return -1; } + + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + if (!(*link = rtnl_link_get(cache, ifindex))) + return -1; + + return 0; } -int if_getirqs(struct interface *i) +int if_get_irqs(struct interface *i) { char dirname[NAME_MAX]; int irq, n = 0; - snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name); + snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", rtnl_link_get_name(i->nl_link)); DIR *dir = opendir(dirname); if (dir) { memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX); @@ -156,11 +163,13 @@ int if_getirqs(struct interface *i) closedir(dir); } + + debug(6, "Found %u IRQs for interface '%s'", n, rtnl_link_get_name(i->nl_link)); return 0; } -int if_setaffinity(struct interface *i, int affinity) +int if_set_affinity(struct interface *i, int affinity) { char filename[NAME_MAX]; FILE *file; @@ -174,10 +183,10 @@ int if_setaffinity(struct interface *i, int affinity) error("Failed to set affinity for IRQ %u", i->irqs[n]); fclose(file); - debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], i->name, affinity); + debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], rtnl_link_get_name(i->nl_link), affinity); } else - error("Failed to set affinity for interface '%s'", i->name); + error("Failed to set affinity for interface '%s'", rtnl_link_get_name(i->nl_link)); } return 0; @@ -186,7 +195,7 @@ int if_setaffinity(struct interface *i, int affinity) struct interface * if_lookup_index(int index) { FOREACH(&interfaces, it) { - if (it->interface->index == index) + if (rtnl_link_get_ifindex(it->interface->nl_link) == index) return it->interface; } diff --git a/server/src/log.c b/server/src/log.c index 068830838..4a7b96073 100644 --- a/server/src/log.c +++ b/server/src/log.c @@ -19,8 +19,8 @@ #ifdef ENABLE_OPAL_ASYNC /* Define RTLAB before including OpalPrint.h for messages to be sent * to the OpalDisplay. Otherwise stdout will be used. */ - #define RTLAB - #include "OpalPrint.h" + #define RTLAB + #include "OpalPrint.h" #endif /** Debug level used by the debug() macro. diff --git a/server/src/msg.c b/server/src/msg.c index e737175ac..7df3df117 100644 --- a/server/src/msg.c +++ b/server/src/msg.c @@ -11,10 +11,10 @@ #include #ifdef __linux__ - #include + #include #elif defined(__PPC__) /* Xilinx toolchain */ - #include - #define bswap_32(x) Xil_EndianSwap32(x) + #include + #define bswap_32(x) Xil_EndianSwap32(x) #endif #include "msg.h" diff --git a/server/src/nl.c b/server/src/nl.c new file mode 100644 index 000000000..ac345971d --- /dev/null +++ b/server/src/nl.c @@ -0,0 +1,104 @@ +/** Netlink related functions. + * + * S2SS uses libnl3 to talk to the Linux kernel to gather networking related information + * + * @author Steffen Vogel + * @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC + * This file is part of S2SS. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#include + +#include +#include + +#include "utils.h" +#include "nl.h" + +/** Singleton for global netlink socket */ +static struct nl_sock *sock = NULL; + +struct nl_sock * nl_init() +{ + int ret; + + if (!sock) { + /* Create connection to netlink */ + sock = nl_socket_alloc(); + if (!sock) + error("Failed to allocate memory"); + + if ((ret = nl_connect(sock, NETLINK_ROUTE))) + error("Failed to connect to kernel: %s", nl_geterror(ret)); + + /* Fill some caches */ + struct nl_cache *cache; + if ((ret = rtnl_link_alloc_cache(sock, AF_UNSPEC, &cache))) + error("Failed to get list of interfaces: %s", nl_geterror(ret)); + + nl_cache_mngt_provide(cache); + } + + return sock; +} + +void nl_shutdown() +{ + nl_close(sock); + nl_socket_free(sock); + + sock = NULL; +} + +static int egress_cb(struct nl_msg *msg, void *arg) +{ + struct rtnl_route **route = (struct rtnl_route **) arg; + + if (rtnl_route_parse(nlmsg_hdr(msg), route)) + return NL_SKIP; + + return NL_STOP; +} + +int nl_get_egress(struct nl_addr *addr) +{ + int ret; + struct nl_sock *sock = nl_init(); + struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0); + + /* Build message */ + struct rtmsg rmsg = { + .rtm_family = nl_addr_get_family(addr), + .rtm_dst_len = nl_addr_get_prefixlen(addr), + }; + + if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO))) + return ret; + if ((ret = nla_put_addr(msg, RTA_DST, addr))) + return ret; + + /* Send message */ + ret = nl_send_auto(sock, msg); + nlmsg_free(msg); + if (ret < 0) + return ret; + + /* Hook into receive chain */ + struct rtnl_route *route = NULL; + struct nl_cb *cb = nl_cb_alloc(NL_CB_VALID); + nl_cb_set(cb, NL_CB_VALID, NL_CB_CUSTOM, egress_cb, &route); + + /* Receive message */ + nl_recvmsgs_report(sock, cb); + nl_wait_for_ack(sock); + nl_cb_put(cb); + + /* Check result */ + if (route && (1 <= rtnl_route_get_nnexthops(route))) { + struct rtnl_nexthop *nh = rtnl_route_nexthop_n(route, 0); + return rtnl_route_nh_get_ifindex(nh); + } + else + return -1; +} diff --git a/server/src/node.c b/server/src/node.c index 94d3e9281..7a2435c6d 100644 --- a/server/src/node.c +++ b/server/src/node.c @@ -14,12 +14,17 @@ /* Node types */ #include "file.h" -#include "socket.h" #ifdef ENABLE_GTFPGA -#include "gtfpga.h" + #include "gtfpga.h" #endif #ifdef ENABLE_OPAL_ASYNC -#include "opal.h" + #include "opal.h" +#endif +#ifdef ENABLE_SOCKET + #include "socket.h" + + #include + #include #endif #define VTABLE(type, name, fnc) { type, name, \ @@ -36,7 +41,9 @@ struct node_vtable vtables[] = { #ifdef ENABLE_GTFPGA VTABLE(GTFPGA, "gtfpga", gtfpga), #endif +#ifdef ENABLE_SOCKET VTABLE(BSD_SOCKET, "socket", socket), +#endif VTABLE(LOG_FILE, "file", file) }; @@ -136,9 +143,12 @@ struct node * node_create() void node_destroy(struct node *n) { switch (n->vt->type) { +#ifdef ENABLE_SOCKET case BSD_SOCKET: - free(n->socket->netem); + rtnl_qdisc_put(n->socket->tc_qdisc); + rtnl_cls_put(n->socket->tc_classifier); break; +#endif case LOG_FILE: free(n->file->path_in); free(n->file->path_out); diff --git a/server/src/path.c b/server/src/path.c index 7cba1b218..72fffc19f 100644 --- a/server/src/path.c +++ b/server/src/path.c @@ -207,7 +207,6 @@ struct path * path_create() hook_add(HOOK_PATH_START, 1, stats_start); - hook_add(HOOK_PATH_STOP, 1, stats_line); hook_add(HOOK_PATH_STOP, 2, stats_show); hook_add(HOOK_PATH_STOP, 3, stats_stop); diff --git a/server/src/server.c b/server/src/server.c index 3f4ac8a1a..4f6a9a114 100644 --- a/server/src/server.c +++ b/server/src/server.c @@ -6,22 +6,25 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ + #include #include -#include #include #include +#include + #include "config.h" #include "utils.h" #include "cfg.h" #include "path.h" #include "node.h" +#include "stats.h" #include "license.h" #ifdef ENABLE_OPAL_ASYNC -#include "opal.h" + #include "opal.h" #endif /** Linked list of nodes */ @@ -43,6 +46,7 @@ static void quit() FOREACH(&nodes, it) node_stop(it->node); + info("De-initializing node types"); node_deinit(); /* Freeing dynamically allocated memory */ @@ -59,7 +63,10 @@ static void realtime_init() { INDENT /* Use FIFO scheduler with real time priority */ if (settings.priority) { - struct sched_param param = { .sched_priority = settings.priority }; + struct sched_param param = { + .sched_priority = settings.priority + }; + if (sched_setscheduler(0, SCHED_FIFO, ¶m)) serror("Failed to set real time priority"); else @@ -98,6 +105,17 @@ static void usage(const char *name) printf(" This type of invocation is used by OPAL-RT Asynchronous processes.\n"); printf(" See in the RT-LAB User Guide for more information.\n\n"); #endif + printf("Supported features:\n"); +#ifdef ENABLE_PCI + printf(" - libpci: GTFPGA PCIe card\n"); +#endif +#ifdef ENABLE_SOCKET + printf(" - libnl3: Network Emulation\n"); +#endif +#ifdef ENABLE_OPAL_ASYNC + printf(" - libOpalAsyncApi: run as OPAL Asynchronous Process\n"); +#endif + printf("\n"); printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); printf(" copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC\n"); @@ -158,15 +176,12 @@ int main(int argc, char *argv[]) /* Run! */ if (settings.stats > 0) { - info("%-32s : %-8s %-8s %-8s %-8s %-8s", - "Source " MAG("=>") " Destination", "#Sent", "#Recv", "#Drop", "#Skip", "#Inval"); - line(); + stats_header(); for (;;) FOREACH(&paths, it) { usleep(settings.stats * 1e6); hook_run(it->path, HOOK_PERIODIC); } - } else pause(); diff --git a/server/src/socket.c b/server/src/socket.c index f948620e8..31a95680f 100644 --- a/server/src/socket.c +++ b/server/src/socket.c @@ -13,24 +13,23 @@ #include #include #include -#include -#include - -#include -#include -#include - -#include -#include #include #include #include +#include +#include +#include +#include +#include + +#include "if.h" +#include "nl.h" +#include "tc.h" #include "config.h" #include "utils.h" #include "socket.h" -#include "if.h" /** Linked list of interfaces */ extern struct list interfaces; @@ -40,27 +39,30 @@ static struct list sockets; int socket_init(int argc, char * argv[], struct settings *set) { INDENT + nl_init(); /* Fill link cache */ list_init(&interfaces, (dtor_cb_t) if_destroy); /* Gather list of used network interfaces */ FOREACH(&sockets, it) { struct socket *s = it->socket; + struct rtnl_link *link; /* Determine outgoing interface */ - int index = if_getegress((struct sockaddr *) &s->remote); - if (index < 0) { + if (if_get_egress((struct sockaddr *) &s->remote, &link)) { char buf[128]; socket_print_addr(buf, sizeof(buf), (struct sockaddr *) &s->remote); error("Failed to get interface for socket address '%s'", buf); } - struct interface *i = if_lookup_index(index); + int ifindex = rtnl_link_get_ifindex(link); + struct interface *i = if_lookup_index(ifindex); if (!i) - i = if_create(index); + i = if_create(link); list_push(&i->sockets, s); } + /** @todo Improve mapping of NIC IRQs per path */ FOREACH(&interfaces, it) if_start(it->interface, set->affinity); @@ -83,11 +85,18 @@ int socket_print(struct node *n, char *buf, int len) char local[INET6_ADDRSTRLEN + 16]; char remote[INET6_ADDRSTRLEN + 16]; + char *layer = NULL; + + switch (s->layer) { + case LAYER_UDP: layer = "udp"; break; + case LAYER_IP: layer = "ip"; break; + case LAYER_ETH: layer = "eth"; break; + } socket_print_addr(local, sizeof(local), (struct sockaddr *) &s->local); socket_print_addr(remote, sizeof(remote), (struct sockaddr *) &s->remote); - return snprintf(buf, len, "local=%s, remote=%s", local, remote); + return snprintf(buf, len, "layer=%s, local=%s, remote=%s", layer, local, remote); } int socket_open(struct node *n) @@ -116,9 +125,9 @@ int socket_open(struct node *n) /* Set fwmark for outgoing packets */ if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark))) - serror("Failed to set fwmark for outgoing packets"); + serror("Failed to set FW mark for outgoing packets"); else - debug(4, "Set fwmark for socket (sd=%u) to %u", s->sd, s->mark); + debug(4, "Set FW mark for socket (sd=%u) to %u", s->sd, s->mark); /* Set socket priority, QoS or TOS IP options */ int prio; @@ -170,11 +179,11 @@ int socket_read(struct node *n, struct msg *pool, int poolsize, int first, int c /* Get size of received packet in bytes */ ioctl(s->sd, FIONREAD, &bytes); - /* Check packet integrity */ + /* Check if packet length is correct */ if (bytes % (cnt * 4) != 0) error("Packet length not dividable by 4: received=%u, cnt=%u", bytes, cnt); if (bytes / cnt > sizeof(struct msg)) - error("Packet length is too large: received=%u, cnt=%u, max=%lu", bytes, cnt, sizeof(struct msg)); + error("Packet length is too large: received=%u, cnt=%u, max=%zu", bytes, cnt, sizeof(struct msg)); for (int i = 0; i < cnt; i++) { /* All messages of a packet must have equal length! */ @@ -283,26 +292,24 @@ int socket_parse(config_setting_t *cfg, struct node *n) if (!config_setting_lookup_string(cfg, "local", &local)) cerror(cfg, "Missing local address for node '%s'", n->name); - + ret = socket_parse_addr(local, (struct sockaddr *) &s->local, s->layer, AI_PASSIVE); - if (ret) + if (ret) { cerror(cfg, "Failed to resolve local address '%s' of node '%s': %s", local, n->name, gai_strerror(ret)); + } ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, s->layer, 0); - if (ret) + if (ret) { cerror(cfg, "Failed to resolve remote address '%s' of node '%s': %s", remote, n->name, gai_strerror(ret)); + } - /** @todo Netem settings are not usable with AF_UNIX */ config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem"); if (cfg_netem) { int enabled = 1; - - if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) { - s->netem = alloc(sizeof(struct netem)); - tc_parse(cfg_netem, s->netem); - } + if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) + tc_parse(cfg_netem, &s->tc_qdisc); } n->socket = s; @@ -312,42 +319,61 @@ int socket_parse(config_setting_t *cfg, struct node *n) return 0; } -int socket_print_addr(char *buf, int len, struct sockaddr *sa) +int socket_print_addr(char *buf, int len, struct sockaddr *saddr) { - switch (sa->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *) sa; - inet_ntop(sin->sin_family, &sin->sin_addr, buf, len); - return snprintf(buf+strlen(buf), len-strlen(buf), ":%hu", ntohs(sin->sin_port)); - } + union sockaddr_union *sa = (union sockaddr_union *) saddr; + + /* Address */ + switch (sa->sa.sa_family) { + case AF_INET6: + inet_ntop(AF_INET6, &sa->sin6.sin6_addr, buf, len); + break; - case AF_PACKET: { - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - char ifname[IF_NAMESIZE]; - - return snprintf(buf, len, "%s%%%s:%#hx", - ether_ntoa((struct ether_addr *) &sll->sll_addr), - if_indextoname(sll->sll_ifindex, ifname), - ntohs(sll->sll_protocol)); - } + case AF_INET: + inet_ntop(AF_INET, &sa->sin.sin_addr, buf, len); + break; + + case AF_PACKET: + snprintf(buf, len, "%02x", sa->sll.sll_addr[0]); + for (int i = 1; i < sa->sll.sll_halen; i++) + strap(buf, len, ":%02x", sa->sll.sll_addr[i]); + break; default: - return snprintf(buf, len, "address family: %u", sa->sa_family); + error("Unknown address family: '%u'", sa->sa.sa_family); + } + + /* Port / Interface */ + switch (sa->sa.sa_family) { + case AF_INET6: + case AF_INET: + strap(buf, len, ":%hu", ntohs(sa->sin.sin_port)); + break; + + case AF_PACKET: { + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + struct rtnl_link *link = rtnl_link_get(cache, sa->sll.sll_ifindex); + if (!link) + error("Failed to get interface for index: %u", sa->sll.sll_ifindex); + + strap(buf, len, "%%%s", rtnl_link_get_name(link)); + strap(buf, len, ":%hu", ntohs(sa->sll.sll_protocol)); + break; + } } return 0; } -int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer layer, int flags) +int socket_parse_addr(const char *addr, struct sockaddr *saddr, enum socket_layer layer, int flags) { /** @todo: Add support for IPv6 */ + union sockaddr_union *sa = (union sockaddr_union *) saddr; char *copy = strdup(addr); int ret; if (layer == LAYER_ETH) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */ - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - /* Split string */ char *node = strtok(copy, "%"); char *ifname = strtok(NULL, ":"); @@ -356,21 +382,27 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer l /* Parse link layer (MAC) address */ struct ether_addr *mac = ether_aton(node); if (!mac) - error("Failed to parse mac address: %s", node); + error("Failed to parse MAC address: %s", node); - memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6); + memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, 6); + + /* Get interface index from name */ + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + struct rtnl_link *link = rtnl_link_get_by_name(cache, ifname); + if (!link) + error("Failed to get network interface: '%s'", ifname); - sll->sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS); - sll->sll_halen = 6; - sll->sll_family = AF_PACKET; - sll->sll_ifindex = if_nametoindex(ifname); + sa->sll.sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS); + sa->sll.sll_halen = 6; + sa->sll.sll_family = AF_PACKET; + sa->sll.sll_ifindex = rtnl_link_get_ifindex(link); ret = 0; } else { /* Format: "192.168.0.10:12001" */ struct addrinfo hint = { .ai_flags = flags, - .ai_family = AF_INET + .ai_family = AF_UNSPEC }; /* Split string */ diff --git a/server/src/stats.c b/server/src/stats.c index d6f901ec4..a06b125ea 100644 --- a/server/src/stats.c +++ b/server/src/stats.c @@ -12,6 +12,13 @@ #include "timing.h" #include "utils.h" +void stats_header() +{ + info("%-32s : %-8s %-8s %-8s %-8s %-8s", + "Source " MAG("=>") " Destination", "#Sent", "#Recv", "#Drop", "#Skip", "#Invalid"); + line(); +} + int stats_line(struct path *p) { char buf[33]; @@ -25,9 +32,9 @@ int stats_line(struct path *p) int stats_show(struct path *p) { - if (p->hist_delay.length) { info("One-way delay:"); hist_print(&p->hist_delay); } - if (p->hist_gap.length) { info("Message gap time:"); hist_print(&p->hist_gap); } - if (p->hist_sequence.length) { info("Sequence number gaps:"); hist_print(&p->hist_sequence); } + if (p->hist_delay.total) { info("One-way delay:"); hist_print(&p->hist_delay); } + if (p->hist_gap.total) { info("Message gap time:"); hist_print(&p->hist_gap); } + if (p->hist_sequence.total) { info("Sequence number gaps:"); hist_print(&p->hist_sequence); } return 0; } diff --git a/server/src/tc.c b/server/src/tc.c index e290ac19f..5f2863d8c 100644 --- a/server/src/tc.c +++ b/server/src/tc.c @@ -8,110 +8,199 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ +#include +#include +#include + +#include + #include "utils.h" #include "if.h" #include "tc.h" +#include "nl.h" -int tc_parse(config_setting_t *cfg, struct netem *em) +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **netem) { - em->valid = 0; + const char *str; + int val; + + struct rtnl_qdisc *ne = rtnl_qdisc_alloc(); + if (!ne) + error("Failed to allocated memory!"); - if (config_setting_lookup_string(cfg, "distribution", &em->distribution)) - em->valid |= TC_NETEM_DISTR; - if (config_setting_lookup_int(cfg, "delay", &em->delay)) - em->valid |= TC_NETEM_DELAY; - if (config_setting_lookup_int(cfg, "jitter", &em->jitter)) - em->valid |= TC_NETEM_JITTER; - if (config_setting_lookup_int(cfg, "loss", &em->loss)) - em->valid |= TC_NETEM_LOSS; - if (config_setting_lookup_int(cfg, "duplicate", &em->duplicate)) - em->valid |= TC_NETEM_DUPL; - if (config_setting_lookup_int(cfg, "corrupt", &em->corrupt)) - em->valid |= TC_NETEM_CORRUPT; + rtnl_tc_set_kind(TC_CAST(ne), "netem"); - /** @todo Validate netem config values */ + if (config_setting_lookup_string(cfg, "distribution", &str)) { + if (rtnl_netem_set_delay_distribution(ne, str)) + cerror(cfg, "Invalid delay distribution '%s' in netem config", str); + } + + if (config_setting_lookup_int(cfg, "limit", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for limit setting", val); + + rtnl_netem_set_limit(ne, val); + } + else + rtnl_netem_set_limit(ne, 0); + + if (config_setting_lookup_int(cfg, "delay", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for delay setting", val); + + rtnl_netem_set_delay(ne, val); + } + + if (config_setting_lookup_int(cfg, "jitter", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for jitter setting", val); + + rtnl_netem_set_jitter(ne, val); + } + + if (config_setting_lookup_int(cfg, "loss", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for loss setting", val); + + rtnl_netem_set_loss(ne, val); + } + + if (config_setting_lookup_int(cfg, "duplicate", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for duplicate setting", val); + + rtnl_netem_set_duplicate(ne, val); + } + + if (config_setting_lookup_int(cfg, "corruption", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for corruption setting", val); + + rtnl_netem_set_corruption_probability(ne, val); + } + + *netem = ne; return 0; } +int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne) +{ + *buf = 0; /* start from the beginning */ + + if (rtnl_netem_get_limit(ne) > 0) + strap(buf, len, "limit %upkts", rtnl_netem_get_limit(ne)); + + if (rtnl_netem_get_delay(ne) > 0) { + strap(buf, len, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0); + + if (rtnl_netem_get_jitter(ne) > 0) { + strap(buf, len, "jitter %.2fms ", rtnl_netem_get_jitter(ne) / 1000.0); + + if (rtnl_netem_get_delay_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_delay_correlation(ne)); + } + } + + if (rtnl_netem_get_loss(ne) > 0) { + strap(buf, len, "loss %u%% ", rtnl_netem_get_loss(ne)); + + if (rtnl_netem_get_loss_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_loss_correlation(ne)); + } + + if (rtnl_netem_get_reorder_probability(ne) > 0) { + strap(buf, len, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne)); + + if (rtnl_netem_get_reorder_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_reorder_correlation(ne)); + } + + if (rtnl_netem_get_corruption_probability(ne) > 0) { + strap(buf, len, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne)); + + if (rtnl_netem_get_corruption_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_corruption_correlation(ne)); + } + + if (rtnl_netem_get_duplicate(ne) > 0) { + strap(buf, len, "duplication %u%% ", rtnl_netem_get_duplicate(ne)); + + if (rtnl_netem_get_duplicate_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_duplicate_correlation(ne)); + } + + return 0; +} + +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_qdisc *q = rtnl_qdisc_alloc(); + + /* This is the default priomap used by the tc-prio qdisc + * We will use the first 'bands' bands internally */ + uint8_t map[] = QDISC_PRIO_DEFAULT_PRIOMAP; + for (int i = 0; i < ARRAY_LEN(map); i++) + map[i] += bands; + + rtnl_tc_set_link(TC_CAST(q), i->nl_link); + rtnl_tc_set_parent(TC_CAST(q), parent); + rtnl_tc_set_handle(TC_CAST(q), handle); + rtnl_tc_set_kind(TC_CAST(q), "prio"); + + rtnl_qdisc_prio_set_bands(q, bands + 3); + rtnl_qdisc_prio_set_priomap(q, map, sizeof(map)); + + int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE | NLM_F_REPLACE); + + *qd = q; + + return ret; +} + +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_qdisc *q = *qd; + + rtnl_tc_set_link(TC_CAST(q), i->nl_link); + rtnl_tc_set_parent(TC_CAST(q), parent); + rtnl_tc_set_handle(TC_CAST(q), handle); + //rtnl_tc_set_kind(TC_CAST(q), "netem"); + + int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); + + *qd = q; + + return ret; +} + +int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_cls *c = rtnl_cls_alloc(); + + rtnl_tc_set_link(TC_CAST(c), i->nl_link); + rtnl_tc_set_handle(TC_CAST(c), mark); + rtnl_tc_set_kind(TC_CAST(c), "fw"); + + rtnl_cls_set_protocol(c, ETH_P_ALL); + + rtnl_fw_set_classid(c, flowid); + rtnl_fw_set_mask(c, 0xFFFFFFFF); + + int ret = rtnl_cls_add(sock, c, NLM_F_CREATE); + + *cls = c; + + return ret; +} + int tc_reset(struct interface *i) { - char cmd[128]; - snprintf(cmd, sizeof(cmd), "tc qdisc replace dev %s root pfifo_fast", i->name); + struct nl_sock *sock = nl_init(); - debug(6, "Reset traffic control for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add reset traffic control for interface '%s'", i->name); - - return 0; -} - -int tc_prio(struct interface *i, tc_hdl_t handle, int bands) -{ - char cmd[128]; - int len = 0; - int priomap[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; - - len += snprintf(cmd+len, sizeof(cmd)-len, - "tc qdisc replace dev %s root handle %u prio bands %u priomap", - i->name, TC_HDL_MAJ(handle), bands + 3); - - for (int i = 0; i < 16; i++) - len += snprintf(cmd+len, sizeof(cmd)-len, " %u", priomap[i] + bands); - - debug(6, "Replace master qdisc for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add prio qdisc for interface '%s'", i->name); - - return 0; -} - -int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em) -{ - int len = 0; - char cmd[256]; - len += snprintf(cmd+len, sizeof(cmd)-len, - "tc qdisc replace dev %s parent %u:%u netem", - i->name, TC_HDL_MAJ(parent), TC_HDL_MIN(parent)); - - if (em->valid & TC_NETEM_DELAY) { - len += snprintf(cmd+len, sizeof(cmd)-len, " delay %u", em->delay); - - if (em->valid & TC_NETEM_JITTER) - len += snprintf(cmd+len, sizeof(cmd)-len, " %u", em->jitter); - if (em->valid & TC_NETEM_DISTR) - len += snprintf(cmd+len, sizeof(cmd)-len, " distribution %s", em->distribution); - } - - if (em->valid & TC_NETEM_LOSS) - len += snprintf(cmd+len, sizeof(cmd)-len, " loss random %u", em->loss); - if (em->valid & TC_NETEM_DUPL) - len += snprintf(cmd+len, sizeof(cmd)-len, " duplicate %u", em->duplicate); - if (em->valid & TC_NETEM_CORRUPT) - len += snprintf(cmd+len, sizeof(cmd)-len, " corrupt %u", em->corrupt); - - debug(6, "Setup netem qdisc for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add netem qdisc for interface '%s'", i->name); - - return 0; -} - -int tc_mark(struct interface *i, tc_hdl_t flowid, int mark) -{ - char cmd[128]; - snprintf(cmd, sizeof(cmd), - "tc filter replace dev %s protocol ip handle %u fw flowid %u:%u", - i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid)); - - debug(7, "Add traffic filter to interface '%s': fwmark %u => flowid %u:%u", - i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid)); - - if (system2(cmd)) - error("Failed to add fw_mark classifier for interface '%s'", i->name); - - return 0; + /* We restore the default pfifo_fast qdisc, by deleting ours */ + return rtnl_qdisc_delete(sock, i->tc_qdisc); }