Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

rewrite of network emulation support using libnl3

Browse source 
iproute2 is now not required anymore. everything is handled by nettling sockets
This commit is contained in:
Steffen Vogel 2015-08-22 17:42:02 +02:00
parent 538a222607
commit 0524afa630
10 changed files with 560 additions and 311 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
server/Makefile
View file

@ -5,7 +5,7 @@ OBJS = path.o node.o hooks.o msg.o cfg.o stats.o
# Helper libs
OBJS += utils.o list.o hist.o log.o timing.o
# Node types
OBJS += file.o socket.o if.o tc.o
OBJS += file.o
VPATH = src
@ -13,33 +13,45 @@ VPATH = src
V ?= 2
# Compiler and linker flags
CC = gcc
LDLIBS = -pthread -lrt -lm -lconfig
CFLAGS = -std=gnu99 -Iinclude/ -MMD -Wall -O3
CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V)
override CFLAGS += -std=gnu99 -Iinclude/ -MMD -Wall -O3
override CFLAGS += -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE=1 -DV=$(V)
# Default include paths for third-party libs
PCIDIR ?= /usr/include
NLDIR ?= /usr/include/libnl3
OPALDIR ?= /usr/opalrt/common
#OPALDIR ?= ../contrib/opal
# Add more compiler flags
ifdef DEBUG
CFLAGS += -O0 -g
override CFLAGS += -O0 -g
endif
ifneq (,$(shell which git))
CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"'
override CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"'
endif
# Enabled GTFPGA support when libpci is available
ifneq (,$(wildcard /usr/include/pci/pci.h))
CFLAGS += -DENABLE_GTFPGA
# Enable Socket node type when libnl3 is available
ifneq (,$(wildcard $(NLDIR)/netlink/netlink.h))
override CFLAGS += -DENABLE_SOCKET -I$(NLDIR)
LDLIBS += -lnl-3 -lnl-route-3
OBJS += nl.o tc.o if.o socket.o
endif
# Enable GTFPGA support when libpci is available
ifneq (,$(wildcard $(PCIDIR)/pci/pci.h))
override CFLAGS += -DENABLE_GTFPGA -I$(PCIDIR)
LDLIBS += -lpci
OBJS += gtfpga.o
endif
# Enable OPAL-RT Asynchronous Process support
OPALDIR = /usr/opalrt/common
#OPALDIR = ../contrib/opal
# Enable OPAL-RT Asynchronous Process support (will result in 32bit binary!!!)
ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h))
override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target
override LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/
override LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a)
override OBJS += opal.o
override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target
LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/,-L/usr/lib/i386-linux-gnu/
LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a)
OBJS += opal.o
endif
.PHONY: all clean strip protected

41
server/include/if.h
View file

@ -14,11 +14,10 @@
#define _IF_H_
#include <sys/types.h>
#include <net/if.h>
#include <sys/socket.h>
#include "list.h"
#define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */
#define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */
#ifndef SO_MARK
@ -26,13 +25,16 @@
#endif
struct socket;
struct nl_addr;
struct rtnl_link;
/** Interface data structure */
struct interface {
/** The index used by the kernel to reference this interface */
int index;
/** Human readable name of this interface */
char name[IF_NAME_MAX];
/** libnl3: Handle of interface */
struct rtnl_link *nl_link;
/** libnl3: Root prio qdisc */
struct rtnl_qdisc *tc_qdisc;
/** List of IRQs of the NIC */
char irqs[IF_IRQ_MAX];
@ -42,11 +44,11 @@ struct interface {
/** Add a new interface to the global list and lookup name, irqs...
*
* @param index The interface index of the OS
* @param link The libnl3 link handle
* @retval >0 Success. A pointer to the new interface.
* @retval 0 Error. The creation failed.
*/
struct interface * if_create(int index);
struct interface * if_create(struct rtnl_link *link);
/** Destroy interface by freeing dynamically allocated memory.
@ -78,20 +80,15 @@ int if_start(struct interface *i, int affinity);
*/
int if_stop(struct interface *i);
/** Get outgoing interface.
/** Lookup routing tables to get the interface on which packets for a certain destination
* will leave the system.
*
* Depending on the address family of the socker address,
* this function tries to determine outgoing interface
* which is used to send packages to a remote host with the specified
* socket address.
*
* For AF_INET the fnuction performs a lookup in the kernel routing table.
* For AF_PACKET the function uses the existing sll_ifindex field of the socket address.
*
* @param sa A destination address for outgoing packets.
* @return The interface index.
* @param[in] sa The destination address for outgoing packets.
* @param[out] link The egress interface.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_getegress(struct sockaddr *sa);
int if_get_egress(struct sockaddr *sa, struct rtnl_link **link);
/** Get all IRQs for this interface.
*
@ -102,7 +99,7 @@ int if_getegress(struct sockaddr *sa);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_getirqs(struct interface *i);
int if_get_irqs(struct interface *i);
/** Change the SMP affinity of NIC interrupts.
*
@ -111,7 +108,7 @@ int if_getirqs(struct interface *i);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_setaffinity(struct interface *i, int affinity);
int if_set_affinity(struct interface *i, int affinity);
/** Search the global list of interfaces for a given index.
*

40
server/include/nl.h Normal file
View file

@ -0,0 +1,40 @@
/**
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC
* This file is part of S2SS. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#ifndef _NL_H_
#define _NL_H_
#include <netlink/netlink.h>
#include <netlink/route/route.h>
#include <netlink/route/link.h>
/** Get libnl3 link object from interface name
*
* @param dev The name of the interface
* @retval 0 Error. Something went wrong.
* @retval >0 A pointer to the libnl3 link object.
*/
struct rtnl_link * nl_get_link(int ifindex, const char *dev);
/**
*
*/
int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop);
/**
*
*/
struct nl_sock * nl_init();
/**
*
*/
void nl_shutdown();
#endif

18
server/include/socket.h
View file

@ -16,6 +16,7 @@
#define _SOCKET_H_
#include <sys/socket.h>
#include <linux/if_packet.h>
#include "node.h"
@ -25,6 +26,13 @@ enum socket_layer {
LAYER_UDP
};
union sockaddr_union {
struct sockaddr sa;
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
struct sockaddr_ll sll;
};
struct socket {
/** The socket descriptor */
int sd;
@ -35,12 +43,14 @@ struct socket {
enum socket_layer layer;
/** Local address of the socket */
struct sockaddr_storage local;
union sockaddr_union local;
/** Remote address of the socket */
struct sockaddr_storage remote;
union sockaddr_union remote;
/** Network emulator settings */
struct netem *netem;
/** libnl3: Network emulator queuing discipline */
struct rtnl_qdisc *tc_qdisc;
/** libnl3: Firewall mark classifier */
struct rtnl_cls *tc_classifier;
/* Linked list _per_interface_ */
struct socket *next;

86
server/include/tc.h
View file

@ -16,65 +16,34 @@
#define _TC_H_
#include <stdint.h>
#include <netlink/route/qdisc.h>
#include <netlink/route/classifier.h>
#include <libconfig.h>
/** A type alias for TC handles.
*
* TC handles are used to construct a tree
* of classes, qdiscs and filters.
*/
typedef uint32_t tc_hdl_t;
/** Concatenate 16 bit minor and majar parts to a 32 bit tc handle */
#define TC_HDL(maj, min) ((maj & 0xFFFF) << 16 | (min & 0xFFFF))
/** Get the major part of a tc handle */
#define TC_HDL_MAJ(h) ((h >> 16) & 0xFFFF)
/** Get the minor part of a tc handle */
#define TC_HDL_MIN(h) ((h >> 0) & 0xFFFF)
/** The root handle */
#define TC_HDL_ROOT (0xFFFFFFFFU)
/* Bitfield for valid fields in struct netem */
#define TC_NETEM_DELAY (1 << 0) /**< netem::delay is valid @see netem::valid */
#define TC_NETEM_JITTER (1 << 1) /**< netem::jitter is valid @see netem::valid */
#define TC_NETEM_DISTR (1 << 2) /**< netem::distribution is valid @see netem::valid */
#define TC_NETEM_LOSS (1 << 3) /**< netem::loss is valid @see netem::valid */
#define TC_NETEM_CORRUPT (1 << 4) /**< netem::corrupt is valid @see netem::valid */
#define TC_NETEM_DUPL (1 << 5) /**< netem::duplicate is valid @see netem::valid */
struct interface;
/** Netem configuration settings.
*
* This struct is used to pass the netem configuration
* from config_parse_netem() to tc_netem()
*/
struct netem {
/** Which fields of this struct contain valid data (TC_NETEM_*). */
char valid;
/** Delay distribution: uniform, normal, pareto, paretonormal */
const char *distribution;
/** Added delay (uS) */
int delay;
/** Delay jitter (uS) */
int jitter;
/** Random loss probability (%) */
int loss;
/** Packet corruption probability (%) */
int corrupt;
/** Packet duplication probability (%) */
int duplicate;
};
/** Parse network emulator (netem) settings.
*
* @param cfg A libconfig object containing the settings.
* @param em A pointer to the netem settings structure (part of the path structure).
* @param ne A pointer to a libnl3 qdisc object where setting will be written to.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_parse(config_setting_t *cfg, struct netem *em);
int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne);
/** Print network emulator (netem) setting into buffer.
*
* @param buf A character buffer to write to.
* @param len The length of the supplied buffer.
* @param tc A pointer to the libnl3 qdisc object where settings will be read from.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne);
/** Remove all queuing disciplines and filters.
*
@ -86,32 +55,35 @@ int tc_reset(struct interface *i);
/** Create a priority (prio) queueing discipline.
*
* @param i The interface
* @param handle The handle for the new qdisc
* @param bands The number of classes for this new qdisc
* @param i[in] The interface
* @param qd[in,out] The libnl3 object of the new prio qdisc.
* @param handle[in] The handle for the new qdisc
* @param bands[in] The number of classes for this new qdisc
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_prio(struct interface *i, tc_hdl_t handle, int bands);
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands);
/** Add a new network emulator (netem) discipline.
*
* @param i The interface
* @param parent Make this qdisc a child of
* @param em The netem settings
* @param i[in] The interface to which this qdisc will be added.
* @param qd[in,out] The libnl3 object of the new prio qdisc.
* @param handle[in] The handle of the new qdisc.
* @param parent[in] Make this qdisc a child of this class
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em);
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
/** Add a new filter based on the netfilter mark.
*
* @param i The interface
* @param i The interface to which this classifier is applied to.
* @param cls[in,out] The libnl3 object of the new prio qdisc.
* @param flowid The destination class for matched traffic
* @param mark The netfilter firewall mark (sometime called 'fwmark')
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_mark(struct interface *i, tc_hdl_t flowid, int mark);
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
#endif /* _TC_H_ */

133
server/src/if.c
View file

@ -10,27 +10,30 @@
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <linux/if_packet.h>
#include <netlink/route/link.h>
#include <netlink/route/route.h>
#include "if.h"
#include "tc.h"
#include "nl.h"
#include "socket.h"
#include "utils.h"
/** Linked list of interfaces. */
struct list interfaces;
struct interface * if_create(int index) {
struct interface * if_create(struct rtnl_link *link)
{
struct interface *i = alloc(sizeof(struct interface));
i->nl_link = link;
i->index = index;
if_indextoname(index, i->name);
debug(3, "Created interface '%s'", rtnl_link_get_name(i->nl_link));
debug(3, "Created interface '%s' (index=%u)", i->name, i->index);
if_get_irqs(i);
list_init(&i->sockets, NULL);
list_push(&interfaces, i);
@ -40,22 +43,24 @@ struct interface * if_create(int index) {
void if_destroy(struct interface *i)
{
/* List members are freed by their belonging nodes. */
/* List members are freed by the nodes they belong to. */
list_destroy(&i->sockets);
rtnl_qdisc_put(i->tc_qdisc);
free(i);
}
int if_start(struct interface *i, int affinity)
{
info("Starting interface '%s' (index=%u)", i->name, i->index);
info("Starting interface '%s' which is used by %u sockets", rtnl_link_get_name(i->nl_link), list_length(&i->sockets));
{ INDENT
/* Assign fwmark's to socket nodes which have netem options */
int mark = 0;
FOREACH(&i->sockets, it) {
struct socket *s = it->socket;
if (s->netem)
if (s->tc_qdisc)
s->mark = 1 + mark++;
}
@ -64,20 +69,24 @@ int if_start(struct interface *i, int affinity)
return 0;
/* Replace root qdisc */
tc_prio(i, TC_HDL(4000, 0), mark);
tc_prio(i, &i->tc_qdisc, TC_HANDLE(4000, 0), mark);
/* Create netem qdisks and appropriate filter per netem node */
FOREACH(&i->sockets, it) {
struct socket *s = it->socket;
if (s->netem) {
tc_mark(i, TC_HDL(4000, s->mark), s->mark);
tc_netem(i, TC_HDL(4000, s->mark), s->netem);
if (s->tc_qdisc) {
tc_mark(i, &s->tc_classifier, TC_HANDLE(4000, s->mark), s->mark);
tc_netem(i, &s->tc_qdisc, TC_HANDLE(8000, s->mark), TC_HANDLE(4000, s->mark));
char buf[256];
tc_print(buf, sizeof(buf), s->tc_qdisc);
debug(5, "Starting network emulation on interface '%s' for FW mark %u: %s",
rtnl_link_get_name(i->nl_link), s->mark, buf);
}
}
/* Set affinity for network interfaces (skip _loopback_ dev) */
if_getirqs(i);
if_setaffinity(i, affinity);
if_set_affinity(i, affinity);
}
return 0;
@ -85,65 +94,55 @@ int if_start(struct interface *i, int affinity)
int if_stop(struct interface *i)
{
info("Stopping interface '%s' (index=%u)", i->name, i->index);
info("Stopping interface '%s'", rtnl_link_get_name(i->nl_link));
{ INDENT
if_setaffinity(i, -1L);
if_set_affinity(i, -1L);
/* Only reset tc if it was initialized before */
FOREACH(&i->sockets, it) {
if (it->socket->netem) {
tc_reset(i);
break;
}
if (i->tc_qdisc)
tc_reset(i);
}
return 0;
}
int if_get_egress(struct sockaddr *sa, struct rtnl_link **link)
{
switch (sa->sa_family) {
case AF_INET:
case AF_INET6: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
struct nl_addr *addr = (sa->sa_family == AF_INET)
? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
: nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
struct rtnl_nexthop *nh;
if (nl_get_nexthop(addr, &nh))
return -1;
*link = nl_get_link(rtnl_route_nh_get_ifindex(nh), NULL);
rtnl_route_nh_free(nh);
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
*link = nl_get_link(sll->sll_ifindex, NULL);
}
}
return 0;
}
int if_getegress(struct sockaddr *sa)
{
switch (sa->sa_family) {
case AF_INET: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
char cmd[128];
char token[32];
snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sin->sin_addr));
debug(8, "System: %s", cmd);
FILE *p = popen(cmd, "r");
if (!p)
return -1;
while (!feof(p) && fscanf(p, "%31s", token)) {
if (!strcmp(token, "dev")) {
fscanf(p, "%31s", token);
break;
}
}
return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token);
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
return sll->sll_ifindex;
}
default:
return -1;
}
}
int if_getirqs(struct interface *i)
int if_get_irqs(struct interface *i)
{
char dirname[NAME_MAX];
int irq, n = 0;
snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name);
snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", rtnl_link_get_name(i->nl_link));
DIR *dir = opendir(dirname);
if (dir) {
memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX);
@ -156,11 +155,13 @@ int if_getirqs(struct interface *i)
closedir(dir);
}
debug(6, "Found %u IRQs for interface '%s'", n, rtnl_link_get_name(i->nl_link));
return 0;
}
int if_setaffinity(struct interface *i, int affinity)
int if_set_affinity(struct interface *i, int affinity)
{
char filename[NAME_MAX];
FILE *file;
@ -174,10 +175,10 @@ int if_setaffinity(struct interface *i, int affinity)
error("Failed to set affinity for IRQ %u", i->irqs[n]);
fclose(file);
debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], i->name, affinity);
debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], rtnl_link_get_name(i->nl_link), affinity);
}
else
error("Failed to set affinity for interface '%s'", i->name);
error("Failed to set affinity for interface '%s'", rtnl_link_get_name(i->nl_link));
}
return 0;
@ -186,7 +187,7 @@ int if_setaffinity(struct interface *i, int affinity)
struct interface * if_lookup_index(int index)
{
FOREACH(&interfaces, it) {
if (it->interface->index == index)
if (rtnl_link_get_ifindex(it->interface->nl_link) == index)
return it->interface;
}

94
server/src/nl.c Normal file
View file

@ -0,0 +1,94 @@
/** Netlink related functions
*
* S2SS uses libnl3 to talk to the Linux kernel to gather networking related information
*
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC
* This file is part of S2SS. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#include <stdio.h>
#include <netlink/route/route.h>
#include "utils.h"
#include "nl.h"
/** Singleton for netlink socket */
static struct nl_sock *sock = NULL;
struct nl_sock * nl_init()
{
if (!sock) {
/* Create connection to netlink */
sock = nl_socket_alloc();
if (!sock)
error("Failed to allocate memory");
nl_connect(sock, NETLINK_ROUTE);
}
return sock;
}
void nl_shutdown()
{
nl_close(sock);
nl_socket_free(sock);
sock = NULL;
}
struct rtnl_link * nl_get_link(int ifindex, const char *dev)
{
struct nl_sock *sock = nl_init();
struct rtnl_link *link;
if (rtnl_link_get_kernel(sock, ifindex, dev, &link))
return NULL;
else
return link;
}
int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop)
{
int ret;
struct rtnl_route *route;
struct rtnl_nexthop *nh;
struct nl_sock *sock = nl_init();
struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0);
struct nl_cache_ops *ops = nl_cache_ops_lookup_safe("route/route");
struct nl_cache *cache = nl_cache_alloc(ops);
struct rtmsg rmsg = {
.rtm_family = nl_addr_get_family(addr),
.rtm_dst_len = nl_addr_get_prefixlen(addr),
};
if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO)) < 0)
return ret;
if ((ret = nla_put_addr(msg, RTA_DST, addr)) < 0)
return ret;
if ((ret = nl_send_auto_complete(sock, msg)) < 0)
return ret;
if ((ret = nl_cache_pickup(sock, cache)) < 0)
return ret;
if (!(route = (struct rtnl_route *) nl_cache_get_first(cache)))
return -1;
if (!(nh = rtnl_route_nexthop_n(route, 0)))
return -1;
*nexthop = rtnl_route_nh_clone(nh);
nl_cache_put(cache);
nl_cache_ops_put(ops);
nlmsg_free(msg);
return 0;
}

15
server/src/node.c
View file

@ -14,14 +14,18 @@
/* Node types */
#include "file.h"
#include "socket.h"
#ifdef ENABLE_GTFPGA
#include "gtfpga.h"
#endif
#ifdef ENABLE_OPAL_ASYNC
#include "opal.h"
#endif
#ifdef ENABLE_SOCKET
#include "socket.h"
#include <netlink/route/qdisc.h>
#include <netlink/route/classifier.h>
#endif
#define VTABLE(type, name, fnc) { type, name, \
fnc ## _parse, fnc ## _print, \
@ -37,7 +41,9 @@ struct node_vtable vtables[] = {
#ifdef ENABLE_GTFPGA
VTABLE(GTFPGA, "gtfpga", gtfpga),
#endif
#ifdef ENABLE_SOCKET
VTABLE(BSD_SOCKET, "socket", socket),
#endif
VTABLE(LOG_FILE, "file", file)
};
@ -137,9 +143,12 @@ struct node * node_create()
void node_destroy(struct node *n)
{
switch (n->vt->type) {
#ifdef ENABLE_SOCKET
case BSD_SOCKET:
free(n->socket->netem);
rtnl_qdisc_put(n->socket->tc_qdisc);
rtnl_cls_put(n->socket->tc_classifier);
break;
#endif
case LOG_FILE:
free(n->file->path_in);
free(n->file->path_out);

138
server/src/socket.c
View file

@ -13,24 +13,23 @@
#include <string.h>
#include <unistd.h>
#include <poll.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <net/if.h>
#include <net/ethernet.h>
#include <netinet/ip.h>
#include <netinet/ether.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <sys/un.h>
#include <netinet/ether.h>
#include <netinet/ip.h>
#include <linux/if_packet.h>
#include <arpa/inet.h>
#include "if.h"
#include "nl.h"
#include "tc.h"
#include "config.h"
#include "utils.h"
#include "socket.h"
#include "if.h"
/** Linked list of interfaces */
extern struct list interfaces;
@ -45,22 +44,24 @@ int socket_init(int argc, char * argv[], struct settings *set)
/* Gather list of used network interfaces */
FOREACH(&sockets, it) {
struct socket *s = it->socket;
struct rtnl_link *link;
/* Determine outgoing interface */
int index = if_getegress((struct sockaddr *) &s->remote);
if (index < 0) {
if (if_get_egress((struct sockaddr *) &s->remote, &link) || !link) {
char buf[128];
socket_print_addr(buf, sizeof(buf), (struct sockaddr *) &s->remote);
error("Failed to get interface for socket address '%s'", buf);
}
int index = rtnl_link_get_ifindex(link);
struct interface *i = if_lookup_index(index);
if (!i)
i = if_create(index);
i = if_create(link);
list_push(&i->sockets, s);
}
/** @todo Improve mapping of NIC IRQs per path */
FOREACH(&interfaces, it)
if_start(it->interface, set->affinity);
@ -83,11 +84,18 @@ int socket_print(struct node *n, char *buf, int len)
char local[INET6_ADDRSTRLEN + 16];
char remote[INET6_ADDRSTRLEN + 16];
char *layer = NULL;
switch (s->layer) {
case LAYER_UDP: layer = "udp"; break;
case LAYER_IP: layer = "ip"; break;
case LAYER_ETH: layer = "eth"; break;
}
socket_print_addr(local, sizeof(local), (struct sockaddr *) &s->local);
socket_print_addr(remote, sizeof(remote), (struct sockaddr *) &s->remote);
return snprintf(buf, len, "local=%s, remote=%s", local, remote);
return snprintf(buf, len, "layer=%s, local=%s, remote=%s", layer, local, remote);
}
int socket_open(struct node *n)
@ -116,9 +124,9 @@ int socket_open(struct node *n)
/* Set fwmark for outgoing packets */
if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark)))
serror("Failed to set fwmark for outgoing packets");
serror("Failed to set FW mark for outgoing packets");
else
debug(4, "Set fwmark for socket (sd=%u) to %u", s->sd, s->mark);
debug(4, "Set FW mark for socket (sd=%u) to %u", s->sd, s->mark);
/* Set socket priority, QoS or TOS IP options */
int prio;
@ -170,11 +178,11 @@ int socket_read(struct node *n, struct msg *pool, int poolsize, int first, int c
/* Get size of received packet in bytes */
ioctl(s->sd, FIONREAD, &bytes);
/* Check packet integrity */
/* Check if packet length is correct */
if (bytes % (cnt * 4) != 0)
error("Packet length not dividable by 4: received=%u, cnt=%u", bytes, cnt);
if (bytes / cnt > sizeof(struct msg))
error("Packet length is too large: received=%u, cnt=%u, max=%lu", bytes, cnt, sizeof(struct msg));
error("Packet length is too large: received=%u, cnt=%u, max=%zu", bytes, cnt, sizeof(struct msg));
for (int i = 0; i < cnt; i++) {
/* All messages of a packet must have equal length! */
@ -283,25 +291,28 @@ int socket_parse(config_setting_t *cfg, struct node *n)
if (!config_setting_lookup_string(cfg, "local", &local))
cerror(cfg, "Missing local address for node '%s'", n->name);
ret = socket_parse_addr(local, (struct sockaddr *) &s->local, s->layer, AI_PASSIVE);
if (ret)
if (ret) {
cerror(cfg, "Failed to resolve local address '%s' of node '%s': %s",
local, n->name, gai_strerror(ret));
}
ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, s->layer, 0);
if (ret)
if (ret) {
cerror(cfg, "Failed to resolve remote address '%s' of node '%s': %s",
remote, n->name, gai_strerror(ret));
}
/** @todo Netem settings are not usable with AF_UNIX */
config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem");
if (cfg_netem) {
int enabled = 1;
if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) {
s->netem = alloc(sizeof(struct netem));
tc_parse(cfg_netem, s->netem);
s->tc_qdisc = rtnl_qdisc_alloc();
if (!s->tc_qdisc)
error("Failed to allocated memory!");
tc_parse(cfg_netem, s->tc_qdisc);
}
}
@ -312,42 +323,60 @@ int socket_parse(config_setting_t *cfg, struct node *n)
return 0;
}
int socket_print_addr(char *buf, int len, struct sockaddr *sa)
int socket_print_addr(char *buf, int len, struct sockaddr *saddr)
{
switch (sa->sa_family) {
case AF_INET: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
inet_ntop(sin->sin_family, &sin->sin_addr, buf, len);
return snprintf(buf+strlen(buf), len-strlen(buf), ":%hu", ntohs(sin->sin_port));
}
union sockaddr_union *sa = (union sockaddr_union *) saddr;
/* Address */
switch (sa->sa.sa_family) {
case AF_INET6:
inet_ntop(AF_INET6, &sa->sin6.sin6_addr, buf, len);
break;
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
char ifname[IF_NAMESIZE];
return snprintf(buf, len, "%s%%%s:%#hx",
ether_ntoa((struct ether_addr *) &sll->sll_addr),
if_indextoname(sll->sll_ifindex, ifname),
ntohs(sll->sll_protocol));
}
case AF_INET:
inet_ntop(AF_INET, &sa->sin.sin_addr, buf, len);
break;
case AF_PACKET:
snprintf(buf, len, "%02x", sa->sll.sll_addr[0]);
for (int i = 1; i < sa->sll.sll_halen; i++)
strap(buf, len, ":%02x", sa->sll.sll_addr[i]);
break;
default:
return snprintf(buf, len, "address family: %u", sa->sa_family);
error("Unknown address family: '%u'", sa->sa.sa_family);
}
/* Port / Interface */
switch (sa->sa.sa_family) {
case AF_INET6:
case AF_INET:
strap(buf, len, ":%hu", ntohs(sa->sin.sin_port));
break;
case AF_PACKET: {
struct rtnl_link *link = nl_get_link(sa->sll.sll_ifindex, NULL);
if (!link)
error("Failed to get interface for index: %u", sa->sll.sll_ifindex);
strap(buf, len, "%%%s", rtnl_link_get_name(link));
strap(buf, len, ":%hu", ntohs(sa->sll.sll_protocol));
break;
}
}
return 0;
}
int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer layer, int flags)
int socket_parse_addr(const char *addr, struct sockaddr *saddr, enum socket_layer layer, int flags)
{
/** @todo: Add support for IPv6 */
union sockaddr_union *sa = (union sockaddr_union *) saddr;
char *copy = strdup(addr);
int ret;
if (layer == LAYER_ETH) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
/* Split string */
char *node = strtok(copy, "%");
char *ifname = strtok(NULL, ":");
@ -356,21 +385,26 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer l
/* Parse link layer (MAC) address */
struct ether_addr *mac = ether_aton(node);
if (!mac)
error("Failed to parse mac address: %s", node);
error("Failed to parse MAC address: %s", node);
memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6);
memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, 6);
/* Get interface index from name */
struct rtnl_link *link = nl_get_link(0, ifname);
if (!link)
error("Failed to get network interface: '%s'", ifname);
sll->sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS);
sll->sll_halen = 6;
sll->sll_family = AF_PACKET;
sll->sll_ifindex = if_nametoindex(ifname);
sa->sll.sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS);
sa->sll.sll_halen = 6;
sa->sll.sll_family = AF_PACKET;
sa->sll.sll_ifindex = rtnl_link_get_ifindex(link);
ret = 0;
}
else { /* Format: "192.168.0.10:12001" */
struct addrinfo hint = {
.ai_flags = flags,
.ai_family = AF_INET
.ai_family = AF_UNSPEC
};
/* Split string */

264
server/src/tc.c
View file

@ -8,110 +8,190 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#include <netlink/route/cls/fw.h>
#include <netlink/route/qdisc/netem.h>
#include <netlink/route/qdisc/prio.h>
#include <linux/if_ether.h>
#include "utils.h"
#include "if.h"
#include "tc.h"
#include "nl.h"
int tc_parse(config_setting_t *cfg, struct netem *em)
int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne)
{
em->valid = 0;
const char *str;
int val;
rtnl_tc_set_kind(TC_CAST(ne), "netem");
if (config_setting_lookup_string(cfg, "distribution", &em->distribution))
em->valid |= TC_NETEM_DISTR;
if (config_setting_lookup_int(cfg, "delay", &em->delay))
em->valid |= TC_NETEM_DELAY;
if (config_setting_lookup_int(cfg, "jitter", &em->jitter))
em->valid |= TC_NETEM_JITTER;
if (config_setting_lookup_int(cfg, "loss", &em->loss))
em->valid |= TC_NETEM_LOSS;
if (config_setting_lookup_int(cfg, "duplicate", &em->duplicate))
em->valid |= TC_NETEM_DUPL;
if (config_setting_lookup_int(cfg, "corrupt", &em->corrupt))
em->valid |= TC_NETEM_CORRUPT;
if (config_setting_lookup_string(cfg, "distribution", &str)) {
if (rtnl_netem_set_delay_distribution(ne, str))
cerror(cfg, "Invalid delay distribution '%s' in netem config", str);
}
if (config_setting_lookup_int(cfg, "limit", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for limit setting", val);
/** @todo Validate netem config values */
rtnl_netem_set_limit(ne, val);
}
else
rtnl_netem_set_limit(ne, 0);
if (config_setting_lookup_int(cfg, "delay", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for delay setting", val);
rtnl_netem_set_delay(ne, val);
}
if (config_setting_lookup_int(cfg, "jitter", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for jitter setting", val);
rtnl_netem_set_jitter(ne, val);
}
if (config_setting_lookup_int(cfg, "loss", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for loss setting", val);
rtnl_netem_set_loss(ne, val);
}
if (config_setting_lookup_int(cfg, "duplicate", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for duplicate setting", val);
rtnl_netem_set_duplicate(ne, val);
}
if (config_setting_lookup_int(cfg, "corruption", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for corruption setting", val);
rtnl_netem_set_corruption_probability(ne, val);
}
return 0;
}
int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne)
{
if (rtnl_netem_get_limit(ne) > 0)
strap(buf, len, "limit %upkts", rtnl_netem_get_limit(ne));
if (rtnl_netem_get_delay(ne) > 0) {
strap(buf, len, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0);
if (rtnl_netem_get_jitter(ne) > 0) {
strap(buf, len, "jitter %f.2ms ", rtnl_netem_get_jitter(ne) / 1000.0);
if (rtnl_netem_get_delay_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_delay_correlation(ne));
}
}
if (rtnl_netem_get_loss(ne) > 0) {
strap(buf, len, "loss %u%% ", rtnl_netem_get_loss(ne));
if (rtnl_netem_get_loss_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_loss_correlation(ne));
}
if (rtnl_netem_get_reorder_probability(ne) > 0) {
strap(buf, len, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne));
if (rtnl_netem_get_reorder_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_reorder_correlation(ne));
}
if (rtnl_netem_get_corruption_probability(ne) > 0) {
strap(buf, len, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne));
if (rtnl_netem_get_corruption_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_corruption_correlation(ne));
}
if (rtnl_netem_get_duplicate(ne) > 0) {
strap(buf, len, "duplication %u%% ", rtnl_netem_get_duplicate(ne));
if (rtnl_netem_get_duplicate_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_duplicate_correlation(ne));
}
return 0;
}
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands)
{
struct nl_sock *sock = nl_init();
struct rtnl_qdisc *q = rtnl_qdisc_alloc();
/* This is the default priomap used by the tc-prio qdisc
* We will use the first 'bands' bands internally */
uint8_t map[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
for (int i = 0; i < ARRAY_LEN(map); i++)
map[i] += bands;
rtnl_tc_set_link(TC_CAST(q), i->nl_link);
rtnl_tc_set_parent(TC_CAST(q), TC_H_ROOT);
rtnl_tc_set_handle(TC_CAST(q), handle);
rtnl_tc_set_kind(TC_CAST(q), "prio");
rtnl_qdisc_prio_set_bands(q, bands + 3);
rtnl_qdisc_prio_set_priomap(q, map, ARRAY_LEN(map));
int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE);
*qd = q;
return ret;
}
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
{
struct nl_sock *sock = nl_init();
struct rtnl_qdisc *q = *qd;
rtnl_tc_set_link(TC_CAST(q), i->nl_link);
rtnl_tc_set_parent(TC_CAST(q), parent);
rtnl_tc_set_handle(TC_CAST(q), handle);
int ret = rtnl_qdisc_add(sock, q, NLM_F_REPLACE);
*qd = q;
return ret;
}
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
{
struct nl_sock *sock = nl_init();
struct rtnl_cls *c = rtnl_cls_alloc();
rtnl_tc_set_link(TC_CAST(c), i->nl_link);
rtnl_tc_set_handle(TC_CAST(c), mark);
rtnl_tc_set_kind(TC_CAST(c), "fw");
rtnl_cls_set_protocol(c, ETH_P_ALL);
rtnl_fw_set_classid(c, flowid);
rtnl_fw_set_mask(c, 0xFFFFFFFF);
int ret = rtnl_cls_add(sock, c, NLM_F_CREATE);
*cls = c;
return ret;
}
int tc_reset(struct interface *i)
{
char cmd[128];
snprintf(cmd, sizeof(cmd), "tc qdisc replace dev %s root pfifo_fast", i->name);
struct nl_sock *sock = nl_init();
debug(6, "Reset traffic control for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add reset traffic control for interface '%s'", i->name);
return 0;
}
int tc_prio(struct interface *i, tc_hdl_t handle, int bands)
{
char cmd[128];
int len = 0;
int priomap[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
len += snprintf(cmd+len, sizeof(cmd)-len,
"tc qdisc replace dev %s root handle %u prio bands %u priomap",
i->name, TC_HDL_MAJ(handle), bands + 3);
for (int i = 0; i < 16; i++)
len += snprintf(cmd+len, sizeof(cmd)-len, " %u", priomap[i] + bands);
debug(6, "Replace master qdisc for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add prio qdisc for interface '%s'", i->name);
return 0;
}
int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em)
{
int len = 0;
char cmd[256];
len += snprintf(cmd+len, sizeof(cmd)-len,
"tc qdisc replace dev %s parent %u:%u netem",
i->name, TC_HDL_MAJ(parent), TC_HDL_MIN(parent));
if (em->valid & TC_NETEM_DELAY) {
len += snprintf(cmd+len, sizeof(cmd)-len, " delay %u", em->delay);
if (em->valid & TC_NETEM_JITTER)
len += snprintf(cmd+len, sizeof(cmd)-len, " %u", em->jitter);
if (em->valid & TC_NETEM_DISTR)
len += snprintf(cmd+len, sizeof(cmd)-len, " distribution %s", em->distribution);
}
if (em->valid & TC_NETEM_LOSS)
len += snprintf(cmd+len, sizeof(cmd)-len, " loss random %u", em->loss);
if (em->valid & TC_NETEM_DUPL)
len += snprintf(cmd+len, sizeof(cmd)-len, " duplicate %u", em->duplicate);
if (em->valid & TC_NETEM_CORRUPT)
len += snprintf(cmd+len, sizeof(cmd)-len, " corrupt %u", em->corrupt);
debug(6, "Setup netem qdisc for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add netem qdisc for interface '%s'", i->name);
return 0;
}
int tc_mark(struct interface *i, tc_hdl_t flowid, int mark)
{
char cmd[128];
snprintf(cmd, sizeof(cmd),
"tc filter replace dev %s protocol ip handle %u fw flowid %u:%u",
i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid));
debug(7, "Add traffic filter to interface '%s': fwmark %u => flowid %u:%u",
i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid));
if (system2(cmd))
error("Failed to add fw_mark classifier for interface '%s'", i->name);
return 0;
/* We restore the default pfifo_fast qdisc, by deleting ours */
return rtnl_qdisc_delete(sock, i->tc_qdisc);
}