1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00

rewrite of network emulation support using libnl3

iproute2 is now not required anymore. everything is handled by nettling sockets
This commit is contained in:
Steffen Vogel 2015-08-22 17:42:02 +02:00
parent 538a222607
commit 0524afa630
10 changed files with 560 additions and 311 deletions

View file

@ -5,7 +5,7 @@ OBJS = path.o node.o hooks.o msg.o cfg.o stats.o
# Helper libs
OBJS += utils.o list.o hist.o log.o timing.o
# Node types
OBJS += file.o socket.o if.o tc.o
OBJS += file.o
VPATH = src
@ -13,33 +13,45 @@ VPATH = src
V ?= 2
# Compiler and linker flags
CC = gcc
LDLIBS = -pthread -lrt -lm -lconfig
CFLAGS = -std=gnu99 -Iinclude/ -MMD -Wall -O3
CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V)
override CFLAGS += -std=gnu99 -Iinclude/ -MMD -Wall -O3
override CFLAGS += -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE=1 -DV=$(V)
# Default include paths for third-party libs
PCIDIR ?= /usr/include
NLDIR ?= /usr/include/libnl3
OPALDIR ?= /usr/opalrt/common
#OPALDIR ?= ../contrib/opal
# Add more compiler flags
ifdef DEBUG
CFLAGS += -O0 -g
override CFLAGS += -O0 -g
endif
ifneq (,$(shell which git))
CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"'
override CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"'
endif
# Enabled GTFPGA support when libpci is available
ifneq (,$(wildcard /usr/include/pci/pci.h))
CFLAGS += -DENABLE_GTFPGA
# Enable Socket node type when libnl3 is available
ifneq (,$(wildcard $(NLDIR)/netlink/netlink.h))
override CFLAGS += -DENABLE_SOCKET -I$(NLDIR)
LDLIBS += -lnl-3 -lnl-route-3
OBJS += nl.o tc.o if.o socket.o
endif
# Enable GTFPGA support when libpci is available
ifneq (,$(wildcard $(PCIDIR)/pci/pci.h))
override CFLAGS += -DENABLE_GTFPGA -I$(PCIDIR)
LDLIBS += -lpci
OBJS += gtfpga.o
endif
# Enable OPAL-RT Asynchronous Process support
OPALDIR = /usr/opalrt/common
#OPALDIR = ../contrib/opal
# Enable OPAL-RT Asynchronous Process support (will result in 32bit binary!!!)
ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h))
override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target
override LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/
override LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a)
override OBJS += opal.o
override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target
LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/,-L/usr/lib/i386-linux-gnu/
LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a)
OBJS += opal.o
endif
.PHONY: all clean strip protected

View file

@ -14,11 +14,10 @@
#define _IF_H_
#include <sys/types.h>
#include <net/if.h>
#include <sys/socket.h>
#include "list.h"
#define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */
#define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */
#ifndef SO_MARK
@ -26,13 +25,16 @@
#endif
struct socket;
struct nl_addr;
struct rtnl_link;
/** Interface data structure */
struct interface {
/** The index used by the kernel to reference this interface */
int index;
/** Human readable name of this interface */
char name[IF_NAME_MAX];
/** libnl3: Handle of interface */
struct rtnl_link *nl_link;
/** libnl3: Root prio qdisc */
struct rtnl_qdisc *tc_qdisc;
/** List of IRQs of the NIC */
char irqs[IF_IRQ_MAX];
@ -42,11 +44,11 @@ struct interface {
/** Add a new interface to the global list and lookup name, irqs...
*
* @param index The interface index of the OS
* @param link The libnl3 link handle
* @retval >0 Success. A pointer to the new interface.
* @retval 0 Error. The creation failed.
*/
struct interface * if_create(int index);
struct interface * if_create(struct rtnl_link *link);
/** Destroy interface by freeing dynamically allocated memory.
@ -78,20 +80,15 @@ int if_start(struct interface *i, int affinity);
*/
int if_stop(struct interface *i);
/** Get outgoing interface.
/** Lookup routing tables to get the interface on which packets for a certain destination
* will leave the system.
*
* Depending on the address family of the socker address,
* this function tries to determine outgoing interface
* which is used to send packages to a remote host with the specified
* socket address.
*
* For AF_INET the fnuction performs a lookup in the kernel routing table.
* For AF_PACKET the function uses the existing sll_ifindex field of the socket address.
*
* @param sa A destination address for outgoing packets.
* @return The interface index.
* @param[in] sa The destination address for outgoing packets.
* @param[out] link The egress interface.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_getegress(struct sockaddr *sa);
int if_get_egress(struct sockaddr *sa, struct rtnl_link **link);
/** Get all IRQs for this interface.
*
@ -102,7 +99,7 @@ int if_getegress(struct sockaddr *sa);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_getirqs(struct interface *i);
int if_get_irqs(struct interface *i);
/** Change the SMP affinity of NIC interrupts.
*
@ -111,7 +108,7 @@ int if_getirqs(struct interface *i);
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int if_setaffinity(struct interface *i, int affinity);
int if_set_affinity(struct interface *i, int affinity);
/** Search the global list of interfaces for a given index.
*

40
server/include/nl.h Normal file
View file

@ -0,0 +1,40 @@
/**
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC
* This file is part of S2SS. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#ifndef _NL_H_
#define _NL_H_
#include <netlink/netlink.h>
#include <netlink/route/route.h>
#include <netlink/route/link.h>
/** Get libnl3 link object from interface name
*
* @param dev The name of the interface
* @retval 0 Error. Something went wrong.
* @retval >0 A pointer to the libnl3 link object.
*/
struct rtnl_link * nl_get_link(int ifindex, const char *dev);
/**
*
*/
int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop);
/**
*
*/
struct nl_sock * nl_init();
/**
*
*/
void nl_shutdown();
#endif

View file

@ -16,6 +16,7 @@
#define _SOCKET_H_
#include <sys/socket.h>
#include <linux/if_packet.h>
#include "node.h"
@ -25,6 +26,13 @@ enum socket_layer {
LAYER_UDP
};
union sockaddr_union {
struct sockaddr sa;
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
struct sockaddr_ll sll;
};
struct socket {
/** The socket descriptor */
int sd;
@ -35,12 +43,14 @@ struct socket {
enum socket_layer layer;
/** Local address of the socket */
struct sockaddr_storage local;
union sockaddr_union local;
/** Remote address of the socket */
struct sockaddr_storage remote;
union sockaddr_union remote;
/** Network emulator settings */
struct netem *netem;
/** libnl3: Network emulator queuing discipline */
struct rtnl_qdisc *tc_qdisc;
/** libnl3: Firewall mark classifier */
struct rtnl_cls *tc_classifier;
/* Linked list _per_interface_ */
struct socket *next;

View file

@ -16,65 +16,34 @@
#define _TC_H_
#include <stdint.h>
#include <netlink/route/qdisc.h>
#include <netlink/route/classifier.h>
#include <libconfig.h>
/** A type alias for TC handles.
*
* TC handles are used to construct a tree
* of classes, qdiscs and filters.
*/
typedef uint32_t tc_hdl_t;
/** Concatenate 16 bit minor and majar parts to a 32 bit tc handle */
#define TC_HDL(maj, min) ((maj & 0xFFFF) << 16 | (min & 0xFFFF))
/** Get the major part of a tc handle */
#define TC_HDL_MAJ(h) ((h >> 16) & 0xFFFF)
/** Get the minor part of a tc handle */
#define TC_HDL_MIN(h) ((h >> 0) & 0xFFFF)
/** The root handle */
#define TC_HDL_ROOT (0xFFFFFFFFU)
/* Bitfield for valid fields in struct netem */
#define TC_NETEM_DELAY (1 << 0) /**< netem::delay is valid @see netem::valid */
#define TC_NETEM_JITTER (1 << 1) /**< netem::jitter is valid @see netem::valid */
#define TC_NETEM_DISTR (1 << 2) /**< netem::distribution is valid @see netem::valid */
#define TC_NETEM_LOSS (1 << 3) /**< netem::loss is valid @see netem::valid */
#define TC_NETEM_CORRUPT (1 << 4) /**< netem::corrupt is valid @see netem::valid */
#define TC_NETEM_DUPL (1 << 5) /**< netem::duplicate is valid @see netem::valid */
struct interface;
/** Netem configuration settings.
*
* This struct is used to pass the netem configuration
* from config_parse_netem() to tc_netem()
*/
struct netem {
/** Which fields of this struct contain valid data (TC_NETEM_*). */
char valid;
/** Delay distribution: uniform, normal, pareto, paretonormal */
const char *distribution;
/** Added delay (uS) */
int delay;
/** Delay jitter (uS) */
int jitter;
/** Random loss probability (%) */
int loss;
/** Packet corruption probability (%) */
int corrupt;
/** Packet duplication probability (%) */
int duplicate;
};
/** Parse network emulator (netem) settings.
*
* @param cfg A libconfig object containing the settings.
* @param em A pointer to the netem settings structure (part of the path structure).
* @param ne A pointer to a libnl3 qdisc object where setting will be written to.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_parse(config_setting_t *cfg, struct netem *em);
int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne);
/** Print network emulator (netem) setting into buffer.
*
* @param buf A character buffer to write to.
* @param len The length of the supplied buffer.
* @param tc A pointer to the libnl3 qdisc object where settings will be read from.
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne);
/** Remove all queuing disciplines and filters.
*
@ -86,32 +55,35 @@ int tc_reset(struct interface *i);
/** Create a priority (prio) queueing discipline.
*
* @param i The interface
* @param handle The handle for the new qdisc
* @param bands The number of classes for this new qdisc
* @param i[in] The interface
* @param qd[in,out] The libnl3 object of the new prio qdisc.
* @param handle[in] The handle for the new qdisc
* @param bands[in] The number of classes for this new qdisc
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_prio(struct interface *i, tc_hdl_t handle, int bands);
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands);
/** Add a new network emulator (netem) discipline.
*
* @param i The interface
* @param parent Make this qdisc a child of
* @param em The netem settings
* @param i[in] The interface to which this qdisc will be added.
* @param qd[in,out] The libnl3 object of the new prio qdisc.
* @param handle[in] The handle of the new qdisc.
* @param parent[in] Make this qdisc a child of this class
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em);
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent);
/** Add a new filter based on the netfilter mark.
*
* @param i The interface
* @param i The interface to which this classifier is applied to.
* @param cls[in,out] The libnl3 object of the new prio qdisc.
* @param flowid The destination class for matched traffic
* @param mark The netfilter firewall mark (sometime called 'fwmark')
* @retval 0 Success. Everything went well.
* @retval <0 Error. Something went wrong.
*/
int tc_mark(struct interface *i, tc_hdl_t flowid, int mark);
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark);
#endif /* _TC_H_ */

View file

@ -10,27 +10,30 @@
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <linux/if_packet.h>
#include <netlink/route/link.h>
#include <netlink/route/route.h>
#include "if.h"
#include "tc.h"
#include "nl.h"
#include "socket.h"
#include "utils.h"
/** Linked list of interfaces. */
struct list interfaces;
struct interface * if_create(int index) {
struct interface * if_create(struct rtnl_link *link)
{
struct interface *i = alloc(sizeof(struct interface));
i->nl_link = link;
i->index = index;
if_indextoname(index, i->name);
debug(3, "Created interface '%s'", rtnl_link_get_name(i->nl_link));
debug(3, "Created interface '%s' (index=%u)", i->name, i->index);
if_get_irqs(i);
list_init(&i->sockets, NULL);
list_push(&interfaces, i);
@ -40,22 +43,24 @@ struct interface * if_create(int index) {
void if_destroy(struct interface *i)
{
/* List members are freed by their belonging nodes. */
/* List members are freed by the nodes they belong to. */
list_destroy(&i->sockets);
rtnl_qdisc_put(i->tc_qdisc);
free(i);
}
int if_start(struct interface *i, int affinity)
{
info("Starting interface '%s' (index=%u)", i->name, i->index);
info("Starting interface '%s' which is used by %u sockets", rtnl_link_get_name(i->nl_link), list_length(&i->sockets));
{ INDENT
/* Assign fwmark's to socket nodes which have netem options */
int mark = 0;
FOREACH(&i->sockets, it) {
struct socket *s = it->socket;
if (s->netem)
if (s->tc_qdisc)
s->mark = 1 + mark++;
}
@ -64,20 +69,24 @@ int if_start(struct interface *i, int affinity)
return 0;
/* Replace root qdisc */
tc_prio(i, TC_HDL(4000, 0), mark);
tc_prio(i, &i->tc_qdisc, TC_HANDLE(4000, 0), mark);
/* Create netem qdisks and appropriate filter per netem node */
FOREACH(&i->sockets, it) {
struct socket *s = it->socket;
if (s->netem) {
tc_mark(i, TC_HDL(4000, s->mark), s->mark);
tc_netem(i, TC_HDL(4000, s->mark), s->netem);
if (s->tc_qdisc) {
tc_mark(i, &s->tc_classifier, TC_HANDLE(4000, s->mark), s->mark);
tc_netem(i, &s->tc_qdisc, TC_HANDLE(8000, s->mark), TC_HANDLE(4000, s->mark));
char buf[256];
tc_print(buf, sizeof(buf), s->tc_qdisc);
debug(5, "Starting network emulation on interface '%s' for FW mark %u: %s",
rtnl_link_get_name(i->nl_link), s->mark, buf);
}
}
/* Set affinity for network interfaces (skip _loopback_ dev) */
if_getirqs(i);
if_setaffinity(i, affinity);
if_set_affinity(i, affinity);
}
return 0;
@ -85,65 +94,55 @@ int if_start(struct interface *i, int affinity)
int if_stop(struct interface *i)
{
info("Stopping interface '%s' (index=%u)", i->name, i->index);
info("Stopping interface '%s'", rtnl_link_get_name(i->nl_link));
{ INDENT
if_setaffinity(i, -1L);
if_set_affinity(i, -1L);
/* Only reset tc if it was initialized before */
FOREACH(&i->sockets, it) {
if (it->socket->netem) {
tc_reset(i);
break;
}
if (i->tc_qdisc)
tc_reset(i);
}
return 0;
}
int if_get_egress(struct sockaddr *sa, struct rtnl_link **link)
{
switch (sa->sa_family) {
case AF_INET:
case AF_INET6: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa;
struct nl_addr *addr = (sa->sa_family == AF_INET)
? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr))
: nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr));
struct rtnl_nexthop *nh;
if (nl_get_nexthop(addr, &nh))
return -1;
*link = nl_get_link(rtnl_route_nh_get_ifindex(nh), NULL);
rtnl_route_nh_free(nh);
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
*link = nl_get_link(sll->sll_ifindex, NULL);
}
}
return 0;
}
int if_getegress(struct sockaddr *sa)
{
switch (sa->sa_family) {
case AF_INET: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
char cmd[128];
char token[32];
snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sin->sin_addr));
debug(8, "System: %s", cmd);
FILE *p = popen(cmd, "r");
if (!p)
return -1;
while (!feof(p) && fscanf(p, "%31s", token)) {
if (!strcmp(token, "dev")) {
fscanf(p, "%31s", token);
break;
}
}
return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token);
}
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
return sll->sll_ifindex;
}
default:
return -1;
}
}
int if_getirqs(struct interface *i)
int if_get_irqs(struct interface *i)
{
char dirname[NAME_MAX];
int irq, n = 0;
snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name);
snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", rtnl_link_get_name(i->nl_link));
DIR *dir = opendir(dirname);
if (dir) {
memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX);
@ -156,11 +155,13 @@ int if_getirqs(struct interface *i)
closedir(dir);
}
debug(6, "Found %u IRQs for interface '%s'", n, rtnl_link_get_name(i->nl_link));
return 0;
}
int if_setaffinity(struct interface *i, int affinity)
int if_set_affinity(struct interface *i, int affinity)
{
char filename[NAME_MAX];
FILE *file;
@ -174,10 +175,10 @@ int if_setaffinity(struct interface *i, int affinity)
error("Failed to set affinity for IRQ %u", i->irqs[n]);
fclose(file);
debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], i->name, affinity);
debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], rtnl_link_get_name(i->nl_link), affinity);
}
else
error("Failed to set affinity for interface '%s'", i->name);
error("Failed to set affinity for interface '%s'", rtnl_link_get_name(i->nl_link));
}
return 0;
@ -186,7 +187,7 @@ int if_setaffinity(struct interface *i, int affinity)
struct interface * if_lookup_index(int index)
{
FOREACH(&interfaces, it) {
if (it->interface->index == index)
if (rtnl_link_get_ifindex(it->interface->nl_link) == index)
return it->interface;
}

94
server/src/nl.c Normal file
View file

@ -0,0 +1,94 @@
/** Netlink related functions
*
* S2SS uses libnl3 to talk to the Linux kernel to gather networking related information
*
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC
* This file is part of S2SS. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#include <stdio.h>
#include <netlink/route/route.h>
#include "utils.h"
#include "nl.h"
/** Singleton for netlink socket */
static struct nl_sock *sock = NULL;
struct nl_sock * nl_init()
{
if (!sock) {
/* Create connection to netlink */
sock = nl_socket_alloc();
if (!sock)
error("Failed to allocate memory");
nl_connect(sock, NETLINK_ROUTE);
}
return sock;
}
void nl_shutdown()
{
nl_close(sock);
nl_socket_free(sock);
sock = NULL;
}
struct rtnl_link * nl_get_link(int ifindex, const char *dev)
{
struct nl_sock *sock = nl_init();
struct rtnl_link *link;
if (rtnl_link_get_kernel(sock, ifindex, dev, &link))
return NULL;
else
return link;
}
int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop)
{
int ret;
struct rtnl_route *route;
struct rtnl_nexthop *nh;
struct nl_sock *sock = nl_init();
struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0);
struct nl_cache_ops *ops = nl_cache_ops_lookup_safe("route/route");
struct nl_cache *cache = nl_cache_alloc(ops);
struct rtmsg rmsg = {
.rtm_family = nl_addr_get_family(addr),
.rtm_dst_len = nl_addr_get_prefixlen(addr),
};
if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO)) < 0)
return ret;
if ((ret = nla_put_addr(msg, RTA_DST, addr)) < 0)
return ret;
if ((ret = nl_send_auto_complete(sock, msg)) < 0)
return ret;
if ((ret = nl_cache_pickup(sock, cache)) < 0)
return ret;
if (!(route = (struct rtnl_route *) nl_cache_get_first(cache)))
return -1;
if (!(nh = rtnl_route_nexthop_n(route, 0)))
return -1;
*nexthop = rtnl_route_nh_clone(nh);
nl_cache_put(cache);
nl_cache_ops_put(ops);
nlmsg_free(msg);
return 0;
}

View file

@ -14,14 +14,18 @@
/* Node types */
#include "file.h"
#include "socket.h"
#ifdef ENABLE_GTFPGA
#include "gtfpga.h"
#endif
#ifdef ENABLE_OPAL_ASYNC
#include "opal.h"
#endif
#ifdef ENABLE_SOCKET
#include "socket.h"
#include <netlink/route/qdisc.h>
#include <netlink/route/classifier.h>
#endif
#define VTABLE(type, name, fnc) { type, name, \
fnc ## _parse, fnc ## _print, \
@ -37,7 +41,9 @@ struct node_vtable vtables[] = {
#ifdef ENABLE_GTFPGA
VTABLE(GTFPGA, "gtfpga", gtfpga),
#endif
#ifdef ENABLE_SOCKET
VTABLE(BSD_SOCKET, "socket", socket),
#endif
VTABLE(LOG_FILE, "file", file)
};
@ -137,9 +143,12 @@ struct node * node_create()
void node_destroy(struct node *n)
{
switch (n->vt->type) {
#ifdef ENABLE_SOCKET
case BSD_SOCKET:
free(n->socket->netem);
rtnl_qdisc_put(n->socket->tc_qdisc);
rtnl_cls_put(n->socket->tc_classifier);
break;
#endif
case LOG_FILE:
free(n->file->path_in);
free(n->file->path_out);

View file

@ -13,24 +13,23 @@
#include <string.h>
#include <unistd.h>
#include <poll.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <net/if.h>
#include <net/ethernet.h>
#include <netinet/ip.h>
#include <netinet/ether.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <sys/un.h>
#include <netinet/ether.h>
#include <netinet/ip.h>
#include <linux/if_packet.h>
#include <arpa/inet.h>
#include "if.h"
#include "nl.h"
#include "tc.h"
#include "config.h"
#include "utils.h"
#include "socket.h"
#include "if.h"
/** Linked list of interfaces */
extern struct list interfaces;
@ -45,22 +44,24 @@ int socket_init(int argc, char * argv[], struct settings *set)
/* Gather list of used network interfaces */
FOREACH(&sockets, it) {
struct socket *s = it->socket;
struct rtnl_link *link;
/* Determine outgoing interface */
int index = if_getegress((struct sockaddr *) &s->remote);
if (index < 0) {
if (if_get_egress((struct sockaddr *) &s->remote, &link) || !link) {
char buf[128];
socket_print_addr(buf, sizeof(buf), (struct sockaddr *) &s->remote);
error("Failed to get interface for socket address '%s'", buf);
}
int index = rtnl_link_get_ifindex(link);
struct interface *i = if_lookup_index(index);
if (!i)
i = if_create(index);
i = if_create(link);
list_push(&i->sockets, s);
}
/** @todo Improve mapping of NIC IRQs per path */
FOREACH(&interfaces, it)
if_start(it->interface, set->affinity);
@ -83,11 +84,18 @@ int socket_print(struct node *n, char *buf, int len)
char local[INET6_ADDRSTRLEN + 16];
char remote[INET6_ADDRSTRLEN + 16];
char *layer = NULL;
switch (s->layer) {
case LAYER_UDP: layer = "udp"; break;
case LAYER_IP: layer = "ip"; break;
case LAYER_ETH: layer = "eth"; break;
}
socket_print_addr(local, sizeof(local), (struct sockaddr *) &s->local);
socket_print_addr(remote, sizeof(remote), (struct sockaddr *) &s->remote);
return snprintf(buf, len, "local=%s, remote=%s", local, remote);
return snprintf(buf, len, "layer=%s, local=%s, remote=%s", layer, local, remote);
}
int socket_open(struct node *n)
@ -116,9 +124,9 @@ int socket_open(struct node *n)
/* Set fwmark for outgoing packets */
if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark)))
serror("Failed to set fwmark for outgoing packets");
serror("Failed to set FW mark for outgoing packets");
else
debug(4, "Set fwmark for socket (sd=%u) to %u", s->sd, s->mark);
debug(4, "Set FW mark for socket (sd=%u) to %u", s->sd, s->mark);
/* Set socket priority, QoS or TOS IP options */
int prio;
@ -170,11 +178,11 @@ int socket_read(struct node *n, struct msg *pool, int poolsize, int first, int c
/* Get size of received packet in bytes */
ioctl(s->sd, FIONREAD, &bytes);
/* Check packet integrity */
/* Check if packet length is correct */
if (bytes % (cnt * 4) != 0)
error("Packet length not dividable by 4: received=%u, cnt=%u", bytes, cnt);
if (bytes / cnt > sizeof(struct msg))
error("Packet length is too large: received=%u, cnt=%u, max=%lu", bytes, cnt, sizeof(struct msg));
error("Packet length is too large: received=%u, cnt=%u, max=%zu", bytes, cnt, sizeof(struct msg));
for (int i = 0; i < cnt; i++) {
/* All messages of a packet must have equal length! */
@ -283,25 +291,28 @@ int socket_parse(config_setting_t *cfg, struct node *n)
if (!config_setting_lookup_string(cfg, "local", &local))
cerror(cfg, "Missing local address for node '%s'", n->name);
ret = socket_parse_addr(local, (struct sockaddr *) &s->local, s->layer, AI_PASSIVE);
if (ret)
if (ret) {
cerror(cfg, "Failed to resolve local address '%s' of node '%s': %s",
local, n->name, gai_strerror(ret));
}
ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, s->layer, 0);
if (ret)
if (ret) {
cerror(cfg, "Failed to resolve remote address '%s' of node '%s': %s",
remote, n->name, gai_strerror(ret));
}
/** @todo Netem settings are not usable with AF_UNIX */
config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem");
if (cfg_netem) {
int enabled = 1;
if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) {
s->netem = alloc(sizeof(struct netem));
tc_parse(cfg_netem, s->netem);
s->tc_qdisc = rtnl_qdisc_alloc();
if (!s->tc_qdisc)
error("Failed to allocated memory!");
tc_parse(cfg_netem, s->tc_qdisc);
}
}
@ -312,42 +323,60 @@ int socket_parse(config_setting_t *cfg, struct node *n)
return 0;
}
int socket_print_addr(char *buf, int len, struct sockaddr *sa)
int socket_print_addr(char *buf, int len, struct sockaddr *saddr)
{
switch (sa->sa_family) {
case AF_INET: {
struct sockaddr_in *sin = (struct sockaddr_in *) sa;
inet_ntop(sin->sin_family, &sin->sin_addr, buf, len);
return snprintf(buf+strlen(buf), len-strlen(buf), ":%hu", ntohs(sin->sin_port));
}
union sockaddr_union *sa = (union sockaddr_union *) saddr;
/* Address */
switch (sa->sa.sa_family) {
case AF_INET6:
inet_ntop(AF_INET6, &sa->sin6.sin6_addr, buf, len);
break;
case AF_PACKET: {
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
char ifname[IF_NAMESIZE];
return snprintf(buf, len, "%s%%%s:%#hx",
ether_ntoa((struct ether_addr *) &sll->sll_addr),
if_indextoname(sll->sll_ifindex, ifname),
ntohs(sll->sll_protocol));
}
case AF_INET:
inet_ntop(AF_INET, &sa->sin.sin_addr, buf, len);
break;
case AF_PACKET:
snprintf(buf, len, "%02x", sa->sll.sll_addr[0]);
for (int i = 1; i < sa->sll.sll_halen; i++)
strap(buf, len, ":%02x", sa->sll.sll_addr[i]);
break;
default:
return snprintf(buf, len, "address family: %u", sa->sa_family);
error("Unknown address family: '%u'", sa->sa.sa_family);
}
/* Port / Interface */
switch (sa->sa.sa_family) {
case AF_INET6:
case AF_INET:
strap(buf, len, ":%hu", ntohs(sa->sin.sin_port));
break;
case AF_PACKET: {
struct rtnl_link *link = nl_get_link(sa->sll.sll_ifindex, NULL);
if (!link)
error("Failed to get interface for index: %u", sa->sll.sll_ifindex);
strap(buf, len, "%%%s", rtnl_link_get_name(link));
strap(buf, len, ":%hu", ntohs(sa->sll.sll_protocol));
break;
}
}
return 0;
}
int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer layer, int flags)
int socket_parse_addr(const char *addr, struct sockaddr *saddr, enum socket_layer layer, int flags)
{
/** @todo: Add support for IPv6 */
union sockaddr_union *sa = (union sockaddr_union *) saddr;
char *copy = strdup(addr);
int ret;
if (layer == LAYER_ETH) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */
struct sockaddr_ll *sll = (struct sockaddr_ll *) sa;
/* Split string */
char *node = strtok(copy, "%");
char *ifname = strtok(NULL, ":");
@ -356,21 +385,26 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer l
/* Parse link layer (MAC) address */
struct ether_addr *mac = ether_aton(node);
if (!mac)
error("Failed to parse mac address: %s", node);
error("Failed to parse MAC address: %s", node);
memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6);
memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, 6);
/* Get interface index from name */
struct rtnl_link *link = nl_get_link(0, ifname);
if (!link)
error("Failed to get network interface: '%s'", ifname);
sll->sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS);
sll->sll_halen = 6;
sll->sll_family = AF_PACKET;
sll->sll_ifindex = if_nametoindex(ifname);
sa->sll.sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS);
sa->sll.sll_halen = 6;
sa->sll.sll_family = AF_PACKET;
sa->sll.sll_ifindex = rtnl_link_get_ifindex(link);
ret = 0;
}
else { /* Format: "192.168.0.10:12001" */
struct addrinfo hint = {
.ai_flags = flags,
.ai_family = AF_INET
.ai_family = AF_UNSPEC
};
/* Split string */

View file

@ -8,110 +8,190 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#include <netlink/route/cls/fw.h>
#include <netlink/route/qdisc/netem.h>
#include <netlink/route/qdisc/prio.h>
#include <linux/if_ether.h>
#include "utils.h"
#include "if.h"
#include "tc.h"
#include "nl.h"
int tc_parse(config_setting_t *cfg, struct netem *em)
int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne)
{
em->valid = 0;
const char *str;
int val;
rtnl_tc_set_kind(TC_CAST(ne), "netem");
if (config_setting_lookup_string(cfg, "distribution", &em->distribution))
em->valid |= TC_NETEM_DISTR;
if (config_setting_lookup_int(cfg, "delay", &em->delay))
em->valid |= TC_NETEM_DELAY;
if (config_setting_lookup_int(cfg, "jitter", &em->jitter))
em->valid |= TC_NETEM_JITTER;
if (config_setting_lookup_int(cfg, "loss", &em->loss))
em->valid |= TC_NETEM_LOSS;
if (config_setting_lookup_int(cfg, "duplicate", &em->duplicate))
em->valid |= TC_NETEM_DUPL;
if (config_setting_lookup_int(cfg, "corrupt", &em->corrupt))
em->valid |= TC_NETEM_CORRUPT;
if (config_setting_lookup_string(cfg, "distribution", &str)) {
if (rtnl_netem_set_delay_distribution(ne, str))
cerror(cfg, "Invalid delay distribution '%s' in netem config", str);
}
if (config_setting_lookup_int(cfg, "limit", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for limit setting", val);
/** @todo Validate netem config values */
rtnl_netem_set_limit(ne, val);
}
else
rtnl_netem_set_limit(ne, 0);
if (config_setting_lookup_int(cfg, "delay", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for delay setting", val);
rtnl_netem_set_delay(ne, val);
}
if (config_setting_lookup_int(cfg, "jitter", &val)) {
if (val <= 0)
cerror(cfg, "Invalid value '%d' for jitter setting", val);
rtnl_netem_set_jitter(ne, val);
}
if (config_setting_lookup_int(cfg, "loss", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for loss setting", val);
rtnl_netem_set_loss(ne, val);
}
if (config_setting_lookup_int(cfg, "duplicate", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for duplicate setting", val);
rtnl_netem_set_duplicate(ne, val);
}
if (config_setting_lookup_int(cfg, "corruption", &val)) {
if (val < 0 || val > 100)
cerror(cfg, "Invalid percentage value '%d' for corruption setting", val);
rtnl_netem_set_corruption_probability(ne, val);
}
return 0;
}
int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne)
{
if (rtnl_netem_get_limit(ne) > 0)
strap(buf, len, "limit %upkts", rtnl_netem_get_limit(ne));
if (rtnl_netem_get_delay(ne) > 0) {
strap(buf, len, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0);
if (rtnl_netem_get_jitter(ne) > 0) {
strap(buf, len, "jitter %f.2ms ", rtnl_netem_get_jitter(ne) / 1000.0);
if (rtnl_netem_get_delay_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_delay_correlation(ne));
}
}
if (rtnl_netem_get_loss(ne) > 0) {
strap(buf, len, "loss %u%% ", rtnl_netem_get_loss(ne));
if (rtnl_netem_get_loss_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_loss_correlation(ne));
}
if (rtnl_netem_get_reorder_probability(ne) > 0) {
strap(buf, len, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne));
if (rtnl_netem_get_reorder_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_reorder_correlation(ne));
}
if (rtnl_netem_get_corruption_probability(ne) > 0) {
strap(buf, len, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne));
if (rtnl_netem_get_corruption_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_corruption_correlation(ne));
}
if (rtnl_netem_get_duplicate(ne) > 0) {
strap(buf, len, "duplication %u%% ", rtnl_netem_get_duplicate(ne));
if (rtnl_netem_get_duplicate_correlation(ne) > 0)
strap(buf, len, "%u%% ", rtnl_netem_get_duplicate_correlation(ne));
}
return 0;
}
int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands)
{
struct nl_sock *sock = nl_init();
struct rtnl_qdisc *q = rtnl_qdisc_alloc();
/* This is the default priomap used by the tc-prio qdisc
* We will use the first 'bands' bands internally */
uint8_t map[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
for (int i = 0; i < ARRAY_LEN(map); i++)
map[i] += bands;
rtnl_tc_set_link(TC_CAST(q), i->nl_link);
rtnl_tc_set_parent(TC_CAST(q), TC_H_ROOT);
rtnl_tc_set_handle(TC_CAST(q), handle);
rtnl_tc_set_kind(TC_CAST(q), "prio");
rtnl_qdisc_prio_set_bands(q, bands + 3);
rtnl_qdisc_prio_set_priomap(q, map, ARRAY_LEN(map));
int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE);
*qd = q;
return ret;
}
int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent)
{
struct nl_sock *sock = nl_init();
struct rtnl_qdisc *q = *qd;
rtnl_tc_set_link(TC_CAST(q), i->nl_link);
rtnl_tc_set_parent(TC_CAST(q), parent);
rtnl_tc_set_handle(TC_CAST(q), handle);
int ret = rtnl_qdisc_add(sock, q, NLM_F_REPLACE);
*qd = q;
return ret;
}
int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark)
{
struct nl_sock *sock = nl_init();
struct rtnl_cls *c = rtnl_cls_alloc();
rtnl_tc_set_link(TC_CAST(c), i->nl_link);
rtnl_tc_set_handle(TC_CAST(c), mark);
rtnl_tc_set_kind(TC_CAST(c), "fw");
rtnl_cls_set_protocol(c, ETH_P_ALL);
rtnl_fw_set_classid(c, flowid);
rtnl_fw_set_mask(c, 0xFFFFFFFF);
int ret = rtnl_cls_add(sock, c, NLM_F_CREATE);
*cls = c;
return ret;
}
int tc_reset(struct interface *i)
{
char cmd[128];
snprintf(cmd, sizeof(cmd), "tc qdisc replace dev %s root pfifo_fast", i->name);
struct nl_sock *sock = nl_init();
debug(6, "Reset traffic control for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add reset traffic control for interface '%s'", i->name);
return 0;
}
int tc_prio(struct interface *i, tc_hdl_t handle, int bands)
{
char cmd[128];
int len = 0;
int priomap[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 };
len += snprintf(cmd+len, sizeof(cmd)-len,
"tc qdisc replace dev %s root handle %u prio bands %u priomap",
i->name, TC_HDL_MAJ(handle), bands + 3);
for (int i = 0; i < 16; i++)
len += snprintf(cmd+len, sizeof(cmd)-len, " %u", priomap[i] + bands);
debug(6, "Replace master qdisc for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add prio qdisc for interface '%s'", i->name);
return 0;
}
int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em)
{
int len = 0;
char cmd[256];
len += snprintf(cmd+len, sizeof(cmd)-len,
"tc qdisc replace dev %s parent %u:%u netem",
i->name, TC_HDL_MAJ(parent), TC_HDL_MIN(parent));
if (em->valid & TC_NETEM_DELAY) {
len += snprintf(cmd+len, sizeof(cmd)-len, " delay %u", em->delay);
if (em->valid & TC_NETEM_JITTER)
len += snprintf(cmd+len, sizeof(cmd)-len, " %u", em->jitter);
if (em->valid & TC_NETEM_DISTR)
len += snprintf(cmd+len, sizeof(cmd)-len, " distribution %s", em->distribution);
}
if (em->valid & TC_NETEM_LOSS)
len += snprintf(cmd+len, sizeof(cmd)-len, " loss random %u", em->loss);
if (em->valid & TC_NETEM_DUPL)
len += snprintf(cmd+len, sizeof(cmd)-len, " duplicate %u", em->duplicate);
if (em->valid & TC_NETEM_CORRUPT)
len += snprintf(cmd+len, sizeof(cmd)-len, " corrupt %u", em->corrupt);
debug(6, "Setup netem qdisc for interface '%s'", i->name);
if (system2(cmd))
error("Failed to add netem qdisc for interface '%s'", i->name);
return 0;
}
int tc_mark(struct interface *i, tc_hdl_t flowid, int mark)
{
char cmd[128];
snprintf(cmd, sizeof(cmd),
"tc filter replace dev %s protocol ip handle %u fw flowid %u:%u",
i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid));
debug(7, "Add traffic filter to interface '%s': fwmark %u => flowid %u:%u",
i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid));
if (system2(cmd))
error("Failed to add fw_mark classifier for interface '%s'", i->name);
return 0;
/* We restore the default pfifo_fast qdisc, by deleting ours */
return rtnl_qdisc_delete(sock, i->tc_qdisc);
}