diff --git a/server/Makefile b/server/Makefile index bc71ff6df..e1e70a444 100644 --- a/server/Makefile +++ b/server/Makefile @@ -5,7 +5,7 @@ OBJS = path.o node.o hooks.o msg.o cfg.o stats.o # Helper libs OBJS += utils.o list.o hist.o log.o timing.o # Node types -OBJS += file.o socket.o if.o tc.o +OBJS += file.o VPATH = src @@ -13,33 +13,45 @@ VPATH = src V ?= 2 # Compiler and linker flags +CC = gcc LDLIBS = -pthread -lrt -lm -lconfig -CFLAGS = -std=gnu99 -Iinclude/ -MMD -Wall -O3 -CFLAGS += -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V) +override CFLAGS += -std=gnu99 -Iinclude/ -MMD -Wall -O3 +override CFLAGS += -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE=1 -DV=$(V) + +# Default include paths for third-party libs +PCIDIR ?= /usr/include +NLDIR ?= /usr/include/libnl3 +OPALDIR ?= /usr/opalrt/common +#OPALDIR ?= ../contrib/opal # Add more compiler flags ifdef DEBUG - CFLAGS += -O0 -g + override CFLAGS += -O0 -g endif ifneq (,$(shell which git)) - CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"' + override CFLAGS += -D_GIT_REV='"$(shell git rev-parse --short HEAD)"' endif -# Enabled GTFPGA support when libpci is available -ifneq (,$(wildcard /usr/include/pci/pci.h)) - CFLAGS += -DENABLE_GTFPGA +# Enable Socket node type when libnl3 is available +ifneq (,$(wildcard $(NLDIR)/netlink/netlink.h)) + override CFLAGS += -DENABLE_SOCKET -I$(NLDIR) + LDLIBS += -lnl-3 -lnl-route-3 + OBJS += nl.o tc.o if.o socket.o +endif + +# Enable GTFPGA support when libpci is available +ifneq (,$(wildcard $(PCIDIR)/pci/pci.h)) + override CFLAGS += -DENABLE_GTFPGA -I$(PCIDIR) LDLIBS += -lpci OBJS += gtfpga.o endif -# Enable OPAL-RT Asynchronous Process support -OPALDIR = /usr/opalrt/common -#OPALDIR = ../contrib/opal +# Enable OPAL-RT Asynchronous Process support (will result in 32bit binary!!!) ifneq (,$(wildcard $(OPALDIR)/include_target/AsyncApi.h)) - override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target - override LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/ - override LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) - override OBJS += opal.o + override CFLAGS += -m32 -DENABLE_OPAL_ASYNC -I$(OPALDIR)/include_target + LDFLAGS += -m32 -Wl,-L/lib/i386-linux-gnu/,-L/usr/lib/i386-linux-gnu/ + LDLIBS += $(addprefix $(OPALDIR)/lib/redhawk/, libOpalAsyncApiCore.a libOpalCore.a libOpalUtils.a libirc.a) + OBJS += opal.o endif .PHONY: all clean strip protected diff --git a/server/include/if.h b/server/include/if.h index b46f15b03..a01ecafa7 100644 --- a/server/include/if.h +++ b/server/include/if.h @@ -14,11 +14,10 @@ #define _IF_H_ #include -#include +#include #include "list.h" -#define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */ #define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */ #ifndef SO_MARK @@ -26,13 +25,16 @@ #endif struct socket; +struct nl_addr; +struct rtnl_link; /** Interface data structure */ struct interface { - /** The index used by the kernel to reference this interface */ - int index; - /** Human readable name of this interface */ - char name[IF_NAME_MAX]; + /** libnl3: Handle of interface */ + struct rtnl_link *nl_link; + /** libnl3: Root prio qdisc */ + struct rtnl_qdisc *tc_qdisc; + /** List of IRQs of the NIC */ char irqs[IF_IRQ_MAX]; @@ -42,11 +44,11 @@ struct interface { /** Add a new interface to the global list and lookup name, irqs... * - * @param index The interface index of the OS + * @param link The libnl3 link handle * @retval >0 Success. A pointer to the new interface. * @retval 0 Error. The creation failed. */ -struct interface * if_create(int index); +struct interface * if_create(struct rtnl_link *link); /** Destroy interface by freeing dynamically allocated memory. @@ -78,20 +80,15 @@ int if_start(struct interface *i, int affinity); */ int if_stop(struct interface *i); -/** Get outgoing interface. +/** Lookup routing tables to get the interface on which packets for a certain destination + * will leave the system. * - * Depending on the address family of the socker address, - * this function tries to determine outgoing interface - * which is used to send packages to a remote host with the specified - * socket address. - * - * For AF_INET the fnuction performs a lookup in the kernel routing table. - * For AF_PACKET the function uses the existing sll_ifindex field of the socket address. - * - * @param sa A destination address for outgoing packets. - * @return The interface index. + * @param[in] sa The destination address for outgoing packets. + * @param[out] link The egress interface. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ -int if_getegress(struct sockaddr *sa); +int if_get_egress(struct sockaddr *sa, struct rtnl_link **link); /** Get all IRQs for this interface. * @@ -102,7 +99,7 @@ int if_getegress(struct sockaddr *sa); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int if_getirqs(struct interface *i); +int if_get_irqs(struct interface *i); /** Change the SMP affinity of NIC interrupts. * @@ -111,7 +108,7 @@ int if_getirqs(struct interface *i); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int if_setaffinity(struct interface *i, int affinity); +int if_set_affinity(struct interface *i, int affinity); /** Search the global list of interfaces for a given index. * diff --git a/server/include/nl.h b/server/include/nl.h new file mode 100644 index 000000000..3132a6276 --- /dev/null +++ b/server/include/nl.h @@ -0,0 +1,40 @@ +/** + * + * @file + * @author Steffen Vogel + * @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC + * This file is part of S2SS. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#ifndef _NL_H_ +#define _NL_H_ + +#include +#include +#include + +/** Get libnl3 link object from interface name + * + * @param dev The name of the interface + * @retval 0 Error. Something went wrong. + * @retval >0 A pointer to the libnl3 link object. + */ +struct rtnl_link * nl_get_link(int ifindex, const char *dev); + +/** + * + */ +int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop); + +/** + * + */ +struct nl_sock * nl_init(); + +/** + * + */ +void nl_shutdown(); + +#endif \ No newline at end of file diff --git a/server/include/socket.h b/server/include/socket.h index 718b9ba69..28946efcf 100644 --- a/server/include/socket.h +++ b/server/include/socket.h @@ -16,6 +16,7 @@ #define _SOCKET_H_ #include +#include #include "node.h" @@ -25,6 +26,13 @@ enum socket_layer { LAYER_UDP }; +union sockaddr_union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + struct sockaddr_ll sll; +}; + struct socket { /** The socket descriptor */ int sd; @@ -35,12 +43,14 @@ struct socket { enum socket_layer layer; /** Local address of the socket */ - struct sockaddr_storage local; + union sockaddr_union local; /** Remote address of the socket */ - struct sockaddr_storage remote; + union sockaddr_union remote; - /** Network emulator settings */ - struct netem *netem; + /** libnl3: Network emulator queuing discipline */ + struct rtnl_qdisc *tc_qdisc; + /** libnl3: Firewall mark classifier */ + struct rtnl_cls *tc_classifier; /* Linked list _per_interface_ */ struct socket *next; diff --git a/server/include/tc.h b/server/include/tc.h index 7d6f2ddf8..397cd00a9 100644 --- a/server/include/tc.h +++ b/server/include/tc.h @@ -16,65 +16,34 @@ #define _TC_H_ #include + +#include +#include + #include -/** A type alias for TC handles. - * - * TC handles are used to construct a tree - * of classes, qdiscs and filters. - */ typedef uint32_t tc_hdl_t; -/** Concatenate 16 bit minor and majar parts to a 32 bit tc handle */ -#define TC_HDL(maj, min) ((maj & 0xFFFF) << 16 | (min & 0xFFFF)) -/** Get the major part of a tc handle */ -#define TC_HDL_MAJ(h) ((h >> 16) & 0xFFFF) -/** Get the minor part of a tc handle */ -#define TC_HDL_MIN(h) ((h >> 0) & 0xFFFF) -/** The root handle */ -#define TC_HDL_ROOT (0xFFFFFFFFU) - -/* Bitfield for valid fields in struct netem */ -#define TC_NETEM_DELAY (1 << 0) /**< netem::delay is valid @see netem::valid */ -#define TC_NETEM_JITTER (1 << 1) /**< netem::jitter is valid @see netem::valid */ -#define TC_NETEM_DISTR (1 << 2) /**< netem::distribution is valid @see netem::valid */ -#define TC_NETEM_LOSS (1 << 3) /**< netem::loss is valid @see netem::valid */ -#define TC_NETEM_CORRUPT (1 << 4) /**< netem::corrupt is valid @see netem::valid */ -#define TC_NETEM_DUPL (1 << 5) /**< netem::duplicate is valid @see netem::valid */ - struct interface; -/** Netem configuration settings. - * - * This struct is used to pass the netem configuration - * from config_parse_netem() to tc_netem() - */ -struct netem { - /** Which fields of this struct contain valid data (TC_NETEM_*). */ - char valid; - - /** Delay distribution: uniform, normal, pareto, paretonormal */ - const char *distribution; - /** Added delay (uS) */ - int delay; - /** Delay jitter (uS) */ - int jitter; - /** Random loss probability (%) */ - int loss; - /** Packet corruption probability (%) */ - int corrupt; - /** Packet duplication probability (%) */ - int duplicate; -}; - /** Parse network emulator (netem) settings. * * @param cfg A libconfig object containing the settings. - * @param em A pointer to the netem settings structure (part of the path structure). + * @param ne A pointer to a libnl3 qdisc object where setting will be written to. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_parse(config_setting_t *cfg, struct netem *em); +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne); + +/** Print network emulator (netem) setting into buffer. + * + * @param buf A character buffer to write to. + * @param len The length of the supplied buffer. + * @param tc A pointer to the libnl3 qdisc object where settings will be read from. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne); /** Remove all queuing disciplines and filters. * @@ -86,32 +55,35 @@ int tc_reset(struct interface *i); /** Create a priority (prio) queueing discipline. * - * @param i The interface - * @param handle The handle for the new qdisc - * @param bands The number of classes for this new qdisc + * @param i[in] The interface + * @param qd[in,out] The libnl3 object of the new prio qdisc. + * @param handle[in] The handle for the new qdisc + * @param bands[in] The number of classes for this new qdisc * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_prio(struct interface *i, tc_hdl_t handle, int bands); +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands); /** Add a new network emulator (netem) discipline. * - * @param i The interface - * @param parent Make this qdisc a child of - * @param em The netem settings + * @param i[in] The interface to which this qdisc will be added. + * @param qd[in,out] The libnl3 object of the new prio qdisc. + * @param handle[in] The handle of the new qdisc. + * @param parent[in] Make this qdisc a child of this class * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em); +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent); /** Add a new filter based on the netfilter mark. * - * @param i The interface + * @param i The interface to which this classifier is applied to. + * @param cls[in,out] The libnl3 object of the new prio qdisc. * @param flowid The destination class for matched traffic * @param mark The netfilter firewall mark (sometime called 'fwmark') * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_mark(struct interface *i, tc_hdl_t flowid, int mark); +int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark); #endif /* _TC_H_ */ diff --git a/server/src/if.c b/server/src/if.c index 0170d283b..90d324238 100644 --- a/server/src/if.c +++ b/server/src/if.c @@ -10,27 +10,30 @@ #include #include #include - -#include #include -#include #include +#include +#include + #include "if.h" #include "tc.h" +#include "nl.h" #include "socket.h" #include "utils.h" /** Linked list of interfaces. */ struct list interfaces; -struct interface * if_create(int index) { +struct interface * if_create(struct rtnl_link *link) +{ struct interface *i = alloc(sizeof(struct interface)); + + i->nl_link = link; - i->index = index; - if_indextoname(index, i->name); + debug(3, "Created interface '%s'", rtnl_link_get_name(i->nl_link)); - debug(3, "Created interface '%s' (index=%u)", i->name, i->index); + if_get_irqs(i); list_init(&i->sockets, NULL); list_push(&interfaces, i); @@ -40,22 +43,24 @@ struct interface * if_create(int index) { void if_destroy(struct interface *i) { - /* List members are freed by their belonging nodes. */ + /* List members are freed by the nodes they belong to. */ list_destroy(&i->sockets); + + rtnl_qdisc_put(i->tc_qdisc); free(i); } int if_start(struct interface *i, int affinity) { - info("Starting interface '%s' (index=%u)", i->name, i->index); + info("Starting interface '%s' which is used by %u sockets", rtnl_link_get_name(i->nl_link), list_length(&i->sockets)); { INDENT /* Assign fwmark's to socket nodes which have netem options */ int mark = 0; FOREACH(&i->sockets, it) { struct socket *s = it->socket; - if (s->netem) + if (s->tc_qdisc) s->mark = 1 + mark++; } @@ -64,20 +69,24 @@ int if_start(struct interface *i, int affinity) return 0; /* Replace root qdisc */ - tc_prio(i, TC_HDL(4000, 0), mark); + tc_prio(i, &i->tc_qdisc, TC_HANDLE(4000, 0), mark); /* Create netem qdisks and appropriate filter per netem node */ FOREACH(&i->sockets, it) { struct socket *s = it->socket; - if (s->netem) { - tc_mark(i, TC_HDL(4000, s->mark), s->mark); - tc_netem(i, TC_HDL(4000, s->mark), s->netem); + if (s->tc_qdisc) { + tc_mark(i, &s->tc_classifier, TC_HANDLE(4000, s->mark), s->mark); + tc_netem(i, &s->tc_qdisc, TC_HANDLE(8000, s->mark), TC_HANDLE(4000, s->mark)); + + char buf[256]; + tc_print(buf, sizeof(buf), s->tc_qdisc); + debug(5, "Starting network emulation on interface '%s' for FW mark %u: %s", + rtnl_link_get_name(i->nl_link), s->mark, buf); } } /* Set affinity for network interfaces (skip _loopback_ dev) */ - if_getirqs(i); - if_setaffinity(i, affinity); + if_set_affinity(i, affinity); } return 0; @@ -85,65 +94,55 @@ int if_start(struct interface *i, int affinity) int if_stop(struct interface *i) { - info("Stopping interface '%s' (index=%u)", i->name, i->index); + info("Stopping interface '%s'", rtnl_link_get_name(i->nl_link)); { INDENT - if_setaffinity(i, -1L); + if_set_affinity(i, -1L); - /* Only reset tc if it was initialized before */ - FOREACH(&i->sockets, it) { - if (it->socket->netem) { - tc_reset(i); - break; - } + if (i->tc_qdisc) + tc_reset(i); + } + + return 0; +} + +int if_get_egress(struct sockaddr *sa, struct rtnl_link **link) +{ + switch (sa->sa_family) { + case AF_INET: + case AF_INET6: { + struct sockaddr_in *sin = (struct sockaddr_in *) sa; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) sa; + + struct nl_addr *addr = (sa->sa_family == AF_INET) + ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr)) + : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr)); + + struct rtnl_nexthop *nh; + if (nl_get_nexthop(addr, &nh)) + return -1; + + *link = nl_get_link(rtnl_route_nh_get_ifindex(nh), NULL); + + rtnl_route_nh_free(nh); + } + + case AF_PACKET: { + struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; + + *link = nl_get_link(sll->sll_ifindex, NULL); } } return 0; } -int if_getegress(struct sockaddr *sa) -{ - switch (sa->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *) sa; - char cmd[128]; - char token[32]; - - snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sin->sin_addr)); - - debug(8, "System: %s", cmd); - - FILE *p = popen(cmd, "r"); - if (!p) - return -1; - - while (!feof(p) && fscanf(p, "%31s", token)) { - if (!strcmp(token, "dev")) { - fscanf(p, "%31s", token); - break; - } - } - - return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token); - } - - case AF_PACKET: { - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - return sll->sll_ifindex; - } - - default: - return -1; - } -} - -int if_getirqs(struct interface *i) +int if_get_irqs(struct interface *i) { char dirname[NAME_MAX]; int irq, n = 0; - snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name); + snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", rtnl_link_get_name(i->nl_link)); DIR *dir = opendir(dirname); if (dir) { memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX); @@ -156,11 +155,13 @@ int if_getirqs(struct interface *i) closedir(dir); } + + debug(6, "Found %u IRQs for interface '%s'", n, rtnl_link_get_name(i->nl_link)); return 0; } -int if_setaffinity(struct interface *i, int affinity) +int if_set_affinity(struct interface *i, int affinity) { char filename[NAME_MAX]; FILE *file; @@ -174,10 +175,10 @@ int if_setaffinity(struct interface *i, int affinity) error("Failed to set affinity for IRQ %u", i->irqs[n]); fclose(file); - debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], i->name, affinity); + debug(5, "Set affinity of IRQ %u for interface '%s' to %#x", i->irqs[n], rtnl_link_get_name(i->nl_link), affinity); } else - error("Failed to set affinity for interface '%s'", i->name); + error("Failed to set affinity for interface '%s'", rtnl_link_get_name(i->nl_link)); } return 0; @@ -186,7 +187,7 @@ int if_setaffinity(struct interface *i, int affinity) struct interface * if_lookup_index(int index) { FOREACH(&interfaces, it) { - if (it->interface->index == index) + if (rtnl_link_get_ifindex(it->interface->nl_link) == index) return it->interface; } diff --git a/server/src/nl.c b/server/src/nl.c new file mode 100644 index 000000000..e634b5cfa --- /dev/null +++ b/server/src/nl.c @@ -0,0 +1,94 @@ +/** Netlink related functions + * + * S2SS uses libnl3 to talk to the Linux kernel to gather networking related information + * + * @author Steffen Vogel + * @copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC + * This file is part of S2SS. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#include + +#include + +#include "utils.h" +#include "nl.h" + +/** Singleton for netlink socket */ +static struct nl_sock *sock = NULL; + +struct nl_sock * nl_init() +{ + if (!sock) { + /* Create connection to netlink */ + sock = nl_socket_alloc(); + if (!sock) + error("Failed to allocate memory"); + + nl_connect(sock, NETLINK_ROUTE); + } + + return sock; +} + +void nl_shutdown() +{ + nl_close(sock); + nl_socket_free(sock); + + sock = NULL; +} + +struct rtnl_link * nl_get_link(int ifindex, const char *dev) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_link *link; + + if (rtnl_link_get_kernel(sock, ifindex, dev, &link)) + return NULL; + else + return link; +} + +int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop) +{ + int ret; + + struct rtnl_route *route; + struct rtnl_nexthop *nh; + + struct nl_sock *sock = nl_init(); + struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0); + + struct nl_cache_ops *ops = nl_cache_ops_lookup_safe("route/route"); + struct nl_cache *cache = nl_cache_alloc(ops); + + struct rtmsg rmsg = { + .rtm_family = nl_addr_get_family(addr), + .rtm_dst_len = nl_addr_get_prefixlen(addr), + }; + + if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO)) < 0) + return ret; + if ((ret = nla_put_addr(msg, RTA_DST, addr)) < 0) + return ret; + + if ((ret = nl_send_auto_complete(sock, msg)) < 0) + return ret; + if ((ret = nl_cache_pickup(sock, cache)) < 0) + return ret; + + if (!(route = (struct rtnl_route *) nl_cache_get_first(cache))) + return -1; + if (!(nh = rtnl_route_nexthop_n(route, 0))) + return -1; + + *nexthop = rtnl_route_nh_clone(nh); + + nl_cache_put(cache); + nl_cache_ops_put(ops); + nlmsg_free(msg); + + return 0; +} diff --git a/server/src/node.c b/server/src/node.c index 491fb444b..7a2435c6d 100644 --- a/server/src/node.c +++ b/server/src/node.c @@ -14,14 +14,18 @@ /* Node types */ #include "file.h" -#include "socket.h" - #ifdef ENABLE_GTFPGA #include "gtfpga.h" #endif #ifdef ENABLE_OPAL_ASYNC #include "opal.h" #endif +#ifdef ENABLE_SOCKET + #include "socket.h" + + #include + #include +#endif #define VTABLE(type, name, fnc) { type, name, \ fnc ## _parse, fnc ## _print, \ @@ -37,7 +41,9 @@ struct node_vtable vtables[] = { #ifdef ENABLE_GTFPGA VTABLE(GTFPGA, "gtfpga", gtfpga), #endif +#ifdef ENABLE_SOCKET VTABLE(BSD_SOCKET, "socket", socket), +#endif VTABLE(LOG_FILE, "file", file) }; @@ -137,9 +143,12 @@ struct node * node_create() void node_destroy(struct node *n) { switch (n->vt->type) { +#ifdef ENABLE_SOCKET case BSD_SOCKET: - free(n->socket->netem); + rtnl_qdisc_put(n->socket->tc_qdisc); + rtnl_cls_put(n->socket->tc_classifier); break; +#endif case LOG_FILE: free(n->file->path_in); free(n->file->path_out); diff --git a/server/src/socket.c b/server/src/socket.c index f948620e8..fd2835091 100644 --- a/server/src/socket.c +++ b/server/src/socket.c @@ -13,24 +13,23 @@ #include #include #include -#include -#include - -#include -#include -#include - -#include -#include #include #include #include +#include +#include +#include +#include +#include + +#include "if.h" +#include "nl.h" +#include "tc.h" #include "config.h" #include "utils.h" #include "socket.h" -#include "if.h" /** Linked list of interfaces */ extern struct list interfaces; @@ -45,22 +44,24 @@ int socket_init(int argc, char * argv[], struct settings *set) /* Gather list of used network interfaces */ FOREACH(&sockets, it) { struct socket *s = it->socket; + struct rtnl_link *link; /* Determine outgoing interface */ - int index = if_getegress((struct sockaddr *) &s->remote); - if (index < 0) { + if (if_get_egress((struct sockaddr *) &s->remote, &link) || !link) { char buf[128]; socket_print_addr(buf, sizeof(buf), (struct sockaddr *) &s->remote); error("Failed to get interface for socket address '%s'", buf); } + int index = rtnl_link_get_ifindex(link); struct interface *i = if_lookup_index(index); if (!i) - i = if_create(index); + i = if_create(link); list_push(&i->sockets, s); } + /** @todo Improve mapping of NIC IRQs per path */ FOREACH(&interfaces, it) if_start(it->interface, set->affinity); @@ -83,11 +84,18 @@ int socket_print(struct node *n, char *buf, int len) char local[INET6_ADDRSTRLEN + 16]; char remote[INET6_ADDRSTRLEN + 16]; + char *layer = NULL; + + switch (s->layer) { + case LAYER_UDP: layer = "udp"; break; + case LAYER_IP: layer = "ip"; break; + case LAYER_ETH: layer = "eth"; break; + } socket_print_addr(local, sizeof(local), (struct sockaddr *) &s->local); socket_print_addr(remote, sizeof(remote), (struct sockaddr *) &s->remote); - return snprintf(buf, len, "local=%s, remote=%s", local, remote); + return snprintf(buf, len, "layer=%s, local=%s, remote=%s", layer, local, remote); } int socket_open(struct node *n) @@ -116,9 +124,9 @@ int socket_open(struct node *n) /* Set fwmark for outgoing packets */ if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark))) - serror("Failed to set fwmark for outgoing packets"); + serror("Failed to set FW mark for outgoing packets"); else - debug(4, "Set fwmark for socket (sd=%u) to %u", s->sd, s->mark); + debug(4, "Set FW mark for socket (sd=%u) to %u", s->sd, s->mark); /* Set socket priority, QoS or TOS IP options */ int prio; @@ -170,11 +178,11 @@ int socket_read(struct node *n, struct msg *pool, int poolsize, int first, int c /* Get size of received packet in bytes */ ioctl(s->sd, FIONREAD, &bytes); - /* Check packet integrity */ + /* Check if packet length is correct */ if (bytes % (cnt * 4) != 0) error("Packet length not dividable by 4: received=%u, cnt=%u", bytes, cnt); if (bytes / cnt > sizeof(struct msg)) - error("Packet length is too large: received=%u, cnt=%u, max=%lu", bytes, cnt, sizeof(struct msg)); + error("Packet length is too large: received=%u, cnt=%u, max=%zu", bytes, cnt, sizeof(struct msg)); for (int i = 0; i < cnt; i++) { /* All messages of a packet must have equal length! */ @@ -283,25 +291,28 @@ int socket_parse(config_setting_t *cfg, struct node *n) if (!config_setting_lookup_string(cfg, "local", &local)) cerror(cfg, "Missing local address for node '%s'", n->name); - + ret = socket_parse_addr(local, (struct sockaddr *) &s->local, s->layer, AI_PASSIVE); - if (ret) + if (ret) { cerror(cfg, "Failed to resolve local address '%s' of node '%s': %s", local, n->name, gai_strerror(ret)); + } ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, s->layer, 0); - if (ret) + if (ret) { cerror(cfg, "Failed to resolve remote address '%s' of node '%s': %s", remote, n->name, gai_strerror(ret)); + } - /** @todo Netem settings are not usable with AF_UNIX */ config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem"); if (cfg_netem) { int enabled = 1; - if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) { - s->netem = alloc(sizeof(struct netem)); - tc_parse(cfg_netem, s->netem); + s->tc_qdisc = rtnl_qdisc_alloc(); + if (!s->tc_qdisc) + error("Failed to allocated memory!"); + + tc_parse(cfg_netem, s->tc_qdisc); } } @@ -312,42 +323,60 @@ int socket_parse(config_setting_t *cfg, struct node *n) return 0; } -int socket_print_addr(char *buf, int len, struct sockaddr *sa) +int socket_print_addr(char *buf, int len, struct sockaddr *saddr) { - switch (sa->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *) sa; - inet_ntop(sin->sin_family, &sin->sin_addr, buf, len); - return snprintf(buf+strlen(buf), len-strlen(buf), ":%hu", ntohs(sin->sin_port)); - } + union sockaddr_union *sa = (union sockaddr_union *) saddr; + + /* Address */ + switch (sa->sa.sa_family) { + case AF_INET6: + inet_ntop(AF_INET6, &sa->sin6.sin6_addr, buf, len); + break; - case AF_PACKET: { - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - char ifname[IF_NAMESIZE]; - - return snprintf(buf, len, "%s%%%s:%#hx", - ether_ntoa((struct ether_addr *) &sll->sll_addr), - if_indextoname(sll->sll_ifindex, ifname), - ntohs(sll->sll_protocol)); - } + case AF_INET: + inet_ntop(AF_INET, &sa->sin.sin_addr, buf, len); + break; + + case AF_PACKET: + snprintf(buf, len, "%02x", sa->sll.sll_addr[0]); + for (int i = 1; i < sa->sll.sll_halen; i++) + strap(buf, len, ":%02x", sa->sll.sll_addr[i]); + break; default: - return snprintf(buf, len, "address family: %u", sa->sa_family); + error("Unknown address family: '%u'", sa->sa.sa_family); + } + + /* Port / Interface */ + switch (sa->sa.sa_family) { + case AF_INET6: + case AF_INET: + strap(buf, len, ":%hu", ntohs(sa->sin.sin_port)); + break; + + case AF_PACKET: { + struct rtnl_link *link = nl_get_link(sa->sll.sll_ifindex, NULL); + if (!link) + error("Failed to get interface for index: %u", sa->sll.sll_ifindex); + + strap(buf, len, "%%%s", rtnl_link_get_name(link)); + strap(buf, len, ":%hu", ntohs(sa->sll.sll_protocol)); + break; + } } return 0; } -int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer layer, int flags) +int socket_parse_addr(const char *addr, struct sockaddr *saddr, enum socket_layer layer, int flags) { /** @todo: Add support for IPv6 */ + union sockaddr_union *sa = (union sockaddr_union *) saddr; char *copy = strdup(addr); int ret; if (layer == LAYER_ETH) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */ - struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - /* Split string */ char *node = strtok(copy, "%"); char *ifname = strtok(NULL, ":"); @@ -356,21 +385,26 @@ int socket_parse_addr(const char *addr, struct sockaddr *sa, enum socket_layer l /* Parse link layer (MAC) address */ struct ether_addr *mac = ether_aton(node); if (!mac) - error("Failed to parse mac address: %s", node); + error("Failed to parse MAC address: %s", node); - memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6); + memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, 6); + + /* Get interface index from name */ + struct rtnl_link *link = nl_get_link(0, ifname); + if (!link) + error("Failed to get network interface: '%s'", ifname); - sll->sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS); - sll->sll_halen = 6; - sll->sll_family = AF_PACKET; - sll->sll_ifindex = if_nametoindex(ifname); + sa->sll.sll_protocol = htons((proto) ? strtol(proto, NULL, 0) : ETH_P_S2SS); + sa->sll.sll_halen = 6; + sa->sll.sll_family = AF_PACKET; + sa->sll.sll_ifindex = rtnl_link_get_ifindex(link); ret = 0; } else { /* Format: "192.168.0.10:12001" */ struct addrinfo hint = { .ai_flags = flags, - .ai_family = AF_INET + .ai_family = AF_UNSPEC }; /* Split string */ diff --git a/server/src/tc.c b/server/src/tc.c index e290ac19f..ea9fc5f82 100644 --- a/server/src/tc.c +++ b/server/src/tc.c @@ -8,110 +8,190 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ +#include +#include +#include + +#include + #include "utils.h" #include "if.h" #include "tc.h" +#include "nl.h" -int tc_parse(config_setting_t *cfg, struct netem *em) +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne) { - em->valid = 0; + const char *str; + int val; + + rtnl_tc_set_kind(TC_CAST(ne), "netem"); - if (config_setting_lookup_string(cfg, "distribution", &em->distribution)) - em->valid |= TC_NETEM_DISTR; - if (config_setting_lookup_int(cfg, "delay", &em->delay)) - em->valid |= TC_NETEM_DELAY; - if (config_setting_lookup_int(cfg, "jitter", &em->jitter)) - em->valid |= TC_NETEM_JITTER; - if (config_setting_lookup_int(cfg, "loss", &em->loss)) - em->valid |= TC_NETEM_LOSS; - if (config_setting_lookup_int(cfg, "duplicate", &em->duplicate)) - em->valid |= TC_NETEM_DUPL; - if (config_setting_lookup_int(cfg, "corrupt", &em->corrupt)) - em->valid |= TC_NETEM_CORRUPT; + if (config_setting_lookup_string(cfg, "distribution", &str)) { + if (rtnl_netem_set_delay_distribution(ne, str)) + cerror(cfg, "Invalid delay distribution '%s' in netem config", str); + } + + if (config_setting_lookup_int(cfg, "limit", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for limit setting", val); - /** @todo Validate netem config values */ + rtnl_netem_set_limit(ne, val); + } + else + rtnl_netem_set_limit(ne, 0); + + if (config_setting_lookup_int(cfg, "delay", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for delay setting", val); + + rtnl_netem_set_delay(ne, val); + } + + if (config_setting_lookup_int(cfg, "jitter", &val)) { + if (val <= 0) + cerror(cfg, "Invalid value '%d' for jitter setting", val); + + rtnl_netem_set_jitter(ne, val); + } + + if (config_setting_lookup_int(cfg, "loss", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for loss setting", val); + + rtnl_netem_set_loss(ne, val); + } + + if (config_setting_lookup_int(cfg, "duplicate", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for duplicate setting", val); + + rtnl_netem_set_duplicate(ne, val); + } + + if (config_setting_lookup_int(cfg, "corruption", &val)) { + if (val < 0 || val > 100) + cerror(cfg, "Invalid percentage value '%d' for corruption setting", val); + + rtnl_netem_set_corruption_probability(ne, val); + } return 0; } +int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne) +{ + if (rtnl_netem_get_limit(ne) > 0) + strap(buf, len, "limit %upkts", rtnl_netem_get_limit(ne)); + + if (rtnl_netem_get_delay(ne) > 0) { + strap(buf, len, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0); + + if (rtnl_netem_get_jitter(ne) > 0) { + strap(buf, len, "jitter %f.2ms ", rtnl_netem_get_jitter(ne) / 1000.0); + + if (rtnl_netem_get_delay_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_delay_correlation(ne)); + } + } + + if (rtnl_netem_get_loss(ne) > 0) { + strap(buf, len, "loss %u%% ", rtnl_netem_get_loss(ne)); + + if (rtnl_netem_get_loss_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_loss_correlation(ne)); + } + + if (rtnl_netem_get_reorder_probability(ne) > 0) { + strap(buf, len, " reorder%u%% ", rtnl_netem_get_reorder_probability(ne)); + + if (rtnl_netem_get_reorder_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_reorder_correlation(ne)); + } + + if (rtnl_netem_get_corruption_probability(ne) > 0) { + strap(buf, len, "corruption %u%% ", rtnl_netem_get_corruption_probability(ne)); + + if (rtnl_netem_get_corruption_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_corruption_correlation(ne)); + } + + if (rtnl_netem_get_duplicate(ne) > 0) { + strap(buf, len, "duplication %u%% ", rtnl_netem_get_duplicate(ne)); + + if (rtnl_netem_get_duplicate_correlation(ne) > 0) + strap(buf, len, "%u%% ", rtnl_netem_get_duplicate_correlation(ne)); + } + + return 0; +} + +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_qdisc *q = rtnl_qdisc_alloc(); + + /* This is the default priomap used by the tc-prio qdisc + * We will use the first 'bands' bands internally */ + uint8_t map[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; + for (int i = 0; i < ARRAY_LEN(map); i++) + map[i] += bands; + + rtnl_tc_set_link(TC_CAST(q), i->nl_link); + rtnl_tc_set_parent(TC_CAST(q), TC_H_ROOT); + rtnl_tc_set_handle(TC_CAST(q), handle); + rtnl_tc_set_kind(TC_CAST(q), "prio"); + + rtnl_qdisc_prio_set_bands(q, bands + 3); + rtnl_qdisc_prio_set_priomap(q, map, ARRAY_LEN(map)); + + int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); + + *qd = q; + + return ret; +} + +int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_qdisc *q = *qd; + + rtnl_tc_set_link(TC_CAST(q), i->nl_link); + rtnl_tc_set_parent(TC_CAST(q), parent); + rtnl_tc_set_handle(TC_CAST(q), handle); + + int ret = rtnl_qdisc_add(sock, q, NLM_F_REPLACE); + + *qd = q; + + return ret; +} + +int tc_mark(struct interface *i, struct rtnl_cls **cls, tc_hdl_t flowid, uint32_t mark) +{ + struct nl_sock *sock = nl_init(); + struct rtnl_cls *c = rtnl_cls_alloc(); + + rtnl_tc_set_link(TC_CAST(c), i->nl_link); + rtnl_tc_set_handle(TC_CAST(c), mark); + rtnl_tc_set_kind(TC_CAST(c), "fw"); + + rtnl_cls_set_protocol(c, ETH_P_ALL); + + rtnl_fw_set_classid(c, flowid); + rtnl_fw_set_mask(c, 0xFFFFFFFF); + + int ret = rtnl_cls_add(sock, c, NLM_F_CREATE); + + *cls = c; + + return ret; +} + int tc_reset(struct interface *i) { - char cmd[128]; - snprintf(cmd, sizeof(cmd), "tc qdisc replace dev %s root pfifo_fast", i->name); + struct nl_sock *sock = nl_init(); - debug(6, "Reset traffic control for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add reset traffic control for interface '%s'", i->name); - - return 0; -} - -int tc_prio(struct interface *i, tc_hdl_t handle, int bands) -{ - char cmd[128]; - int len = 0; - int priomap[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; - - len += snprintf(cmd+len, sizeof(cmd)-len, - "tc qdisc replace dev %s root handle %u prio bands %u priomap", - i->name, TC_HDL_MAJ(handle), bands + 3); - - for (int i = 0; i < 16; i++) - len += snprintf(cmd+len, sizeof(cmd)-len, " %u", priomap[i] + bands); - - debug(6, "Replace master qdisc for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add prio qdisc for interface '%s'", i->name); - - return 0; -} - -int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em) -{ - int len = 0; - char cmd[256]; - len += snprintf(cmd+len, sizeof(cmd)-len, - "tc qdisc replace dev %s parent %u:%u netem", - i->name, TC_HDL_MAJ(parent), TC_HDL_MIN(parent)); - - if (em->valid & TC_NETEM_DELAY) { - len += snprintf(cmd+len, sizeof(cmd)-len, " delay %u", em->delay); - - if (em->valid & TC_NETEM_JITTER) - len += snprintf(cmd+len, sizeof(cmd)-len, " %u", em->jitter); - if (em->valid & TC_NETEM_DISTR) - len += snprintf(cmd+len, sizeof(cmd)-len, " distribution %s", em->distribution); - } - - if (em->valid & TC_NETEM_LOSS) - len += snprintf(cmd+len, sizeof(cmd)-len, " loss random %u", em->loss); - if (em->valid & TC_NETEM_DUPL) - len += snprintf(cmd+len, sizeof(cmd)-len, " duplicate %u", em->duplicate); - if (em->valid & TC_NETEM_CORRUPT) - len += snprintf(cmd+len, sizeof(cmd)-len, " corrupt %u", em->corrupt); - - debug(6, "Setup netem qdisc for interface '%s'", i->name); - - if (system2(cmd)) - error("Failed to add netem qdisc for interface '%s'", i->name); - - return 0; -} - -int tc_mark(struct interface *i, tc_hdl_t flowid, int mark) -{ - char cmd[128]; - snprintf(cmd, sizeof(cmd), - "tc filter replace dev %s protocol ip handle %u fw flowid %u:%u", - i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid)); - - debug(7, "Add traffic filter to interface '%s': fwmark %u => flowid %u:%u", - i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid)); - - if (system2(cmd)) - error("Failed to add fw_mark classifier for interface '%s'", i->name); - - return 0; + /* We restore the default pfifo_fast qdisc, by deleting ours */ + return rtnl_qdisc_delete(sock, i->tc_qdisc); }