diff --git a/server/include/nl.h b/server/include/nl.h index 3132a6276..599d08147 100644 --- a/server/include/nl.h +++ b/server/include/nl.h @@ -1,4 +1,4 @@ -/** +/** Netlink related functions. * * @file * @author Steffen Vogel @@ -14,27 +14,17 @@ #include #include -/** Get libnl3 link object from interface name +/** Get index of outgoing interface for given destination address. * - * @param dev The name of the interface - * @retval 0 Error. Something went wrong. - * @retval >0 A pointer to the libnl3 link object. + * @retval >=0 Interface index of outgoing interface. + * @retval <0 Error. Something went wrong. */ -struct rtnl_link * nl_get_link(int ifindex, const char *dev); +int nl_get_egress(struct nl_addr *addr); -/** - * - */ -int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop); - -/** - * - */ +/** Get or create global netlink socket. */ struct nl_sock * nl_init(); -/** - * - */ +/** Close and free global netlink socket. */ void nl_shutdown(); #endif \ No newline at end of file diff --git a/server/include/tc.h b/server/include/tc.h index 397cd00a9..a5b460154 100644 --- a/server/include/tc.h +++ b/server/include/tc.h @@ -29,11 +29,11 @@ struct interface; /** Parse network emulator (netem) settings. * * @param cfg A libconfig object containing the settings. - * @param ne A pointer to a libnl3 qdisc object where setting will be written to. + * @param[out] ne A pointer to a libnl3 qdisc object where setting will be written to. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne); +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **ne); /** Print network emulator (netem) setting into buffer. * @@ -58,11 +58,12 @@ int tc_reset(struct interface *i); * @param i[in] The interface * @param qd[in,out] The libnl3 object of the new prio qdisc. * @param handle[in] The handle for the new qdisc + * @param parent[in] Make this qdisc a child of this class * @param bands[in] The number of classes for this new qdisc * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands); +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t, int bands); /** Add a new network emulator (netem) discipline. * diff --git a/server/src/if.c b/server/src/if.c index 90d324238..28125c253 100644 --- a/server/src/if.c +++ b/server/src/if.c @@ -56,8 +56,11 @@ int if_start(struct interface *i, int affinity) info("Starting interface '%s' which is used by %u sockets", rtnl_link_get_name(i->nl_link), list_length(&i->sockets)); { INDENT + /* Set affinity for network interfaces (skip _loopback_ dev) */ + if_set_affinity(i, affinity); + /* Assign fwmark's to socket nodes which have netem options */ - int mark = 0; + int ret, mark = 0; FOREACH(&i->sockets, it) { struct socket *s = it->socket; if (s->tc_qdisc) @@ -69,24 +72,25 @@ int if_start(struct interface *i, int affinity) return 0; /* Replace root qdisc */ - tc_prio(i, &i->tc_qdisc, TC_HANDLE(4000, 0), mark); + if ((ret = tc_prio(i, &i->tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, mark))) + ;//error("Failed to setup priority queuing discipline: %s", nl_geterror(ret)); /* Create netem qdisks and appropriate filter per netem node */ FOREACH(&i->sockets, it) { struct socket *s = it->socket; if (s->tc_qdisc) { - tc_mark(i, &s->tc_classifier, TC_HANDLE(4000, s->mark), s->mark); - tc_netem(i, &s->tc_qdisc, TC_HANDLE(8000, s->mark), TC_HANDLE(4000, s->mark)); + if ((ret = tc_mark(i, &s->tc_classifier, TC_HANDLE(1, s->mark), s->mark))) + error("Failed to setup FW mark classifier: %s", nl_geterror(ret)); char buf[256]; tc_print(buf, sizeof(buf), s->tc_qdisc); debug(5, "Starting network emulation on interface '%s' for FW mark %u: %s", rtnl_link_get_name(i->nl_link), s->mark, buf); + + if ((ret = tc_netem(i, &s->tc_qdisc, TC_HANDLE(0x1000+s->mark, 0), TC_HANDLE(1, s->mark)))) + error("Failed to setup netem qdisc: %s", nl_geterror(ret)); } } - - /* Set affinity for network interfaces (skip _loopback_ dev) */ - if_set_affinity(i, affinity); } return 0; @@ -108,6 +112,8 @@ int if_stop(struct interface *i) int if_get_egress(struct sockaddr *sa, struct rtnl_link **link) { + int ifindex = -1; + switch (sa->sa_family) { case AF_INET: case AF_INET6: { @@ -118,22 +124,24 @@ int if_get_egress(struct sockaddr *sa, struct rtnl_link **link) ? nl_addr_build(sin->sin_family, &sin->sin_addr.s_addr, sizeof(sin->sin_addr.s_addr)) : nl_addr_build(sin6->sin6_family, sin6->sin6_addr.s6_addr, sizeof(sin6->sin6_addr)); - struct rtnl_nexthop *nh; - if (nl_get_nexthop(addr, &nh)) - return -1; - - *link = nl_get_link(rtnl_route_nh_get_ifindex(nh), NULL); - - rtnl_route_nh_free(nh); + ifindex = nl_get_egress(addr); + if (ifindex < 0) + error("Netlink error: %s", nl_geterror(ifindex)); + break; } case AF_PACKET: { struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; - *link = nl_get_link(sll->sll_ifindex, NULL); + ifindex = sll->sll_ifindex; + break; } } - + + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + if (!(*link = rtnl_link_get(cache, ifindex))) + return -1; + return 0; } diff --git a/server/src/nl.c b/server/src/nl.c index e634b5cfa..ac345971d 100644 --- a/server/src/nl.c +++ b/server/src/nl.c @@ -1,4 +1,4 @@ -/** Netlink related functions +/** Netlink related functions. * * S2SS uses libnl3 to talk to the Linux kernel to gather networking related information * @@ -11,22 +11,33 @@ #include #include +#include #include "utils.h" #include "nl.h" -/** Singleton for netlink socket */ +/** Singleton for global netlink socket */ static struct nl_sock *sock = NULL; struct nl_sock * nl_init() { + int ret; + if (!sock) { /* Create connection to netlink */ sock = nl_socket_alloc(); if (!sock) error("Failed to allocate memory"); - nl_connect(sock, NETLINK_ROUTE); + if ((ret = nl_connect(sock, NETLINK_ROUTE))) + error("Failed to connect to kernel: %s", nl_geterror(ret)); + + /* Fill some caches */ + struct nl_cache *cache; + if ((ret = rtnl_link_alloc_cache(sock, AF_UNSPEC, &cache))) + error("Failed to get list of interfaces: %s", nl_geterror(ret)); + + nl_cache_mngt_provide(cache); } return sock; @@ -40,55 +51,54 @@ void nl_shutdown() sock = NULL; } -struct rtnl_link * nl_get_link(int ifindex, const char *dev) +static int egress_cb(struct nl_msg *msg, void *arg) { - struct nl_sock *sock = nl_init(); - struct rtnl_link *link; + struct rtnl_route **route = (struct rtnl_route **) arg; - if (rtnl_link_get_kernel(sock, ifindex, dev, &link)) - return NULL; - else - return link; + if (rtnl_route_parse(nlmsg_hdr(msg), route)) + return NL_SKIP; + + return NL_STOP; } -int nl_get_nexthop(struct nl_addr *addr, struct rtnl_nexthop **nexthop) +int nl_get_egress(struct nl_addr *addr) { int ret; - - struct rtnl_route *route; - struct rtnl_nexthop *nh; - struct nl_sock *sock = nl_init(); struct nl_msg *msg = nlmsg_alloc_simple(RTM_GETROUTE, 0); - struct nl_cache_ops *ops = nl_cache_ops_lookup_safe("route/route"); - struct nl_cache *cache = nl_cache_alloc(ops); - + /* Build message */ struct rtmsg rmsg = { .rtm_family = nl_addr_get_family(addr), .rtm_dst_len = nl_addr_get_prefixlen(addr), }; - - if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO)) < 0) + + if ((ret = nlmsg_append(msg, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO))) return ret; - if ((ret = nla_put_addr(msg, RTA_DST, addr)) < 0) + if ((ret = nla_put_addr(msg, RTA_DST, addr))) return ret; - if ((ret = nl_send_auto_complete(sock, msg)) < 0) - return ret; - if ((ret = nl_cache_pickup(sock, cache)) < 0) - return ret; - - if (!(route = (struct rtnl_route *) nl_cache_get_first(cache))) - return -1; - if (!(nh = rtnl_route_nexthop_n(route, 0))) - return -1; - - *nexthop = rtnl_route_nh_clone(nh); - - nl_cache_put(cache); - nl_cache_ops_put(ops); + /* Send message */ + ret = nl_send_auto(sock, msg); nlmsg_free(msg); + if (ret < 0) + return ret; - return 0; + /* Hook into receive chain */ + struct rtnl_route *route = NULL; + struct nl_cb *cb = nl_cb_alloc(NL_CB_VALID); + nl_cb_set(cb, NL_CB_VALID, NL_CB_CUSTOM, egress_cb, &route); + + /* Receive message */ + nl_recvmsgs_report(sock, cb); + nl_wait_for_ack(sock); + nl_cb_put(cb); + + /* Check result */ + if (route && (1 <= rtnl_route_get_nnexthops(route))) { + struct rtnl_nexthop *nh = rtnl_route_nexthop_n(route, 0); + return rtnl_route_nh_get_ifindex(nh); + } + else + return -1; } diff --git a/server/src/socket.c b/server/src/socket.c index fd2835091..31a95680f 100644 --- a/server/src/socket.c +++ b/server/src/socket.c @@ -39,6 +39,7 @@ static struct list sockets; int socket_init(int argc, char * argv[], struct settings *set) { INDENT + nl_init(); /* Fill link cache */ list_init(&interfaces, (dtor_cb_t) if_destroy); /* Gather list of used network interfaces */ @@ -47,14 +48,14 @@ int socket_init(int argc, char * argv[], struct settings *set) struct rtnl_link *link; /* Determine outgoing interface */ - if (if_get_egress((struct sockaddr *) &s->remote, &link) || !link) { + if (if_get_egress((struct sockaddr *) &s->remote, &link)) { char buf[128]; socket_print_addr(buf, sizeof(buf), (struct sockaddr *) &s->remote); error("Failed to get interface for socket address '%s'", buf); } - int index = rtnl_link_get_ifindex(link); - struct interface *i = if_lookup_index(index); + int ifindex = rtnl_link_get_ifindex(link); + struct interface *i = if_lookup_index(ifindex); if (!i) i = if_create(link); @@ -307,13 +308,8 @@ int socket_parse(config_setting_t *cfg, struct node *n) config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem"); if (cfg_netem) { int enabled = 1; - if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) { - s->tc_qdisc = rtnl_qdisc_alloc(); - if (!s->tc_qdisc) - error("Failed to allocated memory!"); - - tc_parse(cfg_netem, s->tc_qdisc); - } + if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) + tc_parse(cfg_netem, &s->tc_qdisc); } n->socket = s; @@ -355,7 +351,8 @@ int socket_print_addr(char *buf, int len, struct sockaddr *saddr) break; case AF_PACKET: { - struct rtnl_link *link = nl_get_link(sa->sll.sll_ifindex, NULL); + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + struct rtnl_link *link = rtnl_link_get(cache, sa->sll.sll_ifindex); if (!link) error("Failed to get interface for index: %u", sa->sll.sll_ifindex); @@ -390,7 +387,8 @@ int socket_parse_addr(const char *addr, struct sockaddr *saddr, enum socket_laye memcpy(&sa->sll.sll_addr, &mac->ether_addr_octet, 6); /* Get interface index from name */ - struct rtnl_link *link = nl_get_link(0, ifname); + struct nl_cache *cache = nl_cache_mngt_require("route/link"); + struct rtnl_link *link = rtnl_link_get_by_name(cache, ifname); if (!link) error("Failed to get network interface: '%s'", ifname); diff --git a/server/src/tc.c b/server/src/tc.c index ea9fc5f82..5f2863d8c 100644 --- a/server/src/tc.c +++ b/server/src/tc.c @@ -19,11 +19,15 @@ #include "tc.h" #include "nl.h" -int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne) +int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **netem) { const char *str; int val; + struct rtnl_qdisc *ne = rtnl_qdisc_alloc(); + if (!ne) + error("Failed to allocated memory!"); + rtnl_tc_set_kind(TC_CAST(ne), "netem"); if (config_setting_lookup_string(cfg, "distribution", &str)) { @@ -75,11 +79,15 @@ int tc_parse(config_setting_t *cfg, struct rtnl_qdisc *ne) rtnl_netem_set_corruption_probability(ne, val); } + *netem = ne; + return 0; } int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne) -{ +{ + *buf = 0; /* start from the beginning */ + if (rtnl_netem_get_limit(ne) > 0) strap(buf, len, "limit %upkts", rtnl_netem_get_limit(ne)); @@ -87,7 +95,7 @@ int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne) strap(buf, len, "delay %.2fms ", rtnl_netem_get_delay(ne) / 1000.0); if (rtnl_netem_get_jitter(ne) > 0) { - strap(buf, len, "jitter %f.2ms ", rtnl_netem_get_jitter(ne) / 1000.0); + strap(buf, len, "jitter %.2fms ", rtnl_netem_get_jitter(ne) / 1000.0); if (rtnl_netem_get_delay_correlation(ne) > 0) strap(buf, len, "%u%% ", rtnl_netem_get_delay_correlation(ne)); @@ -125,26 +133,26 @@ int tc_print(char *buf, size_t len, struct rtnl_qdisc *ne) return 0; } -int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, int bands) +int tc_prio(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hdl_t parent, int bands) { struct nl_sock *sock = nl_init(); struct rtnl_qdisc *q = rtnl_qdisc_alloc(); /* This is the default priomap used by the tc-prio qdisc * We will use the first 'bands' bands internally */ - uint8_t map[] = { 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; + uint8_t map[] = QDISC_PRIO_DEFAULT_PRIOMAP; for (int i = 0; i < ARRAY_LEN(map); i++) map[i] += bands; rtnl_tc_set_link(TC_CAST(q), i->nl_link); - rtnl_tc_set_parent(TC_CAST(q), TC_H_ROOT); + rtnl_tc_set_parent(TC_CAST(q), parent); rtnl_tc_set_handle(TC_CAST(q), handle); rtnl_tc_set_kind(TC_CAST(q), "prio"); rtnl_qdisc_prio_set_bands(q, bands + 3); - rtnl_qdisc_prio_set_priomap(q, map, ARRAY_LEN(map)); + rtnl_qdisc_prio_set_priomap(q, map, sizeof(map)); - int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); + int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE | NLM_F_REPLACE); *qd = q; @@ -159,8 +167,9 @@ int tc_netem(struct interface *i, struct rtnl_qdisc **qd, tc_hdl_t handle, tc_hd rtnl_tc_set_link(TC_CAST(q), i->nl_link); rtnl_tc_set_parent(TC_CAST(q), parent); rtnl_tc_set_handle(TC_CAST(q), handle); + //rtnl_tc_set_kind(TC_CAST(q), "netem"); - int ret = rtnl_qdisc_add(sock, q, NLM_F_REPLACE); + int ret = rtnl_qdisc_add(sock, q, NLM_F_CREATE); *qd = q;