added new list of interfaces for coming netem and irq features

git-svn-id: https://zerberus.eonerc.rwth-aachen.de:8443/svn/s2ss/trunk@78 8ec27952-4edc-4aab-86aa-e87bb2611832

include/cfg.h

@@ -12,6 +12,7 @@
 
 struct node;
 struct path;
+struct interface;
 
 /** Global configuration */
 struct settings {
@@ -42,8 +43,8 @@ struct settings {
  *  - 0 on success
  *  - otherwise an error occured
  */
-int config_parse(const char *filename, config_t *cfg,
-	struct settings *set, struct node **nodes, struct path **paths);
+int config_parse(const char *filename, config_t *cfg, struct settings *set,
+	struct node **nodes, struct path **paths, struct interface **interfaces);
 
 /** Parse the global section of a configuration file.
  *
@@ -64,16 +65,23 @@ int config_parse_global(config_setting_t *cfg, struct settings *set);
  *  - 0 on success
  *  - otherwise an error occured
  */
-int config_parse_path(config_setting_t *cfg, struct path **paths, struct node *nodes);
+int config_parse_path(config_setting_t *cfg,
+	struct path **paths, struct node **nodes);
 
 /** Parse a single node and add it to the global configuration.
  *
  * @param cfg A libconfig object pointing to the node
  * @param nodes Add new nodes to this linked list
+ * @param interfaces Search this list for existing interfaces
+ * @return
+ *  - 0 on success
+ *  - otherwise an error occured
+ */
+int config_parse_node(config_setting_t *cfg,
+	struct node **nodes, struct interface **interfaces);
  * @return
  *  - 0 on success
  *  - otherwise an error occured
  */
-int config_parse_node(config_setting_t *cfg, struct node **nodes);
 
 #endif /* _CFG_H_ */

include/if.h

@@ -8,25 +8,66 @@
 #ifndef _IF_H_
 #define _IF_H_
 
+#include <sys/types.h>
 #include <net/if.h>
 
-/** Get the name of first interface which listens on addr.
+#define IF_NAME_MAX	IFNAMSIZ
+#define IF_IRQ_MAX	3
+
+/** Interface data structure */
+struct interface {
+	/** The index used by the kernel to reference this interface */
+	int index;
+	/** How many nodes use this interface for outgoing packets */
+	int refcnt;
+	/** Human readable name of this interface */
+	char name[IF_NAME_MAX];
+	/** List of IRQs of the NIC */
+	char irqs[IF_IRQ_MAX];
+
+	/** Linked list pointer */
+	struct interface *next;
+};
+
+/** Get outgoing interface
  *
- * @param addr The address whose related interface is searched
- * @return
- *  - A pointer to the interface name (has to be freed by caller)
- *  - NULL if the address is not used on the system
+ * Does a lookup in the kernel routing table to determine
+ * the interface which sends the data to a certain socket
+ * address.
+ *
+ * @param sa A destination address for outgoing packets
+ * @return The interface index
  */
-char* if_addrtoname(struct sockaddr *addr);
+int if_getegress(struct sockaddr_in *sa);
+
+/** Get all IRQs for this interface
+ *
+ * Only MSI IRQs are determined by looking at:
+ *  /sys/class/net/{ifname}/device/msi_irqs/
+ *
+ * @param i A pointer to the interface structure
+ * @return
+ *  - 0 on success
+ *  - otherwise an error occured
+ */
+int if_getirqs(struct interface *i);
 
 /** Change the SMP affinity of NIC interrupts.
  *
- * @param ifname The interface whose IRQs should be pinned
+ * @param i A pointer to the interface structure
  * @param affinity A mask specifying which cores should handle this interrupt.
  * @return
  *  - 0 on success
  *  - otherwise an error occured
  */
-int if_setaffinity(const char *ifname, int affinity);
+int if_setaffinity(struct interface *i, int affinity);
+
+/** Search list of interface for a index
+ *
+ * @param index The interface index to search for
+ * @param interfaces A linked list of all interfaces
+ * @return A pointer to the node or NULL if not found
+ */
+struct interface* if_lookup_index(int index, struct interface *interfaces);
 
 #endif /* _IF_H_ */

include/node.h

@@ -55,10 +55,9 @@ struct node
 	/** Remote address of the socket */
 	struct sockaddr_in remote;
 
-	/** Name of the local interface */
-	const char *ifname;
-	/** Index of the local interface */
-	int ifindex;
+	/** The egress interface */
+	struct interface *interface;
+
 	/** Socket mark for netem, routing and filtering */
 	int mark;
 

include/utils.h

@@ -50,20 +50,7 @@ void print(enum log_level lvl, const char *fmt, ...);
  */
 int resolve_addr(const char *addr, struct sockaddr_in *sa, int flags);
 
-/** Compare two socket addresses based on their family and address.
- *
- * Only the family and the address is compared.
- * Port numbers etc are ignored.
- *
- * @param a First address
- * @param b Second address
- * @return
- *  - 0 if the addresses are equal
- *  - otherwise they are not equal
- */
-int sockaddr_cmp(struct sockaddr *a, struct sockaddr *b);
-
-/** Convert integer into cpu_set_t
+/** Convert integer to cpu_set_t
  *
  * @param set A cpu bitmask
  * @return The opaque cpu_set_t datatype

src/cfg.c

@@ -18,8 +18,8 @@
 #include "path.h"
 #include "utils.h"
 
-int config_parse(const char *filename, config_t *cfg,
-	struct settings *set, struct node **nodes, struct path **paths)
+int config_parse(const char *filename, config_t *cfg, struct settings *set,
+	struct node **nodes, struct path **paths, struct interface **interfaces)
 {
 	if (!config_read_file(cfg, filename)) {
 		error("Failed to parse configuration: %s in %s:%d",
@@ -48,13 +48,13 @@ int config_parse(const char *filename, config_t *cfg,
 	/* Parse nodes */
 	for (int i = 0; i < config_setting_length(cfg_nodes); i++) {
 		config_setting_t *cfg_node = config_setting_get_elem(cfg_nodes, i);
-		config_parse_node(cfg_node, nodes);
+		config_parse_node(cfg_node, nodes, interfaces);
 	}
 
 	/* Parse paths */
 	for (int i = 0; i < config_setting_length(cfg_paths); i++) {
 		config_setting_t *cfg_path = config_setting_get_elem(cfg_paths, i);
-		config_parse_path(cfg_path, paths, *nodes);
+		config_parse_path(cfg_path, paths, nodes);
 	}
 
 	return CONFIG_TRUE;
@@ -99,7 +99,7 @@ int config_parse_global(config_setting_t *cfg, struct settings *set)
 }
 
 int config_parse_path(config_setting_t *cfg,
-	struct path **paths, struct node *nodes)
+	struct path **paths, struct node **nodes)
 {
 	const char *in_str = NULL;
 	const char *out_str = NULL;
@@ -123,11 +123,11 @@ int config_parse_path(config_setting_t *cfg,
 	if (!config_setting_lookup_string(cfg, "out", &out_str))
 		cerror(cfg, "Missing output node for path");
 
-	path->in = node_lookup_name(in_str, nodes);
+	path->in = node_lookup_name(in_str, *nodes);
 	if (!path->in)
 		cerror(cfg, "Invalid input node '%s'");
 
-	path->out = node_lookup_name(out_str, nodes);
+	path->out = node_lookup_name(out_str, *nodes);
 	if (!path->out)
 		cerror(cfg, "Invalid output node '%s'", out_str);
 
@@ -159,7 +159,8 @@ int config_parse_path(config_setting_t *cfg,
 	return 0;
 }
 
-int config_parse_node(config_setting_t *cfg, struct node **nodes)
+int config_parse_node(config_setting_t *cfg,
+	struct node **nodes, struct interface **interfaces)
 {
 	const char *type_str = NULL;
 	const char *remote_str = NULL;
@@ -201,13 +202,22 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes)
 	if (resolve_addr(remote_str, &node->remote, 0))
 		cerror(cfg, "Failed to resolve remote address '%s' of node '%s'", remote_str, node->name);
 
-	if (!config_setting_lookup_string(cfg, "interface", &node->ifname)) {
-		node->ifname = if_addrtoname((struct sockaddr*) &node->local);
+	/* Determine outgoing interface */
+	int index = if_getegress(&node->remote);
+	struct interface *i = if_lookup_index(index, *interfaces);
+	if (!i) {
+		i = malloc(sizeof(struct interface));
+		memset(i, 0, sizeof(struct interface));
+
+		i->index = index;
+
+		list_add(*interfaces, i);
 	}
 
+	node->mark = 1 + i->refcnt++;
+	node->interface = i;
+
 	node->cfg = cfg;
-	node->ifindex = if_nametoindex(node->ifname);
-	node->mark = node->ifindex + 8000;
 
 	list_add(*nodes, node);
 

src/if.c

@@ -10,68 +10,94 @@
 #include <string.h>
 
 #include <dirent.h>
-#include <ifaddrs.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/ip.h>
+#include <arpa/inet.h>
+#include <net/if.h>
 
+#include "if.h"
 #include "utils.h"
 
-char* if_addrtoname(struct sockaddr *addr)
+int if_getegress(struct sockaddr_in *sa)
 {
-	char *ifname = NULL;
-	struct ifaddrs *ifas, *ifa;
+	char cmd[128];
+	char token[32];
 
-	if(getifaddrs(&ifas))
-		return NULL;
+	snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sa->sin_addr));
 
-	for(ifa = ifas; ifa; ifa = ifa->ifa_next) {
-		if (ifa->ifa_addr && !sockaddr_cmp(ifa->ifa_addr, addr)) {
-			ifname = malloc(strlen(ifa->ifa_name) + 1);
-			if (!ifname)
-				goto out;
+	debug(6, "system: %s", cmd);
+	FILE *p = popen(cmd, "r");
+	if (!p)
+		return -1;
 
-			strcpy(ifname, ifa->ifa_name);
-			goto out;
+	while (fscanf(p, "%31s", token)) {
+		if (!strcmp(token, "dev")) {
+			fscanf(p, "%31s", token);
+			break;
 		}
 	}
 
-out:
-	freeifaddrs(ifas);
+	fclose(p);
 
-	return ifname;
+	return if_nametoindex(token);
 }
 
-int if_setaffinity(const char *ifname, int affinity)
+int if_getirqs(struct interface *i)
 {
-	char *dirname[NAME_MAX];
-	char *filename[NAME_MAX];
-
+	char dirname[NAME_MAX];
 	DIR *dir;
-	FILE *file;
 
-	/* Determine IRQs numbers */
-	snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", ifname);
+	snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name);
 	dir = opendir(dirname);
 	if (!dir)
 		return -1;
 
+	memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX);
+
+	int n = 0;
 	struct dirent *entry;
-	while (entry = readdir(dir)) {
-		/* Set SMP affinity */
-		snprintf(filename, sizeof(filename), "/proc/irq/%s/smp_affinity");
+	while((entry = readdir(dir)) && n < IF_IRQ_MAX) {
+		if (entry->d_type & DT_REG) {
+			i->irqs[n++] = atoi(entry->d_name);
+		}
+	}
+
+	closedir(dir);
+	return 0;
+}
+
+int if_setaffinity(struct interface *i, int affinity)
+{
+	char filename[NAME_MAX];
+	FILE *file;
+
+	for (int n = 0; n < IF_IRQ_MAX && i->irqs[n]; n++) {
+		snprintf(filename, sizeof(filename), "/proc/irq/%u/smp_affinity", i->irqs[n]);
+
 		file = fopen(filename, "w");
 		if (!file)
 			continue;
 
-		debug(3, "Setting SMP affinity of IRQ %s (%s) to %8x\n", entry->d_name, ifname, affinity);
+		if (fprintf(file, "%8x", affinity) < 0)
+			error("Failed to set affinity for IRQ %u", i->irqs[n]);
+		else
+			debug(3, "Set affinity of MSI IRQ %u (%s) to %#x", i->irqs[n], i->name, affinity);
 
-		fprintf("%8x", affinity);
 		fclose(file);
 	}
 
-	closedir(dir);
-
 	return 0;
 }
+
+struct interface* if_lookup_index(int index, struct interface *interfaces)
+{
+	for (struct interface *i = interfaces; i; i = i->next) {
+		if (i->index == index) {
+			return i;
+		}
+	}
+
+	return NULL;
+}

src/server.c

@@ -19,6 +19,7 @@
 
 #include "config.h"
 #include "cfg.h"
+#include "if.h"
 #include "msg.h"
 #include "utils.h"
 #include "path.h"
@@ -26,9 +27,10 @@
 
 /** Linked list of nodes */
 static struct node *nodes;
-
 /** Linked list of paths */
 static struct path *paths;
+/** Linked list of interfaces */
+static struct interface *interfaces;
 
 /** Default settings */
 static struct settings settings = {
@@ -126,7 +128,7 @@ int main(int argc, char *argv[])
 
 	/* Parse configuration file */
 	config_init(&config);
-	config_parse(argv[1], &config, &settings, &nodes, &paths);
+	config_parse(argv[1], &config, &settings, &nodes, &paths, &interfaces);
 
 	if (!paths)
 		error("No paths found. Terminating...");

src/utils.c

@@ -81,29 +81,6 @@ int resolve_addr(const char *addr, struct sockaddr_in *sa, int flags)
 	return 0;
 }
 
-int sockaddr_cmp(struct sockaddr *a, struct sockaddr *b)
-{
-	if (a->sa_family == b->sa_family) {
-		switch (a->sa_family) {
-			case AF_INET: {
-				struct sockaddr_in *ai = (struct sockaddr_in *) a;
-				struct sockaddr_in *bi = (struct sockaddr_in *) b;
-
-				return memcmp(&ai->sin_addr, &bi->sin_addr, sizeof(struct in_addr));
-			}
-
-			case AF_INET6: {
-				struct sockaddr_in6 *ai = (struct sockaddr_in6 *) a;
-				struct sockaddr_in6 *bi = (struct sockaddr_in6 *) b;
-
-				return memcmp(&ai->sin6_addr, &bi->sin6_addr, sizeof(struct in6_addr));
-			}
-		}
-	}
-
-	return -1;
-}
-
 cpu_set_t to_cpu_set(int set)
 {
 	cpu_set_t cset;