diff --git a/server/Makefile b/server/Makefile index 7bcd5b9f9..b71f16836 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,15 +1,17 @@ -TARGETS = server send receive random test -SRCS = server.c send.c receive.c random.c node.c path.c utils.c msg.c cfg.c if.c tc.c +TARGETS = server send random receive test +SRCS = server.c send.c receive.c random.c node.c path.c utils.c socket.c msg.c cfg.c if.c tc.c # Default target: build everything all: $(TARGETS) +COMMON = socket.o if.o utils.o msg.o node.o cfg.o tc.o hooks.o + # Dependencies for individual binaries -server: node.o msg.o utils.o path.o cfg.o if.o tc.o hooks.o -send: node.o msg.o utils.o -receive: node.o msg.o utils.o -random: node.o msg.o utils.o -test: node.o msg.o utils.o +server: $(COMMON) path.o +send: $(COMMON) +receive: $(COMMON) +random: utils.o msg.o +test: $(COMMON) VPATH = src diff --git a/server/include/cfg.h b/server/include/cfg.h index b2d88b55b..5e93c42aa 100644 --- a/server/include/cfg.h +++ b/server/include/cfg.h @@ -18,6 +18,9 @@ struct node; struct path; struct interface; +struct socket; +struct opal; +struct gtfpga; struct netem; /** Global configuration */ @@ -72,20 +75,42 @@ int config_parse_path(config_setting_t *cfg, * * @param cfg A libconfig object pointing to the node * @param nodes Add new nodes to this linked list - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int config_parse_node(config_setting_t *cfg, struct node **nodes); +/** Parse node connection details for OPAL type + * + * @param cfg A libconfig object pointing to the node + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int config_parse_opal(config_setting_t *cfg, struct node *n); + +/** Parse node connection details for GTFPGA type + * + * @param cfg A libconfig object pointing to the node + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int config_parse_gtfpga(config_setting_t *cfg, struct node *n); + +/** Parse node connection details for SOCKET type + * + * @param cfg A libconfig object pointing to the node + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int config_parse_socket(config_setting_t *cfg, struct node *n); + /** Parse network emulator (netem) settings. * * @param cfg A libconfig object containing the settings * @param em A pointer to the settings - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int config_parse_netem(config_setting_t *cfg, struct netem *em); diff --git a/server/include/config.h b/server/include/config.h index 4c5f8b38d..c6100ea30 100644 --- a/server/include/config.h +++ b/server/include/config.h @@ -24,4 +24,8 @@ #define HIST_HEIGHT 50 #define HIST_SEQ 17 +/* Protocol numbers */ +#define IPPROTO_S2SS 137 +#define ETH_P_S2SS 0xBABE + #endif /* _CONFIG_H_ */ diff --git a/server/include/gtfpga.h b/server/include/gtfpga.h new file mode 100644 index 000000000..b449d5f60 --- /dev/null +++ b/server/include/gtfpga.h @@ -0,0 +1,16 @@ +/** Node type: GTFPGA (Xilinx ML507) + * + * This file implements the gtfpga subtype for nodes. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#ifndef _GTFPGA_H_ +#define _GTFPGA_H_ + +struct gtfpga { + +}; + +#endif /* _GTFPGA_H_ */ diff --git a/server/include/if.h b/server/include/if.h index 0d103a0f7..a236e6bd3 100644 --- a/server/include/if.h +++ b/server/include/if.h @@ -17,6 +17,8 @@ #define IF_NAME_MAX IFNAMSIZ /**< Maximum length of an interface name */ #define IF_IRQ_MAX 3 /**< Maxmimal number of IRQs of an interface */ +struct socket; + /** Interface data structure */ struct interface { /** The index used by the kernel to reference this interface */ @@ -28,20 +30,57 @@ struct interface { /** List of IRQs of the NIC */ char irqs[IF_IRQ_MAX]; + /** Linked list of associated sockets */ + struct socket *sockets; /** Linked list pointer */ struct interface *next; }; +/** Add a new interface to the global list and lookup name, irqs... + * + * @param index The interface index of the OS + * @retval >0 Success. A pointer to the new interface. + * @retval 0 Error. The creation failed. + */ +struct interface * if_create(int index); + +/** Start interface. + * + * This setups traffic controls queue discs, network emulation and + * maps interface IRQs according to affinity. + * + * @param i A pointer to the interface structure. + * @param affinity Set the IRQ affinity of this interface. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int if_start(struct interface *i, int affinity); + +/** Stop interface + * + * This resets traffic qdiscs ant network emulation + * and maps interface IRQs to all CPUs. + * + * @param i A pointer to the interface structure. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int if_stop(struct interface *i); + /** Get outgoing interface. * - * Does a lookup in the kernel routing table to determine - * the interface which sends the data to a certain socket - * address. + * Depending on the address family of the socker address, + * this function tries to determine outgoing interface + * which is used to send packages to a remote host with the specified + * socket address. * - * @param sa A destination address for outgoing packets - * @return The interface index + * For AF_INET the fnuction performs a lookup in the kernel routing table. + * For AF_PACKET the function uses the existing sll_ifindex field of the socket address. + * + * @param sa A destination address for outgoing packets. + * @return The interface index. */ -int if_getegress(struct sockaddr_in *sa); +int if_getegress(struct sockaddr *sa); /** Get all IRQs for this interface. * @@ -49,9 +88,8 @@ int if_getegress(struct sockaddr_in *sa); * /sys/class/net/{ifname}/device/msi_irqs/ * * @param i A pointer to the interface structure - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int if_getirqs(struct interface *i); @@ -59,18 +97,18 @@ int if_getirqs(struct interface *i); * * @param i A pointer to the interface structure * @param affinity A mask specifying which cores should handle this interrupt. - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int if_setaffinity(struct interface *i, int affinity); -/** Search list of interface for a index. +/** Search the list of interfaces for a given index. * * @param index The interface index to search for * @param interfaces A linked list of all interfaces - * @return A pointer to the node or NULL if not found + * @retval NULL if no interface with index was found. + * @retval >0 Success. A pointer to the interface. */ -struct interface* if_lookup_index(int index, struct interface *interfaces); +struct interface * if_lookup_index(int index); #endif /* _IF_H_ */ diff --git a/server/include/msg.h b/server/include/msg.h index 7a3f2778b..ffb3f019d 100644 --- a/server/include/msg.h +++ b/server/include/msg.h @@ -31,9 +31,8 @@ void msg_swap(struct msg *m); * * @param f The file stream * @param m A pointer to the message - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int msg_fprint(FILE *f, struct msg *m); @@ -41,9 +40,8 @@ int msg_fprint(FILE *f, struct msg *m); * * @param f The file stream * @param m A pointer to the message - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int msg_fscan(FILE *f, struct msg *m); @@ -53,24 +51,4 @@ int msg_fscan(FILE *f, struct msg *m); */ void msg_random(struct msg *m); -/** Send a message to a node. - * - * @param m A pointer to the message - * @param n A pointer to the node - * @return - * - 0 on success - * - otherwise an error occured - */ -int msg_send(struct msg *m, struct node *n); - -/** Receive a message from a node. - * - * @param m A pointer to the message - * @param n A pointer to the node - * @return - * - 0 on success - * - otherwise an error occured - */ -int msg_recv(struct msg *m, struct node *n); - #endif /* _MSG_H_ */ diff --git a/server/include/node.h b/server/include/node.h index 4ce8cdb16..fe31fdf2e 100644 --- a/server/include/node.h +++ b/server/include/node.h @@ -23,35 +23,61 @@ /** Static node initialization */ #define NODE_INIT(n) { \ - .sd = -1, \ .name = n \ } -/** The datastructure for a node. +/* Helper macros for virtual node type */ +#define node_type(n) ((n)->vt->type) +#define node_print(n, b, l) ((n)->vt->print(n, b, l)) +#define node_read(n, m) ((n)->vt->read(n, m)) +#define node_write(n, m) ((n)->vt->write(n, m)) + +/** Node type: layer, protocol, listen/connect */ +enum node_type { + IEEE_802_3, /* BSD socket: AF_PACKET SOCK_DGRAM */ + IP, /* BSD socket: AF_INET SOCK_RAW */ + UDP, /* BSD socket: AF_INET SOCK_DGRAM */ + TCPD, /* BSD socket: AF_INET SOCK_STREAM bind + listen + accept */ + TCP, /* BSD socket: AF_INET SOCK_STREAM bind + connect */ +// OPAL_ASYNC, /* OPAL-RT AsyncApi */ +// GTFPGA, /* Xilinx ML507 GTFPGA card */ + INVALID +}; + +/** C++ like vtable construct for socket_types */ +struct node_vtable { + enum node_type type; + const char *name; + + int (*parse)(config_setting_t *cfg, struct node *n); + int (*print)(struct node *n, char *buf, int len); + + int (*open)(struct node *n); + int (*close)(struct node *n); + int (*read)(struct node *n, struct msg *m); + int (*write)(struct node *n, struct msg *m); +}; + +/** The data structure for a node. * * Every entity which exchanges messages is represented by a node. + * Nodes can be remote machines and simulators or locally running processes. */ struct node { - /** The socket descriptor */ - int sd; /** How many paths are sending / receiving from this node? */ int refcnt; - /** Socket mark for netem, routing and filtering */ - int mark; - /** A short identifier of the node, only used for configuration and logging */ const char *name; - /** Local address of the socket */ - struct sockaddr_in local; - /** Remote address of the socket */ - struct sockaddr_in remote; - - /** The egress interface */ - struct interface *interface; - /** Network emulator settings */ - struct netem *netem; + /** C++ like virtual function call table */ + struct node_vtable const *vt; + /** Virtual data (used by vtable functions) */ + union { + struct socket *socket; + struct opal *opal; + struct gtfpga *gtfpga; + }; /** A pointer to the libconfig object which instantiated this node */ config_setting_t *cfg; @@ -62,21 +88,40 @@ struct node /** Connect and bind the UDP socket of this node. * - * @param n A pointer to the node structure - * @return - * - 0 on success - * - otherwise on error occured - */ -int node_connect(struct node *n); - -/** Disconnect the UDP socket of this node. + * Depending on the type (vtable) of this node, + * a socket is opened, shmem region registred... * * @param n A pointer to the node structure - * @return - * - 0 on success - * - otherwise on error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ -int node_disconnect(struct node *n); +int node_start(struct node *n); + +/** Deferred TCP connection setup + * + * @todo Dirty hack! + * We should check the address of the connecting node! + * We should preserve the original socket for proper shutdown. + */ +int node_start_defer(struct node *n); + +/** Stops a node. + * + * Depending on the type (vtable) of this node, + * a socket is closed, shmem region released... + * + * @param n A pointer to the node structure + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int node_stop(struct node *n); + +/** Lookup string representation of socket type + * + * @param type A string describing the socket type. This must be one of: tcp, tcpd, udp, ip, ieee802.3 + * @return An enumeration value or INVALID (0) + */ +struct node_vtable const * node_lookup_vtable(const char *str); /** Search list of nodes for a name. * diff --git a/server/include/opal.h b/server/include/opal.h new file mode 100644 index 000000000..7b0d74c7b --- /dev/null +++ b/server/include/opal.h @@ -0,0 +1,16 @@ +/** Node type: OPAL (AsyncApi) + * + * This file implements the opal subtype for nodes. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#ifndef _OPAL_H_ +#define _OPAL_H_ + +struct opal { + +}; + +#endif /* _OPAL_H_ */ diff --git a/server/include/path.h b/server/include/path.h index 9eb8216b7..7f71ce8f2 100644 --- a/server/include/path.h +++ b/server/include/path.h @@ -68,18 +68,16 @@ struct path * Start a new pthread for receiving/sending messages over this path. * * @param p A pointer to the path struct - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int path_start(struct path *p); /** Stop a path. * * @param p A pointer to the path struct - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int path_stop(struct path *p); diff --git a/server/include/socket.h b/server/include/socket.h new file mode 100644 index 000000000..baa5e3e0a --- /dev/null +++ b/server/include/socket.h @@ -0,0 +1,105 @@ +/** Node type: socket + * + * This file implements the socket subtype for nodes. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#ifndef _SOCKET_H_ +#define _SOCKET_H_ + +#include + +#include "node.h" + +struct socket { + /** The socket descriptor */ + int sd; + /** Socket mark for netem, routing and filtering */ + int mark; + + /** Local address of the socket */ + struct sockaddr_storage local; + /** Remote address of the socket */ + struct sockaddr_storage remote; + + /** Network emulator settings */ + struct netem *netem; + + /* Linked list _per_interface_ */ + struct socket *next; +}; + +/** Create new socket and connect(), bind(), accept(). + * + * @param n A pointer to the node. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int socket_open(struct node *n); + +/** Close the socket. + * + * @param n A pointer to the node. + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int socket_close(struct node *n); + +/** Send a message over a socket connection. + * + * @param m A pointer to the message + * @param n A pointer to the node + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int socket_write(struct node *n, struct msg *m); + +/** Receive a message over a socket connection. + * + * @param m A pointer to the message + * @param n A pointer to the node + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int socket_read(struct node *n, struct msg *m); + +/** Print details of socket connection + * + * @param n A pointer to the node structure + * @param buf A buffer to be filled. + * @param len The length of the supplied buffer. + * @return The length of the address. + */ +int socket_print(struct node *n, char *buf, int len); + +/** Generate printable socket address depending on the address family + * + * A IPv4 address is formatted as dotted decimals followed by the port/protocol number + * A link layer address is formatted in hexadecimals digits seperated by colons and the inferface name + * + * @param buf A buffer to be filled. + * @param len The length of the supplied buffer. + * @param sa A pointer to the socket address. + * @return The length of the address. + */ +int socket_print_addr(char *buf, int len, struct sockaddr *sa); + +/** Parse a socket address depending on the address family + * + * A IPv4 address has the follwing format: [hostname/ip]:[port/protocol] + * A link layer address has the following format: [mac]%[interface]:[ethertype] + * + * @todo Add support for autodetection of address type + * + * @param str A string specifiying the socket address. See description for allowed formats. + * @param sa A pointer to the resolved address + * @param type Specifies the address type in which the addr is given + * @param flags Flags for getaddrinfo(2) + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int socket_parse_addr(const char *str, struct sockaddr *sa, enum node_type type, int flags); + +#endif /* _SOCKET_H_ */ diff --git a/server/include/tc.h b/server/include/tc.h index b8134b066..a399e77d5 100644 --- a/server/include/tc.h +++ b/server/include/tc.h @@ -67,9 +67,8 @@ struct netem { /** Remove all queuing disciplines and filters. * * @param i The interface - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int tc_reset(struct interface *i); @@ -78,9 +77,8 @@ int tc_reset(struct interface *i); * @param i The interface * @param handle The handle for the new qdisc * @param bands The number of classes for this new qdisc - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int tc_prio(struct interface *i, tc_hdl_t handle, int bands); @@ -89,9 +87,8 @@ int tc_prio(struct interface *i, tc_hdl_t handle, int bands); * @param i The interface * @param parent Make this qdisc a child of * @param em The netem settings - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em); @@ -100,9 +97,8 @@ int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em); * @param i The interface * @param flowid The destination class for matched traffic * @param mark The netfilter firewall mark (sometime called 'fwmark') - * @return - * - 0 on success - * - otherwise an error occured + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. */ int tc_mark(struct interface *i, tc_hdl_t flowid, int mark); diff --git a/server/include/utils.h b/server/include/utils.h index 5f12fee73..5763086e4 100644 --- a/server/include/utils.h +++ b/server/include/utils.h @@ -11,8 +11,6 @@ #include #include #include -#include - #include #ifdef __GNUC__ @@ -22,24 +20,45 @@ #endif /* Some color escape codes for pretty log messages */ -#define RED(str) "\x1B[31m" str "\x1B[0m" /**< Print str in red */ -#define GRN(str) "\x1B[32m" str "\x1B[0m" /**< Print str in green */ -#define YEL(str) "\x1B[33m" str "\x1B[0m" /**< Print str in yellow */ -#define BLU(str) "\x1B[34m" str "\x1B[0m" /**< Print str in blue */ -#define MAG(str) "\x1B[35m" str "\x1B[0m" /**< Print str in magenta */ -#define CYN(str) "\x1B[36m" str "\x1B[0m" /**< Print str in cyan */ -#define WHT(str) "\x1B[37m" str "\x1B[0m" /**< Print str in white */ -#define BLD(str) "\x1B[1m" str "\x1B[0m" /**< Print str in bold */ +#define GRY(str) "\e[30m" str "\e[0m" /**< Print str in gray */ +#define RED(str) "\e[31m" str "\e[0m" /**< Print str in red */ +#define GRN(str) "\e[32m" str "\e[0m" /**< Print str in green */ +#define YEL(str) "\e[33m" str "\e[0m" /**< Print str in yellow */ +#define BLU(str) "\e[34m" str "\e[0m" /**< Print str in blue */ +#define MAG(str) "\e[35m" str "\e[0m" /**< Print str in magenta */ +#define CYN(str) "\e[36m" str "\e[0m" /**< Print str in cyan */ +#define WHT(str) "\e[37m" str "\e[0m" /**< Print str in white */ +#define BLD(str) "\e[1m" str "\e[0m" /**< Print str in bold */ + +#define GFX(chr) "\e(0" chr "\e(B" +#define UP(n) "\e[" ## n ## "A" +#define DOWN(n) "\e[" ## n ## "B" +#define RIGHT(n) "\e[" ## n ## "C" +#define LEFT(n) "\e[" ## n ## "D" + +#define ARRAY_LEN(a) ( sizeof a / sizeof a[0] ) /** The log level which is passed as first argument to print() */ enum log_level { DEBUG, INFO, WARN, ERROR }; /* Forward declarations */ struct settings; -struct sockaddr_in; -struct sockaddr; +struct timespec; -extern int debug; +/* These global variables allow changing the output style and verbosity */ +extern int _debug; +extern int _indent; + +void outdent(int *old); + +#ifdef __GNUC__ + #define INDENT int __attribute__ ((__cleanup__(outdent), unused)) _old_indent = _indent++; +#else + #define INDENT ; +#endif + +/** Reset the wallclock of debugging outputs */ +void epoch_reset(); /** Logs variadic messages to stdout. * @@ -48,16 +67,6 @@ extern int debug; */ void print(enum log_level lvl, const char *fmt, ...); -/** Resolve host/service name by local databases and/or nameservers. - * - * @param addr A string containing the hostname/ip and port seperated by a colon - * @param sa A pointer to the resolved address - * @param flags Flags for gai - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int resolve_addr(const char *addr, struct sockaddr_in *sa, int flags); - /** Convert integer to cpu_set_t. * * @param set A cpu bitmask @@ -77,6 +86,9 @@ void hist_plot(unsigned *hist, int length); /** Dump histogram data in Matlab format */ void hist_dump(unsigned *hist, int length); +/** A system(2) emulator with popen/pclose(2) and proper output handling */ +int system2(const char* cmd, ...); + /** Append an element to a single linked list */ #define list_add(list, elm) do { \ elm->next = list; \ @@ -93,7 +105,7 @@ void hist_dump(unsigned *hist, int length); /** Printf alike debug message with level. */ #define debug(lvl, msg, ...) do { \ - if (lvl <= debug) \ + if (lvl <= _debug) \ print(DEBUG, msg, ##__VA_ARGS__); \ } while (0) diff --git a/server/src/cfg.c b/server/src/cfg.c index aa595c1ec..de173b4a4 100644 --- a/server/src/cfg.c +++ b/server/src/cfg.c @@ -5,11 +5,8 @@ */ #include +#include #include -#include -#include -#include -#include #include "if.h" #include "tc.h" @@ -19,44 +16,53 @@ #include "utils.h" #include "hooks.h" +#include "socket.h" +#include "gtfpga.h" +#include "opal.h" + int config_parse(const char *filename, config_t *cfg, struct settings *set, struct node **nodes, struct path **paths) { config_set_auto_convert(cfg, 1); - if (!config_read_file(cfg, filename)) { + if (!config_read_file(cfg, filename)) error("Failed to parse configuration: %s in %s:%d", config_error_text(cfg), filename, config_error_line(cfg) ); - } - /* Get and check config sections */ config_setting_t *cfg_root = config_root_setting(cfg); - if (!cfg_root || !config_setting_is_group(cfg_root)) - error("Missing global section in config file: %s", filename); - - config_setting_t *cfg_nodes = config_setting_get_member(cfg_root, "nodes"); - if (!cfg_nodes || !config_setting_is_group(cfg_nodes)) - error("Missing node section in config file: %s", filename); - - config_setting_t *cfg_paths = config_setting_get_member(cfg_root, "paths"); - if (!cfg_paths || !config_setting_is_list(cfg_paths)) - error("Missing path section in config file: %s", filename); /* Parse global settings */ - config_parse_global(cfg_root, set); + if (set) { + if (!cfg_root || !config_setting_is_group(cfg_root)) + error("Missing global section in config file: %s", filename); + config_parse_global(cfg_root, set); + } + /* Parse nodes */ - for (int i = 0; i < config_setting_length(cfg_nodes); i++) { - config_setting_t *cfg_node = config_setting_get_elem(cfg_nodes, i); - config_parse_node(cfg_node, nodes); + if (nodes) { + config_setting_t *cfg_nodes = config_setting_get_member(cfg_root, "nodes"); + if (!cfg_nodes || !config_setting_is_group(cfg_nodes)) + error("Missing node section in config file: %s", filename); + + for (int i = 0; i < config_setting_length(cfg_nodes); i++) { + config_setting_t *cfg_node = config_setting_get_elem(cfg_nodes, i); + config_parse_node(cfg_node, nodes); + } } /* Parse paths */ - for (int i = 0; i < config_setting_length(cfg_paths); i++) { - config_setting_t *cfg_path = config_setting_get_elem(cfg_paths, i); - config_parse_path(cfg_path, paths, nodes); + if (paths) { + config_setting_t *cfg_paths = config_setting_get_member(cfg_root, "paths"); + if (!cfg_paths || !config_setting_is_list(cfg_paths)) + error("Missing path section in config file: %s", filename); + + for (int i = 0; i < config_setting_length(cfg_paths); i++) { + config_setting_t *cfg_path = config_setting_get_elem(cfg_paths, i); + config_parse_path(cfg_path, paths, nodes); + } } return CONFIG_TRUE; @@ -69,7 +75,7 @@ int config_parse_global(config_setting_t *cfg, struct settings *set) config_setting_lookup_int(cfg, "debug", &set->debug); config_setting_lookup_float(cfg, "stats", &set->stats); - debug = set->debug; + _debug = set->debug; set->cfg = cfg; @@ -141,8 +147,8 @@ int config_parse_path(config_setting_t *cfg, } } else { + warn("Path '%s' => '%s' is not enabled", p->in->name, p->out->name); free(p); - warn(" Path is not enabled"); } return 0; @@ -150,7 +156,7 @@ int config_parse_path(config_setting_t *cfg, int config_parse_node(config_setting_t *cfg, struct node **nodes) { - const char *remote, *local; + const char *type; int ret; /* Allocate memory */ @@ -161,37 +167,74 @@ int config_parse_node(config_setting_t *cfg, struct node **nodes) memset(n, 0, sizeof(struct node)); /* Required settings */ + n->cfg = cfg; n->name = config_setting_name(cfg); if (!n->name) cerror(cfg, "Missing node name"); + if (!config_setting_lookup_string(cfg, "type", &type)) + cerror(cfg, "Missing node type"); + + n->vt = node_lookup_vtable(type); + if (!n->vt) + cerror(cfg, "Invalid node type"); + + ret = n->vt->parse(cfg, n); + + list_add(*nodes, n); + + return ret; +} + +/** @todo Implement */ +int config_parse_opal(config_setting_t *cfg, struct node *n) +{ + return 0; +} + +/** @todo Implement */ +int config_parse_gtfpga(config_setting_t *cfg, struct node *n) +{ + return 0; +} + +int config_parse_socket(config_setting_t *cfg, struct node *n) +{ + const char *local, *remote; + int ret; + + struct socket *s = (struct socket *) malloc(sizeof(struct socket)); + if (!s) + perror("Failed to allocate memory"); + + memset(s, 0, sizeof(struct socket)); + if (!config_setting_lookup_string(cfg, "remote", &remote)) cerror(cfg, "Missing remote address for node '%s'", n->name); if (!config_setting_lookup_string(cfg, "local", &local)) cerror(cfg, "Missing local address for node '%s'", n->name); - ret = resolve_addr(local, &n->local, AI_PASSIVE); + ret = socket_parse_addr(local, (struct sockaddr *) &s->local, node_type(n), AI_PASSIVE); if (ret) cerror(cfg, "Failed to resolve local address '%s' of node '%s': %s", local, n->name, gai_strerror(ret)); - ret = resolve_addr(remote, &n->remote, 0); + ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, node_type(n), 0); if (ret) cerror(cfg, "Failed to resolve remote address '%s' of node '%s': %s", remote, n->name, gai_strerror(ret)); + /** @todo Netem settings are not usable AF_UNIX */ config_setting_t *cfg_netem = config_setting_get_member(cfg, "netem"); if (cfg_netem) { - n->netem = (struct netem *) malloc(sizeof(struct netem)); - config_parse_netem(cfg_netem, n->netem); + s->netem = (struct netem *) malloc(sizeof(struct netem)); + config_parse_netem(cfg_netem, s->netem); } + + n->socket = s; - n->cfg = cfg; - - list_add(*nodes, n); - - return 0; + return CONFIG_TRUE; } int config_parse_netem(config_setting_t *cfg, struct netem *em) @@ -211,7 +254,7 @@ int config_parse_netem(config_setting_t *cfg, struct netem *em) if (config_setting_lookup_int(cfg, "corrupt", &em->corrupt)) em->valid |= TC_NETEM_CORRUPT; - /** @todo Check netem config values */ + /** @todo Validate netem config values */ return CONFIG_TRUE; } diff --git a/server/src/gtfpga.c b/server/src/gtfpga.c new file mode 100644 index 000000000..1c911e730 --- /dev/null +++ b/server/src/gtfpga.c @@ -0,0 +1,9 @@ +/** Node type: GTFPGA (Xilinx ML507) + * + * This file implements the gtfpga subtype for nodes. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#include "gtfpga.h" diff --git a/server/src/if.c b/server/src/if.c index f9909c375..aaba33ea7 100644 --- a/server/src/if.c +++ b/server/src/if.c @@ -15,56 +15,138 @@ #include #include #include +#include #include "if.h" +#include "tc.h" +#include "socket.h" #include "utils.h" -int if_getegress(struct sockaddr_in *sa) -{ - char cmd[128]; - char token[32]; +/** Linked list of interfaces */ +struct interface *interfaces; - snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sa->sin_addr)); +struct interface * if_create(int index) { + struct interface *i = malloc(sizeof(struct interface)); + if (!i) + error("Failed to allocate memory for interface"); + else + memset(i, 0, sizeof(struct interface)); - debug(8, "System: %s", cmd); + i->index = index; + if_indextoname(index, i->name); - FILE *p = popen(cmd, "r"); - if (!p) + debug(3, "Created interface '%s'", i->name, i->index, i->refcnt); + + list_add(interfaces, i); + + return i; +} + +int if_start(struct interface *i, int affinity) +{ INDENT + if (!i->refcnt) { + warn("Interface '%s' is not used by an active node", i->name); return -1; - - while (!feof(p) && fscanf(p, "%31s", token)) { - if (!strcmp(token, "dev")) { - fscanf(p, "%31s", token); - break; + } + else + info("Starting interface '%s'", i->name); + + { INDENT + int mark = 0; + for (struct socket *s = i->sockets; s; s = s->next) { + if (s->netem) { + s->mark = 1 + mark++; + + /* Set fwmark for outgoing packets */ + if (setsockopt(s->sd, SOL_SOCKET, SO_MARK, &s->mark, sizeof(s->mark))) + perror("Failed to set fwmark for outgoing packets"); + else + debug(4, "Set fwmark for socket->sd = %u to %u", s->sd, s->mark); + + tc_mark(i, TC_HDL(4000, s->mark), s->mark); + tc_netem(i, TC_HDL(4000, s->mark), s->netem); + } } + + /* Create priority qdisc */ + if (mark) + tc_prio(i, TC_HDL(4000, 0), mark); + + /* Set affinity for network interfaces (skip _loopback_ dev) */ + if_getirqs(i); + if_setaffinity(i, affinity); } - return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token); + return 0; +} + +int if_stop(struct interface *i) +{ INDENT + info("Stopping interface '%s'", i->name); + + { INDENT + if_setaffinity(i, -1L); + tc_reset(i); + } + + return 0; +} + +int if_getegress(struct sockaddr *sa) +{ + switch (sa->sa_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *) sa; + char cmd[128]; + char token[32]; + + snprintf(cmd, sizeof(cmd), "ip route get %s", inet_ntoa(sin->sin_addr)); + + debug(8, "System: %s", cmd); + + FILE *p = popen(cmd, "r"); + if (!p) + return -1; + + while (!feof(p) && fscanf(p, "%31s", token)) { + if (!strcmp(token, "dev")) { + fscanf(p, "%31s", token); + break; + } + } + + return (WEXITSTATUS(fclose(p))) ? -1 : if_nametoindex(token); + } + + case AF_PACKET: { + struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; + return sll->sll_ifindex; + } + + default: + return -1; + } } int if_getirqs(struct interface *i) { char dirname[NAME_MAX]; + int irq, n = 0; snprintf(dirname, sizeof(dirname), "/sys/class/net/%s/device/msi_irqs/", i->name); DIR *dir = opendir(dirname); - if (!dir) { - warn("Cannot open IRQs for interface '%s'", i->name); - return -ENOENT; + if (dir) { + memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX); + + struct dirent *entry; + while ((entry = readdir(dir)) && n < IF_IRQ_MAX) { + if ((irq = atoi(entry->d_name))) + i->irqs[n++] = irq; + } + + closedir(dir); } - memset(&i->irqs, 0, sizeof(char) * IF_IRQ_MAX); - - int irq, n = 0; - struct dirent *entry; - while ((entry = readdir(dir)) && n < IF_IRQ_MAX) { - if ((irq = atoi(entry->d_name))) - i->irqs[n++] = irq; - } - - debug(7, "Found %u interrupts for interface '%s'", n, i->name); - - closedir(dir); return 0; } @@ -91,7 +173,7 @@ int if_setaffinity(struct interface *i, int affinity) return 0; } -struct interface* if_lookup_index(int index, struct interface *interfaces) +struct interface * if_lookup_index(int index) { for (struct interface *i = interfaces; i; i = i->next) { if (i->index == index) { @@ -101,3 +183,4 @@ struct interface* if_lookup_index(int index, struct interface *interfaces) return NULL; } + diff --git a/server/src/msg.c b/server/src/msg.c index bbcd259f4..ac3b3f601 100644 --- a/server/src/msg.c +++ b/server/src/msg.c @@ -76,44 +76,3 @@ void msg_random(struct msg *m) m->endian = MSG_ENDIAN_HOST; } - -int msg_send(struct msg *m, struct node *n) -{ - /* Convert headers to network byte order */ - m->sequence = htons(m->sequence); - - if (sendto(n->sd, m, MSG_LEN(m->length), 0, - (struct sockaddr *) &n->remote, - sizeof(struct sockaddr_in)) < 0) - perror("Failed sendto"); - - debug(10, "Message sent to node '%s': version=%u, type=%u, endian=%u, length=%u, sequence=%u", - n->name, m->version, m->type, m->endian, m->length, ntohs(m->sequence)); - - return 0; -} - -int msg_recv(struct msg *m, struct node *n) -{ - /** @todo Fix this for multiple paths calling msg_recv. */ - - /* Receive message from socket */ - if (recv(n->sd, m, sizeof(struct msg), 0) < 0) { - if (errno == EINTR) - return -EINTR; - - perror("Failed recv"); - } - - /* Convert headers to host byte order */ - m->sequence = ntohs(m->sequence); - - /* Convert message to host endianess */ - if (m->endian != MSG_ENDIAN_HOST) - msg_swap(m); - - debug(10, "Message received from node '%s': version=%u, type=%u, endian=%u, length=%u, sequence=%u", - n->name, m->version, m->type, m->endian, m->length, m->sequence); - - return 0; -} diff --git a/server/src/node.c b/server/src/node.c index b9b95461f..e3aa6789b 100644 --- a/server/src/node.c +++ b/server/src/node.c @@ -4,53 +4,106 @@ * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ -#include #include -#include -#include -#include "config.h" +#include "node.h" #include "cfg.h" #include "utils.h" -#include "msg.h" -#include "node.h" -#include "if.h" -int node_connect(struct node *n) -{ - /* Create socket */ - n->sd = socket(AF_INET, SOCK_DGRAM, 0); - if (n->sd < 0) - perror("Failed to create socket"); +/* Node types */ +#include "socket.h" +#include "gtfpga.h" +#include "opal.h" - /* Bind socket for receiving */ - if (bind(n->sd, (struct sockaddr *) &n->local, sizeof(struct sockaddr_in))) - perror("Failed to bind to socket"); +#define VTABLE(type, name, fnc) { type, name, config_parse_ ## fnc, \ + fnc ## _print, \ + fnc ## _open, \ + fnc ## _close, \ + fnc ## _read, \ + fnc ## _write } - debug(1, " We listen for node '%s' at %s:%u", - n->name, inet_ntoa(n->local.sin_addr), - ntohs(n->local.sin_port)); - debug(1, " We sent to node '%s' at %s:%u", - n->name, inet_ntoa(n->remote.sin_addr), - ntohs(n->remote.sin_port)); +/** Vtable for virtual node sub types */ +static const struct node_vtable vtables[] = { + VTABLE(IEEE_802_3, "ieee802.3", socket), + VTABLE(IP, "ip", socket), + VTABLE(UDP, "udp", socket), + VTABLE(TCP, "tcp", socket), + VTABLE(TCPD, "tcpd", socket), + //VTABLE(OPAL, "opal", opal ), + //VTABLE(GTFPGA, "gtfpga", gtfpga), +}; - return 0; -} +/** Linked list of nodes */ +struct node *nodes; -int node_disconnect(struct node *n) -{ - close(n->sd); - - return 0; -} - -struct node* node_lookup_name(const char *str, struct node *nodes) +struct node * node_lookup_name(const char *str, struct node *nodes) { for (struct node *n = nodes; n; n = n->next) { - if (!strcmp(str, n->name)) { + if (!strcmp(str, n->name)) return n; - } } return NULL; } + +struct node_vtable const * node_lookup_vtable(const char *str) +{ + for (int i = 0; i < ARRAY_LEN(vtables); i++) { + if (!strcmp(vtables[i].name, str)) + return &vtables[i]; + } + + return NULL; +} + +int node_start(struct node *n) +{ + int ret; + + char str[256]; + node_print(n, str, sizeof(str)); + + debug(1, "Starting node '%s' of type '%s' (%s)", n->name, n->vt->name, str); + + { INDENT + if (!n->refcnt) + warn("Node '%s' is not used by an active path", n->name); + + ret = n->vt->open(n); + } + + return ret; +} + +int node_start_defer(struct node *n) +{ + switch (node_type(n)) { + case TCPD: + info("Wait for incoming TCP connection from node '%s'...", n->name); + listen(n->socket->sd, 1); + n->socket->sd = accept(n->socket->sd, NULL, NULL); + break; + + case TCP: + info("Connect with TCP to remote node '%s'", n->name); + connect(n->socket->sd, (struct sockaddr *) &n->socket->remote, sizeof(n->socket->remote)); + break; + + default: + break; + } + + return 0; +} + +int node_stop(struct node *n) +{ + int ret; + info("Stopping node '%s'", n->name); + + { INDENT + ret = n->vt->close(n); + } + + return ret; +} diff --git a/server/src/opal.c b/server/src/opal.c new file mode 100644 index 000000000..8539723c0 --- /dev/null +++ b/server/src/opal.c @@ -0,0 +1,9 @@ +/** Node type: OPAL (AsyncApi) + * + * This file implements the opal subtype for nodes. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#include "opal.h" diff --git a/server/src/path.c b/server/src/path.c index df530be73..e5ee226eb 100644 --- a/server/src/path.c +++ b/server/src/path.c @@ -13,12 +13,14 @@ #include -#include "cfg.h" #include "utils.h" #include "path.h" #define sigev_notify_thread_id _sigev_un._tid +/** Linked list of paths */ +struct path *paths; + /** Send messages */ static void * path_send(void *arg) { @@ -50,9 +52,9 @@ static void * path_send(void *arg) perror("Failed to start timer"); while (1) { - sigwait(&set, &sig); + sigwait(&set, &sig); /* blocking wait for next timer tick */ if (p->last) { - msg_send(p->last, p->out); + node_write(p->out, p->last); p->last = NULL; p->sent++; } @@ -66,13 +68,17 @@ static void * path_run(void *arg) { struct path *p = (struct path *) arg; struct msg *m = malloc(sizeof(struct msg)); - if (!m) error("Failed to allocate memory for message!"); + + /* Open deferred TCP connection */ + node_start_defer(p->in); + node_start_defer(p->out); /* Main thread loop */ while (1) { - msg_recv(m, p->in); /* Receive message */ + node_read(p->in, m); /* Receive message */ + p->received++; /* Check header fields */ @@ -126,7 +132,7 @@ static void * path_run(void *arg) /* At fixed rate mode, messages are send by another thread */ if (!p->rate) { - msg_send(m, p->out); + node_write(p->out, m); /* Send message */ p->sent++; } } @@ -137,7 +143,9 @@ static void * path_run(void *arg) } int path_start(struct path *p) -{ +{ INDENT + info("Starting path: %12s " GRN("=>") " %-12s", p->in->name, p->out->name); + /* At fixed rate mode, we start another thread for sending */ if (p->rate) pthread_create(&p->sent_tid, NULL, &path_send, (void *) p); @@ -146,7 +154,9 @@ int path_start(struct path *p) } int path_stop(struct path *p) -{ +{ INDENT + info("Stopping path: %12s " RED("=>") " %-12s", p->in->name, p->out->name); + pthread_cancel(p->recv_tid); pthread_join(p->recv_tid, NULL); diff --git a/server/src/random.c b/server/src/random.c index 6556753ed..7e3849cd6 100644 --- a/server/src/random.c +++ b/server/src/random.c @@ -38,7 +38,8 @@ int main(int argc, char *argv[]) printf(" VALUES is the number of values a message contains\n"); printf(" RATE how many messages per second\n\n"); printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf("Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); exit(EXIT_FAILURE); } diff --git a/server/src/receive.c b/server/src/receive.c index 92a3d285e..e96c5e286 100644 --- a/server/src/receive.c +++ b/server/src/receive.c @@ -17,32 +17,37 @@ #include #include "config.h" +#include "cfg.h" #include "utils.h" #include "node.h" #include "msg.h" -int sd; +static struct settings set; +static struct msg msg = MSG_INIT(0); +extern struct node *nodes; +static struct node *node; void quit(int sig, siginfo_t *si, void *ptr) { - close(sd); + node_stop(node); exit(EXIT_SUCCESS); } int main(int argc, char *argv[]) { + struct config_t config; + if (argc != 2) { - printf("Usage: %s LOCAL\n", argv[0]); - printf(" LOCAL is a IP:PORT combination of the local host\n\n"); + printf("Usage: %s CONFIG NODE\n", argv[0]); + printf(" CONFIG path to a configuration file\n"); + printf(" NODE name of the node which shoud be used\n\n"); printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf("Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); exit(EXIT_FAILURE); } - struct node n = NODE_INIT("remote"); - struct msg m; - /* Setup signals */ struct sigaction sa_quit = { .sa_flags = SA_SIGINFO, @@ -53,22 +58,25 @@ int main(int argc, char *argv[]) sigaction(SIGTERM, &sa_quit, NULL); sigaction(SIGINT, &sa_quit, NULL); - /* Resolve addresses */ - int ret = resolve_addr(argv[1], &n.local, 0); - if (ret) - error("Failed to resolve local address '%s': %s", argv[1], gai_strerror(ret)); + config_init(&config); + config_parse(argv[1], &config, &set, &nodes, NULL); + + node = node_lookup_name(argv[2], nodes); + if (!node) + error("There's no node with the name '%s'", argv[2]); - node_connect(&n); + node_start(node); + node_start_defer(node); /* Print header */ fprintf(stderr, "# %-6s %-8s %-12s\n", "dev_id", "seq_no", "data"); while (1) { - msg_recv(&m, &n); + node_read(node, &msg); - if (m.version != MSG_VERSION) + if (msg.version != MSG_VERSION) continue; - if (m.type != MSG_TYPE_DATA) + if (msg.type != MSG_TYPE_DATA) continue; #if 1 @@ -77,7 +85,7 @@ int main(int argc, char *argv[]) fprintf(stdout, "%17.6f", ts.tv_sec + ts.tv_nsec / 1e9); #endif - msg_fprint(stdout, &m); + msg_fprint(stdout, &msg); } return 0; diff --git a/server/src/send.c b/server/src/send.c index 6d3eaf0a2..51db9cc99 100644 --- a/server/src/send.c +++ b/server/src/send.c @@ -18,33 +18,38 @@ #include #include "config.h" +#include "cfg.h" #include "utils.h" #include "node.h" #include "msg.h" +#include "socket.h" -int sd; +static struct settings set; +static struct msg msg = MSG_INIT(0); +static struct node *node; +extern struct node *nodes; void quit(int sig, siginfo_t *si, void *ptr) { - close(sd); + node_stop(node); exit(EXIT_SUCCESS); } int main(int argc, char *argv[]) { - if (argc != 2 && argc != 3) { - printf("Usage: %s REMOTE [LOCAL]\n", argv[0]); - printf(" REMOTE is a IP:PORT combination of the remote host\n"); - printf(" LOCAL is an optional IP:PORT combination of the local host\n"); + struct config_t config; + + if (argc != 3) { + printf("Usage: %s CONFIG NODE\n", argv[0]); + printf(" CONFIG path to a configuration file\n"); + printf(" NODE name of the node which shoud be used\n\n"); printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf("Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); exit(EXIT_FAILURE); } - struct node n = NODE_INIT("remote"); - struct msg m = MSG_INIT(0); - /* Setup signals */ struct sigaction sa_quit = { .sa_flags = SA_SIGINFO, @@ -55,26 +60,18 @@ int main(int argc, char *argv[]) sigaction(SIGTERM, &sa_quit, NULL); sigaction(SIGINT, &sa_quit, NULL); - /* Resolve addresses */ - int ret = resolve_addr(argv[1], &n.remote, 0); - if (ret) - error("Failed to resolve remote address '%s': %s", argv[1], gai_strerror(ret)); + config_init(&config); + config_parse(argv[1], &config, &set, &nodes, NULL); + + node = node_lookup_name(argv[2], nodes); + if (!node) + error("There's no node with the name '%s'", argv[2]); - if (argc == 3) { - ret = resolve_addr(argv[2], &n.local, AI_PASSIVE); - if (ret) - error("Failed to resolve local address '%s': %s", argv[2], gai_strerror(ret)); - } - else { - n.local.sin_family = AF_INET; - n.local.sin_addr.s_addr = INADDR_ANY; /* all local interfaces */ - n.local.sin_port = 0; /* random port */ - } - - node_connect(&n); + node_start(node); + node_start_defer(node); while (!feof(stdin)) { - msg_fscan(stdin, &m); + msg_fscan(stdin, &msg); #if 1 /* Preprend timestamp */ struct timespec ts; @@ -82,8 +79,8 @@ int main(int argc, char *argv[]) fprintf(stdout, "%17.3f\t", ts.tv_sec + ts.tv_nsec / 1e9); #endif - msg_fprint(stdout, &m); - msg_send(&m, &n); + msg_fprint(stdout, &msg); + node_write(node, &msg); } return 0; diff --git a/server/src/server.c b/server/src/server.c index 20fb0e53f..fb9d77485 100644 --- a/server/src/server.c +++ b/server/src/server.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -22,129 +23,32 @@ #include "node.h" /** Linked list of nodes */ -static struct node *nodes; +extern struct node *nodes; /** Linked list of paths */ -static struct path *paths; +extern struct path *paths; /** Linked list of interfaces */ -static struct interface *interfaces; +extern struct interface *interfaces; /** The global configuration */ -static struct settings settings; -static config_t config; - -static void start() -{ - /* Connect and bind nodes to their sockets, set socket options */ - for (struct node *n = nodes; n; n = n->next) { - if (!n->refcnt) continue; - - /* Determine outgoing interface */ - int index = if_getegress(&n->remote); - if (index < 0) - error("Failed to get egress interface for node '%s'", n->name); - - n->interface = if_lookup_index(index, interfaces); - - /* Create new interface */ - if (!n->interface) { - struct interface *i = malloc(sizeof(struct interface)); - if (!i) - error("Failed to allocate memory for interface"); - else - memset(i, 0, sizeof(struct interface)); - - i->index = index; - if_indextoname(index, i->name); - - debug(3, "Setup interface '%s'", i->name, - i->index, i->refcnt); - - /* Set affinity for network interfaces */ - if (settings.affinity && i->index) { - if_getirqs(i); - if_setaffinity(i, settings.affinity); - } - - list_add(interfaces, i); - n->interface = i; - } - - node_connect(n); - - /* Set fwmark for outgoing packets */ - if (n->netem) { - n->mark = 1 + n->interface->refcnt++; - - if (setsockopt(n->sd, SOL_SOCKET, SO_MARK, &n->mark, sizeof(n->mark))) - perror("Failed to set fwmark for outgoing packets"); - else - debug(4, "Set fwmark of outgoing packets of node '%s' to %u", - n->name, n->mark); - } - -#if 0 /* Set QoS or TOS IP options */ - int prio = SOCKET_PRIO; - if (setsockopt(n->sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio))) - perror("Failed to set socket priority"); - else - debug(4, "Set socket priority for node '%s' to %u", n->name, prio); -#else - int tos = IPTOS_LOWDELAY; - if (setsockopt(n->sd, IPPROTO_IP, IP_TOS, &tos, sizeof(tos))) - perror("Failed to set type of service (QoS)"); - else - debug(4, "Set QoS/TOS IP option for node '%s' to %#x", n->name, tos); -#endif - } - - /* Setup network emulation */ - for (struct interface *i = interfaces; i; i = i->next) { - if (!i->refcnt) continue; - - tc_prio(i, TC_HDL(4000, 0), i->refcnt); - } - - for (struct node *n = nodes; n; n = n->next) { - if (n->netem) { - tc_mark(n->interface, TC_HDL(4000, n->mark), n->mark); - tc_netem(n->interface, TC_HDL(4000, n->mark), n->netem); - } - } - - /* Start on thread per path for asynchronous processing */ - for (struct path *p = paths; p; p = p->next) { - path_start(p); - - info("Path started: %12s " GRN("=>") " %-12s", - p->in->name, p->out->name); - } -} - -static void stop() -{ - /* Join all threads and print statistics */ - for (struct path *p = paths; p; p = p->next) { - path_stop(p); - - info("Path stopped: %12s " RED("=>") " %-12s", - p->in->name, p->out->name); - } - - /* Close all sockets we listen on */ - for (struct node *n = nodes; n; n = n->next) { - node_disconnect(n); - } - - /* Reset interface queues and affinity */ - for (struct interface *i = interfaces; i; i = i->next) { - if_setaffinity(i, -1); - tc_reset(i); - } -} +struct settings settings; +config_t config; static void quit() -{ - stop(); +{ _indent = 0; + info("Stopping paths:"); + for (struct path *p = paths; p; p = p->next) { INDENT + path_stop(p); + } + + info("Stopping nodes:"); + for (struct node *n = nodes; n; n = n->next) { INDENT + node_stop(n); + } + + info("Stopping interfaces:"); + for (struct interface *i = interfaces; i; i = i->next) { INDENT + if_stop(i); + } /** @todo Free nodes and paths */ @@ -153,42 +57,8 @@ static void quit() _exit(EXIT_SUCCESS); } -int main(int argc, char *argv[]) -{ - /* Check arguments */ - if (argc != 2) { - printf("Usage: %s CONFIG\n", argv[0]); - printf(" CONFIG is a required path to a configuration file\n\n"); - printf("Simulator2Simulator Server %s (built on %s, %s)\n", - BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); - exit(EXIT_FAILURE); - } - - info("This is %s %s", BLU("s2ss"), BLU(VERSION)); - - /* Setup exit handler */ - struct sigaction sa_quit = { - .sa_flags = SA_SIGINFO, - .sa_sigaction = quit - }; - - sigemptyset(&sa_quit.sa_mask); - sigaction(SIGTERM, &sa_quit, NULL); - sigaction(SIGINT, &sa_quit, NULL); - atexit(&quit); - - /* Parse configuration file */ - config_init(&config); - config_parse(argv[1], &config, &settings, &nodes, &paths); - - debug(1, "Running with debug level: %u", settings.debug); - - /* Check priviledges */ - if (getuid() != 0) - error("The server requires superuser privileges!"); - +void realtime_init() +{ INDENT /* Check for realtime kernel patch */ struct stat st; if (stat("/sys/kernel/realtime", &st)) @@ -213,10 +83,75 @@ int main(int argc, char *argv[]) else debug(3, "Set affinity to %#x", settings.affinity); } +} + +/* Setup exit handler */ +void signals_init() +{ INDENT + struct sigaction sa_quit = { + .sa_flags = SA_SIGINFO, + .sa_sigaction = quit + }; + + sigemptyset(&sa_quit.sa_mask); + sigaction(SIGTERM, &sa_quit, NULL); + sigaction(SIGINT, &sa_quit, NULL); + atexit(&quit); +} + +void usage(const char *name) +{ + printf("Usage: %s CONFIG\n", name); + printf(" CONFIG is a required path to a configuration file\n\n"); + printf("Simulator2Simulator Server %s (built on %s, %s)\n", + BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); + + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + epoch_reset(); + info("This is Simulator2Simulator Server (S2SS) %s (built on %s, %s)", + BLD(YEL(VERSION)), BLD(MAG(__DATE__)), BLD(MAG(__TIME__))); + + /* Check arguments */ + if (argc != 2) + usage(argv[0]); + + /* Check priviledges */ + if (getuid() != 0) + error("The server requires superuser privileges!"); + + /* Start initialization */ + info("Initialize realtime system:"); + realtime_init(); + info("Setup signals:"); + signals_init(); + info("Parsing configuration:"); + config_init(&config); + + /* Parse configuration and create nodes/paths */ + config_parse(argv[1], &config, &settings, &nodes, &paths); + /* Connect all nodes and start one thread per path */ - start(); + info("Starting nodes:"); + for (struct node *n = nodes; n; n = n->next) { INDENT + node_start(n); + } + info("Starting interfaces:"); + for (struct interface *i = interfaces; i; i = i->next) { INDENT + if_start(i, settings.affinity); + } + + info("Starting pathes:"); + for (struct path *p = paths; p; p = p->next) { INDENT + path_start(p); + } + + /* Run! */ if (settings.stats > 0) { struct path *p = paths; @@ -235,5 +170,7 @@ int main(int argc, char *argv[]) else pause(); + /* Note: quit() is called by exit handler! */ + return 0; } diff --git a/server/src/socket.c b/server/src/socket.c new file mode 100644 index 000000000..96dc7acab --- /dev/null +++ b/server/src/socket.c @@ -0,0 +1,255 @@ +/** Various socket related functions + * + * Parse and print addresses, connect, close, etc... + * + * S2SS uses these functions to setup the network emulation feature. + * + * @author Steffen Vogel + * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "utils.h" +#include "socket.h" +#include "if.h" + +int socket_print(struct node *n, char *buf, int len) +{ + struct socket *s = n->socket; + + char local[INET6_ADDRSTRLEN + 16]; + char remote[INET6_ADDRSTRLEN + 16]; + + socket_print_addr(local, sizeof(local), (struct sockaddr*) &s->local); + socket_print_addr(remote, sizeof(remote), (struct sockaddr*) &s->remote); + + return snprintf(buf, len, "local=%s, remote=%s", local, remote); +} + +int socket_open(struct node *n) +{ + struct socket *s = n->socket; + int af = s->local.ss_family; + + /* Create socket */ + switch (node_type(n)) { + case TCPD: + case TCP: s->sd = socket(af, SOCK_STREAM, 0); break; + case UDP: s->sd = socket(af, SOCK_DGRAM, 0); break; + case IP: s->sd = socket(af, SOCK_RAW, IPPROTO_S2SS); break; + case IEEE_802_3:s->sd = socket(af, SOCK_DGRAM, ETH_P_S2SS); break; + default: + error("Invalid socket type!"); + } + + if (s->sd < 0) + perror("Failed to create socket"); + + /* Bind socket for receiving */ + if (bind(s->sd, (struct sockaddr *) &s->local, sizeof(s->local))) + perror("Failed to bind to socket"); + + /* Determine outgoing interface */ + int index = if_getegress((struct sockaddr *) &s->remote); + if (index < 0) + error("Failed to get egress interface for node '%s'", n->name); + + struct interface *i = if_lookup_index(index); + if (!i) + i = if_create(index); + + list_add(i->sockets, s); + i->refcnt++; + + /* Set socket priority, QoS or TOS IP options */ + int prio; + switch (node_type(n)) { + case TCPD: + case TCP: + case UDP: + case IP: + prio = IPTOS_LOWDELAY; + if (setsockopt(s->sd, IPPROTO_IP, IP_TOS, &prio, sizeof(prio))) + perror("Failed to set type of service (QoS)"); + else + debug(4, "Set QoS/TOS IP option for node '%s' to %#x", n->name, prio); + break; + + default: + prio = SOCKET_PRIO; + if (setsockopt(s->sd, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(prio))) + perror("Failed to set socket priority"); + else + debug(4, "Set socket priority for node '%s' to %u", n->name, prio); + break; + } + + return 0; +} + +int socket_close(struct node *n) +{ + return close(n->socket->sd); +} + +int socket_read(struct node* n, struct msg *m) +{ + /** @todo Fix this for multiple paths calling msg_recv. */ + + /* Receive message from socket */ + if (recv(n->socket->sd, m, sizeof(struct msg), 0) < 0) { + if (errno == EINTR) + return -EINTR; + + perror("Failed recv"); + } + + /* Convert headers to host byte order */ + m->sequence = ntohs(m->sequence); + + /* Convert message to host endianess */ + if (m->endian != MSG_ENDIAN_HOST) + msg_swap(m); + + debug(10, "Message received from node '%s': version=%u, type=%u, endian=%u, length=%u, sequence=%u", + n->name, m->version, m->type, m->endian, m->length, m->sequence); + + return 0; +} + +int socket_write(struct node* n, struct msg *m) +{ + /* Convert headers to network byte order */ + m->sequence = htons(m->sequence); + + if (sendto(n->socket->sd, m, MSG_LEN(m->length), 0, + (struct sockaddr *) &n->socket->remote, + sizeof(struct sockaddr_in)) < 0) + perror("Failed sendto"); + + debug(10, "Message sent to node '%s': version=%u, type=%u, endian=%u, length=%u, sequence=%u", + n->name, m->version, m->type, m->endian, m->length, ntohs(m->sequence)); + + return 0; +} + +int socket_print_addr(char *buf, int len, struct sockaddr *sa) +{ + switch (sa->sa_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *) sa; + inet_ntop(sin->sin_family, &sin->sin_addr, buf, len); + return snprintf(buf+strlen(buf), len-strlen(buf), ":%hu", ntohs(sin->sin_port)); + } + + case AF_PACKET: { + struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; + char ifname[IF_NAMESIZE]; + + return snprintf(buf, len, "%s%%%s:%hu", + ether_ntoa((struct ether_addr *) &sll->sll_addr), + if_indextoname(sll->sll_ifindex, ifname), + ntohs(sll->sll_protocol)); + } + + default: + error("Unsupported address family"); + } + + return 0; +} + +int socket_parse_addr(const char *addr, struct sockaddr *sa, enum node_type type, int flags) +{ + /** @todo: Add support for IPv6 */ + + char *copy = strdup(addr); + int ret; + + if (type == IEEE_802_3) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */ + struct sockaddr_ll *sll = (struct sockaddr_ll *) sa; + + /* Split string */ + char *node = strtok(copy, "%"); + char *ifname = strtok(NULL, "\0"); + + /* Parse link layer (MAC) address */ + struct ether_addr *mac = ether_aton(node); + if (!mac) + error("Failed to parse mac address: %s", node); + + memcpy(&sll->sll_addr, &mac->ether_addr_octet, 6); + + sll->sll_protocol = ETH_P_S2SS; + sll->sll_halen = 6; + sll->sll_family = AF_PACKET; + sll->sll_ifindex = if_nametoindex(ifname); + + ret = 0; + } + else { + //struct sockaddr_in *sin = (struct sockaddr_in *) sa; + struct addrinfo hint = { + .ai_flags = flags, + .ai_family = AF_UNSPEC + }; + + /* Split string */ + char *node = strtok(copy, ":"); + char *service = strtok(NULL, "\0"); + + if (node && !strcmp(node, "*")) + node = NULL; + + if (service && !strcmp(service, "*")) + service = NULL; + + switch (type) { + case IP: + hint.ai_socktype = 0; + hint.ai_protocol = IPPROTO_S2SS; + break; + + case TCPD: + case TCP: + hint.ai_socktype = SOCK_STREAM; + hint.ai_protocol = IPPROTO_TCP; + break; + + case UDP: + hint.ai_socktype = SOCK_DGRAM; + hint.ai_protocol = IPPROTO_UDP; + break; + + case INVALID: + default: + error("Invalid address type"); + } + + /* Lookup address */ + struct addrinfo *result; + ret = getaddrinfo(node, service, &hint, &result); + if (!ret) { + memcpy(sa, result->ai_addr, result->ai_addrlen); + freeaddrinfo(result); + } + } + + free(copy); + + return ret; +} diff --git a/server/src/tc.c b/server/src/tc.c index 2804f74f5..bfda86339 100644 --- a/server/src/tc.c +++ b/server/src/tc.c @@ -20,9 +20,8 @@ int tc_reset(struct interface *i) snprintf(cmd, sizeof(cmd), "tc qdisc del dev %s root", i->name); debug(6, "Reset traffic control for interface '%s'", i->name); - debug(8, "System: %s", cmd); - return system(cmd); + return system2(cmd); } int tc_prio(struct interface *i, tc_hdl_t handle, int bands) @@ -39,9 +38,8 @@ int tc_prio(struct interface *i, tc_hdl_t handle, int bands) len += snprintf(cmd+len, sizeof(cmd)-len, " %u", priomap[i] + bands); debug(6, "Replace master qdisc for interface '%s'", i->name); - debug(8, "System: %s", cmd); - return system(cmd); + return system2(cmd); } int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em) @@ -69,9 +67,8 @@ int tc_netem(struct interface *i, tc_hdl_t parent, struct netem *em) len += snprintf(cmd+len, sizeof(cmd)-len, " corrupt %u", em->corrupt); debug(6, "Setup netem qdisc for interface '%s'", i->name); - debug(8, "System: %s", cmd); - return system(cmd); + return system2(cmd); } int tc_mark(struct interface *i, tc_hdl_t flowid, int mark) @@ -83,8 +80,7 @@ int tc_mark(struct interface *i, tc_hdl_t flowid, int mark) debug(7, "Add traffic filter to interface '%s': fwmark %u => flowid %u:%u", i->name, mark, TC_HDL_MAJ(flowid), TC_HDL_MIN(flowid)); - debug(8, "System: %s", cmd); - return system(cmd); + return system2(cmd); } diff --git a/server/src/test.c b/server/src/test.c index eafb9dc2b..9fa826c89 100644 --- a/server/src/test.c +++ b/server/src/test.c @@ -15,11 +15,15 @@ #include #include "config.h" +#include "cfg.h" #include "msg.h" #include "node.h" #include "utils.h" -int sd; +static struct settings set; +static struct node *node; +extern struct node *nodes; + int running = 1; #define CLOCK_ID CLOCK_MONOTONIC_RAW @@ -36,19 +40,19 @@ void quit(int sig, siginfo_t *si, void *ptr) int main(int argc, char *argv[]) { + config_t config; + if (argc != 4) { - printf("Usage: %s TEST LOCAL REMOTE\n", argv[0]); - printf(" TEST has to be 'rtt' for now\n"); - printf(" LOCAL is a IP:PORT combination of the local host\n"); - printf(" REMOTE is a IP:PORT combination of the remote host\n\n"); + printf("Usage: %s CONFIG NODE\n", argv[0]); + printf(" CONFIG path to a configuration file\n"); + printf(" NODE name of the node which shoud be used\n\n"); printf("Simulator2Simulator Server %s (built on %s %s)\n", BLU(VERSION), MAG(__DATE__), MAG(__TIME__)); - printf("Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Copyright 2014, Institute for Automation of Complex Power Systems, EONERC\n"); + printf(" Steffen Vogel \n"); exit(EXIT_FAILURE); } - struct node n = NODE_INIT("remote"); - /* Setup signals */ struct sigaction sa_quit = { .sa_flags = SA_SIGINFO, @@ -59,16 +63,15 @@ int main(int argc, char *argv[]) sigaction(SIGTERM, &sa_quit, NULL); sigaction(SIGINT, &sa_quit, NULL); - /* Resolve addresses */ - int ret = resolve_addr(argv[2], &n.local, AI_PASSIVE); - if (ret) - error("Failed to resolve local address '%s': %s", argv[1], gai_strerror(ret)); + config_init(&config); + config_parse(argv[1], &config, &set, &nodes, NULL); + + node = node_lookup_name(argv[2], nodes); + if (!node) + error("There's no node with the name '%s'", argv[2]); - ret = resolve_addr(argv[3], &n.remote, 0); - if (ret) - error("Failed to resolve remote address '%s': %s", argv[1], gai_strerror(ret)); - - node_connect(&n); + node_start(node); + node_start_defer(node); if (!strcmp(argv[1], "rtt")) { struct msg m = MSG_INIT(sizeof(struct timespec) / sizeof(float)); @@ -91,10 +94,9 @@ int main(int argc, char *argv[]) fprintf(stdout, "%5s%10s%10s%10s%10s\n", "seq", "rtt", "min", "max", "avg"); while (running) { - clock_gettime(CLOCK_ID, ts1); - msg_send(&m, &n); - msg_recv(&m, &n); + node_write(node, &m); + node_read(node, &m); clock_gettime(CLOCK_ID, ts2); rtt = timespec_delta(ts1, ts2); @@ -127,7 +129,7 @@ int main(int argc, char *argv[]) hist_dump(hist, RTT_HIST); } - close(sd); + node_stop(node); return 0; } diff --git a/server/src/utils.c b/server/src/utils.c index 476506746..0bbf3ff25 100644 --- a/server/src/utils.c +++ b/server/src/utils.c @@ -4,6 +4,7 @@ * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC */ +#include #include #include #include @@ -14,15 +15,25 @@ #include #include -#include -#include - #include "config.h" #include "cfg.h" #include "utils.h" /* This global variable contains the debug level for debug() and assert() macros */ -int debug = V; +int _debug = V; +int _indent = 0; + +struct timespec epoch; + +void outdent(int *old) +{ + _indent = *old; +} + +void epoch_reset() +{ + clock_gettime(CLOCK_REALTIME, &epoch); +} void print(enum log_level lvl, const char *fmt, ...) { @@ -31,57 +42,30 @@ void print(enum log_level lvl, const char *fmt, ...) va_list ap; va_start(ap, fmt); - clock_gettime(CLOCK_REALTIME, &ts); - /* Timestamp */ - printf("%15.4f", ts.tv_sec + ts.tv_nsec / 1e9); + clock_gettime(CLOCK_REALTIME, &ts); + fprintf(stderr, "%8.3f ", timespec_delta(&epoch, &ts)); switch (lvl) { - case DEBUG: printf(" [" BLU("Debug") "] "); break; - case INFO: printf(" [" WHT("Info ") "] "); break; - case WARN: printf(" [" YEL("Warn ") "] "); break; - case ERROR: printf(" [" RED("Error") "] "); break; + case DEBUG: fprintf(stderr, BLD("%-5s "), GRY("Debug")); break; + case INFO: fprintf(stderr, BLD("%-5s "), " " ); break; + case WARN: fprintf(stderr, BLD("%-5s "), YEL(" Warn")); break; + case ERROR: fprintf(stderr, BLD("%-5s "), RED("Error")); break; } - vprintf(fmt, ap); - printf("\n"); + if (_indent) { + for (int i = 0; i < _indent-1; i++) + fprintf(stderr, GFX("\x78") " "); + + fprintf(stderr, GFX("\x74") " "); + } + + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); va_end(ap); } -int resolve_addr(const char *addr, struct sockaddr_in *sa, int flags) -{ - /* Split string */ - char *tmp = strdup(addr); - char *node = strtok(tmp, ":"); - char *service = strtok(NULL, ":"); - - if (node && !strcmp(node, "*")) - node = NULL; - - if (service && !strcmp(service, "*")) - service = NULL; - - /* Get IP */ - struct addrinfo *result; - struct addrinfo hint = { - .ai_flags = flags, - .ai_family = AF_INET, - .ai_socktype = SOCK_DGRAM, - .ai_protocol = 0 - }; - - int ret = getaddrinfo(node, service, &hint, &result); - if (!ret) { - memcpy(sa, result->ai_addr, result->ai_addrlen); - freeaddrinfo(result); - } - - free(tmp); - - return ret; -} - cpu_set_t to_cpu_set(int set) { cpu_set_t cset; @@ -131,7 +115,6 @@ void hist_plot(unsigned *hist, int length) max = i; } - /* Print header */ info("%2s | %5s | %s", "Id", "Value", "Histogram Plot:"); @@ -162,3 +145,30 @@ void hist_dump(unsigned *hist, int length) info("Matlab: hist = [ %s]", buf); } + +/** @todo: Proper way: create additional pipe for stderr in child process */ +int system2(const char *cmd, ...) +{ + char buf[1024]; + + va_list ap; + va_start(ap, cmd); + + vsnprintf(buf, sizeof(buf), cmd, ap); + strncat(buf, " 2>&1", sizeof(buf)); + + va_end(ap); + + debug(1, "System: %s", buf); + + FILE *f = popen(buf, "r"); + if (f == NULL) + perror("Failed to execute: '%s'", cmd); + + while (!feof(f) && fgets(buf, sizeof(buf), f) != NULL) { INDENT + strtok(buf, "\n"); /* strip trailing newline */ + info(buf); + } + + return pclose(f); +}