From 207d904054f8a59df2839cf260868ad574bac187 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 5 Jun 2014 09:35:23 +0000 Subject: [PATCH] added realtime initialization git-svn-id: https://zerberus.eonerc.rwth-aachen.de:8443/svn/s2ss/trunk@48 8ec27952-4edc-4aab-86aa-e87bb2611832 --- Makefile | 2 +- Readme | 1 + etc/server.conf | 7 ++----- include/cfg.h | 6 +++--- include/utils.h | 15 ++++++++++++++- src/cfg.c | 6 +++--- src/rt.c | 50 ------------------------------------------------- src/server.c | 7 +++++-- src/utils.c | 45 +++++++++++++++++++++++++++++++++++++++++++- 9 files changed, 73 insertions(+), 66 deletions(-) delete mode 100644 src/rt.c diff --git a/Makefile b/Makefile index 16c081b29..916c622f8 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ GIT_REV = $(shell git rev-parse --short HEAD) V ?= 4 LDFLAGS = -pthread -lrt -lm -lconfig -lnl-3 -lnl-route-3 -CFLAGS = -g -std=c99 -Iinclude/ -D_XOPEN_SOURCE=500 -DV=$(V) +CFLAGS = -g -std=c99 -Iinclude/ -D_XOPEN_SOURCE=500 -D_GNU_SOURCE -DV=$(V) CFLAGS += -D__GIT_REV__='"$(GIT_REV)"' -D__GIT_TAG__='"$(GIT_TAG)"' .PHONY: all clean doc diff --git a/Readme b/Readme index cff18a32e..b1477f691 100644 --- a/Readme +++ b/Readme @@ -27,6 +27,7 @@ Contacts: - CAP_NET_ADMIN to increase the socket priority - CAP_SYS_NICE to set the realtime priority and cpu affinity + - CAP_IPC_LOC to lock pages for better realtime 3. Examples -------------------------------------------------------------------------------- diff --git a/etc/server.conf b/etc/server.conf index 7ad68fabf..08c905688 100644 --- a/etc/server.conf +++ b/etc/server.conf @@ -2,11 +2,8 @@ name = "s2ss"; # The name of this node -# TODO: this has to be implmented -# debug = 10; # The debugging level (0 - 10) -# affinity = 0xC0; # Mask of cores the server should run on -# nice = -20; # Scheduler priority for the server -# protocol = 0; # Protocol version for UDP message format +affinity = 0x02; # Mask of cores the server should run on +priority = 50; # Scheduler priority for the server nodes = { acs = { diff --git a/include/cfg.h b/include/cfg.h index 6cba8d129..15b02d6a8 100644 --- a/include/cfg.h +++ b/include/cfg.h @@ -10,8 +10,8 @@ #include -#include "path.h" -#include "node.h" +struct node; +struct path; struct config { /// Name of this node @@ -21,7 +21,7 @@ struct config { /// Verbosity level int debug; /// Task priority (lower is better) - int nice; + int priority; /// Core affinity of this task int affinity; /// Protocol version of UDP packages diff --git a/include/utils.h b/include/utils.h index 1edf5e0bb..7c2fd5a67 100644 --- a/include/utils.h +++ b/include/utils.h @@ -13,7 +13,8 @@ #include #include -#include +struct config; +struct sockaddr_in; enum log_level { @@ -41,6 +42,18 @@ void print(enum log_level lvl, const char *fmt, ...); */ int resolve(const char *addr, struct sockaddr_in *sa, int flags); +/** + * @brief Setup various realtime realated things + * + * - Prefault the stack + * - Lock memory + * - Use FIFO scheduler with realtime priority + * - Set CPU affinity + * + * @param g The global configuration + */ +void realtime_init(struct config *g); + /// Check assertion and exit if failed #define assert(exp) do { \ if (!(exp)) { \ diff --git a/src/cfg.c b/src/cfg.c index 06dec81e6..fde624c0a 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -11,6 +11,8 @@ #include #include "cfg.h" +#include "node.h" +#include "path.h" #include "utils.h" int config_parse(config_t *cfg, struct config *g) @@ -71,7 +73,7 @@ int config_parse_global(config_setting_t *c, struct config *g) config_setting_lookup_int(c, "debug", &g->debug); config_setting_lookup_int(c, "affinity", &g->affinity); - config_setting_lookup_int(c, "nice", &g->nice); + config_setting_lookup_int(c, "priority", &g->priority); config_setting_lookup_int(c, "protocol", &g->protocol); return CONFIG_TRUE; @@ -96,8 +98,6 @@ int config_parse_path(config_setting_t *c, struct config *g) if (!config_setting_lookup_string(c, "out", &out_str)) cerror(c, "Missing output node for path"); - if (!config_setting_lookup_int) - info("Loading path from '%s' to '%s'", in_str, out_str); in = node_lookup_name(in_str, g->nodes, g->node_count); diff --git a/src/rt.c b/src/rt.c deleted file mode 100644 index 917219ade..000000000 --- a/src/rt.c +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Realtime related functions - * - * @author Steffen Vogel - * @copyright 2014, Institute for Automation of Complex Power Systems, EONERC - */ - -#include -#include - -#include "config.h" -#include "utils.h" - -void rt_stack_prefault() -{ - unsigned char dummy[MAX_SAFE_STACK]; - - memset(dummy, 0, MAX_SAFE_STACK); - - -} - -void rt_set_priority() -{ - struct sched_param param; - - param.sched_priority = PRIORITY; - if (sched_setscheduler(0, SCHED_FIFO, ¶m) == -1) { - perror("sched_setscheduler failed"); - exit(-1); - } -} - -void rt_lock_memory() -{ - // TODO -} - -void rt_set_affinity() -{ - // TODO -} - -void rt_init() -{ - rt_set_priority(); - rt_set_affinity(); - rt_lock_memory(); - rt_stack_prefault(); -} diff --git a/src/server.c b/src/server.c index 3368945d0..dcbaa3184 100644 --- a/src/server.c +++ b/src/server.c @@ -90,16 +90,19 @@ int main(int argc, char *argv[]) info("This is s2ss %s", VERSION); - // Default settings + /* Default settings */ config.filename = argv[1]; config.debug = 0; - config.nice = 0; + config.priority = 0; config.affinity = 0xC0; config.protocol = 0; config_init(&config.obj); config_parse(&config.obj, &config); + /* Setup various realtime related things */ + realtime_init(&config); + if (config.path_count) info("Parsed %u nodes and %u paths", config.node_count, config.path_count); else diff --git a/src/utils.c b/src/utils.c index 0f8b86148..ba5ec8974 100644 --- a/src/utils.c +++ b/src/utils.c @@ -10,12 +10,19 @@ #include #include #include - +#include +#include #include +#include + #include #include +#include +#include +#include #include "config.h" +#include "cfg.h" #include "utils.h" static const char *log_prefix[] = { @@ -82,3 +89,39 @@ int resolve(const char *addr, struct sockaddr_in *sa, int flags) return 0; } + +void realtime_init(struct config *g) +{ + /* Prefault stack */ + char dummy[MAX_SAFE_STACK]; + memset(dummy, 0, MAX_SAFE_STACK); + + /* Lock memory */ + if(mlockall(MCL_CURRENT | MCL_FUTURE)) + perror("Failed mlockall"); + else + debug(3, "Locked memory"); + + /* Check for realtime kernel patch */ + struct stat st; + if (stat("/sys/kernel/realtime", &st)) + warn("This is not a a realtime patched kernel"); + else + debug(3 ,"This is a realtime patched kernel"); + + /* Use FIFO scheduler with realtime priority */ + struct sched_param param; + param.sched_priority = g->priority; + if (sched_setscheduler(0, SCHED_FIFO, ¶m)) + perror("Failed to set realtime priority"); + else + debug(3, "Set task priority to %u", g->priority); + + /* Pin threads to CPUs by setting the affinity */ + size_t cset = g->affinity; + pid_t pid = getpid(); + if (sched_setaffinity(pid, sizeof(cset), (const struct cpu_set_t *) &cset)) + perror("Failed to set CPU affinity"); + else + debug(3, "Set affinity to %#x", g->affinity); +}