From 132dd5fd6524e332ba2a759b182b1fae336c8bb6 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 26 Jun 2016 15:27:14 +0200 Subject: [PATCH] moved linux realtime code to extra file --- Makefile | 2 +- include/villas/kernel/rt.h | 15 ++++++++ lib/kernel/rt.c | 78 ++++++++++++++++++++++++++++++++++++++ src/fpga-main.c | 4 ++ src/server.c | 41 ++------------------ 5 files changed, 101 insertions(+), 39 deletions(-) create mode 100644 include/villas/kernel/rt.h create mode 100644 lib/kernel/rt.c diff --git a/Makefile b/Makefile index 890a24575..2f6e27731 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DEBUG = 1 # Object files for libvillas LIB_OBJS = sample.o path.o node.o \ - kernel.o \ + kernel.o rt.o \ list.o pool.o queue.o lstack.o \ log.o \ utils.o \ diff --git a/include/villas/kernel/rt.h b/include/villas/kernel/rt.h new file mode 100644 index 000000000..0c0f833cd --- /dev/null +++ b/include/villas/kernel/rt.h @@ -0,0 +1,15 @@ +/** Linux specific real-time optimizations + * + * @file + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#ifndef _RT_H_ +#define _RT_H_ + +int rt_init(int affinity, int priority); + +#endif /* _RT_H_ */ \ No newline at end of file diff --git a/lib/kernel/rt.c b/lib/kernel/rt.c new file mode 100644 index 000000000..e4b80757a --- /dev/null +++ b/lib/kernel/rt.c @@ -0,0 +1,78 @@ +/** Linux specific real-time optimizations + * + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#include + +#include "utils.h" +#include "kernel/kernel.h" +#include "kernel/rt.h" + +int rt_init(int affinity, int priority) +{ INDENT + char isolcpus[255]; + int is_isol, is_rt, ret; + + /* Pin interrupts to our core */ +/* for (int i = 0; i < fpga->vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) { + ret = kernel_irq_setaffinity(fpga->vd.msi_irqs[i], AFFINITY, NULL); + if (ret) + serror("Failed to change affinity of VFIO-MSI interrupt"); + } +*/ + + /* Use FIFO scheduler with real time priority */ + is_rt = kernel_is_rt(); + if (is_rt) + warn("We recommend to use an PREEMPT_RT patched kernel!"); + + struct sched_param param = { + .sched_priority = priority + }; + + if (sched_setscheduler(0, SCHED_FIFO, ¶m)) + serror("Failed to set real time priority"); + + debug(3, "Task priority set to %u", priority); + + /* Pin threads to CPUs by setting the affinity */ + cpu_set_t cset_pin, cset_isol, cset_non_isol; + + is_isol = kernel_get_cmdline_param("isolcpus", isolcpus, sizeof(isolcpus)); + if (is_isol) { + warn("You should reserve some cores for the server (see 'isolcpus')"); + + CPU_ZERO(&cset_isol); + } + else { + ret = cpulist_parse(isolcpus, &cset_isol, 0); + if (ret) + error("Invalid isolcpus cmdline parameter: %s", isolcpus); + } + + cpuset_from_integer(affinity, &cset_pin); + + CPU_XOR(&cset_non_isol, &cset_isol, &cset_pin); + if (CPU_COUNT(&cset_non_isol) > 0) { + char isol[128], pin[128]; + + cpulist_create(isol, sizeof(isol), &cset_isol); + cpulist_create(pin, sizeof(pin), &cset_pin); + + warn("Affinity setting includes cores which are not isolated: affinity=%s isolcpus=%s", pin, isol); + } + + char list[128]; + cpulist_create(list, sizeof(list), &cset_pin); + + if (sched_setaffinity(0, sizeof(cpu_set_t), &cset_pin)) + serror("Failed to set CPU affinity to %s", list); + + debug(3, "Set affinity to %s", list); + + return 0; +} \ No newline at end of file diff --git a/src/fpga-main.c b/src/fpga-main.c index 7891cb7fd..c7f4277a7 100644 --- a/src/fpga-main.c +++ b/src/fpga-main.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,9 @@ int main(int argc, char *argv[]) } } + info("Initialize real-time system"); + rt_init(settings.affinity, settings.priority); + /* Initialize VILLASfpga card */ config_setting_t *cfg_root = config_root_setting(&config); ret = fpga_init(argc, argv, cfg_root); diff --git a/src/server.c b/src/server.c index 864d777d8..6197e1e99 100644 --- a/src/server.c +++ b/src/server.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifdef ENABLE_OPAL_ASYNC #include "opal.h" @@ -27,8 +28,8 @@ struct list paths; /**< List of paths */ struct list nodes; /**< List of nodes */ -struct settings settings; /**< The global configuration */ static config_t config; /**< libconfig handle */ +struct settings settings; /**< The global configuration */ static void quit() { @@ -57,42 +58,6 @@ static void quit() _exit(EXIT_SUCCESS); } -static void realtime_init() -{ INDENT - int ret; - - ret = kernel_has_cmdline("isolcpus"); - if (ret) - warn("You should reserve some cores for the server (see 'isolcpus')"); - - ret = kernel_is_rt(); - if (ret) - warn("We recommend to use an PREEMPT_RT patched kernel!"); - - /* Use FIFO scheduler with real time priority */ - if (settings.priority) { - struct sched_param param = { - .sched_priority = settings.priority - }; - - if (sched_setscheduler(0, SCHED_FIFO, ¶m)) - serror("Failed to set real time priority"); - - debug(3, "Set task priority to %u", settings.priority); - } - warn("Use setting 'priority' to enable real-time scheduling!"); - - /* Pin threads to CPUs by setting the affinity */ - if (settings.affinity) { - cpu_set_t cset = integer_to_cpuset(settings.affinity); - if (sched_setaffinity(0, sizeof(cset), &cset)) - serror("Failed to set CPU affinity to '%#x'", settings.affinity); - - debug(3, "Set affinity to %#x", settings.affinity); - } - warn("Use setting 'affinity' to pin process to isolated CPU cores!"); -} - /* Setup exit handler */ static void signals_init() { INDENT @@ -161,7 +126,7 @@ int main(int argc, char *argv[]) } info("Initialize real-time system"); - realtime_init(); + rt_init(settings.affinity, settings.priority); info("Initialize signals"); signals_init();