diff --git a/common/include/villas/kernel/rt.hpp b/common/include/villas/kernel/rt.hpp index 958dcf71c..92e07ba93 100644 --- a/common/include/villas/kernel/rt.hpp +++ b/common/include/villas/kernel/rt.hpp @@ -28,13 +28,16 @@ #pragma once +#include + namespace villas { namespace kernel { namespace rt { void init(int priority, int affinity); -void setAffinity(int affinity); +void setProcessAffinity(int affinity); +void setThreadAffinity(pthread_t thread, int affinity); void setPriority(int priority); diff --git a/common/lib/kernel/rt.cpp b/common/lib/kernel/rt.cpp index 4e0a6737b..13fe2fa25 100644 --- a/common/lib/kernel/rt.cpp +++ b/common/lib/kernel/rt.cpp @@ -48,7 +48,8 @@ void init(int priority, int affinity) logger->info("Initialize sub-system"); #ifdef __linux__ - int is_rt; + int is_rt, is_isol; + char isolcpus[255]; /* Use FIFO scheduler with real time priority */ is_rt = isPreemptible(); @@ -60,8 +61,21 @@ void init(int priority, int affinity) else logger->warn("You might want to use the 'priority' setting to increase " PROJECT_NAME "'s process priority"); - if (affinity) - setAffinity(affinity); + if (affinity) { + is_isol = get_cmdline_param("isolcpus", isolcpus, sizeof(isolcpus)); + if (is_isol) + logger->warn("You should reserve some cores for " PROJECT_NAME " (see 'isolcpus')"); + else { + CpuSet cset_pin(affinity); + CpuSet cset_isol(isolcpus); + CpuSet cset_non_isol = ~cset_isol & cset_pin; + + if (cset_non_isol.count() > 0) + logger->warn("Affinity setting includes cores which are not isolated: affinity={}, isolcpus={}, non_isolated={}", (std::string) cset_pin, (std::string) cset_isol, (std::string) cset_non_isol); + } + + setProcessAffinity(affinity); + } else logger->warn("You might want to use the 'affinity' setting to pin " PROJECT_NAME " to dedicate CPU cores"); #else @@ -73,27 +87,15 @@ void init(int priority, int affinity) } #ifdef __linux__ -void setAffinity(int affinity) +void setProcessAffinity(int affinity) { - char isolcpus[255]; - int is_isol, ret; + int ret; Logger logger = logging.get("kernel:rt"); /* Pin threads to CPUs by setting the affinity */ CpuSet cset_pin(affinity); - is_isol = get_cmdline_param("isolcpus", isolcpus, sizeof(isolcpus)); - if (is_isol) - logger->warn("You should reserve some cores for " PROJECT_NAME " (see 'isolcpus')"); - else { - CpuSet cset_isol(isolcpus); - CpuSet cset_non_isol = ~cset_isol & cset_pin; - - if (cset_non_isol.count() > 0) - logger->warn("Affinity setting includes cores which are not isolated: affinity={}, isolcpus={}, non_isolated={}", (std::string) cset_pin, (std::string) cset_isol, (std::string) cset_non_isol); - } - ret = sched_setaffinity(0, cset_pin.size(), cset_pin); if (ret) throw SystemError("Failed to set CPU affinity to cores: {}", (std::string) cset_pin); @@ -101,6 +103,21 @@ void setAffinity(int affinity) logger->debug("Set affinity to cores: {}", (std::string) cset_pin); } +void setThreadAffinity(pthread_t thread, int affinity) +{ + int ret; + + Logger logger = logging.get("kernel:rt"); + + CpuSet cset_pin(affinity); + + ret = pthread_setaffinity_np(thread, cset_pin.size(), cset_pin); + if (ret) + throw SystemError("Failed to set CPU affinity to cores: {}", (std::string) cset_pin); + + logger->debug("Set affinity of thread {} to cores: {}", (long unsigned) thread, (std::string) cset_pin); +} + void setPriority(int priority) { int ret;