From 22f1e8be8e579f816dfb462f22ea797031d3998e Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sat, 29 Aug 2020 22:40:49 +0200 Subject: [PATCH 01/46] First peliminary revision of dft algorithm --- lib/hooks/CMakeLists.txt | 1 + lib/hooks/dft.cpp | 186 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 lib/hooks/dft.cpp diff --git a/lib/hooks/CMakeLists.txt b/lib/hooks/CMakeLists.txt index ee5a4d78e..033475d69 100644 --- a/lib/hooks/CMakeLists.txt +++ b/lib/hooks/CMakeLists.txt @@ -24,6 +24,7 @@ set(HOOK_SRC average.cpp cast.cpp decimate.cpp + dft.cpp dp.cpp drop.cpp dump.cpp diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp new file mode 100644 index 000000000..79cb037d5 --- /dev/null +++ b/lib/hooks/dft.cpp @@ -0,0 +1,186 @@ +/** DFT hook. + * + * @author Manuel Pitz + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup hooks Hook functions + * @{ + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace villas { +namespace node { + +class DftHook : public Hook { + +protected: + struct format_type *format; + + //double* smp_memory; + double smp_memory[200]; + + uint smp_mem_pos; + uint smp_mem_size; + + std::complex dftMatrix[200][200]; + std::complex omega; + std::complex M_I; + std::complex dftResults[200]; + double absDftResults[200]; + +public: + DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : + Hook(p, n, fl, prio, en), + smp_mem_pos(0), + smp_mem_size(200), + M_I(0.0,1.0) + { + format = format_type_lookup("villas.human"); + } + + virtual void prepare(){ + + struct signal *freq_sig; + struct signal *ampl_sig; + struct signal *phase_sig; + + /* Add signals */ + freq_sig = signal_create("amplitude", nullptr, SignalType::FLOAT); + ampl_sig = signal_create("phase", nullptr, SignalType::FLOAT); + phase_sig = signal_create("frequency", nullptr, SignalType::FLOAT); + + + if (!freq_sig || !ampl_sig || !phase_sig) + throw RuntimeError("Failed to create new signals"); + + + vlist_push(&signals, freq_sig); + vlist_push(&signals, ampl_sig); + vlist_push(&signals, phase_sig); + + + //offset = vlist_length(&signals) - 1;//needs to be cleaned up + + genDftMatrix(); + + + + state = State::PREPARED; + + } + + + virtual void start() + { + + assert(state == State::PREPARED || state == State::STOPPED); + + //smp_memory = new double[smp_mem_size]; + if (!smp_memory) + throw MemoryAllocationError(); + + for(uint i = 0; i < smp_mem_size; i++) + smp_memory[i] = 0; + + + + state = State::STARTED; + } + + virtual void stop() + { + assert(state == State::STARTED); + + + state = State::STOPPED; + } + + virtual void parse(json_t *cfg) + { + + assert(state != State::STARTED); + + Hook::parse(cfg); + + state = State::PARSED; + } + + virtual Hook::Reason process(sample *smp) + { + assert(state == State::STARTED); + + smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; + smp_mem_pos ++ ; + + + calcDft(); + + for(uint i=0; i p; + +} /* namespace node */ +} /* namespace villas */ + +/** @} */ + From 356fe366249e1f74f7ce049e7fa5a496b90f2e53 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sun, 30 Aug 2020 14:43:58 +0200 Subject: [PATCH 02/46] Fist working version of DFT algorithm --- lib/hooks/dft.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 79cb037d5..9ac16db8c 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -42,22 +42,22 @@ protected: struct format_type *format; //double* smp_memory; - double smp_memory[200]; + double smp_memory[2000]; uint smp_mem_pos; uint smp_mem_size; - std::complex dftMatrix[200][200]; + std::complex dftMatrix[2000][2000]; std::complex omega; std::complex M_I; - std::complex dftResults[200]; - double absDftResults[200]; + std::complex dftResults[2000]; + double absDftResults[2000]; public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), smp_mem_pos(0), - smp_mem_size(200), + smp_mem_size(2000), M_I(0.0,1.0) { format = format_type_lookup("villas.human"); @@ -137,11 +137,13 @@ public: smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; smp_mem_pos ++ ; + if((smp_mem_pos % smp_mem_size) == 0){ + calcDft(); - calcDft(); - - for(uint i=0; i %f\t\t10Hz -> %f\t\t10.1Hz -> %f",absDftResults[99]/1000,absDftResults[100]/1000,absDftResults[101]/1000); } return Reason::OK; } @@ -168,7 +170,7 @@ public: for( uint i=0; i < smp_mem_size; i++){ dftResults[i] = 0; for(uint j=0; j < smp_mem_size; j++){ - dftResults[i]+= smp_memory[( j + smp_mem_pos ) % smp_mem_size] * dftMatrix[i][j]; + dftResults[i]+= smp_memory[( j + smp_mem_pos + 1) % smp_mem_size] * dftMatrix[i][j]; } } } From eb79b7c567c38c28d7a9942b75339fe68525d602 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 31 Aug 2020 11:48:35 +0200 Subject: [PATCH 03/46] tmp commit with added window function --- lib/hooks/dft.cpp | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 9ac16db8c..7c1a300e2 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -33,6 +33,8 @@ #include #include +#define MULTI 10 + namespace villas { namespace node { @@ -42,22 +44,23 @@ protected: struct format_type *format; //double* smp_memory; - double smp_memory[2000]; + double smp_memory[1000]; uint smp_mem_pos; uint smp_mem_size; - std::complex dftMatrix[2000][2000]; + std::complex dftMatrix[1000 * MULTI][1000 * MULTI]; std::complex omega; std::complex M_I; - std::complex dftResults[2000]; - double absDftResults[2000]; + std::complex dftResults[1000 * MULTI]; + double filterWindowCoefficents[1000]; + double absDftResults[1000 * MULTI]; public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), smp_mem_pos(0), - smp_mem_size(2000), + smp_mem_size(1000), M_I(0.0,1.0) { format = format_type_lookup("villas.human"); @@ -87,7 +90,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up genDftMatrix(); - + calcWindow("flatto"); state = State::PREPARED; @@ -143,7 +146,7 @@ public: for(uint i=0; i %f\t\t10Hz -> %f\t\t10.1Hz -> %f",absDftResults[99]/1000,absDftResults[100]/1000,absDftResults[101]/1000); + info("49.5Hz -> %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99]/1000,absDftResults[100]/1000,absDftResults[101]/1000); } return Reason::OK; } @@ -167,13 +170,35 @@ public: } void calcDft(){ - for( uint i=0; i < smp_mem_size; i++){ + //prepare sample window + double tmp_smp_window[1000]; + for(uint i = 0; i< smp_mem_size; i++){ + tmp_smp_window[i] = smp_memory[( i + smp_mem_pos + 1) % smp_mem_size] * filterWindowCoefficents[i]; + } + + + for( uint i=0; i < smp_mem_size * MULTI; i++){ dftResults[i] = 0; - for(uint j=0; j < smp_mem_size; j++){ - dftResults[i]+= smp_memory[( j + smp_mem_pos + 1) % smp_mem_size] * dftMatrix[i][j]; + for(uint j=0; j < smp_mem_size * MULTI; j++){ + dftResults[i]+= tmp_smp_window[j % smp_mem_size] * dftMatrix[i][j]; } } } + + void calcWindow(const char *window_name){ + if(strcmp(window_name, "flattop") == 0){ + for(uint i=0; i < smp_mem_size; i++) + filterWindowCoefficents[i] = 0.21557895 + - 0.41663158 * cos(2 * M_PI * i / ( smp_mem_size - 1 )) + + 0.277263158 * cos(4 * M_PI * i / ( smp_mem_size - 1 )) + - 0.083578947 * cos(6 * M_PI * i / ( smp_mem_size - 1 )) + + 0.006947368 * cos(8 * M_PI * i / ( smp_mem_size - 1 )); + + }else{ + for(uint i=0; i < smp_mem_size; i++) + filterWindowCoefficents[i] = 1; + } + } }; /* Register hook */ From 265dc8b0b788a5425febbcab7761863a57b9ab4a Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 31 Aug 2020 13:56:53 +0200 Subject: [PATCH 04/46] add other window functions --- lib/hooks/dft.cpp | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 7c1a300e2..cb146adfa 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -34,6 +34,7 @@ #include #define MULTI 10 +#define SMP_RATE 100 namespace villas { namespace node { @@ -44,23 +45,23 @@ protected: struct format_type *format; //double* smp_memory; - double smp_memory[1000]; + double smp_memory[SMP_RATE]; uint smp_mem_pos; uint smp_mem_size; - std::complex dftMatrix[1000 * MULTI][1000 * MULTI]; + std::complex dftMatrix[SMP_RATE * MULTI][SMP_RATE * MULTI]; std::complex omega; std::complex M_I; - std::complex dftResults[1000 * MULTI]; - double filterWindowCoefficents[1000]; - double absDftResults[1000 * MULTI]; + std::complex dftResults[SMP_RATE * MULTI]; + double filterWindowCoefficents[SMP_RATE]; + double absDftResults[SMP_RATE * MULTI]; public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), smp_mem_pos(0), - smp_mem_size(1000), + smp_mem_size(SMP_RATE), M_I(0.0,1.0) { format = format_type_lookup("villas.human"); @@ -90,7 +91,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up genDftMatrix(); - calcWindow("flatto"); + calcWindow("hanning"); state = State::PREPARED; @@ -143,10 +144,10 @@ public: if((smp_mem_pos % smp_mem_size) == 0){ calcDft(); - for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99]/1000,absDftResults[100]/1000,absDftResults[101]/1000); + info("49.5Hz -> %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[9] * 2 / (SMP_RATE * MULTI),absDftResults[10] * 2 / (SMP_RATE * MULTI) ,absDftResults[11] * 2 / (SMP_RATE * MULTI) ); } return Reason::OK; } @@ -162,8 +163,8 @@ public: omega = exp((-2 * M_PI * M_I) / (double)smp_mem_size); - for( uint i=0 ; i < smp_mem_size ; i++){ - for( uint j=0 ; j < smp_mem_size ; j++){ + for( uint i=0 ; i < smp_mem_size * MULTI ; i++){ + for( uint j=0 ; j < smp_mem_size * MULTI ; j++){ dftMatrix[i][j] = pow(omega, i * j); } } @@ -171,7 +172,7 @@ public: void calcDft(){ //prepare sample window - double tmp_smp_window[1000]; + double tmp_smp_window[SMP_RATE]; for(uint i = 0; i< smp_mem_size; i++){ tmp_smp_window[i] = smp_memory[( i + smp_mem_pos + 1) % smp_mem_size] * filterWindowCoefficents[i]; } @@ -189,11 +190,19 @@ public: if(strcmp(window_name, "flattop") == 0){ for(uint i=0; i < smp_mem_size; i++) filterWindowCoefficents[i] = 0.21557895 - - 0.41663158 * cos(2 * M_PI * i / ( smp_mem_size - 1 )) - + 0.277263158 * cos(4 * M_PI * i / ( smp_mem_size - 1 )) - - 0.083578947 * cos(6 * M_PI * i / ( smp_mem_size - 1 )) - + 0.006947368 * cos(8 * M_PI * i / ( smp_mem_size - 1 )); - + - 0.41663158 * cos(2 * M_PI * i / ( smp_mem_size )) + + 0.277263158 * cos(4 * M_PI * i / ( smp_mem_size )) + - 0.083578947 * cos(6 * M_PI * i / ( smp_mem_size )) + + 0.006947368 * cos(8 * M_PI * i / ( smp_mem_size )); + }else if(strcmp(window_name, "hanning") == 0 || strcmp(window_name, "hann") == 0){ + double a_0 = 0.5;//this is the hann window + if(strcmp(window_name, "hanning")) + a_0 = 25/46; + + for(uint i=0; i < smp_mem_size; i++) + filterWindowCoefficents[i] = a_0 + - (1 - a_0) * cos(2 * M_PI * i / (smp_mem_size)); + }else{ for(uint i=0; i < smp_mem_size; i++) filterWindowCoefficents[i] = 1; From c60ba7bbc0fad5d518ea65450662578a479d4913 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 31 Aug 2020 18:09:25 +0200 Subject: [PATCH 05/46] working version with 0 padding and simpel signal repeat padding --- lib/hooks/dft.cpp | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index cb146adfa..5afa346f3 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -34,7 +34,7 @@ #include #define MULTI 10 -#define SMP_RATE 100 +#define SMP_RATE 1000 namespace villas { namespace node { @@ -57,6 +57,12 @@ protected: double filterWindowCoefficents[SMP_RATE]; double absDftResults[SMP_RATE * MULTI]; + enum paddingStyle{ + NONE, + ZERO, + SIG_REPEAT + }; + public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), @@ -91,7 +97,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up genDftMatrix(); - calcWindow("hanning"); + calcWindow("ha1nning"); state = State::PREPARED; @@ -142,12 +148,12 @@ public: smp_mem_pos ++ ; if((smp_mem_pos % smp_mem_size) == 0){ - calcDft(); + calcDft(paddingStyle::SIG_REPEAT); for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[9] * 2 / (SMP_RATE * MULTI),absDftResults[10] * 2 / (SMP_RATE * MULTI) ,absDftResults[11] * 2 / (SMP_RATE * MULTI) ); + info("49.5Hz -> %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99] * 2 / (SMP_RATE * MULTI),absDftResults[100] * 2 / (SMP_RATE * MULTI) ,absDftResults[101] * 2 / (SMP_RATE * MULTI) ); } return Reason::OK; } @@ -161,7 +167,7 @@ public: void genDftMatrix(){ using namespace std::complex_literals; - omega = exp((-2 * M_PI * M_I) / (double)smp_mem_size); + omega = exp((-2 * M_PI * M_I) / (double)(smp_mem_size * MULTI)); for( uint i=0 ; i < smp_mem_size * MULTI ; i++){ for( uint j=0 ; j < smp_mem_size * MULTI ; j++){ @@ -170,7 +176,7 @@ public: } } - void calcDft(){ + void calcDft(paddingStyle padding){ //prepare sample window double tmp_smp_window[SMP_RATE]; for(uint i = 0; i< smp_mem_size; i++){ @@ -181,7 +187,15 @@ public: for( uint i=0; i < smp_mem_size * MULTI; i++){ dftResults[i] = 0; for(uint j=0; j < smp_mem_size * MULTI; j++){ - dftResults[i]+= tmp_smp_window[j % smp_mem_size] * dftMatrix[i][j]; + if(padding == paddingStyle::ZERO){ + if(j < (smp_mem_size)){ + dftResults[i]+= tmp_smp_window[j] * dftMatrix[i][j]; + }else{ + dftResults[i]+= 0; + } + }else if(padding == paddingStyle::SIG_REPEAT){//repeate samples + dftResults[i]+= tmp_smp_window[j % smp_mem_size] * dftMatrix[i][j]; + } } } } @@ -201,7 +215,7 @@ public: for(uint i=0; i < smp_mem_size; i++) filterWindowCoefficents[i] = a_0 - - (1 - a_0) * cos(2 * M_PI * i / (smp_mem_size)); + - (1 - a_0) * cos(2 * M_PI * i / ( smp_mem_size )); }else{ for(uint i=0; i < smp_mem_size; i++) From 2d4fd8b7eda4482c8f479d23052d9c6fbb04fa43 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Tue, 1 Sep 2020 21:18:00 +0200 Subject: [PATCH 06/46] added dumping capabilities to export arrays --- lib/hooks/dft.cpp | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 5afa346f3..8c2653095 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -32,8 +32,10 @@ #include #include #include +#include +#include -#define MULTI 10 +#define MULTI 1 #define SMP_RATE 1000 namespace villas { @@ -63,12 +65,15 @@ protected: SIG_REPEAT }; + int skipDft;//this is tmp to skip the dft next time + public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), smp_mem_pos(0), smp_mem_size(SMP_RATE), - M_I(0.0,1.0) + M_I(0.0,1.0), + skipDft(10) { format = format_type_lookup("villas.human"); } @@ -144,16 +149,25 @@ public: { assert(state == State::STARTED); - smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; - smp_mem_pos ++ ; - if((smp_mem_pos % smp_mem_size) == 0){ + + if(skipDft > 0){ + skipDft --; + }else if((smp_mem_pos % smp_mem_size) == 0 && skipDft < 0){ calcDft(paddingStyle::SIG_REPEAT); for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99] * 2 / (SMP_RATE * MULTI),absDftResults[100] * 2 / (SMP_RATE * MULTI) ,absDftResults[101] * 2 / (SMP_RATE * MULTI) ); + + + dumpData("/tmp/absDftResults",absDftResults,smp_mem_size * MULTI); + skipDft = 10; + }else{ + smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; + smp_mem_pos ++ ; + skipDft --; } return Reason::OK; } @@ -163,6 +177,17 @@ public: //delete smp_memory; } + void dumpData(const char *path, double *data, uint size){ + std::ofstream fh; + fh.open(path); + for(uint i = 0 ; i < size ; i++){ + if(i>0)fh << ";"; + fh << data[i]; + + } + fh.close(); + } + void genDftMatrix(){ using namespace std::complex_literals; @@ -180,9 +205,18 @@ public: //prepare sample window double tmp_smp_window[SMP_RATE]; for(uint i = 0; i< smp_mem_size; i++){ - tmp_smp_window[i] = smp_memory[( i + smp_mem_pos + 1) % smp_mem_size] * filterWindowCoefficents[i]; + tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % smp_mem_size] * filterWindowCoefficents[i]; } + dumpData("/tmp/signal_original",tmp_smp_window,smp_mem_size); + for(uint i = 0; i< smp_mem_size; i++){ + tmp_smp_window[i] *= filterWindowCoefficents[i]; + } + dumpData("/tmp/signal_windowed",tmp_smp_window,smp_mem_size); + + dumpData("/tmp/smp_window",smp_memory,smp_mem_size); + + dumpData("/tmp/window",filterWindowCoefficents,smp_mem_size); for( uint i=0; i < smp_mem_size * MULTI; i++){ dftResults[i] = 0; From 69a7810842ce3124a65b5b78b7d9b248feee869c Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 3 Sep 2020 12:05:45 +0200 Subject: [PATCH 07/46] update dft and add more debug info --- lib/hooks/dft.cpp | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 8c2653095..e801b98db 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -35,7 +35,7 @@ #include #include -#define MULTI 1 +#define MULTI 20 #define SMP_RATE 1000 namespace villas { @@ -57,7 +57,11 @@ protected: std::complex M_I; std::complex dftResults[SMP_RATE * MULTI]; double filterWindowCoefficents[SMP_RATE]; - double absDftResults[SMP_RATE * MULTI]; + double absDftResults[SMP_RATE * MULTI/2]; + + + int skipDft;//this is tmp to skip the dft next time + double window_corretion_factor; enum paddingStyle{ NONE, @@ -65,7 +69,6 @@ protected: SIG_REPEAT }; - int skipDft;//this is tmp to skip the dft next time public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : @@ -73,7 +76,8 @@ public: smp_mem_pos(0), smp_mem_size(SMP_RATE), M_I(0.0,1.0), - skipDft(10) + skipDft(10), + window_corretion_factor(0) { format = format_type_lookup("villas.human"); } @@ -102,7 +106,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up genDftMatrix(); - calcWindow("ha1nning"); + calcWindow("flattopz"); state = State::PREPARED; @@ -154,15 +158,15 @@ public: if(skipDft > 0){ skipDft --; }else if((smp_mem_pos % smp_mem_size) == 0 && skipDft < 0){ - calcDft(paddingStyle::SIG_REPEAT); + calcDft(paddingStyle::ZERO); - for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99] * 2 / (SMP_RATE * MULTI),absDftResults[100] * 2 / (SMP_RATE * MULTI) ,absDftResults[101] * 2 / (SMP_RATE * MULTI) ); + info("49.5Hz -> %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99], absDftResults[100] ,absDftResults[101]); - dumpData("/tmp/absDftResults",absDftResults,smp_mem_size * MULTI); + dumpData("/tmp/absDftResults",absDftResults,smp_mem_size * MULTI/2); skipDft = 10; }else{ smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; @@ -205,7 +209,7 @@ public: //prepare sample window double tmp_smp_window[SMP_RATE]; for(uint i = 0; i< smp_mem_size; i++){ - tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % smp_mem_size] * filterWindowCoefficents[i]; + tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % smp_mem_size]; } dumpData("/tmp/signal_original",tmp_smp_window,smp_mem_size); @@ -216,9 +220,7 @@ public: dumpData("/tmp/smp_window",smp_memory,smp_mem_size); - dumpData("/tmp/window",filterWindowCoefficents,smp_mem_size); - - for( uint i=0; i < smp_mem_size * MULTI; i++){ + for( uint i=0; i < smp_mem_size * MULTI / 2; i++){ dftResults[i] = 0; for(uint j=0; j < smp_mem_size * MULTI; j++){ if(padding == paddingStyle::ZERO){ @@ -236,25 +238,32 @@ public: void calcWindow(const char *window_name){ if(strcmp(window_name, "flattop") == 0){ - for(uint i=0; i < smp_mem_size; i++) + for(uint i=0; i < smp_mem_size; i++){ filterWindowCoefficents[i] = 0.21557895 - 0.41663158 * cos(2 * M_PI * i / ( smp_mem_size )) + 0.277263158 * cos(4 * M_PI * i / ( smp_mem_size )) - 0.083578947 * cos(6 * M_PI * i / ( smp_mem_size )) + 0.006947368 * cos(8 * M_PI * i / ( smp_mem_size )); + window_corretion_factor += filterWindowCoefficents[i]; + } }else if(strcmp(window_name, "hanning") == 0 || strcmp(window_name, "hann") == 0){ double a_0 = 0.5;//this is the hann window if(strcmp(window_name, "hanning")) a_0 = 25/46; - for(uint i=0; i < smp_mem_size; i++) + for(uint i=0; i < smp_mem_size; i++){ filterWindowCoefficents[i] = a_0 - (1 - a_0) * cos(2 * M_PI * i / ( smp_mem_size )); - + window_corretion_factor += filterWindowCoefficents[i]; + } }else{ - for(uint i=0; i < smp_mem_size; i++) + for(uint i=0; i < smp_mem_size; i++){ filterWindowCoefficents[i] = 1; + window_corretion_factor += filterWindowCoefficents[i]; + } } + window_corretion_factor /= smp_mem_size; + dumpData("/tmp/filter_window",filterWindowCoefficents,smp_mem_size); } }; From fb5869b9fa0bffd83ac5877271b284c29f87c8c6 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 3 Sep 2020 18:34:45 +0200 Subject: [PATCH 08/46] updated dft --- lib/hooks/dft.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index e801b98db..7c4879086 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -35,7 +35,7 @@ #include #include -#define MULTI 20 +#define MULTI 10 #define SMP_RATE 1000 namespace villas { @@ -106,7 +106,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up genDftMatrix(); - calcWindow("flattopz"); + calcWindow("hanning"); state = State::PREPARED; @@ -169,7 +169,7 @@ public: dumpData("/tmp/absDftResults",absDftResults,smp_mem_size * MULTI/2); skipDft = 10; }else{ - smp_memory[smp_mem_pos % smp_mem_size] = smp->data[1].f; + smp_memory[smp_mem_pos % smp_mem_size] = smp->data[0].f; smp_mem_pos ++ ; skipDft --; } From 41b12a0d88053ec343bbf042669e74ff86cfe704 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Fri, 4 Sep 2020 17:24:14 +0200 Subject: [PATCH 09/46] updated version including cleanup and parsing of configuration --- lib/hooks/dft.cpp | 262 +++++++++++++++++++++++++++++++++------------- 1 file changed, 192 insertions(+), 70 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 7c4879086..2ce714a16 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -35,46 +35,73 @@ #include #include -#define MULTI 10 -#define SMP_RATE 1000 - namespace villas { namespace node { class DftHook : public Hook { protected: + enum paddingType { + ZERO, + SIG_REPEAT + }; + + enum windowType { + NONE, + FLATTOP, + HANN, + HAMMING + }; + + windowType window_type; + paddingType padding_type; + struct format_type *format; - //double* smp_memory; - double smp_memory[SMP_RATE]; + double* smp_memory; + std::complex** dftMatrix; + std::complex* dftResults; + double* filterWindowCoefficents; + double* absDftResults; + double* absDftFreqs; + + double sample_rate; + double start_freqency; + double end_freqency; + double frequency_resolution; + double dft_rate; + uint window_size; + uint sample_rate_i; + uint window_multiplier;//multiplyer for the window to achieve frequency resolution + uint freq_count;//number of requency bins that are calculated uint smp_mem_pos; - uint smp_mem_size; - std::complex dftMatrix[SMP_RATE * MULTI][SMP_RATE * MULTI]; + std::complex omega; std::complex M_I; - std::complex dftResults[SMP_RATE * MULTI]; - double filterWindowCoefficents[SMP_RATE]; - double absDftResults[SMP_RATE * MULTI/2]; int skipDft;//this is tmp to skip the dft next time double window_corretion_factor; - enum paddingStyle{ - NONE, - ZERO, - SIG_REPEAT - }; + public: DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), + window_type(windowType::NONE), + padding_type(paddingType::ZERO), + sample_rate(0), + start_freqency(0), + end_freqency(0), + frequency_resolution(0), + dft_rate(0), + window_size(0), + sample_rate_i(0), + window_multiplier(0), smp_mem_pos(0), - smp_mem_size(SMP_RATE), M_I(0.0,1.0), skipDft(10), window_corretion_factor(0) @@ -105,8 +132,44 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up + + window_multiplier = ceil( ( sample_rate / window_size ) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + + freq_count = ceil( ( end_freqency - start_freqency ) / frequency_resolution) + 1; + + //init sample memory + smp_memory = new double[window_size]; + if (!smp_memory) + throw MemoryAllocationError(); + + for(uint i = 0; i < window_size; i++) + smp_memory[i] = 0; + + + + //init matrix of dft coeffients + dftMatrix = new std::complex*[freq_count]; + if (!dftMatrix) + throw MemoryAllocationError(); + + for(uint i = 0; i < freq_count; i++) { + dftMatrix[i] = new std::complex[window_size * window_multiplier](); + if (!dftMatrix[i]) + throw MemoryAllocationError(); + } + + dftResults = new std::complex[freq_count](); + + filterWindowCoefficents = new double[window_size]; + + absDftResults = new double[freq_count]; + + absDftFreqs = new double[freq_count]; + for(uint i=0; i < freq_count; i++) + absDftFreqs[i] = start_freqency + i * frequency_resolution; + genDftMatrix(); - calcWindow("hanning"); + calcWindow(window_type); state = State::PREPARED; @@ -119,14 +182,6 @@ public: assert(state == State::PREPARED || state == State::STOPPED); - //smp_memory = new double[smp_mem_size]; - if (!smp_memory) - throw MemoryAllocationError(); - - for(uint i = 0; i < smp_mem_size; i++) - smp_memory[i] = 0; - - state = State::STARTED; } @@ -141,12 +196,67 @@ public: virtual void parse(json_t *cfg) { + const char *padding_type_c = nullptr, *window_type_c = nullptr; + int ret; + json_error_t err; assert(state != State::STARTED); Hook::parse(cfg); state = State::PARSED; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: F, s?: F, s?: F, s?: F, s?: F , s?: i, s?: s, s?: s}", + "sample_rate", &sample_rate, + "start_freqency", &start_freqency, + "end_freqency", &end_freqency, + "frequency_resolution", &frequency_resolution, + "dft_rate", &dft_rate, + "window_size", &window_size, + "window_type", &window_type_c, + "padding_type", &padding_type_c + + ); + + if(!window_type_c) { + info("No Window type given, assume no windowing"); + window_type = windowType::NONE; + } else if(strcmp(window_type_c, "flattop") == 0) + window_type = windowType::FLATTOP; + else if(strcmp(window_type_c, "hamming") == 0) + window_type = windowType::HAMMING; + else if(strcmp(window_type_c, "hann") == 0) + window_type = windowType::HANN; + else { + info("Window type %s not recognized, assume no windowing",window_type_c); + window_type = windowType::NONE; + } + + if(!padding_type_c) { + info("No Padding type given, assume no zeropadding"); + padding_type = paddingType::ZERO; + } else if(strcmp(padding_type_c, "signal_repeat") == 0) + padding_type = paddingType::SIG_REPEAT; + else{ + info("Padding type %s not recognized, assume zero padding",padding_type_c); + padding_type = paddingType::ZERO; + } + + + if(end_freqency < 0 || end_freqency > sample_rate){ + error("End frequency must be smaller than sample_rate (%f)",sample_rate); + ret = 1; + } + + if(frequency_resolution > (sample_rate/window_size)){ + error("The maximum frequency resolution with smaple_rate:%f and window_site:%i is %f",sample_rate, window_size, sample_rate/window_size); + ret = 1; + } + + sample_rate_i = ceil(sample_rate); + + if (ret) + throw ConfigError(cfg, err, "node-config-hook-dft"); } virtual Hook::Reason process(sample *smp) @@ -157,19 +267,19 @@ public: if(skipDft > 0){ skipDft --; - }else if((smp_mem_pos % smp_mem_size) == 0 && skipDft < 0){ - calcDft(paddingStyle::ZERO); + }else if((smp_mem_pos % window_size) == 0 && skipDft < 0){ + calcDft(paddingType::ZERO); - for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[99], absDftResults[100] ,absDftResults[101]); + info("49.5Hz -> %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[1], absDftResults[1] ,absDftResults[1]); - dumpData("/tmp/absDftResults",absDftResults,smp_mem_size * MULTI/2); + dumpData("/tmp/absDftResults", absDftResults, freq_count, absDftFreqs); skipDft = 10; }else{ - smp_memory[smp_mem_pos % smp_mem_size] = smp->data[0].f; + smp_memory[smp_mem_pos % window_size] = smp->data[1].f; smp_mem_pos ++ ; skipDft --; } @@ -181,13 +291,19 @@ public: //delete smp_memory; } - void dumpData(const char *path, double *data, uint size){ + void dumpData(const char *path, double *ydata, uint size, double *xdata=nullptr){ std::ofstream fh; fh.open(path); for(uint i = 0 ; i < size ; i++){ if(i>0)fh << ";"; - fh << data[i]; - + fh << ydata[i]; + } + if(xdata){ + fh << "\n"; + for(uint i = 0 ; i < size ; i++){ + if(i>0)fh << ";"; + fh << xdata[i]; + } } fh.close(); } @@ -196,74 +312,80 @@ public: void genDftMatrix(){ using namespace std::complex_literals; - omega = exp((-2 * M_PI * M_I) / (double)(smp_mem_size * MULTI)); + omega = exp((-2 * M_PI * M_I) / (double)(window_size * window_multiplier)); + uint startBin = floor( start_freqency / frequency_resolution ); + - for( uint i=0 ; i < smp_mem_size * MULTI ; i++){ - for( uint j=0 ; j < smp_mem_size * MULTI ; j++){ - dftMatrix[i][j] = pow(omega, i * j); + for( uint i = 0; i < freq_count ; i++){ + for( uint j=0 ; j < window_size * window_multiplier ; j++){ + dftMatrix[i][j] = pow(omega, (i + startBin) * j); } } } - void calcDft(paddingStyle padding){ - //prepare sample window - double tmp_smp_window[SMP_RATE]; - for(uint i = 0; i< smp_mem_size; i++){ - tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % smp_mem_size]; - } - dumpData("/tmp/signal_original",tmp_smp_window,smp_mem_size); + void calcDft(paddingType padding){ - for(uint i = 0; i< smp_mem_size; i++){ + + //prepare sample window The following parts can be combined + double tmp_smp_window[window_size]; + for(uint i = 0; i< window_size; i++){ + tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % window_size]; + } + dumpData("/tmp/signal_original",tmp_smp_window,window_size); + + for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] *= filterWindowCoefficents[i]; } - dumpData("/tmp/signal_windowed",tmp_smp_window,smp_mem_size); + dumpData("/tmp/signal_windowed",tmp_smp_window,window_size); - dumpData("/tmp/smp_window",smp_memory,smp_mem_size); + dumpData("/tmp/smp_window",smp_memory,window_size); - for( uint i=0; i < smp_mem_size * MULTI / 2; i++){ + for( uint i=0; i < freq_count; i++){ dftResults[i] = 0; - for(uint j=0; j < smp_mem_size * MULTI; j++){ - if(padding == paddingStyle::ZERO){ - if(j < (smp_mem_size)){ + for(uint j=0; j < window_size * window_multiplier; j++){ + if(padding == paddingType::ZERO){ + if(j < (window_size)){ dftResults[i]+= tmp_smp_window[j] * dftMatrix[i][j]; }else{ dftResults[i]+= 0; } - }else if(padding == paddingStyle::SIG_REPEAT){//repeate samples - dftResults[i]+= tmp_smp_window[j % smp_mem_size] * dftMatrix[i][j]; + }else if(padding == paddingType::SIG_REPEAT){//repeate samples + dftResults[i]+= tmp_smp_window[j % window_size] * dftMatrix[i][j]; } } } } - void calcWindow(const char *window_name){ - if(strcmp(window_name, "flattop") == 0){ - for(uint i=0; i < smp_mem_size; i++){ + void calcWindow(windowType window_type_in){ + + + if(window_type_in == windowType::FLATTOP){ + for(uint i=0; i < window_size; i++){ filterWindowCoefficents[i] = 0.21557895 - - 0.41663158 * cos(2 * M_PI * i / ( smp_mem_size )) - + 0.277263158 * cos(4 * M_PI * i / ( smp_mem_size )) - - 0.083578947 * cos(6 * M_PI * i / ( smp_mem_size )) - + 0.006947368 * cos(8 * M_PI * i / ( smp_mem_size )); + - 0.41663158 * cos(2 * M_PI * i / ( window_size )) + + 0.277263158 * cos(4 * M_PI * i / ( window_size )) + - 0.083578947 * cos(6 * M_PI * i / ( window_size )) + + 0.006947368 * cos(8 * M_PI * i / ( window_size )); window_corretion_factor += filterWindowCoefficents[i]; } - }else if(strcmp(window_name, "hanning") == 0 || strcmp(window_name, "hann") == 0){ + }else if(window_type_in == windowType::HAMMING || window_type_in == windowType::HANN){ double a_0 = 0.5;//this is the hann window - if(strcmp(window_name, "hanning")) - a_0 = 25/46; + if(window_type_in == windowType::HAMMING) + a_0 = 25./46; - for(uint i=0; i < smp_mem_size; i++){ + for(uint i=0; i < window_size; i++){ filterWindowCoefficents[i] = a_0 - - (1 - a_0) * cos(2 * M_PI * i / ( smp_mem_size )); + - (1 - a_0) * cos(2 * M_PI * i / ( window_size )); window_corretion_factor += filterWindowCoefficents[i]; } }else{ - for(uint i=0; i < smp_mem_size; i++){ + for(uint i=0; i < window_size; i++){ filterWindowCoefficents[i] = 1; window_corretion_factor += filterWindowCoefficents[i]; } } - window_corretion_factor /= smp_mem_size; - dumpData("/tmp/filter_window",filterWindowCoefficents,smp_mem_size); + window_corretion_factor /= window_size; + dumpData("/tmp/filter_window",filterWindowCoefficents,window_size); } }; From fe6e309305d330efda9941dfc1f8ef5a408e19e8 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Fri, 4 Sep 2020 17:59:29 +0200 Subject: [PATCH 10/46] added moving window dft capabilities and realtime checking --- lib/hooks/dft.cpp | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 2ce714a16..3f28f53ae 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -34,6 +34,7 @@ #include #include #include +#include namespace villas { namespace node { @@ -65,13 +66,12 @@ protected: double* absDftResults; double* absDftFreqs; - double sample_rate; + uint sample_rate; double start_freqency; double end_freqency; double frequency_resolution; - double dft_rate; + uint dft_rate; uint window_size; - uint sample_rate_i; uint window_multiplier;//multiplyer for the window to achieve frequency resolution uint freq_count;//number of requency bins that are calculated @@ -99,7 +99,6 @@ public: frequency_resolution(0), dft_rate(0), window_size(0), - sample_rate_i(0), window_multiplier(0), smp_mem_pos(0), M_I(0.0,1.0), @@ -133,7 +132,7 @@ public: //offset = vlist_length(&signals) - 1;//needs to be cleaned up - window_multiplier = ceil( ( sample_rate / window_size ) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + window_multiplier = ceil( ( (double)sample_rate / window_size ) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution freq_count = ceil( ( end_freqency - start_freqency ) / frequency_resolution) + 1; @@ -206,7 +205,7 @@ public: state = State::PARSED; - ret = json_unpack_ex(cfg, &err, 0, "{ s?: F, s?: F, s?: F, s?: F, s?: F , s?: i, s?: s, s?: s}", + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: F, s?: F, s?: F, s?: i , s?: i, s?: s, s?: s}", "sample_rate", &sample_rate, "start_freqency", &start_freqency, "end_freqency", &end_freqency, @@ -244,16 +243,16 @@ public: if(end_freqency < 0 || end_freqency > sample_rate){ - error("End frequency must be smaller than sample_rate (%f)",sample_rate); + error("End frequency must be smaller than sample_rate (%i)",sample_rate); ret = 1; } - if(frequency_resolution > (sample_rate/window_size)){ - error("The maximum frequency resolution with smaple_rate:%f and window_site:%i is %f",sample_rate, window_size, sample_rate/window_size); + if(frequency_resolution > ((double)sample_rate/window_size)){ + error("The maximum frequency resolution with smaple_rate:%i and window_site:%i is %f",sample_rate, window_size, ((double)sample_rate/window_size) ); ret = 1; } - sample_rate_i = ceil(sample_rate); + if (ret) throw ConfigError(cfg, err, "node-config-hook-dft"); @@ -262,12 +261,13 @@ public: virtual Hook::Reason process(sample *smp) { assert(state == State::STARTED); + auto start_time = std::chrono::high_resolution_clock::now(); + smp_memory[smp_mem_pos % window_size] = smp->data[1].f; + smp_mem_pos ++ ; - if(skipDft > 0){ - skipDft --; - }else if((smp_mem_pos % window_size) == 0 && skipDft < 0){ + if((smp_mem_pos % ( sample_rate / dft_rate )) == 0){ calcDft(paddingType::ZERO); for(uint i=0; idata[1].f; - smp_mem_pos ++ ; - skipDft --; } + + + double duration = (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start_time )).count(); + double period = ((1000000) / sample_rate); + if( duration > period ) + warning("Calculation is not Realtime"); + return Reason::OK; } From cb8e31cf84482bd383b5b46053ef20556f7a22c0 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Fri, 4 Sep 2020 21:06:23 +0200 Subject: [PATCH 11/46] add capability to sync dft on time tag of sample --- lib/hooks/dft.cpp | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 3f28f53ae..a84ea76f3 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -34,7 +34,6 @@ #include #include #include -#include namespace villas { namespace node { @@ -74,17 +73,18 @@ protected: uint window_size; uint window_multiplier;//multiplyer for the window to achieve frequency resolution uint freq_count;//number of requency bins that are calculated + bool sync_dft; uint smp_mem_pos; + uint64_t last_sequence; std::complex omega; std::complex M_I; - int skipDft;//this is tmp to skip the dft next time double window_corretion_factor; - + timespec last_dft_cal; @@ -100,10 +100,13 @@ public: dft_rate(0), window_size(0), window_multiplier(0), + freq_count(0), + sync_dft(0), smp_mem_pos(0), + last_sequence(0), M_I(0.0,1.0), - skipDft(10), - window_corretion_factor(0) + window_corretion_factor(0), + last_dft_cal({0,0}) { format = format_type_lookup("villas.human"); } @@ -134,6 +137,7 @@ public: window_multiplier = ceil( ( (double)sample_rate / window_size ) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + freq_count = ceil( ( end_freqency - start_freqency ) / frequency_resolution) + 1; //init sample memory @@ -205,7 +209,7 @@ public: state = State::PARSED; - ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: F, s?: F, s?: F, s?: i , s?: i, s?: s, s?: s}", + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: F, s?: F, s?: F, s?: i , s?: i, s?: s, s?: s, s?: b}", "sample_rate", &sample_rate, "start_freqency", &start_freqency, "end_freqency", &end_freqency, @@ -213,7 +217,8 @@ public: "dft_rate", &dft_rate, "window_size", &window_size, "window_type", &window_type_c, - "padding_type", &padding_type_c + "padding_type", &padding_type_c, + "sync", &sync_dft ); @@ -234,9 +239,9 @@ public: if(!padding_type_c) { info("No Padding type given, assume no zeropadding"); padding_type = paddingType::ZERO; - } else if(strcmp(padding_type_c, "signal_repeat") == 0) + } else if(strcmp(padding_type_c, "signal_repeat") == 0) { padding_type = paddingType::SIG_REPEAT; - else{ + } else { info("Padding type %s not recognized, assume zero padding",padding_type_c); padding_type = paddingType::ZERO; } @@ -261,29 +266,43 @@ public: virtual Hook::Reason process(sample *smp) { assert(state == State::STARTED); - auto start_time = std::chrono::high_resolution_clock::now(); - - smp_memory[smp_mem_pos % window_size] = smp->data[1].f; + smp_memory[smp_mem_pos % window_size] = smp->data[0].f; smp_mem_pos ++ ; - if((smp_mem_pos % ( sample_rate / dft_rate )) == 0){ + + double timediff = 0; + if( sync_dft ){ + timediff = ( smp->ts.origin.tv_sec - last_dft_cal.tv_sec ) + ( smp->ts.origin.tv_nsec - last_dft_cal.tv_nsec ) * 1e-9; + } + + if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || (sync_dft && timediff >= ( 1 / dft_rate ) ) ) { calcDft(paddingType::ZERO); + double maxF = 0; + double maxA = 0; + int maxPos = 0; for(uint i=0; i %f\t\t50Hz -> %f\t\t50.5Hz -> %f",absDftResults[1], absDftResults[1] ,absDftResults[1]); + info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); dumpData("/tmp/absDftResults", absDftResults, freq_count, absDftFreqs); + + last_dft_cal = smp->ts.origin; } - double duration = (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start_time )).count(); - double period = ((1000000) / sample_rate); - if( duration > period ) - warning("Calculation is not Realtime"); + if((smp->sequence - last_sequence) > 1 ) + warning("Calculation is not Realtime. %li sampled missed",smp->sequence - last_sequence); + + last_sequence = smp->sequence; return Reason::OK; } @@ -294,6 +313,7 @@ public: } void dumpData(const char *path, double *ydata, uint size, double *xdata=nullptr){ + std::ofstream fh; fh.open(path); for(uint i = 0 ; i < size ; i++){ From ec962c5e5390daad067e7913d5c3e5c058e53f9b Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 7 Sep 2020 17:23:14 +0200 Subject: [PATCH 12/46] added error lookup for uldaq --- lib/nodes/uldaq.cpp | 134 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 3 deletions(-) diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index e7b1a8d59..d73314c36 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -37,6 +37,123 @@ using namespace villas::utils; static unsigned num_devs = ULDAQ_MAX_DEV_COUNT; static DaqDeviceDescriptor descriptors[ULDAQ_MAX_DEV_COUNT]; +static const struct { + const char *errStr; + uint bitPos; +} errorList[] = { + {"No error has occurred ERR_NO_ERROR" , 0}, + {"Unhandled internal exception ERR_UNHANDLED_EXCEPTION" , 1}, + {"Invalid device handle ERR_BAD_DEV_HANDLE" , 2}, + {"This function cannot be used with this device ERR_BAD_DEV_TYPE" , 3}, + {"Insufficient permission to access this device ERR_USB_DEV_NO_PERMISSION" , 4}, + {"USB interface is already claimed ERR_USB_INTERFACE_CLAIMED" , 5}, + {"Device not found ERR_DEV_NOT_FOUND" , 6}, + {"Device not connected or connection lost ERR_DEV_NOT_CONNECTED" , 7}, + {"Device no longer responding ERR_DEAD_DEV" , 8}, + {"Buffer too small for operation ERR_BAD_BUFFER_SIZE" , 9}, + {"Invalid buffer ERR_BAD_BUFFER" , 10}, + {"Invalid memory type ERR_BAD_MEM_TYPE" , 11}, + {"Invalid memory region ERR_BAD_MEM_REGION" , 12}, + {"Invalid range ERR_BAD_RANGE" , 13}, + {"Invalid analog input channel specified ERR_BAD_AI_CHAN" , 14}, + {"Invalid input mode specified ERR_BAD_INPUT_MODE" , 15}, + {"A background process is already in progress ERR_ALREADY_ACTIVE" , 16}, + {"Invalid trigger type specified ERR_BAD_TRIG_TYPE" , 17}, + {"FIFO overrun, data was not transferred from device fast enough ERR_OVERRUN" , 18}, + {"FIFO underrun, data was not transferred to device fast enough ERR_UNDERRUN" , 19}, + {"Operation timed out ERR_TIMEDOUT" , 20}, + {"Invalid option specified ERR_BAD_OPTION" , 21}, + {"Invalid sampling rate specified ERR_BAD_RATE" , 22}, + {"Sample count cannot be greater than FIFO size for BURSTIO scans ERR_BAD_BURSTIO_COUNT" , 23}, + {"Configuration not supported ERR_CONFIG_NOT_SUPPORTED" , 24}, + {"Invalid configuration value ERR_BAD_CONFIG_VAL" , 25}, + {"Invalid analog input channel type specified ERR_BAD_AI_CHAN_TYPE" , 26}, + {"ADC overrun occurred ERR_ADC_OVERRUN" , 27}, + {"Invalid thermocouple type specified ERR_BAD_TC_TYPE" , 28}, + {"Invalid unit specified ERR_BAD_UNIT" , 29}, + {"Invalid queue size ERR_BAD_QUEUE_SIZE" , 30}, + {"Invalid config item specified ERR_BAD_CONFIG_ITEM" , 31}, + {"Invalid info item specified ERR_BAD_INFO_ITEM" , 32}, + {"Invalid flag specified ERR_BAD_FLAG" , 33}, + {"Invalid sample count specified ERR_BAD_SAMPLE_COUNT" , 34}, + {"Internal error ERR_INTERNAL" , 35}, + {"Invalid coupling mode ERR_BAD_COUPLING_MODE" , 36}, + {"Invalid sensor sensitivity ERR_BAD_SENSOR_SENSITIVITY" , 37}, + {"Invalid IEPE mode ERR_BAD_IEPE_MODE" , 38}, + {"Invalid channel queue specified ERR_BAD_AI_CHAN_QUEUE" , 39}, + {"Invalid gain queue specified ERR_BAD_AI_GAIN_QUEUE" , 40}, + {"Invalid mode queue specified ERR_BAD_AI_MODE_QUEUE" , 41}, + {"FPGA file not found ERR_FPGA_FILE_NOT_FOUND" , 42}, + {"Unable to read FPGA file ERR_UNABLE_TO_READ_FPGA_FILE" , 43}, + {"FPGA not loaded ERR_NO_FPGA" , 44}, + {"Invalid argument ERR_BAD_ARG" , 45}, + {"Minimum slope value reached ERR_MIN_SLOPE_VAL_REACHED" , 46}, + {"Maximum slope value reached ERR_MAX_SLOPE_VAL_REACHED" , 47}, + {"Minimum offset value reached ERR_MIN_OFFSET_VAL_REACHED" , 48}, + {"Maximum offset value reached ERR_MAX_OFFSET_VAL_REACHED" , 49}, + {"Invalid port type specified ERR_BAD_PORT_TYPE" , 50}, + {"Digital I/O is configured incorrectly ERR_WRONG_DIG_CONFIG" , 51}, + {"Invalid bit number ERR_BAD_BIT_NUM" , 52}, + {"Invalid port value specified ERR_BAD_PORT_VAL" , 53}, + {"Invalid re-trigger count ERR_BAD_RETRIG_COUNT" , 54}, + {"Invalid analog output channel specified ERR_BAD_AO_CHAN" , 55}, + {"Invalid D/A output value specified ERR_BAD_DA_VAL" , 56}, + {"Invalid timer specified ERR_BAD_TMR" , 57}, + {"Invalid frequency specified ERR_BAD_FREQUENCY" , 58}, + {"Invalid duty cycle specified ERR_BAD_DUTY_CYCLE" , 59}, + {"Invalid initial delay specified ERR_BAD_INITIAL_DELAY" , 60}, + {"Invalid counter specified ERR_BAD_CTR" , 61}, + {"Invalid counter value specified ERR_BAD_CTR_VAL" , 62}, + {"Invalid DAQ input channel type specified ERR_BAD_DAQI_CHAN_TYPE" , 63}, + {"Invalid number of channels specified ERR_BAD_NUM_CHANS" , 64}, + {"Invalid counter register specified ERR_BAD_CTR_REG" , 65}, + {"Invalid counter measurement type specified ERR_BAD_CTR_MEASURE_TYPE" , 66}, + {"Invalid counter measurement mode specified ERR_BAD_CTR_MEASURE_MODE" , 67}, + {"Invalid debounce time specified ERR_BAD_DEBOUNCE_TIME" , 68}, + {"Invalid debounce mode specified ERR_BAD_DEBOUNCE_MODE" , 69}, + {"Invalid edge detection mode specified ERR_BAD_EDGE_DETECTION" , 70}, + {"Invalid tick size specified ERR_BAD_TICK_SIZE" , 71}, + {"Invalid DAQ output channel type specified ERR_BAD_DAQO_CHAN_TYPE" , 72}, + {"No connection established ERR_NO_CONNECTION_ESTABLISHED" , 73}, + {"Invalid event type specified ERR_BAD_EVENT_TYPE" , 74}, + {"An event handler has already been enabled for this event type ERR_EVENT_ALREADY_ENABLED" , 75}, + {"Invalid event parameter specified ERR_BAD_EVENT_PARAMETER" , 76}, + {"Invalid callback function specified ERR_BAD_CALLBACK_FUCNTION" , 77}, + {"Invalid memory address ERR_BAD_MEM_ADDRESS" , 78}, + {"Memory access denied ERR_MEM_ACCESS_DENIED" , 79}, + {"Device is not available at time of request ERR_DEV_UNAVAILABLE" , 80}, + {"Re-trigger option is not supported for the specified trigger type ERR_BAD_RETRIG_TRIG_TYPE" , 81}, + {"This function cannot be used with this version of the device ERR_BAD_DEV_VER" , 82}, + {"This digital operation is not supported on the specified port ERR_BAD_DIG_OPERATION" , 83}, + {"Invalid digital port index specified ERR_BAD_PORT_INDEX" , 84}, + {"Temperature input has open connection ERR_OPEN_CONNECTION" , 85}, + {"Device is not ready to send data ERR_DEV_NOT_READY" , 86}, + {"Pacer overrun, external clock rate too fast. ERR_PACER_OVERRUN" , 7}, + {"Invalid trigger channel specified ERR_BAD_TRIG_CHANNEL" , 88}, + {"Invalid trigger level specified ERR_BAD_TRIG_LEVEL" , 89}, + {"Invalid channel order ERR_BAD_CHAN_ORDER" , 90}, + {"Temperature input is out of range ERR_TEMP_OUT_OF_RANGE" , 91}, + {"Trigger threshold is out of range ERR_TRIG_THRESHOLD_OUT_OF_RANGE" , 92}, + {"Incompatible firmware version, firmware update required ERR_INCOMPATIBLE_FIRMWARE" , 93}, + {"Specified network interface is not available or disconnected ERR_BAD_NET_IFC" , 94}, + {"Invalid host specified ERR_BAD_NET_HOST" , 95}, + {"Invalid port specified ERR_BAD_NET_PORT" , 96}, + {"Network interface used to obtain the device descriptor not available or disconnected ERR_NET_IFC_UNAVAILABLE" , 97}, + {"Network connection failed ERR_NET_CONNECTION_FAILED" , 98}, + {"Invalid connection code ERR_BAD_CONNECTION_CODE" , 99}, + {"Connection code ignored ERR_CONNECTION_CODE_IGNORED" , 100}, + {"Network device already in use ERR_NET_DEV_IN_USE" , 101}, + {"Invalid network frame ERR_BAD_NET_FRAME" , 102}, + {"Network device did not respond within expected time ERR_NET_TIMEOUT" , 103}, + {"Data socket connection failed ERR_DATA_SOCKET_CONNECTION_FAILED" , 104}, + {"One or more bits on the specified port are used for alarm ERR_PORT_USED_FOR_ALARM" , 105}, + {"The specified bit is used for alarm ERR_BIT_USED_FOR_ALARM" , 106}, + {"Common-mode voltage range exceeded ERR_CMR_EXCEEDED" , 107}, + {"Network buffer overrun, data was not transferred from buffer fast enough ERR_NET_BUFFER_OVERRUN" , 108}, + {"Invalid network buffer ERR_BAD_NET_BUFFER" , 109} +}; + + static const struct { const char *name; AiInputMode mode; @@ -246,7 +363,7 @@ int uldaq_init(struct vnode *n) u->in.queues = nullptr; u->in.sample_rate = 1000; - u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS); + u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS | SO_EXTTIMEBASE); u->in.flags = AINSCAN_FF_DEFAULT; ret = pthread_mutex_init(&u->in.mutex, nullptr); @@ -363,8 +480,13 @@ int uldaq_parse(struct vnode *n, json_t *cfg) return ret; } - -char * uldaq_print(struct vnode *n) +void ul_decode_error(UlError err){ + for( uint i=0; i < sizeof(errorList) / sizeof(errorList[0]) ;i++){ + if(err&(1<_vd; @@ -547,7 +669,9 @@ int uldaq_start(struct vnode *n) /* Start the acquisition */ err = ulAInScan(u->device_handle, 0, 0, (AiInputMode) 0, (Range) 0, u->in.buffer_len / u->in.channel_count, &u->in.sample_rate, u->in.scan_options, u->in.flags, u->in.buffer); + if (err != ERR_NO_ERROR) { + ul_decode_error(err); warning("Failed to start acquisition on DAQ device for node '%s'", node_name(n)); return -1; } @@ -555,11 +679,13 @@ int uldaq_start(struct vnode *n) /* Get the initial status of the acquisition */ err = ulAInScanStatus(u->device_handle, &u->in.status, &u->in.transfer_status); if (err != ERR_NO_ERROR) { + ul_decode_error(err); warning("Failed to retrieve scan status on DAQ device for node '%s'", node_name(n)); return -1; } if (u->in.status != SS_RUNNING) { + ul_decode_error(err); warning("Acquisition did not start on DAQ device for node '%s'", node_name(n)); return -1; } @@ -640,6 +766,8 @@ int uldaq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r return cnt; } + + static struct plugin p; __attribute__((constructor(110))) From 6e0c7a9fe7c7c01a6ee88401ff6db64c11ff1b7f Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sat, 5 Sep 2020 18:42:12 +0000 Subject: [PATCH 13/46] add avg to pps time estimation --- lib/hooks/pps_ts.cpp | 49 +++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index f7038a4e1..3e9b2f1eb 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -38,8 +38,11 @@ class PpsTsHook : public Hook { protected: double lastValue; double thresh; - double realSmpRate; + double realSmpRateAvg; unsigned idx; + uint avg_length;//number of seconds over which the samplerate value will be averaged + uint avg_pos; + double *avg_array; int lastSeqNr; unsigned edgeCounter; @@ -50,8 +53,9 @@ public: Hook(p, n, fl, prio, en), lastValue(0), thresh(1.5), - realSmpRate(0), + realSmpRateAvg(0), idx(0), + avg_length(1), lastSeqNr(0), edgeCounter(0), realTime({ 0, 0 }) @@ -60,23 +64,28 @@ public: virtual void parse(json_t *cfg) { - int ret; - json_error_t err; + int ret; + json_error_t err; - assert(state != State::STARTED); + assert(state != State::STARTED); Hook::parse(cfg); - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f }", - "signal_index", &idx, - "threshold", &thresh - ); - if (ret) - throw ConfigError(cfg, err, "node-config-hook-pps_ts"); + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f, s: i}", + "signal_index", &idx, + "threshold", &thresh, + "avg_length", &avg_length + ); + if (ret) + throw ConfigError(cfg, err, "node-config-hook-pps_ts"); info("parsed config thresh=%f signal_index=%d", thresh, idx); - state = State::PARSED; + avg_array = new double[avg_length]; + avg_array = {0}; + + + state = State::PARSED; } virtual villas::node::Hook::Reason process(sample *smp) @@ -90,8 +99,12 @@ public: /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; if (isEdge) { - if (edgeCounter >= 1) - realSmpRate = seqNr - lastSeqNr; + if (edgeCounter >= 1){ + double currentSmpRate = (double)(seqNr - lastSeqNr); + + realSmpRateAvg += (currentSmpRate - avg_array[avg_pos % avg_length]) / avg_length; + avg_array[avg_pos % avg_length] = currentSmpRate; + } if (edgeCounter == 1) { auto now = time_now(); @@ -103,7 +116,7 @@ public: lastSeqNr = seqNr; - info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRate); + info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); edgeCounter++; } @@ -115,7 +128,7 @@ public: else if (edgeCounter == 2 && isEdge) realTime.tv_nsec = 0; else - realTime.tv_nsec += 1e9 / realSmpRate; + realTime.tv_nsec += 1e9 / realSmpRateAvg; if (realTime.tv_nsec >= 1000000000) { realTime.tv_sec++; @@ -128,6 +141,10 @@ public: return Hook::Reason::OK; } + + ~PpsTsHook(){ + delete avg_array; + } }; /* Register hook */ From 3a7ddb0f753f9e970b29dbc4c555616a94066a7b Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sat, 5 Sep 2020 21:15:28 +0200 Subject: [PATCH 14/46] add avg capabilities --- lib/hooks/pps_ts.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 3e9b2f1eb..e761e72c2 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -42,7 +42,7 @@ protected: unsigned idx; uint avg_length;//number of seconds over which the samplerate value will be averaged uint avg_pos; - double *avg_array; + double* avg_array; int lastSeqNr; unsigned edgeCounter; @@ -56,6 +56,7 @@ public: realSmpRateAvg(0), idx(0), avg_length(1), + avg_pos(0), lastSeqNr(0), edgeCounter(0), realTime({ 0, 0 }) @@ -82,7 +83,7 @@ public: info("parsed config thresh=%f signal_index=%d", thresh, idx); avg_array = new double[avg_length]; - avg_array = {0}; + //avg_array = {0}; state = State::PARSED; @@ -104,6 +105,7 @@ public: realSmpRateAvg += (currentSmpRate - avg_array[avg_pos % avg_length]) / avg_length; avg_array[avg_pos % avg_length] = currentSmpRate; + avg_pos++; } if (edgeCounter == 1) { auto now = time_now(); @@ -116,7 +118,7 @@ public: lastSeqNr = seqNr; - info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); + info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f.10", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); edgeCounter++; } From 3b0d89c525796417d648b7f8b1bf6ce748033e46 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 7 Sep 2020 19:45:25 +0200 Subject: [PATCH 15/46] reanable the average --- lib/hooks/pps_ts.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index e761e72c2..73348565a 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -83,7 +83,8 @@ public: info("parsed config thresh=%f signal_index=%d", thresh, idx); avg_array = new double[avg_length]; - //avg_array = {0}; + for ( uint i = 0; i < avg_length; i++) + avg_array[i] = 0; state = State::PARSED; @@ -102,7 +103,6 @@ public: if (isEdge) { if (edgeCounter >= 1){ double currentSmpRate = (double)(seqNr - lastSeqNr); - realSmpRateAvg += (currentSmpRate - avg_array[avg_pos % avg_length]) / avg_length; avg_array[avg_pos % avg_length] = currentSmpRate; avg_pos++; @@ -118,7 +118,7 @@ public: lastSeqNr = seqNr; - info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f.10", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); + info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); edgeCounter++; } From 18b6797e7d0343730a4c48713b084ba0f1b31070 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 7 Sep 2020 19:46:21 +0200 Subject: [PATCH 16/46] enable external trigger and fix readable uldaq errors --- lib/nodes/uldaq.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index d73314c36..0c9ff89d3 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -363,7 +363,7 @@ int uldaq_init(struct vnode *n) u->in.queues = nullptr; u->in.sample_rate = 1000; - u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS | SO_EXTTIMEBASE); + u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS | SO_EXTCLOCK); u->in.flags = AINSCAN_FF_DEFAULT; ret = pthread_mutex_init(&u->in.mutex, nullptr); @@ -481,7 +481,7 @@ int uldaq_parse(struct vnode *n, json_t *cfg) return ret; } void ul_decode_error(UlError err){ - for( uint i=0; i < sizeof(errorList) / sizeof(errorList[0]) ;i++){ + for( uint i=0; i < ARRAY_LEN(errorList) ;i++){ if(err&(1< Date: Thu, 10 Sep 2020 14:53:50 +0200 Subject: [PATCH 17/46] first non working version of pll pps_ts sync --- lib/hooks/pps_ts.cpp | 68 +++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 73348565a..ad76048e0 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -38,28 +38,26 @@ class PpsTsHook : public Hook { protected: double lastValue; double thresh; - double realSmpRateAvg; + double realSmpRate; unsigned idx; - uint avg_length;//number of seconds over which the samplerate value will be averaged - uint avg_pos; - double* avg_array; int lastSeqNr; unsigned edgeCounter; - + double pll_gain; timespec realTime; + uint64_t last_sequence; public: PpsTsHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), lastValue(0), thresh(1.5), - realSmpRateAvg(0), + realSmpRate(0), idx(0), - avg_length(1), - avg_pos(0), lastSeqNr(0), edgeCounter(0), - realTime({ 0, 0 }) + pll_gain(1), + realTime({ 0, 0 }), + last_sequence(0) { } @@ -72,20 +70,16 @@ public: Hook::parse(cfg); - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f, s: i}", + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f, s?: f}", "signal_index", &idx, "threshold", &thresh, - "avg_length", &avg_length + "pll_gain", &pll_gain ); if (ret) throw ConfigError(cfg, err, "node-config-hook-pps_ts"); info("parsed config thresh=%f signal_index=%d", thresh, idx); - avg_array = new double[avg_length]; - for ( uint i = 0; i < avg_length; i++) - avg_array[i] = 0; - state = State::PARSED; } @@ -100,13 +94,8 @@ public: /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; + if (isEdge) { - if (edgeCounter >= 1){ - double currentSmpRate = (double)(seqNr - lastSeqNr); - realSmpRateAvg += (currentSmpRate - avg_array[avg_pos % avg_length]) / avg_length; - avg_array[avg_pos % avg_length] = currentSmpRate; - avg_pos++; - } if (edgeCounter == 1) { auto now = time_now(); @@ -118,34 +107,61 @@ public: lastSeqNr = seqNr; - info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%ld, smpRate=%f", seqNr, realTime.tv_sec, realTime.tv_nsec, realSmpRateAvg); edgeCounter++; } lastValue = value; + double timeErr; + double changeVal; + if(isEdge){ + timeErr = ( 1e9 - realTime.tv_nsec); + changeVal = 0; + if (edgeCounter >= 1){//pll mode + double targetVal = 0.95e9; + changeVal = pll_gain * abs(targetVal - timeErr); + changeVal = pll_gain; + if(timeErr > targetVal) + realSmpRate -= changeVal; + else if(timeErr < targetVal){ + realSmpRate += changeVal; + } + if(realSmpRate > 10010)realSmpRate = 10010; + if(realSmpRate < 9990)realSmpRate = 9990; + } + //info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%f, smpRate=%f, pll_gain=%f", seqNr, realTime.tv_sec, (1e9 - realTime.tv_nsec + 1e9/realSmpRate), realSmpRate, pll_gain); + } + if (edgeCounter < 2) return Hook::Reason::SKIP_SAMPLE; else if (edgeCounter == 2 && isEdge) realTime.tv_nsec = 0; else - realTime.tv_nsec += 1e9 / realSmpRateAvg; + realTime.tv_nsec += 1e9 / realSmpRate; - if (realTime.tv_nsec >= 1000000000) { + if (realTime.tv_nsec >= 1e9) { realTime.tv_sec++; - realTime.tv_nsec -= 1000000000; + realTime.tv_nsec -= 1e9; + } + + if(isEdge){ + info("Edge detected: seq=%u, realTime.nsec=%ld, timeErr=%f , smpRate=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, realSmpRate, changeVal); } /* Update timestamp */ smp->ts.origin = realTime; smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; + if((smp->sequence - last_sequence) > 1 ) + warning("Calculation is not Realtime. %li sampled missed",smp->sequence - last_sequence); + + last_sequence = smp->sequence; + return Hook::Reason::OK; } ~PpsTsHook(){ - delete avg_array; } }; From 8b558a97db41b539b3feadba0f8238557d076771 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Fri, 11 Sep 2020 18:44:47 +0200 Subject: [PATCH 18/46] update wit niklas --- lib/hooks/pps_ts.cpp | 48 ++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index ad76048e0..4593ec53b 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -40,7 +40,7 @@ protected: double thresh; double realSmpRate; unsigned idx; - int lastSeqNr; + uint64_t lastSeqNr; unsigned edgeCounter; double pll_gain; timespec realTime; @@ -90,55 +90,51 @@ public: /* Get value of PPS signal */ float value = smp->data[idx].f; // TODO check if it is really float - int seqNr = smp->sequence; + uint64_t seqNr = smp->sequence; /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; + lastValue = value; + + + double timeErr; + double changeVal; + static const double targetVal = 0.5e9; if (isEdge) { - if (edgeCounter == 1) { + if (edgeCounter == 2) { auto now = time_now(); if (now.tv_nsec >= 0.5e9) realTime.tv_sec = now.tv_sec + 1; else realTime.tv_sec = now.tv_sec; + realTime.tv_nsec = targetVal; } lastSeqNr = seqNr; - - edgeCounter++; - } - lastValue = value; - - double timeErr; - double changeVal; - if(isEdge){ - timeErr = ( 1e9 - realTime.tv_nsec); - changeVal = 0; - if (edgeCounter >= 1){//pll mode - double targetVal = 0.95e9; - changeVal = pll_gain * abs(targetVal - timeErr); - changeVal = pll_gain; - if(timeErr > targetVal) + timeErr = ( targetVal - realTime.tv_nsec); + if (edgeCounter >= 2){//pll mode + + changeVal = pll_gain * -1 * timeErr; + //changeVal = pll_gain; + /*if(timeErr > 0 ) realSmpRate -= changeVal; - else if(timeErr < targetVal){ + else if(timeErr < 0){*/ realSmpRate += changeVal; - } - if(realSmpRate > 10010)realSmpRate = 10010; - if(realSmpRate < 9990)realSmpRate = 9990; + //} + if(realSmpRate > 10100.)realSmpRate = 10100; + if(realSmpRate < 9900.)realSmpRate = 9900; } //info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%f, smpRate=%f, pll_gain=%f", seqNr, realTime.tv_sec, (1e9 - realTime.tv_nsec + 1e9/realSmpRate), realSmpRate, pll_gain); } if (edgeCounter < 2) return Hook::Reason::SKIP_SAMPLE; - else if (edgeCounter == 2 && isEdge) - realTime.tv_nsec = 0; else - realTime.tv_nsec += 1e9 / realSmpRate; + realTime.tv_nsec += 1.e9 / realSmpRate; if (realTime.tv_nsec >= 1e9) { realTime.tv_sec++; @@ -146,7 +142,7 @@ public: } if(isEdge){ - info("Edge detected: seq=%u, realTime.nsec=%ld, timeErr=%f , smpRate=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, realSmpRate, changeVal); + info("Edge detected: seq=%lu, realTime.nsec=%ld, timeErr=%f , smpRate=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, realSmpRate, changeVal); } /* Update timestamp */ From b628081b233a9c23e44af721fe539faaecddc6c6 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 10 Sep 2020 14:57:45 +0200 Subject: [PATCH 19/46] updated dft algorithm --- lib/hooks/dft.cpp | 62 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index a84ea76f3..a9ff85f1a 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -59,6 +59,7 @@ protected: struct format_type *format; double* smp_memory; + double* pps_memory; std::complex** dftMatrix; std::complex* dftResults; double* filterWindowCoefficents; @@ -149,6 +150,13 @@ public: smp_memory[i] = 0; + pps_memory = new double[window_size]; + if (!pps_memory) + throw MemoryAllocationError(); + + for(uint i = 0; i < window_size; i++) + pps_memory[i] = 0; + //init matrix of dft coeffients dftMatrix = new std::complex*[freq_count]; @@ -268,15 +276,16 @@ public: assert(state == State::STARTED); smp_memory[smp_mem_pos % window_size] = smp->data[0].f; + pps_memory[smp_mem_pos % window_size] = smp->data[1].f; smp_mem_pos ++ ; - - double timediff = 0; + bool runDft = false; if( sync_dft ){ - timediff = ( smp->ts.origin.tv_sec - last_dft_cal.tv_sec ) + ( smp->ts.origin.tv_nsec - last_dft_cal.tv_nsec ) * 1e-9; + if( last_dft_cal.tv_sec != smp->ts.origin.tv_sec ) + runDft = true; } - - if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || (sync_dft && timediff >= ( 1 / dft_rate ) ) ) { + last_dft_cal = smp->ts.origin; + if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || ( runDft ) ) { calcDft(paddingType::ZERO); double maxF = 0; double maxA = 0; @@ -292,10 +301,13 @@ public: } info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); + if(smp_mem_pos>100e3){ + appendDumpData("/tmp/plot/phaseOut",atan2(dftResults[maxPos].imag(), dftResults[maxPos].real())); + appendDumpData("/tmp/plot/voltageOut",(maxA / pow(2,1./2))); + appendDumpData("/tmp/plot/freqOut",maxF); + } - dumpData("/tmp/absDftResults", absDftResults, freq_count, absDftFreqs); - - last_dft_cal = smp->ts.origin; + dumpData("/tmp/plot/absDftResults", absDftResults, freq_count, absDftFreqs); } @@ -312,6 +324,16 @@ public: //delete smp_memory; } + void appendDumpData(const char *path, double ydata){ + std::ofstream fh; + fh.open(path,std::ios_base::app); + if(fh.tellp()>0) + fh << ";"; + fh << ydata; + + fh.close(); + } + void dumpData(const char *path, double *ydata, uint size, double *xdata=nullptr){ std::ofstream fh; @@ -350,17 +372,31 @@ public: //prepare sample window The following parts can be combined double tmp_smp_window[window_size]; + double tmp_smp_pps[window_size]; + + uint lastEdge = 0; + uint edgeCount = 0; for(uint i = 0; i< window_size; i++){ - tmp_smp_window[i] = smp_memory[( i + smp_mem_pos ) % window_size]; + tmp_smp_window[i] = smp_memory[( i + smp_mem_pos) % window_size]; + tmp_smp_pps[i] = pps_memory[( i + smp_mem_pos) % window_size]; + if(edgeCount == 0 || (lastEdge + 1500) < i){ + if(tmp_smp_pps[i] > 2. && i > 5000){ + lastEdge = i; + info("edge detected %i",lastEdge); + edgeCount++; + appendDumpData("/tmp/plot/ppsAmplitude",tmp_smp_window[i]); + } + } } - dumpData("/tmp/signal_original",tmp_smp_window,window_size); + dumpData("/tmp/plot/pps_original",tmp_smp_pps,window_size); + dumpData("/tmp/plot/signal_original",tmp_smp_window,window_size); for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] *= filterWindowCoefficents[i]; } - dumpData("/tmp/signal_windowed",tmp_smp_window,window_size); + dumpData("/tmp/plot/signal_windowed",tmp_smp_window,window_size); - dumpData("/tmp/smp_window",smp_memory,window_size); + dumpData("/tmp/plot/smp_window",smp_memory,window_size); for( uint i=0; i < freq_count; i++){ dftResults[i] = 0; @@ -407,7 +443,7 @@ public: } } window_corretion_factor /= window_size; - dumpData("/tmp/filter_window",filterWindowCoefficents,window_size); + dumpData("/tmp/plot/filter_window",filterWindowCoefficents,window_size); } }; From 34d887617185f4598051acf18580c7ef11b11cde Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 14 Sep 2020 16:03:36 +0200 Subject: [PATCH 20/46] next non working sync --- lib/hooks/pps_ts.cpp | 64 +++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 4593ec53b..bfbd2bec2 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -38,26 +38,28 @@ class PpsTsHook : public Hook { protected: double lastValue; double thresh; - double realSmpRate; + double periodTime;//Period of sample frequency (Villas samples) unsigned idx; uint64_t lastSeqNr; unsigned edgeCounter; double pll_gain; timespec realTime; uint64_t last_sequence; + double y; public: PpsTsHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), lastValue(0), thresh(1.5), - realSmpRate(0), + periodTime(1), idx(0), lastSeqNr(0), edgeCounter(0), pll_gain(1), realTime({ 0, 0 }), - last_sequence(0) + last_sequence(0), + y(0) { } @@ -99,7 +101,16 @@ public: double timeErr; double changeVal; - static const double targetVal = 0.5e9; + static const double targetVal = 0.5; + + + realTime.tv_nsec += periodTime * 1e9 / 10000; + + if (realTime.tv_nsec >= 1e9) { + realTime.tv_sec++; + realTime.tv_nsec -= 1e9; + } + if (isEdge) { if (edgeCounter == 2) { @@ -109,48 +120,53 @@ public: realTime.tv_sec = now.tv_sec + 1; else realTime.tv_sec = now.tv_sec; - realTime.tv_nsec = targetVal; + realTime.tv_nsec = targetVal * 1e9; } lastSeqNr = seqNr; edgeCounter++; - timeErr = ( targetVal - realTime.tv_nsec); + timeErr = ( targetVal - (realTime.tv_nsec / 1e9) ); if (edgeCounter >= 2){//pll mode - changeVal = pll_gain * -1 * timeErr; + changeVal = pll_gain * timeErr; + + double k2 = 1.; + double p = 0.99; + + y = p * timeErr + ( 1 - p) * y; + changeVal -= k2 * y; + //changeVal = pll_gain; /*if(timeErr > 0 ) - realSmpRate -= changeVal; + periodTime -= changeVal; else if(timeErr < 0){*/ - realSmpRate += changeVal; + periodTime += changeVal; //} - if(realSmpRate > 10100.)realSmpRate = 10100; - if(realSmpRate < 9900.)realSmpRate = 9900; + double nominalSmpRate = 1; + if(periodTime > 1.1/nominalSmpRate)periodTime = 1.1/nominalSmpRate; + if(periodTime < 0.9/nominalSmpRate)periodTime = 0.9/nominalSmpRate; } - //info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%f, smpRate=%f, pll_gain=%f", seqNr, realTime.tv_sec, (1e9 - realTime.tv_nsec + 1e9/realSmpRate), realSmpRate, pll_gain); + //info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%f, smpRate=%f, pll_gain=%f", seqNr, realTime.tv_sec, (1e9 - realTime.tv_nsec + 1e9/periodTime), periodTime, pll_gain); + } + + + + + + if(isEdge){ + info("Edge detected: seq=%lu, realTime.nsec=%lu, timeErr=%f , timePeriod=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, periodTime, changeVal); } if (edgeCounter < 2) return Hook::Reason::SKIP_SAMPLE; - else - realTime.tv_nsec += 1.e9 / realSmpRate; - - if (realTime.tv_nsec >= 1e9) { - realTime.tv_sec++; - realTime.tv_nsec -= 1e9; - } - - if(isEdge){ - info("Edge detected: seq=%lu, realTime.nsec=%ld, timeErr=%f , smpRate=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, realSmpRate, changeVal); - } /* Update timestamp */ smp->ts.origin = realTime; smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; if((smp->sequence - last_sequence) > 1 ) - warning("Calculation is not Realtime. %li sampled missed",smp->sequence - last_sequence); + warning("Samples missed: %li sampled missed", smp->sequence - last_sequence); last_sequence = smp->sequence; From ea86b55924ee3428400b018e87acfaa6c7570fff Mon Sep 17 00:00:00 2001 From: Niklas Eiling Date: Mon, 14 Sep 2020 16:11:22 +0200 Subject: [PATCH 21/46] add TimeSync class to lib/hooks/pps_ts.cpp --- lib/hooks/pps_ts.cpp | 140 +++++++++++++++++++++++++++++++------------ 1 file changed, 102 insertions(+), 38 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index bfbd2bec2..5d387cbfd 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -33,6 +33,100 @@ namespace villas { namespace node { +class TimeSync { +protected: + //Algorithms stability parameters + //Examples from Paper: + //const double p = 0.99; + //const double k1 = 1.1; + //const double k2 = 1.0; + //Tuned parameters: + const double p = 1.2; + const double k1 = 1.3; + const double k2 = 0.9; + const double c = 0.7; // ~= mü_max + const double tau = 1.; //Time between synchronizations + //We assume no skew of our internal clock + const double r = 1.; + + //Time estimate + uint64_t x; + //Skew correction + double s; + //Dampening Factor + double y; +public: + TimeSync() : x(0), s(1.), y(0.) + { + //Check stability + assert( tau < (p*(k2-p*(k1-k2)))/ + (c*(k1-p*(k1-k2))*(k1-p*(k1-k2))) ); + assert( 2*k1/3*p > k1-k2 ); + assert( k1-k2 > 0 ); + assert( 2 > p && p > 0 ); + } + + /** Returns the current time Error + * + * @actualTime The actual time from an external clock + * @nanoseconds the nanoseconds since the last synchronization event + */ + int64_t timeError(const uint64_t actualTime, const uint64_t nanoseconds) + { + uint64_t currentEstimate = timeFromNanoseconds(nanoseconds); + return (int64_t)actualTime + (int64_t)tau*1e9 - (int64_t)currentEstimate; + } + + /** synchronizes with external clock + * + * @actualTime nanoseconds at which event should occur + * @return nanoseconds estimate + */ + uint64_t synchronize(const uint64_t actualTime, const uint64_t nanoseconds) + { + double t_err; + uint64_t ret; + //Time estimate + x = timeFromNanoseconds(nanoseconds); + //Time Error + t_err = ((int64_t)actualTime + (int64_t)tau*1e9 - (int64_t)x)/1e9; + //Skew correction or internal frequency / external frequency + s = s + k1*(t_err) - k2*y; + //Dampening Factor + y = p*(t_err) + (1-p)*y; + + //Clip skew correction + if (s > 1.1) { + s = 1.1; + } else if (s < 0.9) { + s = 0.9; + } + printf("time error: %f, x: %lu, s: %f, y: %f\n", t_err, x, s, y); + + ret = x; + //If time estimate gets too large, reset x + if ( x > tau*1e9) { + x -= tau*1e9; + } + return ret; + } + + uint64_t time(const double partOfFullSecond) + { + return x + partOfFullSecond*tau*r*s*1e9; + } + + /** Returns time estimate based on the nanoseconds that passed since + * last synchrsonization + * + * @return nanoesecond estimate + */ + uint64_t timeFromNanoseconds(const uint64_t n) + { + return time(n / 1.e9); + } +}; + class PpsTsHook : public Hook { protected: @@ -46,6 +140,7 @@ protected: timespec realTime; uint64_t last_sequence; double y; + TimeSync ts; public: PpsTsHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : @@ -59,7 +154,8 @@ public: pll_gain(1), realTime({ 0, 0 }), last_sequence(0), - y(0) + y(0), + ts() { } @@ -103,8 +199,8 @@ public: double changeVal; static const double targetVal = 0.5; - - realTime.tv_nsec += periodTime * 1e9 / 10000; + auto now = time_now(); + realTime.tv_nsec = ts.timeFromNanoseconds(now.tv_nsec); if (realTime.tv_nsec >= 1e9) { realTime.tv_sec++; @@ -113,47 +209,15 @@ public: if (isEdge) { - if (edgeCounter == 2) { - auto now = time_now(); - - if (now.tv_nsec >= 0.5e9) - realTime.tv_sec = now.tv_sec + 1; - else - realTime.tv_sec = now.tv_sec; - realTime.tv_nsec = targetVal * 1e9; - } - - lastSeqNr = seqNr; edgeCounter++; - - timeErr = ( targetVal - (realTime.tv_nsec / 1e9) ); - if (edgeCounter >= 2){//pll mode - - changeVal = pll_gain * timeErr; - - double k2 = 1.; - double p = 0.99; - - y = p * timeErr + ( 1 - p) * y; - changeVal -= k2 * y; - - //changeVal = pll_gain; - /*if(timeErr > 0 ) - periodTime -= changeVal; - else if(timeErr < 0){*/ - periodTime += changeVal; - //} - double nominalSmpRate = 1; - if(periodTime > 1.1/nominalSmpRate)periodTime = 1.1/nominalSmpRate; - if(periodTime < 0.9/nominalSmpRate)periodTime = 0.9/nominalSmpRate; - } - //info("Edge detected: seq=%u, realTime.sec=%ld, realTime.nsec=%f, smpRate=%f, pll_gain=%f", seqNr, realTime.tv_sec, (1e9 - realTime.tv_nsec + 1e9/periodTime), periodTime, pll_gain); + currentNanoseconds = now.tv_nsec; + ts.synchronize(0.5e9, currentNanoseconds); } - + if(isEdge){ info("Edge detected: seq=%lu, realTime.nsec=%lu, timeErr=%f , timePeriod=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, periodTime, changeVal); } From 3b2b8de658a72bc6dab798315c03cb840eee903d Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 14 Sep 2020 16:38:15 +0200 Subject: [PATCH 22/46] fix integration --- lib/hooks/pps_ts.cpp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 5d387cbfd..dddb56da9 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -56,7 +56,7 @@ protected: //Dampening Factor double y; public: - TimeSync() : x(0), s(1.), y(0.) + TimeSync() : x(0.5e9), s(1.), y(0.) { //Check stability assert( tau < (p*(k2-p*(k1-k2)))/ @@ -101,7 +101,7 @@ public: } else if (s < 0.9) { s = 0.9; } - printf("time error: %f, x: %lu, s: %f, y: %f\n", t_err, x, s, y); + printf("time error: %f, nanoseconds: %lu, x: %ld, s: %f, y: %f\n", t_err, nanoseconds, x, s, y); ret = x; //If time estimate gets too large, reset x @@ -138,6 +138,7 @@ protected: unsigned edgeCounter; double pll_gain; timespec realTime; + timespec lastEdge; uint64_t last_sequence; double y; TimeSync ts; @@ -153,6 +154,7 @@ public: edgeCounter(0), pll_gain(1), realTime({ 0, 0 }), + lastEdge({0,0}), last_sequence(0), y(0), ts() @@ -188,39 +190,41 @@ public: /* Get value of PPS signal */ float value = smp->data[idx].f; // TODO check if it is really float - uint64_t seqNr = smp->sequence; + //uint64_t seqNr = smp->sequence; /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; lastValue = value; - double timeErr; + /*double timeErr; double changeVal; - static const double targetVal = 0.5; + //static const double targetVal = 0.5;*/ auto now = time_now(); - realTime.tv_nsec = ts.timeFromNanoseconds(now.tv_nsec); + uint64_t timediff = (now.tv_nsec - lastEdge.tv_nsec)+(now.tv_sec - lastEdge.tv_sec)*1e9; + + realTime.tv_nsec = ts.timeFromNanoseconds(timediff); if (realTime.tv_nsec >= 1e9) { realTime.tv_sec++; realTime.tv_nsec -= 1e9; } - if (isEdge) { edgeCounter++; - currentNanoseconds = now.tv_nsec; - ts.synchronize(0.5e9, currentNanoseconds); + ts.synchronize(0.5e9, timediff); + lastEdge = now; } - if(isEdge){ + /*if(isEdge){ info("Edge detected: seq=%lu, realTime.nsec=%lu, timeErr=%f , timePeriod=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, periodTime, changeVal); - } + }*/ + if (edgeCounter < 2) return Hook::Reason::SKIP_SAMPLE; From ca36c12fc75665999636ba77557dbe8bdd809a26 Mon Sep 17 00:00:00 2001 From: Niklas Eiling Date: Tue, 15 Sep 2020 15:06:10 +0200 Subject: [PATCH 23/46] fix wrong full second calculation and cleanup lib/hooks/pps_ts.cpp --- lib/hooks/pps_ts.cpp | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index dddb56da9..0fb5e3046 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -86,14 +86,14 @@ public: { double t_err; uint64_t ret; - //Time estimate - x = timeFromNanoseconds(nanoseconds); + //Time estimate + x = timeFromNanoseconds(nanoseconds); //Time Error t_err = ((int64_t)actualTime + (int64_t)tau*1e9 - (int64_t)x)/1e9; - //Skew correction or internal frequency / external frequency - s = s + k1*(t_err) - k2*y; - //Dampening Factor - y = p*(t_err) + (1-p)*y; + //Skew correction or internal frequency / external frequency + s = s + k1*(t_err) - k2*y; + //Dampening Factor + y = p*(t_err) + (1-p)*y; //Clip skew correction if (s > 1.1) { @@ -132,32 +132,28 @@ class PpsTsHook : public Hook { protected: double lastValue; double thresh; - double periodTime;//Period of sample frequency (Villas samples) unsigned idx; uint64_t lastSeqNr; unsigned edgeCounter; - double pll_gain; timespec realTime; timespec lastEdge; uint64_t last_sequence; - double y; TimeSync ts; + bool converged; public: PpsTsHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), lastValue(0), thresh(1.5), - periodTime(1), idx(0), lastSeqNr(0), edgeCounter(0), - pll_gain(1), realTime({ 0, 0 }), lastEdge({0,0}), last_sequence(0), - y(0), - ts() + ts(), + converged(false) { } @@ -188,44 +184,45 @@ public: { assert(state == State::STARTED); + static const uint64_t targetVal = 0.5e9; /* Get value of PPS signal */ float value = smp->data[idx].f; // TODO check if it is really float //uint64_t seqNr = smp->sequence; /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; + lastValue = value; - /*double timeErr; - double changeVal; - //static const double targetVal = 0.5;*/ - auto now = time_now(); uint64_t timediff = (now.tv_nsec - lastEdge.tv_nsec)+(now.tv_sec - lastEdge.tv_sec)*1e9; + // nsec is between 0.5e9 and 1.5e9 when converged realTime.tv_nsec = ts.timeFromNanoseconds(timediff); + realTime.tv_sec = lastEdge.tv_sec; - if (realTime.tv_nsec >= 1e9) { + // when not converged, nsec might be really high. + // using a loop assures no time jumps. + //TODO: can nsec also become negative? + while (realTime.tv_nsec >= 1e9) { realTime.tv_sec++; realTime.tv_nsec -= 1e9; } if (isEdge) { edgeCounter++; - ts.synchronize(0.5e9, timediff); + ts.synchronize(targetVal, timediff); lastEdge = now; + converged = abs(ts.timeError(targetVal, timediff)) < 0.001; } - - - - /*if(isEdge){ info("Edge detected: seq=%lu, realTime.nsec=%lu, timeErr=%f , timePeriod=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, periodTime, changeVal); }*/ + //if (!converged || edgeCounter < 2) if (edgeCounter < 2) return Hook::Reason::SKIP_SAMPLE; From 58286b7d5a242f2451c47302d3d96849d3fd3e45 Mon Sep 17 00:00:00 2001 From: Niklas Eiling Date: Tue, 15 Sep 2020 15:09:56 +0200 Subject: [PATCH 24/46] fix reading of non-existent config parameter --- lib/hooks/pps_ts.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 0fb5e3046..6a50a931d 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -166,10 +166,9 @@ public: Hook::parse(cfg); - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f, s?: f}", + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f}", "signal_index", &idx, - "threshold", &thresh, - "pll_gain", &pll_gain + "threshold", &thresh ); if (ret) throw ConfigError(cfg, err, "node-config-hook-pps_ts"); From 92482f4fda63d49184048c9b3f8de2a6d120fc7b Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 7 Oct 2020 15:25:47 +0200 Subject: [PATCH 25/46] use internal sampel clock of uldaq hardware --- lib/nodes/uldaq.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index 0c9ff89d3..37b5cfbd7 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -363,7 +363,7 @@ int uldaq_init(struct vnode *n) u->in.queues = nullptr; u->in.sample_rate = 1000; - u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS | SO_EXTCLOCK); + u->in.scan_options = (ScanOption) (SO_DEFAULTIO | SO_CONTINUOUS); u->in.flags = AINSCAN_FF_DEFAULT; ret = pthread_mutex_init(&u->in.mutex, nullptr); From bb16b07a695c48add7206391d069eb33e90a1535 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 7 Oct 2020 15:52:43 +0200 Subject: [PATCH 26/46] finish rebase master --- lib/hooks/dft.cpp | 2 +- lib/nodes/uldaq.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index a9ff85f1a..aedcf1aba 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -90,7 +90,7 @@ protected: public: - DftHook(struct vpath *p, struct node *n, int fl, int prio, bool en = true) : + DftHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), window_type(windowType::NONE), padding_type(paddingType::ZERO), diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index 37b5cfbd7..8e6ce46f8 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -486,7 +486,7 @@ void ul_decode_error(UlError err){ warning("Found error: %s",errorList[i].errStr); } } -char * uldaq_print(struct node *n) +char * uldaq_print(struct vnode *n) { struct uldaq *u = (struct uldaq *) n->_vd; From 3c06705110c3d3b2b0d0daec08431de0f5762c72 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 8 Oct 2020 12:30:20 +0200 Subject: [PATCH 27/46] fix correct selection of uldaq hardware --- lib/nodes/uldaq.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index 8e6ce46f8..178f824f9 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -222,6 +222,12 @@ static const struct { { "unipolar-0.01", UNIPT01VOLTS, 0.0, +0.01 }, { "unipolar-0.005", UNIPT005VOLTS, 0.0, +0.005 } }; +static void ul_decode_error(UlError err){ + for( uint i=0; i < ARRAY_LEN(errorList) ;i++){ + if(err&(1<device_handle) { - u->device_handle = ulCreateDaqDevice(descriptors[0]); + u->device_handle = ulCreateDaqDevice(*u->device_descriptor); if (!u->device_handle) { warning("Unable to create handle for DAQ device for node '%s'", node_name(n)); return -1; @@ -325,6 +331,7 @@ static int uldaq_connect(struct vnode *n) err = ulConnectDaqDevice(u->device_handle); if (err != ERR_NO_ERROR) { warning("Failed to connect to DAQ device for node '%s'", node_name(n)); + ul_decode_error(err); return -1; } } @@ -480,12 +487,7 @@ int uldaq_parse(struct vnode *n, json_t *cfg) return ret; } -void ul_decode_error(UlError err){ - for( uint i=0; i < ARRAY_LEN(errorList) ;i++){ - if(err&(1<_vd; From 27f4a623debacc9f3a93fd94d1dc7fcb66c36028 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 8 Oct 2020 12:31:20 +0200 Subject: [PATCH 28/46] new approach from steffen and manuel to pps sync --- lib/hooks/pps_ts.cpp | 213 ++++++++++++++----------------------------- 1 file changed, 69 insertions(+), 144 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 6a50a931d..6e35f060b 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -33,113 +33,28 @@ namespace villas { namespace node { -class TimeSync { -protected: - //Algorithms stability parameters - //Examples from Paper: - //const double p = 0.99; - //const double k1 = 1.1; - //const double k2 = 1.0; - //Tuned parameters: - const double p = 1.2; - const double k1 = 1.3; - const double k2 = 0.9; - const double c = 0.7; // ~= mü_max - const double tau = 1.; //Time between synchronizations - //We assume no skew of our internal clock - const double r = 1.; - - //Time estimate - uint64_t x; - //Skew correction - double s; - //Dampening Factor - double y; -public: - TimeSync() : x(0.5e9), s(1.), y(0.) - { - //Check stability - assert( tau < (p*(k2-p*(k1-k2)))/ - (c*(k1-p*(k1-k2))*(k1-p*(k1-k2))) ); - assert( 2*k1/3*p > k1-k2 ); - assert( k1-k2 > 0 ); - assert( 2 > p && p > 0 ); - } - - /** Returns the current time Error - * - * @actualTime The actual time from an external clock - * @nanoseconds the nanoseconds since the last synchronization event - */ - int64_t timeError(const uint64_t actualTime, const uint64_t nanoseconds) - { - uint64_t currentEstimate = timeFromNanoseconds(nanoseconds); - return (int64_t)actualTime + (int64_t)tau*1e9 - (int64_t)currentEstimate; - } - - /** synchronizes with external clock - * - * @actualTime nanoseconds at which event should occur - * @return nanoseconds estimate - */ - uint64_t synchronize(const uint64_t actualTime, const uint64_t nanoseconds) - { - double t_err; - uint64_t ret; - //Time estimate - x = timeFromNanoseconds(nanoseconds); - //Time Error - t_err = ((int64_t)actualTime + (int64_t)tau*1e9 - (int64_t)x)/1e9; - //Skew correction or internal frequency / external frequency - s = s + k1*(t_err) - k2*y; - //Dampening Factor - y = p*(t_err) + (1-p)*y; - - //Clip skew correction - if (s > 1.1) { - s = 1.1; - } else if (s < 0.9) { - s = 0.9; - } - printf("time error: %f, nanoseconds: %lu, x: %ld, s: %f, y: %f\n", t_err, nanoseconds, x, s, y); - - ret = x; - //If time estimate gets too large, reset x - if ( x > tau*1e9) { - x -= tau*1e9; - } - return ret; - } - - uint64_t time(const double partOfFullSecond) - { - return x + partOfFullSecond*tau*r*s*1e9; - } - - /** Returns time estimate based on the nanoseconds that passed since - * last synchrsonization - * - * @return nanoesecond estimate - */ - uint64_t timeFromNanoseconds(const uint64_t n) - { - return time(n / 1.e9); - } -}; - class PpsTsHook : public Hook { protected: double lastValue; double thresh; unsigned idx; - uint64_t lastSeqNr; - unsigned edgeCounter; - timespec realTime; - timespec lastEdge; uint64_t last_sequence; - TimeSync ts; - bool converged; + + bool isSynced; + bool isLocked; + struct timespec tsVirt; + double timeErr; // in seconds + double periodEst; // in seconds + double periodErrComp; // in seconds + double period; // in seconds + uintmax_t cntEdges; + uintmax_t cntSmps; + uintmax_t cntSmpsTotal; + unsigned horizonComp; + unsigned horizonEst; + uintmax_t filtLen; + uintmax_t *filtWin; public: PpsTsHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : @@ -147,14 +62,23 @@ public: lastValue(0), thresh(1.5), idx(0), - lastSeqNr(0), - edgeCounter(0), - realTime({ 0, 0 }), - lastEdge({0,0}), last_sequence(0), - ts(), - converged(false) + isSynced(false), + isLocked(false), + timeErr(0.0), + period(0.0), + cntEdges(0), + horizonComp(1), + horizonEst(1), + filtLen(horizonEst + 1) { + filtWin = new uintmax_t[filtLen]; + memset(filtWin, 0, filtLen * sizeof(uintmax_t)); + } + + ~PpsTsHook() + { + delete[] filtWin; } virtual void parse(json_t *cfg) @@ -166,14 +90,18 @@ public: Hook::parse(cfg); - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f}", + double fSmps = 0; + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: f, s: F}", "signal_index", &idx, - "threshold", &thresh + "threshold", &thresh, + "expected_smp_rate", &fSmps ); if (ret) - throw ConfigError(cfg, err, "node-config-hook-pps_ts"); + throw ConfigError(cfg, err, "node-config-hook-pps_ts"); - info("parsed config thresh=%f signal_index=%d", thresh, idx); + period = 1.0 / fSmps; + + info("parsed config thresh=%f signal_index=%d nominal_period=%f", thresh, idx, period); state = State::PARSED; @@ -183,62 +111,59 @@ public: { assert(state == State::STARTED); - static const uint64_t targetVal = 0.5e9; /* Get value of PPS signal */ float value = smp->data[idx].f; // TODO check if it is really float - //uint64_t seqNr = smp->sequence; /* Detect Edge */ bool isEdge = lastValue < thresh && value > thresh; lastValue = value; - - auto now = time_now(); - uint64_t timediff = (now.tv_nsec - lastEdge.tv_nsec)+(now.tv_sec - lastEdge.tv_sec)*1e9; - - // nsec is between 0.5e9 and 1.5e9 when converged - realTime.tv_nsec = ts.timeFromNanoseconds(timediff); - realTime.tv_sec = lastEdge.tv_sec; - - // when not converged, nsec might be really high. - // using a loop assures no time jumps. - //TODO: can nsec also become negative? - while (realTime.tv_nsec >= 1e9) { - realTime.tv_sec++; - realTime.tv_nsec -= 1e9; - } - if (isEdge) { - edgeCounter++; - ts.synchronize(targetVal, timediff); - lastEdge = now; - converged = abs(ts.timeError(targetVal, timediff)) < 0.001; + if (isSynced) { + timeErr += 1.0 - (cntSmps * period); + filtWin[cntEdges % filtLen] = cntSmpsTotal; + /* Estimated sample period over last 'horizonEst' seconds */ + unsigned int tmp = cntEdges < filtLen ? cntEdges : horizonEst; + double cntSmpsAvg = (cntSmpsTotal - filtWin[(cntEdges - tmp) % filtLen]) / tmp; + periodEst = 1.0 / cntSmpsAvg; + info("cntSmpsAvg %f", cntSmpsAvg); + periodErrComp = timeErr / (cntSmpsAvg * horizonComp); + period = periodEst + periodErrComp; + } + else { + tsVirt.tv_sec = time(nullptr); + tsVirt.tv_nsec = 0; + isSynced = true; + cntEdges = 0; + cntSmpsTotal = 0; + } + cntSmps = 0; + cntEdges++; + + info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); } - /*if(isEdge){ - info("Edge detected: seq=%lu, realTime.nsec=%lu, timeErr=%f , timePeriod=%f, changeVal=%f", seqNr,realTime.tv_nsec, timeErr, periodTime, changeVal); - }*/ + cntSmps++; + cntSmpsTotal++; - - //if (!converged || edgeCounter < 2) - if (edgeCounter < 2) + if (cntEdges < 5) return Hook::Reason::SKIP_SAMPLE; - /* Update timestamp */ - smp->ts.origin = realTime; + smp->ts.origin = tsVirt; smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; - if((smp->sequence - last_sequence) > 1 ) + struct timespec tsPeriod = time_from_double(period); + tsVirt = time_add(&tsVirt, &tsPeriod); + + + if ((smp->sequence - last_sequence) > 1) warning("Samples missed: %li sampled missed", smp->sequence - last_sequence); last_sequence = smp->sequence; return Hook::Reason::OK; } - - ~PpsTsHook(){ - } }; /* Register hook */ From 9b58bd15c81df41743230429f1b5af1c8dfe627f Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sat, 10 Oct 2020 20:45:51 +0200 Subject: [PATCH 29/46] add dump via socket --- lib/hooks/dft.cpp | 114 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 14 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index aedcf1aba..1c9a69000 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -35,9 +35,77 @@ #include #include +#include +#include +#include + namespace villas { namespace node { +class dataDump { + protected: + int socketFd; + std::string socketName; + + public: + + dataDump(std::string socketNameIn) : + socketName("") + { + openSocket(socketNameIn); + } + + ~dataDump() { + closeSocket(); + } + + int openSocket(std::string socketNameIn) { + socketName = socketNameIn; + + socketFd = socket(AF_LOCAL,SOCK_STREAM, 0); + if (socketFd < 0) { + info("Error creating socket %s", socketName.c_str()); + return -1; + } + + sockaddr_un socketaddr_un; + socketaddr_un.sun_family = AF_UNIX; + strcpy(socketaddr_un.sun_path, socketName.c_str()); + socketName = socketNameIn; + + //unlink(socketName.c_str()); + //bind(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); + connect(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); + + return 1; + } + + void closeSocket() { + info("Remove socket"); + //unlink(socketName.c_str()); + close(socketFd); + } + + void writeData(uint len, double* yData, double* xData = nullptr) { + ssize_t bytesWritten; + + for (uint i = 0; idata[0].f; pps_memory[smp_mem_pos % window_size] = smp->data[1].f; - smp_mem_pos ++ ; + smp_mem_pos++ ; + //if (smp_mem_pos%1000 == 0) { + originalSignalDump->writeData(1,&(smp->data[0].f)); + ppsSignalDump->writeData(1,&(smp->data[1].f)); + //} bool runDft = false; if( sync_dft ){ @@ -285,9 +367,9 @@ public: runDft = true; } last_dft_cal = smp->ts.origin; - if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || ( runDft ) ) { + if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || ( runDft ))// { calcDft(paddingType::ZERO); - double maxF = 0; + /*double maxF = 0; double maxA = 0; int maxPos = 0; @@ -299,7 +381,7 @@ public: maxPos = i; } } - info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); + //info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); if(smp_mem_pos>100e3){ appendDumpData("/tmp/plot/phaseOut",atan2(dftResults[maxPos].imag(), dftResults[maxPos].real())); @@ -308,7 +390,7 @@ public: } dumpData("/tmp/plot/absDftResults", absDftResults, freq_count, absDftFreqs); - } + }*/ if((smp->sequence - last_sequence) > 1 ) @@ -322,6 +404,8 @@ public: virtual ~DftHook() { //delete smp_memory; + delete originalSignalDump; + delete ppsSignalDump; } void appendDumpData(const char *path, double ydata){ @@ -374,23 +458,25 @@ public: double tmp_smp_window[window_size]; double tmp_smp_pps[window_size]; - uint lastEdge = 0; - uint edgeCount = 0; + //uint lastEdge = 0; + //uint edgeCount = 0; for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] = smp_memory[( i + smp_mem_pos) % window_size]; tmp_smp_pps[i] = pps_memory[( i + smp_mem_pos) % window_size]; - if(edgeCount == 0 || (lastEdge + 1500) < i){ + /*if(edgeCount == 0 || (lastEdge + 1500) < i){ if(tmp_smp_pps[i] > 2. && i > 5000){ - lastEdge = i; + lastEdge = i;dumpData("/tmp/plot/pps_original",tmp_smp_pps,window_size); + dumpData("/tmp/plot/signal_original",tmp_smp_window,window_size); + info("edge detected %i",lastEdge); edgeCount++; appendDumpData("/tmp/plot/ppsAmplitude",tmp_smp_window[i]); } - } + }*/ } - dumpData("/tmp/plot/pps_original",tmp_smp_pps,window_size); - dumpData("/tmp/plot/signal_original",tmp_smp_window,window_size); - + originalSignalDumpSynced->writeData(window_size,tmp_smp_window); + ppsSignalDumpSynced->writeData(window_size,tmp_smp_pps); +return; for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] *= filterWindowCoefficents[i]; } From 6cacb0a2b20da02395483a728cbb07199ebdbf2c Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 12 Oct 2020 15:26:03 +0200 Subject: [PATCH 30/46] Move villasDump from dft to dumper class --- include/villas/dumper.hpp | 42 +++++++++++++ lib/CMakeLists.txt | 1 + lib/dumper.cpp | 89 +++++++++++++++++++++++++++ lib/hooks/dft.cpp | 122 ++++---------------------------------- 4 files changed, 144 insertions(+), 110 deletions(-) create mode 100644 include/villas/dumper.hpp create mode 100644 lib/dumper.cpp diff --git a/include/villas/dumper.hpp b/include/villas/dumper.hpp new file mode 100644 index 000000000..54c33975c --- /dev/null +++ b/include/villas/dumper.hpp @@ -0,0 +1,42 @@ +/** Dump fields and values in a socket to plot them with villasDump.py. + * + * @file + * @author Manuel Pitz + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +namespace villas { +namespace node { + +class Dumper { + protected: + int socketFd; + std::string socketName; + + public: + Dumper(std::string socketNameIn); + ~Dumper(); + int openSocket(std::string socketNameIn); + void closeSocket(); + void writeData(uint len, double* yData, double* xData = nullptr); + }; +} +} \ No newline at end of file diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e65aa0058..de06b38d9 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -48,6 +48,7 @@ endif() set(LIB_SRC config_helper.cpp config.cpp + dumper.cpp mapping.cpp memory.cpp memory/heap.cpp diff --git a/lib/dumper.cpp b/lib/dumper.cpp new file mode 100644 index 000000000..95340ae10 --- /dev/null +++ b/lib/dumper.cpp @@ -0,0 +1,89 @@ +/** Dump fields and values in a socket to plot them with villasDump.py. + * + * @file + * @author Manuel Pitz + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include "villas/dumper.hpp" +#include +#include +#include +#include "villas/log.h" +#include + + + +using namespace villas; +using namespace villas::node; + + +Dumper::Dumper(std::string socketNameIn) : + socketName("") +{ + openSocket(socketNameIn); +} + +Dumper::~Dumper() { + closeSocket(); +} + +int Dumper::openSocket(std::string socketNameIn) { + socketName = socketNameIn; + + socketFd = socket(AF_LOCAL,SOCK_STREAM, 0); + if (socketFd < 0) { + info("Error creating socket %s", socketName.c_str()); + return -1; + } + + sockaddr_un socketaddr_un; + socketaddr_un.sun_family = AF_UNIX; + strcpy(socketaddr_un.sun_path, socketName.c_str()); + socketName = socketNameIn; + + //unlink(socketName.c_str()); + //bind(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); + connect(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); + + return 1; +} + +void Dumper::closeSocket() { + info("Remove socket"); + //unlink(socketName.c_str()); + close(socketFd); +} + +void Dumper::writeData(uint len, double* yData, double* xData) { + ssize_t bytesWritten; + + for (uint i = 0; i - +#include "villas/dumper.hpp" #include #include #include #include #include #include -#include -#include -#include -#include -#include namespace villas { namespace node { -class dataDump { - protected: - int socketFd; - std::string socketName; - - public: - - dataDump(std::string socketNameIn) : - socketName("") - { - openSocket(socketNameIn); - } - - ~dataDump() { - closeSocket(); - } - - int openSocket(std::string socketNameIn) { - socketName = socketNameIn; - - socketFd = socket(AF_LOCAL,SOCK_STREAM, 0); - if (socketFd < 0) { - info("Error creating socket %s", socketName.c_str()); - return -1; - } - - sockaddr_un socketaddr_un; - socketaddr_un.sun_family = AF_UNIX; - strcpy(socketaddr_un.sun_path, socketName.c_str()); - socketName = socketNameIn; - - //unlink(socketName.c_str()); - //bind(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); - connect(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); - - return 1; - } - - void closeSocket() { - info("Remove socket"); - //unlink(socketName.c_str()); - close(socketFd); - } - - void writeData(uint len, double* yData, double* xData = nullptr) { - ssize_t bytesWritten; - - for (uint i = 0; i0) - fh << ";"; - fh << ydata; - - fh.close(); - } - - void dumpData(const char *path, double *ydata, uint size, double *xdata=nullptr){ - - std::ofstream fh; - fh.open(path); - for(uint i = 0 ; i < size ; i++){ - if(i>0)fh << ";"; - fh << ydata[i]; - } - if(xdata){ - fh << "\n"; - for(uint i = 0 ; i < size ; i++){ - if(i>0)fh << ";"; - fh << xdata[i]; - } - } - fh.close(); - } - - void genDftMatrix(){ using namespace std::complex_literals; @@ -480,9 +382,9 @@ return; for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] *= filterWindowCoefficents[i]; } - dumpData("/tmp/plot/signal_windowed",tmp_smp_window,window_size); + //dumpData("/tmp/plot/signal_windowed",tmp_smp_window,window_size); - dumpData("/tmp/plot/smp_window",smp_memory,window_size); + //dumpData("/tmp/plot/smp_window",smp_memory,window_size); for( uint i=0; i < freq_count; i++){ dftResults[i] = 0; @@ -529,7 +431,7 @@ return; } } window_corretion_factor /= window_size; - dumpData("/tmp/plot/filter_window",filterWindowCoefficents,window_size); + //dumpData("/tmp/plot/filter_window",filterWindowCoefficents,window_size); } }; From 7cfa16a9622736f9573f36f7af16ba614afb5081 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 12 Oct 2020 19:27:32 +0200 Subject: [PATCH 31/46] cleanup of dft hoock and update to new dump class --- lib/hooks/dft.cpp | 94 +++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index b361a8d57..65b141a9b 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -52,10 +52,12 @@ protected: HAMMING }; - Dumper* originalSignalDump; - Dumper* ppsSignalDump; - Dumper* originalSignalDumpSynced; - Dumper* ppsSignalDumpSynced; + Dumper* origSigSync; + Dumper* ppsSigSync; + Dumper* windowdSigSync; + Dumper* phasorPhase; + Dumper* phasorAmpitude; + Dumper* phasorFreq; windowType window_type; paddingType padding_type; @@ -70,6 +72,7 @@ protected: double* absDftResults; double* absDftFreqs; + uint64_t dftCalcCnt; uint sample_rate; double start_freqency; double end_freqency; @@ -80,7 +83,7 @@ protected: uint freq_count;//number of requency bins that are calculated bool sync_dft; - uint smp_mem_pos; + uint64_t smp_mem_pos; uint64_t last_sequence; @@ -98,6 +101,7 @@ public: Hook(p, n, fl, prio, en), window_type(windowType::NONE), padding_type(paddingType::ZERO), + dftCalcCnt(0), sample_rate(0), start_freqency(0), end_freqency(0), @@ -115,10 +119,13 @@ public: { format = format_type_lookup("villas.human"); - originalSignalDump = new Dumper("/tmp/plot/originalSignalDump"); - ppsSignalDump = new Dumper("/tmp/plot/ppsSignalDump"); - originalSignalDumpSynced = new Dumper("/tmp/plot/originalSignalDumpSynced"); - ppsSignalDumpSynced = new Dumper("/tmp/plot/ppsSignalDumpSynced"); + origSigSync = new Dumper("/tmp/plot/origSigSync"); + ppsSigSync = new Dumper("/tmp/plot/ppsSigSync"); + windowdSigSync = new Dumper("/tmp/plot/windowdSigSync"); + phasorPhase = new Dumper("/tmp/plot/phasorPhase"); + phasorAmpitude = new Dumper("/tmp/plot/phasorAmpitude"); + phasorFreq = new Dumper("/tmp/plot/phasorFreq"); + } virtual void prepare(){ @@ -187,7 +194,7 @@ public: absDftFreqs = new double[freq_count]; for(uint i=0; i < freq_count; i++) absDftFreqs[i] = start_freqency + i * frequency_resolution; - + genDftMatrix(); calcWindow(window_type); @@ -283,26 +290,24 @@ public: virtual Hook::Reason process(sample *smp) { assert(state == State::STARTED); - + smp_memory[smp_mem_pos % window_size] = smp->data[0].f; pps_memory[smp_mem_pos % window_size] = smp->data[1].f; smp_mem_pos++ ; - //if (smp_mem_pos%1000 == 0) { - originalSignalDump->writeData(1,&(smp->data[0].f)); - ppsSignalDump->writeData(1,&(smp->data[1].f)); - //} bool runDft = false; - if( sync_dft ){ + if( sync_dft ) { if( last_dft_cal.tv_sec != smp->ts.origin.tv_sec ) runDft = true; } last_dft_cal = smp->ts.origin; - if( (( !sync_dft && ( smp_mem_pos % ( sample_rate / dft_rate )) == 0 ) ) || ( runDft ))// { + + if( runDft ) { calcDft(paddingType::ZERO); - /*double maxF = 0; + double maxF = 0; double maxA = 0; int maxPos = 0; + for(uint i=0; its.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); - - if(smp_mem_pos>100e3){ - appendDumpData("/tmp/plot/phaseOut",atan2(dftResults[maxPos].imag(), dftResults[maxPos].real())); - appendDumpData("/tmp/plot/voltageOut",(maxA / pow(2,1./2))); - appendDumpData("/tmp/plot/freqOut",maxF); - } - dumpData("/tmp/plot/absDftResults", absDftResults, freq_count, absDftFreqs); - }*/ + info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); + + if(dftCalcCnt > 1) { + double tmpPhase = atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()); + phasorPhase->writeData(1,&tmpPhase); + //double tmpMaxA = maxA / pow(2,1./2); + //phasorAmpitude->writeData(1,&tmpMaxA); + phasorFreq->writeData(1,&maxF); + } + dftCalcCnt++; + } if((smp->sequence - last_sequence) > 1 ) @@ -335,8 +342,14 @@ public: virtual ~DftHook() { //delete smp_memory; - delete originalSignalDump; - delete ppsSignalDump; + + delete origSigSync; + delete ppsSigSync; + delete windowdSigSync; + delete phasorPhase; + delete phasorAmpitude; + delete phasorFreq; + } void genDftMatrix(){ @@ -365,23 +378,18 @@ public: for(uint i = 0; i< window_size; i++){ tmp_smp_window[i] = smp_memory[( i + smp_mem_pos) % window_size]; tmp_smp_pps[i] = pps_memory[( i + smp_mem_pos) % window_size]; - /*if(edgeCount == 0 || (lastEdge + 1500) < i){ - if(tmp_smp_pps[i] > 2. && i > 5000){ - lastEdge = i;dumpData("/tmp/plot/pps_original",tmp_smp_pps,window_size); - dumpData("/tmp/plot/signal_original",tmp_smp_window,window_size); - - info("edge detected %i",lastEdge); - edgeCount++; - appendDumpData("/tmp/plot/ppsAmplitude",tmp_smp_window[i]); - } - }*/ } - originalSignalDumpSynced->writeData(window_size,tmp_smp_window); - ppsSignalDumpSynced->writeData(window_size,tmp_smp_pps); -return; - for(uint i = 0; i< window_size; i++){ + + origSigSync->writeData(window_size,tmp_smp_window); + ppsSigSync->writeData(window_size,tmp_smp_pps); + + if(dftCalcCnt > 1) + phasorAmpitude->writeData(1,&tmp_smp_window[window_size - 1]); + + for(uint i = 0; i< window_size; i++) { tmp_smp_window[i] *= filterWindowCoefficents[i]; } + windowdSigSync->writeData(window_size,tmp_smp_window); //dumpData("/tmp/plot/signal_windowed",tmp_smp_window,window_size); //dumpData("/tmp/plot/smp_window",smp_memory,window_size); From 1a1e95aa1063068701e675b80d3c28321b30a4c5 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 14 Oct 2020 11:37:37 +0200 Subject: [PATCH 32/46] cleanup code --- include/villas/dumper.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/villas/dumper.hpp b/include/villas/dumper.hpp index 54c33975c..b09814fbe 100644 --- a/include/villas/dumper.hpp +++ b/include/villas/dumper.hpp @@ -38,5 +38,5 @@ class Dumper { void closeSocket(); void writeData(uint len, double* yData, double* xData = nullptr); }; -} -} \ No newline at end of file +} /* namespace node */ +} /* namespace villas */ \ No newline at end of file From 7226fd7310a3a854be3eafba2303bf44ea96475c Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 14 Oct 2020 11:38:24 +0200 Subject: [PATCH 33/46] change default horizon --- lib/hooks/pps_ts.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 6e35f060b..00f1c71ca 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -68,8 +68,8 @@ public: timeErr(0.0), period(0.0), cntEdges(0), - horizonComp(1), - horizonEst(1), + horizonComp(10), + horizonEst(10), filtLen(horizonEst + 1) { filtWin = new uintmax_t[filtLen]; From af9c648e7a08169f7db0e19b6c23c3336d1932c6 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Mon, 19 Oct 2020 20:04:03 +0200 Subject: [PATCH 34/46] add supress warnings for dumper --- include/villas/dumper.hpp | 2 ++ lib/dumper.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/villas/dumper.hpp b/include/villas/dumper.hpp index b09814fbe..0c5b65a93 100644 --- a/include/villas/dumper.hpp +++ b/include/villas/dumper.hpp @@ -30,6 +30,8 @@ class Dumper { protected: int socketFd; std::string socketName; + bool supressRepeatedWarning; + uint64_t warningCounter; public: Dumper(std::string socketNameIn); diff --git a/lib/dumper.cpp b/lib/dumper.cpp index 95340ae10..49f428af1 100644 --- a/lib/dumper.cpp +++ b/lib/dumper.cpp @@ -35,7 +35,9 @@ using namespace villas::node; Dumper::Dumper(std::string socketNameIn) : - socketName("") + socketName(""), + supressRepeatedWarning(true), + warningCounter(0) { openSocket(socketNameIn); } @@ -82,8 +84,9 @@ void Dumper::writeData(uint len, double* yData, double* xData) { str += "\n"; bytesWritten = write(socketFd, str.c_str(), str.length()); - if( (long unsigned int) bytesWritten != str.length() ) { + if( (long unsigned int) bytesWritten != str.length() && (!supressRepeatedWarning || warningCounter<1) ) { warning("Could not send all content to socket %s", socketName.c_str()); + warningCounter++; } } } \ No newline at end of file From 73dad5f73f1458b234ad8252526c59c0cd0f3dc4 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 20:56:51 +0200 Subject: [PATCH 35/46] fix handling of path output signals --- include/villas/node.h | 4 ++-- include/villas/path.h | 4 ++-- lib/mapping.cpp | 2 +- lib/node.cpp | 20 +++++++++----------- lib/path.cpp | 22 ++++++++++++++-------- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/include/villas/node.h b/include/villas/node.h index ceb9286bf..9f7e94f08 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -192,6 +192,8 @@ char * node_name_long(struct vnode *n); */ struct vlist * node_output_signals(struct vnode *n); +struct vlist * node_input_signals(struct vnode *n); + /** Reverse local and remote socket address. * * @see node_type::reverse @@ -218,8 +220,6 @@ bool node_is_valid_name(const char *name); bool node_is_enabled(const struct vnode *n); -struct vlist * node_get_signals(struct vnode *n, enum NodeDir dir); - json_t * node_to_json(struct vnode *); /** @} */ diff --git a/include/villas/path.h b/include/villas/path.h index df4b302f2..28e24d529 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -144,6 +144,8 @@ const char * path_name(struct vpath *p); /** Get a list of signals which is emitted by the path. */ struct vlist * path_output_signals(struct vpath *n); +struct vlist * path_signals(struct vpath *p); + /** Reverse a path */ int path_reverse(struct vpath *p, struct vpath *r); @@ -165,8 +167,6 @@ bool path_is_enabled(const struct vpath *p); bool path_is_reversed(const struct vpath *p); -struct vlist * path_get_signals(struct vpath *p); - json_t * path_to_json(struct vpath *p); /** @} */ diff --git a/lib/mapping.cpp b/lib/mapping.cpp index 756756500..e198a311c 100644 --- a/lib/mapping.cpp +++ b/lib/mapping.cpp @@ -319,7 +319,7 @@ int mapping_entry_prepare(struct mapping_entry *me, struct vlist *nodes) end: if (me->length < 0) { - struct vlist *sigs = node_get_signals(me->node, NodeDir::IN); + struct vlist *sigs = node_input_signals(me->node); me->length = vlist_length(sigs); } diff --git a/lib/node.cpp b/lib/node.cpp index 6677a4d03..79baa2ecd 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -568,14 +568,6 @@ const char * node_name_short(struct vnode *n) return n->name; } -struct vlist * node_output_signals(struct vnode *n) -{ - if (n->output_path) - return path_output_signals(n->output_path); - - return nullptr; -} - int node_reverse(struct vnode *n) { return node_type(n)->reverse ? node_type(n)->reverse(n) : -1; @@ -663,11 +655,17 @@ bool node_is_enabled(const struct vnode *n) return n->enabled; } -struct vlist * node_get_signals(struct vnode *n, enum NodeDir dir) +struct vlist * node_input_signals(struct vnode *n) { - struct vnode_direction *nd = dir == NodeDir::IN ? &n->in : &n->out; + return node_direction_get_signals(&n->in); +} - return node_direction_get_signals(nd); +struct vlist * node_output_signals(struct vnode *n) +{ + if (n->output_path) + return path_output_signals(n->output_path); + + return nullptr; } json_t * node_to_json(struct vnode *n) diff --git a/lib/path.cpp b/lib/path.cpp index 000a66421..576a520ac 100644 --- a/lib/path.cpp +++ b/lib/path.cpp @@ -294,7 +294,7 @@ int path_prepare(struct vpath *p, struct vlist *nodes) vlist_push(&p->sources, ps); } - struct vlist *sigs = node_get_signals(me->node, NodeDir::IN); + struct vlist *sigs = node_input_signals(me->node); /* Update signals of path */ for (unsigned j = 0; j < (unsigned) me->length; j++) { @@ -379,7 +379,7 @@ int path_prepare(struct vpath *p, struct vlist *nodes) #endif /* WITH_HOOKS */ p->logger->info("Prepared path {} with output signals:", path_name(p)); - signal_list_dump(&p->signals); + signal_list_dump(path_output_signals(p)); p->state = State::PREPARED; @@ -758,11 +758,6 @@ const char * path_name(struct vpath *p) return p->_name; } -struct vlist * path_output_signals(struct vpath *p) -{ - return &p->signals; -} - bool path_is_simple(const struct vpath *p) { int ret; @@ -794,11 +789,22 @@ bool path_is_reversed(const struct vpath *p) return p->reverse; } -struct vlist * path_get_signals(struct vpath *p) +struct vlist * path_signals(struct vpath *p) { return &p->signals; } +struct vlist * path_output_signals(struct vpath *p) +{ +#ifdef WITH_HOOKS + Hook *last_hook = (Hook *) vlist_last(&p->hooks); + + return last_hook->getSignals(); +#else + return &p->signals; +#endif +} + json_t * path_to_json(struct vpath *p) { char uuid[37]; From b911e9e7ab03b53cb205195a785ea54b436aa6e8 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 20:57:03 +0200 Subject: [PATCH 36/46] add signal_list_clear() --- include/villas/signal_list.h | 2 ++ lib/signal_list.cpp | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/include/villas/signal_list.h b/include/villas/signal_list.h index 61fbdf91b..2cce28ec9 100644 --- a/include/villas/signal_list.h +++ b/include/villas/signal_list.h @@ -34,6 +34,8 @@ int signal_list_init(struct vlist *list) __attribute__ ((warn_unused_result)); int signal_list_destroy(struct vlist *list) __attribute__ ((warn_unused_result)); +int signal_list_clear(struct vlist *list); + int signal_list_parse(struct vlist *list, json_t *cfg); int signal_list_generate(struct vlist *list, unsigned len, enum SignalType fmt); diff --git a/lib/signal_list.cpp b/lib/signal_list.cpp index 0cceef672..faccbe555 100644 --- a/lib/signal_list.cpp +++ b/lib/signal_list.cpp @@ -42,6 +42,19 @@ int signal_list_init(struct vlist *list) return 0; } +int signal_list_clear(struct vlist *list) +{ + for (size_t i = 0; i < vlist_length(list); i++) { + struct signal *sig = (struct signal *) vlist_at(list, i); + + signal_decref(sig); + } + + vlist_clear(list); + + return 0; +} + int signal_list_destroy(struct vlist *list) { int ret; From 66e3f6f45df5d6136bba40f4612bc1f2c8f1fbb8 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 20:57:57 +0200 Subject: [PATCH 37/46] hooks: update sample signal descriptors in hook_list_process() --- lib/hook_list.cpp | 4 ++++ lib/hooks/average.cpp | 1 - lib/hooks/cast.cpp | 3 --- lib/hooks/dp.cpp | 2 -- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/hook_list.cpp b/lib/hook_list.cpp index 1d7aba314..b1cd544fa 100644 --- a/lib/hook_list.cpp +++ b/lib/hook_list.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include using namespace villas; @@ -129,6 +130,8 @@ skip_add: h->prepare(sigs); sigs = h->getSignals(); + + signal_list_dump(sigs); } } @@ -146,6 +149,7 @@ int hook_list_process(vlist *hs, sample *smps[], unsigned cnt) Hook *h = (Hook *) vlist_at(hs, i); auto ret = h->process(smp); + smp->signals = h->getSignals(); switch (ret) { case Hook::Reason::ERROR: return -1; diff --git a/lib/hooks/average.cpp b/lib/hooks/average.cpp index b5353d714..c89cac123 100644 --- a/lib/hooks/average.cpp +++ b/lib/hooks/average.cpp @@ -172,7 +172,6 @@ public: return Reason::ERROR; sample_data_insert(smp, (union signal_data *) &avg, offset, 1); - smp->signals = &signals; return Reason::OK; } diff --git a/lib/hooks/cast.cpp b/lib/hooks/cast.cpp index be261f51f..200122792 100644 --- a/lib/hooks/cast.cpp +++ b/lib/hooks/cast.cpp @@ -161,9 +161,6 @@ public: signal_data_cast(&smp->data[signal_index], orig_sig, new_sig); - /* Replace signal descriptors of sample */ - smp->signals = &signals; - return Reason::OK; } }; diff --git a/lib/hooks/dp.cpp b/lib/hooks/dp.cpp index 56d729659..b89c1b87b 100644 --- a/lib/hooks/dp.cpp +++ b/lib/hooks/dp.cpp @@ -318,8 +318,6 @@ public: sample_data_insert(smp, (union signal_data *) coeffs, offset, fharmonics_len); } - smp->signals = &signals; - time += timestep; steps++; From 9e97e22d10e0ae6967b9e5c2a79ea6d93eb921c4 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 20:58:16 +0200 Subject: [PATCH 38/46] io: bugfix in state machine --- lib/io.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/io.cpp b/lib/io.cpp index 77a996ee7..1071bd07d 100644 --- a/lib/io.cpp +++ b/lib/io.cpp @@ -505,14 +505,14 @@ FILE * io_stream_input(struct io *io) { int io_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) { - assert(io->state == State::CHECKED || io->state == State::OPENED); + assert(io->state == State::CHECKED || io->state == State::INITIALIZED || io->state == State::OPENED); return io_type(io)->sscan ? io_type(io)->sscan(io, buf, len, rbytes, smps, cnt) : -1; } int io_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) { - assert(io->state == State::CHECKED || io->state == State::OPENED); + assert(io->state == State::CHECKED || io->state == State::INITIALIZED || io->state == State::OPENED); return io_type(io)->sprint ? io_type(io)->sprint(io, buf, len, wbytes, smps, cnt) : -1; } From dd61a919b139c2a63992d98f013011a985c2399a Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 21:00:05 +0200 Subject: [PATCH 39/46] mqtt: bugfixes after last refactoring --- lib/nodes/mqtt.cpp | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/nodes/mqtt.cpp b/lib/nodes/mqtt.cpp index 85906aa73..8c1bac69b 100644 --- a/lib/nodes/mqtt.cpp +++ b/lib/nodes/mqtt.cpp @@ -176,7 +176,7 @@ int mqtt_init(struct vnode *n) int ret; struct mqtt *m = (struct mqtt *) n->_vd; - m->client = mosquitto_new(n->name, 0, (void *) n); + m->client = mosquitto_new(nullptr, true, (void *) n); if (!m->client) return -1; @@ -314,6 +314,26 @@ int mqtt_check(struct vnode *n) return 0; } +int mqtt_prepare(struct vnode *n) +{ + int ret; + struct mqtt *m = (struct mqtt *) n->_vd; + + ret = io_init(&m->io, m->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); + if (ret) + return ret; + + ret = pool_init(&m->pool, 1024, SAMPLE_LENGTH(vlist_length(&n->in.signals))); + if (ret) + return ret; + + ret = queue_signalled_init(&m->queue, 1024); + if (ret) + return ret; + + return 0; +} + char * mqtt_print(struct vnode *n) { struct mqtt *m = (struct mqtt *) n->_vd; @@ -394,18 +414,6 @@ int mqtt_start(struct vnode *n) goto mosquitto_error; } - ret = io_init(&m->io, m->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; - - ret = pool_init(&m->pool, 1024, SAMPLE_LENGTH(vlist_length(&n->in.signals))); - if (ret) - return ret; - - ret = queue_signalled_init(&m->queue, 1024); - if (ret) - return ret; - ret = mosquitto_connect(m->client, m->host, m->port, m->keepalive); if (ret != MOSQ_ERR_SUCCESS) goto mosquitto_error; @@ -567,6 +575,7 @@ static void register_plugin() { p.node.destroy = mqtt_destroy; p.node.parse = mqtt_parse; p.node.check = mqtt_check; + p.node.prepare = mqtt_prepare; p.node.print = mqtt_print; p.node.init = mqtt_init; p.node.destroy = mqtt_destroy; From 9ad4e42b17feb76f8eee7d6879cfac56297ce7f9 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 21:04:14 +0200 Subject: [PATCH 40/46] update VILLAScommon submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index 5e7df9237..ec384970b 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 5e7df923770f3ed45c8618275397b793a0e1d9b1 +Subproject commit ec384970b69e37e9dc6a8547b28282ec29a498e8 From 6a525f7f891fc761c59d7c1a6063db7d85b7fbc6 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 21 Oct 2020 21:04:28 +0200 Subject: [PATCH 41/46] dft: more cleanups --- lib/hooks/dft.cpp | 302 +++++++++++++++++++++++-------------------- lib/hooks/pps_ts.cpp | 4 +- 2 files changed, 164 insertions(+), 142 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 65b141a9b..0b662d99e 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -33,7 +33,6 @@ #include #include - namespace villas { namespace node { @@ -64,8 +63,7 @@ protected: struct format_type *format; - double* smp_memory; - double* pps_memory; + double** smp_memory; std::complex** dftMatrix; std::complex* dftResults; double* filterWindowCoefficents; @@ -90,11 +88,12 @@ protected: std::complex omega; std::complex M_I; - double window_corretion_factor; timespec last_dft_cal; - + int* signal_index;//a list of signal_index to do dft on + uint signalCnt;//number of signal_index given by config file + public: DftHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : @@ -115,7 +114,8 @@ public: last_sequence(0), M_I(0.0,1.0), window_corretion_factor(0), - last_dft_cal({0,0}) + last_dft_cal({0,0}), + signalCnt(0) { format = format_type_lookup("villas.human"); @@ -128,50 +128,57 @@ public: } - virtual void prepare(){ + virtual ~DftHook() + { + delete smp_memory; + delete origSigSync; + delete ppsSigSync; + delete windowdSigSync; + delete phasorPhase; + delete phasorAmpitude; + delete phasorFreq; - struct signal *freq_sig; - struct signal *ampl_sig; - struct signal *phase_sig; + } - /* Add signals */ - freq_sig = signal_create("amplitude", nullptr, SignalType::FLOAT); - ampl_sig = signal_create("phase", nullptr, SignalType::FLOAT); - phase_sig = signal_create("frequency", nullptr, SignalType::FLOAT); + virtual void prepare() { + signal_list_clear(&signals); - if (!freq_sig || !ampl_sig || !phase_sig) - throw RuntimeError("Failed to create new signals"); + smp_memory = new double*[signalCnt]; + for (uint i = 0; i < signalCnt; i++) { + struct signal *freqSig; + struct signal *amplSig; + struct signal *phaseSig; + struct signal *rocofSig; - vlist_push(&signals, freq_sig); - vlist_push(&signals, ampl_sig); - vlist_push(&signals, phase_sig); + /* Add signals */ + freqSig = signal_create("amplitude", nullptr, SignalType::FLOAT); + amplSig = signal_create("phase", nullptr, SignalType::FLOAT); + phaseSig = signal_create("frequency", nullptr, SignalType::FLOAT); + rocofSig = signal_create("rocof", nullptr, SignalType::FLOAT); - - //offset = vlist_length(&signals) - 1;//needs to be cleaned up + if (!freqSig || !amplSig || !phaseSig || !rocofSig) + throw RuntimeError("Failed to create new signals"); + vlist_push(&signals, freqSig); + vlist_push(&signals, amplSig); + vlist_push(&signals, phaseSig); + vlist_push(&signals, rocofSig); - window_multiplier = ceil( ( (double)sample_rate / window_size ) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + smp_memory[i] = new double[window_size]; + if (!smp_memory) + throw MemoryAllocationError(); + for (uint j = 0; j < window_size; j++) + smp_memory[i][j] = 0; + } - freq_count = ceil( ( end_freqency - start_freqency ) / frequency_resolution) + 1; + window_multiplier = ceil(((double)sample_rate / window_size) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + + freq_count = ceil((end_freqency - start_freqency) / frequency_resolution) + 1; //init sample memory - smp_memory = new double[window_size]; - if (!smp_memory) - throw MemoryAllocationError(); - - for(uint i = 0; i < window_size; i++) - smp_memory[i] = 0; - - - pps_memory = new double[window_size]; - if (!pps_memory) - throw MemoryAllocationError(); - - for(uint i = 0; i < window_size; i++) - pps_memory[i] = 0; //init matrix of dft coeffients @@ -179,7 +186,7 @@ public: if (!dftMatrix) throw MemoryAllocationError(); - for(uint i = 0; i < freq_count; i++) { + for (uint i = 0; i < freq_count; i++) { dftMatrix[i] = new std::complex[window_size * window_multiplier](); if (!dftMatrix[i]) throw MemoryAllocationError(); @@ -192,24 +199,21 @@ public: absDftResults = new double[freq_count]; absDftFreqs = new double[freq_count]; - for(uint i=0; i < freq_count; i++) + for (uint i=0; i < freq_count; i++) absDftFreqs[i] = start_freqency + i * frequency_resolution; genDftMatrix(); calcWindow(window_type); - state = State::PREPARED; } - virtual void start() { assert(state == State::PREPARED || state == State::STOPPED); - state = State::STARTED; } @@ -217,7 +221,6 @@ public: { assert(state == State::STARTED); - state = State::STOPPED; } @@ -227,13 +230,15 @@ public: int ret; json_error_t err; + json_t *json_channel_list = nullptr; + assert(state != State::STARTED); Hook::parse(cfg); state = State::PARSED; - ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: F, s?: F, s?: F, s?: i , s?: i, s?: s, s?: s, s?: b}", + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: F, s?: F, s?: F, s?: i , s?: i, s?: s, s?: s, s?: b, s?: o}", "sample_rate", &sample_rate, "start_freqency", &start_freqency, "end_freqency", &end_freqency, @@ -242,47 +247,71 @@ public: "window_size", &window_size, "window_type", &window_type_c, "padding_type", &padding_type_c, - "sync", &sync_dft - + "sync", &sync_dft, + "signal_index", &json_channel_list ); - if(!window_type_c) { + if (json_channel_list != nullptr) { + if (json_channel_list->type == JSON_ARRAY) { + signalCnt = json_array_size(json_channel_list); + signal_index = new int[signalCnt]; + + size_t i; + json_t *json_value; + json_array_foreach(json_channel_list, i, json_value) { + if (!json_is_number(json_value)) + throw ConfigError(json_value, "node-config-hook-dft-channel", "Values must be given as array of integer values!"); + signal_index[i] = json_number_value(json_value); + } + }else if (json_channel_list->type == JSON_INTEGER) { + signalCnt = 1; + signal_index = new int[signalCnt]; + if (!json_is_number(json_channel_list)) + throw ConfigError(json_channel_list, "node-config-hook-dft-channel", "Value must be given as integer value!"); + signal_index[0] = json_number_value(json_channel_list); + } + else + warning("Could not parse channel list. Please check documentation for syntax"); + } + else + throw ConfigError(json_channel_list, "node-config-node-signal", "No parameter channel given."); + + if (!window_type_c) { info("No Window type given, assume no windowing"); window_type = windowType::NONE; - } else if(strcmp(window_type_c, "flattop") == 0) + } else if (strcmp(window_type_c, "flattop") == 0) window_type = windowType::FLATTOP; - else if(strcmp(window_type_c, "hamming") == 0) + else if (strcmp(window_type_c, "hamming") == 0) window_type = windowType::HAMMING; - else if(strcmp(window_type_c, "hann") == 0) + else if (strcmp(window_type_c, "hann") == 0) window_type = windowType::HANN; else { info("Window type %s not recognized, assume no windowing",window_type_c); window_type = windowType::NONE; } - if(!padding_type_c) { + if (!padding_type_c) { info("No Padding type given, assume no zeropadding"); padding_type = paddingType::ZERO; - } else if(strcmp(padding_type_c, "signal_repeat") == 0) { + } else if (strcmp(padding_type_c, "signal_repeat") == 0) { padding_type = paddingType::SIG_REPEAT; - } else { + } + else { info("Padding type %s not recognized, assume zero padding",padding_type_c); padding_type = paddingType::ZERO; } - - if(end_freqency < 0 || end_freqency > sample_rate){ + if (end_freqency < 0 || end_freqency > sample_rate) { error("End frequency must be smaller than sample_rate (%i)",sample_rate); ret = 1; } - if(frequency_resolution > ((double)sample_rate/window_size)){ - error("The maximum frequency resolution with smaple_rate:%i and window_site:%i is %f",sample_rate, window_size, ((double)sample_rate/window_size) ); + if (frequency_resolution > ((double)sample_rate/window_size)) { + error("The maximum frequency resolution with smaple_rate:%i and window_site:%i is %f",sample_rate, window_size, ((double)sample_rate/window_size)); ret = 1; } - if (ret) throw ConfigError(cfg, err, "node-config-hook-dft"); } @@ -290,150 +319,143 @@ public: virtual Hook::Reason process(sample *smp) { assert(state == State::STARTED); - - smp_memory[smp_mem_pos % window_size] = smp->data[0].f; - pps_memory[smp_mem_pos % window_size] = smp->data[1].f; - smp_mem_pos++ ; + for (uint i=0; i< signalCnt; i++) { + smp_memory[i][smp_mem_pos % window_size] = smp->data[signal_index[i]].f; + } + smp_mem_pos++; bool runDft = false; - if( sync_dft ) { - if( last_dft_cal.tv_sec != smp->ts.origin.tv_sec ) + if (sync_dft) { + if (last_dft_cal.tv_sec != smp->ts.origin.tv_sec) runDft = true; } last_dft_cal = smp->ts.origin; - if( runDft ) { - calcDft(paddingType::ZERO); - double maxF = 0; - double maxA = 0; - int maxPos = 0; - + if (runDft) { + for (uint i = 0; i < signalCnt; i++){ + calcDft(paddingType::ZERO, smp_memory[i], smp_mem_pos); + double maxF = 0; + double maxA = 0; + int maxPos = 0; + - for(uint i=0; its.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2)) ); - - if(dftCalcCnt > 1) { - double tmpPhase = atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()); - phasorPhase->writeData(1,&tmpPhase); - //double tmpMaxA = maxA / pow(2,1./2); - //phasorAmpitude->writeData(1,&tmpMaxA); - phasorFreq->writeData(1,&maxF); - } + + //info("sec=%ld, nsec=%ld freq: %f \t phase: %f \t amplitude: %f",last_dft_cal.tv_sec, smp->ts.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2))); + + if (dftCalcCnt > 1) { + //double tmpMaxA = maxA / pow(2,1./2); + //phasorAmpitude->writeData(1,&tmpMaxA); + phasorFreq->writeData(1,&maxF); + + smp->data[i * 4].f = maxF;//frequency + smp->data[i * 4 + 1].f = (maxA / pow(2,1./2));//amplitude + smp->data[i * 4 + 2].f = atan2(dftResults[maxPos].imag(), dftResults[maxPos].real());//phase + smp->data[i * 4 + 3].f = 0;//rocof + + phasorPhase->writeData(1,&(smp->data[i * 4 + 2].f)); + } + } dftCalcCnt++; + smp->length = signalCnt * 4; } - - if((smp->sequence - last_sequence) > 1 ) + + if ((smp->sequence - last_sequence) > 1) warning("Calculation is not Realtime. %li sampled missed",smp->sequence - last_sequence); last_sequence = smp->sequence; - return Reason::OK; + if (runDft) + return Reason::OK; + + return Reason::SKIP_SAMPLE; } - virtual ~DftHook() - { - //delete smp_memory; - - delete origSigSync; - delete ppsSigSync; - delete windowdSigSync; - delete phasorPhase; - delete phasorAmpitude; - delete phasorFreq; - - } - - void genDftMatrix(){ + void genDftMatrix() { using namespace std::complex_literals; omega = exp((-2 * M_PI * M_I) / (double)(window_size * window_multiplier)); - uint startBin = floor( start_freqency / frequency_resolution ); + uint startBin = floor(start_freqency / frequency_resolution); - for( uint i = 0; i < freq_count ; i++){ - for( uint j=0 ; j < window_size * window_multiplier ; j++){ + for (uint i = 0; i < freq_count ; i++) { + for (uint j=0 ; j < window_size * window_multiplier ; j++) { dftMatrix[i][j] = pow(omega, (i + startBin) * j); } } } - void calcDft(paddingType padding){ - + /** mem size needs to be equal to window size **/ + void calcDft(paddingType padding, double* ringBuffer, uint ringBufferPos) { //prepare sample window The following parts can be combined double tmp_smp_window[window_size]; - double tmp_smp_pps[window_size]; - //uint lastEdge = 0; - //uint edgeCount = 0; - for(uint i = 0; i< window_size; i++){ - tmp_smp_window[i] = smp_memory[( i + smp_mem_pos) % window_size]; - tmp_smp_pps[i] = pps_memory[( i + smp_mem_pos) % window_size]; + for (uint i = 0; i< window_size; i++) { + tmp_smp_window[i] = ringBuffer[(i + ringBufferPos) % window_size]; } origSigSync->writeData(window_size,tmp_smp_window); - ppsSigSync->writeData(window_size,tmp_smp_pps); - if(dftCalcCnt > 1) + if (dftCalcCnt > 1) phasorAmpitude->writeData(1,&tmp_smp_window[window_size - 1]); - for(uint i = 0; i< window_size; i++) { + for (uint i = 0; i< window_size; i++) { tmp_smp_window[i] *= filterWindowCoefficents[i]; } windowdSigSync->writeData(window_size,tmp_smp_window); - //dumpData("/tmp/plot/signal_windowed",tmp_smp_window,window_size); - //dumpData("/tmp/plot/smp_window",smp_memory,window_size); - - for( uint i=0; i < freq_count; i++){ + for (uint i=0; i < freq_count; i++) { dftResults[i] = 0; - for(uint j=0; j < window_size * window_multiplier; j++){ - if(padding == paddingType::ZERO){ - if(j < (window_size)){ + for (uint j=0; j < window_size * window_multiplier; j++) { + if (padding == paddingType::ZERO) { + if (j < (window_size)) { dftResults[i]+= tmp_smp_window[j] * dftMatrix[i][j]; - }else{ + } + else{ dftResults[i]+= 0; } - }else if(padding == paddingType::SIG_REPEAT){//repeate samples + } + else if (padding == paddingType::SIG_REPEAT) {//repeate samples dftResults[i]+= tmp_smp_window[j % window_size] * dftMatrix[i][j]; } } } } - void calcWindow(windowType window_type_in){ + void calcWindow(windowType window_type_in) { - - if(window_type_in == windowType::FLATTOP){ - for(uint i=0; i < window_size; i++){ + if (window_type_in == windowType::FLATTOP) { + for (uint i=0; i < window_size; i++) { filterWindowCoefficents[i] = 0.21557895 - - 0.41663158 * cos(2 * M_PI * i / ( window_size )) - + 0.277263158 * cos(4 * M_PI * i / ( window_size )) - - 0.083578947 * cos(6 * M_PI * i / ( window_size )) - + 0.006947368 * cos(8 * M_PI * i / ( window_size )); + - 0.41663158 * cos(2 * M_PI * i / (window_size)) + + 0.277263158 * cos(4 * M_PI * i / (window_size)) + - 0.083578947 * cos(6 * M_PI * i / (window_size)) + + 0.006947368 * cos(8 * M_PI * i / (window_size)); window_corretion_factor += filterWindowCoefficents[i]; } - }else if(window_type_in == windowType::HAMMING || window_type_in == windowType::HANN){ + }else if (window_type_in == windowType::HAMMING || window_type_in == windowType::HANN) { double a_0 = 0.5;//this is the hann window - if(window_type_in == windowType::HAMMING) + if (window_type_in == windowType::HAMMING) a_0 = 25./46; - for(uint i=0; i < window_size; i++){ + for (uint i=0; i < window_size; i++) { filterWindowCoefficents[i] = a_0 - - (1 - a_0) * cos(2 * M_PI * i / ( window_size )); + - (1 - a_0) * cos(2 * M_PI * i / (window_size)); window_corretion_factor += filterWindowCoefficents[i]; } - }else{ - for(uint i=0; i < window_size; i++){ + } + else { + for (uint i=0; i < window_size; i++) { filterWindowCoefficents[i] = 1; window_corretion_factor += filterWindowCoefficents[i]; } diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 00f1c71ca..440ffeee5 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -127,7 +127,7 @@ public: unsigned int tmp = cntEdges < filtLen ? cntEdges : horizonEst; double cntSmpsAvg = (cntSmpsTotal - filtWin[(cntEdges - tmp) % filtLen]) / tmp; periodEst = 1.0 / cntSmpsAvg; - info("cntSmpsAvg %f", cntSmpsAvg); + //info("cntSmpsAvg %f", cntSmpsAvg); periodErrComp = timeErr / (cntSmpsAvg * horizonComp); period = periodEst + periodErrComp; } @@ -141,7 +141,7 @@ public: cntSmps = 0; cntEdges++; - info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); + //info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); } cntSmps++; From fb2d3a0d9e6fecb415201436546bfe41d1248032 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Tue, 3 Nov 2020 18:56:07 +0100 Subject: [PATCH 42/46] cleanup style --- include/villas/dumper.hpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/include/villas/dumper.hpp b/include/villas/dumper.hpp index 0c5b65a93..e11aa9636 100644 --- a/include/villas/dumper.hpp +++ b/include/villas/dumper.hpp @@ -27,18 +27,20 @@ namespace villas { namespace node { class Dumper { - protected: - int socketFd; - std::string socketName; - bool supressRepeatedWarning; - uint64_t warningCounter; - public: - Dumper(std::string socketNameIn); - ~Dumper(); - int openSocket(std::string socketNameIn); - void closeSocket(); - void writeData(uint len, double* yData, double* xData = nullptr); - }; +protected: + int socketFd; + std::string socketName; + bool supressRepeatedWarning; + uint64_t warningCounter; + +public: + Dumper(std::string socketNameIn); + ~Dumper(); + int openSocket(std::string socketNameIn); + void closeSocket(); + void writeData(uint len, double* yData, double* xData = nullptr); +}; + } /* namespace node */ -} /* namespace villas */ \ No newline at end of file +} /* namespace villas */ From 6ba78b34e1a82dd364b83ff3f9b62c74c0feb751 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Tue, 10 Nov 2020 19:00:17 +0100 Subject: [PATCH 43/46] hmm we missed it again. The old version did not take virt time into account --- lib/hooks/pps_ts.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 440ffeee5..821f2e8af 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -121,7 +121,13 @@ public: if (isEdge) { if (isSynced) { - timeErr += 1.0 - (cntSmps * period); + //timeErr += 1.0 - (cntSmps * period); + if(tsVirt.tv_nsec > 0.5e9) + timeErr += 1.0 - (tsVirt.tv_nsec / 1.0e9); + else + timeErr -= (tsVirt.tv_nsec / 1.0e9); + + filtWin[cntEdges % filtLen] = cntSmpsTotal; /* Estimated sample period over last 'horizonEst' seconds */ unsigned int tmp = cntEdges < filtLen ? cntEdges : horizonEst; @@ -141,7 +147,7 @@ public: cntSmps = 0; cntEdges++; - //info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); + info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); } cntSmps++; From d57bef03aa77705a47e523fedb5cb876ccf5e7b2 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 10 Feb 2021 13:06:19 +0100 Subject: [PATCH 44/46] refactoring to comply with code style --- include/villas/dumper.hpp | 2 +- lib/dumper.cpp | 87 ++++++++++++++++---------------- lib/hooks/dft.cpp | 101 ++++++++++++++++++-------------------- lib/hooks/pps_ts.cpp | 2 +- 4 files changed, 91 insertions(+), 101 deletions(-) diff --git a/include/villas/dumper.hpp b/include/villas/dumper.hpp index e11aa9636..b009918a7 100644 --- a/include/villas/dumper.hpp +++ b/include/villas/dumper.hpp @@ -39,7 +39,7 @@ public: ~Dumper(); int openSocket(std::string socketNameIn); void closeSocket(); - void writeData(uint len, double* yData, double* xData = nullptr); + void writeData(uint len, double *yData, double *xData = nullptr); }; } /* namespace node */ diff --git a/lib/dumper.cpp b/lib/dumper.cpp index 49f428af1..9a07ea8a4 100644 --- a/lib/dumper.cpp +++ b/lib/dumper.cpp @@ -1,6 +1,5 @@ /** Dump fields and values in a socket to plot them with villasDump.py. * - * @file * @author Manuel Pitz * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) @@ -21,72 +20,68 @@ * along with this program. If not, see . *********************************************************************************/ -#include "villas/dumper.hpp" +#include #include #include #include -#include "villas/log.h" +#include #include - - using namespace villas; using namespace villas::node; - Dumper::Dumper(std::string socketNameIn) : - socketName(""), - supressRepeatedWarning(true), - warningCounter(0) + socketName(""), + supressRepeatedWarning(true), + warningCounter(0) { - openSocket(socketNameIn); + openSocket(socketNameIn); } Dumper::~Dumper() { - closeSocket(); + closeSocket(); } -int Dumper::openSocket(std::string socketNameIn) { - socketName = socketNameIn; +int Dumper::openSocket(std::string socketNameIn) +{ + socketName = socketNameIn; - socketFd = socket(AF_LOCAL,SOCK_STREAM, 0); - if (socketFd < 0) { - info("Error creating socket %s", socketName.c_str()); - return -1; - } + socketFd = socket(AF_LOCAL,SOCK_STREAM, 0); + if (socketFd < 0) { + info("Error creating socket %s", socketName.c_str()); + return -1; + } - sockaddr_un socketaddr_un; - socketaddr_un.sun_family = AF_UNIX; - strcpy(socketaddr_un.sun_path, socketName.c_str()); - socketName = socketNameIn; + sockaddr_un socketaddr_un; + socketaddr_un.sun_family = AF_UNIX; + strcpy(socketaddr_un.sun_path, socketName.c_str()); + socketName = socketNameIn; - //unlink(socketName.c_str()); - //bind(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); - connect(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); + connect(socketFd, (struct sockaddr *) &socketaddr_un, sizeof(socketaddr_un)); - return 1; + return 1; } -void Dumper::closeSocket() { - info("Remove socket"); - //unlink(socketName.c_str()); - close(socketFd); +void Dumper::closeSocket() +{ + info("Remove socket"); + close(socketFd); } -void Dumper::writeData(uint len, double* yData, double* xData) { - ssize_t bytesWritten; +void Dumper::writeData(uint len, double* yData, double* xData) +{ + ssize_t bytesWritten; - for (uint i = 0; i omega; std::complex M_I; @@ -93,7 +93,7 @@ protected: int* signal_index;//a list of signal_index to do dft on uint signalCnt;//number of signal_index given by config file - + public: DftHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : @@ -140,7 +140,8 @@ public: } - virtual void prepare() { + virtual void prepare() + { signal_list_clear(&signals); @@ -197,11 +198,11 @@ public: filterWindowCoefficents = new double[window_size]; absDftResults = new double[freq_count]; - + absDftFreqs = new double[freq_count]; - for (uint i=0; i < freq_count; i++) + for (uint i = 0; i < freq_count; i++) absDftFreqs[i] = start_freqency + i * frequency_resolution; - + genDftMatrix(); calcWindow(window_type); @@ -211,16 +212,13 @@ public: virtual void start() { - assert(state == State::PREPARED || state == State::STOPPED); - state = State::STARTED; } virtual void stop() { assert(state == State::STARTED); - state = State::STOPPED; } @@ -263,7 +261,8 @@ public: throw ConfigError(json_value, "node-config-hook-dft-channel", "Values must be given as array of integer values!"); signal_index[i] = json_number_value(json_value); } - }else if (json_channel_list->type == JSON_INTEGER) { + } + else if (json_channel_list->type == JSON_INTEGER) { signalCnt = 1; signal_index = new int[signalCnt]; if (!json_is_number(json_channel_list)) @@ -279,7 +278,8 @@ public: if (!window_type_c) { info("No Window type given, assume no windowing"); window_type = windowType::NONE; - } else if (strcmp(window_type_c, "flattop") == 0) + } + else if (strcmp(window_type_c, "flattop") == 0) window_type = windowType::FLATTOP; else if (strcmp(window_type_c, "hamming") == 0) window_type = windowType::HAMMING; @@ -293,13 +293,13 @@ public: if (!padding_type_c) { info("No Padding type given, assume no zeropadding"); padding_type = paddingType::ZERO; - } else if (strcmp(padding_type_c, "signal_repeat") == 0) { - padding_type = paddingType::SIG_REPEAT; } + else if (strcmp(padding_type_c, "signal_repeat") == 0) + padding_type = paddingType::SIG_REPEAT; else { info("Padding type %s not recognized, assume zero padding",padding_type_c); padding_type = paddingType::ZERO; - } + } if (end_freqency < 0 || end_freqency > sample_rate) { error("End frequency must be smaller than sample_rate (%i)",sample_rate); @@ -319,7 +319,7 @@ public: virtual Hook::Reason process(sample *smp) { assert(state == State::STARTED); - for (uint i=0; i< signalCnt; i++) { + for (uint i = 0; i< signalCnt; i++) { smp_memory[i][smp_mem_pos % window_size] = smp->data[signal_index[i]].f; } smp_mem_pos++; @@ -330,16 +330,16 @@ public: runDft = true; } last_dft_cal = smp->ts.origin; - + if (runDft) { - for (uint i = 0; i < signalCnt; i++){ + for (uint i = 0; i < signalCnt; i++) { calcDft(paddingType::ZERO, smp_memory[i], smp_mem_pos); double maxF = 0; double maxA = 0; int maxPos = 0; - - for (uint i=0; its.origin.tv_nsec, maxF, atan2(dftResults[maxPos].imag(), dftResults[maxPos].real()), (maxA / pow(2,1./2))); - + if (dftCalcCnt > 1) { //double tmpMaxA = maxA / pow(2,1./2); //phasorAmpitude->writeData(1,&tmpMaxA); phasorFreq->writeData(1,&maxF); smp->data[i * 4].f = maxF;//frequency - smp->data[i * 4 + 1].f = (maxA / pow(2,1./2));//amplitude + smp->data[i * 4 + 1].f = (maxA / pow(2,1./2));//amplitude smp->data[i * 4 + 2].f = atan2(dftResults[maxPos].imag(), dftResults[maxPos].real());//phase smp->data[i * 4 + 3].f = 0;//rocof phasorPhase->writeData(1,&(smp->data[i * 4 + 2].f)); } - } + } dftCalcCnt++; smp->length = signalCnt * 4; } - - + + if ((smp->sequence - last_sequence) > 1) warning("Calculation is not Realtime. %li sampled missed",smp->sequence - last_sequence); @@ -385,7 +385,7 @@ public: omega = exp((-2 * M_PI * M_I) / (double)(window_size * window_multiplier)); uint startBin = floor(start_freqency / frequency_resolution); - + for (uint i = 0; i < freq_count ; i++) { for (uint j=0 ; j < window_size * window_multiplier ; j++) { @@ -395,39 +395,36 @@ public: } /** mem size needs to be equal to window size **/ - void calcDft(paddingType padding, double* ringBuffer, uint ringBufferPos) { + void calcDft(paddingType padding, double *ringBuffer, uint ringBufferPos) { //prepare sample window The following parts can be combined double tmp_smp_window[window_size]; - - for (uint i = 0; i< window_size; i++) { + + for (uint i = 0; i< window_size; i++) tmp_smp_window[i] = ringBuffer[(i + ringBufferPos) % window_size]; - } origSigSync->writeData(window_size,tmp_smp_window); if (dftCalcCnt > 1) phasorAmpitude->writeData(1,&tmp_smp_window[window_size - 1]); - for (uint i = 0; i< window_size; i++) { + for (uint i = 0; i< window_size; i++) tmp_smp_window[i] *= filterWindowCoefficents[i]; - } + windowdSigSync->writeData(window_size,tmp_smp_window); - for (uint i=0; i < freq_count; i++) { + for (uint i = 0; i < freq_count; i++) { dftResults[i] = 0; for (uint j=0; j < window_size * window_multiplier; j++) { if (padding == paddingType::ZERO) { - if (j < (window_size)) { - dftResults[i]+= tmp_smp_window[j] * dftMatrix[i][j]; - } - else{ - dftResults[i]+= 0; - } + if (j < (window_size)) + dftResults[i] += tmp_smp_window[j] * dftMatrix[i][j]; + else + dftResults[i] += 0; } - else if (padding == paddingType::SIG_REPEAT) {//repeate samples - dftResults[i]+= tmp_smp_window[j % window_size] * dftMatrix[i][j]; - } + else if (padding == paddingType::SIG_REPEAT)//repeate samples + dftResults[i] += tmp_smp_window[j % window_size] * dftMatrix[i][j]; + } } } @@ -435,27 +432,26 @@ public: void calcWindow(windowType window_type_in) { if (window_type_in == windowType::FLATTOP) { - for (uint i=0; i < window_size; i++) { + for (uint i = 0; i < window_size; i++) { filterWindowCoefficents[i] = 0.21557895 - - 0.41663158 * cos(2 * M_PI * i / (window_size)) - + 0.277263158 * cos(4 * M_PI * i / (window_size)) - - 0.083578947 * cos(6 * M_PI * i / (window_size)) - + 0.006947368 * cos(8 * M_PI * i / (window_size)); + - 0.41663158 * cos(2 * M_PI * i / (window_size)) + + 0.277263158 * cos(4 * M_PI * i / (window_size)) + - 0.083578947 * cos(6 * M_PI * i / (window_size)) + + 0.006947368 * cos(8 * M_PI * i / (window_size)); window_corretion_factor += filterWindowCoefficents[i]; } }else if (window_type_in == windowType::HAMMING || window_type_in == windowType::HANN) { double a_0 = 0.5;//this is the hann window if (window_type_in == windowType::HAMMING) a_0 = 25./46; - - for (uint i=0; i < window_size; i++) { - filterWindowCoefficents[i] = a_0 - - (1 - a_0) * cos(2 * M_PI * i / (window_size)); + + for (uint i = 0; i < window_size; i++) { + filterWindowCoefficents[i] = a_0 - (1 - a_0) * cos(2 * M_PI * i / (window_size)); window_corretion_factor += filterWindowCoefficents[i]; } } else { - for (uint i=0; i < window_size; i++) { + for (uint i = 0; i < window_size; i++) { filterWindowCoefficents[i] = 1; window_corretion_factor += filterWindowCoefficents[i]; } @@ -474,4 +470,3 @@ static HookPlugin Date: Wed, 10 Feb 2021 14:54:38 +0100 Subject: [PATCH 45/46] refactoring to comply to code style --- lib/hooks/dft.cpp | 3 +-- lib/hooks/pps_ts.cpp | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 9127de2fd..1388603b5 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -25,7 +25,7 @@ */ #include -#include "villas/dumper.hpp" +#include #include #include #include @@ -84,7 +84,6 @@ protected: uint64_t smp_mem_pos; uint64_t last_sequence; - std::complex omega; std::complex M_I; diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 448e3e058..7baf3e8f8 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -44,10 +44,10 @@ protected: bool isSynced; bool isLocked; struct timespec tsVirt; - double timeErr; // in seconds - double periodEst; // in seconds + double timeErr; // in seconds + double periodEst; // in seconds double periodErrComp; // in seconds - double period; // in seconds + double period; // in seconds uintmax_t cntEdges; uintmax_t cntSmps; uintmax_t cntSmpsTotal; From 83e3dbd78592a27ddc7c97bdbe8913459eb470bb Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 11 Feb 2021 14:31:19 +0100 Subject: [PATCH 46/46] refactoring to comply with code style --- lib/hooks/dft.cpp | 6 +----- lib/hooks/pps_ts.cpp | 6 ++---- lib/nodes/uldaq.cpp | 4 +--- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index 1388603b5..1f5c790c5 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -93,7 +93,6 @@ protected: int* signal_index;//a list of signal_index to do dft on uint signalCnt;//number of signal_index given by config file - public: DftHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), @@ -124,7 +123,6 @@ public: phasorPhase = new Dumper("/tmp/plot/phasorPhase"); phasorAmpitude = new Dumper("/tmp/plot/phasorAmpitude"); phasorFreq = new Dumper("/tmp/plot/phasorFreq"); - } virtual ~DftHook() @@ -136,7 +134,6 @@ public: delete phasorPhase; delete phasorAmpitude; delete phasorFreq; - } virtual void prepare() @@ -174,7 +171,7 @@ public: smp_memory[i][j] = 0; } - window_multiplier = ceil(((double)sample_rate / window_size) / frequency_resolution);//calculate how much zero padding ist needed for a needed resolution + window_multiplier = ceil(((double)sample_rate / window_size) / frequency_resolution); //calculate how much zero padding ist needed for a needed resolution freq_count = ceil((end_freqency - start_freqency) / frequency_resolution) + 1; @@ -206,7 +203,6 @@ public: calcWindow(window_type); state = State::PREPARED; - } virtual void start() diff --git a/lib/hooks/pps_ts.cpp b/lib/hooks/pps_ts.cpp index 7baf3e8f8..75b043cdf 100644 --- a/lib/hooks/pps_ts.cpp +++ b/lib/hooks/pps_ts.cpp @@ -101,7 +101,7 @@ public: period = 1.0 / fSmps; - info("parsed config thresh=%f signal_index=%d nominal_period=%f", thresh, idx, period); + debug(LOG_HOOK | 5, "parsed config thresh=%f signal_index=%d nominal_period=%f", thresh, idx, period); state = State::PARSED; @@ -121,7 +121,6 @@ public: if (isEdge) { if (isSynced) { - //timeErr += 1.0 - (cntSmps * period); if(tsVirt.tv_nsec > 0.5e9) timeErr += 1.0 - (tsVirt.tv_nsec / 1.0e9); else @@ -133,7 +132,6 @@ public: unsigned int tmp = cntEdges < filtLen ? cntEdges : horizonEst; double cntSmpsAvg = (cntSmpsTotal - filtWin[(cntEdges - tmp) % filtLen]) / tmp; periodEst = 1.0 / cntSmpsAvg; - //info("cntSmpsAvg %f", cntSmpsAvg); periodErrComp = timeErr / (cntSmpsAvg * horizonComp); period = periodEst + periodErrComp; } @@ -147,7 +145,7 @@ public: cntSmps = 0; cntEdges++; - info("Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); + debug(LOG_HOOK | 5, "Time Error is: %f periodEst %f periodErrComp %f", timeErr, periodEst, periodErrComp); } cntSmps++; diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index 178f824f9..b5724d609 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -671,7 +671,7 @@ int uldaq_start(struct vnode *n) /* Start the acquisition */ err = ulAInScan(u->device_handle, 0, 0, (AiInputMode) 0, (Range) 0, u->in.buffer_len / u->in.channel_count, &u->in.sample_rate, u->in.scan_options, u->in.flags, u->in.buffer); - + if (err != ERR_NO_ERROR) { ul_decode_error(err); warning("Failed to start acquisition on DAQ device for node '%s'", node_name(n)); @@ -768,8 +768,6 @@ int uldaq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r return cnt; } - - static struct plugin p; __attribute__((constructor(110)))