From e621f6b40f89f7bed16ab529b51576b0f148c8a5 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 28 Mar 2018 14:16:57 +0200 Subject: [PATCH 01/38] wip: started working on new DP-EMT hook --- lib/hooks/dp.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 lib/hooks/dp.c diff --git a/lib/hooks/dp.c b/lib/hooks/dp.c new file mode 100644 index 000000000..2aab9edcd --- /dev/null +++ b/lib/hooks/dp.c @@ -0,0 +1,157 @@ +/** Dynamic Phasor Interface Algorithm hook. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup hooks Hook functions + * @{ + */ + +#include + +#include "villas/hook.h" +#include "villas/plugin.h" +#include "villas/sample.h" +#include "villas/bitset.h" + +struct dp_series { + int idx; + int *harmonics; + int nharmonics; + double f0; + double rate; + enum { + DP_INTERPOLATION_NONE, + DP_INTERPOLATION_STEP, + DP_INTERPOLATION_LINEAR + } interpolation; + + double *history; + + unsigned counter; + unsigned last_sequence; +}; + +struct dp { + struct list series; +}; + +static double dp_series_step(struct dp_series *s, double val) +{ + +} + +static double dp_series_istep(struct dp_series *s, double val) +{ + +} + +static int dp_series_init(struct dp_series *s, int idx, double f0, double rate) +{ + d->idx = idx; + d->f0 = f0; +} + +static int dp_series_destroy(struct dp_series *s) +{ + free(s->history); + + return 0; +} + +static int dp_init(struct hook *h) +{ + struct dp *p = (struct dp *) h->_vd; + + list_init(&p->series); + + return 0; +} + +static int dp_deinit(struct hook *h) +{ + struct dp *p = (struct dp *) h->_vd; + + bitset_destroy(p->mask, (dtor_cb_t) dp_series_destroy, true); + + return 0; +} + +static int dp_parse(struct hook *h, json_t *cfg) +{ + struct dp *p = (struct dp *) h->_vd; + + int ret; + json_error_t err; + const char *mode = NULL; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: F, s?: i }", + "f0", &p->f0, + "mask", &p->mask + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); + + return 0; +} + +static int dp_read(struct hook *h, struct sample *smps[], unsigned *cnt) +{ + struct dp *p = (struct dp *) h->_vd; + + for (unsigned j = 0; j < cnt; j++) { + struct sample *t = smps[j]; + + for (size_t i = 0; i < list_length(&p->series); i++) { + struct dp_series *s = (struct dp_series *) list_at(&p->series, i); + + if (s->idx > t->length) + continue; + } + } + + return 0; +} + +static int dp_write(struct hook *h, struct sample *smps[], unsigned *cnt) +{ + struct dp *p = (struct dp *) h->_vd; + + return 0; +} + +static struct plugin p = { + .name = "dp", + .description = "Transform to dynamic phasor domain", + .type = PLUGIN_TYPE_HOOK, + .hook = { + .flags = HOOK_NODE, + .priority = 99, + .init = dp_init, + .parse = dp_parse, + .read = dp_read, + .write = dp_write, + .size = sizeof(struct dp) + } +}; + +REGISTER_PLUGIN(&p) + +/** @} */ From 8fa823767ca6558438d16fab9270d8d752792444 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 16 Jan 2019 21:43:20 +0100 Subject: [PATCH 02/38] cast: only cast a single signal --- lib/hooks/cast.c | 117 ++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 72 deletions(-) diff --git a/lib/hooks/cast.c b/lib/hooks/cast.c index c74b41563..886cee9e6 100644 --- a/lib/hooks/cast.c +++ b/lib/hooks/cast.c @@ -34,9 +34,7 @@ #include struct cast { - struct vlist operations; - - struct vlist signals; + int index; }; static int cast_init(struct hook *h) @@ -52,17 +50,8 @@ static int cast_init(struct hook *h) else return -1; - ret = vlist_init(&c->signals); - if (ret) - return ret; - - /* Copy original signal list */ - for (int i = 0; i < vlist_length(orig_signals); i++) { - struct signal *orig_sig = vlist_at(orig_signals, i); - struct signal *new_sig = signal_copy(orig_sig); - - vlist_push(&c->signals, new_sig); - } + struct signal *orig_sig = vlist_at(orig_signals, i); + struct signal *new_sig = signal_copy(orig_sig); return 0; } @@ -72,10 +61,6 @@ static int cast_destroy(struct hook *h) int ret; struct cast *c = (struct cast *) h->_vd; - ret = vlist_destroy(&c->signals, (dtor_cb_t) signal_decref, false); - if (ret) - return ret; - return 0; } @@ -86,75 +71,63 @@ static int cast_parse(struct hook *h, json_t *cfg) struct signal *sig; size_t i; - json_t *json_signals; - json_t *json_signal; + json_error_t err; - ret = json_unpack(cfg, "{ s: o }", - "signals", &json_signals + int index = -1; + const char *name = NULL; + + const char *new_name = NULL; + const char *new_unit = NULL; + const char *new_format = NULL; + + ret = json_unpack_ex(cfg, &err, "{ s?: s, s?: i, s?: s, s?: s, s?: s }", + "name", &name, + "index", &index, + "new_format", &new_format, + "new_name", &new_name, + "new_unit", &new_unit ); if (ret) return ret; - if (json_is_array(json_signals)) + /* Find matching original signal descriptor */ + if (index >= 0 && name != NULL) return -1; - json_array_foreach(json_signals, i, json_signal) { - int index = -1; - const char *name = NULL; + if (index < 0 && name == NULL) + return -1; - const char *new_name = NULL; - const char *new_unit = NULL; - const char *new_format = NULL; + sig = name + ? vlist_lookup(&c->signals, name) + : vlist_at_safe(&c->signals, index); + if (!sig) + return -1; - ret = json_unpack(json_signal, "{ s?: s, s?: i, s?: s, s?: s, s?: s }", - "name", &name, - "index", &index, - "new_format", &new_format, - "new_name", &new_name, - "new_unit", &new_unit - ); - if (ret) - return ret; + /* Cast to new format */ + if (new_format) { + enum signal_type fmt; - /* Find matching original signal descriptor */ - if (index >= 0 && name != NULL) + fmt = signal_type_from_str(new_format); + if (fmt == SIGNAL_TYPE_INVALID) return -1; - if (index < 0 && name == NULL) - return -1; + sig->type = fmt; + } - sig = name - ? vlist_lookup(&c->signals, name) - : vlist_at_safe(&c->signals, index); - if (!sig) - return -1; + /* Set new name */ + if (new_name) { + if (sig->name) + free(sig->name); - /* Cast to new format */ - if (new_format) { - enum signal_type fmt; + sig->name = strdup(new_name); + } - fmt = signal_type_from_str(new_format); - if (fmt == SIGNAL_TYPE_INVALID) - return -1; + /* Set new unit */ + if (new_unit) { + if (sig->unit) + free(sig->unit); - sig->type = fmt; - } - - /* Set new name */ - if (new_name) { - if (sig->name) - free(sig->name); - - sig->name = strdup(new_name); - } - - /* Set new unit */ - if (new_unit) { - if (sig->unit) - free(sig->unit); - - sig->unit = strdup(new_unit); - } + sig->unit = strdup(new_unit); } return 0; @@ -183,7 +156,7 @@ static int cast_process(struct hook *h, struct sample *smps[], unsigned *cnt) static struct plugin p = { .name = "cast", - .description = "Cast signals", + .description = "Cast signals types", .type = PLUGIN_TYPE_HOOK, .hook = { .flags = HOOK_NODE_READ | HOOK_PATH, From 54bd0af030c19ff92f214896b73389214851b30f Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 16 Jan 2019 21:43:40 +0100 Subject: [PATCH 03/38] wip: continue working on the dp hook --- lib/hooks/CMakeLists.txt | 1 + lib/hooks/dp.c | 228 ++++++++++++++++++++++++++++----------- 2 files changed, 167 insertions(+), 62 deletions(-) diff --git a/lib/hooks/CMakeLists.txt b/lib/hooks/CMakeLists.txt index 2872a0354..afc12824d 100644 --- a/lib/hooks/CMakeLists.txt +++ b/lib/hooks/CMakeLists.txt @@ -36,6 +36,7 @@ set(HOOK_SRC cast.c average.c dump.c + dp.c ) if(WITH_IO) diff --git a/lib/hooks/dp.c b/lib/hooks/dp.c index 2aab9edcd..b0aafc5fc 100644 --- a/lib/hooks/dp.c +++ b/lib/hooks/dp.c @@ -1,7 +1,7 @@ /** Dynamic Phasor Interface Algorithm hook. * * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * * VILLASnode @@ -24,131 +24,235 @@ * @{ */ +#include +#include #include -#include "villas/hook.h" -#include "villas/plugin.h" -#include "villas/sample.h" -#include "villas/bitset.h" +#include +#include +#include -struct dp_series { - int idx; - int *harmonics; - int nharmonics; - double f0; - double rate; - enum { - DP_INTERPOLATION_NONE, - DP_INTERPOLATION_STEP, - DP_INTERPOLATION_LINEAR - } interpolation; +#define J _Complex_I - double *history; +struct delay { + double *data; + size_t steps; + size_t mask; + size_t pos; +} - unsigned counter; - unsigned last_sequence; -}; +int delay_init(struct delay *w, double dt, double period) +{ + size_t len = LOG2_CEIL(period / dt); + + /* Allocate memory for ciruclar history buffer */ + w->data = alloc(len * sizeof(double)); + if (!w->data) + return -1; + + w->pos = 0; + w->mask = len - 1; + + return 0; +} + +int delay_destroy(struct delay *w) +{ + free(w->data); +} + +double delay_update(struct delay *w, double in) +{ + double out = w->data[pos & w->mask]; + + w->data[w->pos++] + + return out; +} struct dp { - struct list series; -}; + int index; + int inverse; -static double dp_series_step(struct dp_series *s, double val) -{ + double f0; + double dt; + double t; + double complex *coeffs; + int *fharmonics; + int fharmonics_len; + + struct window history; } -static double dp_series_istep(struct dp_series *s, double val) +static void dp_step(struct dp *d, double *in, float complex *out) { + double newest = *in; + double oldest = delay_update(&d->hist, newest); + for (int i = 0; i < d->fharmonics_len; i++) { + double pi_fharm = 2.0 * M_PI * d->fharmonics[i]; + + /* Recursive update */ + d->coeffs[i] = (d->coeffs[i] + (newest - oldest)) * cexp(pi_fharm); + + /* Correction for stationary phasor */ + double complex correction = cexp(pi_fharm * (d->t - (d->history_len + 1))); + double complex result = 2.0 / d->history_len * d->coeffs[i] / correction; + + /* DC component */ + if (i == 0) + result /= 2.0; + + out[i] = result; + } } -static int dp_series_init(struct dp_series *s, int idx, double f0, double rate) +static void dp_istep(struct dp *d, complex float *in, double *out) { - d->idx = idx; - d->f0 = f0; + double complex value = 0; + + /* Reconstruct the original signal */ + for (int i = 0; i < d->fharmonics_len; i++) { + double freq = d->fharmonics[i]; + double complex coeff = in[i]; + + value += coeff * cexp(2.0 * M_PI * freq * d->t); + } + + *out = creal(value); } -static int dp_series_destroy(struct dp_series *s) +static int dp_start(struct hook *h) { - free(s->history); + struct dp *d = (struct dp *) h->_vd; + + d->t = 0; + + double cycle = 1.0 / d->f0; + + /* Delay for one cycle */ + ret = delay_init(&d->history, d->dt, cycle); + if (ret) + return ret; return 0; } static int dp_init(struct hook *h) { - struct dp *p = (struct dp *) h->_vd; + struct dp *d = (struct dp *) h->_vd; - list_init(&p->series); + /* Default values */ + d->inverse = 0; return 0; } -static int dp_deinit(struct hook *h) +static int dp_destroy(struct hook *h) { - struct dp *p = (struct dp *) h->_vd; + struct dp *d = (struct dp *) h->_vd; - bitset_destroy(p->mask, (dtor_cb_t) dp_series_destroy, true); + /* Release memory */ + free(d->history); + free(d->fharmonics); + free(d->coeffs); return 0; } static int dp_parse(struct hook *h, json_t *cfg) { - struct dp *p = (struct dp *) h->_vd; + struct dp *d = (struct dp *) h->_vd; int ret; json_error_t err; - const char *mode = NULL; + json_t *json_harmonics, *json_harmonic; + size_t i; - ret = json_unpack_ex(cfg, &err, 0, "{ s: F, s?: i }", - "f0", &p->f0, - "mask", &p->mask + double rate = -1, dt = -1; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: F, s: F, s: o, s?: b }", + "index", &d->index, + "f0", &d->f0, + "dt", &dt, + "rate", &rate, + "harmonics", &json_harmonics, + "inverse", &d->inverse ); if (ret) jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); - return 0; -} + if (rate > 0) + d->dt = 1. / rate; + else if (dt > 0) + d->dt = dt; + else + error("Either on of the settings 'dt' or 'rate' must be gived for hook '%s'", plugin_name(h->_vt)); -static int dp_read(struct hook *h, struct sample *smps[], unsigned *cnt) -{ - struct dp *p = (struct dp *) h->_vd; + if (!json_is_array(json_harmonics)) + error("Setting 'harmonics' of hook '%s' must be a list of integers", plugin_name(h->_vt)); - for (unsigned j = 0; j < cnt; j++) { - struct sample *t = smps[j]; + d->fharmonics_len = json_array_size(json_harmonics); + d->fharmonics = alloc(d->fharmonics_len * sizeof(double)); + d->coeffs = alloc(d->fharmonics_len * sizeof(double complex)); + if (!d->fharmonics || !d->coeffs) + return -1; - for (size_t i = 0; i < list_length(&p->series); i++) { - struct dp_series *s = (struct dp_series *) list_at(&p->series, i); + json_array_foreach(json_harmonics, i, json_harmonic) { + if (!json_is_integer(json_harmonic)) + error("Setting 'harmonics' of hook '%s' must be a list of integers", plugin_name(h->_vt)); - if (s->idx > t->length) - continue; - } + d->fharmonics[i] = d->f0 * json_integer_value(json_harmonic); } return 0; } -static int dp_write(struct hook *h, struct sample *smps[], unsigned *cnt) +static int dp_process(struct hook *h, struct sample *smps[], unsigned *cnt) { - struct dp *p = (struct dp *) h->_vd; + struct dp *d = (struct dp *) h->_vd; + + for (unsigned j = 0; j < *cnt; j++) { + struct sample *smp = smps[j]; + + if (d->index > smp->length) + continue; + + struct signal *s = (struct signal *) vlist_at(smp->signals, d->index); + + if (d->inverse) { + if (s->type != SIGNAL_TYPE_FLOAT) + return -1; + + dp_istep(d, &smp->data[d->index].z, &smp->data[d->index].f); + } + else { + if (s->type != SIGNAL_TYPE_COMPLEX) + return -1; + + dp_step(d, &smp->data[d->index].f, &smp->data[d->index].z); + } + } + + d->t += d->dt; return 0; } static struct plugin p = { .name = "dp", - .description = "Transform to dynamic phasor domain", + .description = "Transform to/from dynamic phasor domain", .type = PLUGIN_TYPE_HOOK, .hook = { - .flags = HOOK_NODE, - .priority = 99, - .init = dp_init, - .parse = dp_parse, - .read = dp_read, - .write = dp_write, - .size = sizeof(struct dp) + .flags = HOOK_PATH, + .priority = 99, + .init = dp_init, + .destroy = dp_destroy, + .start = dp_start, + .parse = dp_parse, + .process = dp_process, + .size = sizeof(struct dp) } }; From fac3feecdab46c5f1bf39b6c2529efb875a5d84d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 16 Jan 2019 21:43:49 +0100 Subject: [PATCH 04/38] update comments --- include/villas/node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/villas/node.h b/include/villas/node.h index c97bf7914..75ed11670 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -57,7 +57,7 @@ struct node_direction { int builtin; /**< This node should use built-in hooks by default. */ int vectorize; /**< Number of messages to send / recv at once (scatter / gather) */ - struct vlist hooks; /**< List of write hooks (struct hook). */ + struct vlist hooks; /**< List of read / write hooks (struct hook). */ struct vlist signals; /**< Signal description. */ json_t *cfg; /**< A JSON object containing the configuration of the node. */ From d3e4616b32259359f5da6dfccdfb110ed6b76a4a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 17 Feb 2019 22:46:52 +0100 Subject: [PATCH 05/38] add README for running integration tests --- tests/integration/README.md | 20 ++++++++++++++++++++ tests/integration/run.sh | 1 - 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/integration/README.md delete mode 120000 tests/integration/run.sh diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 000000000..85c6cbfd7 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,20 @@ +# Integration Tests + +Run tests: + +``` +$ BUILDDIR=/VILLASnode/build/ /VILLASnode/tools/integration-tests.sh +``` + +There are two options for the test script: + +``` +-v Show full test output +-f FILTER Filter test cases +``` + +Example: + +``` +$ BUILDDIR=/VILLASnode/build/ /VILLASnode/tools/integration-tests.sh -f pipe-loopback-socket -v +``` diff --git a/tests/integration/run.sh b/tests/integration/run.sh deleted file mode 120000 index 33f51f9ce..000000000 --- a/tests/integration/run.sh +++ /dev/null @@ -1 +0,0 @@ -../../tools/integration-tests.sh \ No newline at end of file From 11ff9831400fc637f9ba843dfef808224f3b61d9 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 17 Feb 2019 22:49:44 +0100 Subject: [PATCH 06/38] hooks: disable broken cast hook --- lib/hooks/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hooks/CMakeLists.txt b/lib/hooks/CMakeLists.txt index afc12824d..c9b2f1de7 100644 --- a/lib/hooks/CMakeLists.txt +++ b/lib/hooks/CMakeLists.txt @@ -33,7 +33,7 @@ set(HOOK_SRC limit_rate.c scale.c fix.c - cast.c +# cast.c average.c dump.c dp.c From 70158411ae5a9ee53ecc7834e343848decaa26da Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 17 Feb 2019 23:43:59 +0100 Subject: [PATCH 07/38] dp: wip --- lib/hooks/dp.c | 65 ++++++++++++++++---------------------------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/lib/hooks/dp.c b/lib/hooks/dp.c index b0aafc5fc..2f8298fd6 100644 --- a/lib/hooks/dp.c +++ b/lib/hooks/dp.c @@ -31,45 +31,10 @@ #include #include #include +#include #define J _Complex_I -struct delay { - double *data; - size_t steps; - size_t mask; - size_t pos; -} - -int delay_init(struct delay *w, double dt, double period) -{ - size_t len = LOG2_CEIL(period / dt); - - /* Allocate memory for ciruclar history buffer */ - w->data = alloc(len * sizeof(double)); - if (!w->data) - return -1; - - w->pos = 0; - w->mask = len - 1; - - return 0; -} - -int delay_destroy(struct delay *w) -{ - free(w->data); -} - -double delay_update(struct delay *w, double in) -{ - double out = w->data[pos & w->mask]; - - w->data[w->pos++] - - return out; -} - struct dp { int index; int inverse; @@ -82,13 +47,13 @@ struct dp { int *fharmonics; int fharmonics_len; - struct window history; -} + struct window window; +}; static void dp_step(struct dp *d, double *in, float complex *out) { double newest = *in; - double oldest = delay_update(&d->hist, newest); + double oldest = window_update(&d->window, newest); for (int i = 0; i < d->fharmonics_len; i++) { double pi_fharm = 2.0 * M_PI * d->fharmonics[i]; @@ -97,8 +62,8 @@ static void dp_step(struct dp *d, double *in, float complex *out) d->coeffs[i] = (d->coeffs[i] + (newest - oldest)) * cexp(pi_fharm); /* Correction for stationary phasor */ - double complex correction = cexp(pi_fharm * (d->t - (d->history_len + 1))); - double complex result = 2.0 / d->history_len * d->coeffs[i] / correction; + double complex correction = cexp(pi_fharm * (d->t - (d->window.steps + 1))); + double complex result = 2.0 / d->window.steps * d->coeffs[i] / correction; /* DC component */ if (i == 0) @@ -125,14 +90,24 @@ static void dp_istep(struct dp *d, complex float *in, double *out) static int dp_start(struct hook *h) { + int ret; struct dp *d = (struct dp *) h->_vd; d->t = 0; - double cycle = 1.0 / d->f0; + ret = window_init(&d->window, (1.0 / d->f0) / d->dt, 0.0); + if (ret) + return ret; - /* Delay for one cycle */ - ret = delay_init(&d->history, d->dt, cycle); + return 0; +} + +static int dp_stop(struct hook *h) +{ + int ret; + struct dp *d = (struct dp *) h->_vd; + + ret = window_destroy(&d->window); if (ret) return ret; @@ -154,7 +129,6 @@ static int dp_destroy(struct hook *h) struct dp *d = (struct dp *) h->_vd; /* Release memory */ - free(d->history); free(d->fharmonics); free(d->coeffs); @@ -250,6 +224,7 @@ static struct plugin p = { .init = dp_init, .destroy = dp_destroy, .start = dp_start, + .stop = dp_stop, .parse = dp_parse, .process = dp_process, .size = sizeof(struct dp) From 82954a2853a4898493bd0060115bdf48d018ce4e Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 17 Feb 2019 23:44:22 +0100 Subject: [PATCH 08/38] ci: fix docker build --- .gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf7939af5..9a04b8e92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,9 @@ before_script: docker-dev: stage: prepare script: - - docker build -f packaging/docker/Dockerfile.dev -t ${DOCKER_IMAGE_DEV}:${DOCKER_TAG_DEV} . + - docker build + --file packaging/docker/Dockerfile.dev + --tag ${DOCKER_IMAGE_DEV}:${DOCKER_TAG_DEV} . tags: - shell - linux @@ -170,7 +172,10 @@ deploy:packages: docker: stage: docker script: - - docker build -f packaging/docker/Dockerfile.app -t ${DOCKER_IMAGE}:${DOCKER_TAG} . + - docker build + --build-arg BUILDER_IMAGE=${DOCKER_IMAGE_DEV}:${DOCKER_TAG_DEV} + --file packaging/docker/Dockerfile.app + --tag ${DOCKER_IMAGE}:${DOCKER_TAG} . - docker push ${DOCKER_IMAGE}:${DOCKER_TAG} - docker push ${DOCKER_IMAGE_DEV}:${DOCKER_TAG_DEV} tags: From 5b553f249669e2ac40cbe0f772011bcca6ae95fd Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:09:33 +0100 Subject: [PATCH 09/38] refactor: use strtok_r() instead of strtok() --- lib/mapping.c | 22 +++++++++++----------- lib/nodes/infiniband.c | 11 +++++------ lib/nodes/influxdb.c | 6 +++--- lib/nodes/socket.c | 12 +++++++----- lib/nodes/stats.c | 8 ++++---- lib/nodes/websocket.c | 6 +++--- 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/lib/mapping.c b/lib/mapping.c index 5c89fd4ee..7069d013b 100644 --- a/lib/mapping.c +++ b/lib/mapping.c @@ -32,14 +32,14 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *nodes) { - char *cpy, *node, *type, *field, *end; + char *cpy, *node, *type, *field, *end, *lasts; cpy = strdup(str); if (!cpy) return -1; if (nodes) { - node = strtok(cpy, "."); + node = strtok_r(cpy, ".", &lasts); if (!node) { warning("Missing node name"); goto invalid_format; @@ -51,14 +51,14 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n goto invalid_format; } - type = strtok(NULL, ".["); + type = strtok_r(NULL, ".[", &lasts); if (!type) type = "data"; } else { me->node = NULL; - type = strtok(cpy, ".["); + type = strtok_r(cpy, ".[", &lasts); if (!type) goto invalid_format; } @@ -67,11 +67,11 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n me->type = MAPPING_TYPE_STATS; me->length = 1; - char *metric = strtok(NULL, "."); + char *metric = strtok_r(NULL, ".", &lasts); if (!metric) goto invalid_format; - type = strtok(NULL, "."); + type = strtok_r(NULL, ".", &lasts); if (!type) goto invalid_format; @@ -87,7 +87,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n me->type = MAPPING_TYPE_HEADER; me->length = 1; - field = strtok(NULL, "."); + field = strtok_r(NULL, ".", &lasts); if (!field) { warning("Missing header type"); goto invalid_format; @@ -106,7 +106,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n me->type = MAPPING_TYPE_TIMESTAMP; me->length = 2; - field = strtok(NULL, "."); + field = strtok_r(NULL, ".", &lasts); if (!field) { warning("Missing timestamp type"); goto invalid_format; @@ -127,7 +127,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n me->type = MAPPING_TYPE_DATA; - first_str = strtok(NULL, "-]"); + first_str = strtok_r(NULL, "-]", &lasts); if (first_str) { if (me->node) first = vlist_lookup_index(&me->node->in.signals, first_str); @@ -148,7 +148,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n goto end; } - last_str = strtok(NULL, "]"); + last_str = strtok_r(NULL, "]", &lasts); if (last_str) { if (me->node) last = vlist_lookup_index(&me->node->in.signals, last_str); @@ -175,7 +175,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n goto invalid_format; end: /* Check that there is no garbage at the end */ - end = strtok(NULL, ""); + end = strtok_r(NULL, "", &lasts); if (end) goto invalid_format; diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c index 063542b57..a1cf20ae3 100644 --- a/lib/nodes/infiniband.c +++ b/lib/nodes/infiniband.c @@ -174,8 +174,7 @@ int ib_parse(struct node *n, json_t *cfg) struct infiniband *ib = (struct infiniband *) n->_vd; int ret; - char *local = NULL; - char *remote = NULL; + char *local = NULL, *remote = NULL, *lasts; const char *transport_mode = "RC"; int timeout = 1000; int recv_cq_size = 128; @@ -255,8 +254,8 @@ int ib_parse(struct node *n, json_t *cfg) debug(LOG_IB | 4, "Set buffer subtraction to %i in node %s", buffer_subtraction, node_name(n)); // Translate IP:PORT to a struct addrinfo - char* ip_adr = strtok(local, ":"); - char* port = strtok(NULL, ":"); + char* ip_adr = strtok_r(local, ":", &lasts); + char* port = strtok_r(NULL, ":", &lasts); ret = getaddrinfo(ip_adr, port, NULL, &ib->conn.src_addr); if (ret) @@ -327,8 +326,8 @@ int ib_parse(struct node *n, json_t *cfg) // If node will send data, set remote address if (ib->is_source) { // Translate address info - char* ip_adr = strtok(remote, ":"); - char* port = strtok(NULL, ":"); + char *ip_adr = strtok_r(remote, ":", &lasts); + char *port = strtok_r(NULL, ":", &lasts); ret = getaddrinfo(ip_adr, port, NULL, &ib->conn.dst_addr); if (ret) diff --git a/lib/nodes/influxdb.c b/lib/nodes/influxdb.c index 2b22c3b63..cf57192d3 100644 --- a/lib/nodes/influxdb.c +++ b/lib/nodes/influxdb.c @@ -40,7 +40,7 @@ int influxdb_parse(struct node *n, json_t *json) json_error_t err; int ret; - char *tmp, *host, *port; + char *tmp, *host, *port, *lasts; const char *server, *key; ret = json_unpack_ex(json, &err, 0, "{ s: s, s: s, s?: o }", @@ -52,8 +52,8 @@ int influxdb_parse(struct node *n, json_t *json) tmp = strdup(server); - host = strtok(tmp, ":"); - port = strtok(NULL, ""); + host = strtok_r(tmp, ":", &lasts); + port = strtok_r(NULL, "", &lasts); i->key = strdup(key); i->host = strdup(host); diff --git a/lib/nodes/socket.c b/lib/nodes/socket.c index 9d8fb2c28..f56b1f610 100644 --- a/lib/nodes/socket.c +++ b/lib/nodes/socket.c @@ -626,9 +626,10 @@ int socket_parse_address(const char *addr, struct sockaddr *saddr, enum socket_l #ifdef WITH_SOCKET_LAYER_ETH else if (layer == SOCKET_LAYER_ETH) { /* Format: "ab:cd:ef:12:34:56%ifname:protocol" */ /* Split string */ - char *node = strtok(copy, "%"); - char *ifname = strtok(NULL, ":"); - char *proto = strtok(NULL, "\0"); + char *lasts; + char *node = strtok_r(copy, "%", &lasts); + char *ifname = strtok_r(NULL, ":", &lasts); + char *proto = strtok_r(NULL, "\0", &lasts); /* Parse link layer (MAC) address */ struct ether_addr *mac = ether_aton(node); @@ -659,8 +660,9 @@ int socket_parse_address(const char *addr, struct sockaddr *saddr, enum socket_l }; /* Split string */ - char *node = strtok(copy, ":"); - char *service = strtok(NULL, "\0"); + char *lasts; + char *node = strtok_r(copy, ":", &lasts); + char *service = strtok_r(NULL, "\0", &lasts); if (node && !strcmp(node, "*")) node = NULL; diff --git a/lib/nodes/stats.c b/lib/nodes/stats.c index 540c33abe..e60455758 100644 --- a/lib/nodes/stats.c +++ b/lib/nodes/stats.c @@ -51,7 +51,7 @@ int stats_node_signal_parse(struct stats_node_signal *s, json_t *cfg) int ret; const char *stats; - char *metric, *type, *node, *cpy; + char *metric, *type, *node, *cpy, *lasts; ret = json_unpack_ex(cfg, &err, 0, "{ s: s }", "stats", &stats @@ -61,15 +61,15 @@ int stats_node_signal_parse(struct stats_node_signal *s, json_t *cfg) cpy = strdup(stats); - node = strtok(cpy, "."); + node = strtok_r(cpy, ".", &lasts); if (!node) goto invalid_format; - metric = strtok(NULL, "."); + metric = strtok_r(NULL, ".", &lasts); if (!metric) goto invalid_format; - type = strtok(NULL, "."); + type = strtok_r(NULL, ".", &lasts); if (!type) goto invalid_format; diff --git a/lib/nodes/websocket.c b/lib/nodes/websocket.c index 308e7e16d..7f67b25c7 100644 --- a/lib/nodes/websocket.c +++ b/lib/nodes/websocket.c @@ -203,7 +203,7 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi */ /* Get path of incoming request */ - char *node, *format; + char *node, *format, *lasts; char uri[64]; lws_hdr_copy(wsi, uri, sizeof(uri), WSI_TOKEN_GET_URI); /* The path component of the*/ @@ -213,14 +213,14 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi return -1; } - node = strtok(uri, "/."); + node = strtok_r(uri, "/.", &lasts); if (!node) { websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_POLICY_VIOLATION, "Unknown node"); warning("Failed to tokenize request URI"); return -1; } - format = strtok(NULL, ""); + format = strtok_r(NULL, "", &lasts); if (!format) format = "villas.web"; From a2b5825a4b4a2bc2dfd37abf0e6214cc62a48959 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:10:45 +0100 Subject: [PATCH 10/38] config: simplify command line configuration for arrays --- lib/config_helper.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/config_helper.c b/lib/config_helper.c index 23b924598..6c3b7471a 100644 --- a/lib/config_helper.c +++ b/lib/config_helper.c @@ -165,9 +165,26 @@ int json_to_config(json_t *json, config_setting_t *parent) } #endif /* LIBCONFIG_FOUND */ +void json_object_extend_key_value_token(json_t *obj, const char *key, const char *value) +{ + char *str = strdup(value); + char *delim = ","; + + char *lasts; + char *token = strtok_r(str, delim, &lasts); + + while (token) { + json_object_extend_key_value(obj, key, token); + + token = strtok_r(NULL, delim, &lasts); + } + + free(str); +} + void json_object_extend_key_value(json_t *obj, const char *key, const char *value) { - char *end, *cpy, *key1, *key2; + char *end, *cpy, *key1, *key2, *lasts; double real; long integer; @@ -178,8 +195,8 @@ void json_object_extend_key_value(json_t *obj, const char *key, const char *valu subobj = obj; cpy = strdup(key); - key1 = strtok(cpy, "."); - key2 = strtok(NULL, "."); + key1 = strtok_r(cpy, ".", &lasts); + key2 = strtok_r(NULL, ".", &lasts); while (key1 && key2) { existing = json_object_get(subobj, key1); @@ -193,7 +210,7 @@ void json_object_extend_key_value(json_t *obj, const char *key, const char *valu } key1 = key2; - key2 = strtok(NULL, "."); + key2 = strtok_r(NULL, ".", &lasts); } /* Try to parse as integer */ @@ -254,7 +271,7 @@ json_t * json_load_cli(int argc, const char *argv[]) const char *key = NULL; const char *value = NULL; const char *sep; - char *cpy; + char *cpy, *lasts; json_t *json = json_object(); @@ -274,10 +291,10 @@ json_t * json_load_cli(int argc, const char *argv[]) if (sep) { cpy = strdup(key); - key = strtok(cpy, "="); - value = strtok(NULL, ""); + key = strtok_r(cpy, "=", &lasts); + value = strtok_r(NULL, "", &lasts); - json_object_extend_key_value(json, key, value); + json_object_extend_key_value_token(json, key, value); free(cpy); key = NULL; @@ -291,7 +308,7 @@ json_t * json_load_cli(int argc, const char *argv[]) value = opt; - json_object_extend_key_value(json, key, value); + json_object_extend_key_value_token(json, key, value); key = NULL; } } @@ -324,17 +341,17 @@ int json_object_extend(json_t *obj, json_t *merge) int json_object_extend_str(json_t *obj, const char *str) { - char *key, *value, *cpy; + char *key, *value, *cpy, *lasts; cpy = strdup(str); - key = strtok(cpy, "="); - value = strtok(NULL, ""); + key = strtok_r(cpy, "=", &lasts); + value = strtok_r(NULL, "", &lasts); if (!key || !value) return -1; - json_object_extend_key_value(obj, key, value); + json_object_extend_key_value_token(obj, key, value); free(cpy); From 64357a0ce08fbf0617445626cb7f4bc42d0814a7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:10:51 +0100 Subject: [PATCH 11/38] remote whitespace --- lib/io.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/io.c b/lib/io.c index 740d08cb8..75bc916e9 100644 --- a/lib/io.c +++ b/lib/io.c @@ -115,7 +115,6 @@ int io_init_auto(struct io *io, const struct format_type *fmt, int len, int flag struct vlist *signals; signals = alloc(sizeof(struct vlist)); - signals->state = STATE_DESTROYED; ret = vlist_init(signals); From d19ae55c71ecdb3ba6de6a88991b6ceecdc7a5ec Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:11:16 +0100 Subject: [PATCH 12/38] hook: propagate error in hook_process() --- src/villas-hook.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/villas-hook.cpp b/src/villas-hook.cpp index 45f9288fd..2d30b86e9 100644 --- a/src/villas-hook.cpp +++ b/src/villas-hook.cpp @@ -215,7 +215,9 @@ check: if (optarg == endptr) unsigned send = recv; - hook_process(&h, smps, (unsigned *) &send); + ret = hook_process(&h, smps, (unsigned *) &send); + if (ret < 0) + throw RuntimeError("Failed to process samples"); sent = io_print(&io, smps, send); if (sent < 0) From 40d4e55546b8c91d0c609a8a99c2cd9f403a0dfb Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:11:34 +0100 Subject: [PATCH 13/38] dp: fix signal handling (for now) --- lib/hooks/dp.c | 53 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/hooks/dp.c b/lib/hooks/dp.c index 2f8298fd6..e727b2452 100644 --- a/lib/hooks/dp.c +++ b/lib/hooks/dp.c @@ -48,6 +48,8 @@ struct dp { int fharmonics_len; struct window window; + + struct vlist *signals; }; static void dp_step(struct dp *d, double *in, float complex *out) @@ -94,6 +96,10 @@ static int dp_start(struct hook *h) struct dp *d = (struct dp *) h->_vd; d->t = 0; + d->signals = NULL; + + for (int i = 0; i < d->fharmonics_len; i++) + d->coeffs[i] = 0; ret = window_init(&d->window, (1.0 / d->f0) / d->dt, 0.0); if (ret) @@ -146,7 +152,7 @@ static int dp_parse(struct hook *h, json_t *cfg) double rate = -1, dt = -1; - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: F, s: F, s: o, s?: b }", + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: F, s?: F, s?: F, s: o, s?: b }", "index", &d->index, "f0", &d->f0, "dt", &dt, @@ -193,20 +199,47 @@ static int dp_process(struct hook *h, struct sample *smps[], unsigned *cnt) if (d->index > smp->length) continue; - struct signal *s = (struct signal *) vlist_at(smp->signals, d->index); + if (!d->signals) { + struct signal *orig_sig, *new_sig; - if (d->inverse) { - if (s->type != SIGNAL_TYPE_FLOAT) + d->signals = alloc(sizeof(struct vlist)); + d->signals->state = STATE_DESTROYED; + + vlist_copy(d->signals, smp->signals); + + orig_sig = vlist_at(smp->signals, d->index); + if (!orig_sig) return -1; + if (d->inverse) { + if (orig_sig->type != SIGNAL_TYPE_COMPLEX) + return -1; + } + else { + if (orig_sig->type != SIGNAL_TYPE_FLOAT) + return -1; + } + + new_sig = signal_copy(orig_sig); + if (!new_sig) + return -1; + + if (d->inverse) + new_sig->type = SIGNAL_TYPE_FLOAT; + else + new_sig->type = SIGNAL_TYPE_COMPLEX; + + int ret = vlist_set(d->signals, d->index, new_sig); + if (ret) + return ret; + } + + smp->signals = d->signals; + + if (d->inverse) dp_istep(d, &smp->data[d->index].z, &smp->data[d->index].f); - } - else { - if (s->type != SIGNAL_TYPE_COMPLEX) - return -1; - + else dp_step(d, &smp->data[d->index].f, &smp->data[d->index].z); - } } d->t += d->dt; From 4147826a96361b96ec14d743297fc2cb7dd924c1 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:11:47 +0100 Subject: [PATCH 14/38] tests: started with test for new dp hook --- tests/integration/hook-dp.sh | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 tests/integration/hook-dp.sh diff --git a/tests/integration/hook-dp.sh b/tests/integration/hook-dp.sh new file mode 100755 index 000000000..b09219163 --- /dev/null +++ b/tests/integration/hook-dp.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Integration test for dp hook. +# +# @author Steffen Vogel +# @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################## + +set -x + +INPUT_FILE=$(mktemp) +OUTPUT_FILE=$(mktemp) +EXPECT_FILE=$(mktemp) + +NUM_SAMPLES=40000 +RATE=5000 +F0=50 + +OPTS="-d debug -o f0=${F0} -o rate=${RATE} -o index=0 -o harmonics=0,1,3,5" + +villas-signal sine -v1 -l ${NUM_SAMPLES} -f ${F0} -r ${RATE} -n > ${INPUT_FILE} + +villas-hook dp -o inverse=false ${OPTS} < ${INPUT_FILE} > ${OUTPUT_FILE} + +cat ${OUTPUT_FILE} + +exit 1 + +# Compare only the data values +villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} + +RC=$? + +rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} + +exit $RC From 520ed3283d4ab26e5e5afb3341f86f8ef1360484 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 01:12:50 +0100 Subject: [PATCH 15/38] update VILLAScommon submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index 43943147c..09832330c 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 43943147c771d6be1c840e93cd469e5e596bec94 +Subproject commit 09832330c1e88710a60d3ed5db785c1403c50735 From fa6d3d87eef0d87b3f9840b424e7aa7c81aa3a9b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 18:05:05 +0100 Subject: [PATCH 16/38] python: add Sample class to parse villas.human format --- python/villas/node/sample.py | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 python/villas/node/sample.py diff --git a/python/villas/node/sample.py b/python/villas/node/sample.py new file mode 100644 index 000000000..8a37e09ac --- /dev/null +++ b/python/villas/node/sample.py @@ -0,0 +1,83 @@ +import re + +class Timestamp: + """Parsing the VILLASnode human-readable timestamp format""" + + def __init__(self, seconds = 0, nanoseconds = None, offset = None, sequence = None): + self.seconds = seconds + self.nanoseconds = nanoseconds + self.offset = offset + self.sequence = sequence + + @classmethod + def parse(self, ts): + m = re.match('(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?(?:e[+-]?\d+)?)?(?:\((\d+)\))?', ts) + + seconds = int(m.group(1)); # Mandatory + nanoseconds = int(m.group(2)) if m.group(2) else None + offset = float(m.group(3)) if m.group(3) else None + sequence = int(m.group(4)) if m.group(4) else None + + return Timestamp(seconds, nanoseconds, offset, sequence) + + def __str__(self): + str = "%u" % (self.seconds) + + if self.nanoseconds is not None: + str += ".%09u" % self.nanoseconds + if self.offset is not None: + str += "+%u" % self.offset + if self.sequence is not None: + str += "(%u)" % self.sequence + + return str + + def __float__(self): + sum = float(self.seconds) + + if self.nanoseconds is not None: + sum += self.nanoseconds * 1e-9 + if self.offset is not None: + sum += self.offset + + return sum + + def __cmp__(self, other): + return cmp(float(self), float(other)) + +class Sample: + """Parsing a VILLASnode sample from a file (not a UDP package!!)""" + + def __init__(self, ts, values): + self.ts = ts + self.values = values + + @classmethod + def parse(self, line): + csv = line.split() + + ts = Timestamp.parse(csv[0]) + vs = [ ] + + for value in csv[1:]: + try: + v = float(value) + except ValueError: + value = value.lower() + try: + v = complex(value) + except: + if value.endswith('i'): + v = complex(value.replace('i', 'j')) + else: + raise ValueError() + + vs.append(v) + + return Sample(ts, vs) + + def __str__(self): + return '%s %s' % (self.ts, " ".join(map(str, self.values))) + + def __cmp__(self, other): + return cmp(self.ts, other.ts) From c4651cf56f9a026fd785c3e41142c21f20ef406c Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Feb 2019 18:05:18 +0100 Subject: [PATCH 17/38] python: fix URL in setup.py --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 9afba3213..f2259cdeb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -13,7 +13,7 @@ setup( description = 'Python-support for VILLASnode simulation-data gateway', license = 'GPL-3.0', keywords = 'simulation power system real-time villas', - url = 'https://git.rwth-aachen.de/acs/public/villas/dataprocessing', + url = 'https://git.rwth-aachen.de/acs/public/villas/VILLASnode', packages = [ 'villas.node' ], long_description = long_description, long_description_content_type = 'text/markdown', From 78bcb67245773bd226ad6edc47b1f0e045bc9b33 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 09:06:48 +0100 Subject: [PATCH 18/38] refactor: _init2() -> _prepare() --- include/villas/node.h | 2 +- include/villas/path.h | 2 +- lib/node.c | 8 ++++---- lib/path.c | 2 +- lib/super_node.cpp | 4 ++-- src/villas-pipe.cpp | 2 +- src/villas-signal.cpp | 2 +- src/villas-test-rtt.cpp | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/villas/node.h b/include/villas/node.h index 75ed11670..1616d7084 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -103,7 +103,7 @@ struct node { int node_init(struct node *n, struct node_type *vt); /** Do initialization after parsing the configuration */ -int node_init2(struct node *n); +int node_prepare(struct node *n); /** Parse settings of a node. * diff --git a/include/villas/path.h b/include/villas/path.h index 622c263a3..6024139c3 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -112,7 +112,7 @@ struct path { /** Initialize internal data structures. */ int path_init(struct path *p); -int path_init2(struct path *p); +int path_prepare(struct path *p); /** Check if path configuration is proper. */ int path_check(struct path *p); diff --git a/lib/node.c b/lib/node.c index cec2e5b3a..b85beffbb 100644 --- a/lib/node.c +++ b/lib/node.c @@ -41,7 +41,7 @@ #include #endif /* WITH_NETEM */ -static int node_direction_init2(struct node_direction *nd, struct node *n) +int node_direction_prepare(struct node_direction *nd, struct node *n) { #ifdef WITH_HOOKS int ret; @@ -254,17 +254,17 @@ int node_init(struct node *n, struct node_type *vt) return 0; } -int node_init2(struct node *n) +int node_prepare(struct node *n) { int ret; assert(n->state == STATE_CHECKED); - ret = node_direction_init2(&n->in, n); + ret = node_direction_prepare(&n->in, n); if (ret) return ret; - ret = node_direction_init2(&n->out, n); + ret = node_direction_prepare(&n->in, n); if (ret) return ret; diff --git a/lib/path.c b/lib/path.c index 97528c938..8365d266b 100644 --- a/lib/path.c +++ b/lib/path.c @@ -404,7 +404,7 @@ int path_init_poll(struct path *p) return 0; } -int path_init2(struct path *p) +int path_prepare(struct path *p) { int ret; diff --git a/lib/super_node.cpp b/lib/super_node.cpp index 180636f27..de842388e 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -356,7 +356,7 @@ void SuperNode::startNodes() for (size_t i = 0; i < vlist_length(&nodes); i++) { auto *n = (struct node *) vlist_at(&nodes, i); - ret = node_init2(n); + ret = node_prepare(n); if (ret) throw RuntimeError("Failed to prepare node: {}", node_name(n)); @@ -379,7 +379,7 @@ void SuperNode::startPaths() auto *p = (struct path *) vlist_at(&paths, i); if (p->enabled) { - ret = path_init2(p); + ret = path_prepare(p); if (ret) throw RuntimeError("Failed to prepare path: {}", path_name(p)); diff --git a/src/villas-pipe.cpp b/src/villas-pipe.cpp index 51df16d56..0d679656f 100644 --- a/src/villas-pipe.cpp +++ b/src/villas-pipe.cpp @@ -395,7 +395,7 @@ check: if (optarg == endptr) if (ret) throw RuntimeError("Invalid node configuration"); - ret = node_init2(node); + ret = node_prepare(node); if (ret) throw RuntimeError("Failed to start node {}: reason={}", node_name(node), ret); diff --git a/src/villas-signal.cpp b/src/villas-signal.cpp index 594b2cdb0..4040cf7b6 100644 --- a/src/villas-signal.cpp +++ b/src/villas-signal.cpp @@ -237,7 +237,7 @@ int main(int argc, char *argv[]) if (ret) throw RuntimeError("Failed to initialize pool"); - ret = node_init2(&n); + ret = node_prepare(&n); if (ret) throw RuntimeError("Failed to start node {}: reason={}", node_name(&n), ret); diff --git a/src/villas-test-rtt.cpp b/src/villas-test-rtt.cpp index 8ae798a2c..8c6c1812f 100644 --- a/src/villas-test-rtt.cpp +++ b/src/villas-test-rtt.cpp @@ -167,7 +167,7 @@ check: if (optarg == endptr) if (ret) throw RuntimeError("Failed to start node-type {}: reason={}", node_type_name(node->_vt), ret); - ret = node_init2(node); + ret = node_prepare(node); if (ret) throw RuntimeError("Failed to start node {}: reason={}", node_name(node), ret); From f851aacf3ccd94b0ac31b22977696d6e4b288c14 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 09:10:18 +0100 Subject: [PATCH 19/38] refactor: rename signal_list_{init,destroy}() and node_list_parse() --- include/villas/node.h | 2 +- include/villas/signal.h | 4 ++-- lib/node.c | 6 +++--- lib/path.c | 4 ++-- lib/signal.c | 22 ++++++++++++++++++++++ 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/include/villas/node.h b/include/villas/node.h index 1616d7084..0673f2ef9 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -123,7 +123,7 @@ int node_parse(struct node *n, json_t *cfg, const char *name); * @param nodes The nodes will be added to this list. * @param all This list contains all valid nodes. */ -int node_parse_list(struct vlist *list, json_t *cfg, struct vlist *all); +int node_list_parse(struct vlist *list, json_t *cfg, struct vlist *all); /** Parse the list of signal definitions. */ int node_parse_signals(struct vlist *list, json_t *cfg); diff --git a/include/villas/signal.h b/include/villas/signal.h index d57f85851..d7af0e2f7 100644 --- a/include/villas/signal.h +++ b/include/villas/signal.h @@ -106,10 +106,10 @@ int signal_parse(struct signal *s, json_t *cfg); /** Initialize signal from a mapping_entry. */ int signal_init_from_mapping(struct signal *s, const struct mapping_entry *me, unsigned index); +int signal_list_init(struct vlist *list); +int signal_list_destroy(struct vlist *list); int signal_list_parse(struct vlist *list, json_t *cfg); - int signal_list_generate(struct vlist *list, unsigned len, enum signal_type fmt); - void signal_list_dump(const struct vlist *list, const union signal_data *data, int len); enum signal_type signal_type_from_str(const char *str); diff --git a/lib/node.c b/lib/node.c index b85beffbb..9a23bf183 100644 --- a/lib/node.c +++ b/lib/node.c @@ -78,7 +78,7 @@ static int node_direction_init(struct node_direction *nd, struct node *n) return ret; #endif /* WITH_HOOKS */ - ret = vlist_init(&nd->signals); + ret = signal_list_init(&nd->signals); if (ret) return ret; @@ -95,7 +95,7 @@ static int node_direction_destroy(struct node_direction *nd, struct node *n) return ret; #endif /* WITH_HOOKS */ - ret = vlist_destroy(&nd->signals, (dtor_cb_t) signal_decref, false); + ret = signal_list_destroy(&nd->signals); if (ret) return ret; @@ -716,7 +716,7 @@ struct memory_type * node_memory_type(struct node *n, struct memory_type *parent return node_type(n)->memory_type ? node_type(n)->memory_type(n, parent) : &memory_hugepage; } -int node_parse_list(struct vlist *list, json_t *cfg, struct vlist *all) +int node_list_parse(struct vlist *list, json_t *cfg, struct vlist *all) { struct node *node; const char *str; diff --git a/lib/path.c b/lib/path.c index 8365d266b..7bdb04b3f 100644 --- a/lib/path.c +++ b/lib/path.c @@ -333,7 +333,7 @@ int path_init(struct path *p) if (ret) return ret; - ret = vlist_init(&p->signals); + ret = signal_list_init(&p->signals); if (ret) return ret; @@ -553,7 +553,7 @@ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes) /* Output node(s) */ if (json_out) { - ret = node_parse_list(&destinations, json_out, nodes); + ret = node_list_parse(&destinations, json_out, nodes); if (ret) jerror(&err, "Failed to parse output nodes"); } diff --git a/lib/signal.c b/lib/signal.c index 4cd6e89d4..47016714d 100644 --- a/lib/signal.c +++ b/lib/signal.c @@ -210,6 +210,28 @@ int signal_parse(struct signal *s, json_t *cfg) /* Signal list */ +int signal_list_init(struct vlist *list) +{ + int ret; + + ret = vlist_init(list); + if (ret) + return ret; + + return 0; +} + +int signal_list_destroy(struct vlist *list) +{ + int ret; + + ret = vlist_destroy(list, (dtor_cb_t) signal_decref, false); + if (ret) + return ret; + + return 0; +} + int signal_list_parse(struct vlist *list, json_t *cfg) { int ret; From e8de9df993e58da2caf013fd46bc30fcb3019e28 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 09:23:31 +0100 Subject: [PATCH 20/38] update VILLAScommon submodule with new list functions --- common | 2 +- include/villas/plugin.h | 2 +- lib/node.c | 2 +- lib/nodes/websocket.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common b/common index 09832330c..0c35a58b7 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 09832330c1e88710a60d3ed5db785c1403c50735 +Subproject commit 0c35a58b79b00726d13675b7a6231dd35bf62a20 diff --git a/include/villas/plugin.h b/include/villas/plugin.h index 017b02189..d2a4c9571 100644 --- a/include/villas/plugin.h +++ b/include/villas/plugin.h @@ -53,7 +53,7 @@ __attribute__((constructor(110))) static void UNIQUE(__ctor)() {\ } \ __attribute__((destructor(110))) static void UNIQUE(__dtor)() { \ if (plugins.state != STATE_DESTROYED) \ - vlist_remove(&plugins, p); \ + vlist_remove_all(&plugins, p); \ } extern struct vlist plugins; diff --git a/lib/node.c b/lib/node.c index 9a23bf183..a25e0ff22 100644 --- a/lib/node.c +++ b/lib/node.c @@ -533,7 +533,7 @@ int node_destroy(struct node *n) return ret; } - vlist_remove(&node_type(n)->instances, n); + vlist_remove_all(&node_type(n)->instances, n); if (n->_vd) free(n->_vd); diff --git a/lib/nodes/websocket.c b/lib/nodes/websocket.c index 7f67b25c7..9a17ca0c6 100644 --- a/lib/nodes/websocket.c +++ b/lib/nodes/websocket.c @@ -267,7 +267,7 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi } if (connections.state == STATE_INITIALIZED) - vlist_remove(&connections, c); + vlist_remove_all(&connections, c); if (c->state == WEBSOCKET_CONNECTION_STATE_INITIALIZED) websocket_connection_destroy(c); From 3b99227537951eed2b48829e13533b5f86b30d24 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 09:39:41 +0100 Subject: [PATCH 21/38] path: separated path_{source, destination} --- include/villas/path.h | 15 -- include/villas/path_destination.h | 60 ++++++++ include/villas/path_source.h | 62 ++++++++ lib/CMakeLists.txt | 2 + lib/path.c | 247 +++--------------------------- lib/path_destination.c | 110 +++++++++++++ lib/path_source.c | 159 +++++++++++++++++++ 7 files changed, 417 insertions(+), 238 deletions(-) create mode 100644 include/villas/path_destination.h create mode 100644 include/villas/path_source.h create mode 100644 lib/path_destination.c create mode 100644 lib/path_source.c diff --git a/include/villas/path.h b/include/villas/path.h index 6024139c3..1abe9ab0c 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -49,21 +49,6 @@ extern "C" { struct stats; struct node; -struct path_source { - struct node *node; - - bool masked; - - struct pool pool; - struct vlist mappings; /**< List of mappings (struct mapping_entry). */ -}; - -struct path_destination { - struct node *node; - - struct queue queue; -}; - /** The register mode determines under which condition the path is triggered. */ enum path_mode { PATH_MODE_ANY, /**< The path is triggered whenever one of the sources receives samples. */ diff --git a/include/villas/path_destination.h b/include/villas/path_destination.h new file mode 100644 index 000000000..fede6ffb6 --- /dev/null +++ b/include/villas/path_destination.h @@ -0,0 +1,60 @@ +/** Path destination + * + * @file + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** A path connects one input node to multiple output nodes (1-to-n). + * + * @addtogroup path Path + * @{ + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declarations */ +struct path; +struct sample; + +struct path_destination { + struct node *node; + + struct queue queue; +}; + +int path_destination_init(struct path_destination *pd, int queuelen); + +int path_destination_destroy(struct path_destination *pd); + +void path_destination_enqueue(struct path *p, struct sample *smps[], unsigned cnt); + +void path_destination_write(struct path_destination *pd, struct path *p); + +#ifdef __cplusplus +} +#endif + +/** @} */ diff --git a/include/villas/path_source.h b/include/villas/path_source.h new file mode 100644 index 000000000..4114dcda2 --- /dev/null +++ b/include/villas/path_source.h @@ -0,0 +1,62 @@ +/** Message source + * + * @file + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** A path connects one input node to multiple output nodes (1-to-n). + * + * @addtogroup path Path + * @{ + */ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declarations */ +struct path; +struct sample; + +struct path_source { + struct node *node; + + bool masked; + + struct pool pool; + struct vlist mappings; /**< List of mappings (struct mapping_entry). */ +}; + +int path_source_init(struct path_source *ps); + +int path_source_destroy(struct path_source *ps); + +int path_source_read(struct path_source *ps, struct path *p, int i); + +#ifdef __cplusplus +} +#endif + +/** @} */ diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 22a797a8e..85318fcbb 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -46,6 +46,8 @@ set(LIB_SRC memory/managed.c sample.c path.c + path_source.c + path_destination.c node.c memory.c plugin.c diff --git a/lib/path.c b/lib/path.c index 7bdb04b3f..9f1288bb3 100644 --- a/lib/path.c +++ b/lib/path.c @@ -29,7 +29,6 @@ #include #include -#include #include #include #include @@ -39,221 +38,9 @@ #include #include #include - -/* Forward declaration */ -static void path_destination_enqueue(struct path *p, struct sample *smps[], unsigned cnt); - -static int path_source_init(struct path_source *ps) -{ - int ret; - int pool_size = MAX(DEFAULT_QUEUE_LENGTH, ps->node->in.vectorize); - - if (ps->node->_vt->pool_size) - pool_size = ps->node->_vt->pool_size; - - ret = pool_init(&ps->pool, pool_size, SAMPLE_LENGTH(vlist_length(&ps->node->in.signals)), node_memory_type(ps->node, &memory_hugepage)); - if (ret) - return ret; - - return 0; -} - -static int path_source_destroy(struct path_source *ps) -{ - int ret; - - ret = pool_destroy(&ps->pool); - if (ret) - return ret; - - ret = vlist_destroy(&ps->mappings, NULL, true); - if (ret) - return ret; - - return 0; -} - -static int path_source_read(struct path_source *ps, struct path *p, int i) -{ - int recv, tomux, allocated, cnt, toenqueue, enqueued = 0; - unsigned release; - - cnt = ps->node->in.vectorize; - - struct sample *read_smps[cnt]; - struct sample *muxed_smps[cnt]; - struct sample **tomux_smps; - - /* Fill smps[] free sample blocks from the pool */ - allocated = sample_alloc_many(&ps->pool, read_smps, cnt); - if (allocated != cnt) - warning("Pool underrun for path source %s", node_name(ps->node)); - - /* Read ready samples and store them to blocks pointed by smps[] */ - release = allocated; - - recv = node_read(ps->node, read_smps, allocated, &release); - if (recv == 0) { - enqueued = 0; - goto out2; - } - else if (recv < 0) { - if (ps->node->state == STATE_STOPPING) { - p->state = STATE_STOPPING; - - enqueued = -1; - goto out2; - } - else - error("Failed to read samples from node %s", node_name(ps->node)); - } - else if (recv < allocated) - warning("Partial read for path %s: read=%u, expected=%u", path_name(p), recv, allocated); - - bitset_set(&p->received, i); - - if (p->mode == PATH_MODE_ANY) { /* Mux all samples */ - tomux_smps = read_smps; - tomux = recv; - } - else { /* Mux only last sample and discard others */ - tomux_smps = read_smps + recv - 1; - tomux = 1; - } - - for (int i = 0; i < tomux; i++) { - muxed_smps[i] = i == 0 - ? sample_clone(p->last_sample) - : sample_clone(muxed_smps[i-1]); - - if (p->original_sequence_no) - muxed_smps[i]->sequence = tomux_smps[i]->sequence; - else { - muxed_smps[i]->sequence = p->last_sequence++; - muxed_smps[i]->flags |= SAMPLE_HAS_SEQUENCE; - } - - muxed_smps[i]->ts = tomux_smps[i]->ts; - muxed_smps[i]->flags |= tomux_smps[i]->flags & (SAMPLE_HAS_TS_ORIGIN | SAMPLE_HAS_TS_RECEIVED); - - mapping_remap(&ps->mappings, muxed_smps[i], tomux_smps[i], NULL); - } - - sample_copy(p->last_sample, muxed_smps[tomux-1]); - - debug(15, "Path %s received = %s", path_name(p), bitset_dump(&p->received)); - -#ifdef WITH_HOOKS - toenqueue = hook_process_list(&p->hooks, muxed_smps, tomux); - if (toenqueue != tomux) { - int skipped = tomux - toenqueue; - - debug(LOG_NODES | 10, "Hooks skipped %u out of %u samples for path %s", skipped, tomux, path_name(p)); - } -#else - toenqueue = tomux; -#endif - - if (bitset_test(&p->mask, i)) { - /* Check if we received an update from all nodes/ */ - if ((p->mode == PATH_MODE_ANY) || - (p->mode == PATH_MODE_ALL && !bitset_cmp(&p->mask, &p->received))) { - path_destination_enqueue(p, muxed_smps, toenqueue); - - /* Reset bitset of updated nodes */ - bitset_clear_all(&p->received); - - enqueued = toenqueue; - } - } - - sample_decref_many(muxed_smps, tomux); -out2: sample_decref_many(read_smps, release); - - return enqueued; -} - -static int path_destination_init(struct path_destination *pd, int queuelen) -{ - int ret; - - ret = queue_init(&pd->queue, queuelen, &memory_hugepage); - if (ret) - return ret; - - return 0; -} - -static int path_destination_destroy(struct path_destination *pd) -{ - int ret; - - ret = queue_destroy(&pd->queue); - if (ret) - return ret; - - return 0; -} - -static void path_destination_enqueue(struct path *p, struct sample *smps[], unsigned cnt) -{ - unsigned enqueued, cloned; - - struct sample *clones[cnt]; - - cloned = sample_clone_many(clones, smps, cnt); - if (cloned < cnt) - warning("Pool underrun in path %s", path_name(p)); - - for (size_t i = 0; i < vlist_length(&p->destinations); i++) { - struct path_destination *pd = (struct path_destination *) vlist_at(&p->destinations, i); - - enqueued = queue_push_many(&pd->queue, (void **) clones, cloned); - if (enqueued != cnt) - warning("Queue overrun for path %s", path_name(p)); - - /* Increase reference counter of these samples as they are now also owned by the queue. */ - sample_incref_many(clones, cloned); - - debug(LOG_PATH | 15, "Enqueued %u samples to destination %s of path %s", enqueued, node_name(pd->node), path_name(p)); - } - - sample_decref_many(clones, cloned); -} - -static void path_destination_write(struct path_destination *pd, struct path *p) -{ - int cnt = pd->node->out.vectorize; - int sent; - int released; - int allocated; - unsigned release; - - struct sample *smps[cnt]; - - /* As long as there are still samples in the queue */ - while (1) { - allocated = queue_pull_many(&pd->queue, (void **) smps, cnt); - if (allocated == 0) - break; - else if (allocated < cnt) - debug(LOG_PATH | 5, "Queue underrun for path %s: allocated=%u expected=%u", path_name(p), allocated, cnt); - - debug(LOG_PATH | 15, "Dequeued %u samples from queue of node %s which is part of path %s", allocated, node_name(pd->node), path_name(p)); - - release = allocated; - - sent = node_write(pd->node, smps, allocated, &release); - if (sent < 0) - error("Failed to sent %u samples to node %s: reason=%d", cnt, node_name(pd->node), sent); - else if (sent < allocated) - warning("Partial write to node %s: written=%d, expected=%d", node_name(pd->node), sent, allocated); - - released = sample_decref_many(smps, release); - - debug(LOG_PATH | 15, "Released %d samples back to memory pool", released); - } -} +#include +#include +#include static void * path_run_single(void *arg) { @@ -337,7 +124,7 @@ int path_init(struct path *p) if (ret) return ret; - ret = vlist_init(&p->hooks); + ret = hook_list_init(&p->hooks); if (ret) return ret; @@ -411,8 +198,10 @@ int path_prepare(struct path *p) assert(p->state == STATE_CHECKED); #ifdef WITH_HOOKS + int m = p->builtin ? HOOK_PATH | HOOK_BUILTIN : 0; + /* Add internal hooks if they are not already in the list */ - ret = hook_init_builtin_list(&p->hooks, p->builtin, HOOK_PATH, p, NULL); + ret = hook_list_prepare(&p->hooks, &p->signals, m, p, NULL); if (ret) return ret; @@ -646,7 +435,7 @@ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes) #ifdef WITH_HOOKS if (json_hooks) { - ret = hook_parse_list(&p->hooks, json_hooks, HOOK_PATH, p, NULL); + ret = hook_list_parse(&p->hooks, json_hooks, HOOK_PATH, p, NULL); if (ret) return ret; } @@ -846,15 +635,27 @@ int path_stop(struct path *p) int path_destroy(struct path *p) { + int ret; + if (p->state == STATE_DESTROYED) return 0; #ifdef WITH_HOOKS - vlist_destroy(&p->hooks, (dtor_cb_t) hook_destroy, true); + ret = hook_list_destroy(&p->hooks); + if (ret) + return ret; #endif - vlist_destroy(&p->sources, (dtor_cb_t) path_source_destroy, true); - vlist_destroy(&p->destinations, (dtor_cb_t) path_destination_destroy, true); - vlist_destroy(&p->signals, (dtor_cb_t) signal_decref, false); + ret = signal_list_destroy(&p->signals); + if (ret) + return ret; + + ret = vlist_destroy(&p->sources, (dtor_cb_t) path_source_destroy, true); + if (ret) + return ret; + + ret = vlist_destroy(&p->destinations, (dtor_cb_t) path_destination_destroy, true); + if (ret) + return ret; if (p->reader.pfds) free(p->reader.pfds); diff --git a/lib/path_destination.c b/lib/path_destination.c new file mode 100644 index 000000000..fa0d2c032 --- /dev/null +++ b/lib/path_destination.c @@ -0,0 +1,110 @@ +/** Path destination + * + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +int path_destination_init(struct path_destination *pd, int queuelen) +{ + int ret; + + ret = queue_init(&pd->queue, queuelen, &memory_hugepage); + if (ret) + return ret; + + return 0; +} + +int path_destination_destroy(struct path_destination *pd) +{ + int ret; + + ret = queue_destroy(&pd->queue); + if (ret) + return ret; + + return 0; +} + +void path_destination_enqueue(struct path *p, struct sample *smps[], unsigned cnt) +{ + unsigned enqueued, cloned; + + struct sample *clones[cnt]; + + cloned = sample_clone_many(clones, smps, cnt); + if (cloned < cnt) + warning("Pool underrun in path %s", path_name(p)); + + for (size_t i = 0; i < vlist_length(&p->destinations); i++) { + struct path_destination *pd = (struct path_destination *) vlist_at(&p->destinations, i); + + enqueued = queue_push_many(&pd->queue, (void **) clones, cloned); + if (enqueued != cnt) + warning("Queue overrun for path %s", path_name(p)); + + /* Increase reference counter of these samples as they are now also owned by the queue. */ + sample_incref_many(clones, cloned); + + debug(LOG_PATH | 15, "Enqueued %u samples to destination %s of path %s", enqueued, node_name(pd->node), path_name(p)); + } + + sample_decref_many(clones, cloned); +} + +void path_destination_write(struct path_destination *pd, struct path *p) +{ + int cnt = pd->node->out.vectorize; + int sent; + int released; + int allocated; + unsigned release; + + struct sample *smps[cnt]; + + /* As long as there are still samples in the queue */ + while (1) { + allocated = queue_pull_many(&pd->queue, (void **) smps, cnt); + if (allocated == 0) + break; + else if (allocated < cnt) + debug(LOG_PATH | 5, "Queue underrun for path %s: allocated=%u expected=%u", path_name(p), allocated, cnt); + + debug(LOG_PATH | 15, "Dequeued %u samples from queue of node %s which is part of path %s", allocated, node_name(pd->node), path_name(p)); + + release = allocated; + + sent = node_write(pd->node, smps, allocated, &release); + if (sent < 0) + error("Failed to sent %u samples to node %s: reason=%d", cnt, node_name(pd->node), sent); + else if (sent < allocated) + warning("Partial write to node %s: written=%d, expected=%d", node_name(pd->node), sent, allocated); + + released = sample_decref_many(smps, release); + + debug(LOG_PATH | 15, "Released %d samples back to memory pool", released); + } +} diff --git a/lib/path_source.c b/lib/path_source.c new file mode 100644 index 000000000..ae8225588 --- /dev/null +++ b/lib/path_source.c @@ -0,0 +1,159 @@ +/** Path source + * + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +int path_source_init(struct path_source *ps) +{ + int ret; + int pool_size = MAX(DEFAULT_QUEUE_LENGTH, ps->node->in.vectorize); + + if (ps->node->_vt->pool_size) + pool_size = ps->node->_vt->pool_size; + + ret = pool_init(&ps->pool, pool_size, SAMPLE_LENGTH(vlist_length(&ps->node->in.signals)), node_memory_type(ps->node, &memory_hugepage)); + if (ret) + return ret; + + return 0; +} + +int path_source_destroy(struct path_source *ps) +{ + int ret; + + ret = pool_destroy(&ps->pool); + if (ret) + return ret; + + ret = vlist_destroy(&ps->mappings, NULL, true); + if (ret) + return ret; + + return 0; +} + +int path_source_read(struct path_source *ps, struct path *p, int i) +{ + int recv, tomux, allocated, cnt, toenqueue, enqueued = 0; + unsigned release; + + cnt = ps->node->in.vectorize; + + struct sample *read_smps[cnt]; + struct sample *muxed_smps[cnt]; + struct sample **tomux_smps; + + /* Fill smps[] free sample blocks from the pool */ + allocated = sample_alloc_many(&ps->pool, read_smps, cnt); + if (allocated != cnt) + warning("Pool underrun for path source %s", node_name(ps->node)); + + /* Read ready samples and store them to blocks pointed by smps[] */ + release = allocated; + + recv = node_read(ps->node, read_smps, allocated, &release); + if (recv == 0) { + enqueued = 0; + goto out2; + } + else if (recv < 0) { + if (ps->node->state == STATE_STOPPING) { + p->state = STATE_STOPPING; + + enqueued = -1; + goto out2; + } + else + error("Failed to read samples from node %s", node_name(ps->node)); + } + else if (recv < allocated) + warning("Partial read for path %s: read=%u, expected=%u", path_name(p), recv, allocated); + + bitset_set(&p->received, i); + + if (p->mode == PATH_MODE_ANY) { /* Mux all samples */ + tomux_smps = read_smps; + tomux = recv; + } + else { /* Mux only last sample and discard others */ + tomux_smps = read_smps + recv - 1; + tomux = 1; + } + + for (int i = 0; i < tomux; i++) { + muxed_smps[i] = i == 0 + ? sample_clone(p->last_sample) + : sample_clone(muxed_smps[i-1]); + + if (p->original_sequence_no) + muxed_smps[i]->sequence = tomux_smps[i]->sequence; + else { + muxed_smps[i]->sequence = p->last_sequence++; + muxed_smps[i]->flags |= SAMPLE_HAS_SEQUENCE; + } + + muxed_smps[i]->ts = tomux_smps[i]->ts; + muxed_smps[i]->flags |= tomux_smps[i]->flags & (SAMPLE_HAS_TS_ORIGIN | SAMPLE_HAS_TS_RECEIVED); + + mapping_remap(&ps->mappings, muxed_smps[i], tomux_smps[i], NULL); + } + + sample_copy(p->last_sample, muxed_smps[tomux-1]); + + debug(15, "Path %s received = %s", path_name(p), bitset_dump(&p->received)); + +#ifdef WITH_HOOKS + toenqueue = hook_list_process(&p->hooks, muxed_smps, tomux); + if (toenqueue != tomux) { + int skipped = tomux - toenqueue; + + debug(LOG_NODES | 10, "Hooks skipped %u out of %u samples for path %s", skipped, tomux, path_name(p)); + } +#else + toenqueue = tomux; +#endif + + if (bitset_test(&p->mask, i)) { + /* Check if we received an update from all nodes/ */ + if ((p->mode == PATH_MODE_ANY) || + (p->mode == PATH_MODE_ALL && !bitset_cmp(&p->mask, &p->received))) { + path_destination_enqueue(p, muxed_smps, toenqueue); + + /* Reset bitset of updated nodes */ + bitset_clear_all(&p->received); + + enqueued = toenqueue; + } + } + + sample_decref_many(muxed_smps, tomux); +out2: sample_decref_many(read_smps, release); + + return enqueued; +} From 7798f09a351ae6d3c6f411f0052511770bec036f Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:06:24 +0100 Subject: [PATCH 22/38] inline *_type() functions --- include/villas/hook.h | 6 ++++++ include/villas/node.h | 6 +++++- lib/hook.c | 38 +++++++++++++++++++------------------- lib/node.c | 7 ------- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/include/villas/hook.h b/include/villas/hook.h index 28ecfddf6..bfc348ac1 100644 --- a/include/villas/hook.h +++ b/include/villas/hook.h @@ -81,6 +81,12 @@ int hook_process_list(struct vlist *hs, struct sample *smps[], unsigned cnt); /** Compare two hook functions with their priority. Used by vlist_sort() */ int hook_cmp_priority(const void *a, const void *b); +static inline +struct hook_type * hook_type(struct hook *h) +{ + return h->_vt; +} + /** Parses an object of hooks * * Example: diff --git a/include/villas/node.h b/include/villas/node.h index 0673f2ef9..59a3b111f 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -199,7 +199,11 @@ int node_poll_fds(struct node *n, int fds[]); int node_netem_fds(struct node *n, int fds[]); -struct node_type * node_type(struct node *n); +static inline +struct node_type * node_type(struct node *n) +{ + return n->_vt; +} struct memory_type * node_memory_type(struct node *n, struct memory_type *parent); diff --git a/lib/hook.c b/lib/hook.c index 17b693e7a..85bd71b2f 100644 --- a/lib/hook.c +++ b/lib/hook.c @@ -45,7 +45,7 @@ int hook_init(struct hook *h, struct hook_type *vt, struct path *p, struct node h->_vt = vt; h->_vd = alloc(vt->size); - ret = h->_vt->init ? h->_vt->init(h) : 0; + ret = hook_type(h)->init ? hook_type(h)->init(h) : 0; if (ret) return ret; @@ -66,9 +66,9 @@ int hook_parse(struct hook *h, json_t *cfg) "enabled", &h->enabled ); if (ret) - jerror(&err, "Failed to parse configuration of hook '%s'", hook_type_name(h->_vt)); + jerror(&err, "Failed to parse configuration of hook '%s'", hook_type_name(hook_type(h))); - ret = h->_vt->parse ? h->_vt->parse(h, cfg) : 0; + ret = hook_type(h)->parse ? hook_type(h)->parse(h, cfg) : 0; if (ret) return ret; @@ -84,7 +84,7 @@ int hook_destroy(struct hook *h) assert(h->state != STATE_DESTROYED); - ret = h->_vt->destroy ? h->_vt->destroy(h) : 0; + ret = hook_type(h)->destroy ? hook_type(h)->destroy(h) : 0; if (ret) return ret; @@ -101,10 +101,10 @@ int hook_start(struct hook *h) if (!h->enabled) return 0; - if (h->_vt->start) { - debug(LOG_HOOK | 10, "Start hook %s: priority=%d", hook_type_name(h->_vt), h->priority); + if (hook_type(h)->start) { + debug(LOG_HOOK | 10, "Start hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return h->_vt->start(h); + return hook_type(h)->start(h); } else return 0; @@ -115,10 +115,10 @@ int hook_stop(struct hook *h) if (!h->enabled) return 0; - if (h->_vt->stop) { - debug(LOG_HOOK | 10, "Stopping hook %s: priority=%d", hook_type_name(h->_vt), h->priority); + if (hook_type(h)->stop) { + debug(LOG_HOOK | 10, "Stopping hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return h->_vt->stop(h); + return hook_type(h)->stop(h); } else return 0; @@ -129,10 +129,10 @@ int hook_periodic(struct hook *h) if (!h->enabled) return 0; - if (h->_vt->periodic) { - debug(LOG_HOOK | 10, "Periodic hook %s: priority=%d", hook_type_name(h->_vt), h->priority); + if (hook_type(h)->periodic) { + debug(LOG_HOOK | 10, "Periodic hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return h->_vt->periodic(h); + return hook_type(h)->periodic(h); } else return 0; @@ -143,10 +143,10 @@ int hook_restart(struct hook *h) if (!h->enabled) return 0; - if (h->_vt->restart) { - debug(LOG_HOOK | 10, "Restarting hook %s: priority=%d", hook_type_name(h->_vt), h->priority); + if (hook_type(h)->restart) { + debug(LOG_HOOK | 10, "Restarting hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return h->_vt->restart(h); + return hook_type(h)->restart(h); } else return 0; @@ -157,10 +157,10 @@ int hook_process(struct hook *h, struct sample *smps[], unsigned *cnt) if (!h->enabled) return 0; - if (h->_vt->process) { - debug(LOG_HOOK | 10, "Process hook %s: priority=%d, cnt=%d", hook_type_name(h->_vt), h->priority, *cnt); + if (hook_type(h)->process) { + debug(LOG_HOOK | 10, "Process hook %s: priority=%d, cnt=%d", hook_type_name(hook_type(h)), h->priority, *cnt); - return h->_vt->process(h, smps, cnt); + return hook_type(h)->process(h, smps, cnt); } else return 0; diff --git a/lib/node.c b/lib/node.c index a25e0ff22..d3659d39a 100644 --- a/lib/node.c +++ b/lib/node.c @@ -704,13 +704,6 @@ int node_netem_fds(struct node *n, int fds[]) return node_type(n)->netem_fds ? node_type(n)->netem_fds(n, fds) : -1; } -struct node_type * node_type(struct node *n) -{ - assert(n->state != STATE_DESTROYED); - - return n->_vt; -} - struct memory_type * node_memory_type(struct node *n, struct memory_type *parent) { return node_type(n)->memory_type ? node_type(n)->memory_type(n, parent) : &memory_hugepage; From 0acce9e37590b490dba7cf444db178c09dbe1919 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:07:07 +0100 Subject: [PATCH 23/38] node: separate node_direction --- include/villas/node.h | 12 +- include/villas/node_direction.h | 74 ++++++++++ lib/CMakeLists.txt | 3 +- lib/node.c | 171 ----------------------- lib/node_direction.c | 233 ++++++++++++++++++++++++++++++++ 5 files changed, 310 insertions(+), 183 deletions(-) create mode 100644 include/villas/node_direction.h create mode 100644 lib/node_direction.c diff --git a/include/villas/node.h b/include/villas/node.h index 59a3b111f..8f166b2e9 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -52,17 +53,6 @@ extern "C" { struct rtnl_cls; #endif /* WITH_NETEM */ -struct node_direction { - int enabled; - int builtin; /**< This node should use built-in hooks by default. */ - int vectorize; /**< Number of messages to send / recv at once (scatter / gather) */ - - struct vlist hooks; /**< List of read / write hooks (struct hook). */ - struct vlist signals; /**< Signal description. */ - - json_t *cfg; /**< A JSON object containing the configuration of the node. */ -}; - /** The data structure for a node. * * Every entity which exchanges messages is represented by a node. diff --git a/include/villas/node_direction.h b/include/villas/node_direction.h new file mode 100644 index 000000000..1fd86ccfb --- /dev/null +++ b/include/villas/node_direction.h @@ -0,0 +1,74 @@ +/** Node direction + * + * @file + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** + * @addtogroup node Node + * @{ + */ + +#pragma once + +#include + +#include +#include + +/* Forward declarations */ +struct node; + +enum node_dir { + NODE_DIR_IN, /**< VILLASnode is receiving/reading */ + NODE_DIR_OUT /**< VILLASnode is sending/writing */ +}; + +struct node_direction { + enum state state; + enum node_dir direction; + + int enabled; + int builtin; /**< This node should use built-in hooks by default. */ + int vectorize; /**< Number of messages to send / recv at once (scatter / gather) */ + + struct vlist hooks; /**< List of read / write hooks (struct hook). */ + struct vlist signals; /**< Signal description. */ + + json_t *cfg; /**< A JSON object containing the configuration of the node. */ +}; + +int node_direction_init(struct node_direction *nd, enum node_dir dir, struct node *n); + +int node_direction_parse(struct node_direction *nd, struct node *n, json_t *cfg); + +int node_direction_check(struct node_direction *nd, struct node *n); + +int node_direction_prepare(struct node_direction *nd, struct node *n); + +int node_direction_start(struct node_direction *nd, struct node *n); + +int node_direction_stop(struct node_direction *nd, struct node *n); + +int node_direction_destroy(struct node_direction *nd, struct node *n); + +struct vlist * node_direction_get_signals(struct node_direction *nd); + +/** @} */ diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 85318fcbb..055ac29aa 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -49,9 +49,10 @@ set(LIB_SRC path_source.c path_destination.c node.c + node_type.c + node_direction.c memory.c plugin.c - node_type.c stats.c mapping.c shmem.c diff --git a/lib/node.c b/lib/node.c index d3659d39a..b49c01eb3 100644 --- a/lib/node.c +++ b/lib/node.c @@ -41,177 +41,6 @@ #include #endif /* WITH_NETEM */ -int node_direction_prepare(struct node_direction *nd, struct node *n) -{ -#ifdef WITH_HOOKS - int ret; - int m = nd == &n->out - ? HOOK_NODE_WRITE - : HOOK_NODE_READ; - - /* Add internal hooks if they are not already in the list */ - ret = hook_init_builtin_list(&nd->hooks, nd->builtin, m, NULL, n); - if (ret) - return ret; - - /* We sort the hooks according to their priority before starting the path */ - vlist_sort(&nd->hooks, hook_cmp_priority); -#endif /* WITH_HOOKS */ - - return 0; -} - -static int node_direction_init(struct node_direction *nd, struct node *n) -{ - int ret; - - nd->enabled = 1; - nd->vectorize = 1; - nd->builtin = 1; - - nd->hooks.state = STATE_DESTROYED; - nd->signals.state = STATE_DESTROYED; - -#ifdef WITH_HOOKS - ret = vlist_init(&nd->hooks); - if (ret) - return ret; -#endif /* WITH_HOOKS */ - - ret = signal_list_init(&nd->signals); - if (ret) - return ret; - - return 0; -} - -static int node_direction_destroy(struct node_direction *nd, struct node *n) -{ - int ret = 0; - -#ifdef WITH_HOOKS - ret = vlist_destroy(&nd->hooks, (dtor_cb_t) hook_destroy, true); - if (ret) - return ret; -#endif /* WITH_HOOKS */ - - ret = signal_list_destroy(&nd->signals); - if (ret) - return ret; - - return ret; -} - -static int node_direction_parse(struct node_direction *nd, struct node *n, json_t *cfg) -{ - int ret; - - json_error_t err; - json_t *json_hooks = NULL; - json_t *json_signals = NULL; - - nd->cfg = cfg; - - ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: i, s?: b, s?: b }", - "hooks", &json_hooks, - "signals", &json_signals, - "vectorize", &nd->vectorize, - "builtin", &nd->builtin, - "enabled", &nd->enabled - ); - if (ret) - jerror(&err, "Failed to parse node %s", node_name(n)); - - if (n->_vt->flags & NODE_TYPE_PROVIDES_SIGNALS) { - if (json_signals) - error("Node %s does not support signal definitions", node_name(n)); - } - else if (json_is_array(json_signals)) { - ret = signal_list_parse(&nd->signals, json_signals); - if (ret) - error("Failed to parse signal definition of node %s", node_name(n)); - } - else { - int count = DEFAULT_SAMPLE_LENGTH; - const char *type_str = "float"; - - if (json_is_object(json_signals)) { - json_unpack_ex(json_signals, &err, 0, "{ s: i, s: s }", - "count", &count, - "type", &type_str - ); - } - else - warning("No signal definition found for node %s. Using the default config of %d floating point signals.", node_name(n), DEFAULT_SAMPLE_LENGTH); - - int type = signal_type_from_str(type_str); - if (type < 0) - error("Invalid signal type %s", type_str); - - signal_list_generate(&nd->signals, count, type); - } - -#ifdef WITH_HOOKS - int m = nd == &n->out - ? HOOK_NODE_WRITE - : HOOK_NODE_READ; - - if (json_hooks) { - ret = hook_parse_list(&nd->hooks, json_hooks, m, NULL, n); - if (ret < 0) - return ret; - } -#endif /* WITH_HOOKS */ - - return 0; -} - -static int node_direction_check(struct node_direction *nd, struct node *n) -{ - if (nd->vectorize <= 0) - error("Invalid setting 'vectorize' with value %d for node %s. Must be natural number!", nd->vectorize, node_name(n)); - - if (node_type(n)->vectorize && node_type(n)->vectorize < nd->vectorize) - error("Invalid value for setting 'vectorize'. Node type requires a number smaller than %d!", - node_type(n)->vectorize); - - return 0; -} - -static int node_direction_start(struct node_direction *nd, struct node *n) -{ -#ifdef WITH_HOOKS - int ret; - - for (size_t i = 0; i < vlist_length(&nd->hooks); i++) { - struct hook *h = (struct hook *) vlist_at(&nd->hooks, i); - - ret = hook_start(h); - if (ret) - return ret; - } -#endif /* WITH_HOOKS */ - - return 0; -} - -static int node_direction_stop(struct node_direction *nd, struct node *n) -{ -#ifdef WITH_HOOKS - int ret; - - for (size_t i = 0; i < vlist_length(&nd->hooks); i++) { - struct hook *h = (struct hook *) vlist_at(&nd->hooks, i); - - ret = hook_stop(h); - if (ret) - return ret; - } -#endif /* WITH_HOOKS */ - - return 0; -} - int node_init(struct node *n, struct node_type *vt) { int ret; diff --git a/lib/node_direction.c b/lib/node_direction.c new file mode 100644 index 000000000..7029f7549 --- /dev/null +++ b/lib/node_direction.c @@ -0,0 +1,233 @@ + +/** Node direction + * + * @author Steffen Vogel + * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include + +int node_direction_prepare(struct node_direction *nd, struct node *n) +{ + assert(nd->state == STATE_CHECKED); + +#ifdef WITH_HOOKS + int ret; + int t = nd->direction == NODE_DIR_OUT ? HOOK_NODE_WRITE : HOOK_NODE_READ; + int m = nd->builtin ? t | HOOK_BUILTIN : 0; + + ret = hook_list_prepare(&nd->hooks, &nd->signals, m, NULL, n); + if (ret) + return ret; +#endif /* WITH_HOOKS */ + + nd->state = STATE_PREPARED; + + return 0; +} + +int node_direction_init(struct node_direction *nd, enum node_dir dir, struct node *n) +{ + int ret; + + assert(nd->state == STATE_DESTROYED); + + nd->direction = dir; + nd->enabled = 1; + nd->vectorize = 1; + nd->builtin = 1; + + nd->hooks.state = STATE_DESTROYED; + nd->signals.state = STATE_DESTROYED; + +#ifdef WITH_HOOKS + ret = hook_list_init(&nd->hooks); + if (ret) + return ret; +#endif /* WITH_HOOKS */ + + ret = signal_list_init(&nd->signals); + if (ret) + return ret; + + nd->state = STATE_INITIALIZED; + + return 0; +} + +int node_direction_destroy(struct node_direction *nd, struct node *n) +{ + int ret = 0; + + assert(nd->state != STATE_DESTROYED && nd->state != STATE_STARTED); + +#ifdef WITH_HOOKS + ret = hook_list_destroy(&nd->hooks); + if (ret) + return ret; +#endif /* WITH_HOOKS */ + + ret = signal_list_destroy(&nd->signals); + if (ret) + return ret; + + nd->state = STATE_DESTROYED; + + return 0; +} + +int node_direction_parse(struct node_direction *nd, struct node *n, json_t *cfg) +{ + int ret; + + assert(nd->state == STATE_INITIALIZED); + + json_error_t err; + json_t *json_hooks = NULL; + json_t *json_signals = NULL; + + nd->cfg = cfg; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: i, s?: b, s?: b }", + "hooks", &json_hooks, + "signals", &json_signals, + "vectorize", &nd->vectorize, + "builtin", &nd->builtin, + "enabled", &nd->enabled + ); + if (ret) + jerror(&err, "Failed to parse node %s", node_name(n)); + + if (n->_vt->flags & NODE_TYPE_PROVIDES_SIGNALS) { + if (json_signals) + error("Node %s does not support signal definitions", node_name(n)); + } + else if (json_is_array(json_signals)) { + ret = signal_list_parse(&nd->signals, json_signals); + if (ret) + error("Failed to parse signal definition of node %s", node_name(n)); + } + else { + int count = DEFAULT_SAMPLE_LENGTH; + const char *type_str = "float"; + + if (json_is_object(json_signals)) { + json_unpack_ex(json_signals, &err, 0, "{ s: i, s: s }", + "count", &count, + "type", &type_str + ); + } + else + warning("No signal definition found for node %s. Using the default config of %d floating point signals.", node_name(n), DEFAULT_SAMPLE_LENGTH); + + int type = signal_type_from_str(type_str); + if (type < 0) + error("Invalid signal type %s", type_str); + + signal_list_generate(&nd->signals, count, type); + } + +#ifdef WITH_HOOKS + if (json_hooks) { + int m = nd->direction == NODE_DIR_OUT ? HOOK_NODE_WRITE : HOOK_NODE_READ; + + ret = hook_list_parse(&nd->hooks, json_hooks, m, NULL, n); + if (ret < 0) + return ret; + } +#endif /* WITH_HOOKS */ + + nd->state = STATE_PARSED; + + return 0; +} + +int node_direction_check(struct node_direction *nd, struct node *n) +{ + assert(nd->state == STATE_PARSED); + + if (nd->vectorize <= 0) + error("Invalid setting 'vectorize' with value %d for node %s. Must be natural number!", nd->vectorize, node_name(n)); + + if (node_type(n)->vectorize && node_type(n)->vectorize < nd->vectorize) + error("Invalid value for setting 'vectorize'. Node type requires a number smaller than %d!", + node_type(n)->vectorize); + + nd->state = STATE_CHECKED; + + return 0; +} + +int node_direction_start(struct node_direction *nd, struct node *n) +{ + assert(nd->state == STATE_PREPARED); + +#ifdef WITH_HOOKS + int ret; + + for (size_t i = 0; i < vlist_length(&nd->hooks); i++) { + struct hook *h = (struct hook *) vlist_at(&nd->hooks, i); + + ret = hook_start(h); + if (ret) + return ret; + } +#endif /* WITH_HOOKS */ + + nd->state = STATE_STARTED; + + return 0; +} + +int node_direction_stop(struct node_direction *nd, struct node *n) +{ + assert(nd->state == STATE_STARTED); + +#ifdef WITH_HOOKS + int ret; + + for (size_t i = 0; i < vlist_length(&nd->hooks); i++) { + struct hook *h = (struct hook *) vlist_at(&nd->hooks, i); + + ret = hook_stop(h); + if (ret) + return ret; + } +#endif /* WITH_HOOKS */ + + nd->state = STATE_STOPPED; + + return 0; +} + +struct vlist * node_direction_get_signals(struct node_direction *nd) +{ +#ifdef WITH_HOOKS + assert(nd->state == STATE_PREPARED); + + struct hook *h = vlist_last(&nd->hooks); + + return h->signals; +#else + return &nd->signals; +#endif +} From e65ffc78ddb9d2083af787a5ef1452d43e2f0134 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:08:15 +0100 Subject: [PATCH 24/38] node: add enabled setting --- include/villas/node.h | 3 +++ lib/node.c | 9 ++++++++- lib/path.c | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/villas/node.h b/include/villas/node.h index 8f166b2e9..e7d34eac3 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -60,6 +60,7 @@ extern "C" { */ struct node { char *name; /**< A short identifier of the node, only used for configuration and logging */ + int enabled; enum state state; @@ -198,6 +199,8 @@ struct node_type * node_type(struct node *n) struct memory_type * node_memory_type(struct node *n, struct memory_type *parent); int node_is_valid_name(const char *name); +bool node_is_enabled(const struct node *n); + #ifdef __cplusplus } diff --git a/lib/node.c b/lib/node.c index b49c01eb3..f9752f903 100644 --- a/lib/node.c +++ b/lib/node.c @@ -53,6 +53,7 @@ int node_init(struct node *n, struct node_type *vt) n->name = NULL; n->_name = NULL; n->_name_long = NULL; + n->enabled = 1; #ifdef __linux__ n->fwmark = -1; @@ -113,8 +114,9 @@ int node_parse(struct node *n, json_t *json, const char *name) n->name = strdup(name); - ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: { s?: o } }", + ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: b, s?: { s?: o } }", "type", &type, + "enabled", &n->enabled, "in", "signals", &json_signals ); @@ -603,4 +605,9 @@ int node_is_valid_name(const char *name) } return 0; +bool node_is_enabled(const struct node *n) +{ + return n->enabled; +} + } diff --git a/lib/path.c b/lib/path.c index 9f1288bb3..ddaf6926a 100644 --- a/lib/path.c +++ b/lib/path.c @@ -375,6 +375,9 @@ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes) vlist_push(&p->sources, ps); } + if (!node_is_enabled(ps->node)) + error("Source %s of path %s is not enabled", node_name(ps->node), path_name(p)); + vlist_push(&ps->mappings, me); } @@ -385,6 +388,9 @@ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes) pd->node = n; + if (!node_is_enabled(pd->node)) + error("Destination %s of path %s is not enabled", node_name(pd->node), path_name(p)); + vlist_push(&p->destinations, pd); } From 34137545bcf60ad7208c56837bbb50f25a50a89b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:08:56 +0100 Subject: [PATCH 25/38] path: use path_type() --- lib/path.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/path.c b/lib/path.c index ddaf6926a..4a0795dcd 100644 --- a/lib/path.c +++ b/lib/path.c @@ -216,10 +216,10 @@ int path_prepare(struct path *p) for (size_t i = 0; i < vlist_length(&p->destinations); i++) { struct path_destination *pd = (struct path_destination *) vlist_at(&p->destinations, i); - if (pd->node->_vt->pool_size > pool_size) - pool_size = pd->node->_vt->pool_size; + if (node_type(pd->node)->pool_size > pool_size) + pool_size = node_type(pd->node)->pool_size; - if (pd->node->_vt->memory_type) + if (node_type(pd->node)->memory_type) pool_mt = node_memory_type(pd->node, &memory_hugepage); ret = path_destination_init(pd, p->queuelen); @@ -484,7 +484,7 @@ int path_check(struct path *p) for (size_t i = 0; i < vlist_length(&p->sources); i++) { struct path_source *ps = (struct path_source *) vlist_at(&p->sources, i); - if (!ps->node->_vt->poll_fds) + if (!node_type(ps->node)->poll_fds) error("Node %s can not be used in polling mode with path %s", node_name(ps->node), path_name(p)); } } @@ -502,14 +502,14 @@ int path_check(struct path *p) for (size_t i = 0; i < vlist_length(&p->sources); i++) { struct path_source *ps = (struct path_source *) vlist_at(&p->sources, i); - if (!ps->node->_vt->read) + if (!node_type(ps->node)->read) error("Node %s is not supported as a source for path %s", node_name(ps->node), path_name(p)); } for (size_t i = 0; i < vlist_length(&p->destinations); i++) { struct path_destination *pd = (struct path_destination *) vlist_at(&p->destinations, i); - if (!pd->node->_vt->write) + if (!node_type(pd->node)->write) error("Destiation node %s is not supported as a sink for path %s", node_name(pd->node), path_name(p)); } From 7e5b4ac5ce82ebb5d03f2a6b186dcf1dbcc6fb3c Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:10:44 +0100 Subject: [PATCH 26/38] refactor: add {node, path}_is_{enabled, reversed}() --- include/villas/node.h | 3 ++- include/villas/path.h | 6 +++++- lib/node.c | 8 +++++--- lib/path.c | 28 +++++++++++++++++++--------- lib/super_node.cpp | 4 ++-- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/include/villas/node.h b/include/villas/node.h index e7d34eac3..2b2f7fb08 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -198,7 +198,8 @@ struct node_type * node_type(struct node *n) struct memory_type * node_memory_type(struct node *n, struct memory_type *parent); -int node_is_valid_name(const char *name); +bool node_is_valid_name(const char *name); + bool node_is_enabled(const struct node *n); diff --git a/include/villas/path.h b/include/villas/path.h index 1abe9ab0c..0eaaf4b99 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -157,7 +157,11 @@ int path_uses_node(struct path *p, struct node *n); */ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes); -int path_is_simple(struct path *p); +bool path_is_simple(const struct path *p); + +bool path_is_enabled(const struct path *p); + +bool path_is_reversed(const struct path *p); /** @} */ diff --git a/lib/node.c b/lib/node.c index f9752f903..c08043198 100644 --- a/lib/node.c +++ b/lib/node.c @@ -595,16 +595,18 @@ invalid2: return 0; } -int node_is_valid_name(const char *name) +bool node_is_valid_name(const char *name) { for (const char *p = name; *p; p++) { if (isalnum(*p) || (*p == '_') || (*p == '-')) continue; - return -1; + return false; } - return 0; + return true; +} + bool node_is_enabled(const struct node *n) { return n->enabled; diff --git a/lib/path.c b/lib/path.c index 4a0795dcd..63d2e05b4 100644 --- a/lib/path.c +++ b/lib/path.c @@ -545,8 +545,8 @@ int path_start(struct path *p) p->poll ? "yes" : "no", mask, p->rate, - p->enabled ? "yes" : "no", - p->reverse ? "yes" : "no", + path_is_enabled(p) ? "yes" : "no", + path_is_reversed(p) ? "yes" : "no", p->queuelen, vlist_length(&p->hooks), vlist_length(&p->sources), @@ -723,22 +723,32 @@ int path_uses_node(struct path *p, struct node *n) return -1; } -int path_is_simple(struct path *p) +bool path_is_simple(const struct path *p) { int ret; const char *in = NULL, *out = NULL; ret = json_unpack(p->cfg, "{ s: s, s: s }", "in", &in, "out", &out); if (ret) - return ret; + return false; ret = node_is_valid_name(in); - if (ret) - return ret; + if (!ret) + return false; ret = node_is_valid_name(out); - if (ret) - return ret; + if (!ret) + return false; - return 0; + return true; +} + +bool path_is_enabled(const struct path *p) +{ + return p->enabled; +} + +bool path_is_reversed(const struct path *p) +{ + return p->reverse; } diff --git a/lib/super_node.cpp b/lib/super_node.cpp index de842388e..0eba7eb04 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -218,7 +218,7 @@ int SuperNode::parseJson(json_t *j) const char *type; ret = node_is_valid_name(name); - if (ret) + if (!ret) throw RuntimeError("Invalid name for node: {}", name); ret = json_unpack_ex(json_node, &err, 0, "{ s: s }", "type", &type); @@ -266,7 +266,7 @@ parse: path *p = (path *) alloc(sizeof(path)); if (p->reverse) { /* Only simple paths can be reversed */ ret = path_is_simple(p); - if (ret) + if (!ret) throw RuntimeError("Complex paths can not be reversed!"); /* Parse a second time with in/out reversed */ From 842be5a9cc88ca4e130c02e98473f1f2ff72d315 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:11:19 +0100 Subject: [PATCH 27/38] path: refactor path_init_poll() -> path_prepare_poll() --- lib/path.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/path.c b/lib/path.c index 63d2e05b4..7db489b25 100644 --- a/lib/path.c +++ b/lib/path.c @@ -146,7 +146,7 @@ int path_init(struct path *p) return 0; } -int path_init_poll(struct path *p) +static int path_prepare_poll(struct path *p) { int fds[16], ret, n = 0, m; @@ -282,7 +282,7 @@ int path_prepare(struct path *p) /* Prepare poll() */ if (p->poll) { - ret = path_init_poll(p); + ret = path_prepare_poll(p); if (ret) return ret; } From 9836bc029690864e4e2c82410ea9e706e537c986 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:13:28 +0100 Subject: [PATCH 28/38] add prepare state --- common | 2 +- include/villas/super_node.hpp | 4 ++ lib/node.c | 4 +- lib/path.c | 4 +- lib/super_node.cpp | 75 +++++++++++++++++++++++++---------- src/villas-node.cpp | 1 + 6 files changed, 66 insertions(+), 24 deletions(-) diff --git a/common b/common index 0c35a58b7..3a4beccee 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 0c35a58b79b00726d13675b7a6231dd35bf62a20 +Subproject commit 3a4beccee89f42a85b175b5557205d9ab756bfb5 diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index 805832d74..de484788f 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -88,10 +88,14 @@ public: int check(); /** Initialize after parsing the configuration file. */ + void prepare(); void start(); void stop(); void run(); + void preparePaths(); + void prepareNodes(); + void startPaths(); void startNodes(); void startNodeTypes(); diff --git a/lib/node.c b/lib/node.c index c08043198..5c6c7a838 100644 --- a/lib/node.c +++ b/lib/node.c @@ -98,6 +98,8 @@ int node_prepare(struct node *n) if (ret) return ret; + n->state = STATE_PREPARED; + return 0; } @@ -220,7 +222,7 @@ int node_start(struct node *n) { int ret; - assert(n->state == STATE_CHECKED); + assert(n->state == STATE_PREPARED); assert(node_type(n)->state == STATE_STARTED); info("Starting node %s", node_name_long(n)); diff --git a/lib/path.c b/lib/path.c index 7db489b25..22b12f4c5 100644 --- a/lib/path.c +++ b/lib/path.c @@ -287,6 +287,8 @@ int path_prepare(struct path *p) return ret; } + p->state = STATE_PREPARED; + return 0; } @@ -528,7 +530,7 @@ int path_start(struct path *p) int ret; char *mode, *mask; - assert(p->state == STATE_CHECKED); + assert(p->state == STATE_PREPARED); switch (p->mode) { case PATH_MODE_ANY: mode = "any"; break; diff --git a/lib/super_node.cpp b/lib/super_node.cpp index 0eba7eb04..f4780ec1e 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -356,18 +356,12 @@ void SuperNode::startNodes() for (size_t i = 0; i < vlist_length(&nodes); i++) { auto *n = (struct node *) vlist_at(&nodes, i); - ret = node_prepare(n); - if (ret) - throw RuntimeError("Failed to prepare node: {}", node_name(n)); + if (!node_is_enabled(n)) + continue; - int refs = vlist_count(&paths, (cmp_cb_t) path_uses_node, n); - if (refs > 0) { - ret = node_start(n); - if (ret) - throw RuntimeError("Failed to start node: {}", node_name(n)); - } - else - logger->warn("No path is using the node {}. Skipping...", node_name(n)); + ret = node_start(n); + if (ret) + throw RuntimeError("Failed to start node: {}", node_name(n)); } } @@ -378,20 +372,59 @@ void SuperNode::startPaths() for (size_t i = 0; i < vlist_length(&paths); i++) { auto *p = (struct path *) vlist_at(&paths, i); - if (p->enabled) { - ret = path_prepare(p); - if (ret) - throw RuntimeError("Failed to prepare path: {}", path_name(p)); + if (!path_is_enabled(p)) + continue; - ret = path_start(p); - if (ret) - throw RuntimeError("Failed to start path: {}", path_name(p)); - } - else - logger->warn("Path {} is disabled. Skipping...", path_name(p)); + ret = path_start(p); + if (ret) + throw RuntimeError("Failed to start path: {}", path_name(p)); } } +void SuperNode::prepareNodes() +{ + int ret, refs; + + for (size_t i = 0; i < vlist_length(&nodes); i++) { + auto *n = (struct node *) vlist_at(&nodes, i); + + refs = vlist_count(&paths, (cmp_cb_t) path_uses_node, n); + if (refs <= 0) { + logger->warn("No path is using the node {}. Skipping...", node_name(n)); + n->enabled = false; + } + + if (!node_is_enabled(n)) + continue; + + ret = node_prepare(n); + if (ret) + throw RuntimeError("Failed to prepare node: {}", node_name(n)); + } +} + +void SuperNode::preparePaths() +{ + int ret; + + for (size_t i = 0; i < vlist_length(&paths); i++) { + auto *p = (struct path *) vlist_at(&paths, i); + + if (!path_is_enabled(p)) + continue; + + ret = path_prepare(p); + if (ret) + throw RuntimeError("Failed to prepare path: {}", path_name(p)); + } +} + +void SuperNode::prepare() +{ + prepareNodes(); + preparePaths(); +} + void SuperNode::start() { int ret; diff --git a/src/villas-node.cpp b/src/villas-node.cpp index 91568425f..5c5a14e8e 100644 --- a/src/villas-node.cpp +++ b/src/villas-node.cpp @@ -184,6 +184,7 @@ int main(int argc, char *argv[]) if (ret) throw RuntimeError("Failed to verify configuration"); + sn.prepare(); sn.start(); sn.run(); sn.stop(); From af99a50786efeca3ddf4001e3e3150c83a381cc8 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 24 Feb 2019 11:14:44 +0100 Subject: [PATCH 29/38] whitespace and documentation fixes --- include/villas/hook.h | 2 ++ include/villas/hook_type.h | 8 ++++---- lib/node.c | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/villas/hook.h b/include/villas/hook.h index bfc348ac1..2d81af6d4 100644 --- a/include/villas/hook.h +++ b/include/villas/hook.h @@ -70,9 +70,11 @@ int hook_parse(struct hook *h, json_t *cfg); int hook_destroy(struct hook *h); int hook_start(struct hook *h); + int hook_stop(struct hook *h); int hook_periodic(struct hook *h); + int hook_restart(struct hook *h); int hook_process(struct hook *h, struct sample *smps[], unsigned *cnt); diff --git a/include/villas/hook_type.h b/include/villas/hook_type.h index d7ff2f642..a7089af19 100644 --- a/include/villas/hook_type.h +++ b/include/villas/hook_type.h @@ -1,6 +1,6 @@ /** Hook funktions * - * Every path can register a hook function which is called for every received + * Every path or node can register a hook function which is called for every received * message. This can be used to debug the data flow, get statistics * or alter the message. * @@ -62,11 +62,11 @@ struct hook_type { int (*parse)(struct hook *h, json_t *cfg); - int (*init)(struct hook *h); /**< Called before path is started to parsed. */ + int (*init)(struct hook *h); /**< Called before hook is started to parsed. */ int (*destroy)(struct hook *h); /**< Called after path has been stopped to release memory allocated by HOOK_INIT */ - int (*start)(struct hook *h); /**< Called whenever a path is started; before threads are created. */ - int (*stop)(struct hook *h); /**< Called whenever a path is stopped; after threads are destoyed. */ + int (*start)(struct hook *h); /**< Called whenever a hook is started; before threads are created. */ + int (*stop)(struct hook *h); /**< Called whenever a hook is stopped; after threads are destoyed. */ int (*periodic)(struct hook *h);/**< Called periodically. Period is set by global 'stats' option in the configuration file. */ int (*restart)(struct hook *h); /**< Called whenever a new simulation case is started. This is detected by a sequence no equal to zero. */ diff --git a/lib/node.c b/lib/node.c index 5c6c7a838..a2248a843 100644 --- a/lib/node.c +++ b/lib/node.c @@ -331,9 +331,8 @@ int node_restart(struct node *n) info("Restarting node %s", node_name(n)); - if (node_type(n)->restart) { + if (node_type(n)->restart) ret = node_type(n)->restart(n); - } else { ret = node_type(n)->stop ? node_type(n)->stop(n) : 0; if (ret) From 3bab0c9c063fb933e212d8a46954473949c2448e Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 8 Mar 2019 15:21:01 +0100 Subject: [PATCH 30/38] node, path: add getters for signals --- include/villas/node.h | 1 + include/villas/path.h | 2 ++ lib/node.c | 5 +++++ lib/path.c | 5 +++++ 4 files changed, 13 insertions(+) diff --git a/include/villas/node.h b/include/villas/node.h index 2b2f7fb08..0275343d5 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -202,6 +202,7 @@ bool node_is_valid_name(const char *name); bool node_is_enabled(const struct node *n); +struct vlist * node_get_signals(struct node *n, enum node_dir dir); #ifdef __cplusplus } diff --git a/include/villas/path.h b/include/villas/path.h index 0eaaf4b99..e899fe036 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -163,6 +163,8 @@ bool path_is_enabled(const struct path *p); bool path_is_reversed(const struct path *p); +struct vlist * path_get_signals(struct path *p); + /** @} */ #ifdef __cplusplus diff --git a/lib/node.c b/lib/node.c index a2248a843..8a38ff267 100644 --- a/lib/node.c +++ b/lib/node.c @@ -613,4 +613,9 @@ bool node_is_enabled(const struct node *n) return n->enabled; } +struct vlist * node_get_signals(struct node *n, enum node_dir dir) +{ + struct node_direction *nd = dir == NODE_DIR_IN ? &n->in : &n->out; + + return node_direction_get_signals(nd); } diff --git a/lib/path.c b/lib/path.c index 22b12f4c5..177573529 100644 --- a/lib/path.c +++ b/lib/path.c @@ -754,3 +754,8 @@ bool path_is_reversed(const struct path *p) { return p->reverse; } + +struct vlist * path_get_signals(struct path *p) +{ + return &p->signals; +} From 3f6ddaa653d192f77093a7e45c531c2caab5ef8c Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 8 Mar 2019 15:21:22 +0100 Subject: [PATCH 31/38] signal: add signal_list_copy() --- include/villas/signal.h | 1 + lib/signal.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/include/villas/signal.h b/include/villas/signal.h index d7af0e2f7..93d054437 100644 --- a/include/villas/signal.h +++ b/include/villas/signal.h @@ -111,6 +111,7 @@ int signal_list_destroy(struct vlist *list); int signal_list_parse(struct vlist *list, json_t *cfg); int signal_list_generate(struct vlist *list, unsigned len, enum signal_type fmt); void signal_list_dump(const struct vlist *list, const union signal_data *data, int len); +int signal_list_copy(struct vlist *dst, const struct vlist *src); enum signal_type signal_type_from_str(const char *str); diff --git a/lib/signal.c b/lib/signal.c index 47016714d..bc8c96060 100644 --- a/lib/signal.c +++ b/lib/signal.c @@ -307,6 +307,22 @@ void signal_list_dump(const struct vlist *list, const union signal_data *data, i } } +int signal_list_copy(struct vlist *dst, const struct vlist *src) +{ + assert(src->state == STATE_INITIALIZED); + assert(dst->state == STATE_INITIALIZED); + + for (size_t i = 0; i < vlist_length(src); i++) { + struct signal *s = (struct signal *) vlist_at_safe(src, i); + + signal_incref(s); + + vlist_push(dst, s); + } + + return 0; +} + /* Signal type */ enum signal_type signal_type_from_str(const char *str) From c9061433382ba486db760e4edd0238a012f7acbd Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 8 Mar 2019 15:21:46 +0100 Subject: [PATCH 32/38] sample: add sample_data_{insert, remove}() --- include/villas/sample.h | 4 ++++ lib/sample.c | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/include/villas/sample.h b/include/villas/sample.h index f5bc0d35f..9659e306a 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -136,6 +136,10 @@ int sample_decref_many(struct sample *smps[], int cnt); enum signal_type sample_format(const struct sample *s, unsigned idx); +void sample_data_insert(struct sample *smp, const union signal_data *src, size_t offset, size_t len); + +void sample_data_remove(struct sample *smp, size_t offset, size_t len); + #ifdef __cplusplus } #endif diff --git a/lib/sample.c b/lib/sample.c index afe6ce98f..f18225afb 100644 --- a/lib/sample.c +++ b/lib/sample.c @@ -301,3 +301,20 @@ void sample_dump(struct sample *s) if (s->signals) signal_list_dump(s->signals, s->data, s->length); } + +void sample_data_insert(struct sample *smp, const union signal_data *src, size_t offset, size_t len) +{ + memmove(&smp->data[offset + len], &smp->data[offset], sizeof(smp->data[0]) * (smp->length - offset)); + memcpy(&smp->data[offset], src, sizeof(smp->data[0]) * len); + + smp->length += len; +} + +void sample_data_remove(struct sample *smp, size_t offset, size_t len) +{ + size_t sz = sizeof(smp->data[0]) * len; + + memmove(&smp->data[offset], &smp->data[offset + len], sz); + + smp->length -= len; +} From 4942d8ee741d38defdb451ef109a2e8a92b85e43 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:32:22 +0100 Subject: [PATCH 33/38] bunch of bugfixes and refactoring for preperation of hooks --- include/villas/hook.h | 25 ++++- include/villas/hook_type.h | 2 + include/villas/mapping.h | 12 ++- lib/hook.c | 198 +++++++++++++++++++++++++++---------- lib/mapping.c | 70 +++++++------ lib/node.c | 12 +-- lib/node_direction.c | 2 +- lib/path.c | 18 ++-- lib/path_source.c | 2 +- lib/super_node.cpp | 18 ++-- src/villas-hook.cpp | 4 + 11 files changed, 250 insertions(+), 113 deletions(-) diff --git a/include/villas/hook.h b/include/villas/hook.h index 2d81af6d4..4451ad2c0 100644 --- a/include/villas/hook.h +++ b/include/villas/hook.h @@ -26,6 +26,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ + /** * @addtogroup hooks User-defined hook functions * @ingroup path @@ -35,6 +36,7 @@ #pragma once #include +#include #include #ifdef __cplusplus @@ -44,7 +46,6 @@ extern "C" { /* Forward declarations */ struct path; struct sample; -struct vlist; /** Descriptor for user defined hooks. See hooks[]. */ struct hook { @@ -56,6 +57,8 @@ struct hook { struct path *path; struct node *node; + struct vlist signals; + struct hook_type *_vt; /**< C++ like Vtable pointer. */ void *_vd; /**< Private data for this hook. This pointer can be used to pass data between consecutive calls of the callback. */ @@ -63,10 +66,13 @@ struct hook { }; int hook_init(struct hook *h, struct hook_type *vt, struct path *p, struct node *n); -int hook_init_builtin_list(struct vlist *l, bool builtin, int mask, struct path *p, struct node *n); + +int hook_init_signals(struct hook *h, struct vlist *signals); int hook_parse(struct hook *h, json_t *cfg); +int hook_prepare(struct hook *h, struct vlist *signals); + int hook_destroy(struct hook *h); int hook_start(struct hook *h); @@ -78,7 +84,6 @@ int hook_periodic(struct hook *h); int hook_restart(struct hook *h); int hook_process(struct hook *h, struct sample *smps[], unsigned *cnt); -int hook_process_list(struct vlist *hs, struct sample *smps[], unsigned cnt); /** Compare two hook functions with their priority. Used by vlist_sort() */ int hook_cmp_priority(const void *a, const void *b); @@ -89,6 +94,10 @@ struct hook_type * hook_type(struct hook *h) return h->_vt; } +int hook_list_init(struct vlist *hs); + +int hook_list_destroy(struct vlist *hs); + /** Parses an object of hooks * * Example: @@ -103,7 +112,15 @@ struct hook_type * hook_type(struct hook *h) * hooks = [ "print" ] * } */ -int hook_parse_list(struct vlist *list, json_t *cfg, int mask, struct path *p, struct node *n); +int hook_list_parse(struct vlist *hs, json_t *cfg, int mask, struct path *p, struct node *n); + +int hook_list_prepare(struct vlist *hs, struct vlist *sigs, int mask, struct path *p, struct node *n); + +int hook_list_prepare_signals(struct vlist *hs, struct vlist *signals); + +int hook_list_add(struct vlist *hs, int mask, struct path *p, struct node *n); + +int hook_list_process(struct vlist *hs, struct sample *smps[], unsigned cnt); #ifdef __cplusplus } diff --git a/include/villas/hook_type.h b/include/villas/hook_type.h index a7089af19..632b8705d 100644 --- a/include/villas/hook_type.h +++ b/include/villas/hook_type.h @@ -65,6 +65,8 @@ struct hook_type { int (*init)(struct hook *h); /**< Called before hook is started to parsed. */ int (*destroy)(struct hook *h); /**< Called after path has been stopped to release memory allocated by HOOK_INIT */ + int (*init_signals)(struct hook *h); + int (*start)(struct hook *h); /**< Called whenever a hook is started; before threads are created. */ int (*stop)(struct hook *h); /**< Called whenever a hook is stopped; after threads are destoyed. */ diff --git a/include/villas/mapping.h b/include/villas/mapping.h index 6c9522af3..9515da8f0 100644 --- a/include/villas/mapping.h +++ b/include/villas/mapping.h @@ -88,18 +88,20 @@ struct mapping_entry { }; }; -int mapping_remap(const struct vlist *m, struct sample *remapped, const struct sample *original, const struct stats *s); - -int mapping_update(const struct mapping_entry *e, struct sample *remapped, const struct sample *original, const struct stats *s); +int mapping_update(const struct mapping_entry *e, struct sample *remapped, const struct sample *original); int mapping_parse(struct mapping_entry *e, json_t *cfg, struct vlist *nodes); int mapping_parse_str(struct mapping_entry *e, const char *str, struct vlist *nodes); -int mapping_parse_list(struct vlist *l, json_t *cfg, struct vlist *nodes); - int mapping_to_str(const struct mapping_entry *me, unsigned index, char **str); +int mapping_list_parse(struct vlist *ml, json_t *cfg, struct vlist *nodes); + +int mapping_list_prepare(struct vlist *ml); + +int mapping_list_remap(const struct vlist *ml, struct sample *remapped, const struct sample *original); + #ifdef __cplusplus } #endif diff --git a/lib/hook.c b/lib/hook.c index 85bd71b2f..751f32729 100644 --- a/lib/hook.c +++ b/lib/hook.c @@ -42,6 +42,12 @@ int hook_init(struct hook *h, struct hook_type *vt, struct path *p, struct node h->path = p; h->node = n; + h->signals.state = STATE_DESTROYED; + + ret = signal_list_init(&h->signals); + if (ret) + return ret; + h->_vt = vt; h->_vd = alloc(vt->size); @@ -49,7 +55,30 @@ int hook_init(struct hook *h, struct hook_type *vt, struct path *p, struct node if (ret) return ret; - h->state = STATE_INITIALIZED; + // We dont need to parse builtin hooks + h->state = hook_type(h)->flags & HOOK_BUILTIN ? STATE_PARSED : STATE_INITIALIZED; + + return 0; +} + +int hook_prepare(struct hook *h, struct vlist *signals) +{ + int ret; + + assert(h->state == STATE_PARSED); + + if (!h->enabled) + return 0; + + ret = signal_list_copy(&h->signals, signals); + if (ret) + return -1; + + ret = hook_type(h)->init_signals ? hook_type(h)->init_signals(h) : 0; + if (ret) + return ret; + + h->state = STATE_PREPARED; return 0; } @@ -82,7 +111,11 @@ int hook_destroy(struct hook *h) { int ret; - assert(h->state != STATE_DESTROYED); + assert(h->state != STATE_DESTROYED && h->state != STATE_STARTED); + + ret = signal_list_destroy(&h->signals); + if (ret) + return ret; ret = hook_type(h)->destroy ? hook_type(h)->destroy(h) : 0; if (ret) @@ -98,34 +131,46 @@ int hook_destroy(struct hook *h) int hook_start(struct hook *h) { + int ret; + assert(h->state == STATE_PREPARED); + if (!h->enabled) return 0; - if (hook_type(h)->start) { - debug(LOG_HOOK | 10, "Start hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); + debug(LOG_HOOK | 10, "Start hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return hook_type(h)->start(h); - } - else - return 0; + ret = hook_type(h)->start ? hook_type(h)->start(h) : 0; + if (ret) + return ret; + + h->state = STATE_STARTED; + + return 0; } int hook_stop(struct hook *h) { + int ret; + assert(h->state == STATE_STARTED); + if (!h->enabled) return 0; - if (hook_type(h)->stop) { - debug(LOG_HOOK | 10, "Stopping hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); + debug(LOG_HOOK | 10, "Stopping hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return hook_type(h)->stop(h); - } - else - return 0; + ret = hook_type(h)->stop ? hook_type(h)->stop(h) : 0; + if (ret) + return ret; + + h->state = STATE_STOPPED; + + return 0; } int hook_periodic(struct hook *h) { + assert(h->state == STATE_STARTED); + if (!h->enabled) return 0; @@ -140,47 +185,36 @@ int hook_periodic(struct hook *h) int hook_restart(struct hook *h) { + int ret; + assert(h->state == STATE_STARTED); + if (!h->enabled) return 0; - if (hook_type(h)->restart) { - debug(LOG_HOOK | 10, "Restarting hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); + debug(LOG_HOOK | 10, "Restarting hook %s: priority=%d", hook_type_name(hook_type(h)), h->priority); - return hook_type(h)->restart(h); - } - else - return 0; + ret = hook_type(h)->restart ? hook_type(h)->restart(h) : 0; + if (ret) + return ret; + + return 0; } int hook_process(struct hook *h, struct sample *smps[], unsigned *cnt) { + int ret; + assert(h->state == STATE_STARTED); + if (!h->enabled) return 0; - if (hook_type(h)->process) { - debug(LOG_HOOK | 10, "Process hook %s: priority=%d, cnt=%d", hook_type_name(hook_type(h)), h->priority, *cnt); + debug(LOG_HOOK | 10, "Process hook %s: priority=%d, cnt=%d", hook_type_name(hook_type(h)), h->priority, *cnt); - return hook_type(h)->process(h, smps, cnt); - } - else - return 0; -} + ret = hook_type(h)->process ? hook_type(h)->process(h, smps, cnt) : 0; + if (ret) + return ret; -int hook_process_list(struct vlist *hs, struct sample *smps[], unsigned cnt) -{ - unsigned ret; - - for (size_t i = 0; i < vlist_length(hs); i++) { - struct hook *h = (struct hook *) vlist_at(hs, i); - - ret = hook_process(h, smps, &cnt); - if (ret || !cnt) - /* Abort hook processing if earlier hooks removed all samples - * or they returned something non-zero */ - break; - } - - return cnt; + return 0; } int hook_cmp_priority(const void *a, const void *b) @@ -191,7 +225,29 @@ int hook_cmp_priority(const void *a, const void *b) return ha->priority - hb->priority; } -int hook_parse_list(struct vlist *list, json_t *cfg, int mask, struct path *o, struct node *n) +int hook_list_init(struct vlist *hs) +{ + int ret; + + ret = vlist_init(hs); + if (ret) + return ret; + + return 0; +} + +int hook_list_destroy(struct vlist *hs) +{ + int ret; + + ret = vlist_destroy(hs, (dtor_cb_t) hook_destroy, true); + if (ret) + return ret; + + return 0; +} + +int hook_list_parse(struct vlist *hs, json_t *cfg, int mask, struct path *o, struct node *n) { if (!json_is_array(cfg)) error("Hooks must be configured as a list of objects"); @@ -225,17 +281,42 @@ int hook_parse_list(struct vlist *list, json_t *cfg, int mask, struct path *o, s if (ret) jerror(&err, "Failed to parse hook configuration"); - vlist_push(list, h); + vlist_push(hs, h); } return 0; } -int hook_init_builtin_list(struct vlist *l, bool builtin, int mask, struct path *p, struct node *n) +int hook_list_prepare(struct vlist *hs, struct vlist *sigs, int m, struct path *p, struct node *n) { int ret; - assert(l->state == STATE_INITIALIZED); + /* Add internal hooks if they are not already in the list */ + ret = hook_list_add(hs, m, p, n); + if (ret) + return ret; + + /* We sort the hooks according to their priority */ + vlist_sort(hs, hook_cmp_priority); + + for (size_t i = 0; i < vlist_length(hs); i++) { + struct hook *h = (struct hook *) vlist_at(hs, i); + + ret = hook_prepare(h, sigs); + if (ret) + return ret; + + sigs = &h->signals; + } + + return 0; +} + +int hook_list_add(struct vlist *hs, int mask, struct path *p, struct node *n) +{ + int ret; + + assert(hs->state == STATE_INITIALIZED); for (size_t i = 0; i < vlist_length(&plugins); i++) { struct plugin *q = (struct plugin *) vlist_at(&plugins, i); @@ -246,11 +327,7 @@ int hook_init_builtin_list(struct vlist *l, bool builtin, int mask, struct path if (q->type != PLUGIN_TYPE_HOOK) continue; - if (builtin && - vt->flags & HOOK_BUILTIN && - vt->flags & mask) - { - + if ((vt->flags & mask) == mask) { h = (struct hook *) alloc(sizeof(struct hook)); if (!h) return -1; @@ -259,7 +336,7 @@ int hook_init_builtin_list(struct vlist *l, bool builtin, int mask, struct path if (ret) return ret; - vlist_push(l, h); + vlist_push(hs, h); } } @@ -270,3 +347,20 @@ const char * hook_type_name(struct hook_type *vt) { return plugin_name(vt); } + +int hook_list_process(struct vlist *hs, struct sample *smps[], unsigned cnt) +{ + unsigned ret; + + for (size_t i = 0; i < vlist_length(hs); i++) { + struct hook *h = (struct hook *) vlist_at(hs, i); + + ret = hook_process(h, smps, &cnt); + if (ret || !cnt) + /* Abort hook processing if earlier hooks removed all samples + * or they returned something non-zero */ + break; + } + + return cnt; +} diff --git a/lib/mapping.c b/lib/mapping.c index 7069d013b..5108471e7 100644 --- a/lib/mapping.c +++ b/lib/mapping.c @@ -144,7 +144,7 @@ int mapping_parse_str(struct mapping_entry *me, const char *str, struct vlist *n else { /* Map all signals */ me->data.offset = 0; - me->length = me->node ? vlist_length(&me->node->in.signals) : 0; + me->length = -1; goto end; } @@ -201,9 +201,9 @@ int mapping_parse(struct mapping_entry *me, json_t *cfg, struct vlist *nodes) return mapping_parse_str(me, str, nodes); } -int mapping_parse_list(struct vlist *l, json_t *cfg, struct vlist *nodes) +int mapping_list_parse(struct vlist *ml, json_t *cfg, struct vlist *nodes) { - int ret, off; + int ret; size_t i; json_t *json_entry; @@ -218,7 +218,6 @@ int mapping_parse_list(struct vlist *l, json_t *cfg, struct vlist *nodes) else return -1; - off = 0; json_array_foreach(json_mapping, i, json_entry) { struct mapping_entry *me = (struct mapping_entry *) alloc(sizeof(struct mapping_entry)); @@ -226,10 +225,7 @@ int mapping_parse_list(struct vlist *l, json_t *cfg, struct vlist *nodes) if (ret) goto out; - me->offset = off; - off += me->length; - - vlist_push(l, me); + vlist_push(ml, me); } ret = 0; @@ -239,22 +235,16 @@ out: json_decref(json_mapping); return ret; } -int mapping_update(const struct mapping_entry *me, struct sample *remapped, const struct sample *original, const struct stats *s) +int mapping_update(const struct mapping_entry *me, struct sample *remapped, const struct sample *original) { - int len = me->length; - int off = me->offset; - - /* me->length == 0 means that we want to take all values */ - if (!len) - len = original->length; - - if (len + off > remapped->capacity) + if (me->length + me->offset > remapped->capacity) return -1; switch (me->type) { - case MAPPING_TYPE_STATS: - remapped->data[off++] = stats_get_value(s, me->stats.metric, me->stats.type); + case MAPPING_TYPE_STATS: { + remapped->data[me->offset] = stats_get_value(me->node->stats, me->stats.metric, me->stats.type); break; + } case MAPPING_TYPE_TIMESTAMP: { const struct timespec *ts; @@ -270,8 +260,8 @@ int mapping_update(const struct mapping_entry *me, struct sample *remapped, cons return -1; } - remapped->data[off++].i = ts->tv_sec; - remapped->data[off++].i = ts->tv_nsec; + remapped->data[me->offset + 0].i = ts->tv_sec; + remapped->data[me->offset + 1].i = ts->tv_nsec; break; } @@ -279,11 +269,13 @@ int mapping_update(const struct mapping_entry *me, struct sample *remapped, cons case MAPPING_TYPE_HEADER: switch (me->header.type) { case MAPPING_HEADER_TYPE_LENGTH: - remapped->data[off++].i = original->length; + remapped->data[me->offset].i = original->length; break; + case MAPPING_HEADER_TYPE_SEQUENCE: - remapped->data[off++].i = original->sequence; + remapped->data[me->offset].i = original->sequence; break; + default: return -1; } @@ -291,11 +283,11 @@ int mapping_update(const struct mapping_entry *me, struct sample *remapped, cons break; case MAPPING_TYPE_DATA: - for (int j = me->data.offset; j < len + me->data.offset; j++) { + for (int j = me->data.offset, i = me->offset; j < me->length + me->data.offset; j++, i++) { if (j >= original->length) - remapped->data[off++].f = 0; + remapped->data[i].f = -1; else - remapped->data[off++] = original->data[j]; + remapped->data[i] = original->data[j]; } break; @@ -304,14 +296,14 @@ int mapping_update(const struct mapping_entry *me, struct sample *remapped, cons return 0; } -int mapping_remap(const struct vlist *m, struct sample *remapped, const struct sample *original, const struct stats *s) +int mapping_list_remap(const struct vlist *ml, struct sample *remapped, const struct sample *original) { int ret; - for (size_t i = 0; i < vlist_length(m); i++) { - struct mapping_entry *me = (struct mapping_entry *) vlist_at(m, i); + for (size_t i = 0; i < vlist_length(ml); i++) { + struct mapping_entry *me = (struct mapping_entry *) vlist_at(ml, i); - ret = mapping_update(me, remapped, original, s); + ret = mapping_update(me, remapped, original); if (ret) return ret; } @@ -319,6 +311,24 @@ int mapping_remap(const struct vlist *m, struct sample *remapped, const struct s return 0; } +int mapping_list_prepare(struct vlist *ml) +{ + for (size_t i = 0, off = 0; i < vlist_length(ml); i++) { + struct mapping_entry *me = (struct mapping_entry *) vlist_at(ml, i); + + if (me->length < 0) { + struct vlist *sigs = node_get_signals(me->node, NODE_DIR_IN); + + me->length = vlist_length(sigs); + } + + me->offset = off; + off += me->length; + } + + return 0; +} + int mapping_to_str(const struct mapping_entry *me, unsigned index, char **str) { const char *type; diff --git a/lib/node.c b/lib/node.c index 8a38ff267..b1f94f6ec 100644 --- a/lib/node.c +++ b/lib/node.c @@ -65,11 +65,11 @@ int node_init(struct node *n, struct node_type *vt) #endif /* WITH_NETEM */ /* Default values */ - ret = node_direction_init(&n->in, n); + ret = node_direction_init(&n->in, NODE_DIR_IN, n); if (ret) return ret; - ret = node_direction_init(&n->out, n); + ret = node_direction_init(&n->out, NODE_DIR_OUT, n); if (ret) return ret; @@ -94,7 +94,7 @@ int node_prepare(struct node *n) if (ret) return ret; - ret = node_direction_prepare(&n->in, n); + ret = node_direction_prepare(&n->out, n); if (ret) return ret; @@ -418,7 +418,7 @@ int node_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rel #ifdef WITH_HOOKS /* Run read hooks */ - int rread = hook_process_list(&n->in.hooks, smps, nread); + int rread = hook_list_process(&n->in.hooks, smps, nread); int skipped = nread - rread; if (skipped > 0 && n->stats != NULL) { @@ -448,7 +448,7 @@ int node_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *re #ifdef WITH_HOOKS /* Run write hooks */ - cnt = hook_process_list(&n->out.hooks, smps, cnt); + cnt = hook_list_process(&n->out.hooks, smps, cnt); if (cnt <= 0) return cnt; #endif /* WITH_HOOKS */ @@ -479,7 +479,7 @@ int node_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *re char * node_name(struct node *n) { if (!n->_name) - strcatf(&n->_name, CLR_RED("%s") "(" CLR_YEL("%s") ")", n->name, node_type_name(n->_vt)); + strcatf(&n->_name, CLR_RED("%s") "(" CLR_YEL("%s") ")", n->name, node_type_name(node_type(n))); return n->_name; } diff --git a/lib/node_direction.c b/lib/node_direction.c index 7029f7549..6d9361021 100644 --- a/lib/node_direction.c +++ b/lib/node_direction.c @@ -226,7 +226,7 @@ struct vlist * node_direction_get_signals(struct node_direction *nd) struct hook *h = vlist_last(&nd->hooks); - return h->signals; + return &h->signals; #else return &nd->signals; #endif diff --git a/lib/path.c b/lib/path.c index 177573529..767fd2abf 100644 --- a/lib/path.c +++ b/lib/path.c @@ -241,19 +241,21 @@ int path_prepare(struct path *p) if (ps->masked) bitset_set(&p->mask, i); + ret = mapping_list_prepare(&ps->mappings); + if (ret) + return ret; + for (size_t i = 0; i < vlist_length(&ps->mappings); i++) { struct mapping_entry *me = (struct mapping_entry *) vlist_at(&ps->mappings, i); + struct vlist *sigs = node_get_signals(me->node, NODE_DIR_IN); - int off = me->offset; - int len = me->length; - - for (int j = 0; j < len; j++) { + for (int j = 0; j < me->length; j++) { struct signal *sig; /* For data mappings we simple refer to the existing * signal descriptors of the source node. */ if (me->type == MAPPING_TYPE_DATA) { - sig = (struct signal *) vlist_at_safe(&me->node->in.signals, me->data.offset + j); + sig = (struct signal *) vlist_at_safe(sigs, me->data.offset + j); if (!sig) { warning("Failed to create signal description for path %s", path_name(p)); continue; @@ -270,8 +272,8 @@ int path_prepare(struct path *p) return -1; } - vlist_extend(&p->signals, off + j + 1, NULL); - vlist_set(&p->signals, off + j, sig); + vlist_extend(&p->signals, me->offset + j + 1, NULL); + vlist_set(&p->signals, me->offset + j, sig); } } } @@ -328,7 +330,7 @@ int path_parse(struct path *p, json_t *cfg, struct vlist *nodes) jerror(&err, "Failed to parse path configuration"); /* Input node(s) */ - ret = mapping_parse_list(&sources, json_in, nodes); + ret = mapping_list_parse(&sources, json_in, nodes); if (ret) error("Failed to parse input mapping of path %s", path_name(p)); diff --git a/lib/path_source.c b/lib/path_source.c index ae8225588..21bd21f36 100644 --- a/lib/path_source.c +++ b/lib/path_source.c @@ -121,7 +121,7 @@ int path_source_read(struct path_source *ps, struct path *p, int i) muxed_smps[i]->ts = tomux_smps[i]->ts; muxed_smps[i]->flags |= tomux_smps[i]->flags & (SAMPLE_HAS_TS_ORIGIN | SAMPLE_HAS_TS_RECEIVED); - mapping_remap(&ps->mappings, muxed_smps[i], tomux_smps[i], NULL); + mapping_list_remap(&ps->mappings, muxed_smps[i], tomux_smps[i]); } sample_copy(p->last_sample, muxed_smps[tomux-1]); diff --git a/lib/super_node.cpp b/lib/super_node.cpp index f4780ec1e..78c241e04 100644 --- a/lib/super_node.cpp +++ b/lib/super_node.cpp @@ -420,12 +420,6 @@ void SuperNode::preparePaths() } void SuperNode::prepare() -{ - prepareNodes(); - preparePaths(); -} - -void SuperNode::start() { int ret; @@ -437,6 +431,18 @@ void SuperNode::start() kernel::rt::init(priority, affinity); + prepareNodes(); + preparePaths(); + + state = STATE_PREPARED; +} + +void SuperNode::start() +{ + int ret; + + assert(state == STATE_PREPARED); + #ifdef WITH_API api.start(); #endif diff --git a/src/villas-hook.cpp b/src/villas-hook.cpp index 2d30b86e9..f4fee472b 100644 --- a/src/villas-hook.cpp +++ b/src/villas-hook.cpp @@ -194,6 +194,10 @@ check: if (optarg == endptr) if (ret) throw RuntimeError("Failed to parse hook config"); + ret = hook_prepare(&h, io.signals); + if (ret) + throw RuntimeError("Failed to prepare hook"); + ret = hook_start(&h); if (ret) throw RuntimeError("Failed to start hook"); From df6317d5932e51e7749fdac66e92bed9f855a75c Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:32:31 +0100 Subject: [PATCH 34/38] fix format string --- src/villas-signal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/villas-signal.cpp b/src/villas-signal.cpp index 4040cf7b6..0bbd8aeee 100644 --- a/src/villas-signal.cpp +++ b/src/villas-signal.cpp @@ -109,7 +109,7 @@ json_t * parse_cli(int argc, char *argv[]) continue; check: if (optarg == endptr) - logger->warn("Failed to parse parse option argument '-%c %s'", c, optarg); + logger->warn("Failed to parse parse option argument '-{} {}'", c, optarg); } if (argc != optind + 1) From 4229531cf1e54c38c49aad12977974a9beb3728a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:32:59 +0100 Subject: [PATCH 35/38] hooks: rewrite a bunch of hooks to the new signal list method --- lib/hooks/CMakeLists.txt | 2 +- lib/hooks/average.c | 111 +++++++++++++++++++++--- lib/hooks/cast.c | 129 ++++++++++++++-------------- lib/hooks/dp.c | 178 +++++++++++++++++++++++++++------------ lib/hooks/print.c | 13 +-- lib/hooks/scale.c | 92 ++++++++++++++------ 6 files changed, 361 insertions(+), 164 deletions(-) diff --git a/lib/hooks/CMakeLists.txt b/lib/hooks/CMakeLists.txt index c9b2f1de7..afc12824d 100644 --- a/lib/hooks/CMakeLists.txt +++ b/lib/hooks/CMakeLists.txt @@ -33,7 +33,7 @@ set(HOOK_SRC limit_rate.c scale.c fix.c -# cast.c + cast.c average.c dump.c dp.c diff --git a/lib/hooks/average.c b/lib/hooks/average.c index f50f8a982..338c03005 100644 --- a/lib/hooks/average.c +++ b/lib/hooks/average.c @@ -29,43 +29,129 @@ #include #include #include +#include struct average { - int mask; int offset; + + struct bitset mask; + struct vlist signal_names; }; +static int average_init(struct hook *h) +{ + int ret; + struct average *a = (struct average *) h->_vd; + + ret = vlist_init(&a->signal_names); + if (ret) + return ret; + + ret = bitset_init(&a->mask, 128); + if (ret) + return ret; + + bitset_clear_all(&a->mask); + + return 0; +} + +static int average_destroy(struct hook *h) +{ + int ret; + struct average *a = (struct average *) h->_vd; + + ret = vlist_destroy(&a->signal_names, NULL, true); + if (ret) + return ret; + + ret = bitset_destroy(&a->mask); + if (ret) + return ret; + + return 0; +} + +static int average_prepare(struct hook *h) +{ + int ret; + struct average *a = (struct average *) h->_vd; + struct signal *avg_sig; + + /* Setup mask */ + for (size_t i = 0; i < vlist_length(&a->signal_names); i++) { + char *signal_name = (char *) vlist_at_safe(&a->signal_names, i); + + int index = vlist_lookup_index(&a->signal_names, signal_name); + if (index < 0) + return -1; + + bitset_set(&a->mask, index); + } + + /* Add averaged signal */ + avg_sig = signal_create("average", NULL, SIGNAL_TYPE_FLOAT); + if (!avg_sig) + return -1; + + ret = vlist_insert(&h->signals, a->offset, avg_sig); + if (ret) + return ret; + + return 0; +} + static int average_parse(struct hook *h, json_t *cfg) { - struct average *p = (struct average *) h->_vd; + struct average *a = (struct average *) h->_vd; int ret; + size_t i; json_error_t err; + json_t *json_signals, *json_signal; - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: i }", - "offset", &p->offset, - "mask", &p->mask + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: o }", + "offset", &a->offset, + "signals", &json_signals ); if (ret) jerror(&err, "Failed to parse configuration of hook '%s'", hook_type_name(h->_vt)); + if (!json_is_array(json_signals)) + error("Setting 'signals' of hook '%s' must be a list of signal names", hook_type_name(h->_vt)); + + json_array_foreach(json_signals, i, json_signal) { + switch (json_typeof(json_signal)) { + case JSON_STRING: + vlist_push(&a->signal_names, strdup(json_string_value(json_signal))); + break; + + case JSON_INTEGER: + bitset_set(&a->mask, json_integer_value(json_signal)); + break; + + default: + error("Invalid value for setting 'signals' in hook '%s'", hook_type_name(h->_vt)); + } + } + return 0; } static int average_process(struct hook *h, struct sample *smps[], unsigned *cnt) { - struct average *p = (struct average *) h->_vd; + struct average *a = (struct average *) h->_vd; for (int i = 0; i < *cnt; i++) { struct sample *smp = smps[i]; - double sum = 0; + double avg, sum = 0; int n = 0; for (int k = 0; k < smp->length; k++) { - if (!(p->mask & (1 << k))) + if (!bitset_test(&a->mask, k)) continue; - switch (sample_format(smps[i], k)) { + switch (sample_format(smp, k)) { case SIGNAL_TYPE_INTEGER: sum += smp->data[k].i; break; @@ -84,7 +170,9 @@ static int average_process(struct hook *h, struct sample *smps[], unsigned *cnt) n++; } - smp->data[p->offset].f = sum / n; + avg = n == 0 ? 0 : sum / n; + sample_data_insert(smp, (union signal_data *) &avg, a->offset, 1); + smp->signals = &h->signals; } return 0; @@ -99,6 +187,9 @@ static struct plugin p = { .priority = 99, .parse = average_parse, .process = average_process, + .init = average_init, + .init_signals = average_prepare, + .destroy = average_destroy, .size = sizeof(struct average) } }; diff --git a/lib/hooks/cast.c b/lib/hooks/cast.c index 886cee9e6..7d5382815 100644 --- a/lib/hooks/cast.c +++ b/lib/hooks/cast.c @@ -29,38 +29,58 @@ #include #include -#include -#include #include struct cast { - int index; + int signal_index; + char *signal_name; + + enum signal_type new_type; + char *new_name; + char *new_unit; }; -static int cast_init(struct hook *h) +static int cast_prepare(struct hook *h) { - int ret; + struct signal *orig_sig, *new_sig; struct cast *c = (struct cast *) h->_vd; - struct vlist *orig_signals; - if (h->node) - orig_signals = &h->node->in.signals; - else if (h->path) - orig_signals = &h->path->signals; - else - return -1; + if (c->signal_name) { + c->signal_index = vlist_lookup_index(&h->signals, c->signal_name); + if (c->signal_index < 0) + return -1; + } - struct signal *orig_sig = vlist_at(orig_signals, i); - struct signal *new_sig = signal_copy(orig_sig); + char *name, *unit; + enum signal_type type; + + orig_sig = vlist_at_safe(&h->signals, c->signal_index); + + type = c->new_type != SIGNAL_TYPE_AUTO ? c->new_type : orig_sig->type; + name = c->new_name ? c->new_name : orig_sig->name; + unit = c->new_unit ? c->new_unit : orig_sig->unit; + + new_sig = signal_create(name, unit, type); + + vlist_set(&h->signals, c->signal_index, new_sig); + signal_decref(orig_sig); return 0; } static int cast_destroy(struct hook *h) { - int ret; struct cast *c = (struct cast *) h->_vd; + if (c->signal_name) + free(c->signal_name); + + if (c->new_name) + free(c->new_name); + + if (c->new_unit) + free(c->new_unit); + return 0; } @@ -68,67 +88,50 @@ static int cast_parse(struct hook *h, json_t *cfg) { int ret; struct cast *c = (struct cast *) h->_vd; - struct signal *sig; - size_t i; json_error_t err; - - int index = -1; - const char *name = NULL; + json_t *json_signal; const char *new_name = NULL; const char *new_unit = NULL; - const char *new_format = NULL; + const char *new_type = NULL; - ret = json_unpack_ex(cfg, &err, "{ s?: s, s?: i, s?: s, s?: s, s?: s }", - "name", &name, - "index", &index, - "new_format", &new_format, + ret = json_unpack_ex(cfg, &err, 0, "{ s: o, s?: s, s?: s, s?: s }", + "signal", &json_signal, + "new_type", &new_type, "new_name", &new_name, "new_unit", &new_unit ); if (ret) return ret; - /* Find matching original signal descriptor */ - if (index >= 0 && name != NULL) - return -1; + switch (json_typeof(json_signal)) { + case JSON_STRING: + c->signal_name = strdup(json_string_value(json_signal)); + break; - if (index < 0 && name == NULL) - return -1; + case JSON_INTEGER: + c->signal_name = NULL; + c->signal_index = json_integer_value(json_signal); + break; - sig = name - ? vlist_lookup(&c->signals, name) - : vlist_at_safe(&c->signals, index); - if (!sig) - return -1; + default: + error("Invalid value for setting 'signal' in hook '%s'", hook_type_name(h->_vt)); + } - /* Cast to new format */ - if (new_format) { - enum signal_type fmt; - - fmt = signal_type_from_str(new_format); - if (fmt == SIGNAL_TYPE_INVALID) + if (new_type) { + c->new_type = signal_type_from_str(new_type); + if (c->new_type == SIGNAL_TYPE_INVALID) return -1; - - sig->type = fmt; } + else + c->new_type = SIGNAL_TYPE_AUTO; // We use this constant to indicate that we dont want to change the type - /* Set new name */ - if (new_name) { - if (sig->name) - free(sig->name); + if (new_name) + c->new_name = strdup(new_name); - sig->name = strdup(new_name); - } - - /* Set new unit */ - if (new_unit) { - if (sig->unit) - free(sig->unit); - - sig->unit = strdup(new_unit); - } + if (new_unit) + c->new_unit = strdup(new_unit); return 0; } @@ -140,15 +143,13 @@ static int cast_process(struct hook *h, struct sample *smps[], unsigned *cnt) for (int i = 0; i < *cnt; i++) { struct sample *smp = smps[i]; - for (int j = 0; j < smp->length; j++) { - struct signal *orig_sig = vlist_at(smp->signals, j); - struct signal *new_sig = vlist_at(&c->signals, j); + struct signal *orig_sig = vlist_at(smp->signals, c->signal_index); + struct signal *new_sig = vlist_at(&h->signals, c->signal_index); - signal_data_cast(&smp->data[j], orig_sig, new_sig); - } + signal_data_cast(&smp->data[c->signal_index], orig_sig, new_sig); /* Replace signal descriptors of sample */ - smp->signals = &c->signals; + smp->signals = &h->signals; } return 0; @@ -161,8 +162,8 @@ static struct plugin p = { .hook = { .flags = HOOK_NODE_READ | HOOK_PATH, .priority = 99, - .init = cast_init, .destroy = cast_destroy, + .init_signals = cast_prepare, .parse = cast_parse, .process = cast_process, .size = sizeof(struct cast) diff --git a/lib/hooks/dp.c b/lib/hooks/dp.c index e727b2452..d540b6332 100644 --- a/lib/hooks/dp.c +++ b/lib/hooks/dp.c @@ -36,7 +36,10 @@ #define J _Complex_I struct dp { - int index; + char *signal_name; + int signal_index; + + int offset; int inverse; double f0; @@ -48,30 +51,31 @@ struct dp { int fharmonics_len; struct window window; - - struct vlist *signals; }; static void dp_step(struct dp *d, double *in, float complex *out) { + int n = d->window.steps; + double r = 0.9999999999; + double complex om, corr; double newest = *in; double oldest = window_update(&d->window, newest); for (int i = 0; i < d->fharmonics_len; i++) { - double pi_fharm = 2.0 * M_PI * d->fharmonics[i]; + om = 2.0 * M_PI * J * d->fharmonics[i] / n; /* Recursive update */ - d->coeffs[i] = (d->coeffs[i] + (newest - oldest)) * cexp(pi_fharm); + //d->coeffs[i] = cexp(om) * (d->coeffs[i] + (newest - oldest)); + d->coeffs[i] = d->coeffs[i] * r * cexp(om) - powf(r, n) * oldest + newest; /* Correction for stationary phasor */ - double complex correction = cexp(pi_fharm * (d->t - (d->window.steps + 1))); - double complex result = 2.0 / d->window.steps * d->coeffs[i] / correction; + corr = cexp(-om * (d->t - (d->window.steps + 1))); + + out[i] = (2.0 / d->window.steps) * (d->coeffs[i] * corr); /* DC component */ - if (i == 0) - result /= 2.0; - - out[i] = result; + if (d->fharmonics[i] == 0) + out[i] /= 2.0; } } @@ -96,7 +100,6 @@ static int dp_start(struct hook *h) struct dp *d = (struct dp *) h->_vd; d->t = 0; - d->signals = NULL; for (int i = 0; i < d->fharmonics_len; i++) d->coeffs[i] = 0; @@ -138,6 +141,9 @@ static int dp_destroy(struct hook *h) free(d->fharmonics); free(d->coeffs); + if (d->signal_name) + free(d->signal_name); + return 0; } @@ -147,13 +153,13 @@ static int dp_parse(struct hook *h, json_t *cfg) int ret; json_error_t err; - json_t *json_harmonics, *json_harmonic; + json_t *json_harmonics, *json_harmonic, *json_signal; size_t i; double rate = -1, dt = -1; - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: F, s?: F, s?: F, s: o, s?: b }", - "index", &d->index, + ret = json_unpack_ex(cfg, &err, 0, "{ s: o, s: F, s?: F, s?: F, s: o, s?: b }", + "signal", &json_signal, "f0", &d->f0, "dt", &dt, "rate", &rate, @@ -173,6 +179,20 @@ static int dp_parse(struct hook *h, json_t *cfg) if (!json_is_array(json_harmonics)) error("Setting 'harmonics' of hook '%s' must be a list of integers", plugin_name(h->_vt)); + switch (json_typeof(json_signal)) { + case JSON_STRING: + d->signal_name = strdup(json_string_value(json_signal)); + break; + + case JSON_INTEGER: + d->signal_name = NULL; + d->signal_index = json_integer_value(json_signal); + break; + + default: + error("Invalid value for setting 'signal' in hook '%s'", hook_type_name(h->_vt)); + } + d->fharmonics_len = json_array_size(json_harmonics); d->fharmonics = alloc(d->fharmonics_len * sizeof(double)); d->coeffs = alloc(d->fharmonics_len * sizeof(double complex)); @@ -183,7 +203,79 @@ static int dp_parse(struct hook *h, json_t *cfg) if (!json_is_integer(json_harmonic)) error("Setting 'harmonics' of hook '%s' must be a list of integers", plugin_name(h->_vt)); - d->fharmonics[i] = d->f0 * json_integer_value(json_harmonic); + d->fharmonics[i] = json_integer_value(json_harmonic); + } + + return 0; +} + +static int dp_prepare(struct hook *h) +{ + int ret; + struct dp *d = (struct dp *) h->_vd; + + char *new_sig_name; + struct signal *orig_sig, *new_sig; + + if (d->signal_name) { + d->signal_index = vlist_lookup_index(&h->signals, d->signal_name); + if (d->signal_index < 0) + return -1; + } + + if (d->inverse) { + /* Remove complex-valued coefficient signals */ + for (int i = 0; i < d->fharmonics_len; i++) { + orig_sig = vlist_at_safe(&h->signals, d->signal_index + i); + if (!orig_sig) + return -1; + + /** @todo: SIGNAL_TYPE_AUTO is bad here */ + if (orig_sig->type != SIGNAL_TYPE_COMPLEX && orig_sig->type != SIGNAL_TYPE_AUTO) + return -1; + + ret = vlist_remove(&h->signals, d->signal_index + i); + if (ret) + return -1; + + signal_decref(orig_sig); + } + + /* Add new real-valued reconstructed signals */ + new_sig = signal_create("dp", "idp", SIGNAL_TYPE_FLOAT); + if (!new_sig) + return -1; + + ret = vlist_insert(&h->signals, d->offset, new_sig); + if (ret) + return -1; + } + else { + orig_sig = vlist_at_safe(&h->signals, d->signal_index); + if (!orig_sig) + return -1; + + /** @todo: SIGNAL_TYPE_AUTO is bad here */ + if (orig_sig->type != SIGNAL_TYPE_FLOAT && orig_sig->type != SIGNAL_TYPE_AUTO) + return -1; + + ret = vlist_remove(&h->signals, d->signal_index); + if (ret) + return -1; + + for (int i = 0; i < d->fharmonics_len; i++) { + new_sig_name = strf("%s_harm%d", orig_sig->name, i); + + new_sig = signal_create(new_sig_name, orig_sig->unit, SIGNAL_TYPE_COMPLEX); + if (!new_sig) + return -1; + + ret = vlist_insert(&h->signals, d->offset + i, new_sig); + if (ret) + return -1; + } + + signal_decref(orig_sig); } return 0; @@ -196,50 +288,29 @@ static int dp_process(struct hook *h, struct sample *smps[], unsigned *cnt) for (unsigned j = 0; j < *cnt; j++) { struct sample *smp = smps[j]; - if (d->index > smp->length) + if (d->signal_index > smp->length) continue; - if (!d->signals) { - struct signal *orig_sig, *new_sig; + if (d->inverse) { + double signal; + float complex *coeffs = &smp->data[d->signal_index].z; - d->signals = alloc(sizeof(struct vlist)); - d->signals->state = STATE_DESTROYED; + dp_istep(d, coeffs, &signal); - vlist_copy(d->signals, smp->signals); + sample_data_remove(smp, d->signal_index, d->fharmonics_len); + sample_data_insert(smp, (union signal_data *) &signal, d->offset, 1); + } + else { + double signal = smp->data[d->signal_index].f; + float complex coeffs[d->fharmonics_len]; - orig_sig = vlist_at(smp->signals, d->index); - if (!orig_sig) - return -1; + dp_step(d, &signal, coeffs); - if (d->inverse) { - if (orig_sig->type != SIGNAL_TYPE_COMPLEX) - return -1; - } - else { - if (orig_sig->type != SIGNAL_TYPE_FLOAT) - return -1; - } - - new_sig = signal_copy(orig_sig); - if (!new_sig) - return -1; - - if (d->inverse) - new_sig->type = SIGNAL_TYPE_FLOAT; - else - new_sig->type = SIGNAL_TYPE_COMPLEX; - - int ret = vlist_set(d->signals, d->index, new_sig); - if (ret) - return ret; + sample_data_remove(smp, d->signal_index, 1); + sample_data_insert(smp, (union signal_data *) coeffs, d->offset, d->fharmonics_len); } - smp->signals = d->signals; - - if (d->inverse) - dp_istep(d, &smp->data[d->index].z, &smp->data[d->index].f); - else - dp_step(d, &smp->data[d->index].f, &smp->data[d->index].z); + smp->signals = &h->signals; } d->t += d->dt; @@ -252,9 +323,10 @@ static struct plugin p = { .description = "Transform to/from dynamic phasor domain", .type = PLUGIN_TYPE_HOOK, .hook = { - .flags = HOOK_PATH, + .flags = HOOK_PATH | HOOK_NODE_READ | HOOK_NODE_WRITE, .priority = 99, .init = dp_init, + .init_signals = dp_prepare, .destroy = dp_destroy, .start = dp_start, .stop = dp_stop, diff --git a/lib/hooks/print.c b/lib/hooks/print.c index ce930ff00..a88a541db 100644 --- a/lib/hooks/print.c +++ b/lib/hooks/print.c @@ -56,18 +56,7 @@ static int print_start(struct hook *h) struct print *p = (struct print *) h->_vd; int ret; - struct vlist *signals; - - if (h->node) - signals = &h->node->in.signals; - else if (h->path) - signals = &h->path->signals; - else - signals = NULL; - - ret = signals - ? io_init(&p->io, p->format, signals, SAMPLE_HAS_ALL) - : io_init_auto(&p->io, p->format, DEFAULT_SAMPLE_LENGTH, SAMPLE_HAS_ALL); + ret = io_init(&p->io, p->format, &h->signals, SAMPLE_HAS_ALL); if (ret) return ret; diff --git a/lib/hooks/scale.c b/lib/hooks/scale.c index 343b54f49..3b8b7e59f 100644 --- a/lib/hooks/scale.c +++ b/lib/hooks/scale.c @@ -31,63 +31,105 @@ #include struct scale { + char *signal_name; + int signal_index; + double scale; double offset; }; static int scale_init(struct hook *h) { - struct scale *p = (struct scale *) h->_vd; + struct scale *s = (struct scale *) h->_vd; - p->scale = 1; - p->offset = 0; + s->scale = 1; + s->offset = 0; + + return 0; +} + +static int scale_prepare(struct hook *h) +{ + struct scale *s = (struct scale *) h->_vd; + + if (s->signal_name) { + s->signal_index = vlist_lookup_index(&h->signals, s->signal_name); + if (s->signal_index < 0) + return -1; + } + + return 0; +} + +static int scale_destroy(struct hook *h) +{ + struct scale *s = (struct scale *) h->_vd; + + if (s->signal_name) + free(s->signal_name); return 0; } static int scale_parse(struct hook *h, json_t *cfg) { - struct scale *p = (struct scale *) h->_vd; + struct scale *s = (struct scale *) h->_vd; int ret; + json_t *json_signal; json_error_t err; - ret = json_unpack_ex(cfg, &err, 0, "{ s?: F, s?: F }", - "scale", &p->scale, - "offset", &p->offset + ret = json_unpack_ex(cfg, &err, 0, "{ s?: F, s?: F, s: o }", + "scale", &s->scale, + "offset", &s->offset, + "signal", &json_signal ); if (ret) jerror(&err, "Failed to parse configuration of hook '%s'", hook_type_name(h->_vt)); + switch (json_typeof(json_signal)) { + case JSON_STRING: + s->signal_name = strdup(json_string_value(json_signal)); + break; + + case JSON_INTEGER: + s->signal_name = NULL; + s->signal_index = json_integer_value(json_signal); + break; + + default: + error("Invalid value for setting 'signal' in hook '%s'", hook_type_name(h->_vt)); + } + return 0; } static int scale_process(struct hook *h, struct sample *smps[], unsigned *cnt) { - struct scale *p = (struct scale *) h->_vd; + struct scale *s = (struct scale *) h->_vd; for (int i = 0; i < *cnt; i++) { - for (int k = 0; k < smps[i]->length; k++) { + struct sample *smp = smps[i]; + int k = s->signal_index; - switch (sample_format(smps[i], k)) { - case SIGNAL_TYPE_INTEGER: - smps[i]->data[k].i = smps[i]->data[k].i * p->scale + p->offset; - break; + switch (sample_format(smp, k)) { + case SIGNAL_TYPE_INTEGER: + smp->data[k].i = smp->data[k].i * s->scale + s->offset; + break; - case SIGNAL_TYPE_FLOAT: - smps[i]->data[k].f = smps[i]->data[k].f * p->scale + p->offset; - break; + case SIGNAL_TYPE_FLOAT: + smp->data[k].f = smp->data[k].f * s->scale + s->offset; + break; - case SIGNAL_TYPE_COMPLEX: - smps[i]->data[k].z = smps[i]->data[k].z * p->scale + p->offset; - break; + case SIGNAL_TYPE_COMPLEX: + smp->data[k].z = smp->data[k].z * s->scale + s->offset; + break; - case SIGNAL_TYPE_BOOLEAN: - smps[i]->data[k].b = smps[i]->data[k].b * p->scale + p->offset; - break; + case SIGNAL_TYPE_BOOLEAN: + smp->data[k].b = smp->data[k].b * s->scale + s->offset; + break; - default: { } - } + default: { } } } @@ -102,6 +144,8 @@ static struct plugin p = { .flags = HOOK_PATH, .priority = 99, .init = scale_init, + .init_signals = scale_prepare, + .destroy = scale_destroy, .parse = scale_parse, .process = scale_process, .size = sizeof(struct scale) From c8ada99e0324ab799cd2d9278784acf02385832c Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:33:13 +0100 Subject: [PATCH 36/38] tests: update hook tests --- tests/integration/hook-average.sh | 28 +++++++++------- tests/integration/hook-cast.sh | 56 +++++++++++++------------------ tests/integration/hook-dp.sh | 19 ++++++----- tests/integration/hook-scale.sh | 45 +++++++++++++------------ 4 files changed, 74 insertions(+), 74 deletions(-) diff --git a/tests/integration/hook-average.sh b/tests/integration/hook-average.sh index 33b9d9224..aca7d593a 100755 --- a/tests/integration/hook-average.sh +++ b/tests/integration/hook-average.sh @@ -40,26 +40,30 @@ cat < ${INPUT_FILE} EOF cat < ${EXPECT_FILE} -1548104309.033621000(0) -0.488878 0.590769 -1.000000 0.597649 0.100588 -1548104309.133998900(1) -0.492331 0.952914 -1.000000 0.196137 0.200966 -1548104309.233542500(2) -0.486250 0.950063 -1.000000 -0.202037 0.300509 -1548104309.334019400(3) -0.479840 0.582761 -1.000000 -0.603945 0.400986 -1548104309.433952200(4) 0.513039 -0.005774 1.000000 -0.996324 0.500919 -1548104309.533756400(5) 0.524631 -0.591455 1.000000 -0.597107 0.600723 -1548104309.637440300(6) 0.507441 -0.959248 1.000000 -0.182372 0.704407 -1548104309.736158700(7) 0.511616 -0.944805 1.000000 0.212502 0.803126 -1548104309.833614900(8) 0.507615 -0.584824 1.000000 0.602327 0.900582 -1548104309.934288200(9) -0.469575 0.007885 -1.000000 0.994980 0.001255 +# seconds.nanoseconds+offset(sequence) average signal0 signal1 signal2 signal3 signal4 +1548104309.033621000(0) 0.062250 0.022245 0.590769 -1.000000 0.597649 0.100588 +1548104309.133998900(1) 0.073071 0.015339 0.952914 -1.000000 0.196137 0.200966 +1548104309.233542500(2) 0.015207 0.027500 0.950063 -1.000000 -0.202037 0.300509 +1548104309.334019400(3) -0.115976 0.040320 0.582761 -1.000000 -0.603945 0.400986 +1548104309.433952200(4) 0.104980 0.026079 -0.005774 1.000000 -0.996324 0.500919 +1548104309.533756400(5) 0.092285 0.049262 -0.591455 1.000000 -0.597107 0.600723 +1548104309.637440300(6) 0.115534 0.014883 -0.959248 1.000000 -0.182372 0.704407 +1548104309.736158700(7) 0.218811 0.023232 -0.944805 1.000000 0.212502 0.803126 +1548104309.833614900(8) 0.386663 0.015231 -0.584824 1.000000 0.602327 0.900582 +1548104309.934288200(9) 0.012994 0.060849 0.007885 -1.000000 0.994980 0.001255 EOF # Average over first and third signal (mask = 0b101 = 5) -villas-hook -o mask=5 -o offset=0 average < ${INPUT_FILE} > ${OUTPUT_FILE} +villas-hook -o mask=5 -o offset=0 -o signals=0,1,2,3,4 average < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} - RC=$? +cat ${INPUT_FILE} +echo +cat ${OUTPUT_FILE} + rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} exit $RC diff --git a/tests/integration/hook-cast.sh b/tests/integration/hook-cast.sh index be186e76b..dd08a522a 100755 --- a/tests/integration/hook-cast.sh +++ b/tests/integration/hook-cast.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Integration test for convert hook. +# Integration test for cast hook. # # @author Steffen Vogel # @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC @@ -22,50 +22,42 @@ # along with this program. If not, see . ################################################################################## -# We skip this test for now -echo "Test not yet supported" -exit 99 - INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) EXPECT_FILE=$(mktemp) cat < ${INPUT_FILE} -1490500399.776379108(0) 0.000000 0.000000 0.000000 0.000000 -1490500399.876379108(1) 0.587785 0.587785 0.587785 0.587785 -1490500399.976379108(2) 0.951057 0.951057 0.951057 0.951057 -1490500400.076379108(3) 0.951057 0.951057 0.951057 0.951057 -1490500400.176379108(4) 0.587785 0.587785 0.587785 0.587785 -1490500400.276379108(5) 0.000000 0.000000 0.000000 0.000000 -1490500400.376379108(6) -0.587785 -0.587785 -0.587785 -0.587785 -1490500400.476379108(7) -0.951057 -0.951057 -0.951057 -0.951057 -1490500400.576379108(8) -0.951057 -0.951057 -0.951057 -0.951057 -1490500400.676379108(9) -0.587785 -0.587785 -0.587785 -0.587785 +# seconds.nanoseconds+offset(sequence) signal0 signal1 signal2 signal3 signal4 +1551015508.801653200(0) 0.022245 0.000000 -1.000000 1.000000 0.000000 +1551015508.901653200(1) 0.015339 58.778500 -1.000000 0.600000 0.100000 +1551015509.001653200(2) 0.027500 95.105700 -1.000000 0.200000 0.200000 +1551015509.101653200(3) 0.040320 95.105700 -1.000000 -0.200000 0.300000 +1551015509.201653200(4) 0.026079 58.778500 -1.000000 -0.600000 0.400000 +1551015509.301653200(5) 0.049262 0.000000 1.000000 -1.000000 0.500000 +1551015509.401653200(6) 0.014883 -58.778500 1.000000 -0.600000 0.600000 +1551015509.501653200(7) 0.023232 -95.105700 1.000000 -0.200000 0.700000 +1551015509.601653200(8) 0.015231 -95.105700 1.000000 0.200000 0.800000 +1551015509.701653200(9) 0.060849 -58.778500 1.000000 0.600000 0.900000 EOF cat < ${EXPECT_FILE} -1490500399.776379108(0) 0.000000 0 0 0.000000 -1490500399.876379108(1) 0.587785 58 58 0.587785 -1490500399.976379108(2) 0.951057 95 95 0.951057 -1490500400.076379108(3) 0.951057 95 95 0.951057 -1490500400.176379108(4) 0.587785 58 58 0.587785 -1490500400.276379108(5) 0.000000 0 0 0.000000 -1490500400.376379108(6) -0.587785 -58 -58 -0.587785 -1490500400.476379108(7) -0.951057 -95 -95 -0.951057 -1490500400.576379108(8) -0.951057 -95 -95 -0.951057 -1490500400.676379108(9) -0.587785 -58 -58 -0.587785 +# seconds.nanoseconds+offset(sequence) signal0 test[V] signal2 signal3 signal4 +1551015508.801653200(0) 0.022245 0 -1.000000 1.000000 0.000000 +1551015508.901653200(1) 0.015339 58 -1.000000 0.600000 0.100000 +1551015509.001653200(2) 0.027500 95 -1.000000 0.200000 0.200000 +1551015509.101653200(3) 0.040320 95 -1.000000 -0.200000 0.300000 +1551015509.201653200(4) 0.026079 58 -1.000000 -0.600000 0.400000 +1551015509.301653200(5) 0.049262 0 1.000000 -1.000000 0.500000 +1551015509.401653200(6) 0.014883 -58 1.000000 -0.600000 0.600000 +1551015509.501653200(7) 0.023232 -95 1.000000 -0.200000 0.700000 +1551015509.601653200(8) 0.015231 -95 1.000000 0.200000 0.800000 +1551015509.701653200(9) 0.060849 -58 1.000000 0.600000 0.900000 EOF -cat ${INPUT_FILE} | \ -villas-hook -o scale=100 scale | \ -villas-hook cast | \ -tee ${OUTPUT_FILE} - -cat ${OUTPUT_FILE} +villas-hook cast -o new_name=test -o new_unit=V -o new_type=integer -o signal=1 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} - RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-dp.sh b/tests/integration/hook-dp.sh index b09219163..3b3f9c417 100755 --- a/tests/integration/hook-dp.sh +++ b/tests/integration/hook-dp.sh @@ -24,27 +24,30 @@ set -x -INPUT_FILE=$(mktemp) -OUTPUT_FILE=$(mktemp) -EXPECT_FILE=$(mktemp) +#INPUT_FILE=$(mktemp) +#OUTPUT_FILE=$(mktemp) +#RECON_FILE=$(mktemp) -NUM_SAMPLES=40000 +INPUT_FILE=in +OUTPUT_FILE=out +RECON_FILE=recon + +NUM_SAMPLES=10000 RATE=5000 F0=50 -OPTS="-d debug -o f0=${F0} -o rate=${RATE} -o index=0 -o harmonics=0,1,3,5" +OPTS="-o f0=${F0} -o rate=${RATE} -o signal=0 -o harmonics=0,1,3,5,7" villas-signal sine -v1 -l ${NUM_SAMPLES} -f ${F0} -r ${RATE} -n > ${INPUT_FILE} villas-hook dp -o inverse=false ${OPTS} < ${INPUT_FILE} > ${OUTPUT_FILE} -cat ${OUTPUT_FILE} +villas-hook dp -o inverse=true ${OPTS} < ${OUTPUT_FILE} > ${RECON_FILE} -exit 1 +exit 0 # Compare only the data values villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} - RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-scale.sh b/tests/integration/hook-scale.sh index 0e222dff8..0518974ce 100755 --- a/tests/integration/hook-scale.sh +++ b/tests/integration/hook-scale.sh @@ -27,36 +27,37 @@ OUTPUT_FILE=$(mktemp) EXPECT_FILE=$(mktemp) cat < ${INPUT_FILE} -1490500399.776379108(0) 0.000000 0.000000 0.000000 0.000000 -1490500399.876379108(1) 0.587785 0.587785 0.587785 0.587785 -1490500399.976379108(2) 0.951057 0.951057 0.951057 0.951057 -1490500400.076379108(3) 0.951057 0.951057 0.951057 0.951057 -1490500400.176379108(4) 0.587785 0.587785 0.587785 0.587785 -1490500400.276379108(5) 0.000000 0.000000 0.000000 0.000000 -1490500400.376379108(6) -0.587785 -0.587785 -0.587785 -0.587785 -1490500400.476379108(7) -0.951057 -0.951057 -0.951057 -0.951057 -1490500400.576379108(8) -0.951057 -0.951057 -0.951057 -0.951057 -1490500400.676379108(9) -0.587785 -0.587785 -0.587785 -0.587785 +# seconds.nanoseconds(sequence) random sine square triangle ramp +1551015508.801653200(0) 0.022245 0.000000 -1.000000 1.000000 0.000000 +1551015508.901653200(1) 0.015339 0.587785 -1.000000 0.600000 0.100000 +1551015509.001653200(2) 0.027500 0.951057 -1.000000 0.200000 0.200000 +1551015509.101653200(3) 0.040320 0.951057 -1.000000 -0.200000 0.300000 +1551015509.201653200(4) 0.026079 0.587785 -1.000000 -0.600000 0.400000 +1551015509.301653200(5) 0.049262 0.000000 1.000000 -1.000000 0.500000 +1551015509.401653200(6) 0.014883 -0.587785 1.000000 -0.600000 0.600000 +1551015509.501653200(7) 0.023232 -0.951057 1.000000 -0.200000 0.700000 +1551015509.601653200(8) 0.015231 -0.951057 1.000000 0.200000 0.800000 +1551015509.701653200(9) 0.060849 -0.587785 1.000000 0.600000 0.900000 EOF cat < ${EXPECT_FILE} -1490500399.776379108-1.490500e+09(0) -10.000000 -10.000000 -10.000000 -10.000000 -1490500399.876379108-1.490500e+09(1) -4.122150 -4.122150 -4.122150 -4.122150 -1490500399.976379108-1.490500e+09(2) -0.489430 -0.489430 -0.489430 -0.489430 -1490500400.076379108-1.490500e+09(3) -0.489430 -0.489430 -0.489430 -0.489430 -1490500400.176379108-1.490500e+09(4) -4.122150 -4.122150 -4.122150 -4.122150 -1490500400.276379108-1.490500e+09(5) -10.000000 -10.000000 -10.000000 -10.000000 -1490500400.376379108-1.490500e+09(6) -15.877850 -15.877850 -15.877850 -15.877850 -1490500400.476379108-1.490500e+09(7) -19.510570 -19.510570 -19.510570 -19.510570 -1490500400.576379108-1.490500e+09(8) -19.510570 -19.510570 -19.510570 -19.510570 -1490500400.676379108-1.490500e+09(9) -15.877850 -15.877850 -15.877850 -15.877850 +# seconds.nanoseconds+offset(sequence) signal0 signal1 signal2 signal3 signal4 +1551015508.801653200(0) 0.022245 0.000000 -1.000000 1.000000 55.000000 +1551015508.901653200(1) 0.015339 0.587785 -1.000000 0.600000 65.000000 +1551015509.001653200(2) 0.027500 0.951057 -1.000000 0.200000 75.000000 +1551015509.101653200(3) 0.040320 0.951057 -1.000000 -0.200000 85.000000 +1551015509.201653200(4) 0.026079 0.587785 -1.000000 -0.600000 95.000000 +1551015509.301653200(5) 0.049262 0.000000 1.000000 -1.000000 105.000000 +1551015509.401653200(6) 0.014883 -0.587785 1.000000 -0.600000 115.000000 +1551015509.501653200(7) 0.023232 -0.951057 1.000000 -0.200000 125.000000 +1551015509.601653200(8) 0.015231 -0.951057 1.000000 0.200000 135.000000 +1551015509.701653200(9) 0.060849 -0.587785 1.000000 0.600000 145.000000 EOF -villas-hook -o scale=10 -o offset=-10 scale < ${INPUT_FILE} > ${OUTPUT_FILE} +villas-hook scale -o scale=100 -o offset=55 -o signal=signal4 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} - RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} From 80605a9455d57bb1222dd75d626d742093780bbc Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:33:23 +0100 Subject: [PATCH 37/38] tests: fix mapping unit test --- tests/unit/mapping.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/mapping.cpp b/tests/unit/mapping.cpp index 37c92bda0..68995eb9d 100644 --- a/tests/unit/mapping.cpp +++ b/tests/unit/mapping.cpp @@ -88,7 +88,7 @@ Test(mapping, parse_nodes) cr_assert_eq(m.node, vlist_lookup(&nodes, "carrot")); cr_assert_eq(m.type, MAPPING_TYPE_DATA); cr_assert_eq(m.data.offset, 0); - cr_assert_eq(m.length, vlist_length(&m.node->in.signals)); + cr_assert_eq(m.length, -1); ret = mapping_parse_str(&m, "carrot.data[sole]", &nodes); cr_assert_eq(ret, 0); @@ -151,13 +151,13 @@ Test(mapping, parse) cr_assert_eq(ret, 0); cr_assert_eq(m.type, MAPPING_TYPE_DATA); cr_assert_eq(m.data.offset, 0); - cr_assert_eq(m.length, 0); + cr_assert_eq(m.length, -1); ret = mapping_parse_str(&m, "data[]", nullptr); cr_assert_eq(ret, 0); cr_assert_eq(m.type, MAPPING_TYPE_DATA); cr_assert_eq(m.data.offset, 0); - cr_assert_eq(m.length, 0); + cr_assert_eq(m.length, -1); ret = mapping_parse_str(&m, "data[1.1-2f]", nullptr); cr_assert_neq(ret, 0); @@ -182,5 +182,5 @@ Test(mapping, parse) /* Negative length of chunk */ ret = mapping_parse_str(&m, "data[5-3]", nullptr); - cr_assert_eq(ret, -1); + cr_assert_neq(ret, 0); } From 6a2829aa4113d4de6a22beee19a1ce42977a3b65 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 9 Mar 2019 00:33:44 +0100 Subject: [PATCH 38/38] update VILLAScommon submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index 3a4beccee..4c9231b18 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 3a4beccee89f42a85b175b5557205d9ab756bfb5 +Subproject commit 4c9231b18c7f1025f7c56d3b7b7ac0d805b62363