diff --git a/.dockerignore b/.dockerignore index 0f1a95af5..024325812 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,3 @@ * !build/release/packaging/rpm/* -!thirdparty/libxil/ -!thirdparty/criterion/ -!thirdparty/libwebsockets/ -!thirdparty/nanomsg/ -!thirdparty/libzmq/ +!thirdparty/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7f1aabf66..f91392254 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /build/ *~ +.clang_complete diff --git a/Dockerfile.dev b/Dockerfile.dev index 6c004d252..b5abc8616 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -18,12 +18,12 @@ # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program. If not, see . ################################################################################### @@ -47,7 +47,7 @@ RUN dnf -y install \ jq \ iproute \ python-pip \ - valgrind gdb \ + valgrind gdb gdb-gdbserver \ xmlto rubygem-asciidoctor # 32bit versions of some standard libraries for RT-LAB code @@ -70,6 +70,7 @@ RUN dnf -y install \ libnl3-devel \ libcurl-devel \ jansson-devel \ + hdf5-devel \ libwebsockets-devel \ zeromq-devel \ nanomsg-devel \ @@ -88,6 +89,5 @@ EXPOSE 443 ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64 -ENTRYPOINT villas WORKDIR /villas ENTRYPOINT bash diff --git a/Dockerfile.dev-centos b/Dockerfile.dev-centos new file mode 100644 index 000000000..ee3e20a0e --- /dev/null +++ b/Dockerfile.dev-centos @@ -0,0 +1,106 @@ +# Dockerfile for VILLASnode development. +# +# This Dockerfile builds an image which contains all library dependencies +# and tools to build VILLASnode. +# However, VILLASnode itself it not part of the image. +# +# This image can be used for developing VILLASnode +# by running: +# make docker +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################### + +FROM centos:7 +MAINTAINER Steffen Vogel + +# Some of the dependencies are only available in our own repo +ADD https://villas.fein-aachen.org/packages/villas.repo /etc/yum.repos.d/ + +# Enable Extra Packages for Enterprise Linux (EPEL) and Software collection repo +RUN yum -y install epel-release centos-release-scl + +# Toolchain +RUN yum -y install \ + devtoolset-6-toolchain \ + gcc gcc-c++ \ + pkgconfig make cmake \ + autoconf automake autogen libtool \ + flex bison \ + texinfo git curl tar + +# Several tools only needed for developement and testing +RUN yum -y install \ + doxygen dia graphviz \ + openssh-clients \ + rpmdevtools rpm-build \ + jq \ + iproute \ + python-pip \ + valgrind gdb gdb-gdbserver \ + xmlto rubygem-asciidoctor + +# 32bit versions of some standard libraries for RT-LAB code +RUN yum -y install \ + libstdc++-devel.i686 \ + libuuid-devel.i686 \ + glibc-devel.i686 + +# Tools for debugging, coverage, profiling +RUN pip install \ + gcovr + +# Dependencies +RUN yum -y install \ + openssl openssl-devel \ + libconfig-devel \ + libnl3-devel \ + hdf5-devel \ + zeromq-devel \ + nanomsg-devel \ + libxil-devel + +# Build & Install Criterion +COPY thirdparty/criterion /tmp/criterion +RUN cd /tmp/criterion && cmake . && make install && rm -rf /tmp/* + +# Build & Install Jansson +COPY thirdparty/jansson /tmp/jansson +RUN cd /tmp/jansson && cmake -DJANSSON_BUILD_DOCS=OFF . && make install && rm -rf /tmp/* + +# Build & Install libwebsockets +COPY thirdparty/libwebsockets /tmp/libwebsockets +RUN cd /tmp/libwebsockets && cmake -DLWS_IPV6=1 -DLWS_WITH_STATIC=0 -DLWS_WITHOUT_TESTAPPS=1 -DLWS_WITH_HTTP2=1 . && make install && rm -rf /tmp/* + +# Build & Install libcurl +COPY thirdparty/libcurl /tmp/libcurl +RUN cd /tmp/libcurl/ && ./buildconf && ./configure && make install && rm -rf /tmp/* + +# Workaround for libnl3's search path for netem distributions +RUN ln -s /usr/lib64/tc /usr/lib/tc + +# Expose ports for HTTP and WebSocket frontend +EXPOSE 80 +EXPOSE 443 + +ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64 + +WORKDIR /villas +ENTRYPOINT scl enable devtoolset-6 bash diff --git a/Dockerfile.dev-ubuntu b/Dockerfile.dev-ubuntu index 461653afa..3e9363490 100644 --- a/Dockerfile.dev-ubuntu +++ b/Dockerfile.dev-ubuntu @@ -18,12 +18,12 @@ # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program. If not, see . ################################################################################### @@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y \ jq \ iproute \ python-pip \ - valgrind gdb \ + valgrind gdb gdbserver \ xmlto asciidoctor # 32bit versions of some standard libraries for RT-LAB code @@ -74,6 +74,7 @@ RUN apt-get update && apt-get install -y \ libnl-3-dev libnl-route-3-dev \ libcurl4-openssl-dev \ libjansson-dev \ + libhdf5-dev \ libzmq3-dev \ libnanomsg-dev @@ -98,6 +99,5 @@ ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64 # Workaround for libnl3's search path for netem distributions RUN ln -s /usr/lib64/tc /usr/lib/tc -ENTRYPOINT villas WORKDIR /villas ENTRYPOINT bash diff --git a/Makefile b/Makefile index 07dd7ff9b..394002113 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ V ?= 2 # Platform PLATFORM ?= $(shell uname) +# Enable supprot for libconfig configuration files +WITH_CONFIG ?= 1 + # Common flags LDLIBS = CFLAGS += -I. -Iinclude -Iinclude/villas @@ -116,7 +119,11 @@ else endif # pkg-config dependencies -PKGS = libconfig openssl +PKGS = openssl jansson + +ifeq ($(WITH_CONFIG),1) + PKGS += libconfig +endif ######## Targets ######## @@ -148,7 +155,8 @@ CFLAGS += $(addprefix -DWITH_, $(call escape,$(PKGS))) install: $(addprefix install-,$(filter-out thirdparty doc clients,$(MODULES))) clean: $(addprefix clean-, $(filter-out thirdparty doc clients,$(MODULES))) -.PHONY: all everything clean install FORCE +.PHONY: all everything clean install +-include $(wildcard $(SRCDIR)/Makefile.*) -include $(wildcard $(BUILDDIR)/**/*.d) --include $(addsuffix /Makefile.inc,$(MODULES)) +-include $(patsubst %,$(SRCDIR)/%/Makefile.inc,$(MODULES)) diff --git a/Makefile.complete b/Makefile.complete new file mode 100644 index 000000000..47676e496 --- /dev/null +++ b/Makefile.complete @@ -0,0 +1,49 @@ +# Makefile for clang autocompletion +# +# This Makefile produces .clang_complete files containing compiler flags +# which are used by clang autocompletion tools such as: +# +# - https://atom.io/packages/autocomplete-clang +# - https://github.com/Rip-Rip/clang_complete +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################### + +SRCMODULES = src lib plugins tools tests/unit + +CLANG_COMPLETES = $(patsubst %,$(SRCDIR)/%/.clang_complete,$(SRCMODULES)) + +tests/unit/.clang_complete: FLAGS = $(TEST_CFLAGS) +src/.clang_complete: FLAGS = $(SRC_CFLAGS) +tools/.clang_complete: FLAGS = $(TOOLS_CFLAGS) +plugins/.clang_complete: FLAGS = $(PLUGIN_CFLAGS) +lib/.clang_complete: FLAGS = $(LIB_CFLAGS) + +%/.clang_complete: + echo "$(FLAGS)" > $@ + +clang-complete: $(CLANG_COMPLETES) + +clean: clean-clang-complete + +clean-clang-complete: + rm -f $(CLANG_COMPLETES) + +.PHONY: clang-complete diff --git a/etc/Makefile.inc b/etc/Makefile.inc index 25044f61e..f12aa55cc 100644 --- a/etc/Makefile.inc +++ b/etc/Makefile.inc @@ -22,8 +22,8 @@ etc: -install-etc: - install -D -t $(DESTDIR)/etc/villas/node etc/*.conf +install-etc: | $(DESTDIR)/etc/villas/node/ + install -D -t $(DESTDIR)/etc/villas/node $(SRCDIR)/etc/*.conf clean-etc: diff --git a/etc/example.conf b/etc/example.conf index 944b7d408..78e1a546d 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -193,7 +193,7 @@ nodes = { signal_node = { type = "signal", - signal = "sine", # One of "sine", "ramp", "triangle", "random", "mixed" + signal = "sine", # One of "sine", "square", "ramp", "triangle", "random", "mixed" values = 4, # Number of values per sample amplitude = 2.3, # Amplitude of generated signals frequency = 10, # Frequency of generated signals @@ -204,6 +204,7 @@ nodes = { type = "loopback", # A loopback node will receive exactly the same data which has been sent to it. # The internal implementation is based on queue. queuelen = 1024 # The queue length of the internal queue which buffers the samples. + samplelen = 64 # Each buffered sample can contain up to 64 values. } }; diff --git a/etc/fpga-simple.conf b/etc/fpga-simple.conf index 7143d9c56..b2e22ed9f 100644 --- a/etc/fpga-simple.conf +++ b/etc/fpga-simple.conf @@ -44,6 +44,11 @@ fpgas = { vlnv = "xilinx.com:ip:axis_interconnect:2.1" baseaddr = 0x0000; numports = 3; + + paths = ( + { in = "dma_0", out = "rtds_0" }, + { in = "rtds_0", out = "dma_0" } + ) }, rtds_0 = { vlnv = "acs.eonerc.rwth-aachen.de:user:rtds_axis:1.0" @@ -57,12 +62,6 @@ fpgas = { irq = 0 } } - - /* Configure switch_0 */ - paths = ( - { in = "dma_0", out = "rtds_0" }, - { in = "rtds_0", out = "dma_0" } - ) } } diff --git a/etc/fpga.conf b/etc/fpga.conf index df9745ac2..a0f54f06b 100644 --- a/etc/fpga.conf +++ b/etc/fpga.conf @@ -56,6 +56,18 @@ fpgas = { vlnv = "xilinx.com:ip:axis_interconnect:2.1" baseaddr = 0x5000; num_ports = 10; + + paths = ( + // { in = "fifo_mm_s_0", out = "fifo_mm_s_0" }, # Loopback fifo_mm_s_0 + // { in = "dma_0", out = "dma_0" }, # Loopback dma_0 + // { in = "dma_1", out = "dma_1" } # Loopback dma_1 + // { in = "rtds_axis_0", out = "fifo_mm_s_0", reverse = true } # Linux <-> RTDS + // { in = "rtds_axis_0", out = "dma_0", reverse = true } # Linux (dma_0) <-> RTDS + { in = "rtds_axis_0", out = "dma_1", reverse = true } # Linux (dma_1) <-> RTDS + // { in = "rtds_axis_0", out = "fifo_mm_s_0", reverse = true } # Linux (fifo_mm_s_0) <-> RTDS + // { in = "dma_0", out = "hls_dft_0", reverse = true } # DFT <-> Linux + // { in = "rtds_axis_0", out = "hls_dft_0", reverse = true }, # DFT <-> RTDS + ) }, axi_reset_0 = { vlnv = "xilinx.com:ip:axi_gpio:2.0"; @@ -117,21 +129,6 @@ fpgas = { port = 6; }, } - - ############ Switch config ############ - # Requires a single IP core with VLNV: - # xilinx.com:ip:axis_interconnect - paths = ( - // { in = "fifo_mm_s_0", out = "fifo_mm_s_0" }, # Loopback fifo_mm_s_0 - // { in = "dma_0", out = "dma_0" }, # Loopback dma_0 - // { in = "dma_1", out = "dma_1" } # Loopback dma_1 - // { in = "rtds_axis_0", out = "fifo_mm_s_0", reverse = true } # Linux <-> RTDS - // { in = "rtds_axis_0", out = "dma_0", reverse = true } # Linux (dma_0) <-> RTDS - { in = "rtds_axis_0", out = "dma_1", reverse = true } # Linux (dma_1) <-> RTDS - // { in = "rtds_axis_0", out = "fifo_mm_s_0", reverse = true } # Linux (fifo_mm_s_0) <-> RTDS - // { in = "dma_0", out = "hls_dft_0", reverse = true } # DFT <-> Linux - // { in = "rtds_axis_0", out = "hls_dft_0", reverse = true }, # DFT <-> RTDS - ) } } diff --git a/etc/js/config.js b/etc/js/config.js new file mode 100644 index 000000000..454aa6bcc --- /dev/null +++ b/etc/js/config.js @@ -0,0 +1,88 @@ +/** Example Javascript config + * + * This example demonstrates how you can use Javascript code and NodeJS + * to script configuration files. + * + * To use this configuration, run the following command: + * + * villas node <(node /etc/villas/node/js/config.js) + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +var glob = require('glob'); +var global = require(__dirname + '/global.js') + +/* List of plugins + * + * Additional node-types, hooks or VILLASfpga IP cores + * can be loaded by compiling them into a shared library and + * adding them to this list + */ +global.plugins = glob.sync('/usr?(/local)/share/villas/node/plugins/*.so'); + +global.nodes = { + loopback_node : { + vectorize : 1, + type : "loopback", // A loopback node will receive exactly the same data which has been sent to it. + // The internal implementation is based on queue. + queuelen : 10240 // The queue length of the internal queue which buffers the samples. + }, + socket_node : { + type : "socket", + + local : "*:12000", + remote : "127.0.0.1:12000" + } +}; + +global.paths = [ + { + in : "test_node", + out : "socket_node", + queuelen : 10000 + }, + { + in : "socket_node", + out : "test_node", + queuelen : 10000, + hooks : [ + { + type : "stats", + warmup : 100, + verbose : true, + format : "human", + output : "./stats.log" + }, + { + type : "convert" + } + ] + } +]; + +// Convert Javascript Object to JSON string +var json = JSON.stringify(global, null, 4); + +// Some log message +process.stderr.write('Configuration file successfully generated\n'); + +// Print JSON to stdout +process.stdout.write(json); \ No newline at end of file diff --git a/etc/js/global.js b/etc/js/global.js new file mode 100644 index 000000000..194e0b654 --- /dev/null +++ b/etc/js/global.js @@ -0,0 +1,65 @@ +/** Global configuration file for VILLASnode. + * + * The syntax of this file is similar to JSON. + * A detailed description of the format can be found here: + * http://www.hyperrealm.com/libconfig/libconfig_manual.html//Configuration-Files + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +os = require('os'); + +var logfile = "/var/log/villas-node_" + new Date().toISOString() + ".log" + +module.exports = { + affinity : 0x01, // Mask of cores the server should run on + // This also maps the NIC interrupts to those cores! + + priority : 50, // Priority for the server tasks. + // Usually the server is using a real-time FIFO + // scheduling algorithm + + // See: https://github.com/docker/docker/issues/22380 + // on why we cant use real-time scheduling in Docker + + stats : 3, // The interval in seconds to print path statistics. + // A value of 0 disables the statistics. + + name : os.hostname(), // The name of this VILLASnode. Might by used by node-types + // to identify themselves (default is the hostname). + + log : { + level : 5, // The level of verbosity for debug messages + // Higher number => increased verbosity + + faciltities : [ "path", "socket" ], // The list of enabled debug faciltities. + // If omitted, all faciltities are enabled + // For a full list of available faciltities, check lib/log.c + + file : logfile, // File for logs + }, + + http : { + enabled : true, // Do not listen on port if true + + htdocs : "/villas/web/socket/", // Root directory of internal webserver + port : 80 // Port for HTTP connections + } +}; \ No newline at end of file diff --git a/include/villas/advio.h b/include/villas/advio.h index b8b734d8f..a1741a9f9 100644 --- a/include/villas/advio.h +++ b/include/villas/advio.h @@ -47,6 +47,7 @@ typedef struct advio AFILE; /* The remaining functions from stdio are just replaced macros */ #define afeof(af) feof((af)->file) +#define afgets(ln, sz, af) fgets(ln, sz, (af)->file) #define aftell(af) ftell((af)->file) #define afileno(af) fileno((af)->file) #define afread(ptr, sz, nitems, af) fread(ptr, sz, nitems, (af)->file) @@ -60,6 +61,9 @@ typedef struct advio AFILE; #define auri(af) ((af)->uri) #define ahash(af) ((af)->hash) +/** Check if a URI is pointing to a local file. */ +int aislocal(const char *uri); + AFILE *afopen(const char *url, const char *mode); int afclose(AFILE *file); @@ -71,7 +75,6 @@ int afseek(AFILE *file, long offset, int origin); void arewind(AFILE *file); /** Download contens from remote file - * * * @param resume Do a partial download and append to the local file */ diff --git a/include/villas/api.h b/include/villas/api.h index d31d94c18..b56627306 100644 --- a/include/villas/api.h +++ b/include/villas/api.h @@ -25,9 +25,11 @@ #include #include +#include #include "list.h" #include "common.h" +#include "queue_signalled.h" #include "api/session.h" @@ -48,10 +50,13 @@ struct api_action; typedef int (*api_cb_t)(struct api_action *c, json_t *args, json_t **resp, struct api_session *s); struct api { - struct list sessions; /**< List of currently active connections */ - enum state state; + struct list sessions; /**< List of currently active connections */ + struct queue_signalled pending; /**< A queue of api_sessions which have pending requests. */ + + pthread_t thread; + struct super_node *super_node; }; diff --git a/include/villas/api/session.h b/include/villas/api/session.h index ccf69b1f7..244d8f732 100644 --- a/include/villas/api/session.h +++ b/include/villas/api/session.h @@ -27,7 +27,8 @@ #include #include "common.h" -#include "web/buffer.h" +#include "queue.h" +#include "buffer.h" enum api_version { API_VERSION_UNKOWN = 0, @@ -41,34 +42,28 @@ enum api_mode { /** A connection via HTTP REST or WebSockets to issue API actions. */ struct api_session { - enum api_mode mode; + enum state state; + enum api_version version; + enum api_mode mode; int runs; struct { - struct web_buffer body; /**< HTTP body / WS payload */ - } request; - - struct { - struct web_buffer body; /**< HTTP body / WS payload */ - struct web_buffer headers; /**< HTTP headers */ - } response; - - struct { - char name[64]; - char ip[64]; - } peer; - - bool completed; /**< Did we receive the complete body yet? */ - - enum state state; + struct buffer buffer; + struct queue queue; + } request, response; + struct lws *wsi; struct api *api; + + char *_name; }; -int api_session_init(struct api_session *s, struct api *a, enum api_mode m); +int api_session_init(struct api_session *s, enum api_mode m); int api_session_destroy(struct api_session *s); int api_session_run_command(struct api_session *s, json_t *req, json_t **resp); + +char * api_session_name(struct api_session *s); diff --git a/include/villas/buffer.h b/include/villas/buffer.h new file mode 100644 index 000000000..8899f5fb0 --- /dev/null +++ b/include/villas/buffer.h @@ -0,0 +1,50 @@ +/** A simple growing buffer. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +#include "common.h" + +struct buffer { + enum state state; + + char *buf; + size_t len; + size_t size; +}; + +int buffer_init(struct buffer *b, size_t size); + +int buffer_destroy(struct buffer *b); + +void buffer_clear(struct buffer *b); + +int buffer_append(struct buffer *b, const char *data, size_t len); + +int buffer_parse_json(struct buffer *b, json_t **j); + +int buffer_append_json(struct buffer *b, json_t *j); \ No newline at end of file diff --git a/include/villas/compat.h b/include/villas/compat.h index 9d8377eb5..ad574fda2 100644 --- a/include/villas/compat.h +++ b/include/villas/compat.h @@ -31,11 +31,15 @@ size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags); #define le16toh(x) OSSwapLittleToHostInt16(x) #define le32toh(x) OSSwapLittleToHostInt32(x) + #define le64toh(x) OSSwapLittleToHostInt64(x) #define be16toh(x) OSSwapBigToHostInt16(x) #define be32toh(x) OSSwapBigToHostInt32(x) + #define be64toh(x) OSSwapBigToHostInt64(x) #define htole16(x) OSSwapHostToLittleInt16(x) #define htole32(x) OSSwapHostToLittleInt32(x) + #define htole64(x) OSSwapHostToLittleInt64(x) #define htobe16(x) OSSwapHostToBigInt16(x) #define htobe32(x) OSSwapHostToBigInt32(x) + #define htobe64(x) OSSwapHostToBigInt64(x) #endif /* __MACH__ */ diff --git a/include/villas/config_helper.h b/include/villas/config_helper.h index 423ac7dcb..6be9afe4b 100644 --- a/include/villas/config_helper.h +++ b/include/villas/config_helper.h @@ -33,5 +33,12 @@ json_t * config_to_json(config_setting_t *cfg); /* Convert a jansson object into a libconfig object. */ int json_to_config(json_t *json, config_setting_t *parent); -/* Create a libconfig object from command line parameters. */ -int config_read_cli(config_t *cfg, int argc, char *argv[]); +/* Create a JSON object from command line parameters. */ +json_t * json_load_cli(int argc, char *argv[]); + +int json_object_extend_str(json_t *orig, const char *str); + +void json_object_extend_key_value(json_t *obj, const char *key, const char *value); + +/* Merge two JSON objects recursively. */ +int json_object_extend(json_t *orig, json_t *merge); diff --git a/include/villas/fpga/card.h b/include/villas/fpga/card.h index ba2b3ce3f..5ca255501 100644 --- a/include/villas/fpga/card.h +++ b/include/villas/fpga/card.h @@ -29,8 +29,6 @@ #pragma once -#include - #include "common.h" #include "kernel/pci.h" #include "kernel/vfio.h" @@ -64,17 +62,15 @@ struct fpga_card { struct fpga_ip *intc; struct fpga_ip *reset; struct fpga_ip *sw; - - config_setting_t *cfg; }; /** Initialize FPGA card and its IP components. */ int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc); /** Parse configuration of FPGA card including IP cores from config. */ -int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg); +int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name); -int fpga_card_parse_list(struct list *l, config_setting_t *cfg); +int fpga_card_parse_list(struct list *l, json_t *cfg); /** Check if the FPGA card configuration is plausible. */ int fpga_card_check(struct fpga_card *c); diff --git a/include/villas/fpga/ip.h b/include/villas/fpga/ip.h index 64994a391..918173114 100644 --- a/include/villas/fpga/ip.h +++ b/include/villas/fpga/ip.h @@ -30,7 +30,6 @@ #pragma once #include -#include #include "common.h" @@ -60,7 +59,7 @@ struct fpga_ip_type { enum fpga_ip_types type; int (*init)(struct fpga_ip *c); - int (*parse)(struct fpga_ip *c); + int (*parse)(struct fpga_ip *c, json_t *cfg); int (*check)(struct fpga_ip *c); int (*start)(struct fpga_ip *c); int (*stop)(struct fpga_ip *c); @@ -87,15 +86,13 @@ struct fpga_ip { int irq; /**< The interrupt number of the FPGA IP component. */ struct fpga_card *card; /**< The FPGA to which this IP instance belongs to. */ - - config_setting_t *cfg; }; /** Initialize IP core. */ int fpga_ip_init(struct fpga_ip *c, struct fpga_ip_type *vt); /** Parse IP core configuration from configuration file */ -int fpga_ip_parse(struct fpga_ip *c, config_setting_t *cfg); +int fpga_ip_parse(struct fpga_ip *c, json_t *cfg, const char *name); /** Check configuration of IP core. */ int fpga_ip_check(struct fpga_ip *c); diff --git a/include/villas/fpga/ips/dft.h b/include/villas/fpga/ips/dft.h index 046082f4f..03c08dbdf 100644 --- a/include/villas/fpga/ips/dft.h +++ b/include/villas/fpga/ips/dft.h @@ -25,7 +25,7 @@ struct dft { int decimation; }; -int dft_parse(struct fpga_ip *c); +int dft_parse(struct fpga_ip *c, json_t *cfg); int dft_start(struct fpga_ip *c); diff --git a/include/villas/fpga/ips/model.h b/include/villas/fpga/ips/model.h index 63b91af21..dadefe4fd 100644 --- a/include/villas/fpga/ips/model.h +++ b/include/villas/fpga/ips/model.h @@ -33,20 +33,20 @@ enum model_xsg_block_type { XSG_BLOCK_INFO = 0x2000 }; -enum model_param_type { - MODEL_PARAM_TYPE_UFIX, - MODEL_PARAM_TYPE_FIX, - MODEL_PARAM_TYPE_FLOAT, - MODEL_PARAM_TYPE_BOOLEAN +enum model_parameter_type { + MODEL_PARAMETER_TYPE_UFIX, + MODEL_PARAMETER_TYPE_FIX, + MODEL_PARAMETER_TYPE_FLOAT, + MODEL_PARAMETER_TYPE_BOOLEAN }; -enum model_param_direction { - MODEL_PARAM_IN, - MODEL_PARAM_OUT, - MODEL_PARAM_INOUT +enum model_parameter_direction { + MODEL_PARAMETER_IN, + MODEL_PARAMETER_OUT, + MODEL_PARAMETER_INOUT }; -union model_param_value { +union model_parameter_value { uint32_t ufix; int32_t fix; float flt; @@ -75,16 +75,16 @@ struct model_info { char *value; }; -struct model_param { +struct model_parameter { char *name; /**< Name of the parameter */ - enum model_param_direction direction; /**< Read / Write / Read-write? */ - enum model_param_type type; /**< Data type. Integers are represented by MODEL_GW_TYPE_(U)FIX with model_gw::binpt == 0 */ + enum model_parameter_direction direction; /**< Read / Write / Read-write? */ + enum model_parameter_type type; /**< Data type. Integers are represented by MODEL_GW_TYPE_(U)FIX with model_gw::binpt == 0 */ int binpt; /**< Binary point for type == MODEL_GW_TYPE_(U)FIX */ uintptr_t offset; /**< Register offset to model::baseaddress */ - union model_param_value default_value; + union model_parameter_value default_value; struct fpga_ip *ip; /**< A pointer to the model structure to which this parameters belongs to. */ }; @@ -93,7 +93,7 @@ struct model_param { int model_init(struct fpga_ip *c); /** Parse model */ -int model_parse(struct fpga_ip *c); +int model_parse(struct fpga_ip *c, json_t *cfg); /** Destroy a model */ int model_destroy(struct fpga_ip *c); @@ -102,17 +102,17 @@ int model_destroy(struct fpga_ip *c); void model_dump(struct fpga_ip *c); /** Add a new parameter to the model */ -void model_param_add(struct fpga_ip *c, const char *name, enum model_param_direction dir, enum model_param_type type); +void model_parameter_add(struct fpga_ip *c, const char *name, enum model_parameter_direction dir, enum model_parameter_type type); /** Remove an existing parameter by its name */ -int model_param_remove(struct fpga_ip *c, const char *name); +int model_parameter_remove(struct fpga_ip *c, const char *name); /** Read a model parameter. * * Note: the data type of the register is taken into account. * All datatypes are converted to double. */ -int model_param_read(struct model_param *p, double *v); +int model_parameter_read(struct model_parameter *p, double *v); /** Update a model parameter. * @@ -120,8 +120,8 @@ int model_param_read(struct model_param *p, double *v); * The double argument will be converted to the respective data type of the * GatewayIn/Out block. */ -int model_param_write(struct model_param *p, double v); +int model_parameter_write(struct model_parameter *p, double v); -int model_param_update(struct model_param *p, struct model_param *u); +int model_parameter_update(struct model_parameter *p, struct model_parameter *u); /** @} */ diff --git a/include/villas/fpga/ips/switch.h b/include/villas/fpga/ips/switch.h index 9c46c2dcb..19b8b5417 100644 --- a/include/villas/fpga/ips/switch.h +++ b/include/villas/fpga/ips/switch.h @@ -13,6 +13,7 @@ #pragma once +#include #include #include "list.h" @@ -41,7 +42,7 @@ int switch_init_paths(struct fpga_ip *c); int switch_destroy(struct fpga_ip *c); -int switch_parse(struct fpga_ip *c); +int switch_parse(struct fpga_ip *c, json_t *cfg); int switch_connect(struct fpga_ip *c, struct fpga_ip *mi, struct fpga_ip *si); diff --git a/include/villas/hist.h b/include/villas/hist.h index b3fcd3a6e..567472dac 100644 --- a/include/villas/hist.h +++ b/include/villas/hist.h @@ -93,12 +93,8 @@ char * hist_dump(struct hist *h); /** Prints Matlab struct containing all infos to file. */ int hist_dump_matlab(struct hist *h, FILE *f); -#ifdef WITH_JSON - /** Write the histogram in JSON format to fiel \p f. */ int hist_dump_json(struct hist *h, FILE *f); /** Build a libjansson / JSON object of the histogram. */ json_t * hist_json(struct hist *h); - -#endif /* WITH_JSON */ diff --git a/include/villas/hook.h b/include/villas/hook.h index b73a0c49a..05606064c 100644 --- a/include/villas/hook.h +++ b/include/villas/hook.h @@ -52,21 +52,13 @@ struct hook { void *_vd; /**< Private data for this hook. This pointer can be used to pass data between consecutive calls of the callback. */ int priority; /**< A priority to change the order of execution within one type of hook. */ + + json_t *cfg; /**< A JSON object containing the configuration of the hook. */ }; -/** Save references to global nodes, paths and settings */ int hook_init(struct hook *h, struct hook_type *vt, struct path *p); -/** Parse a single hook. - * - * A hook definition is composed of the hook name and optional parameters - * seperated by a colon. - * - * Examples: - * "print:stdout" - */ -int hook_parse(struct hook *h, config_setting_t *cfg); - +int hook_parse(struct hook *h, json_t *cfg); int hook_parse_cli(struct hook *h, int argc, char *argv[]); int hook_destroy(struct hook *h); @@ -77,11 +69,11 @@ int hook_stop(struct hook *h); int hook_periodic(struct hook *h); int hook_restart(struct hook *h); -int hook_read(struct hook *h, struct sample *smps[], size_t *cnt); -int hook_write(struct hook *h, struct sample *smps[], size_t *cnt); +int hook_read(struct hook *h, struct sample *smps[], unsigned *cnt); +int hook_write(struct hook *h, struct sample *smps[], unsigned *cnt); -size_t hook_read_list(struct list *hs, struct sample *smps[], size_t cnt); -size_t hook_write_list(struct list *hs, struct sample *smps[], size_t cnt); +int hook_read_list(struct list *hs, struct sample *smps[], unsigned cnt); +int hook_write_list(struct list *hs, struct sample *smps[], unsigned cnt); /** Compare two hook functions with their priority. Used by list_sort() */ int hook_cmp_priority(const void *a, const void *b); @@ -100,4 +92,4 @@ int hook_cmp_priority(const void *a, const void *b); * hooks = [ "print" ] * } */ -int hook_parse_list(struct list *list, config_setting_t *cfg, struct path *p); +int hook_parse_list(struct list *list, json_t *cfg, struct path *p); diff --git a/include/villas/hook_type.h b/include/villas/hook_type.h index a6c45ece1..edebde0a5 100644 --- a/include/villas/hook_type.h +++ b/include/villas/hook_type.h @@ -37,7 +37,7 @@ #include #include -#include +#include /* Forward declarations */ struct hook; @@ -49,7 +49,7 @@ struct hook_type { size_t size; /**< Size of allocation for struct hook::_vd */ - int (*parse)(struct hook *h, config_setting_t *cfg); + int (*parse)(struct hook *h, json_t *cfg); int (*parse_cli)(struct hook *h, int argc, char *argv[]); int (*init)(struct hook *h); /**< Called before path is started to parseHOOK_DESTROYs. */ @@ -61,6 +61,6 @@ struct hook_type { int (*periodic)(struct hook *h);/**< Called periodically. Period is set by global 'stats' option in the configuration file. */ int (*restart)(struct hook *h); /**< Called whenever a new simulation case is started. This is detected by a sequence no equal to zero. */ - int (*read)(struct hook *h, struct sample *smps[], size_t *cnt); /**< Called for every single received samples. */ - int (*write)(struct hook *h, struct sample *smps[], size_t *cnt); /**< Called for every single sample which will be sent. */ + int (*read)(struct hook *h, struct sample *smps[], unsigned *cnt); /**< Called for every single received samples. */ + int (*write)(struct hook *h, struct sample *smps[], unsigned *cnt); /**< Called for every single sample which will be sent. */ }; diff --git a/include/villas/io.h b/include/villas/io.h new file mode 100644 index 000000000..969daca15 --- /dev/null +++ b/include/villas/io.h @@ -0,0 +1,98 @@ +/** Read / write sample data in different formats. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include "advio.h" +#include "common.h" + +/* Forward declarations */ +struct sample; +struct io_format; + +enum io_flags { + IO_FLUSH = (1 << 8) /**< Flush the output stream after each chunk of samples. */ +}; + +struct io { + enum state state; + int flags; + + enum { + IO_MODE_STDIO, + IO_MODE_ADVIO, + IO_MODE_CUSTOM + } mode; + + /** A format type can use this file handle or overwrite the + * format::{open,close,eof,rewind} functions and the private + * data in io::_vd. + */ + union { + struct { + FILE *input; + FILE *output; + } stdio; + struct { + AFILE *input; + AFILE *output; + } advio; + }; + + struct { + char *input; + char *output; + } buffer; + + void *_vd; + struct io_format *_vt; +}; + +int io_init(struct io *io, struct io_format *fmt, int flags); + +int io_destroy(struct io *io); + +int io_open(struct io *io, const char *uri); + +int io_close(struct io *io); + +int io_print(struct io *io, struct sample *smps[], unsigned cnt); + +int io_scan(struct io *io, struct sample *smps[], unsigned cnt); + +int io_eof(struct io *io); + +void io_rewind(struct io *io); + +int io_flush(struct io *io); + + +int io_stream_open(struct io *io, const char *uri); + +int io_stream_close(struct io *io); + +int io_stream_eof(struct io *io); + +void io_stream_rewind(struct io *io); + +int io_stream_flush(struct io *io); diff --git a/include/villas/webmsg.h b/include/villas/io/csv.h similarity index 61% rename from include/villas/webmsg.h rename to include/villas/io/csv.h index c5800eb75..993305664 100644 --- a/include/villas/webmsg.h +++ b/include/villas/io/csv.h @@ -1,4 +1,4 @@ -/** Message related functions +/** Comma-separated values. * * @file * @author Steffen Vogel @@ -23,29 +23,13 @@ #pragma once +#include + /* Forward declarations. */ -struct webmsg; +struct sample; -/** Swaps the byte-order of the message. - * - * Message are always transmitted in network (big endian) byte order. - * - * @param m A pointer to the message - */ -void webmsg_hdr_ntoh(struct webmsg *m); +#define CSV_SEPARATOR '\t' -void webmsg_hdr_hton(struct webmsg *m); +int csv_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags); -void webmsg_ntoh(struct webmsg *m); - -void webmsg_hton(struct webmsg *m); - -/** Check the consistency of a message. - * - * The functions checks the header fields of a message. - * - * @param m A pointer to the message - * @retval 0 The message header is valid. - * @retval <0 The message header is invalid. - */ -int msg_verify(struct webmsg *m); +int csv_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags); diff --git a/include/villas/sample_io_json.h b/include/villas/io/json.h similarity index 78% rename from include/villas/sample_io_json.h rename to include/villas/io/json.h index 0f0e4e7ac..045a1201e 100644 --- a/include/villas/sample_io_json.h +++ b/include/villas/io/json.h @@ -26,10 +26,10 @@ #include "sample.h" -int sample_io_json_pack(json_t **j, struct sample *s, int flags); +int json_pack_sample(json_t **j, struct sample *s, int flags); -int sample_io_json_unpack(json_t *j, struct sample *s, int *flags); +int json_unpack_sample(json_t *j, struct sample *s, int *flags); -int sample_io_json_fprint(FILE *f, struct sample *s, int flags); +int json_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags); -int sample_io_json_fscan(FILE *f, struct sample *s, int *flags); +int json_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags); diff --git a/include/villas/msg.h b/include/villas/io/msg.h similarity index 85% rename from include/villas/msg.h rename to include/villas/io/msg.h index 194ffe1b3..9c520d06d 100644 --- a/include/villas/msg.h +++ b/include/villas/io/msg.h @@ -23,12 +23,16 @@ #pragma once +#include + /* Forward declarations. */ struct msg; struct sample; +struct io; -/** The maximum length of a packet which contains stuct msg. */ -#define MSG_MAX_PACKET_LEN 1500 +enum msg_flags { + MSG_WEB = (1 << 16) /**< Use webmsg format (everying little endian) */ +}; /** Swaps the byte-order of the message. * @@ -60,9 +64,8 @@ int msg_to_sample(struct msg *msg, struct sample *smp); /** Copy fields form \p smp into \p msg. */ int msg_from_sample(struct msg *msg, struct sample *smp); - /** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ -ssize_t msg_buffer_from_samples(struct sample *smps[], unsigned cnt, char *buf, size_t len); +int msg_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags); /** Read struct sample's from buffer \p buf into samples \p smps. */ -int msg_buffer_to_samples(struct sample *smps[], unsigned cnt, char *buf, size_t len); +int msg_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags); diff --git a/include/villas/msg_format.h b/include/villas/io/msg_format.h similarity index 100% rename from include/villas/msg_format.h rename to include/villas/io/msg_format.h diff --git a/include/villas/io/raw.h b/include/villas/io/raw.h new file mode 100644 index 000000000..797503fed --- /dev/null +++ b/include/villas/io/raw.h @@ -0,0 +1,60 @@ +/** RAW IO format + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +/* Forward declarations */ +struct sample; + +enum raw_flags { + RAW_FAKE = (1 << 16), /**< Treat the first three values as: sequenceno, seconds, nanoseconds */ + + RAW_BE_INT = (1 << 17), /**< Byte-order for integer data: big-endian if set. */ + RAW_BE_FLT = (1 << 18), /**< Byte-order for floating point data: big-endian if set. */ + RAW_BE_HDR = (1 << 19), /**< Byte-order for fake header fields: big-endian if set. */ + + /** Byte-order for all fields: big-endian if set. */ + RAW_BE = RAW_BE_INT | RAW_BE_FLT | RAW_BE_HDR, + + /** Mix floating and integer types. + * + * io_raw_sscan() parses all values as single / double precission fp. + * io_raw_sprint() uses sample::format to determine the type. + */ + RAW_AUTO = (1 << 22), + RAW_FLT = (1 << 23), /**< Data-type: floating point otherwise integer. */ + + //RAW_1 = (0 << 24), /**< Pack each value as a single bit. */ + RAW_8 = (3 << 24), /**< Pack each value as a byte. */ + RAW_16 = (4 << 24), /**< Pack each value as a word. */ + RAW_32 = (5 << 24), /**< Pack each value as a double word. */ + RAW_64 = (6 << 24) /**< Pack each value as a quad word. */ +}; + +/** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ +int raw_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags); + +/** Read struct sample's from buffer \p buf into samples \p smps. */ +int raw_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags); diff --git a/include/villas/io/villas.h b/include/villas/io/villas.h new file mode 100644 index 000000000..94aadc40d --- /dev/null +++ b/include/villas/io/villas.h @@ -0,0 +1,36 @@ +/** The VILLASframework sample format + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include "io.h" + +int villas_print(struct io *io, struct sample *smps[], unsigned cnt); + +int villas_scan(struct io *io, struct sample *smps[], unsigned cnt); + +int villas_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags); + +int villas_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags); diff --git a/include/villas/io_format.h b/include/villas/io_format.h new file mode 100644 index 000000000..d82bd5c17 --- /dev/null +++ b/include/villas/io_format.h @@ -0,0 +1,146 @@ +/** Read / write sample data in different formats. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +/* Forward declarations */ +struct sample; +struct io; + +enum io_format_flags { + IO_FORMAT_NANOSECONDS = (1 << 0), /**< Include nanoseconds in output. */ + IO_FORMAT_OFFSET = (1 << 1), /**< Include offset / delta between received and send timestamps. */ + IO_FORMAT_SEQUENCE = (1 << 2), /**< Include sequence number in output. */ + IO_FORMAT_VALUES = (1 << 3), /**< Include values in output. */ + IO_FORMAT_ALL = 15, /**< Enable all output options. */ + + IO_FORMAT_BINARY = (1 << 8) +}; + +struct io_format { + int (*init)(struct io *io); + int (*destroy)(struct io *io); + + /** @{ + * High-level interface + */ + + /** Open an IO stream. + * + * @see fopen() + */ + int (*open)(struct io *io, const char *uri); + + /** Close an IO stream. + * + * @see fclose() + */ + int (*close)(struct io *io); + + /** Check if end-of-file was reached. + * + * @see feof() + */ + int (*eof)(struct io *io); + + /** Rewind an IO stream. + * + * @see rewind() + */ + void (*rewind)(struct io *io); + + /** Flush buffered data to disk. + * + * @see fflush() + */ + int (*flush)(struct io *io); + + int (*print)(struct io *io, struct sample *smps[], unsigned cnt); + int (*scan)( struct io *io, struct sample *smps[], unsigned cnt); + /** @} */ + + /** @{ + * Low-level interface + */ + + /** @see io_format_sscan */ + int (*sscan)(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags); + + /** @see io_format_sprint */ + int (*sprint)(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags); + + /** @see io_format_fscan */ + int (*fscan)(FILE *f, struct sample *smps[], unsigned cnt, int *flags); + + /** @see io_format_fprint */ + int (*fprint)(FILE *f, struct sample *smps[], unsigned cnt, int flags); + + /** @} */ + + size_t size; /**< Number of bytes to allocate for io::_vd */ + int flags; /**< A set of flags which is automatically used. */ +}; + +struct io_format * io_format_lookup(const char *name); + +/** Parse samples from the buffer \p buf with a length of \p len bytes. + * + * @param buf[in] The buffer of data which should be parsed / de-serialized. + * @param len[in] The length of the buffer \p buf. + * @param rbytes[out] The number of bytes which have been read from \p buf. + * @param smps[out] The array of pointers to samples. + * @param cnt[in] The number of pointers in the array \p smps. + * + * @retval >=0 The number of samples which have been parsed from \p buf and written into \p smps. + * @retval <0 Something went wrong. + */ +int io_format_sscan(struct io_format *fmt, char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags); + +/** Print \p cnt samples from \p smps into buffer \p buf of length \p len. + * + * @param buf[out] The buffer which should be filled with serialized data. + * @param len[in] The length of the buffer \p buf. + * @param rbytes[out] The number of bytes which have been written to \p buf. Ignored if NULL. + * @param smps[in] The array of pointers to samples. + * @param cnt[in] The number of pointers in the array \p smps. + * + * @retval >=0 The number of samples from \p smps which have been written into \p buf. + * @retval <0 Something went wrong. + */ +int io_format_sprint(struct io_format *fmt, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags); + +/** Parse up to \p cnt samples from stream \p f into array \p smps. + * + * @retval >=0 The number of samples which have been parsed from \p f and written into \p smps. + * @retval <0 Something went wrong. + */ +int io_format_fscan(struct io_format *fmt, FILE *f, struct sample *smps[], unsigned cnt, int *flags); + +/** Print \p cnt samples from \p smps to stream \p f. + * + * @retval >=0 The number of samples from \p smps which have been written to \p f. + * @retval <0 Something went wrong. + */ +int io_format_fprint(struct io_format *fmt, FILE *f, struct sample *smps[], unsigned cnt, int flags); diff --git a/include/villas/kernel/rt.h b/include/villas/kernel/rt.h index a6408ff1d..44c22a0b5 100644 --- a/include/villas/kernel/rt.h +++ b/include/villas/kernel/rt.h @@ -1,5 +1,6 @@ /** Linux specific real-time optimizations * + * @see: https://wiki.linuxfoundation.org/realtime/documentation/howto/applications/application_base * @file * @author Steffen Vogel * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC @@ -31,6 +32,8 @@ int rt_set_affinity(int affinity); int rt_set_priority(int priority); +int rt_lock_memory(); + /** Checks for realtime (PREEMPT_RT) patched kernel. * * See https://rt.wiki.kernel.org diff --git a/include/villas/kernel/tc.h b/include/villas/kernel/tc.h index 0a861a8e1..da3cd712c 100644 --- a/include/villas/kernel/tc.h +++ b/include/villas/kernel/tc.h @@ -35,7 +35,7 @@ #include #include -#include +#include typedef uint32_t tc_hdl_t; @@ -43,12 +43,12 @@ struct interface; /** Parse network emulator (netem) settings. * - * @param cfg A libconfig object containing the settings. + * @param cfg A jansson object containing the settings. * @param[out] ne A pointer to a libnl3 qdisc object where setting will be written to. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **ne); +int tc_parse(struct rtnl_qdisc **ne, json_t *cfg); /** Print network emulator (netem) setting into buffer. * diff --git a/include/villas/log.h b/include/villas/log.h index e5699203e..5234f71bd 100644 --- a/include/villas/log.h +++ b/include/villas/log.h @@ -98,6 +98,7 @@ struct log { int level; long facilities; /**< Debug facilities used by the debug() macro. */ const char *path; /**< Path of the log file. */ + char *prefix; /**< Prefix each line with this string. */ FILE *file; /**< Send all log output to this file / stdout / stderr. */ }; diff --git a/include/villas/log_config.h b/include/villas/log_config.h index 7e898f82b..86c2281ba 100644 --- a/include/villas/log_config.h +++ b/include/villas/log_config.h @@ -1,4 +1,4 @@ -/** Logging routines that depend on libconfig. +/** Logging routines that depend on jansson. * * @file * @author Steffen Vogel @@ -25,14 +25,13 @@ struct log; -#include +#include #include "log.h" /** Parse logging configuration. */ -int log_parse(struct log *l, config_setting_t *cfg); +int log_parse(struct log *l, json_t *cfg); /** Print configuration error and exit. */ -void cerror(config_setting_t *cfg, const char *fmt, ...) +void jerror(json_error_t *err, const char *fmt, ...) __attribute__ ((format(printf, 2, 3))); - diff --git a/include/villas/mapping.h b/include/villas/mapping.h index 346c6f4ce..a814f0b93 100644 --- a/include/villas/mapping.h +++ b/include/villas/mapping.h @@ -23,7 +23,7 @@ #pragma once -#include +#include #include "stats.h" #include "common.h" @@ -87,7 +87,7 @@ struct mapping { int mapping_init(struct mapping *m); -int mapping_parse(struct mapping *m, config_setting_t *cfg); +int mapping_parse(struct mapping *m, json_t *cfg); int mapping_check(struct mapping *m); @@ -95,6 +95,6 @@ int mapping_destroy(struct mapping *m); int mapping_remap(struct mapping *m, struct sample *orig, struct sample *remapped, struct stats *s); -int mapping_entry_parse(struct mapping_entry *e, config_setting_t *cfg); +int mapping_entry_parse(struct mapping_entry *e, json_t *cfg); int mapping_entry_parse_str(struct mapping_entry *e, const char *str); diff --git a/include/villas/node.h b/include/villas/node.h index 9d99ff9c4..b32142f8b 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -28,7 +28,7 @@ #include #include -#include +#include #include "node_type.h" #include "sample.h" @@ -43,8 +43,7 @@ */ struct node { - const char *name; /**< A short identifier of the node, only used for configuration and logging */ - + char *name; /**< A short identifier of the node, only used for configuration and logging */ char *_name; /**< Singleton: A string used to print to screen. */ char *_name_long; /**< Singleton: A string used to print to screen. */ @@ -60,18 +59,18 @@ struct node struct node_type *_vt; /**< Virtual functions (C++ OOP style) */ void *_vd; /**< Virtual data (used by struct node::_vt functions) */ - config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this node */ + json_t *cfg; /**< A JSON object containing the configuration of the node. */ }; int node_init(struct node *n, struct node_type *vt); /** Parse settings of a node. * - * @param cfg A libconfig object pointing to the node. + * @param cfg A JSON object containing the configuration of the node. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int node_parse(struct node *n, config_setting_t *cfg); +int node_parse(struct node *n, json_t *cfg, const char *name); /** Parse settings of a node from cmdline. */ int node_parse_cli(struct node *n, int argc, char *argv[]); @@ -131,10 +130,10 @@ int node_write(struct node *n, struct sample *smps[], unsigned cnt); * out = [ "sintef", "scedu" ] * out = "acs" * - * @param cfg The libconfig object handle for "out". + * @param cfg A JSON array or string. See examples above. * @param nodes The nodes will be added to this list. * @param all This list contains all valid nodes. */ -int node_parse_list(struct list *list, config_setting_t *cfg, struct list *all); +int node_parse_list(struct list *list, json_t *cfg, struct list *all); /** @} */ diff --git a/include/villas/node_type.h b/include/villas/node_type.h index 9b8ca66c3..3b00957a3 100644 --- a/include/villas/node_type.h +++ b/include/villas/node_type.h @@ -26,7 +26,7 @@ #pragma once -#include +#include #include "list.h" #include "common.h" @@ -78,11 +78,11 @@ struct node_type { /** Parse node connection details. * * @param n A pointer to the node object. - * @param cfg A libconfig object pointing to the node. + * @param cfg A JSON object containing the configuration of the node. * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ - int (*parse)(struct node *n, config_setting_t *cfg); + int (*parse)(struct node *n, json_t *cfg); /** Parse node from command line arguments. */ int (*parse_cli)(struct node *n, int argc, char *argv[]); @@ -118,10 +118,10 @@ struct node_type { * Indexes used to address @p m will wrap around after len messages. * Some node-types might only support to receive one message at a time. * - * @param n A pointer to the node object. - * @param smps An array of pointers to memory blocks where the function should store received samples. - * @param cnt The number of messages which should be received. - * @return The number of messages actually received. + * @param n A pointer to the node object. + * @param smps An array of pointers to memory blocks where the function should store received samples. + * @param cnt The number of messages which should be received. + * @return The number of messages actually received. */ int (*read) (struct node *n, struct sample *smps[], unsigned cnt); @@ -132,10 +132,10 @@ struct node_type { * The messages have to be stored in a circular buffer / array m. * So the indexes will wrap around after len. * - * @param n A pointer to the node object. - * @param smps An array of pointers to memory blocks where samples read from. - * @param cnt The number of messages which should be sent. - * @return The number of messages actually sent. + * @param n A pointer to the node object. + * @param smps An array of pointers to memory blocks where samples read from. + * @param cnt The number of messages which should be sent. + * @return The number of messages actually sent. */ int (*write)(struct node *n, struct sample *smps[], unsigned cnt); @@ -143,7 +143,7 @@ struct node_type { * * This is not supported by all node-types! * - * @param n A pointer to the node object. + * @param n A pointer to the node object. */ int (*reverse)(struct node *n); }; diff --git a/include/villas/nodes/file.h b/include/villas/nodes/file.h index 78ab29c0c..624a73ee5 100644 --- a/include/villas/nodes/file.h +++ b/include/villas/nodes/file.h @@ -31,54 +31,48 @@ #pragma once -#include "advio.h" +#include "io.h" #include "node.h" +#include "task.h" #define FILE_MAX_PATHLEN 512 -enum { - FILE_READ, - FILE_WRITE -}; - struct file { - struct file_direction { - AFILE *handle; /**< libc: stdio file handle. */ + struct io io; /**< Format and file IO */ + struct io_format *format; - const char *mode; /**< libc: fopen() mode. */ - const char *fmt; /**< Format string for file name. */ - - char *uri; /**< Real file name. */ - } read, write; + char *uri_tmpl; /**< Format string for file name. */ + char *uri; /**< Real file name. */ + char *mode; /**< File access mode. */ int flush; /**< Flush / upload file contents after each write. */ + struct task task; /**< Timer file descriptor. Blocks until 1 / rate seconds are elapsed. */ + double rate; /**< The read rate. */ - enum read_epoch_mode { + enum epoch_mode { FILE_EPOCH_DIRECT, FILE_EPOCH_WAIT, FILE_EPOCH_RELATIVE, FILE_EPOCH_ABSOLUTE, FILE_EPOCH_ORIGINAL - } read_epoch_mode; /**< Specifies how file::offset is calculated. */ - - struct timespec read_first; /**< The first timestamp in the file file::{read,write}::uri */ - struct timespec read_epoch; /**< The epoch timestamp from the configuration. */ - struct timespec read_offset; /**< An offset between the timestamp in the input file and the current time */ - + } epoch_mode; /**< Specifies how file::offset is calculated. */ + enum { FILE_EOF_EXIT, /**< Terminate when EOF is reached. */ FILE_EOF_REWIND, /**< Rewind the file when EOF is reached. */ FILE_EOF_WAIT /**< Blocking wait when EOF is reached. */ - } read_eof; /**< Should we rewind the file when we reach EOF? */ - int read_timer; /**< Timer file descriptor. Blocks until 1 / rate seconds are elapsed. */ - double read_rate; /**< The read rate. */ + } eof; + + struct timespec first; /**< The first timestamp in the file file::{read,write}::uri */ + struct timespec epoch; /**< The epoch timestamp from the configuration. */ + struct timespec offset; /**< An offset between the timestamp in the input file and the current time */ }; /** @see node_type::print */ char * file_print(struct node *n); /** @see node_type::parse */ -int file_parse(struct node *n, config_setting_t *cfg); +int file_parse(struct node *n, json_t *cfg); /** @see node_type::open */ int file_start(struct node *n); diff --git a/include/villas/nodes/fpga.h b/include/villas/nodes/fpga.h index ece227bf5..fe235bb3b 100644 --- a/include/villas/nodes/fpga.h +++ b/include/villas/nodes/fpga.h @@ -44,7 +44,7 @@ int fpga_init(struct super_node *sn); int fpga_deinit(); /** @see node_type::parse */ -int fpga_parse(struct node *n, config_setting_t *cfg); +int fpga_parse(struct node *n, json_t *cfg); struct fpga_card * fpga_lookup_card(const char *name); diff --git a/include/villas/nodes/loopback.h b/include/villas/nodes/loopback.h index e92e8fe39..f11d82446 100644 --- a/include/villas/nodes/loopback.h +++ b/include/villas/nodes/loopback.h @@ -29,8 +29,6 @@ #pragma once -#include - #include "queue_signalled.h" #include "pool.h" @@ -53,7 +51,7 @@ struct loopback { char * loopback_print(struct node *n); /** @see node_type::parse */ -int loopback_parse(struct node *n, config_setting_t *cfg); +int loopback_parse(struct node *n, json_t *cfg); /** @see node_type::open */ int loopback_open(struct node *n); diff --git a/include/villas/nodes/nanomsg.h b/include/villas/nodes/nanomsg.h index 18109487d..d4ba6070c 100644 --- a/include/villas/nodes/nanomsg.h +++ b/include/villas/nodes/nanomsg.h @@ -34,6 +34,12 @@ #include "node.h" #include "list.h" +/** The maximum length of a packet which contains stuct msg. */ +#define NANOMSG_MAX_PACKET_LEN 1500 + +/* Forward declarations */ +struct io_format; + struct nanomsg { struct { int socket; @@ -44,13 +50,15 @@ struct nanomsg { int socket; struct list endpoints; } subscriber; + + struct io_format *format; }; /** @see node_type::print */ char * nanomsg_print(struct node *n); /** @see node_type::parse */ -int nanomsg_parse(struct node *n, config_setting_t *cfg); +int nanomsg_parse(struct node *n, json_t *cfg); /** @see node_type::open */ int nanomsg_start(struct node *n); diff --git a/include/villas/nodes/ngsi.h b/include/villas/nodes/ngsi.h index 9794b1a47..a64ac459a 100644 --- a/include/villas/nodes/ngsi.h +++ b/include/villas/nodes/ngsi.h @@ -39,9 +39,9 @@ #include #include "list.h" -#include "msg.h" #include "super_node.h" #include "node.h" +#include "task.h" struct node; @@ -54,7 +54,7 @@ struct ngsi { double timeout; /**< HTTP timeout in seconds */ double rate; /**< Rate used for polling. */ - int tfd; /**< Timer */ + struct task task; /**< Timer for periodic events. */ int ssl_verify; /**< Boolean flag whether SSL server certificates should be verified or not. */ struct curl_slist *headers; /**< List of HTTP request headers for libcurl */ @@ -77,7 +77,7 @@ int ngsi_init(struct super_node *sn); int ngsi_deinit(); /** @see node_type::parse */ -int ngsi_parse(struct node *n, config_setting_t *cfg); +int ngsi_parse(struct node *n, json_t *cfg); /** @see node_type::print */ char * ngsi_print(struct node *n); diff --git a/include/villas/nodes/opal.h b/include/villas/nodes/opal.h index 663a71631..61fc4f6db 100644 --- a/include/villas/nodes/opal.h +++ b/include/villas/nodes/opal.h @@ -67,7 +67,7 @@ int opal_init(struct super_node *sn); int opal_deinit(); /** @see node_type::parse */ -int opal_parse(struct node *n, config_setting_t *cfg); +int opal_parse(struct node *n, json_t *cfg); /** @see node_type::print */ char * opal_print(struct node *n); diff --git a/include/villas/nodes/shmem.h b/include/villas/nodes/shmem.h index c001d1d22..5af608e86 100644 --- a/include/villas/nodes/shmem.h +++ b/include/villas/nodes/shmem.h @@ -51,7 +51,7 @@ struct shmem { char * shmem_print(struct node *n); /** @see node_type::parse */ -int shmem_parse(struct node *n, config_setting_t *cfg); +int shmem_parse(struct node *n, json_t *cfg); /** @see node_type::open */ int shmem_open(struct node *n); diff --git a/include/villas/nodes/signal.h b/include/villas/nodes/signal.h index ed47edf04..3d0e07521 100644 --- a/include/villas/nodes/signal.h +++ b/include/villas/nodes/signal.h @@ -29,9 +29,8 @@ #pragma once -#include - #include "timing.h" +#include "task.h" /* Forward declarations */ struct node; @@ -50,7 +49,7 @@ enum signal_type { * @see node_type */ struct signal { - int tfd; /**< timerfd file descriptor. */ + struct task task; /**< Timer for periodic events. */ int rt; /**< Real-time mode? */ enum signal_type type; /**< Signal type */ @@ -71,7 +70,7 @@ struct signal { char * signal_print(struct node *n); /** @see node_type::parse */ -int signal_parse(struct node *n, config_setting_t *cfg); +int signal_parse(struct node *n, json_t *cfg); /** @see node_type::open */ int signal_open(struct node *n); diff --git a/include/villas/nodes/socket.h b/include/villas/nodes/socket.h index bf0700d90..b142c12ac 100644 --- a/include/villas/nodes/socket.h +++ b/include/villas/nodes/socket.h @@ -39,8 +39,22 @@ #include #endif +#ifdef WITH_LIBNL_ROUTE_30 + #include "kernel/if.h" + #include "kernel/nl.h" + #include "kernel/tc.h" + + #define WITH_NETEM +#endif /* WITH_LIBNL_ROUTE_30 */ + #include "node.h" +/* Forward declarations */ +struct io_format; + +/** The maximum length of a packet which contains stuct msg. */ +#define SOCKET_MAX_PACKET_LEN 1500 + enum socket_layer { SOCKET_LAYER_ETH, SOCKET_LAYER_IP, @@ -80,6 +94,8 @@ struct socket { union sockaddr_union local; /**< Local address of the socket */ union sockaddr_union remote; /**< Remote address of the socket */ + struct io_format *format; + /* Multicast options */ struct multicast { int enabled; /**< Is multicast enabled? */ @@ -88,8 +104,10 @@ struct socket { struct ip_mreq mreq; /**< A multicast group to join. */ } multicast; +#ifdef WITH_NETEM struct rtnl_qdisc *tc_qdisc; /**< libnl3: Network emulator queuing discipline */ struct rtnl_cls *tc_classifier; /**< libnl3: Firewall mark classifier */ +#endif /* WITH_NETEM */ }; @@ -112,7 +130,7 @@ int socket_write(struct node *n, struct sample *smps[], unsigned cnt); int socket_read(struct node *n, struct sample *smps[], unsigned cnt); /** @see node_type::parse */ -int socket_parse(struct node *n, config_setting_t *cfg); +int socket_parse(struct node *n, json_t *cfg); /** @see node_type::print */ char * socket_print(struct node *n); diff --git a/include/villas/nodes/test_rtt.h b/include/villas/nodes/test_rtt.h new file mode 100644 index 000000000..8c38a1cbe --- /dev/null +++ b/include/villas/nodes/test_rtt.h @@ -0,0 +1,73 @@ +/** Node type: Node-type for testing Round-trip Time. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** + * @addtogroup test-rtt Node-type for testing Round-trip Time. + * @ingroup node + * @{ + */ + +#pragma once + +#include "node.h" +#include "list.h" +#include "io.h" +#include "task.h" + +struct test_rtt_case { + struct task task; + double rate; + int values; + int counter; + int limit; /**< The number of samples we take per test. */ +}; + +struct test_rtt { + double cooldown; /**< Number of seconds to wait beween tests. */ + int current; /**< Index of current test in test_rtt::cases */ + + struct io io; /**< The format of the output file */ + struct list cases; /**< List of test cases */ + + const char *output; /**< The directory where we place the results. */ +}; + +/** @see node_type::print */ +char * test_rtt_print(struct node *n); + +/** @see node_type::parse */ +int test_rtt_parse(struct node *n, json_t *cfg); + +/** @see node_type::open */ +int test_rtt_start(struct node *n); + +/** @see node_type::close */ +int test_rtt_stop(struct node *n); + +/** @see node_type::read */ +int test_rtt_read(struct node *n, struct sample *smps[], unsigned cnt); + +/** @see node_type::write */ +int test_rtt_write(struct node *n, struct sample *smps[], unsigned cnt); + +/** @} */ diff --git a/include/villas/nodes/websocket.h b/include/villas/nodes/websocket.h index 5316756ec..02e80a75d 100644 --- a/include/villas/nodes/websocket.h +++ b/include/villas/nodes/websocket.h @@ -48,7 +48,6 @@ struct lws; /** Internal data per websocket node */ struct websocket { - struct list connections; /**< List of active libwebsocket connections in server mode (struct websocket_connection). */ struct list destinations; /**< List of websocket servers connect to in client mode (struct websocket_destination). */ struct pool pool; @@ -57,24 +56,31 @@ struct websocket { /* Internal datastructures */ struct websocket_connection { - struct node *node; - struct lws *wsi; - - struct queue queue; /**< For samples which are sent to the WebSocket */ - - struct { - char name[64]; - char ip[64]; - } peer; - + enum websocket_connection_state { + STATE_DISCONNECTED, + STATE_CONNECTING, + STATE_RECONNECTING, + STATE_ESTABLISHED, + STATE_SHUTDOWN, + STATE_ERROR + } state; /**< The current status of this connection. */ + enum { WEBSOCKET_MODE_CLIENT, WEBSOCKET_MODE_SERVER, } mode; - enum state state; + struct lws *wsi; + struct node *node; + struct io_format *format; /**< The IO format used for this connection. */ + struct queue queue; /**< For samples which are sent to the WebSocket */ - char *buf; /**< A buffer which is used to construct the messages. */ + struct websocket_destination *destination; + + struct { + struct buffer recv; /**< A buffer for reconstructing fragmented messags. */ + struct buffer send; /**< A buffer for contsructing messages before calling lws_write() */ + } buffers; char *_name; }; diff --git a/include/villas/nodes/zeromq.h b/include/villas/nodes/zeromq.h index 88841e3d9..3de66acd1 100644 --- a/include/villas/nodes/zeromq.h +++ b/include/villas/nodes/zeromq.h @@ -30,19 +30,26 @@ #pragma once #include +#include -#include "node.h" #include "list.h" #if ZMQ_VERSION_MAJOR > 4 || (ZMQ_VERSION_MAJOR == 4 && ZMQ_VERSION_MINOR >= 2) #define ZMQ_BUILD_DISH 1 #endif +/* Forward declarations */ +struct io_format; +struct node; +struct sample; + struct zeromq { int ipv6; char *filter; + struct io_format *format; + struct { int enabled; struct { @@ -74,7 +81,7 @@ struct zeromq { char * zeromq_print(struct node *n); /** @see node_type::parse */ -int zeromq_parse(struct node *n, config_setting_t *cfg); +int zeromq_parse(struct node *n, json_t *cfg); /** @see node_type::init */ int zeromq_init(); diff --git a/include/villas/path.h b/include/villas/path.h index 7e75afa2b..73f22c542 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -30,7 +30,7 @@ #pragma once #include -#include +#include #include "list.h" #include "queue.h" @@ -83,7 +83,8 @@ struct path struct stats *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ struct super_node *super_node; /**< The super node this path belongs to. */ - config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path. */ + + json_t *cfg; /**< A JSON object containing the configuration of the path. */ }; /** Initialize internal data structures. */ @@ -141,12 +142,12 @@ int path_uses_node(struct path *p, struct node *n); /** Parse a single path and add it to the global configuration. * - * @param cfg A libconfig object pointing to the path + * @param cfg A JSON object containing the configuration of the path. * @param p Pointer to the allocated memory for this path * @param nodes A linked list of all existing nodes * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes); +int path_parse(struct path *p, json_t *cfg, struct list *nodes); /** @} */ diff --git a/include/villas/plugin.h b/include/villas/plugin.h index ff4792fa3..9b71b7e46 100644 --- a/include/villas/plugin.h +++ b/include/villas/plugin.h @@ -23,6 +23,7 @@ #pragma once +#include "io_format.h" #include "hook.h" #include "api.h" #include "common.h" @@ -53,6 +54,7 @@ enum plugin_type { PLUGIN_TYPE_HOOK, PLUGIN_TYPE_NODE, PLUGIN_TYPE_API, + PLUGIN_TYPE_IO, PLUGIN_TYPE_FPGA_IP, PLUGIN_TYPE_MODEL_CBUILDER }; @@ -71,6 +73,7 @@ struct plugin { int (*unload)(struct plugin *p); union { + struct io_format io; struct api_action api; struct node_type node; #ifdef WITH_FPGA @@ -91,7 +94,7 @@ int plugin_init(struct plugin *p); int plugin_destroy(struct plugin *p); -int plugin_parse(struct plugin *p, config_setting_t *cfg); +int plugin_parse(struct plugin *p, json_t *cfg); int plugin_load(struct plugin *p); diff --git a/include/villas/sample.h b/include/villas/sample.h index 707058739..62bb67624 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -42,20 +42,22 @@ struct pool; #define SAMPLE_LEN(len) (sizeof(struct sample) + SAMPLE_DATA_LEN(len)) /** The length of a sample data portion of a sample datastructure with \p values values in bytes. */ -#define SAMPLE_DATA_LEN(len) ((len) * sizeof(float)) +#define SAMPLE_DATA_LEN(len) ((len) * sizeof(double)) /** The offset to the beginning of the data section. */ #define SAMPLE_DATA_OFFSET(smp) ((char *) (smp) + offsetof(struct sample, data)) enum sample_data_format { - SAMPLE_DATA_FORMAT_FLOAT= 0, - SAMPLE_DATA_FORMAT_INT = 1 + SAMPLE_DATA_FORMAT_FLOAT = 0, + SAMPLE_DATA_FORMAT_INT = 1 }; struct sample { int sequence; /**< The sequence number of this sample. */ int length; /**< The number of values in sample::values which are valid. */ int capacity; /**< The number of values in sample::values for which memory is reserved. */ + + int id; atomic_int refcnt; /**< Reference counter. */ off_t pool_off; /**< This sample belongs to this memory pool (relative pointer). */ @@ -68,12 +70,16 @@ struct sample { struct timespec sent; /**< The point in time when this data was send for the last time. */ } ts; - uint64_t format; /**< A long bitfield indicating the number representation of the first 64 values in sample::data[] */ + /** A long bitfield indicating the number representation of the first 64 values in sample::data[]. + * + * @see sample_data_format + */ + uint64_t format; /** The values. */ union { - float f; /**< Floating point values. */ - uint32_t i; /**< Integer values. */ + double f; /**< Floating point values. */ + uint64_t i; /**< Integer values. */ } data[]; /**< Data is in host endianess! */ }; diff --git a/include/villas/sample_io.h b/include/villas/sample_io.h deleted file mode 100644 index d900a0e05..000000000 --- a/include/villas/sample_io.h +++ /dev/null @@ -1,89 +0,0 @@ -/** Read / write sample data in different formats. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include -#include - -/* Forward declarations */ -struct sample; - -enum sample_io_format { - SAMPLE_IO_FORMAT_VILLAS, - SAMPLE_IO_FORMAT_JSON, - SAMPLE_IO_FORMAT_HDF5, - SAMPLE_IO_FORMAT_COMTRADE -}; - -/** These flags define the format which is used by sample_io_fscan() and sample_io_fprint(). */ -enum sample_flags { - SAMPLE_IO_NANOSECONDS = (1 << 0), - SAMPLE_IO_OFFSET = (1 << 1), - SAMPLE_IO_SEQUENCE = (1 << 2), - SAMPLE_IO_VALUES = (1 << 3), - SAMPLE_IO_ALL = 16-1 -}; - -/* Not implemented yet */ -#if 0 -struct sample_io_handle { - enum sample_io_format format; - int flags - - FILE *file; - - struct list fields; -}; - -int sample_io_init(struct sample_io *io, enum sample_io_format fmt, char *mode, int flags); - -int sample_io_destroy(struct sample_io *io); - -int sample_io_open(struct sample_io *io); - -int sample_io_close(struct sample_io *io); - -int sample_io_write(struct sample_io *io, struct sample *smps[], size_t cnt); - -int sample_io_read(struct sample_io *io, struct sample *smps[], size_t cnt); - -int sample_io_eof(struct sample_io *io); -int sample_io_rewind(struct sample_io *io); -#endif - -/* Lowlevel interface */ - -int sample_io_fprint(FILE *f, struct sample *s, enum sample_io_format fmt, int flags); - -int sample_io_fscan(FILE *f, struct sample *s, enum sample_io_format fmt, int *flags); - -/* VILLASnode human readable format */ - -int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags); - -int sample_io_villas_scan(const char *line, struct sample *s, int *fl); - -int sample_io_villas_fprint(FILE *f, struct sample *s, int flags); - -int sample_io_villas_fscan(FILE *f, struct sample *s, int *flags); diff --git a/include/villas/stats.h b/include/villas/stats.h index 643f7f3e2..a3c377d63 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -25,6 +25,7 @@ #define _STATS_H_ #include +#include #include "hist.h" @@ -61,6 +62,8 @@ struct stats { struct stats_delta *delta; }; +int stats_lookup_format(const char *str); + int stats_init(struct stats *s, int buckets, int warmup); int stats_destroy(struct stats *s); @@ -71,11 +74,7 @@ void stats_collect(struct stats_delta *s, struct sample *smps[], size_t cnt); int stats_commit(struct stats *s, struct stats_delta *d); -#ifdef WITH_JSON - #include - json_t * stats_json(struct stats *s); -#endif void stats_reset(struct stats *s); diff --git a/include/villas/super_node.h b/include/villas/super_node.h index cf435dfa3..bcd4169f2 100644 --- a/include/villas/super_node.h +++ b/include/villas/super_node.h @@ -1,8 +1,4 @@ -/** Configuration file parser. - * - * The server program is configured by a single file. - * This config file is parsed with a third-party library: - * libconfig http://www.hyperrealm.com/libconfig/ +/** The super node object holding the state of the application. * * @file * @author Steffen Vogel @@ -27,8 +23,6 @@ #pragma once -#include - #include "list.h" #include "api.h" #include "web.h" @@ -50,6 +44,8 @@ struct super_node { struct api api; struct web web; + char *name; /**< A name of this super node. Usually the hostname. */ + struct { int argc; char **argv; @@ -59,8 +55,7 @@ struct super_node { char *uri; /**< URI of configuration */ - config_t cfg; /**< Pointer to configuration file */ - json_t *json; /**< JSON representation of the same config. */ + json_t *cfg; /**< JSON representation of the configuration. */ }; /* Compatibility with libconfig < 1.5 */ @@ -84,7 +79,7 @@ int super_node_parse_uri(struct super_node *sn, const char *uri); * @retval 0 Success. Everything went well. * @retval <0 Error. Something went wrong. */ -int super_node_parse(struct super_node *sn, config_setting_t *cfg); +int super_node_parse_json(struct super_node *sn, json_t *cfg); /** Check validity of super node configuration. */ int super_node_check(struct super_node *sn); diff --git a/include/villas/task.h b/include/villas/task.h new file mode 100644 index 000000000..518ae46ba --- /dev/null +++ b/include/villas/task.h @@ -0,0 +1,71 @@ +/** Run tasks periodically. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#include + +/** We can choose between two periodic task implementations */ +//#define PERIODIC_TASK_IMPL NANOSLEEP +#define TIMERFD 1 +#define CLOCK_NANOSLEEP 2 +#define NANOSLEEP 3 + +#if defined(__MACH__) + #define PERIODIC_TASK_IMPL NANOSLEEP +#else + #define PERIODIC_TASK_IMPL TIMERFD +#endif + +struct task { + struct timespec period; +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP + struct timespec next_period; +#elif PERIODIC_TASK_IMPL == TIMERFD + int fd; +#else + #error "Invalid period task implementation" +#endif + int clock; +}; + +/** Create a new task with the given rate. */ +int task_init(struct task *t, double rate, int clock); + +int task_destroy(struct task *t); + +/** Wait until task elapsed + * + * @retval 0 An error occured. Maybe the task was stopped. + * @retval >0 The nummer of runs this task already fired. + */ +uint64_t task_wait_until_next_period(struct task *t); + +/** Wait until a fixed time in the future is reached + * + * @param until A pointer to a time in the future. + */ +int task_wait_until(struct task *t, const struct timespec *until); \ No newline at end of file diff --git a/include/villas/timing.h b/include/villas/timing.h index 6cb00c19e..1ef49e123 100644 --- a/include/villas/timing.h +++ b/include/villas/timing.h @@ -28,30 +28,6 @@ #include -#ifdef __linux__ - #include -#endif - -/** Create a new timer with the given rate. */ -int timerfd_create_rate(double rate); - -/** Wait until timer elapsed - * - * @param fd A file descriptor which was created by timerfd_create(3). - * @retval 0 An error occured. Maybe the timer was stopped. - * @retval >0 The nummer of runs this timer already fired. - */ -uint64_t timerfd_wait(int fd); - -/** Wait until a fixed time in the future is reached - * - * @param fd A file descriptor which was created by timerfd_create(3). - * @param until A pointer to a time in the future. - * @retval 0 An error occured. Maybe the timer was stopped. - * @retval >0 The nummer of runs this timer already fired. - */ -uint64_t timerfd_wait_until(int fd, const struct timespec *until); - /** Get delta between two timespec structs */ struct timespec time_diff(const struct timespec *start, const struct timespec *end); diff --git a/include/villas/web.h b/include/villas/web.h index 307d1c72b..a299146e4 100644 --- a/include/villas/web.h +++ b/include/villas/web.h @@ -23,7 +23,6 @@ #pragma once -#include #include #include "common.h" @@ -40,9 +39,9 @@ struct web { struct lws_vhost *vhost; /**< The libwebsockets vhost. */ int port; /**< Port of the build in HTTP / WebSocket server. */ - const char *htdocs; /**< The root directory for files served via HTTP. */ - const char *ssl_cert; /**< Path to the SSL certitifcate for HTTPS / WSS. */ - const char *ssl_private_key; /**< Path to the SSL private key for HTTPS / WSS. */ + char *htdocs; /**< The root directory for files served via HTTP. */ + char *ssl_cert; /**< Path to the SSL certitifcate for HTTPS / WSS. */ + char *ssl_private_key; /**< Path to the SSL private key for HTTPS / WSS. */ pthread_t thread; }; @@ -60,4 +59,4 @@ int web_start(struct web *w); int web_stop(struct web *w); /** Parse HTTPd and WebSocket related options */ -int web_parse(struct web *w, config_setting_t *lcs); +int web_parse(struct web *w, json_t *cfg); diff --git a/include/villas/web/buffer.h b/include/villas/web/buffer.h deleted file mode 100644 index 88a3d755f..000000000 --- a/include/villas/web/buffer.h +++ /dev/null @@ -1,72 +0,0 @@ -/** WebSocket buffer. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include - -#include - -#include "common.h" - -struct web_buffer { - char *buffer; /**< A pointer to the buffer. Usually resized via realloc() */ - size_t size; /**< The allocated size of the buffer. */ - size_t len; /**< The used length of the buffer. */ - size_t prefix; /**< The used length of the buffer. */ - - enum lws_write_protocol protocol; - - enum state state; -}; - -/** Initialize a libwebsockets buffer. */ -int web_buffer_init(struct web_buffer *b, enum lws_write_protocol prot); - -/** Destroy a libwebsockets buffer. */ -int web_buffer_destroy(struct web_buffer *b); - -/** Flush the buffers contents to lws_write() */ -int web_buffer_write(struct web_buffer *b, struct lws *w); - -/** Copy \p len bytes from the beginning of the buffer and copy them to \p out. - * - * @param out The destination buffer. If NULL, we just remove \p len bytes from the buffer. - */ -int web_buffer_read(struct web_buffer *b, char *out, size_t len); - -/** Parse JSON from the beginning of the buffer. - * - * @retval -1 The buffer is empty. - * @retval -2 The buffer contains malformed JSON. - */ -int web_buffer_read_json(struct web_buffer *b, json_t **req); - -/** Append \p len bytes of \p in at the end of the buffer. - * - * The buffer is automatically resized. - */ -int web_buffer_append(struct web_buffer *b, const char *in, size_t len); - -/** Append the serialized represetnation of the JSON object \p res at the end of the buffer. */ -int web_buffer_append_json(struct web_buffer *b, json_t *res); diff --git a/include/villas/webmsg_format.h b/include/villas/webmsg_format.h deleted file mode 100644 index d58d3d773..000000000 --- a/include/villas/webmsg_format.h +++ /dev/null @@ -1,89 +0,0 @@ -/** Binary websocket message format. - * - * Note: Messages sent by the 'websocket' node-type are always send in little endian byte-order! - * This is different from the messages send with the 'socket' node-type! - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include - -/** The current version number for the message format */ -#define WEBMSG_VERSION 2 - -/** @todo Implement more message types */ -#define WEBMSG_TYPE_DATA 0 /**< Message contains float values */ -#define WEBMSG_TYPE_START 1 /**< Message marks the beginning of a new simulation case */ -#define WEBMSG_TYPE_STOP 2 /**< Message marks the end of a simulation case */ - -/** The total size in bytes of a message */ -#define WEBMSG_LEN(values) (sizeof(struct webmsg) + WEBMSG_DATA_LEN(values)) - -/** The length of \p values values in bytes. */ -#define WEBMSG_DATA_LEN(values) (sizeof(float) * (values)) - -/** The offset to the first data value in a message. */ -#define WEBMSG_DATA_OFFSET(msg) ((char *) (msg) + offsetof(struct webmsg, data)) - -/** Initialize a message with default values */ -#define WEBMSG_INIT(len, seq) (struct webmsg) {\ - .version = WEBMSG_VERSION, \ - .type = WEBMSG_TYPE_DATA, \ - .length = len, \ - .sequence = seq \ -} - -/** The timestamp of a message in struct timespec format */ -#define WEBMSG_TS(msg) (struct timespec) {\ - .tv_sec = (msg)->ts.sec, \ - .tv_nsec = (msg)->ts.nsec \ -} - -/** This message format is used by all clients - * - * @diafile msg_format.dia - **/ -struct webmsg -{ - unsigned rsvd1 : 2; /**< Reserved bits */ - unsigned type : 2; /**< Data or control message (see MSG_TYPE_*) */ - unsigned version: 4; /**< Specifies the format of the remaining message (see MGS_VERSION) */ - - uint8_t id; /**< The node index from / to which this sample received / sent to. - * Corresponds to the index of the node in the http://localhost/nodes.json array. */ - - uint16_t length; /**< The number of values in msg::data[]. */ - uint32_t sequence; /**< The sequence number is incremented by one for consecutive messages. */ - - /** A timestamp per message. */ - struct { - uint32_t sec; /**< Seconds since 1970-01-01 00:00:00 */ - uint32_t nsec; /**< Nanoseconds of the current second. */ - } ts; - - /** The message payload. */ - union { - float f; /**< Floating point values. */ - uint32_t i; /**< Integer values. */ - } data[]; -} __attribute__((packed)); diff --git a/lib/Makefile.inc b/lib/Makefile.inc index de447f6c1..7c762a738 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -28,6 +28,7 @@ LIB_CFLAGS = $(CFLAGS) -fPIC -include lib/hooks/Makefile.inc -include lib/nodes/Makefile.inc +-include lib/io/Makefile.inc -include $(patsubst %, lib/Makefile.%.inc, $(SONAMES)) diff --git a/lib/Makefile.villas.inc b/lib/Makefile.villas.inc index 58f5754e0..b3a09c715 100644 --- a/lib/Makefile.villas.inc +++ b/lib/Makefile.villas.inc @@ -26,15 +26,14 @@ LIB = $(BUILDDIR)/$(LIB_NAME).so.$(LIB_ABI_VERSION) WITH_WEB ?= 1 WITH_API ?= 1 -WITH_JSON ?= 1 # Object files for libvillas LIB_SRCS += $(addprefix lib/kernel/, kernel.c rt.c) \ $(addprefix lib/, sample.c path.c node.c hook.c log.c log_config.c \ utils.c super_node.c hist.c timing.c pool.c list.c queue.c \ queue_signalled.c memory.c advio.c plugin.c node_type.c stats.c \ - mapping.c sample_io.c shmem.c config_helper.c crypt.c compat.c \ - log_table.c log_helper.c \ + mapping.c io.c shmem.c config_helper.c crypt.c compat.c \ + log_table.c log_helper.c io_format.c task.c buffer.c \ ) LIB_LDFLAGS = -shared @@ -46,14 +45,10 @@ endif LIB_PKGS += openssl libcurl -ifeq ($(WITH_JSON),1) - LIB_SRCS += lib/sample_io_json.c - LIB_PKGS += jansson - LIB_CFLAGS += -DWITH_JSON -endif - ifeq ($(WITH_WEB),1) - -include lib/web/Makefile.inc + LIB_SRCS += lib/web.c + LIB_PKGS += libwebsockets + LIB_CFLAGS += -DWITH_WEB endif ifeq ($(WITH_API),1) @@ -74,11 +69,11 @@ $(LIB): $(LIB_OBJS) ln -srf $@ $(BUILDDIR)/$(LIB_NAME).so # Install -install-libvillas: libvillas +install-libvillas: libvillas | $(DESTDIR)$(PREFIX)/include/villas/ install -m 0755 -D -T $(LIB) $(DESTDIR)$(PREFIX)/lib/$(LIB_NAME).so.$(LIB_ABI_VERSION) install -m 0644 -D -t $(DESTDIR)$(PREFIX)/include/villas/ include/villas/*.h ln -srf $(DESTDIR)$(PREFIX)/lib/$(LIB_NAME).so.$(LIB_ABI_VERSION) $(DESTDIR)$(PREFIX)/lib/$(LIB_NAME).so - ldconfig + if [ "$(PLATFORM)" == "Linux" ]; then ldconfig; fi clean-libvillas: rm -f $(LIB) diff --git a/lib/advio.c b/lib/advio.c index 25206d403..ae95fd094 100644 --- a/lib/advio.c +++ b/lib/advio.c @@ -169,6 +169,23 @@ static int advio_xferinfo(void *p, curl_off_t dl_total_bytes, curl_off_t dl_byte return 0; } +int aislocal(const char *uri) +{ + char *sep; + const char *supported_schemas[] = { "file", "http", "https", "tftp", "ftp", "scp", "sftp", "smb", "smbs" }; + + sep = strstr(uri, "://"); + if (!sep) + return 1; /* no schema, we assume its a local file */ + + for (int i = 0; i < ARRAY_LEN(supported_schemas); i++) { + if (!strncmp(supported_schemas[i], uri, sep - uri)) + return 0; + } + + return -1; /* none of the supported schemas match. this is an invalid uri */ +} + AFILE * afopen(const char *uri, const char *mode) { int ret; @@ -184,7 +201,7 @@ AFILE * afopen(const char *uri, const char *mode) if (!af->uri) goto out2; } - else { + else { /* Open local file by prepending file:// schema. */ if (strlen(uri) <= 1) return NULL; @@ -242,14 +259,19 @@ int afclose(AFILE *af) int ret; ret = afflush(af); - + if (ret) + return ret; + curl_easy_cleanup(af->curl); - fclose(af->file); + + ret = fclose(af->file); + if (ret) + return ret; free(af->uri); free(af); - return ret; + return 0; } int afseek(AFILE *af, long offset, int origin) @@ -416,6 +438,7 @@ int adownload(AFILE *af, int resume) case CURLE_FILE_COULDNT_READ_FILE: case CURLE_TFTP_NOTFOUND: case CURLE_REMOTE_FILE_NOT_FOUND: + info("File does not exist."); goto notexist; /* If libcurl does not know the protocol, we will try it with the stdio */ @@ -425,7 +448,7 @@ int adownload(AFILE *af, int resume) return -1; default: - error("ADVIO: Failed to download file: %s: %s", af->uri, curl_easy_strerror(res)); + error("Failed to download file: %s: %s", af->uri, curl_easy_strerror(res)); return -1; } diff --git a/lib/api.c b/lib/api.c index 5fe95dfa8..deb39c049 100644 --- a/lib/api.c +++ b/lib/api.c @@ -30,9 +30,13 @@ #include "assert.h" #include "compat.h" +/* Forward declarations */ +static void * worker(void *ctx); + int api_ws_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { - int ret; + int ret, pulled, pushed; + json_t *req, *resp; struct web *w = lws_context_user(lws_get_context(wsi)); struct api_session *s = (struct api_session *) user; @@ -46,21 +50,23 @@ int api_ws_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void * /* Parse request URI */ char uri[64]; - lws_hdr_copy(wsi, uri, sizeof(uri), WSI_TOKEN_GET_URI); /* The path component of the*/ + lws_hdr_copy(wsi, uri, sizeof(uri), WSI_TOKEN_GET_URI); ret = sscanf(uri, "/v%d", (int *) &s->version); if (ret != 1) return -1; - ret = api_session_init(s, w->api, API_MODE_WS); + ret = api_session_init(s, API_MODE_WS); if (ret) return -1; - list_push(&w->api->sessions, s); + s->wsi = wsi; + s->api = w->api; - lws_get_peer_addresses(wsi, lws_get_socket_fd(wsi), s->peer.name, sizeof(s->peer.name), s->peer.ip, sizeof(s->peer.ip)); + list_push(&s->api->sessions, s); + + debug(LOG_API, "Initiated API session: %s", api_session_name(s)); - debug(LOG_API, "New API session initiated: version=%d, mode=websocket, remote=%s (%s)", s->version, s->peer.name, s->peer.ip); break; case LWS_CALLBACK_CLOSED: @@ -70,28 +76,44 @@ int api_ws_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void * list_remove(&w->api->sessions, s); - debug(LOG_API, "Closed API session"); - - break; - - case LWS_CALLBACK_SERVER_WRITEABLE: - web_buffer_write(&s->response.body, wsi); - - if (s->completed && s->response.body.len == 0) - return -1; + debug(LOG_API, "Closing API session: %s", api_session_name(s)); break; case LWS_CALLBACK_RECEIVE: - web_buffer_append(&s->request.body, in, len); + if (lws_is_first_fragment(wsi)) + buffer_clear(&s->request.buffer); + + buffer_append(&s->request.buffer, in, len); + + if (lws_is_final_fragment(wsi)) { + ret = buffer_parse_json(&s->request.buffer, &req); + if (ret) + break; + + pushed = queue_push(&s->request.queue, req); + if (pushed != 1) + warn("Queue overun in API session"); - json_t *req, *resp; - while (web_buffer_read_json(&s->request.body, &req) >= 0) { - api_session_run_command(s, req, &resp); - - web_buffer_append_json(&s->response.body, resp); - lws_callback_on_writable(wsi); + pushed = queue_signalled_push(&w->api->pending, s); + if (pushed != 1) + warn("Queue overrun in API"); } + + break; + + case LWS_CALLBACK_SERVER_WRITEABLE: + pulled = queue_pull(&s->response.queue, (void **) &resp); + if (pulled < 1) + break; + + char pad[LWS_PRE]; + + buffer_clear(&s->response.buffer); + buffer_append(&s->response.buffer, pad, sizeof(pad)); + buffer_append_json(&s->response.buffer, resp); + + lws_write(wsi, (unsigned char *) s->response.buffer.buf + LWS_PRE, s->response.buffer.len - LWS_PRE, LWS_WRITE_TEXT); break; default: @@ -103,7 +125,8 @@ int api_ws_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void * int api_http_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { - int ret; + int ret, pulled, pushed; + json_t *resp, *req; struct web *w = lws_context_user(lws_get_context(wsi)); struct api_session *s = (struct api_session *) user; @@ -120,32 +143,16 @@ int api_http_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void if (ret != 1) return -1; - ret = api_session_init(s, w->api, API_MODE_HTTP); + ret = api_session_init(s, API_MODE_HTTP); if (ret) return -1; + + s->wsi = wsi; + s->api = w->api; - list_push(&w->api->sessions, s); - - lws_get_peer_addresses(wsi, lws_get_socket_fd(wsi), s->peer.name, sizeof(s->peer.name), s->peer.ip, sizeof(s->peer.ip)); + list_push(&s->api->sessions, s); - debug(LOG_API, "New API session initiated: version=%d, mode=http, remote=%s (%s)", s->version, s->peer.name, s->peer.ip); - - /* Prepare HTTP response header */ - const char headers[] = "HTTP/1.1 200 OK\r\n" - "Content-type: application/json\r\n" - "User-agent: " USER_AGENT "\r\n" - "Access-Control-Allow-Origin: *\r\n" - "Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n" - "Access-Control-Allow-Headers: Content-Type\r\n" - "Access-Control-Max-Age: 86400\r\n" - "\r\n"; - - web_buffer_append(&s->response.headers, headers, sizeof(headers)-1); - lws_callback_on_writable(wsi); - - /* Only HTTP POST requests wait for body */ - if (lws_hdr_total_length(wsi, WSI_TOKEN_POST_URI) == 0) - s->completed = true; + debug(LOG_API, "Initiated API session: %s", api_session_name(s)); break; @@ -163,27 +170,50 @@ int api_http_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void break; case LWS_CALLBACK_HTTP_BODY: - web_buffer_append(&s->request.body, in, len); + buffer_append(&s->request.buffer, in, len); - json_t *req, *resp; - while (web_buffer_read_json(&s->request.body, &req) == 1) { - api_session_run_command(s, req, &resp); - - web_buffer_append_json(&s->response.body, resp); - lws_callback_on_writable(wsi); - } break; case LWS_CALLBACK_HTTP_BODY_COMPLETION: - s->completed = true; + ret = buffer_parse_json(&s->request.buffer, &req); + if (ret) + break; + + buffer_clear(&s->request.buffer); + + pushed = queue_push(&s->request.queue, req); + if (pushed != 1) + warn("Queue overrun for API session: %s", api_session_name(s)); + + pushed = queue_signalled_push(&w->api->pending, s); + if (pushed != 1) + warn("Queue overrun for API"); + break; case LWS_CALLBACK_HTTP_WRITEABLE: - web_buffer_write(&s->response.headers, wsi); - web_buffer_write(&s->response.body, wsi); + pulled = queue_pull(&s->response.queue, (void **) &resp); + if (pulled) { + const char headers[] = "HTTP/1.1 200 OK\r\n" + "Content-type: application/json\r\n" + "User-agent: " USER_AGENT "\r\n" + "Access-Control-Allow-Origin: *\r\n" + "Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n" + "Access-Control-Allow-Headers: Content-Type\r\n" + "Access-Control-Max-Age: 86400\r\n" + "\r\n"; + + buffer_clear(&s->response.buffer); + buffer_append_json(&s->response.buffer, resp); + + lws_write(wsi, (unsigned char *) headers, strlen(headers), LWS_WRITE_HTTP_HEADERS); + lws_write(wsi, (unsigned char *) s->response.buffer.buf, s->response.buffer.len, LWS_WRITE_HTTP); + + debug(LOG_API, "Closing API session: %s", api_session_name(s)); - if (s->completed && s->response.body.len == 0) return -1; /* Close connection */ + } + break; default: @@ -216,7 +246,13 @@ int api_destroy(struct api *a) int api_start(struct api *a) { + int ret; + info("Starting API sub-system"); + + ret = pthread_create(&a->thread, NULL, worker, a); + if (ret) + error("Failed to start API worker thread"); a->state = STATE_STARTED; @@ -225,16 +261,57 @@ int api_start(struct api *a) int api_stop(struct api *a) { + int ret; + info("Stopping API sub-system"); - for (int i = 0; i < 10 && list_length(&a->sessions) > 0; i++) { + for (int i = 0; i < 10 && list_length(&a->sessions) > 0; i++) { INDENT info("Wait for API requests to complete"); - usleep(100 * 1e-3); + usleep(1 * 1e6); } + + if (a->state != STATE_STARTED) + return 0; - list_destroy(&a->sessions, (dtor_cb_t) api_session_destroy, false); + ret = list_destroy(&a->sessions, (dtor_cb_t) api_session_destroy, false); + if (ret) + return ret; + + ret = pthread_cancel(a->thread); + if (ret) + serror("Failed to cancel API worker thread"); + + ret = pthread_join(a->thread, NULL); + if (ret) + serror("Failed to join API worker thread"); a->state = STATE_STOPPED; return 0; } + +static void * worker(void *ctx) +{ + int pulled; + + struct api *a = ctx; + struct api_session *s; + + json_t *req, *resp; + + for (;;) { + pulled = queue_signalled_pull(&a->pending, (void **) &s); + if (pulled != 1) + continue; + + queue_pull(&s->request.queue, (void **) &req); + + api_session_run_command(s, req, &resp); + + queue_push(&s->response.queue, resp); + + lws_callback_on_writable(s->wsi); + } + + return NULL; +} diff --git a/lib/api/Makefile.inc b/lib/api/Makefile.inc index f83c84d67..a9d0a41c4 100644 --- a/lib/api/Makefile.inc +++ b/lib/api/Makefile.inc @@ -20,10 +20,6 @@ # along with this program. If not, see . ################################################################################### -ifndef WITH_JSON - $(error API support requires JSON) -endif - LIB_SRCS += lib/api.c LIB_SRCS += $(wildcard lib/api/*.c) diff --git a/lib/api/actions/capabiltities.c b/lib/api/actions/capabiltities.c index ff7048a9b..2e2dbbb7e 100644 --- a/lib/api/actions/capabiltities.c +++ b/lib/api/actions/capabiltities.c @@ -27,6 +27,7 @@ static int api_capabilities(struct api_action *h, json_t *args, json_t **resp, s json_t *json_hooks = json_array(); json_t *json_apis = json_array(); json_t *json_nodes = json_array(); + json_t *json_ios = json_array(); json_t *json_name; for (size_t i = 0; i < list_length(&plugins); i++) { @@ -38,15 +39,17 @@ static int api_capabilities(struct api_action *h, json_t *args, json_t **resp, s case PLUGIN_TYPE_NODE: json_array_append_new(json_nodes, json_name); break; case PLUGIN_TYPE_HOOK: json_array_append_new(json_hooks, json_name); break; case PLUGIN_TYPE_API: json_array_append_new(json_apis, json_name); break; - default: { } + case PLUGIN_TYPE_IO: json_array_append_new(json_ios, json_name); break; + default: json_decref(json_name); } } - *resp = json_pack("{ s: s, s: o, s: o, s: o }", + *resp = json_pack("{ s: s, s: o, s: o, s: o, s: o }", "build", BUILDID, "hooks", json_hooks, "node-types", json_nodes, - "apis", json_apis); + "apis", json_apis, + "formats", json_ios); return 0; } diff --git a/lib/api/actions/config.c b/lib/api/actions/config.c index db42997d1..6eb977051 100644 --- a/lib/api/actions/config.c +++ b/lib/api/actions/config.c @@ -20,19 +20,14 @@ * along with this program. If not, see . *********************************************************************************/ -#include - #include "api.h" #include "utils.h" #include "plugin.h" -#include "config_helper.h" #include "super_node.h" static int api_config(struct api_action *h, json_t *args, json_t **resp, struct api_session *s) { - config_setting_t *cfg_root = config_root_setting(&s->api->super_node->cfg); - - *resp = cfg_root ? config_to_json(cfg_root) : json_object(); + *resp = s->api->super_node->cfg; return 0; } diff --git a/lib/api/actions/nodes.c b/lib/api/actions/nodes.c index 121dd9dce..d5679d2aa 100644 --- a/lib/api/actions/nodes.c +++ b/lib/api/actions/nodes.c @@ -26,8 +26,6 @@ #include "node.h" #include "super_node.h" #include "utils.h" -#include "config_helper.h" - #include "api.h" static int api_nodes(struct api_action *r, json_t *args, json_t **resp, struct api_session *s) @@ -47,7 +45,7 @@ static int api_nodes(struct api_action *r, json_t *args, json_t **resp, struct a /* Add all additional fields of node here. * This can be used for metadata */ - json_object_update(json_node, config_to_json(n->cfg)); + json_object_update(json_node, n->cfg); json_array_append_new(json_nodes, json_node); } diff --git a/lib/api/actions/paths.c b/lib/api/actions/paths.c index e89e21ce5..922ebe067 100644 --- a/lib/api/actions/paths.c +++ b/lib/api/actions/paths.c @@ -25,7 +25,6 @@ #include "plugin.h" #include "path.h" #include "utils.h" -#include "config_helper.h" #include "stats.h" #include "super_node.h" @@ -47,7 +46,7 @@ static int api_paths(struct api_action *r, json_t *args, json_t **resp, struct a /* Add all additional fields of node here. * This can be used for metadata */ - json_object_update(json_path, config_to_json(p->cfg)); + json_object_update(json_path, p->cfg); json_array_append_new(json_paths, json_path); } diff --git a/lib/api/actions/restart.c b/lib/api/actions/restart.c index b9473a950..b11bf5e64 100644 --- a/lib/api/actions/restart.c +++ b/lib/api/actions/restart.c @@ -64,12 +64,10 @@ static int api_restart(struct api_action *h, json_t *args, json_t **resp, struct /* We pass some env variables to the new process */ setenv("VILLAS_API_RESTART_COUNT", buf, 1); - setenv("VILLAS_API_RESTART_REMOTE", s->peer.ip, 1); - *resp = json_pack("{ s: i, s: s, s: s }", + *resp = json_pack("{ s: i, s: s }", "restarts", cnt, - "config", config, - "remote", s->peer.ip + "config", config ); /* Register exit handler */ diff --git a/lib/api/actions/shutdown.c b/lib/api/actions/shutdown.c index 36e670ebc..c99760de6 100644 --- a/lib/api/actions/shutdown.c +++ b/lib/api/actions/shutdown.c @@ -32,7 +32,7 @@ static int api_shutdown(struct api_action *h, json_t *args, json_t **resp, struc static struct plugin p = { .name = "shutdown", - .description = "stop VILLASnode", + .description = "quit VILLASnode", .type = PLUGIN_TYPE_API, .api.cb = api_shutdown }; diff --git a/lib/api/session.c b/lib/api/session.c index ee27f4064..3d05d385c 100644 --- a/lib/api/session.c +++ b/lib/api/session.c @@ -24,33 +24,61 @@ #include "web.h" #include "plugin.h" +#include "memory.h" -int api_session_init(struct api_session *s, struct api *a, enum api_mode m) +int api_session_init(struct api_session *s, enum api_mode m) { + int ret; + + s->runs = 0; s->mode = m; - s->api = a; + + ret = buffer_init(&s->request.buffer, 0); + if (ret) + return ret; + + ret = buffer_init(&s->response.buffer, 0); + if (ret) + return ret; + + ret = queue_init(&s->request.queue, 128, &memtype_heap); + if (ret) + return ret; - s->completed = false; - - web_buffer_init(&s->request.body, s->mode == API_MODE_HTTP ? LWS_WRITE_HTTP : LWS_WRITE_TEXT); - web_buffer_init(&s->response.body, s->mode == API_MODE_HTTP ? LWS_WRITE_HTTP : LWS_WRITE_TEXT); - - if (s->mode == API_MODE_HTTP) - web_buffer_init(&s->response.headers, LWS_WRITE_HTTP_HEADERS); + ret = queue_init(&s->response.queue, 128, &memtype_heap); + if (ret) + return ret; + + s->_name = NULL; return 0; } int api_session_destroy(struct api_session *s) { + int ret; + if (s->state == STATE_DESTROYED) return 0; - web_buffer_destroy(&s->request.body); - web_buffer_destroy(&s->response.body); + ret = buffer_destroy(&s->request.buffer); + if (ret) + return ret; - if (s->mode == API_MODE_HTTP) - web_buffer_destroy(&s->response.headers); + ret = buffer_destroy(&s->response.buffer); + if (ret) + return ret; + + ret = queue_destroy(&s->request.queue); + if (ret) + return ret; + + ret = queue_destroy(&s->response.queue); + if (ret) + return ret; + + if (s->_name) + free(s->_name); s->state = STATE_DESTROYED; @@ -90,7 +118,7 @@ int api_session_run_command(struct api_session *s, json_t *json_in, json_t **jso goto out; } - debug(LOG_API, "Running API request: %s", p->name); + debug(LOG_API, "Running API request: action=%s, id=%s", action, id); ret = p->api.cb(&p->api, json_args, &json_resp, s); if (ret) @@ -107,7 +135,31 @@ int api_session_run_command(struct api_session *s, json_t *json_in, json_t **jso if (json_resp) json_object_set(*json_out, "response", json_resp); -out: debug(LOG_API, "API request completed with code: %d", ret); +out: debug(LOG_API, "Completed API request: action=%s, id=%s, code=%d", action, id, ret); + + s->runs++; return 0; } + +char * api_session_name(struct api_session *s) +{ + if (!s->_name) { + char *mode; + + switch (s->mode) { + case API_MODE_WS: mode = "ws"; break; + case API_MODE_HTTP: mode = "http"; break; + default: mode = "unknown"; break; + } + + char name[128]; + char ip[128]; + + lws_get_peer_addresses(s->wsi, lws_get_socket_fd(s->wsi), name, sizeof(name), ip, sizeof(ip)); + + s->_name = strcatf(&s->_name, "version=%d, mode=%s, runs=%d, remote.name=%s, remote.ip=%s", s->version, mode, s->runs, name, ip); + } + + return s->_name; +} diff --git a/lib/buffer.c b/lib/buffer.c new file mode 100644 index 000000000..439952b8a --- /dev/null +++ b/lib/buffer.c @@ -0,0 +1,99 @@ +/** A simple growing buffer. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include "buffer.h" +#include "common.h" + +int buffer_init(struct buffer *b, size_t size) +{ + b->len = 0; + b->size = size; + b->buf = malloc(size); + if (!b->buf) + return -1; + + b->state = STATE_INITIALIZED; + + return 0; +} + +int buffer_destroy(struct buffer *b) +{ + if (b->buf) + free(b->buf); + + b->state = STATE_DESTROYED; + + return 0; +} + +void buffer_clear(struct buffer *b) +{ + b->len = 0; +} + +int buffer_append(struct buffer *b, const char *data, size_t len) +{ + if (b->len + len > b->size) { + b->size = b->len + len; + b->buf = realloc(b->buf, b->size); + if (!b->buf) + return -1; + } + + memcpy(b->buf + b->len, data, len); + + b->len += len; + + return 0; +} + +int buffer_parse_json(struct buffer *b, json_t **j) +{ + *j = json_loadb(b->buf, b->len, 0, NULL); + if (!*j) + return -1; + + return 0; +} + +int buffer_append_json(struct buffer *b, json_t *j) +{ + size_t len; + +retry: len = json_dumpb(j, b->buf + b->len, b->size - b->len, 0); + if (b->size < b->len + len) { + b->buf = realloc(b->buf, b->len + len); + if (!b->buf) + return -1; + + b->size = b->len + len; + goto retry; + } + + b->len += len; + + return 0; +} diff --git a/lib/config_helper.c b/lib/config_helper.c index 2353c2123..5ee6dc735 100644 --- a/lib/config_helper.c +++ b/lib/config_helper.c @@ -20,10 +20,13 @@ * along with this program. If not, see . *********************************************************************************/ +#include + #include "config_helper.h" #include "utils.h" -#ifdef WITH_JSON +#ifdef WITH_LIBCONFIG + static int json_to_config_type(int type) { switch (type) { @@ -139,24 +142,180 @@ int json_to_config(json_t *json, config_setting_t *parent) return 0; } -#endif /* WITH_JSON */ +#endif /* WITH_LIBCONFIG */ -int config_read_cli(config_t *cfg, int argc, char *argv[]) +void json_object_extend_key_value(json_t *obj, const char *key, const char *value) { - int ret; - char *str = NULL; + char *end, *cpy, *key1, *key2; - for (int i = 0; i < argc; i++) - str = strcatf(&str, "%s", argv[i]); + double real; + long integer; - if (!str) - return 0; + json_t *arr, *new, *existing, *subobj; - config_set_auto_convert(cfg, 1); + /* Is the key pointing to an object? */ + subobj = obj; + cpy = strdup(key); - ret = config_read_string(cfg, str); + key1 = strtok(cpy, "."); + key2 = strtok(NULL, "."); - free(str); + while (key1 && key2) { + existing = json_object_get(subobj, key1); + if (existing) + subobj = existing; + else { + new = json_object(); + json_object_set(subobj, key1, new); - return ret != CONFIG_TRUE; + subobj = new; + } + + key1 = key2; + key2 = strtok(NULL, "."); + } + + /* Try to parse as integer */ + integer = strtol(value, &end, 0); + if (*end == 0) { + new = json_integer(integer); + goto success; + } + + /* Try to parse as floating point */ + real = strtod(value, &end); + if (*end == 0) { + new = json_real(real); + goto success; + } + + /* Try to parse special types */ + if (!strcmp(value, "true")) { + new = json_true(); + goto success; + } + + if (!strcmp(value, "false")) { + new = json_false(); + goto success; + } + + if (!strcmp(value, "null")) { + new = json_null(); + goto success; + } + + /* Fallback to string */ + new = json_string(value); + +success: + /* Does the key already exist? + * If yes, transform to array. */ + existing = json_object_get(subobj, key1); + if (existing) { + if (json_is_array(existing)) + arr = existing; + else { + arr = json_array(); + json_object_set(subobj, key1, arr); + json_array_append(arr, existing); + } + + json_array_append(arr, new); + } + else + json_object_set(subobj, key1, new); +} + +json_t * json_load_cli(int argc, char *argv[]) +{ + char *opt; + char *key = NULL; + char *value = NULL; + char *sep, *cpy; + + json_t *json = json_object(); + + for (int i = 1; i < argc; i++) { + opt = argv[i]; + + /* Long Option */ + if (opt[0] == '-' && opt[1] == '-') { + /* Option without value? Abort. */ + if (key != NULL) + return NULL; + + key = opt + 2; + + /* Does this option has the form "--option=value"? */ + sep = strchr(key, '='); + if (sep) { + cpy = strdup(key); + + key = strtok(cpy, "="); + value = strtok(NULL, ""); + + json_object_extend_key_value(json, key, value); + + free(cpy); + key = NULL; + } + } + /* Value */ + else { + /* Value without key. Abort. */ + if (key == NULL) + return NULL; + + value = opt; + + json_object_extend_key_value(json, key, value); + key = NULL; + } + } + + return json; +} + +int json_object_extend(json_t *obj, json_t *merge) +{ + const char *key; + void *tmp; + int ret; + json_t *merge_value, *obj_value; + + if (!json_is_object(obj) || !json_is_object(merge)) + return -1; + + json_object_foreach_safe(merge, tmp, key, merge_value) { + obj_value = json_object_get(obj, key); + if (obj_value && json_is_object(obj_value)) + ret = json_object_extend(obj_value, merge_value); + else + ret = json_object_set(obj, key, merge_value); + + if (ret) + return ret; + } + + return 0; +} + +int json_object_extend_str(json_t *obj, const char *str) +{ + char *key, *value, *cpy; + + cpy = strdup(str); + + key = strtok(cpy, "="); + value = strtok(NULL, ""); + + if (!key || !value) + return -1; + + json_object_extend_key_value(obj, key, value); + + free(cpy); + + return 0; } diff --git a/lib/fpga/card.c b/lib/fpga/card.c index 4f9c3f8ea..d0aa8c566 100644 --- a/lib/fpga/card.c +++ b/lib/fpga/card.c @@ -55,48 +55,59 @@ int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container * return 0; } -int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg) +int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name) { int ret; - const char *slot, *id, *err; - config_setting_t *cfg_ips, *cfg_slot, *cfg_id; - c->name = config_setting_name(cfg); + json_t *cfg_ips; + json_t *cfg_slot = NULL; + json_t *cfg_id = NULL; + json_error_t err; - config_setting_lookup_int(cfg, "affinity", &c->affinity); - config_setting_lookup_bool(cfg, "do_reset", &c->do_reset); + c->name = strdup(name); + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: b, s?: o, s?: o, s: o }" + "affinity", &c->affinity, + "do_reset", &c->do_reset, + "slot", &cfg_slot, + "id", &cfg_id, + "ips", &cfg_ips + ); + if (ret) + jerror(&err, "Failed to parse FPGA vard configuration"); - cfg_slot = config_setting_get_member(cfg, "slot"); if (cfg_slot) { - slot = config_setting_get_string(cfg_slot); + const char *err, *slot; + + slot = json_string_value(cfg_slot); if (slot) { ret = pci_device_parse_slot(&c->filter, slot, &err); if (ret) - cerror(cfg_slot, "Failed to parse PCI slot: %s", err); + error("Failed to parse PCI slot: %s", err); } else - cerror(cfg_slot, "PCI slot must be a string"); + error("PCI slot must be a string"); } - cfg_id = config_setting_get_member(cfg, "id"); if (cfg_id) { - id = config_setting_get_string(cfg_id); + const char *err, *id; + + id = json_string_value(cfg_id); if (id) { ret = pci_device_parse_id(&c->filter, (char*) id, &err); if (ret) - cerror(cfg_id, "Failed to parse PCI id: %s", err); + error("Failed to parse PCI id: %s", err); } else - cerror(cfg_slot, "PCI ID must be a string"); + error("PCI ID must be a string"); } - cfg_ips = config_setting_get_member(cfg, "ips"); - if (!cfg_ips) - cerror(cfg, "FPGA configuration is missing ips section"); - - for (int i = 0; i < config_setting_length(cfg_ips); i++) { - config_setting_t *cfg_ip = config_setting_get_elem(cfg_ips, i); + if (!json_is_object(cfg_ips)) + error("FPGA card IPs section must be an object"); + const char *name_ip; + json_t *cfg_ip; + json_object_foreach(cfg_ips, name_ip, cfg_ip) { const char *vlnv; struct fpga_ip_type *vt; @@ -104,45 +115,45 @@ int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg) ip->card = c; - if (!config_setting_lookup_string(cfg, "vlnv", &vlnv)) - cerror(cfg, "FPGA IP core %s is missing the VLNV identifier", c->name); + ret = json_unpack_ex(cfg_ip, &err, 0, "{ s: s }", "vlnv", &vlnv); + if (ret) + error("Failed to parse FPGA IP '%s' of card '%s'", name_ip, name); vt = fpga_ip_type_lookup(vlnv); if (!vt) - cerror(cfg, "FPGA IP core VLNV identifier '%s' is invalid", vlnv); + error("FPGA IP core VLNV identifier '%s' is invalid", vlnv); ret = fpga_ip_init(ip, vt); if (ret) error("Failed to initalize FPGA IP core"); - ret = fpga_ip_parse(ip, cfg_ip); + ret = fpga_ip_parse(ip, cfg_ip, name_ip); if (ret) - cerror(cfg_ip, "Failed to parse FPGA IP core"); + error("Failed to parse FPGA IP core"); list_push(&c->ips, ip); } - c->cfg = cfg; c->state = STATE_PARSED; return 0; } -int fpga_card_parse_list(struct list *cards, config_setting_t *cfg) +int fpga_card_parse_list(struct list *cards, json_t *cfg) { int ret; - if (!config_setting_is_group(cfg)) - cerror(cfg, "FPGA configuration section must be a group"); - - for (int i = 0; i < config_setting_length(cfg); i++) { - config_setting_t *cfg_fpga = config_setting_get_elem(cfg, i); + if (!json_is_object(cfg)) + error("FPGA card configuration section must be a JSON object"); + const char *name; + json_t *cfg_fpga; + json_object_foreach(cfg, name, cfg_fpga) { struct fpga_card *c = alloc(sizeof(struct fpga_card)); - ret = fpga_card_parse(c, cfg_fpga); + ret = fpga_card_parse(c, cfg_fpga, name); if (ret) - cerror(cfg_fpga, "Failed to parse FPGA card configuration"); + error("Failed to parse FPGA card configuration"); list_push(cards, c); } diff --git a/lib/fpga/ip.c b/lib/fpga/ip.c index b407d6c36..de83738ef 100644 --- a/lib/fpga/ip.c +++ b/lib/fpga/ip.c @@ -20,8 +20,6 @@ * along with this program. If not, see . *********************************************************************************/ -#include - #include "log_config.h" #include "log.h" #include "plugin.h" @@ -46,34 +44,33 @@ int fpga_ip_init(struct fpga_ip *c, struct fpga_ip_type *vt) return ret; } -int fpga_ip_parse(struct fpga_ip *c, config_setting_t *cfg) +int fpga_ip_parse(struct fpga_ip *c, json_t *cfg, const char *name) { - int ret; - long long baseaddr; + int ret, baseaddr = -1; assert(c->state != STATE_STARTED && c->state != STATE_DESTROYED); - c->cfg = cfg; + c->name = strdup(name); + c->baseaddr = -1; + c->irq = -1; + c->port = -1; - c->name = config_setting_name(cfg); - if (!c->name) - cerror(cfg, "IP is missing a name"); + json_error_t err; - /* Common settings */ - if (config_setting_lookup_int64(cfg, "baseaddr", &baseaddr)) - c->baseaddr = baseaddr; - else - c->baseaddr = -1; + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: i, s?: i }", + "baseaddr", &baseaddr, + "irq", &c->irq, + "port", &c->port + ); + if (ret) + jerror(&err, "Failed to parse configuration for FPGA IP '%s'", name); - if (!config_setting_lookup_int(cfg, "irq", &c->irq)) - c->irq = -1; - if (!config_setting_lookup_int(cfg, "port", &c->port)) - c->port = -1; + c->baseaddr = baseaddr; /* Type sepecific settings */ - ret = c->_vt && c->_vt->parse ? c->_vt->parse(c) : 0; + ret = c->_vt && c->_vt->parse ? c->_vt->parse(c, cfg) : 0; if (ret) - error("Failed to parse settings for IP core '%s'", c->name); + error("Failed to parse settings for IP core '%s'", name); c->state = STATE_PARSED; diff --git a/lib/fpga/ips/dft.c b/lib/fpga/ips/dft.c index c3c1ec809..9657287e6 100644 --- a/lib/fpga/ips/dft.c +++ b/lib/fpga/ips/dft.c @@ -13,33 +13,40 @@ #include "fpga/card.h" #include "fpga/ips/dft.h" -int dft_parse(struct fpga_ip *c) +int dft_parse(struct fpga_ip *c, json_t *cfg) { struct dft *dft = c->_vd; - config_setting_t *cfg_harms; + int ret; - if (!config_setting_lookup_int(c->cfg, "period", &dft->period)) - cerror(c->cfg, "DFT IP core requires 'period' setting"); + json_t *cfg_harms; + json_error_t err; - if (!config_setting_lookup_int(c->cfg, "decimation", &dft->decimation)) - cerror(c->cfg, "DFT IP core requires 'decimation' setting"); + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: i, s: o }", + "period", &dft->period, + "decimation", &dft->decimation, + "harmonics", &cfg_harms + ); + if (ret) + jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); - cfg_harms = config_setting_get_member(c->cfg, "harmonics"); - if (!cfg_harms) - cerror(c->cfg, "DFT IP core requires 'harmonics' setting!"); + if (!json_is_array(cfg_harms)) + error("DFT IP core requires 'harmonics' to be an array of integers!"); - if (!config_setting_is_array(cfg_harms)) - cerror(c->cfg, "DFT IP core requires 'harmonics' to be an array of integers!"); - - dft->num_harmonics = config_setting_length(cfg_harms); + dft->num_harmonics = json_array_size(cfg_harms); if (dft->num_harmonics <= 0) - cerror(cfg_harms, "DFT IP core requires 'harmonics' to contain at least 1 integer!"); + error("DFT IP core requires 'harmonics' to contain at least 1 value!"); dft->fharmonics = alloc(sizeof(float) * dft->num_harmonics); - for (int i = 0; i < dft->num_harmonics; i++) - dft->fharmonics[i] = (float) config_setting_get_int_elem(cfg_harms, i) / dft->period; + size_t index; + json_t *cfg_harm; + json_array_foreach(cfg_harms, index, cfg_harm) { + if (!json_is_real(cfg_harm)) + error("DFT IP core requires all 'harmonics' values to be of floating point type"); + + dft->fharmonics[index] = (float) json_number_value(cfg_harm) / dft->period; + } return 0; } diff --git a/lib/fpga/ips/fifo.c b/lib/fpga/ips/fifo.c index 4ab7a9c5b..efe387e2f 100644 --- a/lib/fpga/ips/fifo.c +++ b/lib/fpga/ips/fifo.c @@ -91,16 +91,21 @@ ssize_t fifo_read(struct fpga_ip *c, char *buf, size_t len) return nextlen; } -int fifo_parse(struct fpga_ip *c) +int fifo_parse(struct fpga_ip *c, json_t *cfg) { struct fifo *fifo = c->_vd; - int baseaddr_axi4; + int baseaddr_axi4 = -1, ret; - if (config_setting_lookup_int(c->cfg, "baseaddr_axi4", &baseaddr_axi4)) - fifo->baseaddr_axi4 = baseaddr_axi4; - else - fifo->baseaddr_axi4 = -1; + json_error_t err; + + fifo->baseaddr_axi4 = -1; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i }", "baseaddr_axi4", &baseaddr_axi4); + if (ret) + jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); + + fifo->baseaddr_axi4 = baseaddr_axi4; return 0; } diff --git a/lib/fpga/ips/model.c b/lib/fpga/ips/model.c index a54781904..f8a0cb770 100644 --- a/lib/fpga/ips/model.c +++ b/lib/fpga/ips/model.c @@ -18,7 +18,7 @@ #include "fpga/card.h" #include "fpga/ips/model.h" -static int model_param_destroy(struct model_param *p) +static int model_parameter_destroy(struct model_parameter *p) { free(p->name); @@ -64,7 +64,7 @@ static int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameter if (length < 4) break; /* block is to small to describe a gateway */ - struct model_param *e, *p = alloc(sizeof(struct model_param)); + struct model_parameter *e, *p = alloc(sizeof(struct model_parameter)); p->name = copy_string(3); p->default_value.flt = *((float *) &data[1]); @@ -75,7 +75,7 @@ static int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameter e = list_lookup(parameters, p->name); if (e) - model_param_update(e, p); + model_parameter_update(e, p); else list_push(parameters, p); break; @@ -133,28 +133,40 @@ static int model_xsg_map_read(uint32_t *map, size_t len, void *baseaddr) return i; } -int model_parse(struct fpga_ip *c) +int model_parse(struct fpga_ip *c, json_t *cfg) { struct model *m = c->_vd; - config_setting_t *cfg_params, *cfg_param; + int ret; + + json_t *cfg_params; + json_error_t err; if (strcmp(c->vlnv.library, "hls") == 0) m->type = MODEL_TYPE_HLS; else if (strcmp(c->vlnv.library, "sysgen") == 0) m->type = MODEL_TYPE_XSG; else - cerror(c->cfg, "Invalid model type: %s", c->vlnv.library); + error("Unsupported model type: %s", c->vlnv.library); + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o }", "parameters", &cfg_params); + if (ret) + jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); - cfg_params = config_setting_get_member(c->cfg, "parameters"); if (cfg_params) { - for (int i = 0; i < config_setting_length(cfg_params); i++) { - cfg_param = config_setting_get_elem(cfg_params, i); + if (!json_is_object(cfg_params)) + error("Setting 'parameters' must be a JSON object"); - struct model_param *p = alloc(sizeof(struct model_param)); + const char *name; + json_t *value; + json_object_foreach(cfg_params, name, value) { + if (!json_is_real(value)) + error("Parameters of FPGA IP '%s' must be of type floating point", c->name); - p->name = config_setting_name(cfg_param); - p->default_value.flt = config_setting_get_float(cfg_param); + struct model_parameter *p = alloc(sizeof(struct model_parameter)); + + p->name = strdup(name); + p->default_value.flt = json_real_value(value); list_push(&m->parameters, p); } @@ -206,12 +218,12 @@ int model_init(struct fpga_ip *c) /* Set default values for parameters */ for (size_t i = 0; i < list_length(&m->parameters); i++) { - struct model_param *p = list_at(&m->parameters, i); + struct model_parameter *p = list_at(&m->parameters, i); p->ip = c; - if (p->direction == MODEL_PARAM_IN) { - model_param_write(p, p->default_value.flt); + if (p->direction == MODEL_PARAMETER_IN) { + model_parameter_write(p, p->default_value.flt); info("Set parameter '%s' updated to default value: %f", p->name, p->default_value.flt); } } @@ -226,7 +238,7 @@ int model_destroy(struct fpga_ip *c) { struct model *m = c->_vd; - list_destroy(&m->parameters, (dtor_cb_t) model_param_destroy, true); + list_destroy(&m->parameters, (dtor_cb_t) model_parameter_destroy, true); list_destroy(&m->infos, (dtor_cb_t) model_info_destroy, true); if (m->xsg.map != NULL) @@ -245,9 +257,9 @@ void model_dump(struct fpga_ip *c) { INDENT info("Parameters:"); for (size_t i = 0; i < list_length(&m->parameters); i++) { INDENT - struct model_param *p = list_at(&m->parameters, i); + struct model_parameter *p = list_at(&m->parameters, i); - if (p->direction == MODEL_PARAM_IN) + if (p->direction == MODEL_PARAMETER_IN) info("%#jx: %s (%s) = %.3f %s %u", p->offset, p->name, @@ -256,7 +268,7 @@ void model_dump(struct fpga_ip *c) param_type[p->type], p->binpt ); - else if (p->direction == MODEL_PARAM_OUT) + else if (p->direction == MODEL_PARAMETER_OUT) info("%#jx: %s (%s)", p->offset, p->name, @@ -273,52 +285,52 @@ void model_dump(struct fpga_ip *c) } } -int model_param_read(struct model_param *p, double *v) +int model_parameter_read(struct model_parameter *p, double *v) { struct fpga_ip *c = p->ip; - union model_param_value *ptr = (union model_param_value *) (c->card->map + c->baseaddr + p->offset); + union model_parameter_value *ptr = (union model_parameter_value *) (c->card->map + c->baseaddr + p->offset); switch (p->type) { - case MODEL_PARAM_TYPE_UFIX: + case MODEL_PARAMETER_TYPE_UFIX: *v = (double) ptr->ufix / (1 << p->binpt); break; - case MODEL_PARAM_TYPE_FIX: + case MODEL_PARAMETER_TYPE_FIX: *v = (double) ptr->fix / (1 << p->binpt); break; - case MODEL_PARAM_TYPE_FLOAT: + case MODEL_PARAMETER_TYPE_FLOAT: *v = (double) ptr->flt; break; - case MODEL_PARAM_TYPE_BOOLEAN: + case MODEL_PARAMETER_TYPE_BOOLEAN: *v = (double) ptr->ufix ? 1 : 0; } return 0; } -int model_param_write(struct model_param *p, double v) +int model_parameter_write(struct model_parameter *p, double v) { struct fpga_ip *c = p->ip; - union model_param_value *ptr = (union model_param_value *) (c->card->map + c->baseaddr + p->offset); + union model_parameter_value *ptr = (union model_parameter_value *) (c->card->map + c->baseaddr + p->offset); switch (p->type) { - case MODEL_PARAM_TYPE_UFIX: + case MODEL_PARAMETER_TYPE_UFIX: ptr->ufix = (uint32_t) (v * (1 << p->binpt)); break; - case MODEL_PARAM_TYPE_FIX: + case MODEL_PARAMETER_TYPE_FIX: ptr->fix = (int32_t) (v * (1 << p->binpt)); break; - case MODEL_PARAM_TYPE_FLOAT: + case MODEL_PARAMETER_TYPE_FLOAT: ptr->flt = (float) v; break; - case MODEL_PARAM_TYPE_BOOLEAN: + case MODEL_PARAMETER_TYPE_BOOLEAN: ptr->bol = (bool) v; break; } @@ -326,10 +338,10 @@ int model_param_write(struct model_param *p, double v) return 0; } -void model_param_add(struct fpga_ip *c, const char *name, enum model_param_direction dir, enum model_param_type type) +void model_parameter_add(struct fpga_ip *c, const char *name, enum model_parameter_direction dir, enum model_parameter_type type) { struct model *m = c->_vd; - struct model_param *p = alloc(sizeof(struct model_param)); + struct model_parameter *p = alloc(sizeof(struct model_parameter)); p->name = strdup(name); p->type = type; @@ -338,10 +350,10 @@ void model_param_add(struct fpga_ip *c, const char *name, enum model_param_direc list_push(&m->parameters, p); } -int model_param_remove(struct fpga_ip *c, const char *name) +int model_parameter_remove(struct fpga_ip *c, const char *name) { struct model *m = c->_vd; - struct model_param *p; + struct model_parameter *p; p = list_lookup(&m->parameters, name); if (!p) @@ -352,7 +364,7 @@ int model_param_remove(struct fpga_ip *c, const char *name) return 0; } -int model_param_update(struct model_param *p, struct model_param *u) +int model_parameter_update(struct model_parameter *p, struct model_parameter *u) { if (strcmp(p->name, u->name) != 0) return -1; diff --git a/lib/fpga/ips/switch.c b/lib/fpga/ips/switch.c index a901fc740..04f50b438 100644 --- a/lib/fpga/ips/switch.c +++ b/lib/fpga/ips/switch.c @@ -88,43 +88,41 @@ int switch_destroy(struct fpga_ip *c) return 0; } -int switch_parse(struct fpga_ip *c) +int switch_parse(struct fpga_ip *c, json_t *cfg) { - struct fpga_card *f = c->card; struct sw *sw = c->_vd; + int ret; + size_t index; + json_error_t err; + json_t *cfg_path, *cfg_paths = NULL; + list_init(&sw->paths); - config_setting_t *cfg_sw, *cfg_path; + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: o }", + "num_ports", &sw->num_ports, + "paths", &cfg_paths + ); + if (ret) + jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); - if (!config_setting_lookup_int(c->cfg, "num_ports", &sw->num_ports)) - cerror(c->cfg, "Switch IP '%s' requires 'num_ports' option", c->name); - - cfg_sw = config_setting_get_member(f->cfg, "paths"); - if (!cfg_sw) + if (!cfg_paths) return 0; /* no switch config available */ - for (int i = 0; i < config_setting_length(cfg_sw); i++) { - cfg_path = config_setting_get_elem(cfg_sw, i); + if (!json_is_array(cfg_paths)) + error("Setting 'paths' of FPGA IP '%s' should be an array of JSON objects", c->name); + json_array_foreach(cfg_paths, index, cfg_path) { struct sw_path *p = alloc(sizeof(struct sw_path)); - int reverse; + int reverse = 0; - if (!config_setting_lookup_bool(cfg_path, "reverse", &reverse)) - reverse = 0; - - if (!config_setting_lookup_string(cfg_path, "in", &p->in) && - !config_setting_lookup_string(cfg_path, "from", &p->in) && - !config_setting_lookup_string(cfg_path, "src", &p->in) && - !config_setting_lookup_string(cfg_path, "source", &p->in)) - cerror(cfg_path, "Path is missing 'in' setting"); - - if (!config_setting_lookup_string(cfg_path, "out", &p->out) && - !config_setting_lookup_string(cfg_path, "to", &p->out) && - !config_setting_lookup_string(cfg_path, "dst", &p->out) && - !config_setting_lookup_string(cfg_path, "dest", &p->out) && - !config_setting_lookup_string(cfg_path, "sink", &p->out)) - cerror(cfg_path, "Path is missing 'out' setting"); + ret = json_unpack_ex(cfg_path, &err, 0, "{ s?: b, s: s, s: s }", + "reverse", &reverse, + "in", &p->in, + "out", &p->out + ); + if (ret) + jerror(&err, "Failed to parse path %zu of FPGA IP '%s'", index, c->name); list_push(&sw->paths, p); diff --git a/lib/hist.c b/lib/hist.c index 448feffac..0b65d7efc 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -139,11 +139,11 @@ void hist_print(struct hist *h, int details) hist_cnt_t missed = h->total - h->higher - h->lower; stats("Counted values: %ju (%ju between %f and %f)", h->total, missed, h->low, h->high); - stats("Highest: %f", h->highest); - stats("Lowest: %f", h->lowest); - stats("Mu: %f", hist_mean(h)); - stats("Variance: %f", hist_var(h)); - stats("Stddev: %f", hist_stddev(h)); + stats("Highest: %g", h->highest); + stats("Lowest: %g", h->lowest); + stats("Mu: %g", hist_mean(h)); + stats("Variance: %g", hist_var(h)); + stats("Stddev: %g", hist_stddev(h)); if (details > 0 && h->total - h->higher - h->lower > 0) { char *buf = hist_dump(h); @@ -212,7 +212,6 @@ char * hist_dump(struct hist *h) return buf; } -#ifdef WITH_JSON json_t * hist_json(struct hist *h) { json_t *json_buckets, *json_hist; @@ -257,7 +256,6 @@ int hist_dump_json(struct hist *h, FILE *f) return ret; } -#endif /* WITH_JSON */ int hist_dump_matlab(struct hist *h, FILE *f) { diff --git a/lib/hook.c b/lib/hook.c index f286b2dc3..6a8d0e848 100644 --- a/lib/hook.c +++ b/lib/hook.c @@ -21,11 +21,10 @@ *********************************************************************************/ #include #include -#include #include "timing.h" #include "config.h" -#include "msg.h" +#include "io/msg.h" #include "hook.h" #include "path.h" #include "utils.h" @@ -54,18 +53,24 @@ int hook_init(struct hook *h, struct hook_type *vt, struct path *p) return 0; } -int hook_parse(struct hook *h, config_setting_t *cfg) +int hook_parse(struct hook *h, json_t *cfg) { int ret; + json_error_t err; assert(h->state != STATE_DESTROYED); - config_setting_lookup_int(cfg, "priority", &h->priority); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i }", + "priority", &h->priority + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); ret = h->_vt->parse ? h->_vt->parse(h, cfg) : 0; if (ret) return ret; + h->cfg = cfg; h->state = STATE_PARSED; return 0; @@ -83,21 +88,11 @@ int hook_parse_cli(struct hook *h, int argc, char *argv[]) h->state = STATE_PARSED; } else { - config_t cfg; - config_setting_t *cfg_root; + h->cfg = json_load_cli(argc, argv); + if (!h->cfg) + return -1; - config_init(&cfg); - - ret = config_read_cli(&cfg, argc, argv); - if (ret) - goto out; - - cfg_root = config_root_setting(&cfg); - - ret = hook_parse(h, cfg_root); - -out: - config_destroy(&cfg); + ret = hook_parse(h, h->cfg); } return ret; @@ -149,23 +144,23 @@ int hook_restart(struct hook *h) return h->_vt->restart ? h->_vt->restart(h) : 0; } -int hook_read(struct hook *h, struct sample *smps[], size_t *cnt) +int hook_read(struct hook *h, struct sample *smps[], unsigned *cnt) { - debug(LOG_HOOK | 10, "Running hook %s: type=read, priority=%d, cnt=%zu", plugin_name(h->_vt), h->priority, *cnt); + debug(LOG_HOOK | 10, "Running hook %s: type=read, priority=%d, cnt=%u", plugin_name(h->_vt), h->priority, *cnt); return h->_vt->read ? h->_vt->read(h, smps, cnt) : 0; } -int hook_write(struct hook *h, struct sample *smps[], size_t *cnt) +int hook_write(struct hook *h, struct sample *smps[], unsigned *cnt) { - debug(LOG_HOOK | 10, "Running hook %s: type=write, priority=%d, cnt=%zu", plugin_name(h->_vt), h->priority, *cnt); + debug(LOG_HOOK | 10, "Running hook %s: type=write, priority=%d, cnt=%u", plugin_name(h->_vt), h->priority, *cnt); return h->_vt->write ? h->_vt->write(h, smps, cnt) : 0; } -size_t hook_read_list(struct list *hs, struct sample *smps[], size_t cnt) +int hook_read_list(struct list *hs, struct sample *smps[], unsigned cnt) { - size_t ret; + unsigned ret; for (size_t i = 0; i < list_length(hs); i++) { struct hook *h = list_at(hs, i); @@ -180,9 +175,9 @@ size_t hook_read_list(struct list *hs, struct sample *smps[], size_t cnt) return cnt; } -size_t hook_write_list(struct list *hs, struct sample *smps[], size_t cnt) +int hook_write_list(struct list *hs, struct sample *smps[], unsigned cnt) { - size_t ret; + unsigned ret; for (size_t i = 0; i < list_length(hs); i++) { struct hook *h = list_at(hs, i); @@ -205,38 +200,36 @@ int hook_cmp_priority(const void *a, const void *b) return ha->priority - hb->priority; } -int hook_parse_list(struct list *list, config_setting_t *cfg, struct path *o) +int hook_parse_list(struct list *list, json_t *cfg, struct path *o) { - struct plugin *p; + if (!json_is_array(cfg)) + error("Hooks must be configured as a list of objects"); - int ret; - const char *type; + size_t index; + json_t *cfg_hook; + json_array_foreach(cfg, index, cfg_hook) { + int ret; + const char *type; + struct plugin *p; + json_error_t err; - if (!config_setting_is_list(cfg)) - cerror(cfg, "Hooks must be configured as a list of objects"); - - for (int i = 0; i < config_setting_length(cfg); i++) { - config_setting_t *cfg_hook = config_setting_get_elem(cfg, i); - - if (!config_setting_is_group(cfg_hook)) - cerror(cfg_hook, "The 'hooks' setting must be an array of strings."); - - if (!config_setting_lookup_string(cfg_hook, "type", &type)) - cerror(cfg_hook, "Missing setting 'type' for hook"); + ret = json_unpack_ex(cfg_hook, &err, 0, "{ s: s }", "type", &type); + if (ret) + jerror(&err, "Failed to parse hook"); p = plugin_lookup(PLUGIN_TYPE_HOOK, type); if (!p) - continue; /* We ignore all non hook settings in this libconfig object setting */ + jerror(&err, "Unkown hook type '%s'", type); struct hook *h = alloc(sizeof(struct hook)); ret = hook_init(h, &p->hook, o); if (ret) - cerror(cfg_hook, "Failed to initialize hook"); + jerror(&err, "Failed to initialize hook"); ret = hook_parse(h, cfg_hook); if (ret) - cerror(cfg_hook, "Failed to parse hook configuration"); + jerror(&err, "Failed to parse hook configuration"); list_push(list, h); } diff --git a/lib/hooks/convert.c b/lib/hooks/convert.c index db83afe65..853e1815e 100644 --- a/lib/hooks/convert.c +++ b/lib/hooks/convert.c @@ -49,29 +49,33 @@ static int convert_init(struct hook *h) return 0; } -static int convert_parse(struct hook *h, config_setting_t *cfg) +static int convert_parse(struct hook *h, json_t *cfg) { struct convert *p = h->_vd; - const char *mode; + int ret; + json_error_t err; + const char *mode = NULL; - config_setting_lookup_float(cfg, "scale", &p->scale); - config_setting_lookup_int64(cfg, "mask", &p->mask); - - if (!config_setting_lookup_string(cfg, "mode", &mode)) - cerror(cfg, "Missing setting 'mode' for hook '%s'", plugin_name(h->_vt)); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: F, s?: i, s: s }", + "scale", &p->scale, + "mask", &p->mask, + "mode", &mode + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); if (!strcmp(mode, "fixed")) p->mode = TO_FIXED; else if (!strcmp(mode, "float")) p->mode = TO_FLOAT; else - error("Invalid parameter '%s' for hook 'convert'", mode); + error("Invalid 'mode' setting '%s' for hook '%s'", mode, plugin_name(h->_vt)); return 0; } -static int convert_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int convert_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct convert *p = h->_vd; diff --git a/lib/hooks/decimate.c b/lib/hooks/decimate.c index b97bcf670..58a74df80 100644 --- a/lib/hooks/decimate.c +++ b/lib/hooks/decimate.c @@ -41,20 +41,23 @@ static int decimate_init(struct hook *h) return 0; } -static int decimate_parse(struct hook *h, config_setting_t *cfg) +static int decimate_parse(struct hook *h, json_t *cfg) { struct decimate *p = h->_vd; - if (!cfg) - error("Missing configuration for hook: '%s'", plugin_name(h->_vt)); + int ret; + json_error_t err; - if (!config_setting_lookup_int(cfg, "ratio", &p->ratio)) - cerror(cfg, "Missing setting 'ratio' for hook '%s'", plugin_name(h->_vt)); + ret = json_unpack_ex(cfg, &err, 0, "{ s: i }", + "ratio", &p->ratio + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); return 0; } -static int decimate_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int decimate_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct decimate *p = h->_vd; diff --git a/lib/hooks/drop.c b/lib/hooks/drop.c index 844bb620d..531927f63 100644 --- a/lib/hooks/drop.c +++ b/lib/hooks/drop.c @@ -53,7 +53,7 @@ static int drop_stop(struct hook *h) return 0; } -static int drop_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int drop_read(struct hook *h, struct sample *smps[], unsigned *cnt) { int i, ok, dist; diff --git a/lib/hooks/fix_ts.c b/lib/hooks/fix_ts.c index 8fa2190f6..8c016dd58 100644 --- a/lib/hooks/fix_ts.c +++ b/lib/hooks/fix_ts.c @@ -29,7 +29,7 @@ #include "timing.h" #include "sample.h" -int fix_ts_read(struct hook *h, struct sample *smps[], size_t *cnt) +int fix_ts_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct timespec now; diff --git a/lib/hooks/jitter_calc.c b/lib/hooks/jitter_calc.c index 5d04faff6..de7f3d947 100644 --- a/lib/hooks/jitter_calc.c +++ b/lib/hooks/jitter_calc.c @@ -31,15 +31,52 @@ #include "timing.h" #include "sample.h" -#define CALC_GPS_NTP_DELAY 0 /* @todo move to global config file */ #define GPS_NTP_DELAY_WIN_SIZE 16 -static int64_t jitter_val[GPS_NTP_DELAY_WIN_SIZE] = {0}; -static int64_t delay_series[GPS_NTP_DELAY_WIN_SIZE] = {0}; -static int64_t moving_avg[GPS_NTP_DELAY_WIN_SIZE] = {0}; -static int64_t moving_var[GPS_NTP_DELAY_WIN_SIZE] = {0}; -static int64_t delay_mov_sum = 0, delay_mov_sum_sqrd = 0; -static int curr_count = 0; +struct jitter_calc { + int64_t *jitter_val; + int64_t *delay_series; + int64_t *moving_avg; + int64_t *moving_var; + int64_t delay_mov_sum; + int64_t delay_mov_sum_sqrd; + int curr_count; +}; + +int jitter_calc_init(struct hook *h) +{ + struct jitter_calc *j = h->_vd; + + size_t sz = GPS_NTP_DELAY_WIN_SIZE * sizeof(int64_t); + + j->jitter_val = alloc(sz); + j->delay_series = alloc(sz); + j->moving_avg = alloc(sz); + j->moving_var = alloc(sz); + + memset(j->jitter_val, 0, sz); + memset(j->delay_series, 0, sz); + memset(j->moving_avg, 0, sz); + memset(j->moving_var, 0, sz); + + j->delay_mov_sum = 0; + j->delay_mov_sum_sqrd = 0; + j->curr_count = 0; + + return 0; +} + +int jitter_calc_deinit(struct hook *h) +{ + struct jitter_calc *j = h->_vd; + + free(j->jitter_val); + free(j->delay_series); + free(j->moving_avg); + free(j->moving_var); + + return 0; +} /** * Hook to calculate jitter between GTNET-SKT GPS timestamp and Villas node NTP timestamp. @@ -49,9 +86,10 @@ static int curr_count = 0; * is high (i.e. several mins depending on GPS_NTP_DELAY_WIN_SIZE), * the variance value will overrun the 64bit value. */ -int hook_jitter_ts(struct hook *h, struct sample *smps[], size_t *cnt) +int jitter_calc_read(struct hook *h, struct sample *smps[], unsigned *cnt) { - /* @todo save data for each node, not just displaying on the screen, doesn't work for more than one node!!! */ + struct jitter_calc *j = h->_vd; + struct timespec now = time_now(); int64_t delay_sec, delay_nsec, curr_delay_us; @@ -60,44 +98,45 @@ int hook_jitter_ts(struct hook *h, struct sample *smps[], size_t *cnt) delay_nsec = now.tv_nsec - smps[i]->ts.origin.tv_nsec; /* Calc on microsec instead of nenosec delay as variance formula overflows otherwise.*/ - curr_delay_us = delay_sec*1000000 + delay_nsec/1000; + curr_delay_us = delay_sec * 1000000 + delay_nsec / 1000; - delay_mov_sum = delay_mov_sum + curr_delay_us - delay_series[curr_count]; - moving_avg[curr_count] = delay_mov_sum/(GPS_NTP_DELAY_WIN_SIZE); /* Will be valid after GPS_NTP_DELAY_WIN_SIZE initial values */ + j->delay_mov_sum = j->delay_mov_sum + curr_delay_us - j->delay_series[j->curr_count]; + j->moving_avg[j->curr_count] = j->delay_mov_sum / GPS_NTP_DELAY_WIN_SIZE; /* Will be valid after GPS_NTP_DELAY_WIN_SIZE initial values */ - delay_mov_sum_sqrd = delay_mov_sum_sqrd + (curr_delay_us*curr_delay_us) - (delay_series[curr_count]*delay_series[curr_count]); - moving_var[curr_count] = (delay_mov_sum_sqrd - (delay_mov_sum*delay_mov_sum)/GPS_NTP_DELAY_WIN_SIZE)/(GPS_NTP_DELAY_WIN_SIZE-1); + j->delay_mov_sum_sqrd = j->delay_mov_sum_sqrd + (curr_delay_us * curr_delay_us) - (j->delay_series[j->curr_count] * j->delay_series[j->curr_count]); + j->moving_var[j->curr_count] = (j->delay_mov_sum_sqrd - (j->delay_mov_sum * j->delay_mov_sum) / GPS_NTP_DELAY_WIN_SIZE) / (GPS_NTP_DELAY_WIN_SIZE - 1); - delay_series[curr_count] = curr_delay_us; /* Update the last delay value */ + j->delay_series[j->curr_count] = curr_delay_us; /* Update the last delay value */ /* Jitter calc formula as used in Wireshark according to RFC3550 (RTP) D(i,j) = (Rj-Ri)-(Sj-Si) = (Rj-Sj)-(Ri-Si) J(i) = J(i-1)+(|D(i-1,i)|-J(i-1))/16 */ - jitter_val[(curr_count+1)%GPS_NTP_DELAY_WIN_SIZE] = jitter_val[curr_count] + (labs(curr_delay_us) - jitter_val[curr_count])/16; + j->jitter_val[(j->curr_count + 1) % GPS_NTP_DELAY_WIN_SIZE] = j->jitter_val[j->curr_count] + (labs(curr_delay_us) - j->jitter_val[j->curr_count]) / 16; - stats("%s: jitter=%" PRId64 " usec, moving average=%" PRId64 " usec, moving variance=%" PRId64 " usec", __FUNCTION__, jitter_val[(curr_count+1)%GPS_NTP_DELAY_WIN_SIZE], moving_avg[curr_count], moving_var[curr_count]); + stats("%s: jitter=%" PRId64 " usec, moving average=%" PRId64 " usec, moving variance=%" PRId64 " usec", __FUNCTION__, j->jitter_val[(j->curr_count + 1) % GPS_NTP_DELAY_WIN_SIZE], j->moving_avg[j->curr_count], j->moving_var[j->curr_count]); - curr_count++; - if(curr_count >= GPS_NTP_DELAY_WIN_SIZE) - curr_count = 0; + j->curr_count++; + if(j->curr_count >= GPS_NTP_DELAY_WIN_SIZE) + j->curr_count = 0; } + return 0; } -#if CALC_GPS_NTP_DELAY == 1 static struct plugin p = { .name = "jitter_calc", .description = "Calc jitter, mean and variance of GPS vs NTP TS", .type = PLUGIN_TYPE_HOOK, .hook = { .priority = 0, - .builtin = true, - .read = hook_jitter_ts, + .init = jitter_calc_init, + .destroy= jitter_calc_deinit, + .read = jitter_calc_read, + .size = sizeof(struct jitter_calc) } }; REGISTER_PLUGIN(&p) -#endif /** @} */ diff --git a/lib/hooks/map.c b/lib/hooks/map.c index 1c203ccd8..7b75f02d7 100644 --- a/lib/hooks/map.c +++ b/lib/hooks/map.c @@ -51,15 +51,18 @@ static int map_destroy(struct hook *h) return mapping_destroy(&p->mapping); } -static int map_parse(struct hook *h, config_setting_t *cfg) +static int map_parse(struct hook *h, json_t *cfg) { int ret; struct map *p = h->_vd; - config_setting_t *cfg_mapping; + json_error_t err; + json_t *cfg_mapping; - cfg_mapping = config_setting_lookup(cfg, "mapping"); - if (!cfg_mapping || !config_setting_is_array(cfg_mapping)) - return -1; + ret = json_unpack_ex(cfg, &err, 0, "{ s: o }", + "map", &cfg_mapping + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); ret = mapping_parse(&p->mapping, cfg_mapping); if (ret) @@ -68,7 +71,7 @@ static int map_parse(struct hook *h, config_setting_t *cfg) return 0; } -static int map_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int map_read(struct hook *h, struct sample *smps[], unsigned *cnt) { int ret; struct map *p = h->_vd; diff --git a/lib/hooks/print.c b/lib/hooks/print.c index a2e403d85..f00338510 100644 --- a/lib/hooks/print.c +++ b/lib/hooks/print.c @@ -27,10 +27,12 @@ #include "hook.h" #include "plugin.h" #include "sample.h" -#include "sample_io.h" +#include "io.h" struct print { - FILE *output; + struct io io; + struct io_format *format; + const char *uri; }; @@ -38,8 +40,8 @@ static int print_init(struct hook *h) { struct print *p = h->_vd; - p->output = stdout; p->uri = NULL; + p->format = io_format_lookup("villas"); return 0; } @@ -47,12 +49,15 @@ static int print_init(struct hook *h) static int print_start(struct hook *h) { struct print *p = h->_vd; + int ret; - if (p->uri) { - p->output = fopen(p->uri, "w+"); - if (!p->output) - error("Failed to open file %s for writing", p->uri); - } + ret = io_init(&p->io, p->format, IO_FORMAT_ALL); + if (ret) + return ret; + + ret = io_open(&p->io, p->uri); + if (ret) + return ret; return 0; } @@ -60,28 +65,47 @@ static int print_start(struct hook *h) static int print_stop(struct hook *h) { struct print *p = h->_vd; + int ret; - if (p->uri) - fclose(p->output); + ret = io_close(&p->io); + if (ret) + return ret; + + ret = io_destroy(&p->io); + if (ret) + return ret; return 0; } -static int print_parse(struct hook *h, config_setting_t *cfg) +static int print_parse(struct hook *h, json_t *cfg) { struct print *p = h->_vd; + const char *format = NULL; + int ret; + json_error_t err; - config_setting_lookup_string(cfg, "output", &p->uri); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s }", + "output", &p->uri, + "format", &format + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); + + if (format) { + p->format = io_format_lookup(format); + if (!p->format) + jerror(&err, "Invalid format '%s'", format); + } return 0; } -static int print_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int print_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct print *p = h->_vd; - for (int i = 0; i < *cnt; i++) - sample_io_villas_fprint(p->output, smps[i], SAMPLE_IO_ALL); + io_print(&p->io, smps, *cnt); return 0; } diff --git a/lib/hooks/restart.c b/lib/hooks/restart.c index e83acff5a..76ce49fd8 100644 --- a/lib/hooks/restart.c +++ b/lib/hooks/restart.c @@ -52,7 +52,7 @@ static int restart_stop(struct hook *h) return 0; } -static int restart_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int restart_read(struct hook *h, struct sample *smps[], unsigned *cnt) { int i; struct restart *r = h->_vd; diff --git a/lib/hooks/shift_seq.c b/lib/hooks/shift_seq.c index aeab1de1b..459fed9a1 100644 --- a/lib/hooks/shift_seq.c +++ b/lib/hooks/shift_seq.c @@ -32,17 +32,23 @@ struct shift { int offset; }; -static int shift_seq_parse(struct hook *h, config_setting_t *cfg) +static int shift_seq_parse(struct hook *h, json_t *cfg) { struct shift *p = h->_vd; - if (!config_setting_lookup_int(cfg, "offset", &p->offset)) - cerror(cfg, "Missing setting 'offset' for hook '%s'", plugin_name(h->_vt)); + json_error_t err; + int ret; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: i }", + "offset", &p->offset + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); return 0; } -static int shift_seq_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int shift_seq_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct shift *p = h->_vd; diff --git a/lib/hooks/shift_ts.c b/lib/hooks/shift_ts.c index e2c6ab6b8..9c815279c 100644 --- a/lib/hooks/shift_ts.c +++ b/lib/hooks/shift_ts.c @@ -47,13 +47,22 @@ static int shift_ts_init(struct hook *h) return 0; } -static int shift_ts_parse(struct hook *h, config_setting_t *cfg) +static int shift_ts_parse(struct hook *h, json_t *cfg) { struct shift_ts *p = h->_vd; + double offset; + const char *mode = NULL; + int ret; + json_error_t err; - const char *mode; + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s: f }", + "mode", &mode, + "offset", &offset + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); - if (config_setting_lookup_string(cfg, "mode", &mode)) { + if (mode) { if (!strcmp(mode, "origin")) p->mode = SHIFT_ORIGIN; else if (!strcmp(mode, "received")) @@ -61,19 +70,15 @@ static int shift_ts_parse(struct hook *h, config_setting_t *cfg) else if (!strcmp(mode, "sent")) p->mode = SHIFT_SENT; else - cerror(cfg, "Invalid mode parameter '%s' for hook '%s'", mode, plugin_name(h->_vt)); + jerror(&err, "Invalid mode parameter '%s' for hook '%s'", mode, plugin_name(h->_vt)); } - double offset; - if (!config_setting_lookup_float(cfg, "offset", &offset)) - cerror(cfg, "Missing setting 'offset' for hook '%s'", plugin_name(h->_vt)); - p->offset = time_from_double(offset); return 0; } -static int shift_ts_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int shift_ts_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct shift_ts *p = h->_vd; diff --git a/lib/hooks/skip_first.c b/lib/hooks/skip_first.c index a1fb84eb9..c908da62d 100644 --- a/lib/hooks/skip_first.c +++ b/lib/hooks/skip_first.c @@ -24,8 +24,6 @@ * @{ */ -#include - #include "hook.h" #include "plugin.h" #include "timing.h" @@ -53,23 +51,33 @@ struct skip_first { }; }; -static int skip_first_parse(struct hook *h, config_setting_t *cfg) +static int skip_first_parse(struct hook *h, json_t *cfg) { struct skip_first *p = h->_vd; double seconds; - if (config_setting_lookup_float(cfg, "seconds", &seconds)) { + int ret; + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: F }", "seconds", &seconds); + if (!ret) { p->seconds.wait = time_from_double(seconds); p->mode = HOOK_SKIP_MODE_SECONDS; - } - else if (config_setting_lookup_int(cfg, "samples", &p->samples.wait)) { - p->mode = HOOK_SKIP_MODE_SAMPLES; - } - else - cerror(cfg, "Missing setting 'seconds' or 'samples' for hook '%s'", plugin_name(h->_vt)); - return 0; + return 0; + } + + ret = json_unpack_ex(cfg, &err, 0, "{ s: i }", "samples", &p->samples.wait); + if (!ret) { + p->mode = HOOK_SKIP_MODE_SAMPLES; + + return 0; + } + + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); + + return -1; } static int skip_first_restart(struct hook *h) @@ -81,7 +89,7 @@ static int skip_first_restart(struct hook *h) return 0; } -static int skip_first_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int skip_first_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct skip_first *p = h->_vd; diff --git a/lib/hooks/stats_collect.c b/lib/hooks/stats_collect.c index 0d2c03105..782c1dc26 100644 --- a/lib/hooks/stats_collect.c +++ b/lib/hooks/stats_collect.c @@ -25,6 +25,7 @@ */ #include "common.h" +#include "advio.h" #include "hook.h" #include "plugin.h" #include "stats.h" @@ -38,8 +39,8 @@ struct stats_collect { int warmup; int buckets; - FILE *output; - const char *uri; + AFILE *output; + char *uri; }; static int stats_collect_init(struct hook *h) @@ -58,8 +59,7 @@ static int stats_collect_init(struct hook *h) p->warmup = 500; p->buckets = 20; p->uri = NULL; - p->output = stdout; - + return 0; } @@ -67,6 +67,9 @@ static int stats_collect_destroy(struct hook *h) { struct stats_collect *p = h->_vd; + if (p->uri) + free(p->uri); + return stats_destroy(&p->stats); } @@ -75,11 +78,11 @@ static int stats_collect_start(struct hook *h) struct stats_collect *p = h->_vd; if (p->uri) { - p->output = fopen(p->uri, "w+"); + p->output = afopen(p->uri, "w+"); if (!p->output) error("Failed to open file %s for writing", p->uri); } - + return stats_init(&p->stats, p->buckets, p->warmup); } @@ -87,10 +90,10 @@ static int stats_collect_stop(struct hook *h) { struct stats_collect *p = h->_vd; - stats_print(&p->stats, p->output, p->format, p->verbose); + stats_print(&p->stats, p->uri ? p->output->file : stdout, p->format, p->verbose); if (p->uri) - fclose(p->output); + afclose(p->output); return 0; } @@ -108,36 +111,46 @@ static int stats_collect_periodic(struct hook *h) { struct stats_collect *p = h->_vd; - stats_print_periodic(&p->stats, p->output, p->format, p->verbose, h->path); + stats_print_periodic(&p->stats, p->uri ? p->output->file : stdout, p->format, p->verbose, h->path); return 0; } -static int stats_collect_parse(struct hook *h, config_setting_t *cfg) +static int stats_collect_parse(struct hook *h, json_t *cfg) { struct stats_collect *p = h->_vd; - const char *format; - if (config_setting_lookup_string(cfg, "format", &format)) { - if (!strcmp(format, "human")) - p->format = STATS_FORMAT_HUMAN; - else if (!strcmp(format, "json")) - p->format = STATS_FORMAT_JSON; - else if (!strcmp(format, "matlab")) - p->format = STATS_FORMAT_MATLAB; - else - cerror(cfg, "Invalid statistic output format: %s", format); + int ret, fmt; + json_error_t err; + + const char *format = NULL; + const char *uri = NULL; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: b, s?: i, s?: i, s?: s }", + "format", &format, + "verbose", &p->verbose, + "warmup", &p->warmup, + "buckets", &p->buckets, + "output", &uri + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); + + if (format) { + fmt = stats_lookup_format(format); + if (fmt < 0) + jerror(&err, "Invalid statistic output format: %s", format); + + p->format = fmt; } - config_setting_lookup_bool(cfg, "verbose", &p->verbose); - config_setting_lookup_int(cfg, "warmup", &p->warmup); - config_setting_lookup_int(cfg, "buckets", &p->buckets); - config_setting_lookup_string(cfg, "output", &p->uri); + if (uri) + p->uri = strdup(uri); return 0; } -static int stats_collect_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int stats_collect_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct stats_collect *p = h->_vd; diff --git a/lib/hooks/stats_send.c b/lib/hooks/stats_send.c index 6a95105a5..4bfc72241 100644 --- a/lib/hooks/stats_send.c +++ b/lib/hooks/stats_send.c @@ -53,35 +53,45 @@ static int stats_send_init(struct hook *h) return 0; } -static int stats_send_parse(struct hook *h, config_setting_t *cfg) +static int stats_send_parse(struct hook *h, json_t *cfg) { struct stats_send *p = h->_vd; assert(h->path && h->path->super_node); - const char *dest, *mode; + const char *dest = NULL; + const char *mode = NULL; - if (config_setting_lookup_string(cfg, "destination", &dest)) { + int ret; + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: s, s?: i }" + "destination", &dest, + "mode", &mode, + "decimation", &p->decimation + ); + if (ret) + jerror(&err, "Failed to parse configuration of hook '%s'", plugin_name(h->_vt)); + + if (dest) { assert(h->path); p->dest = list_lookup(&h->path->super_node->nodes, dest); if (!p->dest) - cerror(cfg, "Invalid destination node '%s' for hook '%s'", dest, plugin_name(h->_vt)); + jerror(&err, "Invalid destination node '%s' for hook '%s'", dest, plugin_name(h->_vt)); } else - cerror(cfg, "Missing setting 'destination' for hook '%s'", plugin_name(h->_vt)); + jerror(&err, "Missing setting 'destination' for hook '%s'", plugin_name(h->_vt)); - if (config_setting_lookup_string(cfg, "mode", &mode)) { + if (mode) { if (!strcmp(mode, "periodic")) p->mode = STATS_SEND_MODE_PERIODIC; else if (!strcmp(mode, "read")) p->mode = STATS_SEND_MODE_READ; else - cerror(cfg, "Invalid value '%s' for setting 'mode' of hook '%s'", mode, plugin_name(h->_vt)); + jerror(&err, "Invalid value '%s' for setting 'mode' of hook '%s'", mode, plugin_name(h->_vt)); } - config_setting_lookup_int(cfg, "decimation", &p->decimation); - return 0; } @@ -115,7 +125,7 @@ static int stats_send_periodic(struct hook *h) return 0; } -static int stats_send_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int stats_send_read(struct hook *h, struct sample *smps[], unsigned *cnt) { struct stats_send *p = h->_vd; diff --git a/lib/hooks/ts.c b/lib/hooks/ts.c index c9eb7f0c4..3f078b4de 100644 --- a/lib/hooks/ts.c +++ b/lib/hooks/ts.c @@ -29,7 +29,7 @@ #include "timing.h" #include "sample.h" -static int ts_read(struct hook *h, struct sample *smps[], size_t *cnt) +static int ts_read(struct hook *h, struct sample *smps[], unsigned *cnt) { for (int i = 0; i < *cnt; i++) smps[i]->ts.origin = smps[i]->ts.received; diff --git a/lib/io.c b/lib/io.c new file mode 100644 index 000000000..6abf70bff --- /dev/null +++ b/lib/io.c @@ -0,0 +1,293 @@ +/** Reading and writing simulation samples in various formats. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include "io.h" +#include "io_format.h" +#include "utils.h" +#include "sample.h" +#include "plugin.h" + +int io_init(struct io *io, struct io_format *fmt, int flags) +{ + io->_vt = fmt; + io->_vd = alloc(fmt->size); + + io->flags = flags | io->_vt->flags; + + return io->_vt->init ? io->_vt->init(io) : 0; +} + +int io_destroy(struct io *io) +{ + int ret; + + ret = io->_vt->destroy ? io->_vt->destroy(io) : 0; + if (ret) + return ret; + + free(io->_vd); + + return 0; +} + +int io_stream_open(struct io *io, const char *uri) +{ + int ret; + + if (uri) { + if (aislocal(uri)) { + io->mode = IO_MODE_STDIO; + + io->stdio.output = fopen(uri, "a+"); + if (io->stdio.output == NULL) + return -1; + + io->stdio.input = fopen(uri, "r"); + if (io->stdio.input == NULL) + return -1; + } + else { + io->mode = IO_MODE_ADVIO; + + io->advio.output = afopen(uri, "a+"); + if (io->advio.output == NULL) + return -1; + + io->advio.input = afopen(uri, "a+"); + if (io->advio.input == NULL) + return -2; + } + } + else { + io->mode = IO_MODE_STDIO; + io->flags |= IO_FLUSH; + + io->stdio.input = stdin; + io->stdio.output = stdout; + } + + if (io->mode == IO_MODE_STDIO) { + ret = setvbuf(io->stdio.input, NULL, _IOLBF, BUFSIZ); + if (ret) + return -1; + + ret = setvbuf(io->stdio.output, NULL, _IOLBF, BUFSIZ); + if (ret) + return -1; + } + + return 0; +} + +int io_stream_close(struct io *io) +{ + int ret; + + switch (io->mode) { + case IO_MODE_ADVIO: + ret = afclose(io->advio.input); + if (ret) + return ret; + + ret = afclose(io->advio.output); + if (ret) + return ret; + + return 0; + + case IO_MODE_STDIO: + if (io->stdio.input == stdin) + return 0; + + ret = fclose(io->stdio.input); + if (ret) + return ret; + + ret = fclose(io->stdio.output); + if (ret) + return ret; + + return 0; + + case IO_MODE_CUSTOM: + return 0; + } + + return -1; +} + +int io_stream_flush(struct io *io) +{ + switch (io->mode) { + case IO_MODE_ADVIO: + return afflush(io->advio.output); + case IO_MODE_STDIO: + return fflush(io->stdio.output); + case IO_MODE_CUSTOM: + return 0; + } + + return -1; +} + +int io_stream_eof(struct io *io) +{ + switch (io->mode) { + case IO_MODE_ADVIO: + return afeof(io->advio.input); + case IO_MODE_STDIO: + return feof(io->stdio.input); + case IO_MODE_CUSTOM: + return 0; + } + + return -1; +} + +void io_stream_rewind(struct io *io) +{ + switch (io->mode) { + case IO_MODE_ADVIO: + return arewind(io->advio.input); + case IO_MODE_STDIO: + return rewind(io->stdio.input); + case IO_MODE_CUSTOM: { } + } +} + +int io_open(struct io *io, const char *uri) +{ + return io->_vt->open + ? io->_vt->open(io, uri) + : io_stream_open(io, uri); +} + +int io_close(struct io *io) +{ + return io->_vt->close + ? io->_vt->close(io) + : io_stream_close(io); +} + +int io_flush(struct io *io) +{ + return io->_vt->flush + ? io->_vt->flush(io) + : io_stream_flush(io); +} + +int io_eof(struct io *io) +{ + return io->_vt->eof + ? io->_vt->eof(io) + : io_stream_eof(io); +} + +void io_rewind(struct io *io) +{ + io->_vt->rewind + ? io->_vt->rewind(io) + : io_stream_rewind(io); +} + +int io_print(struct io *io, struct sample *smps[], unsigned cnt) +{ + int ret; + + if (io->_vt->print) + ret = io->_vt->print(io, smps, cnt); + else { + FILE *f = io->mode == IO_MODE_ADVIO + ? io->advio.output->file + : io->stdio.output; + + //flockfile(f); + + if (io->_vt->fprint) + ret = io->_vt->fprint(f, smps, cnt, io->flags); + else if (io->_vt->sprint) { + char buf[4096]; + size_t wbytes; + + ret = io->_vt->sprint(buf, sizeof(buf), &wbytes, smps, cnt, io->flags); + + fwrite(buf, wbytes, 1, f); + } + else + ret = -1; + + //funlockfile(f); + } + + if (io->flags & IO_FLUSH) + io_flush(io); + + return ret; +} + +int io_scan(struct io *io, struct sample *smps[], unsigned cnt) +{ + int ret; + + if (io->_vt->scan) + ret = io->_vt->scan(io, smps, cnt); + else { + FILE *f = io->mode == IO_MODE_ADVIO + ? io->advio.input->file + : io->stdio.input; + + //flockfile(f); + + int flags = io->flags; + + if (io->_vt->fscan) + return io->_vt->fscan(f, smps, cnt, &flags); + else if (io->_vt->sscan) { + size_t bytes, rbytes; + char buf[4096]; + + bytes = fread(buf, 1, sizeof(buf), f); + + ret = io->_vt->sscan(buf, bytes, &rbytes, smps, cnt, &flags); + } + else + ret = -1; + + //funlockfile(f); + } + + return ret; +} + +struct io_format * io_format_lookup(const char *name) +{ + struct plugin *p; + + p = plugin_lookup(PLUGIN_TYPE_IO, name); + if (!p) + return NULL; + + return &p->io; +} diff --git a/lib/web/Makefile.inc b/lib/io/Makefile.inc similarity index 76% rename from lib/web/Makefile.inc rename to lib/io/Makefile.inc index 6ee6486af..d75555287 100644 --- a/lib/web/Makefile.inc +++ b/lib/io/Makefile.inc @@ -20,8 +20,18 @@ # along with this program. If not, see . ################################################################################### -LIB_SRCS += lib/web.c -LIB_SRCS += $(wildcard lib/web/*.c) +LIB_SRCS += $(addprefix lib/io/,json.c villas.c csv.c raw.c msg.c) -LIB_PKGS += libwebsockets -LIB_CFLAGS += -DWITH_WEB +WITH_HDF5 ?= 0 + +ifeq ($(PLATFORM),Darwin) + HDF5_PREFIX ?= /opt/local +else + HDF5_PREFIX ?= /usr +endif + +ifeq ($(WITH_HDF5),1) +ifneq ($(wildcard $(HDF5_PREFIX)/include/hdf5_hl.h),) + LIB_SRCS += lib/formats/hdf5.c +endif +endif diff --git a/lib/io/csv.c b/lib/io/csv.c new file mode 100644 index 000000000..aae43a75b --- /dev/null +++ b/lib/io/csv.c @@ -0,0 +1,202 @@ +/** Comma-separated values. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include "io/csv.h" +#include "plugin.h" +#include "sample.h" +#include "timing.h" + +size_t csv_sprint_single(char *buf, size_t len, struct sample *s, int flags) +{ + size_t off = snprintf(buf, len, "%ld", s->ts.origin.tv_sec); + + if (flags & IO_FORMAT_NANOSECONDS) + off += snprintf(buf + off, len - off, "%c%09llu", CSV_SEPARATOR, (unsigned long long) s->ts.origin.tv_nsec); + + if (flags & IO_FORMAT_SEQUENCE) + off += snprintf(buf + off, len - off, "%c%u", CSV_SEPARATOR, s->sequence); + + for (int i = 0; i < s->length; i++) { + switch ((s->format >> i) & 0x1) { + case SAMPLE_DATA_FORMAT_FLOAT: + off += snprintf(buf + off, len - off, "%c%.6f", CSV_SEPARATOR, s->data[i].f); + break; + case SAMPLE_DATA_FORMAT_INT: + off += snprintf(buf + off, len - off, "%c%" PRId64, CSV_SEPARATOR, s->data[i].i); + break; + } + } + + off += snprintf(buf + off, len - off, "\n"); + + return off; +} + +size_t csv_sscan_single(const char *buf, size_t len, struct sample *s, int *flags) +{ + const char *ptr = buf; + char *end; + + s->ts.origin.tv_sec = strtoul(ptr, &end, 10); + if (end == ptr || *end == '\n') + goto out; + + ptr = end; + + s->ts.origin.tv_nsec = strtoul(ptr, &end, 10); + if (end == ptr || *end == '\n') + goto out; + + ptr = end; + + s->sequence = strtoul(ptr, &end, 10); + if (end == ptr || *end == '\n') + goto out; + + for (ptr = end, s->length = 0; + s->length < s->capacity; + ptr = end, s->length++) { + if (*end == '\n') + goto out; + + switch (s->format & (1 << s->length)) { + case SAMPLE_DATA_FORMAT_FLOAT: + s->data[s->length].f = strtod(ptr, &end); + break; + case SAMPLE_DATA_FORMAT_INT: + s->data[s->length].i = strtol(ptr, &end, 10); + break; + } + + /* There are no valid FP values anymore. */ + if (end == ptr) + goto out; + } + +out: if (*end == '\n') + end++; + + s->ts.received = time_now(); + + return end - buf; +} + +int csv_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + int i; + size_t off = 0; + + for (i = 0; i < cnt && off < len; i++) + off += csv_sprint_single(buf + off, len - off, smps[i], flags); + + if (wbytes) + *wbytes = off; + + return i; +} + +int csv_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + int i; + size_t off = 0; + + for (i = 0; i < cnt && off < len; i++) + off += csv_sscan_single(buf + off, len - off, smps[i], flags); + + if (rbytes) + *rbytes = off; + + return i; +} + +int csv_fprint_single(FILE *f, struct sample *s, int flags) +{ + int ret; + char line[4096]; + + ret = csv_sprint_single(line, sizeof(line), s, flags); + if (ret < 0) + return ret; + + fputs(line, f); + + return 0; +} + +int csv_fscan_single(FILE *f, struct sample *s, int *flags) +{ + char *ptr, line[4096]; + +skip: if (fgets(line, sizeof(line), f) == NULL) + return -1; /* An error occured */ + + /* Skip whitespaces, empty and comment lines */ + for (ptr = line; isspace(*ptr); ptr++); + if (*ptr == '\0' || *ptr == '#') + goto skip; + + return csv_sscan_single(line, strlen(line), s, flags); +} + +int csv_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags) +{ + int ret, i; + for (i = 0; i < cnt; i++) { + ret = csv_fprint_single(f, smps[i], flags); + if (ret < 0) + break; + } + + return i; +} + +int csv_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags) +{ + int ret, i; + for (i = 0; i < cnt; i++) { + ret = csv_fscan_single(f, smps[i], flags); + if (ret < 0) { + warn("Failed to read CSV line: %d", ret); + break; + } + } + + return i; +} + +static struct plugin p = { + .name = "csv", + .description = "Tabulator-separated values", + .type = PLUGIN_TYPE_IO, + .io = { + .fprint = csv_fprint, + .fscan = csv_fscan, + .sprint = csv_sprint, + .sscan = csv_sscan, + .size = 0 + } +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/io/h5pt.c b/lib/io/h5pt.c new file mode 100644 index 000000000..8a5dcb981 --- /dev/null +++ b/lib/io/h5pt.c @@ -0,0 +1,170 @@ +/** HDF5 Packet Table IO format based-on the H5PT high-level API. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include + +#include "plugin.h" +#include "io.h" + +enum h5pt_flags { + FORMAT_H5PT_COMPRESS = (1 << 16) +}; + +struct h5pt { + hsize_t index; + hsize_t num_packets; + + hid_t fid; /**< File identifier. */ + hid_t ptable; /**< Packet table identifier. */ +} + +hid_t h5pt_create_datatype(int values) +{ + hid_t dt = H5Tcreate(H5T_COMPOUND, sizeof()); +} + +int h5pt_open(struct io *io, const char *uri, const char *mode) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + /* Create a file using default properties */ + h5->fid = H5Fcreate(uri, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* Create a fixed-length packet table within the file */ + /* This table's "packets" will be simple integers. */ + h5->ptable = H5PTcreate_fl(fid, "Packet Test Dataset", H5T_NATIVE_INT, 1, 1); + if (h5->ptable == H5I_INVALID_HID) + return -1; + + err = H5PTis_valid(h5->ptable); + if (err < 0) + return err; + + err = H5PTget_num_packets(h5->ptable, h5->num_packets); + if (err < 0) + return err; + + io_format_h5pt_rewind(io); + + return 0; +} + +int h5pt_close(struct io *io) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + /* Close dataset */ + err = H5PTclose(h5->ptable); + if (err < 0) + goto out; + + /* Close file */ +out: err = H5Fclose(h5->fid); + if (err < 0) + return err; + + return 0; +} + +int h5pt_rewind(struct io *io) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + err = H5PTcreate_index(h5->ptable); + if (err < 0) + return err; + + h5->index = 0; + + return 0; +} + +int h5pt_eof(struct io *io) +{ + struct h5pt *h5 = io->_vd; + + return h5->index >= h5->num_packets; +} + +int h5pt_flush(struct io *io) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + err = H5Fflush(h5->fid, H5F_SCOPE_GLOBAL); + if (err < 0) + return err; + + return 0; +} + +int h5pt_print(struct io *io, struct sample *smps[], size_t cnt) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + /* Write one packet to the packet table */ + err = H5PTappend(h5->ptable, smps, cnt); + if (err < 0) + goto out; + + return 0; +} + +int h5pt_scan(struct io *io, struct sample *smps[], size_t cnt) +{ + struct h5pt *h5 = io->_vd; + + herr_t err; + + H5PTget_next(h5->ptable, cnt, smps) + + return 0; +} + +static struct plugin p = { + .name = "hdf5", + .description = "HDF5 Packet Table", + .type = PLUGIN_TYPE_FORMAT, + .format = { + .open = h5pt_open, + .close = h5pt_close, + .rewind = h5pt_rewind, + .eof = h5pt_eof, + .flush = h5pt_flush, + .print = h5pt_print, + .scan = h5pt_scan, + + .flags = IO_FORMAT_BINARY, + .size = sizeof(struct h5pt) + } +}; diff --git a/lib/io/json.c b/lib/io/json.c new file mode 100644 index 000000000..f9b9374c7 --- /dev/null +++ b/lib/io/json.c @@ -0,0 +1,245 @@ +/** JSON serializtion of sample data. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include "plugin.h" +#include "io/json.h" + +int json_pack_sample(json_t **j, struct sample *smp, int flags) +{ + json_t *json_smp; + json_error_t err; + + json_smp = json_pack_ex(&err, 0, "{ s: { s: [ I, I ], s: [ I, I ], s: [ I, I ] } }", + "ts", + "origin", smp->ts.origin.tv_sec, smp->ts.origin.tv_nsec, + "received", smp->ts.received.tv_sec, smp->ts.received.tv_nsec, + "sent", smp->ts.sent.tv_sec, smp->ts.sent.tv_nsec); + + if (flags & IO_FORMAT_SEQUENCE) { + json_t *json_sequence = json_integer(smp->sequence); + + json_object_set(json_smp, "sequence", json_sequence); + } + + if (flags & IO_FORMAT_VALUES) { + json_t *json_data = json_array(); + + for (int i = 0; i < smp->length; i++) { + json_t *json_value = sample_get_data_format(smp, i) + ? json_integer(smp->data[i].i) + : json_real(smp->data[i].f); + + json_array_append(json_data, json_value); + } + + json_object_set(json_smp, "data", json_data); + } + + *j = json_smp; + + return 0; +} + +int json_pack_samples(json_t **j, struct sample *smps[], unsigned cnt, int flags) +{ + int ret; + json_t *json_smps = json_array(); + + for (int i = 0; i < cnt; i++) { + json_t *json_smp; + + ret = json_pack_sample(&json_smp, smps[i], flags); + if (ret) + break; + + json_array_append(json_smps, json_smp); + } + + *j = json_smps; + + return cnt; +} + +int json_unpack_sample(json_t *json_smp, struct sample *smp, int *flags) +{ + int ret; + json_t *json_data, *json_value; + size_t i; + + ret = json_unpack(json_smp, "{ s: { s: [ I, I ], s: [ I, I ], s: [ I, I ] }, s: I, s: o }", + "ts", + "origin", &smp->ts.origin.tv_sec, &smp->ts.origin.tv_nsec, + "received", &smp->ts.received.tv_sec, &smp->ts.received.tv_nsec, + "sent", &smp->ts.sent.tv_sec, &smp->ts.sent.tv_nsec, + "sequence", &smp->sequence, + "data", &json_data); + + if (ret) + return ret; + + if (!json_is_array(json_data)) + return -1; + + smp->length = 0; + + json_array_foreach(json_data, i, json_value) { + if (i >= smp->capacity) + break; + + switch (json_typeof(json_value)) { + case JSON_REAL: + smp->data[i].f = json_real_value(json_value); + sample_set_data_format(smp, i, SAMPLE_DATA_FORMAT_FLOAT); + break; + + case JSON_INTEGER: + smp->data[i].f = json_integer_value(json_value); + sample_set_data_format(smp, i, SAMPLE_DATA_FORMAT_INT); + break; + + default: + return -2; + } + + smp->length++; + } + + return 0; +} + +int json_unpack_samples(json_t *json_smps, struct sample *smps[], unsigned cnt, int *flags) +{ + int ret; + json_t *json_smp; + size_t i; + + if (!json_is_array(json_smps)) + return -1; + + json_array_foreach(json_smps, i, json_smp) { + if (i >= cnt) + break; + + ret = json_unpack_sample(json_smp, smps[i], flags); + if (ret < 0) + break; + } + + return i; +} + +int json_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + int ret; + json_t *json; + size_t wr; + + ret = json_pack_samples(&json, smps, cnt, flags); + if (ret < 0) + return ret; + + wr = json_dumpb(json, buf, len, 0); + + json_decref(json); + + if (wbytes) + *wbytes = wr; + + return ret; +} + +int json_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + int ret; + json_t *json; + json_error_t err; + + json = json_loadb(buf, len, 0, &err); + if (!json) + return -1; + + ret = json_unpack_samples(json, smps, cnt, flags); + if (ret < 0) + return ret; + + json_decref(json); + + if (rbytes) + *rbytes = err.position; + + return ret; +} + +int json_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags) +{ + int ret, i; + json_t *json; + + for (i = 0; i < cnt; i++) { + ret = json_pack_sample(&json, smps[i], flags); + if (ret) + return ret; + + ret = json_dumpf(json, f, 0); + fputc('\n', f); + + json_decref(json); + } + + return i; +} + +int json_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags) +{ + int i, ret; + json_t *json; + json_error_t err; + + for (i = 0; i < cnt; i++) { +skip: json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); + if (!json) + break; + + ret = json_unpack_sample(json, smps[i], flags); + if (ret) + goto skip; + + json_decref(json); + } + + return i; +} + +static struct plugin p = { + .name = "json", + .description = "Javascript Object Notation", + .type = PLUGIN_TYPE_IO, + .io = { + .fscan = json_fscan, + .fprint = json_fprint, + .sscan = json_sscan, + .sprint = json_sprint, + .size = 0 + }, +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/io/msg.c b/lib/io/msg.c new file mode 100644 index 000000000..a213d37eb --- /dev/null +++ b/lib/io/msg.c @@ -0,0 +1,228 @@ +/** Message related functions. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include "io/msg.h" +#include "io/msg_format.h" +#include "sample.h" +#include "utils.h" +#include "plugin.h" + +void msg_ntoh(struct msg *m) +{ + msg_hdr_ntoh(m); + + for (int i = 0; i < m->length; i++) + m->data[i].i = ntohl(m->data[i].i); +} + +void msg_hton(struct msg *m) +{ + for (int i = 0; i < m->length; i++) + m->data[i].i = htonl(m->data[i].i); + + msg_hdr_hton(m); +} + +void msg_hdr_hton(struct msg *m) +{ + m->length = htons(m->length); + m->sequence = htonl(m->sequence); + m->ts.sec = htonl(m->ts.sec); + m->ts.nsec = htonl(m->ts.nsec); +} + +void msg_hdr_ntoh(struct msg *m) +{ + m->length = ntohs(m->length); + m->sequence = ntohl(m->sequence); + m->ts.sec = ntohl(m->ts.sec); + m->ts.nsec = ntohl(m->ts.nsec); +} + +int msg_verify(struct msg *m) +{ + if (m->version != MSG_VERSION) + return -1; + else if (m->type != MSG_TYPE_DATA) + return -2; + else if (m->rsvd1 != 0) + return -3; + else + return 0; +} + +int msg_to_sample(struct msg *msg, struct sample *smp) +{ + int ret; + + ret = msg_verify(msg); + if (ret) + return -1; + + smp->length = MIN(msg->length, smp->capacity); + smp->sequence = msg->sequence; + smp->id = msg->id; + smp->ts.origin = MSG_TS(msg); + smp->ts.received.tv_sec = -1; + smp->ts.received.tv_nsec = -1; + smp->format = 0; + + for (int i = 0; i < smp->length; i++) { + switch (sample_get_data_format(smp, i)) { + case SAMPLE_DATA_FORMAT_FLOAT: smp->data[i].f = msg->data[i].f; break; + case SAMPLE_DATA_FORMAT_INT: smp->data[i].i = msg->data[i].i; break; + } + } + + return 0; +} + +int msg_from_sample(struct msg *msg, struct sample *smp) +{ + *msg = MSG_INIT(smp->length, smp->sequence); + + msg->ts.sec = smp->ts.origin.tv_sec; + msg->ts.nsec = smp->ts.origin.tv_nsec; + msg->id = smp->id; + + for (int i = 0; i < smp->length; i++) { + switch (sample_get_data_format(smp, i)) { + case SAMPLE_DATA_FORMAT_FLOAT: msg->data[i].f = smp->data[i].f; break; + case SAMPLE_DATA_FORMAT_INT: msg->data[i].i = smp->data[i].i; break; + } + } + + return 0; +} + +int msg_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + int ret, i = 0; + char *ptr = buf; + + for (i = 0; i < cnt; i++) { + struct msg *msg = (struct msg *) ptr; + struct sample *smp = smps[i]; + + if (ptr + MSG_LEN(smp->length) > buf + len) + break; + + ret = msg_from_sample(msg, smp); + if (ret) + return ret; + + if (flags & MSG_WEB) { + /** @todo convert to little endian */ + } + else + msg_hton(msg); + + ptr += MSG_LEN(smp->length); + } + + if (wbytes) + *wbytes = ptr - buf; + + return i; +} + +int msg_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + int ret, i = 0, values; + char *ptr = buf; + + if (len % 4 != 0) { + warn("Packet size is invalid: %zd Must be multiple of 4 bytes.", len); + return -1; + } + + for (i = 0; i < cnt; i++) { + struct msg *msg = (struct msg *) ptr; + struct sample *smp = smps[i]; + + /* Complete buffer has been parsed */ + if (ptr == buf + len) + break; + + /* Check if header is still in buffer bounaries */ + if (ptr + sizeof(struct msg) > buf + len) { + warn("Invalid msg received: reason=1"); + break; + } + + values = (*flags & MSG_WEB) ? msg->length : ntohs(msg->length); + + /* Check if remainder of message is in buffer boundaries */ + if (ptr + MSG_LEN(values) > buf + len) { + warn("Invalid msg received: reason=2, msglen=%zu, len=%zu, ptr=%p, buf=%p, i=%u", MSG_LEN(values), len, ptr, buf, i); + break; + } + + if (*flags & MSG_WEB) + ; + else + msg_ntoh(msg); + + ret = msg_to_sample(msg, smp); + if (ret) { + warn("Invalid msg received: reason=3, ret=%d", ret); + break; + } + + ptr += MSG_LEN(smp->length); + } + + if (rbytes) + *rbytes = ptr - buf; + + return i; +} + +static struct plugin p1 = { + .name = "msg", + .description = "VILLAS binary network format", + .type = PLUGIN_TYPE_IO, + .io = { + .sprint = msg_sprint, + .sscan = msg_sscan, + .size = 0, + .flags = IO_FORMAT_BINARY + }, +}; + +/** The WebSocket node-type usually uses little endian byte order intead of network byte order */ +static struct plugin p2 = { + .name = "webmsg", + .description = "VILLAS binary network format for WebSockets", + .type = PLUGIN_TYPE_IO, + .io = { + .sprint = msg_sprint, + .sscan = msg_sscan, + .size = 0, + .flags = IO_FORMAT_BINARY | MSG_WEB + }, +}; + +REGISTER_PLUGIN(&p1); +REGISTER_PLUGIN(&p2); diff --git a/lib/io/raw.c b/lib/io/raw.c new file mode 100644 index 000000000..dc757f8cb --- /dev/null +++ b/lib/io/raw.c @@ -0,0 +1,244 @@ +/** RAW IO format + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include "sample.h" +#include "plugin.h" +#include "utils.h" +#include "compat.h" +#include "io/raw.h" + +/** Convert float to host byte order */ +#define SWAP_FLT_TOH(o, n) ({ \ + union { float f; uint32_t i; } x = { .f = n }; \ + x.i = (o) ? be32toh(x.i) : le32toh(x.i); x.f; \ +}) + +/** Convert double to host byte order */ +#define SWAP_DBL_TOH(o, n) ({ \ + union { float f; uint64_t i; } x = { .f = n }; \ + x.i = (o) ? be64toh(x.i) : le64toh(x.i); x.f; \ +}) + +/** Convert float to big/little endian byte order */ +#define SWAP_FLT_TOE(o, n) ({ \ + union { float f; uint32_t i; } x = { .f = n }; \ + x.i = (o) ? htobe32(x.i) : htole32(x.i); x.f; \ +}) + +/** Convert double to big/little endian byte order */ +#define SWAP_DBL_TOE(o, n) ({ \ + union { double f; uint64_t i; } x = { .f = n }; \ + x.i = (o) ? htobe64(x.i) : htole64(x.i); x.f; \ +}) + +/** Convert integer of varying width to host byte order */ +#define SWAP_INT_TOH(o, b, n) (o ? be ## b ## toh(n) : le ## b ## toh(n)) + +/** Convert integer of varying width to big/little endian byte order */ +#define SWAP_INT_TOE(o, b, n) (o ? htobe ## b (n) : htole ## b (n)) + +int raw_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + + int i, o = 0; + size_t nlen; + + int8_t *i8 = (void *) buf; + int16_t *i16 = (void *) buf; + int32_t *i32 = (void *) buf; + int64_t *i64 = (void *) buf; + float *f32 = (void *) buf; + double *f64 = (void *) buf; + + int bits = 1 << (flags >> 24); + + for (i = 0; i < cnt; i++) { + nlen = (smps[i]->length + o + (flags & RAW_FAKE) ? 3 : 0) * (bits / 8); + if (nlen >= len) + break; + + /* First three values are sequence, seconds and nano-seconds timestamps */ + if (flags & RAW_FAKE) { + switch (bits) { + case 32: + i32[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 32, smps[i]->sequence); + i32[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 32, smps[i]->ts.origin.tv_sec); + i32[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 32, smps[i]->ts.origin.tv_nsec); + break; + case 64: + i64[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 64, smps[i]->sequence); + i64[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 64, smps[i]->ts.origin.tv_sec); + i64[o++] = SWAP_INT_TOE(flags & RAW_BE_HDR, 64, smps[i]->ts.origin.tv_nsec); + break; + } + } + + for (int j = 0; j < smps[i]->length; j++) { + enum { INT, FLT } fmt; + + if (flags & RAW_AUTO) + fmt = smps[i]->format & (1 << i) ? INT : FLT; + else if (flags & RAW_FLT) + fmt = FLT; + else + fmt = INT; + + switch (fmt) { + case FLT: + switch (bits) { + case 32: f32[o++] = SWAP_FLT_TOE(flags & RAW_BE_FLT, smps[i]->data[j].f); break; + case 64: f64[o++] = SWAP_DBL_TOE(flags & RAW_BE_FLT, smps[i]->data[j].f); break; + } + break; + + case INT: + switch (bits) { + case 8: i8 [o++] = smps[i]->data[j].i; break; + case 16: i16[o++] = SWAP_INT_TOE(flags & RAW_BE_INT, 16, smps[i]->data[j].i); break; + case 32: i32[o++] = SWAP_INT_TOE(flags & RAW_BE_INT, 32, smps[i]->data[j].i); break; + case 64: i64[o++] = SWAP_INT_TOE(flags & RAW_BE_INT, 64, smps[i]->data[j].i); break; + } + break; + } + } + } + + if (wbytes) + *wbytes = o * (bits / 8); + + return i; +} + +int raw_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + /* The raw format can not encode multiple samples in one buffer + * as there is no support for framing. */ + struct sample *smp = smps[0]; + + int8_t *i8 = (void *) buf; + int16_t *i16 = (void *) buf; + int32_t *i32 = (void *) buf; + int64_t *i64 = (void *) buf; + float *f32 = (void *) buf; + double *f64 = (void *) buf; + + int off, bits = 1 << (*flags >> 24); + + smp->length = len / (bits / 8); + + if (*flags & RAW_FAKE) { + off = 3; + + if (smp->length < off) { +// warn("Node %s received a packet with no fake header. Skipping...", node_name(n)); + return 0; + } + + smp->length -= off; + + switch (bits) { + case 32: + smp->sequence = SWAP_INT_TOH(*flags & RAW_BE_HDR, 32, i32[0]); + smp->ts.origin.tv_sec = SWAP_INT_TOH(*flags & RAW_BE_HDR, 32, i32[1]); + smp->ts.origin.tv_nsec = SWAP_INT_TOH(*flags & RAW_BE_HDR, 32, i32[2]); + break; + + case 64: + smp->sequence = SWAP_INT_TOH(*flags & RAW_BE_HDR, 64, i64[0]); + smp->ts.origin.tv_sec = SWAP_INT_TOH(*flags & RAW_BE_HDR, 64, i64[1]); + smp->ts.origin.tv_nsec = SWAP_INT_TOH(*flags & RAW_BE_HDR, 64, i64[2]); + break; + } + } + else { + off = 0; + + smp->sequence = 0; + smp->ts.origin.tv_sec = 0; + smp->ts.origin.tv_nsec = 0; + } + + if (smp->length > smp->capacity) { + warn("Received more values than supported: length=%u, capacity=%u", smp->length, smp->capacity); + smp->length = smp->capacity; + } + + for (int i = 0; i < smp->length; i++) { + int fmt = *flags & RAW_FLT ? SAMPLE_DATA_FORMAT_FLOAT + : SAMPLE_DATA_FORMAT_INT; + + sample_set_data_format(smp, i, fmt); + + switch (fmt) { + case SAMPLE_DATA_FORMAT_FLOAT: + switch (bits) { + case 32: smp->data[i].f = SWAP_FLT_TOH(*flags & RAW_BE_FLT, f32[i+off]); break; + case 64: smp->data[i].f = SWAP_DBL_TOH(*flags & RAW_BE_FLT, f64[i+off]); break; + } + break; + + case SAMPLE_DATA_FORMAT_INT: + switch (bits) { + case 8: smp->data[i].i = i8[i]; break; + case 16: smp->data[i].i = (int16_t) SWAP_INT_TOH(*flags & RAW_BE_INT, 16, i16[i+off]); break; + case 32: smp->data[i].i = (int32_t) SWAP_INT_TOH(*flags & RAW_BE_INT, 32, i32[i+off]); break; + case 64: smp->data[i].i = (int64_t) SWAP_INT_TOH(*flags & RAW_BE_INT, 64, i64[i+off]); break; + } + break; + } + } + + smp->ts.received.tv_sec = 0; + smp->ts.received.tv_nsec = 0; + + if (rbytes) + *rbytes = len; + + return 1; +} + +#define REGISTER_FORMAT_RAW(i, n, f) \ +static struct plugin i = { \ + .name = n, \ + .description = "RAW binary data", \ + .type = PLUGIN_TYPE_IO, \ + .io = { \ + .flags = f | IO_FORMAT_BINARY, \ + .sprint = raw_sprint, \ + .sscan = raw_sscan \ + } \ +}; \ +REGISTER_PLUGIN(& i); + +/* Feel free to add additional format identifiers here to suit your needs */ +REGISTER_FORMAT_RAW(p, "raw", 0) +REGISTER_FORMAT_RAW(p_f32, "raw-flt32", RAW_32 | RAW_FLT) +REGISTER_FORMAT_RAW(p_f64, "raw-flt64", RAW_64 | RAW_FLT) +REGISTER_FORMAT_RAW(p_i8, "raw-int8", RAW_8) +REGISTER_FORMAT_RAW(p_i16be, "raw-int16-be", RAW_16 | RAW_BE) +REGISTER_FORMAT_RAW(p_i16le, "raw-int16-le", RAW_16) +REGISTER_FORMAT_RAW(p_i32be, "raw-int32-be", RAW_32 | RAW_BE) +REGISTER_FORMAT_RAW(p_i32le, "raw-int32-le", RAW_32) +REGISTER_FORMAT_RAW(p_i64be, "raw-int64-be", RAW_64 | RAW_BE) +REGISTER_FORMAT_RAW(p_i64le, "raw-int64-le", RAW_64) +REGISTER_FORMAT_RAW(p_gtnet, "gtnet", RAW_32 | RAW_FLT | RAW_BE) +REGISTER_FORMAT_RAW(p_gtnef, "gtnet-fake", RAW_32 | RAW_FLT | RAW_BE | RAW_FAKE) diff --git a/lib/sample_io.c b/lib/io/villas.c similarity index 54% rename from lib/sample_io.c rename to lib/io/villas.c index ea2d14b98..568dd339d 100644 --- a/lib/sample_io.c +++ b/lib/io/villas.c @@ -21,60 +21,37 @@ *********************************************************************************/ #include +#include +#include +#include "io.h" +#include "plugin.h" +#include "utils.h" #include "timing.h" #include "sample.h" -#include "sample_io.h" +#include "io/villas.h" -#ifdef WITH_JSON - #include "sample_io_json.h" -#endif - -int sample_io_fprint(FILE *f, struct sample *s, enum sample_io_format fmt, int flags) -{ - switch (fmt) { - case SAMPLE_IO_FORMAT_VILLAS: return sample_io_villas_fprint(f, s, flags); -#ifdef WITH_JSON - case SAMPLE_IO_FORMAT_JSON: return sample_io_json_fprint(f, s, flags); -#endif - default: - return -1; - } -} - -int sample_io_fscan(FILE *f, struct sample *s, enum sample_io_format fmt, int *flags) -{ - switch (fmt) { - case SAMPLE_IO_FORMAT_VILLAS: return sample_io_villas_fscan(f, s, flags); -#ifdef WITH_JSON - case SAMPLE_IO_FORMAT_JSON: return sample_io_json_fscan(f, s, flags); -#endif - default: - return -1; - } -} - -int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags) +size_t villas_sprint_single(char *buf, size_t len, struct sample *s, int flags) { size_t off = snprintf(buf, len, "%llu", (unsigned long long) s->ts.origin.tv_sec); - if (flags & SAMPLE_IO_NANOSECONDS) + if (flags & IO_FORMAT_NANOSECONDS) off += snprintf(buf + off, len - off, ".%09llu", (unsigned long long) s->ts.origin.tv_nsec); - if (flags & SAMPLE_IO_OFFSET) + if (flags & IO_FORMAT_OFFSET) off += snprintf(buf + off, len - off, "%+e", time_delta(&s->ts.origin, &s->ts.received)); - if (flags & SAMPLE_IO_SEQUENCE) + if (flags & IO_FORMAT_SEQUENCE) off += snprintf(buf + off, len - off, "(%u)", s->sequence); - if (flags & SAMPLE_IO_VALUES) { + if (flags & IO_FORMAT_VALUES) { for (int i = 0; i < s->length; i++) { switch ((s->format >> i) & 0x1) { case SAMPLE_DATA_FORMAT_FLOAT: - off += snprintf(buf + off, len - off, "\t%.6f", s->data[i].f); + off += snprintf(buf + off, len - off, "\t%.6lf", s->data[i].f); break; case SAMPLE_DATA_FORMAT_INT: - off += snprintf(buf + off, len - off, "\t%d", s->data[i].i); + off += snprintf(buf + off, len - off, "\t%" PRIi64, s->data[i].i); break; } } @@ -82,15 +59,15 @@ int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags) off += snprintf(buf + off, len - off, "\n"); - return 0; /* trailing '\0' */ + return off; } -int sample_io_villas_scan(const char *line, struct sample *s, int *fl) +size_t villas_sscan_single(const char *buf, size_t len, struct sample *s, int *flags) { char *end; - const char *ptr = line; + const char *ptr = buf; - int flags = 0; + int fl = 0; double offset = 0; /* Format: Seconds.NanoSeconds+Offset(SequenceNumber) Value1 Value2 ... @@ -101,8 +78,8 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) /* Mandatory: seconds */ s->ts.origin.tv_sec = (uint32_t) strtoul(ptr, &end, 10); - if (ptr == end) - return -2; + if (ptr == end || *end == '\n') + return -1; /* Optional: nano seconds */ if (*end == '.') { @@ -110,7 +87,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) s->ts.origin.tv_nsec = (uint32_t) strtoul(ptr, &end, 10); if (ptr != end) - flags |= SAMPLE_IO_NANOSECONDS; + fl |= IO_FORMAT_NANOSECONDS; else return -3; } @@ -123,7 +100,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) offset = strtof(ptr, &end); /* offset is ignored for now */ if (ptr != end) - flags |= SAMPLE_IO_OFFSET; + fl |= IO_FORMAT_OFFSET; else return -4; } @@ -134,7 +111,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) s->sequence = strtoul(ptr, &end, 10); if (ptr != end) - flags |= SAMPLE_IO_SEQUENCE; + fl |= IO_FORMAT_SEQUENCE; else return -5; @@ -145,7 +122,9 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) for (ptr = end, s->length = 0; s->length < s->capacity; ptr = end, s->length++) { - + if (*end == '\n') + break; + switch (s->format & (1 << s->length)) { case SAMPLE_DATA_FORMAT_FLOAT: s->data[s->length].f = strtod(ptr, &end); @@ -155,40 +134,59 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) break; } - if (end == ptr) /* There are no valid FP values anymore */ + /* There are no valid FP values anymore. */ + if (end == ptr) break; } + + if (*end == '\n') + end++; if (s->length > 0) - flags |= SAMPLE_IO_VALUES; + fl |= IO_FORMAT_VALUES; - if (fl) - *fl = flags; - if (flags & SAMPLE_IO_OFFSET) { + if (flags) + *flags = fl; + + if (fl & IO_FORMAT_OFFSET) { struct timespec off = time_from_double(offset); s->ts.received = time_add(&s->ts.origin, &off); } else - s->ts.received = s->ts.origin; + s->ts.received = time_now(); - return 0; + return end - buf; } -int sample_io_villas_fprint(FILE *f, struct sample *s, int flags) +int villas_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) { - char line[4096]; - int ret; + int i; + size_t off = 0; - ret = sample_io_villas_print(line, sizeof(line), s, flags); - if (ret) - return ret; + for (i = 0; i < cnt && off < len; i++) + off += villas_sprint_single(buf + off, len - off, smps[i], flags); + + if (wbytes) + *wbytes = off; - fputs(line, f); - - return 0; + return i; } -int sample_io_villas_fscan(FILE *f, struct sample *s, int *fl) +int villas_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + int i; + size_t off = 0; + + for (i = 0; i < cnt && off < len; i++) + off += villas_sscan_single(buf + off, len - off, smps[i], flags); + + if (rbytes) + *rbytes = off; + + return i; +} + +int villas_fscan_single(FILE *f, struct sample *s, int *flags) { char *ptr, line[4096]; @@ -200,5 +198,81 @@ skip: if (fgets(line, sizeof(line), f) == NULL) if (*ptr == '\0' || *ptr == '#') goto skip; - return sample_io_villas_scan(line, s, fl); + return villas_sscan_single(line, strlen(line), s, flags); } + +int villas_fprint_single(FILE *f, struct sample *s, int flags) +{ + int ret; + char line[4096]; + + ret = villas_sprint_single(line, sizeof(line), s, flags); + if (ret < 0) + return ret; + + fputs(line, f); + + return 0; +} + +int villas_fprint(FILE *f, struct sample *smps[], unsigned cnt, int flags) +{ + int ret, i; + + for (i = 0; i < cnt; i++) { + ret = villas_fprint_single(f, smps[i], flags); + if (ret < 0) + return ret; + } + + return i; +} + +int villas_fscan(FILE *f, struct sample *smps[], unsigned cnt, int *flags) +{ + int ret, i; + + for (i = 0; i < cnt; i++) { + ret = villas_fscan_single(f, smps[i], flags); + if (ret < 0) + return ret; + } + + return i; +} + +int villas_open(struct io *io, const char *uri) +{ + int ret; + + ret = io_stream_open(io, uri); + if (ret) + return ret; + + FILE *f = io->mode == IO_MODE_ADVIO + ? io->advio.output->file + : io->stdio.output; + + fprintf(f, "# %-20s\t\t%s\n", "sec.nsec+offset", "data[]"); + + if (io->flags & IO_FLUSH) + io_flush(io); + + return 0; +} + +static struct plugin p = { + .name = "villas", + .description = "VILLAS human readable format", + .type = PLUGIN_TYPE_IO, + .io = { + .open = villas_open, + .fprint = villas_fprint, + .fscan = villas_fscan, + .sprint = villas_sprint, + .sscan = villas_sscan, + .size = 0 + } +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/io_format.c b/lib/io_format.c new file mode 100644 index 000000000..bbf3d1bd8 --- /dev/null +++ b/lib/io_format.c @@ -0,0 +1,51 @@ +/** Read / write sample data in different formats. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include "io_format.h" + +int io_format_sscan(struct io_format *fmt, char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int *flags) +{ + if (!flags) + flags = &fmt->flags; + + return fmt->sscan ? fmt->sscan(buf, len, rbytes, smps, cnt, flags) : -1; +} + +int io_format_sprint(struct io_format *fmt, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + flags |= fmt->flags; + + return fmt->sprint ? fmt->sprint(buf, len, wbytes, smps, cnt, flags) : -1; +} + +int io_format_fscan(struct io_format *fmt, FILE *f, struct sample *smps[], unsigned cnt, int *flags) +{ + return fmt->sprint ? fmt->fscan(f, smps, cnt, flags) : -1; +} + +int io_format_fprint(struct io_format *fmt, FILE *f, struct sample *smps[], unsigned cnt, int flags) +{ + return fmt->fprint ? fmt->fprint(f, smps, cnt, flags) : -1; +} \ No newline at end of file diff --git a/lib/kernel/rt.c b/lib/kernel/rt.c index d296cbde0..8675c90df 100644 --- a/lib/kernel/rt.c +++ b/lib/kernel/rt.c @@ -22,6 +22,7 @@ #include #include +#include #include "config.h" #include "utils.h" @@ -52,6 +53,8 @@ int rt_init(int priority, int affinity) rt_set_affinity(affinity); else warn("You might want to use the 'affinity' setting to pin VILLASnode to dedicate CPU cores"); + + rt_lock_memory(); #else warn("This platform is not optimized for real-time execution"); #endif @@ -62,6 +65,19 @@ int rt_init(int priority, int affinity) #ifdef __linux__ +int rt_lock_memory() +{ + int ret; + +#ifdef _POSIX_MEMLOCK + ret = mlockall(MCL_CURRENT | MCL_FUTURE); + if (ret) + error("Failed to lock memory"); +#endif + + return 0; +} + int rt_set_affinity(int affinity) { char isolcpus[255]; diff --git a/lib/kernel/tc.c b/lib/kernel/tc.c index a1e2f0e33..311aefb26 100644 --- a/lib/kernel/tc.c +++ b/lib/kernel/tc.c @@ -35,10 +35,32 @@ #include "utils.h" -int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **netem) +int tc_parse(struct rtnl_qdisc **netem, json_t *cfg) { const char *str; - int val; + int ret, val; + + json_t *cfg_distribution = NULL; + json_t *cfg_limit = NULL; + json_t *cfg_delay = NULL; + json_t *cfg_jitter = NULL; + json_t *cfg_loss = NULL; + json_t *cfg_duplicate = NULL; + json_t *cfg_corruption = NULL; + + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: i, s?: i, s?: i, s?: i, s?: i, s?: i }", + "distribution", &cfg_distribution, + "limit", &cfg_limit, + "delay", &cfg_delay, + "jitter", &cfg_jitter, + "loss", &cfg_loss, + "duplicate", &cfg_duplicate, + "corruption", &cfg_corruption + ); + if (ret) + jerror(&err, "Failed to parse setting network emulation settings"); struct rtnl_qdisc *ne = rtnl_qdisc_alloc(); if (!ne) @@ -46,51 +68,67 @@ int tc_parse(config_setting_t *cfg, struct rtnl_qdisc **netem) rtnl_tc_set_kind(TC_CAST(ne), "netem"); - if (config_setting_lookup_string(cfg, "distribution", &str)) { + if (cfg_distribution) { + str = json_string_value(cfg_distribution); + if (!str) + error("Setting 'distribution' must be a JSON string"); + if (rtnl_netem_set_delay_distribution(ne, str)) - cerror(cfg, "Invalid delay distribution '%s' in netem config", str); + error("Invalid delay distribution '%s' in netem config", str); } - if (config_setting_lookup_int(cfg, "limit", &val)) { - if (val <= 0) - cerror(cfg, "Invalid value '%d' for limit setting", val); + if (cfg_limit) { + val = json_integer_value(cfg_limit); + + if (!json_is_integer(cfg_limit) || val <= 0) + error("Setting 'limit' must be a positive integer"); rtnl_netem_set_limit(ne, val); } else rtnl_netem_set_limit(ne, 0); - if (config_setting_lookup_int(cfg, "delay", &val)) { - if (val <= 0) - cerror(cfg, "Invalid value '%d' for delay setting", val); + if (cfg_delay) { + val = json_integer_value(cfg_delay); + + if (!json_is_integer(cfg_delay) || val <= 0) + error("Setting 'delay' must be a positive integer"); rtnl_netem_set_delay(ne, val); } - if (config_setting_lookup_int(cfg, "jitter", &val)) { - if (val <= 0) - cerror(cfg, "Invalid value '%d' for jitter setting", val); + if (cfg_jitter) { + val = json_integer_value(cfg_jitter); + + if (!json_is_integer(cfg_jitter) || val <= 0) + error("Setting 'jitter' must be a positive integer"); rtnl_netem_set_jitter(ne, val); } - if (config_setting_lookup_int(cfg, "loss", &val)) { - if (val < 0 || val > 100) - cerror(cfg, "Invalid percentage value '%d' for loss setting", val); + if (cfg_loss) { + val = json_integer_value(cfg_loss); + + if (!json_is_integer(cfg_loss) || val < 0 || val > 100) + error("Setting 'loss' must be a positive integer within the range [ 0, 100 ]"); rtnl_netem_set_loss(ne, val); } - if (config_setting_lookup_int(cfg, "duplicate", &val)) { - if (val < 0 || val > 100) - cerror(cfg, "Invalid percentage value '%d' for duplicate setting", val); + if (cfg_duplicate) { + val = json_integer_value(cfg_duplicate); + + if (!json_is_integer(cfg_duplicate) || val < 0 || val > 100) + error("Setting 'duplicate' must be a positive integer within the range [ 0, 100 ]"); rtnl_netem_set_duplicate(ne, val); } - if (config_setting_lookup_int(cfg, "corruption", &val)) { - if (val < 0 || val > 100) - cerror(cfg, "Invalid percentage value '%d' for corruption setting", val); + if (cfg_corruption) { + val = json_integer_value(cfg_corruption); + + if (!json_is_integer(cfg_corruption) || val < 0 || val > 100) + error("Setting 'corruption' must be a positive integer within the range [ 0, 100 ]"); rtnl_netem_set_corruption_probability(ne, val); } diff --git a/lib/log.c b/lib/log.c index 2dcfb8902..02b37db54 100644 --- a/lib/log.c +++ b/lib/log.c @@ -39,19 +39,23 @@ #include "OpalPrint.h" #endif -/** The global log instance. */ struct log *global_log; -struct log default_log = { - .level = V, - .facilities = LOG_ALL, - .file = NULL, - .path = NULL, - .epoch = { -1 , -1 }, - .window = { - .ws_row = LOG_HEIGHT, - .ws_col = LOG_WIDTH - } -}; + +/* We register a default log instance */ +__attribute__((constructor)) +void register_default_log() +{ + int ret; + static struct log default_log; + + ret = log_init(&default_log, V, LOG_ALL); + if (ret) + error("Failed to initalize log"); + + ret = log_start(&default_log); + if (ret) + error("Failed to start log"); +} /** List of debug facilities as strings */ static const char *facilities_strs[] = { @@ -156,6 +160,7 @@ int log_init(struct log *l, int level, long facilitites) int log_start(struct log *l) { l->epoch = time_now(); + l->prefix = getenv("VILLAS_LOG_PREFIX"); l->file = l->path ? fopen(l->path, "a+") : stderr; if (!l->file) { @@ -186,7 +191,7 @@ int log_destroy(struct log *l) { default_log.epoch = l->epoch; - global_log = NULL; + global_log = &default_log; l->state = STATE_DESTROYED; @@ -256,9 +261,13 @@ void log_vprint(struct log *l, const char *lvl, const char *fmt, va_list ap) { struct timespec ts = time_now(); char *buf = alloc(512); + + /* Optional prefix */ + if (l->prefix) + strcatf(&buf, "%s", l->prefix); /* Timestamp & Severity */ - strcatf(&buf, "%10.3f %5s ", time_delta(&l->epoch, &ts), lvl); + strcatf(&buf, "%10.3f %-5s ", time_delta(&l->epoch, &ts), lvl); /* Indention */ #ifdef __GNUC__ @@ -275,7 +284,7 @@ void log_vprint(struct log *l, const char *lvl, const char *fmt, va_list ap) #ifdef ENABLE_OPAL_ASYNC OpalPrint("VILLASnode: %s\n", buf); #endif - fprintf(l->file ? l->file : stderr, "\33[2K\r%s\n", buf); + fprintf(l->file ? l->file : stderr, "%s\n", buf); free(buf); } diff --git a/lib/log_config.c b/lib/log_config.c index d60ccd654..f8eaf21c0 100644 --- a/lib/log_config.c +++ b/lib/log_config.c @@ -1,4 +1,4 @@ -/** Logging routines that depend on libconfig. +/** Logging routines that depend on jansson. * * @author Steffen Vogel * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC @@ -28,35 +28,39 @@ #include "log.h" #include "log_config.h" #include "utils.h" +#include "string.h" -int log_parse(struct log *l, config_setting_t *cfg) +int log_parse(struct log *l, json_t *cfg) { - const char *fac, *pth; - int lvl; + const char *facilities = NULL; + const char *path = NULL; + int ret; - if (!config_setting_is_group(cfg)) - cerror(cfg, "Setting 'log' must be a group."); + json_error_t err; - if (config_setting_lookup_int(cfg, "level", &lvl)) - l->level = lvl; + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: s, s?: s }", + "level", &l->level, + "file", &path, + "facilities", &facilities + ); + if (ret) + jerror(&err, "Failed to parse logging configuration"); - if (config_setting_lookup_string(cfg, "file", &pth)) - l->path = pth; + if (path) + l->path = strdup(path); - if (config_setting_lookup_string(cfg, "facilities", &fac)) - log_set_facility_expression(l, fac); + if (facilities) + log_set_facility_expression(l, facilities); l->state = STATE_PARSED; return 0; } -void cerror(config_setting_t *cfg, const char *fmt, ...) +void jerror(json_error_t *err, const char *fmt, ...) { va_list ap; char *buf = NULL; - const char *file; - int line; struct log *l = global_log ? global_log : &default_log; @@ -64,12 +68,10 @@ void cerror(config_setting_t *cfg, const char *fmt, ...) vstrcatf(&buf, fmt, ap); va_end(ap); - line = config_setting_source_line(cfg); - file = config_setting_source_file(cfg); - if (!file) - file = config_setting_get_hook(config_root_setting(cfg->config)); - - log_print(l, LOG_LVL_ERROR, "%s in %s:%u", buf, file, line); + log_print(l, LOG_LVL_ERROR, "%s:", buf); + { INDENT + log_print(l, LOG_LVL_ERROR, "%s in %s:%d:%d", err->text, err->source, err->line, err->column); + } free(buf); diff --git a/lib/log_helper.c b/lib/log_helper.c index d5868d670..cc9f3cd5a 100644 --- a/lib/log_helper.c +++ b/lib/log_helper.c @@ -30,7 +30,7 @@ void debug(long class, const char *fmt, ...) { va_list ap; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; int lvl = class & 0xFF; int fac = class & ~0xFF; @@ -46,7 +46,7 @@ void info(const char *fmt, ...) { va_list ap; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; va_start(ap, fmt); log_vprint(l, LOG_LVL_INFO, fmt, ap); @@ -57,7 +57,7 @@ void warn(const char *fmt, ...) { va_list ap; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; va_start(ap, fmt); log_vprint(l, LOG_LVL_WARN, fmt, ap); @@ -68,7 +68,7 @@ void stats(const char *fmt, ...) { va_list ap; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; va_start(ap, fmt); log_vprint(l, LOG_LVL_STATS, fmt, ap); @@ -79,7 +79,7 @@ void error(const char *fmt, ...) { va_list ap; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; va_start(ap, fmt); log_vprint(l, LOG_LVL_ERROR, fmt, ap); @@ -94,7 +94,7 @@ void serror(const char *fmt, ...) va_list ap; char *buf = NULL; - struct log *l = global_log ? global_log : &default_log; + struct log *l = global_log; va_start(ap, fmt); vstrcatf(&buf, fmt, ap); diff --git a/lib/mapping.c b/lib/mapping.c index ae0896d17..cc5d8f295 100644 --- a/lib/mapping.c +++ b/lib/mapping.c @@ -162,11 +162,11 @@ invalid_format: return -1; } -int mapping_entry_parse(struct mapping_entry *e, config_setting_t *cfg) +int mapping_entry_parse(struct mapping_entry *e, json_t *cfg) { const char *str; - str = config_setting_get_string(cfg); + str = json_string_value(cfg); if (!str) return -1; @@ -195,27 +195,23 @@ int mapping_destroy(struct mapping *m) return 0; } -int mapping_parse(struct mapping *m, config_setting_t *cfg) +int mapping_parse(struct mapping *m, json_t *cfg) { int ret; assert(m->state == STATE_INITIALIZED); - if (!config_setting_is_array(cfg)) + if (!json_is_array(cfg)) return -1; m->real_length = 0; - for (int i = 0; i < config_setting_length(cfg); i++) { - config_setting_t *cfg_mapping; - - cfg_mapping = config_setting_get_elem(cfg, i); - if (!cfg_mapping) - return -1; - + size_t index; + json_t *cfg_entry; + json_array_foreach(cfg, index, cfg_entry) { struct mapping_entry *e = alloc(sizeof(struct mapping_entry)); - ret = mapping_entry_parse(e, cfg_mapping); + ret = mapping_entry_parse(e, cfg_entry); if (ret) return ret; diff --git a/lib/msg.c b/lib/msg.c deleted file mode 100644 index 08622f8ce..000000000 --- a/lib/msg.c +++ /dev/null @@ -1,152 +0,0 @@ -/** Message related functions. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -#include "msg.h" -#include "msg_format.h" -#include "sample.h" -#include "utils.h" - -void msg_ntoh(struct msg *m) -{ - msg_hdr_ntoh(m); - - for (int i = 0; i < m->length; i++) - m->data[i].i = ntohl(m->data[i].i); -} - -void msg_hton(struct msg *m) -{ - for (int i = 0; i < m->length; i++) - m->data[i].i = htonl(m->data[i].i); - - msg_hdr_hton(m); -} - -void msg_hdr_hton(struct msg *m) -{ - m->length = htons(m->length); - m->sequence = htonl(m->sequence); - m->ts.sec = htonl(m->ts.sec); - m->ts.nsec = htonl(m->ts.nsec); -} - -void msg_hdr_ntoh(struct msg *m) -{ - m->length = ntohs(m->length); - m->sequence = ntohl(m->sequence); - m->ts.sec = ntohl(m->ts.sec); - m->ts.nsec = ntohl(m->ts.nsec); -} - -int msg_verify(struct msg *m) -{ - if (m->version != MSG_VERSION) - return -1; - else if (m->type != MSG_TYPE_DATA) - return -2; - else if (m->rsvd1 != 0) - return -3; - else - return 0; -} - -int msg_to_sample(struct msg *msg, struct sample *smp) -{ - int ret; - - msg_ntoh(msg); - - ret = msg_verify(msg); - if (ret) - return -1; - - smp->length = MIN(msg->length, smp->capacity); - smp->sequence = msg->sequence; - smp->ts.origin = MSG_TS(msg); - smp->ts.received.tv_sec = -1; - smp->ts.received.tv_nsec = -1; - - memcpy(smp->data, msg->data, SAMPLE_DATA_LEN(smp->length)); - - return 0; -} - -int msg_from_sample(struct msg *msg, struct sample *smp) -{ - *msg = MSG_INIT(smp->length, smp->sequence); - - msg->ts.sec = smp->ts.origin.tv_sec; - msg->ts.nsec = smp->ts.origin.tv_nsec; - - memcpy(msg->data, smp->data, MSG_DATA_LEN(smp->length)); - - msg_hton(msg); - - return 0; -} - -ssize_t msg_buffer_from_samples(struct sample *smps[], unsigned cnt, char *buf, size_t len) -{ - int ret, i = 0; - char *ptr = buf; - - struct msg *msg = (struct msg *) ptr; - struct sample *smp = smps[i]; - - while (ptr < buf + len && i < cnt) { - ret = msg_from_sample(msg, smp); - if (ret) - return ret; - - ptr += MSG_LEN(smp->length); - - msg = (struct msg *) ptr; - smp = smps[++i]; - } - - return ptr - buf; -} - -int msg_buffer_to_samples(struct sample *smps[], unsigned cnt, char *buf, size_t len) -{ - int ret, i = 0; - char *ptr = buf; - - struct msg *msg = (struct msg *) ptr; - struct sample *smp = smps[i]; - - while (ptr < buf + len && i < cnt) { - ret = msg_to_sample(msg, smp); - if (ret) - return ret; - - ptr += MSG_LEN(smp->length); - - msg = (struct msg *) ptr; - smp = smps[++i]; - } - - return i; -} diff --git a/lib/node.c b/lib/node.c index c8a2c43e9..bb3e02d84 100644 --- a/lib/node.c +++ b/lib/node.c @@ -21,7 +21,6 @@ *********************************************************************************/ #include -#include #include "sample.h" #include "node.h" @@ -38,6 +37,10 @@ int node_init(struct node *n, struct node_type *vt) n->_vt = vt; n->_vd = alloc(vt->size); + + n->name = NULL; + n->_name = NULL; + n->_name_long = NULL; n->id = max_id++; @@ -51,29 +54,32 @@ int node_init(struct node *n, struct node_type *vt) return 0; } -int node_parse(struct node *n, config_setting_t *cfg) +int node_parse(struct node *n, json_t *cfg, const char *name) { struct plugin *p; - const char *type, *name; int ret; - name = config_setting_name(cfg); + json_error_t err; - if (!config_setting_lookup_string(cfg, "type", &type)) - cerror(cfg, "Missing node type"); + const char *type; + + n->name = strdup(name); + + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: i }", + "type", &type, + "vectorize", &n->vectorize + ); + if (ret) + jerror(&err, "Failed to parse node '%s'", node_name(n)); p = plugin_lookup(PLUGIN_TYPE_NODE, type); assert(&p->node == n->_vt); - config_setting_lookup_int(cfg, "vectorize", &n->vectorize); - - n->name = name; - n->cfg = cfg; - ret = n->_vt->parse ? n->_vt->parse(n, cfg) : 0; if (ret) - cerror(cfg, "Failed to parse node '%s'", node_name(n)); + error("Failed to parse node '%s'", node_name(n)); + n->cfg = cfg; n->state = STATE_PARSED; return ret; @@ -85,10 +91,9 @@ int node_parse_cli(struct node *n, int argc, char *argv[]) assert(n->_vt); - n->vectorize = 1; - n->name = "cli"; - if (n->_vt->parse_cli) { + n->name = strdup("cli"); + ret = n->_vt->parse_cli(n, argc, argv); if (ret) return ret; @@ -96,21 +101,11 @@ int node_parse_cli(struct node *n, int argc, char *argv[]) n->state = STATE_PARSED; } else { - config_t cfg; - config_setting_t *cfg_root; + n->cfg = json_load_cli(argc, argv); + if (!n->cfg) + return -1; - config_init(&cfg); - - ret = config_read_cli(&cfg, argc, argv); - if (ret) - goto out; - - cfg_root = config_root_setting(&cfg); - - ret = node_parse(n, cfg_root); - -out: - config_destroy(&cfg); + ret = node_parse(n, n->cfg, "cli"); } return ret; @@ -188,6 +183,9 @@ int node_destroy(struct node *n) if (n->_name_long) free(n->_name_long); + if (n->name) + free(n->name); + n->state = STATE_DESTROYED; return 0; @@ -204,12 +202,18 @@ int node_read(struct node *n, struct sample *smps[], unsigned cnt) if (n->_vt->vectorize > 0 && n->_vt->vectorize < cnt) { while (cnt - nread > 0) { readd = n->_vt->read(n, &smps[nread], MIN(cnt - nread, n->_vt->vectorize)); + if (readd < 0) + return readd; + nread += readd; debug(LOG_NODES | 5, "Received %u samples from node %s", readd, node_name(n)); } } else { nread = n->_vt->read(n, smps, cnt); + if (nread < 0) + return nread; + debug(LOG_NODES | 5, "Received %u samples from node %s", nread, node_name(n)); } @@ -230,12 +234,18 @@ int node_write(struct node *n, struct sample *smps[], unsigned cnt) if (n->_vt->vectorize > 0 && n->_vt->vectorize < cnt) { while (cnt - nsent > 0) { sent = n->_vt->write(n, &smps[nsent], MIN(cnt - nsent, n->_vt->vectorize)); + if (sent < 0) + return sent; + nsent += sent; debug(LOG_NODES | 5, "Sent %u samples to node %s", sent, node_name(n)); } } else { nsent = n->_vt->write(n, smps, cnt); + if (nsent < 0) + return nsent; + debug(LOG_NODES | 5, "Sent %u samples to node %s", nsent, node_name(n)); } @@ -276,47 +286,57 @@ int node_reverse(struct node *n) return n->_vt->reverse ? n->_vt->reverse(n) : -1; } -int node_parse_list(struct list *list, config_setting_t *cfg, struct list *all) +int node_parse_list(struct list *list, json_t *cfg, struct list *all) { - const char *str; struct node *node; + const char *str; + char *allstr = NULL; - switch (config_setting_type(cfg)) { - case CONFIG_TYPE_STRING: - str = config_setting_get_string(cfg); - if (str) { - node = list_lookup(all, str); - if (node) - list_push(list, node); - else - cerror(cfg, "Unknown outgoing node '%s'", str); - } - else - cerror(cfg, "Invalid outgoing node"); + size_t index; + json_t *elm; + + switch (json_typeof(cfg)) { + case JSON_STRING: + str = json_string_value(cfg); + node = list_lookup(all, str); + if (!node) + goto invalid2; + + list_push(list, node); break; - case CONFIG_TYPE_ARRAY: - for (int i = 0; i < config_setting_length(cfg); i++) { - config_setting_t *elm = config_setting_get_elem(cfg, i); + case JSON_ARRAY: + json_array_foreach(cfg, index, elm) { + if (!json_is_string(elm)) + goto invalid; - str = config_setting_get_string(elm); - if (str) { - node = list_lookup(all, str); - if (!node) - cerror(elm, "Unknown outgoing node '%s'", str); - else if (node->_vt->write == NULL) - cerror(cfg, "Output node '%s' is not supported as a sink.", node_name(node)); + node = list_lookup(all, json_string_value(elm)); + if (!node) - list_push(list, node); - } - else - cerror(cfg, "Invalid outgoing node"); + + list_push(list, node); } break; default: - cerror(cfg, "Invalid output node(s)"); + goto invalid; } + return 0; + +invalid: + error("The node list must be an a single or an array of strings referring to the keys of the 'nodes' section"); + + return -1; + +invalid2: + for (size_t i = 0; i < list_length(all); i++) { + struct node *n = list_at(all, i); + + strcatf(&allstr, " %s", node_name_short(n)); + } + + error("Unknown node '%s'. Choose of one of: %s", str, allstr); + return 0; } diff --git a/lib/nodes/Makefile.inc b/lib/nodes/Makefile.inc index 54c0ee57f..8f4a6984a 100644 --- a/lib/nodes/Makefile.inc +++ b/lib/nodes/Makefile.inc @@ -23,12 +23,13 @@ LIB_SRCS += $(addprefix lib/nodes/, cbuilder.c loopback.c) ifeq ($(PLATFORM),Linux) - WITH_FILE ?= 1 - WITH_SIGNAL ?= 1 - WITH_NGSI ?= 1 WITH_FPGA ?= 1 endif +WITH_TEST_RTT ?= 0 +WITH_FILE ?= 1 +WITH_SIGNAL ?= 1 +WITH_NGSI ?= 1 WITH_WEBSOCKET ?= 1 WITH_SOCKET ?= 1 WITH_ZEROMQ ?= 1 @@ -53,6 +54,12 @@ ifeq ($(WITH_SIGNAL),1) LIB_CFLAGS += -DWITH_SIGNAL endif +# Enable RTT test node-tyoe +ifeq ($(WITH_TEST_RTT),1) + LIB_SRCS += lib/nodes/test_rtt.c + LIB_CFLAGS += -DWITH_TEST_RTT +endif + # Enable VILLASfpga support when libxil is available ifeq ($(WITH_FPGA),1) ifeq ($(shell $(PKGCONFIG) libxil; echo $$?),0) @@ -83,8 +90,7 @@ endif # Enable Socket node type when libnl3 is available ifeq ($(WITH_SOCKET),1) - LIB_SRCS += $(addprefix lib/nodes/, socket.c) - LIB_SRCS += $(addprefix lib/, msg.c) + LIB_SRCS += lib/nodes/socket.c LIB_CFLAGS += -DWITH_SOCKET # libnl3 is optional but required for network emulation and IRQ pinning @@ -128,7 +134,7 @@ endif # Enable WebSocket support ifeq ($(WITH_WEBSOCKET),1) ifeq ($(shell $(PKGCONFIG) libwebsockets jansson; echo $$?),0) - LIB_SRCS += lib/nodes/websocket.c lib/webmsg.c + LIB_SRCS += lib/nodes/websocket.c LIB_PKGS += libwebsockets jansson LIB_CFLAGS += -DWITH_WEBSOCKET endif diff --git a/lib/nodes/cbuilder.c b/lib/nodes/cbuilder.c index 48e4032e9..759851d28 100644 --- a/lib/nodes/cbuilder.c +++ b/lib/nodes/cbuilder.c @@ -11,33 +11,43 @@ #include "nodes/cbuilder.h" -int cbuilder_parse(struct node *n, config_setting_t *cfg) +int cbuilder_parse(struct node *n, json_t *cfg) { struct cbuilder *cb = n->_vd; - config_setting_t *cfg_params; + json_t *cfg_param, *cfg_params = NULL; const char *model; - if (!config_setting_lookup_float(cfg, "timestep", &cb->timestep)) - cerror(cfg, "CBuilder model requires 'timestep' setting"); + int ret; + size_t index; + json_error_t err; - if (!config_setting_lookup_string(cfg, "model", &model)) - cerror(cfg, "CBuilder model requires 'model' setting"); + ret = json_unpack_ex(cfg, &err, 0, "{ s: f, s: s, s: b }", + "timestep", &cb->timestep, + "model", &model, + "parameters", &cfg_params + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); cb->model = (struct cbuilder_model *) plugin_lookup(PLUGIN_TYPE_MODEL_CBUILDER, model); if (!cb->model) - cerror(cfg, "Unknown model '%s'", model); + error("Unknown model '%s' of node %s", model, node_name(n)); - cfg_params = config_setting_get_member(cfg, "parameters"); if (cfg_params) { - if (!config_setting_is_array(cfg_params)) - cerror(cfg_params, "Model parameters must be an array of numbers!"); + if (!json_is_array(cfg_params)) + error("Setting 'parameters' of node %s must be an JSON array of numbers!", node_name(n)); - cb->paramlen = config_setting_length(cfg_params); + cb->paramlen = json_array_size(cfg_params); cb->params = alloc(cb->paramlen * sizeof(double)); - for (int i = 0; i < cb->paramlen; i++) - cb->params[i] = config_setting_get_float_elem(cfg_params, i); + json_array_foreach(cfg_params, index, cfg_param) { + if (json_is_number(cfg_param)) + error("Setting 'parameters' of node %s must be an JSON array of numbers!", node_name(n)); + + cb->params[index] = json_number_value(cfg_params); + + } } return 0; @@ -84,8 +94,15 @@ int cbuilder_read(struct node *n, struct sample *smps[], unsigned cnt) pthread_mutex_lock(&cb->mtx); while (cb->read >= cb->step) pthread_cond_wait(&cb->cv, &cb->mtx); + + float data[smp->capacity]; - smp->length = cb->model->read(&smp->data[0].f, 16); + smp->length = cb->model->read(data, smp->capacity); + + /* Cast float -> double */ + for (int i = 0; i < smp->length; i++) + smp->data[i].f = data[i]; + smp->sequence = cb->step; cb->read = cb->step; @@ -101,8 +118,10 @@ int cbuilder_write(struct node *n, struct sample *smps[], unsigned cnt) struct sample *smp = smps[0]; pthread_mutex_lock(&cb->mtx); + + float flt = smp->data[0].f; - cb->model->write(&smp->data[0].f, smp->length); + cb->model->write(&flt, smp->length); cb->model->code(); cb->step++; diff --git a/lib/nodes/file.c b/lib/nodes/file.c index db4a977b6..061f546d4 100644 --- a/lib/nodes/file.c +++ b/lib/nodes/file.c @@ -29,19 +29,7 @@ #include "timing.h" #include "queue.h" #include "plugin.h" -#include "sample_io.h" - -int file_reverse(struct node *n) -{ - struct file *f = n->_vd; - struct file_direction tmp; - - tmp = f->read; - f->read = f->write; - f->write = tmp; - - return 0; -} +#include "io.h" static char * file_format_name(const char *format, struct timespec *ts) { @@ -56,34 +44,13 @@ static char * file_format_name(const char *format, struct timespec *ts) return buf; } -static AFILE * file_reopen(struct file_direction *dir) -{ - if (dir->handle) - afclose(dir->handle); - - return afopen(dir->uri, dir->mode); -} - -static int file_parse_direction(config_setting_t *cfg, struct file *f, int d) -{ - struct file_direction *dir = (d == FILE_READ) ? &f->read : &f->write; - - if (!config_setting_lookup_string(cfg, "uri", &dir->fmt)) - return -1; - - if (!config_setting_lookup_string(cfg, "mode", &dir->mode)) - dir->mode = (d == FILE_READ) ? "r" : "w+"; - - return 0; -} - -static struct timespec file_calc_read_offset(const struct timespec *first, const struct timespec *epoch, enum read_epoch_mode mode) +static struct timespec file_calc_offset(const struct timespec *first, const struct timespec *epoch, enum epoch_mode mode) { /* Get current time */ struct timespec now = time_now(); struct timespec offset; - /* Set read_offset depending on epoch_mode */ + /* Set offset depending on epoch_mode */ switch (mode) { case FILE_EPOCH_DIRECT: /* read first value at now + epoch */ offset = time_diff(first, &now); @@ -96,7 +63,7 @@ static struct timespec file_calc_read_offset(const struct timespec *first, const case FILE_EPOCH_RELATIVE: /* read first value at first + epoch */ return *epoch; - case FILE_EPOCH_ABSOLUTE: /* read first value at f->read_epoch */ + case FILE_EPOCH_ABSOLUTE: /* read first value at f->epoch */ return time_diff(first, epoch); default: @@ -104,68 +71,68 @@ static struct timespec file_calc_read_offset(const struct timespec *first, const } } -int file_parse(struct node *n, config_setting_t *cfg) +int file_parse(struct node *n, json_t *cfg) { struct file *f = n->_vd; - config_setting_t *cfg_in, *cfg_out; + int ret; + json_error_t err; - cfg_out = config_setting_get_member(cfg, "out"); - if (cfg_out) { - if (file_parse_direction(cfg_out, f, FILE_WRITE)) - cerror(cfg_out, "Failed to parse output file for node %s", node_name(n)); + const char *uri_tmpl = NULL; + const char *format = "villas"; + const char *eof = NULL; + const char *epoch_mode = NULL; + double epoch_flt = 0; - if (!config_setting_lookup_bool(cfg_out, "flush", &f->flush)) - f->flush = 0; + /* Default values */ + f->rate = 0; + f->eof = FILE_EOF_EXIT; + f->epoch_mode = FILE_EPOCH_DIRECT; + f->flush = 0; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: b, s?: s, s?: f, s?: s, s?: f, s?: s }", + "uri", &uri_tmpl, + "flush", &f->flush, + "eof", &eof, + "rate", &f->rate, + "epoch_mode", &epoch_mode, + "epoch", &epoch_flt, + "format", &format + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + + f->epoch = time_from_double(epoch_flt); + f->uri_tmpl = uri_tmpl ? strdup(uri_tmpl) : NULL; + + f->format = io_format_lookup(format); + if (!f->format) + error("Invalid format '%s' for node %s", format, node_name(n)); + + if (eof) { + if (!strcmp(eof, "exit")) + f->eof = FILE_EOF_EXIT; + else if (!strcmp(eof, "rewind")) + f->eof = FILE_EOF_REWIND; + else if (!strcmp(eof, "wait")) + f->eof = FILE_EOF_WAIT; + else + error("Invalid mode '%s' for 'eof' setting of node %s", eof, node_name(n)); } - cfg_in = config_setting_get_member(cfg, "in"); - if (cfg_in) { - const char *eof; - - if (file_parse_direction(cfg_in, f, FILE_READ)) - cerror(cfg_in, "Failed to parse input file for node %s", node_name(n)); - - /* More read specific settings */ - if (config_setting_lookup_string(cfg_in, "eof", &eof)) { - if (!strcmp(eof, "exit")) - f->read_eof = FILE_EOF_EXIT; - else if (!strcmp(eof, "rewind")) - f->read_eof = FILE_EOF_REWIND; - else if (!strcmp(eof, "wait")) - f->read_eof = FILE_EOF_WAIT; - else - cerror(cfg_in, "Invalid mode '%s' for 'eof' setting", eof); - } + if (epoch_mode) { + if (!strcmp(epoch_mode, "direct")) + f->epoch_mode = FILE_EPOCH_DIRECT; + else if (!strcmp(epoch_mode, "wait")) + f->epoch_mode = FILE_EPOCH_WAIT; + else if (!strcmp(epoch_mode, "relative")) + f->epoch_mode = FILE_EPOCH_RELATIVE; + else if (!strcmp(epoch_mode, "absolute")) + f->epoch_mode = FILE_EPOCH_ABSOLUTE; + else if (!strcmp(epoch_mode, "original")) + f->epoch_mode = FILE_EPOCH_ORIGINAL; else - f->read_eof = FILE_EOF_EXIT; - - if (!config_setting_lookup_float(cfg_in, "rate", &f->read_rate)) - f->read_rate = 0; /* Disable fixed rate sending. Using timestamps of file instead */ - - double epoch_flt; - if (!config_setting_lookup_float(cfg_in, "epoch", &epoch_flt)) - epoch_flt = 0; - - f->read_epoch = time_from_double(epoch_flt); - - const char *epoch_mode; - if (config_setting_lookup_string(cfg_in, "epoch_mode", &epoch_mode)) { - if (!strcmp(epoch_mode, "direct")) - f->read_epoch_mode = FILE_EPOCH_DIRECT; - else if (!strcmp(epoch_mode, "wait")) - f->read_epoch_mode = FILE_EPOCH_WAIT; - else if (!strcmp(epoch_mode, "relative")) - f->read_epoch_mode = FILE_EPOCH_RELATIVE; - else if (!strcmp(epoch_mode, "absolute")) - f->read_epoch_mode = FILE_EPOCH_ABSOLUTE; - else if (!strcmp(epoch_mode, "original")) - f->read_epoch_mode = FILE_EPOCH_ORIGINAL; - else - cerror(cfg_in, "Invalid value '%s' for setting 'epoch_mode'", epoch_mode); - } - else - f->read_epoch_mode = FILE_EPOCH_DIRECT; + error("Invalid value '%s' for setting 'epoch_mode' of node %s", epoch_mode, node_name(n)); } n->_vd = f; @@ -178,53 +145,46 @@ char * file_print(struct node *n) struct file *f = n->_vd; char *buf = NULL; - if (f->read.fmt) { - const char *epoch_str = NULL; - switch (f->read_epoch_mode) { - case FILE_EPOCH_DIRECT: epoch_str = "direct"; break; - case FILE_EPOCH_WAIT: epoch_str = "wait"; break; - case FILE_EPOCH_RELATIVE: epoch_str = "relative"; break; - case FILE_EPOCH_ABSOLUTE: epoch_str = "absolute"; break; - case FILE_EPOCH_ORIGINAL: epoch_str = "original"; break; - } + const char *epoch_str = NULL; + const char *eof_str = NULL; - const char *eof_str = NULL; - switch (f->read_eof) { - case FILE_EOF_EXIT: eof_str = "exit"; break; - case FILE_EOF_WAIT: eof_str = "wait"; break; - case FILE_EOF_REWIND: eof_str = "rewind"; break; - } - - strcatf(&buf, "in=%s, mode=%s, eof=%s, epoch_mode=%s, epoch=%.2f", - f->read.uri ? f->read.uri : f->read.fmt, - f->read.mode, - eof_str, - epoch_str, - time_to_double(&f->read_epoch) - ); - - if (f->read_rate) - strcatf(&buf, ", rate=%.1f", f->read_rate); + switch (f->epoch_mode) { + case FILE_EPOCH_DIRECT: epoch_str = "direct"; break; + case FILE_EPOCH_WAIT: epoch_str = "wait"; break; + case FILE_EPOCH_RELATIVE: epoch_str = "relative"; break; + case FILE_EPOCH_ABSOLUTE: epoch_str = "absolute"; break; + case FILE_EPOCH_ORIGINAL: epoch_str = "original"; break; } - if (f->write.fmt) { - strcatf(&buf, ", out=%s, mode=%s", - f->write.uri ? f->write.uri : f->write.fmt, - f->write.mode - ); + switch (f->eof) { + case FILE_EOF_EXIT: eof_str = "exit"; break; + case FILE_EOF_WAIT: eof_str = "wait"; break; + case FILE_EOF_REWIND: eof_str = "rewind"; break; } - if (f->read_first.tv_sec || f->read_first.tv_nsec) - strcatf(&buf, ", first=%.2f", time_to_double(&f->read_first)); + strcatf(&buf, "uri=%s, format=%s, flush=%s, eof=%s, epoch_mode=%s, epoch=%.2f", + f->uri ? f->uri : f->uri_tmpl, + plugin_name(f->format), + f->flush ? "yes" : "no", + eof_str, + epoch_str, + time_to_double(&f->epoch) + ); - if (f->read_offset.tv_sec || f->read_offset.tv_nsec) - strcatf(&buf, ", offset=%.2f", time_to_double(&f->read_offset)); + if (f->rate) + strcatf(&buf, ", rate=%.1f", f->rate); - if ((f->read_first.tv_sec || f->read_first.tv_nsec) && - (f->read_offset.tv_sec || f->read_offset.tv_nsec)) { + if (f->first.tv_sec || f->first.tv_nsec) + strcatf(&buf, ", first=%.2f", time_to_double(&f->first)); + + if (f->offset.tv_sec || f->offset.tv_nsec) + strcatf(&buf, ", offset=%.2f", time_to_double(&f->offset)); + + if ((f->first.tv_sec || f->first.tv_nsec) && + (f->offset.tv_sec || f->offset.tv_nsec)) { struct timespec eta, now = time_now(); - eta = time_add(&f->read_first, &f->read_offset); + eta = time_add(&f->first, &f->offset); eta = time_diff(&now, &eta); if (eta.tv_sec || eta.tv_nsec) @@ -239,50 +199,45 @@ int file_start(struct node *n) struct file *f = n->_vd; struct timespec now = time_now(); - int ret; + int ret, flags; - if (f->read.fmt) { - /* Prepare file name */ - f->read.uri = file_format_name(f->read.fmt, &now); + /* Prepare file name */ + f->uri = file_format_name(f->uri_tmpl, &now); - /* Open file */ - f->read.handle = file_reopen(&f->read); - if (!f->read.handle) - serror("Failed to open file for reading: '%s'", f->read.uri); + /* Open file */ + flags = IO_FORMAT_ALL; + if (f->flush) + flags |= IO_FLUSH; - /* Create timer */ - f->read_timer = f->read_rate - ? timerfd_create_rate(f->read_rate) - : timerfd_create(CLOCK_REALTIME, 0); - if (f->read_timer < 0) - serror("Failed to create timer"); + ret = io_init(&f->io, f->format, flags); + if (ret) + return ret; - arewind(f->read.handle); + ret = io_open(&f->io, f->uri); + if (ret) + return ret; - /* Get timestamp of first line */ - if (f->read_epoch_mode != FILE_EPOCH_ORIGINAL) { - struct sample s; - s.capacity = 0; + /* Create timer */ + ret = task_init(&f->task, f->rate, CLOCK_REALTIME); + if (ret) + serror("Failed to create timer"); - ret = sample_io_villas_fscan(f->read.handle->file, &s, NULL); - if (ret < 0) - error("Failed to read first timestamp of node %s", node_name(n)); + /* Get timestamp of first line */ + if (f->epoch_mode != FILE_EPOCH_ORIGINAL) { + io_rewind(&f->io); - f->read_first = s.ts.origin; - f->read_offset = file_calc_read_offset(&f->read_first, &f->read_epoch, f->read_epoch_mode); - arewind(f->read.handle); - } + struct sample s = { .capacity = 0 }; + struct sample *smps[] = { &s }; + + ret = io_scan(&f->io, smps, 1); + if (ret != 1) + error("Failed to read first timestamp of node %s", node_name(n)); + + f->first = s.ts.origin; + f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); } - if (f->write.fmt) { - /* Prepare file name */ - f->write.uri = file_format_name(f->write.fmt, &now); - - /* Open file */ - f->write.handle = file_reopen(&f->write); - if (!f->write.handle) - serror("Failed to open file for writing: '%s'", f->write.uri); - } + io_rewind(&f->io); return 0; } @@ -290,16 +245,19 @@ int file_start(struct node *n) int file_stop(struct node *n) { struct file *f = n->_vd; + int ret; - if (f->read_timer) - close(f->read_timer); - if (f->read.handle) - afclose(f->read.handle); - if (f->write.handle) - afclose(f->write.handle); + task_destroy(&f->task); - free(f->read.uri); - free(f->write.uri); + ret = io_close(&f->io); + if (ret) + return ret; + + ret = io_destroy(&f->io); + if (ret) + return ret; + + free(f->uri); return 0; } @@ -307,78 +265,78 @@ int file_stop(struct node *n) int file_read(struct node *n, struct sample *smps[], unsigned cnt) { struct file *f = n->_vd; - struct sample *s = smps[0]; - int values, flags; - uint64_t ex; + int ret; + uint64_t steps; - assert(f->read.handle); assert(cnt == 1); -retry: values = sample_io_villas_fscan(f->read.handle->file, s, &flags); /* Get message and timestamp */ - if (values < 0) { - if (afeof(f->read.handle)) { - switch (f->read_eof) { +retry: ret = io_scan(&f->io, smps, cnt); + if (ret <= 0) { + if (io_eof(&f->io)) { + switch (f->eof) { case FILE_EOF_REWIND: info("Rewind input file of node %s", node_name(n)); - f->read_offset = file_calc_read_offset(&f->read_first, &f->read_epoch, f->read_epoch_mode); - arewind(f->read.handle); + f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); + io_rewind(&f->io); goto retry; case FILE_EOF_WAIT: - usleep(10000); /* We wait 10ms before fetching again. */ - adownload(f->read.handle, 1); + /* We wait 10ms before fetching again. */ + usleep(100000); + + /* Try to download more data if this is a remote file. */ + if (f->io.mode == IO_MODE_ADVIO) + adownload(f->io.advio.input, 1); + goto retry; case FILE_EOF_EXIT: info("Reached end-of-file of node %s", node_name(n)); + killme(SIGTERM); pause(); - } } else - warn("Failed to read messages from node %s: reason=%d", node_name(n), values); + warn("Failed to read messages from node %s: reason=%d", node_name(n), ret); return 0; } - if (f->read_epoch_mode != FILE_EPOCH_ORIGINAL) { - if (!f->read_rate || aftell(f->read.handle) == 0) { - s->ts.origin = time_add(&s->ts.origin, &f->read_offset); + /* We dont wait in FILE_EPOCH_ORIGINAL mode */ + if (f->epoch_mode == FILE_EPOCH_ORIGINAL) + return cnt; - ex = timerfd_wait_until(f->read_timer, &s->ts.origin); - } - else { /* Wait with fixed rate delay */ - ex = timerfd_wait(f->read_timer); + if (f->rate) { + steps = task_wait_until_next_period(&f->task); - s->ts.origin = time_now(); - } + smps[0]->ts.origin = time_now(); + } + else { + smps[0]->ts.origin = time_add(&smps[0]->ts.origin, &f->offset); - /* Check for overruns */ - if (ex == 0) - serror("Failed to wait for timer"); - else if (ex != 1) - warn("Overrun: %" PRIu64, ex - 1); + steps = task_wait_until(&f->task, &smps[0]->ts.origin); } - return 1; + /* Check for overruns */ + if (steps == 0) + serror("Failed to wait for timer"); + else if (steps != 1) + warn("Missed steps: %" PRIu64, steps - 1); + + return cnt; } int file_write(struct node *n, struct sample *smps[], unsigned cnt) { struct file *f = n->_vd; - struct sample *s = smps[0]; - assert(f->write.handle); assert(cnt == 1); - sample_io_villas_fprint(f->write.handle->file, s, SAMPLE_IO_ALL & ~SAMPLE_IO_OFFSET); + io_print(&f->io, smps, cnt); - if (f->flush) - afflush(f->write.handle); - - return 1; + return cnt; } static struct plugin p = { @@ -388,7 +346,6 @@ static struct plugin p = { .node = { .vectorize = 1, .size = sizeof(struct file), - .reverse = file_reverse, .parse = file_parse, .print = file_print, .start = file_start, diff --git a/lib/nodes/fpga.c b/lib/nodes/fpga.c index 557d86620..76856e87e 100644 --- a/lib/nodes/fpga.c +++ b/lib/nodes/fpga.c @@ -34,7 +34,6 @@ void fpga_dump(struct fpga *f) int fpga_init(struct super_node *sn) { int ret; - config_setting_t *cfg, *cfg_fpgas; ret = pci_init(&pci); if (ret) @@ -44,6 +43,9 @@ int fpga_init(struct super_node *sn) if (ret) error("Failed to initiliaze VFIO sub-system"); +#if 0 + json_t *cfg, *cfg_fpgas; + /* Parse FPGA configuration */ cfg = config_root_setting(&sn->cfg); cfg_fpgas = config_setting_lookup(cfg, "fpgas"); @@ -53,7 +55,7 @@ int fpga_init(struct super_node *sn) ret = fpga_card_parse_list(&cards, cfg_fpgas); if (ret) cerror(cfg, "Failed to parse VILLASfpga config"); - +#endif return 0; } @@ -76,7 +78,7 @@ int fpga_deinit() return 0; } -int fpga_parse(struct node *n, config_setting_t *cfg) +int fpga_parse(struct node *n, json_t *cfg) { struct fpga *f = n->_vd; struct fpga_card *card; @@ -85,11 +87,18 @@ int fpga_parse(struct node *n, config_setting_t *cfg) char *cpy, *card_name, *ip_name; const char *dm; - if (!config_setting_lookup_string(cfg, "datamover", &dm)) - cerror(cfg, "Node '%s' is missing the 'datamover' setting", node_name(n)); + json_error_t err; + int ret; - if (!config_setting_lookup_bool(cfg, "use_irqs", &f->use_irqs)) - f->use_irqs = false; + /* Default values */ + f->use_irqs = false; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: b }", + "datamover", &dm, + "use_irqs", &f->use_irqs + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); cpy = strdup(dm); /* strtok can not operate on type const char * */ @@ -98,13 +107,13 @@ int fpga_parse(struct node *n, config_setting_t *cfg) card = list_lookup(&cards, card_name); if (!card) - cerror(cfg, "There is no FPGA card named '%s", card_name); + error("There is no FPGA card named '%s' used by node %s", card_name, node_name(n)); ip = list_lookup(&card->ips, ip_name); if (!ip) - cerror(cfg, "There is no datamover named '%s' on the FPGA card '%s'", ip_name, card_name); + error("There is no datamover named '%s' on the FPGA card '%s' used by node %s", ip_name, card_name, node_name(n)); if (ip->_vt->type != FPGA_IP_TYPE_DM_DMA && ip->_vt->type != FPGA_IP_TYPE_DM_FIFO) - cerror(cfg, "The IP '%s' on FPGA card '%s' is not a datamover", ip_name, card_name); + error("The IP '%s' on FPGA card '%s' is not a datamover", ip_name, card_name); free(cpy); diff --git a/lib/nodes/loopback.c b/lib/nodes/loopback.c index dcb3c2761..995a8d055 100644 --- a/lib/nodes/loopback.c +++ b/lib/nodes/loopback.c @@ -27,15 +27,23 @@ #include "nodes/loopback.h" #include "memory.h" -int loopback_parse(struct node *n, config_setting_t *cfg) +int loopback_parse(struct node *n, json_t *cfg) { struct loopback *l = n->_vd; - if (!config_setting_lookup_int(cfg, "queuelen", &l->queuelen)) - l->queuelen = DEFAULT_QUEUELEN; + json_error_t err; + int ret; - if (!config_setting_lookup_int(cfg, "samplelen", &l->samplelen)) - l->samplelen = DEFAULT_SAMPLELEN; + /* Default values */ + l->queuelen = DEFAULT_QUEUELEN; + l->samplelen = DEFAULT_SAMPLELEN; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: i }", + "queuelen", &l->queuelen, + "samplelen", &l->samplelen + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); return 0; } diff --git a/lib/nodes/nanomsg.c b/lib/nodes/nanomsg.c index c442ff2ea..734d84152 100644 --- a/lib/nodes/nanomsg.c +++ b/lib/nodes/nanomsg.c @@ -27,7 +27,7 @@ #include "plugin.h" #include "nodes/nanomsg.h" #include "utils.h" -#include "msg.h" +#include "io_format.h" int nanomsg_reverse(struct node *n) { @@ -46,25 +46,28 @@ int nanomsg_reverse(struct node *n) return 0; } -static int nanomsg_parse_endpoints(struct list *l, config_setting_t *cfg) +static int nanomsg_parse_endpoints(struct list *l, json_t *cfg) { const char *ep; - switch (config_setting_type(cfg)) { - case CONFIG_TYPE_LIST: - case CONFIG_TYPE_ARRAY: - for (int j = 0; j < config_setting_length(cfg); j++) { - const char *ep = config_setting_get_string_elem(cfg, j); + size_t index; + json_t *cfg_val; + + switch (json_typeof(cfg)) { + case JSON_ARRAY: + json_array_foreach(cfg, index, cfg_val) { + ep = json_string_value(cfg_val); + if (!ep) + return -1; list_push(l, strdup(ep)); } break; - case CONFIG_TYPE_STRING: - ep = config_setting_get_string(cfg); + case JSON_STRING: + ep = json_string_value(cfg); list_push(l, strdup(ep)); - break; default: @@ -74,29 +77,44 @@ static int nanomsg_parse_endpoints(struct list *l, config_setting_t *cfg) return 0; } -int nanomsg_parse(struct node *n, config_setting_t *cfg) +int nanomsg_parse(struct node *n, json_t *cfg) { int ret; struct nanomsg *m = n->_vd; - config_setting_t *cfg_pub, *cfg_sub; + const char *format = "villas"; + + json_error_t err; + + json_t *cfg_pub = NULL; + json_t *cfg_sub = NULL; list_init(&m->publisher.endpoints); list_init(&m->subscriber.endpoints); - cfg_pub = config_setting_lookup(cfg, "publish"); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: s }", + "publish", &cfg_pub, + "subscribe", &cfg_sub, + "format", &format + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + if (cfg_pub) { ret = nanomsg_parse_endpoints(&m->publisher.endpoints, cfg_pub); if (ret < 0) - cerror(cfg_pub, "Invalid type for 'publish' setting of node %s", node_name(n)); + error("Invalid type for 'publish' setting of node %s", node_name(n)); } - cfg_sub = config_setting_lookup(cfg, "subscribe"); if (cfg_sub) { ret = nanomsg_parse_endpoints(&m->subscriber.endpoints, cfg_sub); if (ret < 0) - cerror(cfg_pub, "Invalid type for 'subscribe' setting of node %s", node_name(n)); + error("Invalid type for 'subscribe' setting of node %s", node_name(n)); } + + m->format = io_format_lookup(format); + if (!m->format) + error("Invalid format '%s' for node %s", format, node_name(n)); return 0; } @@ -107,7 +125,7 @@ char * nanomsg_print(struct node *n) char *buf = NULL; - strcatf(&buf, "subscribe=[ "); + strcatf(&buf, "format=%s, subscribe=[ ", plugin_name(m->format)); for (size_t i = 0; i < list_length(&m->subscriber.endpoints); i++) { char *ep = list_at(&m->subscriber.endpoints, i); @@ -176,11 +194,11 @@ int nanomsg_stop(struct node *n) int ret; struct nanomsg *m = n->_vd; - ret = nn_shutdown(m->subscriber.socket, 0); + ret = nn_close(m->subscriber.socket); if (ret < 0) return ret; - ret = nn_shutdown(m->publisher.socket, 0); + ret = nn_close(m->publisher.socket); if (ret < 0) return ret; @@ -196,17 +214,16 @@ int nanomsg_deinit() int nanomsg_read(struct node *n, struct sample *smps[], unsigned cnt) { - int ret; struct nanomsg *m = n->_vd; - - char data[MSG_MAX_PACKET_LEN]; + int bytes; + char data[NANOMSG_MAX_PACKET_LEN]; /* Receive payload */ - ret = nn_recv(m->subscriber.socket, data, sizeof(data), 0); - if (ret < 0) - return ret; + bytes = nn_recv(m->subscriber.socket, data, sizeof(data), 0); + if (bytes < 0) + return -1; - return msg_buffer_to_samples(smps, cnt, data, ret); + return io_format_sscan(m->format, data, bytes, NULL, smps, cnt, NULL); } int nanomsg_write(struct node *n, struct sample *smps[], unsigned cnt) @@ -214,15 +231,15 @@ int nanomsg_write(struct node *n, struct sample *smps[], unsigned cnt) int ret; struct nanomsg *m = n->_vd; - ssize_t sent; + size_t wbytes; - char data[MSG_MAX_PACKET_LEN]; + char data[NANOMSG_MAX_PACKET_LEN]; - sent = msg_buffer_from_samples(smps, cnt, data, sizeof(data)); - if (sent < 0) + ret = io_format_sprint(m->format, data, sizeof(data), &wbytes, smps, cnt, IO_FORMAT_ALL); + if (ret <= 0) return -1; - ret = nn_send(m->publisher.socket, data, sent, 0); + ret = nn_send(m->publisher.socket, data, wbytes, 0); if (ret < 0) return ret; diff --git a/lib/nodes/ngsi.c b/lib/nodes/ngsi.c index 1483da599..0c2ba042d 100644 --- a/lib/nodes/ngsi.c +++ b/lib/nodes/ngsi.c @@ -197,26 +197,31 @@ static int ngsi_parse_entity(json_t *entity, struct ngsi *i, struct sample *smps return cnt; } -static int ngsi_parse_mapping(struct list *mapping, config_setting_t *cfg) +static int ngsi_parse_mapping(struct list *mapping, json_t *cfg) { - if (!config_setting_is_array(cfg)) + if (!json_is_array(cfg)) return -1; list_init(mapping); - for (int j = 0; j < config_setting_length(cfg); j++) { - const char *token = config_setting_get_string_elem(cfg, j); + size_t index; + json_t *cfg_token; + + json_array_foreach(cfg, index, cfg_token) { + const char *token; + + token = json_string_value(cfg_token); if (!token) return -2; struct ngsi_attribute *a = alloc(sizeof(struct ngsi_attribute)); - a->index = j; + a->index = index; /* Parse Attribute: AttributeName(AttributeType) */ int bytes; if (sscanf(token, "%m[^(](%m[^)])%n", &a->name, &a->type, &bytes) != 2) - cerror(cfg, "Invalid mapping token: '%s'", token); + error("Invalid mapping token: '%s'", token); token += bytes; @@ -241,7 +246,7 @@ static int ngsi_parse_mapping(struct list *mapping, config_setting_t *cfg) .name = "index", .type = "integer" }; - assert(asprintf(&i.value, "%u", j)); + assert(asprintf(&i.value, "%zu", index)); list_push(&a->metadata, memdup(&s, sizeof(s))); list_push(&a->metadata, memdup(&i, sizeof(i))); @@ -387,19 +392,6 @@ out: json_decref(request); int ngsi_init(struct super_node *sn) { - config_setting_t *cfg; - - cfg = config_root_setting(&sn->cfg); - - const char *tname; - if (config_setting_lookup_string(cfg, "name", &tname)) { - name = strdup(tname); - } - else { - name = alloc(128); /** @todo missing free */ - gethostname((char *) name, 128); - } - return curl_global_init(CURL_GLOBAL_ALL); } @@ -412,37 +404,36 @@ int ngsi_deinit() return 0; } -int ngsi_parse(struct node *n, config_setting_t *cfg) +int ngsi_parse(struct node *n, json_t *cfg) { struct ngsi *i = n->_vd; - if (!config_setting_lookup_string(cfg, "access_token", &i->access_token)) - i->access_token = NULL; /* disabled by default */ + int ret; + json_error_t err; + json_t *cfg_mapping; - if (!config_setting_lookup_string(cfg, "endpoint", &i->endpoint)) - cerror(cfg, "Missing NGSI endpoint for node %s", node_name(n)); + /* Default values */ + i->access_token = NULL; /* disabled by default */ + i->ssl_verify = 1; /* verify by default */ + i->timeout = 1; /* default value */ + i->rate = 5; /* default value */ - if (!config_setting_lookup_string(cfg, "entity_id", &i->entity_id)) - cerror(cfg, "Missing NGSI entity ID for node %s", node_name(n)); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s: s, s: s, s: s, s?: b, s?: f, s?: f }", + "access_token", &i->access_token, + "endpoint", &i->endpoint, + "entity_id", &i->entity_id, + "entity_type", &i->entity_type, + "ssl_verify", &i->ssl_verify, + "timeout", &i->timeout, + "rate", &i->rate, + "mapping", &cfg_mapping + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); - if (!config_setting_lookup_string(cfg, "entity_type", &i->entity_type)) - cerror(cfg, "Missing NGSI entity type for node %s", node_name(n)); - - if (!config_setting_lookup_bool(cfg, "ssl_verify", &i->ssl_verify)) - i->ssl_verify = 1; /* verify by default */ - - if (!config_setting_lookup_float(cfg, "timeout", &i->timeout)) - i->timeout = 1; /* default value */ - - if (!config_setting_lookup_float(cfg, "rate", &i->rate)) - i->rate = 5; /* default value */ - - config_setting_t *cfg_mapping = config_setting_get_member(cfg, "mapping"); - if (!cfg_mapping) - cerror(cfg, "Missing mapping for node %s", node_name(n)); - - if (ngsi_parse_mapping(&i->mapping, cfg_mapping)) - cerror(cfg_mapping, "Invalid mapping for node %s", node_name(n)); + ret = ngsi_parse_mapping(&i->mapping, cfg_mapping); + if (ret) + error("Invalid setting 'mapping' of node %s", node_name(n)); return 0; } @@ -497,13 +488,13 @@ int ngsi_start(struct node *n) i->headers = curl_slist_append(i->headers, buf); } - /* Create timer */ + /* Create task */ if (i->timeout > 1 / i->rate) warn("Timeout is to large for given rate: %f", i->rate); - i->tfd = timerfd_create_rate(i->rate); - if (i->tfd < 0) - serror("Failed to create timer"); + ret = task_init(&i->task, i->rate, CLOCK_MONOTONIC); + if (ret) + serror("Failed to create task"); i->headers = curl_slist_append(i->headers, "Accept: application/json"); i->headers = curl_slist_append(i->headers, "Content-Type: application/json"); @@ -548,8 +539,8 @@ int ngsi_read(struct node *n, struct sample *smps[], unsigned cnt) struct ngsi *i = n->_vd; int ret; - if (timerfd_wait(i->tfd) == 0) - perror("Failed to wait for timer"); + if (task_wait_until_next_period(&i->task) == 0) + perror("Failed to wait for task"); json_t *rentity; json_t *entity = ngsi_build_entity(i, NULL, 0, 0); @@ -602,4 +593,3 @@ static struct plugin p = { REGISTER_PLUGIN(&p) LIST_INIT_STATIC(&p.node.instances) - diff --git a/lib/nodes/opal.c b/lib/nodes/opal.c index 6fe47f826..e48004be9 100644 --- a/lib/nodes/opal.c +++ b/lib/nodes/opal.c @@ -144,13 +144,20 @@ int opal_print_global() return 0; } -int opal_parse(struct node *n, config_setting_t *cfg) +int opal_parse(struct node *n, json_t *cfg) { struct opal *o = n->_vd; - config_setting_lookup_int(cfg, "send_id", &o->send_id); - config_setting_lookup_int(cfg, "recv_id", &o->recv_id); - config_setting_lookup_bool(cfg, "reply", &o->reply); + int ret; + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s: i, s: b }", + "send_id", &o->send_id, + "recv_id", &o->recv_id, + "reply", &o->reply + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); return 0; } diff --git a/lib/nodes/shmem.c b/lib/nodes/shmem.c index 1fcbc6054..6b16a843f 100644 --- a/lib/nodes/shmem.c +++ b/lib/nodes/shmem.c @@ -36,44 +36,49 @@ #include "timing.h" #include "utils.h" -int shmem_parse(struct node *n, config_setting_t *cfg) +int shmem_parse(struct node *n, json_t *cfg) { struct shmem *shm = n->_vd; + const char *val; - if (!config_setting_lookup_string(cfg, "out_name", &shm->out_name)) - cerror(cfg, "Missing shared memory output queue name"); + int ret; + json_t *cfg_exec = NULL; + json_error_t err; - if (!config_setting_lookup_string(cfg, "in_name", &shm->in_name)) - cerror(cfg, "Missing shared memory input queue name"); + /* Default values */ + shm->conf.queuelen = MAX(DEFAULT_SHMEM_QUEUELEN, n->vectorize); + shm->conf.samplelen = DEFAULT_SHMEM_SAMPLELEN; + shm->conf.polling = false; + shm->exec = NULL; - if (!config_setting_lookup_int(cfg, "queuelen", &shm->conf.queuelen)) - shm->conf.queuelen = MAX(DEFAULT_SHMEM_QUEUELEN, n->vectorize); + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s: s, s?: i, s?: i, s?: b, s?: o }", + "out_name", &shm->out_name, + "in_name", &shm->in_name, + "queuelen", &shm->conf.queuelen, + "samplelen", &shm->conf.samplelen, + "polling", &shm->conf.polling, + "exec", &cfg_exec + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); - if (!config_setting_lookup_int(cfg, "samplelen", &shm->conf.samplelen)) - shm->conf.samplelen = DEFAULT_SHMEM_SAMPLELEN; + if (cfg_exec) { + if (!json_is_array(cfg_exec)) + error("Setting 'exec' of node %s must be a JSON array of strings", node_name(n)); - if (!config_setting_lookup_bool(cfg, "polling", &shm->conf.polling)) - shm->conf.polling = false; + shm->exec = alloc(sizeof(char *) * (json_array_size(cfg_exec) + 1)); - config_setting_t *exec_cfg = config_setting_lookup(cfg, "exec"); - if (!exec_cfg) - shm->exec = NULL; - else { - if (!config_setting_is_array(exec_cfg)) - cerror(exec_cfg, "Invalid format for exec"); + size_t index; + json_t *cfg_val; + json_array_foreach(cfg_exec, index, cfg_val) { + val = json_string_value(cfg_exec); + if (!val) + error("Setting 'exec' of node %s must be a JSON array of strings", node_name(n)); - shm->exec = alloc(sizeof(char *) * (config_setting_length(exec_cfg) + 1)); - - int i; - for (i = 0; i < config_setting_length(exec_cfg); i++) { - const char *elm = config_setting_get_string_elem(exec_cfg, i); - if (!elm) - cerror(exec_cfg, "Invalid format for exec"); - - shm->exec[i] = strdup(elm); + shm->exec[index] = strdup(val); } - shm->exec[i] = NULL; + shm->exec[index] = NULL; } return 0; diff --git a/lib/nodes/signal.c b/lib/nodes/signal.c index e3d11d489..cba5dff57 100644 --- a/lib/nodes/signal.c +++ b/lib/nodes/signal.c @@ -45,43 +45,46 @@ enum signal_type signal_lookup_type(const char *type) return -1; } -int signal_parse(struct node *n, config_setting_t *cfg) +int signal_parse(struct node *n, json_t *cfg) { struct signal *s = n->_vd; int ret; - const char *type; + const char *type = NULL; - if (!config_setting_lookup_string(cfg, "signal", &type)) - s->type = SIGNAL_TYPE_MIXED; - else { + json_error_t err; + + s->rt = 1; + s->limit = -1; + s->values = 1; + s->rate = 10; + s->frequency = 1; + s->amplitude = 1; + s->stddev = 0.02; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: b, s?: i, s?: i, s?: f, s?: f, s?: f, s?: f }", + "signal", &type, + "realtime", &s->rt, + "limit", &s->limit, + "values", &s->values, + "rate", &s->rate, + "frequency", &s->frequency, + "amplitude", &s->amplitude, + "stddev", &s->stddev + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + + + if (type) { ret = signal_lookup_type(type); if (ret == -1) - cerror(cfg, "Unknown signal type '%s'", type); + error("Unknown signal type '%s' of node %s", type, node_name(n)); s->type = ret; } - - if (!config_setting_lookup_bool(cfg, "realtime", &s->rt)) - s->rt = 1; - - if (!config_setting_lookup_int(cfg, "limit", &s->limit)) - s->limit = -1; - - if (!config_setting_lookup_int(cfg, "values", &s->values)) - s->values = 1; - - if (!config_setting_lookup_float(cfg, "rate", &s->rate)) - s->rate = 10; - - if (!config_setting_lookup_float(cfg, "frequency", &s->frequency)) - s->frequency = 1; - - if (!config_setting_lookup_float(cfg, "amplitude", &s->amplitude)) - s->amplitude = 1; - - if (!config_setting_lookup_float(cfg, "stddev", &s->stddev)) - s->stddev = 0.02; + else + s->type = SIGNAL_TYPE_MIXED; return 0; } @@ -154,29 +157,33 @@ check: if (optarg == endptr) int signal_open(struct node *n) { + int ret; struct signal *s = n->_vd; s->counter = 0; s->started = time_now(); - /* Setup timer */ + /* Setup task */ if (s->rt) { - s->tfd = timerfd_create_rate(s->rate); - if (s->tfd < 0) - return -1; + ret = task_init(&s->task, s->rate, CLOCK_MONOTONIC); + if (ret) + return ret; } - else - s->tfd = -1; return 0; } int signal_close(struct node *n) { + int ret; struct signal* s = n->_vd; - close(s->tfd); - + if (s->rt) { + ret = task_destroy(&s->task); + if (ret) + return ret; + } + return 0; } @@ -193,7 +200,7 @@ int signal_read(struct node *n, struct sample *smps[], unsigned cnt) /* Throttle output if desired */ if (s->rt) { /* Block until 1/p->rate seconds elapsed */ - steps = timerfd_wait(s->tfd); + steps = task_wait_until_next_period(&s->task); if (steps > 1) warn("Missed steps: %u", steps); @@ -226,7 +233,7 @@ int signal_read(struct node *n, struct sample *smps[], unsigned cnt) } if (s->limit > 0 && s->counter >= s->limit) { - info("Reached limit"); + info("Reached limit of node %s", node_name(n)); killme(SIGTERM); pause(); } diff --git a/lib/nodes/socket.c b/lib/nodes/socket.c index f972f7be6..4035694c4 100644 --- a/lib/nodes/socket.c +++ b/lib/nodes/socket.c @@ -34,16 +34,13 @@ #include "config.h" #include "utils.h" -#ifdef WITH_LIBNL_ROUTE_30 +#ifdef WITH_NETEM #include "kernel/if.h" #include "kernel/nl.h" #include "kernel/tc.h" +#endif /* WITH_NETEM */ - #define WITH_NETEM -#endif /* WITH_LIBNL_ROUTE_30 */ - -#include "msg.h" -#include "msg_format.h" +#include "io_format.h" #include "sample.h" #include "queue.h" #include "plugin.h" @@ -127,7 +124,7 @@ int socket_deinit() char * socket_print(struct node *n) { struct socket *s = n->_vd; - char *layer = NULL, *header = NULL, *endian = NULL, *buf; + char *layer = NULL, *buf; switch (s->layer) { case SOCKET_LAYER_UDP: layer = "udp"; break; @@ -135,25 +132,10 @@ char * socket_print(struct node *n) case SOCKET_LAYER_ETH: layer = "eth"; break; } - switch (s->header) { - case SOCKET_HEADER_NONE: header = "none"; break; - case SOCKET_HEADER_FAKE: header = "fake"; break; - case SOCKET_HEADER_DEFAULT: header = "default"; break; - } - - if (s->header == SOCKET_HEADER_DEFAULT) - endian = "auto"; - else { - switch (s->endian) { - case SOCKET_ENDIAN_LITTLE: endian = "little"; break; - case SOCKET_ENDIAN_BIG: endian = "big"; break; - } - } - char *local = socket_print_addr((struct sockaddr *) &s->local); char *remote = socket_print_addr((struct sockaddr *) &s->remote); - buf = strf("layer=%s, header=%s, endian=%s, local=%s, remote=%s", layer, header, endian, local, remote); + buf = strf("layer=%s, format=%s, local=%s, remote=%s", layer, plugin_name(s->format), local, remote); if (s->multicast.enabled) { char group[INET_ADDRSTRLEN]; @@ -320,125 +302,23 @@ int socket_destroy(struct node *n) return 0; } -static int socket_read_none(struct node *n, struct sample *smps[], unsigned cnt) -{ - int length; - struct socket *s = n->_vd; - - char buf[MSG_MAX_PACKET_LEN]; - uint32_t *values = (uint32_t *) buf; - ssize_t bytes; - - struct sample *smp = smps[0]; - - if (cnt < 1) - return 0; - - union sockaddr_union src; - socklen_t srclen = sizeof(src); - - /* Receive next sample */ - bytes = recvfrom(s->sd, buf, sizeof(buf), 0, &src.sa, &srclen); - if (bytes == 0) - error("Remote node %s closed the connection", node_name(n)); /** @todo Should we really hard fail here? */ - else if (bytes < 0) - serror("Failed recv from node %s", node_name(n)); - else if (bytes % 4 != 0) { - warn("Packet size is invalid: %zd Must be multiple of 4 bytes.", bytes); - recv(s->sd, NULL, 0, 0); /* empty receive buffer */ - return -1; - } - - length = bytes / 4; - - /* Strip IP header from packet */ - if (s->layer == SOCKET_LAYER_IP) { - struct ip *iphdr = (struct ip *) buf; - - length -= iphdr->ip_hl; - values += iphdr->ip_hl; - } - - /* SOCK_RAW IP sockets to not provide the IP protocol number via recvmsg() - * So we simply set it ourself. */ - if (s->layer == SOCKET_LAYER_IP) { - switch (src.sa.sa_family) { - case AF_INET: src.sin.sin_port = s->remote.sin.sin_port; break; - case AF_INET6: src.sin6.sin6_port = s->remote.sin6.sin6_port; break; - } - } - - if (s->verify_source && socket_compare_addr(&src.sa, &s->remote.sa) != 0) { - char *buf = socket_print_addr((struct sockaddr *) &src); - warn("Received packet from unauthorized source: %s", buf); - free(buf); - - return 0; - } - - /* Convert packet contents to host endianess */ - for (int i = 0; i < length; i++) - values[i] = s->endian == SOCKET_ENDIAN_BIG - ? be32toh(values[i]) - : le32toh(values[i]); - - if (s->header == SOCKET_HEADER_FAKE) { - if (length < 3) { - warn("Node %s received a packet with no fake header. Skipping...", node_name(n)); - return 0; - } - - smp->sequence = values[0]; - smp->ts.origin.tv_sec = values[1]; - smp->ts.origin.tv_nsec = values[2]; - - values += 3; - length -= 3; - } - else { - smp->sequence = n->sequence++; /* Fake sequence no generated by VILLASnode */ - smp->ts.origin.tv_sec = 0; - smp->ts.origin.tv_nsec = 0; - } - - if (length > smp->capacity) { - warn("Node %s received more values than supported. Dropping %u values", node_name(n), length - smp->capacity); - length = smp->capacity; - } - - memcpy(smp->data, values, SAMPLE_DATA_LEN(length)); - - smp->ts.received.tv_sec = 0; - smp->ts.received.tv_nsec = 0; - - smp->length = length; - - return 1; /* GTNET-SKT sends every sample in a single packet */ -} - -static int socket_read_villas(struct node *n, struct sample *smps[], unsigned cnt) +int socket_read(struct node *n, struct sample *smps[], unsigned cnt) { int ret; struct socket *s = n->_vd; - char buf[MSG_MAX_PACKET_LEN]; + char buf[SOCKET_MAX_PACKET_LEN]; char *bufptr = buf; ssize_t bytes; + size_t rbytes; union sockaddr_union src; socklen_t srclen = sizeof(src); /* Receive next sample */ bytes = recvfrom(s->sd, bufptr, sizeof(buf), 0, &src.sa, &srclen); - if (bytes == 0) - error("Remote node %s closed the connection", node_name(n)); /** @todo Should we really hard fail here? */ - else if (bytes < 0) + if (bytes < 0) serror("Failed recv from node %s", node_name(n)); - else if (bytes % 4 != 0) { - warn("Packet size is invalid: %zd Must be multiple of 4 bytes.", bytes); - recv(s->sd, NULL, 0, 0); /* empty receive buffer */ - return -1; - } /* Strip IP header from packet */ if (s->layer == SOCKET_LAYER_IP) { @@ -465,114 +345,79 @@ static int socket_read_villas(struct node *n, struct sample *smps[], unsigned cn return 0; } - ret = msg_buffer_to_samples(smps, cnt, bufptr, bytes); - if (ret < 0) - warn("Received invalid packet from node: %s reason=%d", node_name(n), ret); + ret = io_format_sscan(s->format, bufptr, bytes, &rbytes, smps, cnt, NULL); + + if (bytes != rbytes) + warn("Received invalid packet from node: %s bytes=%zu, rbytes=%zu", node_name(n), bytes, rbytes); return ret; } -static int socket_write_none(struct node *n, struct sample *smps[], unsigned cnt) -{ - struct socket *s = n->_vd; - - int sent = 0; - ssize_t bytes; - - if (cnt < 1) - return 0; - - for (int i = 0; i < cnt; i++) { - int off = s->header == SOCKET_HEADER_FAKE ? 3 : 0; - int len = smps[i]->length + off; - uint32_t data[len]; - - /* First three values are sequence, seconds and nano-seconds timestamps */ - if (s->header == SOCKET_HEADER_FAKE) { - data[0] = smps[i]->sequence; - data[1] = smps[i]->ts.origin.tv_sec; - data[2] = smps[i]->ts.origin.tv_nsec; - } - - for (int j = 0; j < smps[i]->length; j++) - data[off + j] = s->endian == SOCKET_ENDIAN_BIG - ? htobe32(smps[i]->data[j].i) - : htole32(smps[i]->data[j].i); - - bytes = sendto(s->sd, data, len * sizeof(data[0]), 0, - (struct sockaddr *) &s->remote, sizeof(s->remote)); - if (bytes < 0) - serror("Failed send to node %s", node_name(n)); - - sent++; - } - - return sent; -} - -static int socket_write_villas(struct node *n, struct sample *smps[], unsigned cnt) -{ - struct socket *s = n->_vd; - - char data[MSG_MAX_PACKET_LEN]; - ssize_t bytes = 0, sent; - - sent = msg_buffer_from_samples(smps, cnt, data, sizeof(data)); - if (sent < 0) - return -1; - - /* Send message */ - bytes = sendto(s->sd, data, sent, 0, (struct sockaddr *) &s->remote, sizeof(s->remote)); - if (bytes < 0) - serror("Failed send to node %s", node_name(n)); - - return cnt; -} - -int socket_read(struct node *n, struct sample *smps[], unsigned cnt) -{ - struct socket *s = n->_vd; - - switch (s->header) { - case SOCKET_HEADER_NONE: - case SOCKET_HEADER_FAKE: - return socket_read_none(n, smps, cnt); - - case SOCKET_HEADER_DEFAULT: - return socket_read_villas(n, smps, cnt); - } - - return -1; -} - int socket_write(struct node *n, struct sample *smps[], unsigned cnt) { struct socket *s = n->_vd; - switch (s->header) { - case SOCKET_HEADER_NONE: - case SOCKET_HEADER_FAKE: - return socket_write_none(n, smps, cnt); + char data[SOCKET_MAX_PACKET_LEN]; + int ret; + ssize_t bytes; + size_t wbytes; - case SOCKET_HEADER_DEFAULT: - return socket_write_villas(n, smps, cnt); - } + ret = io_format_sprint(s->format, data, sizeof(data), &wbytes, smps, cnt, IO_FORMAT_ALL); + if (ret < 0) + return -1; - return -1; + /* Send message */ + bytes = sendto(s->sd, data, wbytes, 0, (struct sockaddr *) &s->remote, sizeof(s->remote)); + if (bytes < 0) + serror("Failed send to node %s", node_name(n)); + + if (bytes != wbytes) + warn("Partial send to node %s", node_name(n)); + + return cnt; } -int socket_parse(struct node *n, config_setting_t *cfg) +int socket_parse(struct node *n, json_t *cfg) { - config_setting_t *cfg_multicast; - const char *local, *remote, *layer, *hdr, *endian; - int ret; - struct socket *s = n->_vd; + const char *local, *remote; + const char *endian = NULL; + const char *layer = NULL; + const char *header = NULL; + const char *format = "msg"; + + int ret; + + json_t *cfg_multicast = NULL; + json_error_t err; + + /* Default values */ + s->layer = SOCKET_LAYER_UDP; + s->header = SOCKET_HEADER_DEFAULT; + s->endian = SOCKET_ENDIAN_BIG; + s->verify_source = 0; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s, s?: s, s: s, s: s, s?: b, s?: o, s?: s }", + "layer", &layer, + "header", &header, + "endian", &endian, + "remote", &remote, + "local", &local, + "verify_source", &s->verify_source, + "multicast", &cfg_multicast, + "format", &format + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + + /* Format */ + s->format = io_format_lookup(format); + if (!s->format) + error("Invalid format '%s' for node %s", format, node_name(n)); + /* IP layer */ - if (!config_setting_lookup_string(cfg, "layer", &layer)) - s->layer = SOCKET_LAYER_UDP; - else { + if (layer) { if (!strcmp(layer, "ip")) s->layer = SOCKET_LAYER_IP; #ifdef __linux__ @@ -582,103 +427,89 @@ int socket_parse(struct node *n, config_setting_t *cfg) else if (!strcmp(layer, "udp")) s->layer = SOCKET_LAYER_UDP; else - cerror(cfg, "Invalid layer '%s' for node %s", layer, node_name(n)); + error("Invalid layer '%s' for node %s", layer, node_name(n)); } /* Application header */ - if (!config_setting_lookup_string(cfg, "header", &hdr)) - s->header = SOCKET_HEADER_DEFAULT; - else { - if (!strcmp(hdr, "gtnet-skt") || (!strcmp(hdr, "none"))) + if (header) { + if (!strcmp(header, "gtnet-skt") || (!strcmp(header, "none"))) s->header = SOCKET_HEADER_NONE; - else if (!strcmp(hdr, "gtnet-skt:fake") || (!strcmp(hdr, "fake"))) + else if (!strcmp(header, "gtnet-skt:fake") || (!strcmp(header, "fake"))) s->header = SOCKET_HEADER_FAKE; - else if (!strcmp(hdr, "villas") || !strcmp(hdr, "default")) + else if (!strcmp(header, "villas") || !strcmp(header, "default")) s->header = SOCKET_HEADER_DEFAULT; else - cerror(cfg, "Invalid application header type '%s' for node %s", hdr, node_name(n)); + error("Invalid application header type '%s' for node %s", header, node_name(n)); } - if (!config_setting_lookup_string(cfg, "endian", &endian)) - s->endian = SOCKET_ENDIAN_BIG; - else { + if (endian) { if (!strcmp(endian, "big") || !strcmp(endian, "network")) s->endian = SOCKET_ENDIAN_BIG; else if (!strcmp(endian, "little")) s->endian = SOCKET_ENDIAN_LITTLE; else - cerror(cfg, "Invalid endianness type '%s' for node %s", endian, node_name(n)); + error("Invalid endianness type '%s' for node %s", endian, node_name(n)); } - if (!config_setting_lookup_string(cfg, "remote", &remote)) - cerror(cfg, "Missing remote address for node %s", node_name(n)); - - if (!config_setting_lookup_string(cfg, "local", &local)) - cerror(cfg, "Missing local address for node %s", node_name(n)); - - if (!config_setting_lookup_bool(cfg, "verify_source", &s->verify_source)) - s->verify_source = 0; - ret = socket_parse_addr(local, (struct sockaddr *) &s->local, s->layer, AI_PASSIVE); if (ret) { - cerror(cfg, "Failed to resolve local address '%s' of node %s: %s", + error("Failed to resolve local address '%s' of node %s: %s", local, node_name(n), gai_strerror(ret)); } ret = socket_parse_addr(remote, (struct sockaddr *) &s->remote, s->layer, 0); if (ret) { - cerror(cfg, "Failed to resolve remote address '%s' of node %s: %s", + error("Failed to resolve remote address '%s' of node %s: %s", remote, node_name(n), gai_strerror(ret)); } - cfg_multicast = config_setting_get_member(cfg, "multicast"); if (cfg_multicast) { - const char *group, *interface; + const char *group, *interface = NULL; - if (!config_setting_lookup_bool(cfg_multicast, "enabled", &s->multicast.enabled)) - s->multicast.enabled = true; + /* Default values */ + s->multicast.enabled = true; + s->multicast.mreq.imr_interface.s_addr = INADDR_ANY; + s->multicast.loop = 0; + s->multicast.ttl = 255; - if (!config_setting_lookup_string(cfg_multicast, "group", &group)) - cerror(cfg_multicast, "The multicast group requires a 'group' setting."); - else { - ret = inet_aton(group, &s->multicast.mreq.imr_multiaddr); - if (!ret) { - cerror(cfg_multicast, "Failed to resolve multicast group address '%s' of node %s", - group, node_name(n)); - } + ret = json_unpack_ex(cfg_multicast, &err, 0, "{ s?: b, s: s, s?: s, s?: b, s?: i }", + "enabled", &s->multicast.enabled, + "group", &group, + "interface", &interface, + "loop", &s->multicast.loop, + "ttl", &s->multicast.ttl + ); + if (ret) + jerror(&err, "Failed to parse setting 'multicast' of node %s", node_name(n)); + + ret = inet_aton(group, &s->multicast.mreq.imr_multiaddr); + if (!ret) { + error("Failed to resolve multicast group address '%s' of node %s", + group, node_name(n)); } - if (!config_setting_lookup_string(cfg_multicast, "interface", &interface)) - s->multicast.mreq.imr_interface.s_addr = INADDR_ANY; - else { + if (interface) { ret = inet_aton(group, &s->multicast.mreq.imr_interface); if (!ret) { - cerror(cfg_multicast, "Failed to resolve multicast interface address '%s' of node %s", + error("Failed to resolve multicast interface address '%s' of node %s", interface, node_name(n)); } } - - int loop; - if (!config_setting_lookup_bool(cfg_multicast, "loop", &loop)) - s->multicast.loop = 0; - else - s->multicast.loop = loop; - - int ttl; - if (!config_setting_lookup_int(cfg_multicast, "ttl", &ttl)) - s->multicast.ttl = 255; - else - s->multicast.ttl = ttl; } #ifdef WITH_NETEM - config_setting_t *cfg_netem; + json_t *cfg_netem; - cfg_netem = config_setting_get_member(cfg, "netem"); + cfg_netem = json_object_get(cfg, "netem"); if (cfg_netem) { int enabled = 1; - if (!config_setting_lookup_bool(cfg_netem, "enabled", &enabled) || enabled) - tc_parse(cfg_netem, &s->tc_qdisc); + + ret = json_unpack_ex(cfg_netem, &err, 0, "{ s?: b }", "enabled", &enabled); + if (ret) + jerror(&err, "Failed to parse setting 'netem' of node %s", node_name(n)); + + if (enabled) + tc_parse(&s->tc_qdisc, cfg_netem); else s->tc_qdisc = NULL; } diff --git a/lib/nodes/test_rtt.c b/lib/nodes/test_rtt.c new file mode 100644 index 000000000..7a718d46f --- /dev/null +++ b/lib/nodes/test_rtt.c @@ -0,0 +1,255 @@ +/** Node type: Node-type for testing Round-trip Time. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include "timing.h" +#include "plugin.h" +#include "nodes/test_rtt.h" + +static int test_rtt_case_start(struct test_rtt *t, int caseno) +{ + int ret; + char fn[512]; + + struct test_rtt_case *c = list_at(&t->cases, caseno); + + /* Open file */ + snprintf(fn, sizeof(fn), "%s/test_rtt_%d_%.0f.log", t->output, c->values, c->rate); + ret = io_init(&t->io, NULL, FORMAT_ALL & ~FORMAT_VALUES); + if (ret) + return ret; + + /* Start timer. */ + ret = task_init(&t->timer, c->rate); + if (ret) + serror("Failed to create timer"); + + return 0; +} + +static int test_rtt_case_stop(struct test_rtt_case *c) +{ + /* Close file */ + io_close(&c->io); + + /* Stop timer. */ + task_destroy(&c->task); + + return 0; +} + +int test_rtt_parse(struct node *n, json_t *cfg) +{ + struct test_rtt *t = n->_vd; + struct plugin *p; + + int ret, numrates, numvalues, limit = 1000; + int *rates, *values; + + const char *format = "villas"; + const char *output = "."; + + size_t index; + json_t *cfg_rates, *cfg_values, *cfg_val; + json_error_t err; + + t->cooldown = 1.0; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: s, s?: s, s: f, s: o, s: o }", + "limit", &limit, + "output", &output, + "format", &format, + "cooldown", &t->cooldown, + "rates", &cfg_rates, + "values", &cfg_values + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + + t->output = strdup(output); + + if (cfg_rates) { + if (!json_is_array(cfg_rates) || json_array_size(cfg_rates) < 1) + error("The 'rates' setting of node %s must be an array of integers with at least one element.", node_name(n)); + + numrates = json_array_size(cfg_rates); + rates = alloc(sizeof(rates[0]) * numrates); + + json_array_foreach(cfg_rates, index, cfg_val) { + if (!json_is_integer(cfg_val)) + error("The 'rates' setting of node %s must be an array of integers", node_name(n)); + + rates[index] = json_integer_value(cfg_val); + } + } + + if (cfg_values) { + if (!json_is_array(cfg_values) || json_array_size(cfg_values) < 1) + error("The 'values' setting of node %s must be an array of integers with at least one element.", node_name(n)); + + numvalues = json_array_size(cfg_values); + rates = alloc(sizeof(rates[0]) * numvalues); + + json_array_foreach(cfg_values, index, cfg_val) { + if (!json_is_integer(cfg_val)) + error("The 'values' setting of node %s must be an array of integers", node_name(n)); + + values[index] = json_integer_value(cfg_val); + } + } + + /* Initialize IO module */ + p = plugin_lookup(PLUGIN_TYPE_FORMAT, format); + if (!p) + error("Invalid value for setting 'format' of node %s", node_name(n)); + + ret = io_init(&t->io, &p->io, FORMAT_ALL & ~FORMAT_VALUES); + if (ret) + return ret; + + /* Generate list of test cases */ + list_init(&t->cases); + + for (int i = 0; i < numrates; i++) { + for (int j = 0; j < numvalues; j++) { + struct test_rtt_case *c = alloc(sizeof(struct test_rtt_case)); + + c->rate = rates[i]; + c->values = values[j]; + c->limit = limit; + + list_push(&t->cases, c); + } + } + + return 0; +} + +char * test_rtt_print(struct node *n) +{ + struct test_rtt *t = n->_vd; + + return strf("output=%s, cooldown=%f, #cases=%zu", t->output, t->cooldown, list_length(&t->cases)); +} + +int test_rtt_start(struct node *n) +{ + struct test_rtt *t = n->_vd; + + t->current = 0; + + test_rtt_case_start(t); + + return 0; +} + +int test_rtt_stop(struct node *n) +{ + int ret; + struct test_rtt *t = n->_vd; + + ret = test_rtt_case_stop(t->current); + if (ret) + error("Failed to stop test case"); + + list_destroy(&t->cases, NULL, true); + + return 0; +} + +int test_rtt_read(struct node *n, struct sample *smps[], unsigned cnt) +{ + int ret, i; + uint64_t steps; + + struct test_rtt *t = n->_vd; + struct test_rtt_case *c = t->current; + + /* Wait */ + steps = task_wait_until_next_period(&c->task); + if (steps > 1) + warn("Skipped %zu samples", steps - 1); + + struct timespec now = time_now(); + + /* Prepare samples. */ + for (i = 0; i < cnt; i++) { + if (c->counter >= c->limit) { + info("Reached limit. Terminating."); + killme(SIGTERM); + pause(); + } + + int values = c->values; + if (smps[i]->capacity < MAX(2, values)) { + values = smps[i]->capacity; + warn("Sample capacity too small. Limiting to %d values.", values); + } + + smps[i]->data[0].i = c->values; + smps[i]->data[1].f = c->rate; + + smps[i]->length = values; + smps[i]->sequence = c->counter++; + smps[i]->ts.origin = now; + } + + return i; +} + +int test_rtt_write(struct node *n, struct sample *smps[], unsigned cnt) +{ + struct test_rtt *t = n->_vd; + + int i; + + for (i = 0; i < cnt; i++) { + if (smps[i]->length != c->values) { + warn("Discarding invalid sample"); + continue; + } + + io_print(&t->io, smps[i], 1); + } + + return i; +} + +static struct plugin p = { + .name = "test_rtt", + .description = "Test round-trip time with loopback", + .type = PLUGIN_TYPE_NODE, + .node = { + .vectorize = 0, + .size = sizeof(struct test_rtt), + .parse = test_rtt_parse, + .print = test_rtt_print, + .start = test_rtt_start, + .stop = test_rtt_stop, + .read = test_rtt_read, + .write = test_rtt_write + } +}; + +REGISTER_PLUGIN(&p) +LIST_INIT_STATIC(&p.node.instances) diff --git a/lib/nodes/websocket.c b/lib/nodes/websocket.c index 7240c9beb..95f99a641 100644 --- a/lib/nodes/websocket.c +++ b/lib/nodes/websocket.c @@ -26,15 +26,12 @@ #include #include -#include - #include "super_node.h" -#include "webmsg.h" -#include "webmsg_format.h" #include "timing.h" #include "utils.h" +#include "buffer.h" #include "plugin.h" - +#include "io_format.h" #include "nodes/websocket.h" /* Private static storage */ @@ -47,76 +44,26 @@ static struct plugin p; static char * websocket_connection_name(struct websocket_connection *c) { if (!c->_name) { - strcatf(&c->_name, "%s (%s)", c->peer.name, c->peer.ip); + if (c->wsi) { + char name[128]; + char ip[128]; + + lws_get_peer_addresses(c->wsi, lws_get_socket_fd(c->wsi), name, sizeof(name), ip, sizeof(ip)); + + strcatf(&c->_name, "remote.ip=%s, remote.name=%s", ip, name); + } + else if (c->destination) + strcatf(&c->_name, "%s:%d", c->destination->info.address, c->destination->info.port); if (c->node) - strcatf(&c->_name, " for node %s", node_name(c->node)); + strcatf(&c->_name, ", node=%s", node_name(c->node)); - strcatf(&c->_name, " in %s mode", c->mode == WEBSOCKET_MODE_CLIENT ? "client" : "server"); + strcatf(&c->_name, ", mode=%s", c->mode == WEBSOCKET_MODE_CLIENT ? "client" : "server"); } return c->_name; } -static int websocket_connection_init(struct websocket_connection *c, struct lws *wsi) -{ - int ret; - - lws_get_peer_addresses(wsi, lws_get_socket_fd(wsi), c->peer.name, sizeof(c->peer.name), c->peer.ip, sizeof(c->peer.ip)); - - info("LWS: New connection %s", websocket_connection_name(c)); - - c->state = STATE_INITIALIZED; - c->wsi = wsi; - c->buf = NULL; - - if (c->node) { - struct websocket *w = c->node->_vd; - - list_push(&w->connections, c); - } - else - list_push(&connections, c); - - ret = queue_init(&c->queue, DEFAULT_QUEUELEN, &memtype_hugepage); - if (ret) - return ret; - - return 0; -} - -static int websocket_connection_destroy(struct websocket_connection *c) -{ - int ret; - - if (c->state == STATE_DESTROYED) - return 0; - - info("LWS: Connection %s closed", websocket_connection_name(c)); - - if (c->node) { - struct websocket *w = c->node->_vd; - list_remove(&w->connections, c); - } - else - list_remove(&connections, c); - - if (c->_name) - free(c->_name); - - ret = queue_destroy(&c->queue); - if (ret) - return ret; - - if (c->buf) - free(c->buf); - - c->state = STATE_DESTROYED; - c->wsi = NULL; - - return ret; -} - static void websocket_destination_destroy(struct websocket_destination *d) { free(d->uri); @@ -127,27 +74,19 @@ static void websocket_destination_destroy(struct websocket_destination *d) static int websocket_connection_write(struct websocket_connection *c, struct sample *smps[], unsigned cnt) { - int ret; - - switch (c->state) { - case STATE_INITIALIZED: - c->state = STATE_STARTED; - /* fall through */ - - case STATE_STARTED: - for (int i = 0; i < cnt; i++) { - sample_get(smps[i]); /* increase reference count */ - - ret = queue_push(&c->queue, (void **) smps[i]); - if (ret != 1) - warn("Queue overrun in websocket connection: %s", websocket_connection_name(c)); - } - - lws_callback_on_writable(c->wsi); - break; - - default: { } - } + int pushed; + + pushed = queue_push_many(&c->queue, (void **) smps, cnt); + if (pushed < cnt) + warn("Queue overrun in WebSocket connection: %s", websocket_connection_name(c)); + + sample_get_many(smps, cnt); + + debug(LOG_WEBSOCKET | 10, "Enqueued %u samples to %s", pushed, websocket_connection_name(c)); + + /* Client connections which are currently conecting don't have an associate c->wsi yet */ + if (c->wsi) + lws_callback_on_writable(c->wsi); return 0; } @@ -156,52 +95,63 @@ static void websocket_connection_close(struct websocket_connection *c, struct lw { lws_close_reason(wsi, status, (unsigned char *) reason, strlen(reason)); - char *msg = strf("LWS: Closing connection"); - - if (c) - msg = strcatf(&msg, " with %s", websocket_connection_name(c)); - - msg = strcatf(&msg, ": status=%u, reason=%s", status, reason); - - warn("%s", msg); - - free(msg); + debug(LOG_WEBSOCKET | 10, "Closing WebSocket connection with %s: status=%u, reason=%s", websocket_connection_name(c), status, reason); } int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { - int ret; + int ret, recvd, pulled, cnt = 128; struct websocket_connection *c = user; - - struct webmsg *msg; - struct sample *smp; - + switch (reason) { case LWS_CALLBACK_CLIENT_ESTABLISHED: - ret = websocket_connection_init(c, wsi); - if (ret) - return -1; + c->wsi = wsi; + c->state = STATE_ESTABLISHED; + + buffer_init(&c->buffers.recv, 1 << 12); + buffer_init(&c->buffers.send, 1 << 12); - return 0; + debug(LOG_WEBSOCKET | 10, "Established WebSocket connection: %s", websocket_connection_name(c)); + + /* Schedule writable callback in case we have something to send */ + if (queue_available(&c->queue) > 0) + lws_callback_on_writable(wsi); + + break; case LWS_CALLBACK_ESTABLISHED: - c->state = STATE_DESTROYED; + c->wsi = wsi; + c->state = STATE_ESTABLISHED; c->mode = WEBSOCKET_MODE_SERVER; + c->_name = NULL; + + /* We use the URI to associate this connection to a node + * and choose a protocol. + * + * Example: ws://example.com/node_1.json + * Will select the node with the name 'node_1' + * and format 'json'. + * + * If the node name is omitted, this connection + * will receive sample data from all websocket nodes + * (catch all). + */ /* Get path of incoming request */ + char *node, *format = NULL; char uri[64]; + lws_hdr_copy(wsi, uri, sizeof(uri), WSI_TOKEN_GET_URI); /* The path component of the*/ if (strlen(uri) <= 0) { websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_PROTOCOL_ERR, "Invalid URL"); return -1; } - - if ((uri[0] == '/' && uri[1] == 0) || uri[0] == 0){ - /* Catch all connection */ + + node = strtok(uri, "/."); + if (strlen(node) == 0) c->node = NULL; - } else { - char *node = uri + 1; + format = strtok(NULL, ""); /* Search for node whose name matches the URI. */ c->node = list_lookup(&p.node.instances, node); @@ -210,145 +160,148 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi return -1; } } + + if (!format) + format = "webmsg"; - ret = websocket_connection_init(c, wsi); + c->format = io_format_lookup(format); + if (!c->format) { + websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_PROTOCOL_ERR, "Invalid format"); + return -1; + } + + buffer_init(&c->buffers.recv, 1 << 12); + buffer_init(&c->buffers.send, 1 << 12); + + ret = queue_init(&c->queue, DEFAULT_QUEUELEN, &memtype_hugepage); if (ret) return -1; - return 0; + list_push(&connections, c); + + debug(LOG_WEBSOCKET | 10, "Established WebSocket connection: %s", websocket_connection_name(c)); + break; + + case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: + c->state = STATE_ERROR; + + warn("Failed to establish WebSocket connection: %s, reason=%s", websocket_connection_name(c), in ? (char *) in : "unkown"); + + return -1; + case LWS_CALLBACK_CLOSED: - websocket_connection_destroy(c); + debug(LOG_WEBSOCKET | 10, "Closed WebSocket connection: %s", websocket_connection_name(c)); + + if (c->state != STATE_SHUTDOWN) { + /** @todo Attempt reconnect here */ + } + + list_remove(&connections, c); + + if (c->_name) + free(c->_name); + + ret = queue_destroy(&c->queue); + if (ret) + return ret; + + buffer_destroy(&c->buffers.recv); + buffer_destroy(&c->buffers.send); + + c->wsi = NULL; if (c->mode == WEBSOCKET_MODE_CLIENT) free(c); - return 0; + break; case LWS_CALLBACK_CLIENT_WRITEABLE: - case LWS_CALLBACK_SERVER_WRITEABLE: - if (c->state == STATE_STOPPED) { - websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_NORMAL, "Goodbye"); - return -1; - } - - if (c->node && c->node->state != STATE_STARTED) { + case LWS_CALLBACK_SERVER_WRITEABLE: { + size_t wbytes; + + if (c->state == STATE_SHUTDOWN) { websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_GOINGAWAY, "Node stopped"); return -1; } - size_t msglen, buflen = LWS_PRE; + struct sample **smps = alloca(cnt * sizeof(struct sample *)); - while (queue_pull(&c->queue, (void **) &smp)) { - msglen = WEBMSG_LEN(smp->length); + pulled = queue_pull_many(&c->queue, (void **) smps, cnt); + if (pulled > 0) { + io_format_sprint(c->format, c->buffers.send.buf + LWS_PRE, c->buffers.send.size - LWS_PRE, &wbytes, smps, pulled, IO_FORMAT_ALL); + + ret = lws_write(wsi, (unsigned char *) c->buffers.send.buf + LWS_PRE, wbytes, c->format->flags & IO_FORMAT_BINARY ? LWS_WRITE_BINARY : LWS_WRITE_TEXT); - c->buf = realloc(c->buf, buflen + msglen); - if (!c->buf) - serror("realloc failed:"); + sample_put_many(smps, pulled); - msg = (struct webmsg *) (c->buf + buflen); - buflen += msglen; - - msg->version = WEBMSG_VERSION; - msg->type = WEBMSG_TYPE_DATA; - msg->length = smp->length; - msg->sequence = smp->sequence; - msg->id = smp->source->id; - msg->ts.sec = smp->ts.origin.tv_sec; - msg->ts.nsec = smp->ts.origin.tv_nsec; - - memcpy(&msg->data, &smp->data, WEBMSG_DATA_LEN(smp->length)); - - webmsg_hton(msg); - - sample_put(smp); + debug(LOG_WEBSOCKET | 10, "Send %d samples to connection: %s, bytes=%d", pulled, websocket_connection_name(c), ret); } - - ret = lws_write(wsi, (unsigned char *) c->buf + LWS_PRE, buflen - LWS_PRE, LWS_WRITE_BINARY); - if (ret < 0) { - warn("Failed lws_write() for connection %s", websocket_connection_name(c)); - return -1; - } - - return 0; + + if (queue_available(&c->queue) > 0) + lws_callback_on_writable(wsi); + + break; + } case LWS_CALLBACK_CLIENT_RECEIVE: - case LWS_CALLBACK_RECEIVE: - if (!lws_frame_is_binary(wsi)) { - websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_UNACCEPTABLE_OPCODE, "Binary data expected"); + case LWS_CALLBACK_RECEIVE: + if (!c->node) { + websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_PROTOCOL_ERR, "Catch-all connection can not receive."); return -1; } - - if (len < WEBMSG_LEN(0)) { - websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_PROTOCOL_ERR, "Invalid packet"); + + if (lws_is_first_fragment(wsi)) + buffer_clear(&c->buffers.recv); + + ret = buffer_append(&c->buffers.recv, in, len); + if (ret) { + websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_UNACCEPTABLE_OPCODE, "Failed to process data"); return -1; } + + /* We dont try to parse the frame yet, as we have to wait for the remaining fragments */ + if (lws_is_final_fragment(wsi)) { + struct timespec ts_recv = time_now(); - struct timespec ts_recv = time_now(); - - msg = (struct webmsg *) in; - while ((char *) msg < (char *) in + len) { - struct node *dest; - - /* Convert message to host byte-order */ - webmsg_ntoh(msg); - - /* Find destination node of this message */ - if (c->node) - dest = c->node; - else { - dest = NULL; - - for (int i = 0; i < list_length(&p.node.instances); i++) { - struct node *n = list_at(&p.node.instances, i); - - if (n->id == msg->id) { - dest = n; - break; - } - } - - if (!dest) { - warn("Ignoring message due to invalid node id"); - goto next; - } - } - - struct websocket *w = dest->_vd; - - ret = sample_alloc(&w->pool, &smp, 1); - if (ret != 1) { + struct websocket *w = c->node->_vd; + struct sample **smps = alloca(cnt * sizeof(struct sample *)); + + ret = sample_alloc(&w->pool, smps, cnt); + if (ret != cnt) { warn("Pool underrun for connection: %s", websocket_connection_name(c)); break; } - smp->ts.origin = WEBMSG_TS(msg); - smp->ts.received = ts_recv; - - smp->sequence = msg->sequence; - smp->length = msg->length; - if (smp->length > smp->capacity) { - smp->length = smp->capacity; - warn("Dropping values for connection: %s", websocket_connection_name(c)); - } - - memcpy(&smp->data, &msg->data, SAMPLE_DATA_LEN(smp->length)); - - ret = queue_signalled_push(&w->queue, (void **) smp); - if (ret != 1) { - warn("Queue overrun for connection %s", websocket_connection_name(c)); + recvd = io_format_sscan(c->format, c->buffers.recv.buf, c->buffers.recv.len, NULL, smps, cnt, NULL); + if (recvd < 0) { + warn("Failed to parse sample data received on connection: %s", websocket_connection_name(c)); break; } - /* Next message */ -next: msg = (struct webmsg *) ((char *) msg + WEBMSG_LEN(msg->length)); + debug(LOG_WEBSOCKET | 10, "Received %d samples to connection: %s", recvd, websocket_connection_name(c)); + + /* Set receive timestamp */ + for (int i = 0; i < recvd; i++) + smps[i]->ts.received = ts_recv; + + ret = queue_signalled_push_many(&w->queue, (void **) smps, recvd); + if (ret != recvd) + warn("Queue overrun for connection: %s", websocket_connection_name(c)); + + if (c->state == STATE_SHUTDOWN) { + websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_GOINGAWAY, "Node stopped"); + return -1; + } } - return 0; + break; default: - return 0; + break; } + + return 0; } int websocket_init(struct super_node *sn) @@ -368,14 +321,16 @@ int websocket_deinit() for (size_t i = 0; i < list_length(&connections); i++) { struct websocket_connection *c = list_at(&connections, i); - c->state = STATE_STOPPED; + c->state = STATE_SHUTDOWN; lws_callback_on_writable(c->wsi); } /* Wait for all connections to be closed */ - while (list_length(&connections) > 0) - usleep(0.2*1e6); + while (list_length(&connections) > 0) { + info("Waiting for WebSocket connection shutdown"); + sleep(1); + } list_destroy(&connections, (dtor_cb_t) websocket_destination_destroy, true); @@ -387,9 +342,7 @@ int websocket_start(struct node *n) int ret; struct websocket *w = n->_vd; - size_t blocklen = LWS_PRE + WEBMSG_LEN(DEFAULT_WEBSOCKET_SAMPLELEN); - - ret = pool_init(&w->pool, DEFAULT_WEBSOCKET_QUEUELEN, blocklen, &memtype_hugepage); + ret = pool_init(&w->pool, DEFAULT_WEBSOCKET_QUEUELEN, SAMPLE_LEN(DEFAULT_WEBSOCKET_SAMPLELEN), &memtype_hugepage); if (ret) return ret; @@ -399,16 +352,25 @@ int websocket_start(struct node *n) for (int i = 0; i < list_length(&w->destinations); i++) { struct websocket_destination *d = list_at(&w->destinations, i); - struct websocket_connection *c = alloc(sizeof(struct websocket_connection)); - c->state = STATE_DESTROYED; + c->state = STATE_CONNECTING; c->mode = WEBSOCKET_MODE_CLIENT; c->node = n; + c->destination = d; + c->_name = NULL; + + c->format = io_format_lookup("webmsg"); /** @todo We could parse the format from the URI */ d->info.context = web->context; d->info.vhost = web->vhost; d->info.userdata = c; + + ret = queue_init(&c->queue, DEFAULT_QUEUELEN, &memtype_hugepage); + if (ret) + return -1; + + list_push(&connections, c); lws_client_connect_via_info(&d->info); } @@ -420,19 +382,38 @@ int websocket_stop(struct node *n) { int ret; struct websocket *w = n->_vd; + + /* Wait for all connections to be closed */ + for (;;) { + int connecting = 0; + + for (int i = 0; i < list_length(&w->destinations); i++) { + struct websocket_destination *d = list_at(&w->destinations, i); + struct websocket_connection *c = d->info.userdata; + + if (c->state == STATE_CONNECTING) + connecting++; + } + + if (connecting == 0) + break; + + debug(LOG_WEBSOCKET | 10, "Waiting for %d client connections to be established", connecting); + sleep(1); + } + + for (size_t i = 0; i < list_length(&connections); i++) { + struct websocket_connection *c = list_at(&connections, i); + + if (c->node != n) + continue; - for (size_t i = 0; i < list_length(&w->connections); i++) { - struct websocket_connection *c = list_at(&w->connections, i); - - c->state = STATE_STOPPED; + if (c->state != STATE_CONNECTING) + c->state = STATE_SHUTDOWN; lws_callback_on_writable(c->wsi); } - /* Wait for all connections to be closed */ - while (list_length(&w->connections) > 0) - sleep(1); - ret = queue_signalled_destroy(&w->queue); if (ret) return ret; @@ -448,7 +429,6 @@ int websocket_destroy(struct node *n) { struct websocket *w = n->_vd; - list_destroy(&w->connections, NULL, false); list_destroy(&w->destinations, (dtor_cb_t) websocket_destination_destroy, true); return 0; @@ -461,11 +441,9 @@ int websocket_read(struct node *n, struct sample *smps[], unsigned cnt) struct websocket *w = n->_vd; struct sample *cpys[cnt]; - do { - avail = queue_signalled_pull_many(&w->queue, (void **) cpys, cnt); - if (avail < 0) - return avail; - } while (avail == 0); + avail = queue_signalled_pull_many(&w->queue, (void **) cpys, cnt); + if (avail < 0) + return avail; for (int i = 0; i < avail; i++) { sample_copy(smps[i], cpys[i]); @@ -491,18 +469,14 @@ int websocket_write(struct node *n, struct sample *smps[], unsigned cnt) sample_copy(cpys[i], smps[i]); cpys[i]->source = n; - } - - for (size_t i = 0; i < list_length(&w->connections); i++) { - struct websocket_connection *c = list_at(&w->connections, i); - - websocket_connection_write(c, cpys, cnt); + cpys[i]->id = n->id; } for (size_t i = 0; i < list_length(&connections); i++) { struct websocket_connection *c = list_at(&connections, i); - - websocket_connection_write(c, cpys, cnt); + + if (c->node == n || c->node == NULL) + websocket_connection_write(c, cpys, cnt); } sample_put_many(cpys, avail); @@ -510,46 +484,49 @@ int websocket_write(struct node *n, struct sample *smps[], unsigned cnt) return cnt; } -int websocket_parse(struct node *n, config_setting_t *cfg) +int websocket_parse(struct node *n, json_t *cfg) { struct websocket *w = n->_vd; - config_setting_t *cfg_dests; int ret; - list_init(&w->connections); + size_t index; + json_t *cfg_dests = NULL; + json_t *cfg_dest; + json_error_t err; + list_init(&w->destinations); - cfg_dests = config_setting_get_member(cfg, "destinations"); - if (cfg_dests) { - if (!config_setting_is_array(cfg_dests)) - cerror(cfg_dests, "The 'destinations' setting must be an array of URLs"); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o }", "destinations", &cfg_dests); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); - for (int i = 0; i < config_setting_length(cfg_dests); i++) { + if (cfg_dests) { + if (!json_is_array(cfg_dests)) + error("The 'destinations' setting of node %s must be an array of URLs", node_name(n)); + + json_array_foreach(cfg_dests, index, cfg_dest) { const char *uri, *prot, *ads, *path; - uri = config_setting_get_string_elem(cfg_dests, i); + uri = json_string_value(cfg_dest); if (!uri) - cerror(cfg_dests, "The 'destinations' setting must be an array of URLs"); + error("The 'destinations' setting of node %s must be an array of URLs", node_name(n)); struct websocket_destination *d = alloc(sizeof(struct websocket_destination)); d->uri = strdup(uri); - if (!d->uri) - serror("Failed to allocate memory"); ret = lws_parse_uri(d->uri, &prot, &ads, &d->info.port, &path); if (ret) - cerror(cfg_dests, "Failed to parse websocket URI: '%s'", uri); + error("Failed to parse WebSocket URI: '%s'", uri); d->info.ssl_connection = !strcmp(prot, "https"); d->info.address = strdup(ads); + d->info.path = strdup(path); d->info.host = d->info.address; d->info.origin = d->info.address; d->info.ietf_version_or_minus_one = -1; d->info.protocol = "live"; - ret = asprintf((char **) &d->info.path, "/%s", path); - list_push(&w->destinations, d); } } @@ -568,7 +545,7 @@ char * websocket_print(struct node *n) for (size_t i = 0; i < list_length(&w->destinations); i++) { struct websocket_destination *d = list_at(&w->destinations, i); - buf = strcatf(&buf, "%s://%s:%d%s ", + buf = strcatf(&buf, "%s://%s:%d/%s ", d->info.ssl_connection ? "wss" : "ws", d->info.address, d->info.port, diff --git a/lib/nodes/zeromq.c b/lib/nodes/zeromq.c index 9f49b42a3..721a834e4 100644 --- a/lib/nodes/zeromq.c +++ b/lib/nodes/zeromq.c @@ -27,10 +27,11 @@ #endif #include "nodes/zeromq.h" +#include "node.h" #include "utils.h" #include "queue.h" #include "plugin.h" -#include "msg.h" +#include "io_format.h" static void *context; @@ -89,83 +90,98 @@ int zeromq_reverse(struct node *n) return 0; } -int zeromq_parse(struct node *n, config_setting_t *cfg) +int zeromq_parse(struct node *n, json_t *cfg) { struct zeromq *z = n->_vd; - const char *ep, *type, *filter; + int ret; + const char *ep = NULL; + const char *type = NULL; + const char *filter = NULL; + const char *format = "villas"; - config_setting_t *cfg_pub, *cfg_curve; + size_t index; + json_t *cfg_pub = NULL; + json_t *cfg_curve = NULL; + json_t *cfg_val; + json_error_t err; list_init(&z->publisher.endpoints); - if (config_setting_lookup_string(cfg, "subscribe", &ep)) - z->subscriber.endpoint = strdup(ep); - else - z->subscriber.endpoint = NULL; + z->curve.enabled = false; + z->ipv6 = 0; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: o, s?: o, s?: s, s?: s, s?: b, s?: s }", + "subscribe", &ep, + "publish", &cfg_pub, + "curve", &cfg_curve, + "filter", &filter, + "pattern", &type, + "ipv6", &z->ipv6, + "format", &format + ); + if (ret) + jerror(&err, "Failed to parse configuration of node %s", node_name(n)); + + z->subscriber.endpoint = ep ? strdup(ep) : NULL; + z->filter = filter ? strdup(filter) : NULL; + + z->format = io_format_lookup(format); + if (!z->format) + error("Invalid format '%s' for node %s", format, node_name(n)); - cfg_pub = config_setting_lookup(cfg, "publish"); if (cfg_pub) { - switch (config_setting_type(cfg_pub)) { - case CONFIG_TYPE_LIST: - case CONFIG_TYPE_ARRAY: - for (int j = 0; j < config_setting_length(cfg_pub); j++) { - const char *ep = config_setting_get_string_elem(cfg_pub, j); + switch (json_typeof(cfg_pub)) { + case JSON_ARRAY: + json_array_foreach(cfg_pub, index, cfg_val) { + ep = json_string_value(cfg_pub); + if (!ep) + error("All 'publish' settings must be strings"); list_push(&z->publisher.endpoints, strdup(ep)); } break; - case CONFIG_TYPE_STRING: - ep = config_setting_get_string(cfg_pub); + case JSON_STRING: + ep = json_string_value(cfg_pub); list_push(&z->publisher.endpoints, strdup(ep)); break; default: - cerror(cfg_pub, "Invalid type for ZeroMQ publisher setting"); + error("Invalid type for ZeroMQ publisher setting"); } } - cfg_curve = config_setting_lookup(cfg, "curve"); if (cfg_curve) { - if (!config_setting_is_group(cfg_curve)) - cerror(cfg_curve, "The curve setting must be a group"); - const char *public_key, *secret_key; - if (!config_setting_lookup_string(cfg_curve, "public_key", &public_key)) - cerror(cfg_curve, "Setting 'curve.public_key' is missing"); + z->curve.enabled = true; - if (!config_setting_lookup_string(cfg_curve, "secret_key", &secret_key)) - cerror(cfg_curve, "Setting 'curve.secret_key' is missing"); - - if (!config_setting_lookup_bool(cfg_curve, "enabled", &z->curve.enabled)) - z->curve.enabled = true; + ret = json_unpack_ex(cfg_curve, &err, 0, "{ s: s, s: s, s?: b }", + "public_key", &public_key, + "secret_key", &secret_key, + "enabled", &z->curve.enabled + ); + if (ret) + jerror(&err, "Failed to parse setting 'curve' of node %s", node_name(n)); if (strlen(secret_key) != 40) - cerror(cfg_curve, "Setting 'curve.secret_key' must be a Z85 encoded CurveZMQ key"); + error("Setting 'curve.secret_key' of node %s must be a Z85 encoded CurveZMQ key", node_name(n)); if (strlen(public_key) != 40) - cerror(cfg_curve, "Setting 'curve.public_key' must be a Z85 encoded CurveZMQ key"); + error("Setting 'curve.public_key' of node %s must be a Z85 encoded CurveZMQ key", node_name(n)); strncpy(z->curve.server.public_key, public_key, 41); strncpy(z->curve.server.secret_key, secret_key, 41); } - else - z->curve.enabled = false; /** @todo We should fix this. Its mostly done. */ if (z->curve.enabled) - cerror(cfg_curve, "CurveZMQ support is currently broken"); + error("CurveZMQ support is currently broken"); - if (config_setting_lookup_string(cfg, "filter", &filter)) - z->filter = strdup(filter); - else - z->filter = NULL; - - if (config_setting_lookup_string(cfg, "pattern", &type)) { + if (type) { if (!strcmp(type, "pubsub")) z->pattern = ZEROMQ_PATTERN_PUBSUB; #ifdef ZMQ_BUILD_DISH @@ -173,12 +189,9 @@ int zeromq_parse(struct node *n, config_setting_t *cfg) z->pattern = ZEROMQ_PATTERN_RADIODISH; #endif else - cerror(cfg, "Invalid type for ZeroMQ node: %s", node_name_short(n)); + error("Invalid type for ZeroMQ node: %s", node_name_short(n)); } - if (!config_setting_lookup_bool(cfg, "ipv6", &z->ipv6)) - z->ipv6 = 0; - return 0; } @@ -196,7 +209,13 @@ char * zeromq_print(struct node *n) #endif } - strcatf(&buf, "pattern=%s, ipv6=%s, crypto=%s, subscribe=%s, publish=[ ", pattern, z->ipv6 ? "yes" : "no", z->curve.enabled ? "yes" : "no", z->subscriber.endpoint); + strcatf(&buf, "format=%s, pattern=%s, ipv6=%s, crypto=%s, subscribe=%s, publish=[ ", + plugin_name(z->format), + pattern, + z->ipv6 ? "yes" : "no", + z->curve.enabled ? "yes" : "no", + z->subscriber.endpoint + ); for (size_t i = 0; i < list_length(&z->publisher.endpoints); i++) { char *ep = list_at(&z->publisher.endpoints, i); @@ -410,7 +429,7 @@ int zeromq_read(struct node *n, struct sample *smps[], unsigned cnt) if (ret < 0) return ret; - recv = msg_buffer_to_samples(smps, cnt, zmq_msg_data(&m), zmq_msg_size(&m)); + recv = io_format_sscan(z->format, zmq_msg_data(&m), zmq_msg_size(&m), NULL, smps, cnt, NULL); ret = zmq_msg_close(&m); if (ret) @@ -424,16 +443,16 @@ int zeromq_write(struct node *n, struct sample *smps[], unsigned cnt) int ret; struct zeromq *z = n->_vd; - ssize_t sent; + size_t wbytes; zmq_msg_t m; - char data[1500]; + char data[4096]; - sent = msg_buffer_from_samples(smps, cnt, data, sizeof(data)); - if (sent < 0) + ret = io_format_sprint(z->format, data, sizeof(data), &wbytes, smps, cnt, IO_FORMAT_ALL); + if (ret <= 0) return -1; - ret = zmq_msg_init_size(&m, sent); + ret = zmq_msg_init_size(&m, wbytes); if (z->filter) { switch (z->pattern) { @@ -451,7 +470,7 @@ int zeromq_write(struct node *n, struct sample *smps[], unsigned cnt) } } - memcpy(zmq_msg_data(&m), data, sent); + memcpy(zmq_msg_data(&m), data, wbytes); ret = zmq_msg_send(&m, z->publisher.socket, 0); if (ret < 0) diff --git a/lib/path.c b/lib/path.c index 50fd6b6d9..db2661eca 100644 --- a/lib/path.c +++ b/lib/path.c @@ -25,7 +25,6 @@ #include #include #include -#include #include "config.h" #include "utils.h" @@ -63,7 +62,7 @@ static void path_read(struct path *p) if (recv < 0) error("Failed to receive message from node %s", node_name(ps->node)); else if (recv < ready) - warn("Partial read for path %s: read=%u expected=%u", path_name(p), recv, ready); + warn("Partial read for path %s: read=%u, expected=%u", path_name(p), recv, ready); /* Run preprocessing hooks for vector of samples */ enqueue = hook_read_list(&p->hooks, smps, recv); @@ -73,7 +72,7 @@ static void path_read(struct path *p) if (p->stats) stats_update(p->stats->delta, STATS_SKIPPED, recv - enqueue); } - + /* Keep track of the lowest index that wasn't enqueued; * all following samples must be freed here */ for (size_t i = 0; i < list_length(&p->destinations); i++) { @@ -123,7 +122,7 @@ static void path_write(struct path *p) if (sent < 0) error("Failed to sent %u samples to node %s", cnt, node_name(pd->node)); else if (sent < tosend) - warn("Partial write to node %s", node_name(pd->node)); + warn("Partial write to node %s: written=%d, expected=%d", node_name(pd->node), sent, tosend); released = sample_put_many(smps, sent); @@ -165,6 +164,8 @@ int path_init(struct path *p, struct super_node *sn) list_init(&p->hooks); list_init(&p->destinations); + + p->_name = NULL; /* Default values */ p->reverse = 0; @@ -180,53 +181,56 @@ int path_init(struct path *p, struct super_node *sn) return 0; } -int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes) +int path_parse(struct path *p, json_t *cfg, struct list *nodes) { - config_setting_t *cfg_out, *cfg_hooks; - const char *in; int ret; + const char *in; + + json_error_t err; + json_t *cfg_out = NULL; + json_t *cfg_hooks = NULL; struct node *source; struct list destinations = { .state = STATE_DESTROYED }; list_init(&destinations); - /* Input node */ - if (!config_setting_lookup_string(cfg, "in", &in)) - cerror(cfg, "Missing input node for path"); + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: o, s?: o, s?: b, s?: b, s?: i, s?: i }", + "in", &in, + "out", &cfg_out, + "hooks", &cfg_hooks, + "reverse", &p->reverse, + "enabled", &p->enabled, + "samplelen", &p->samplelen, + "queuelen", &p->queuelen + ); + if (ret) + jerror(&err, "Failed to parse path configuration"); + /* Input node */ source = list_lookup(nodes, in); if (!source) - cerror(cfg, "Invalid input node '%s'", in); + jerror(&err, "Invalid input node '%s'", in); /* Output node(s) */ - cfg_out = config_setting_get_member(cfg, "out"); if (cfg_out) { ret = node_parse_list(&destinations, cfg_out, nodes); if (ret) - cerror(cfg_out, "Failed to parse output nodes"); + jerror(&err, "Failed to parse output nodes"); } /* Optional settings */ - cfg_hooks = config_setting_get_member(cfg, "hooks"); if (cfg_hooks) { ret = hook_parse_list(&p->hooks, cfg_hooks, p); if (ret) return ret; } - config_setting_lookup_bool(cfg, "reverse", &p->reverse); - config_setting_lookup_bool(cfg, "enabled", &p->enabled); - config_setting_lookup_int(cfg, "samplelen", &p->samplelen); - config_setting_lookup_int(cfg, "queuelen", &p->queuelen); - if (!IS_POW2(p->queuelen)) { p->queuelen = LOG2_CEIL(p->queuelen); warn("Queue length should always be a power of 2. Adjusting to %d", p->queuelen); } - p->cfg = cfg; - p->source = alloc(sizeof(struct path_source)); p->source->node = source; p->source->samplelen = p->samplelen; @@ -235,7 +239,7 @@ int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes) struct node *n = list_at(&destinations, i); struct path_destination *pd = alloc(sizeof(struct path_destination)); - + pd->node = n; pd->queuelen = p->queuelen; @@ -244,6 +248,9 @@ int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes) list_destroy(&destinations, NULL, false); + p->cfg = cfg; + p->state = STATE_PARSED; + return 0; } @@ -281,7 +288,7 @@ int path_init2(struct path *p) if (vt->builtin) { struct hook *h = alloc(sizeof(struct hook)); - + ret = hook_init(h, vt, p); if (ret) { free(h); @@ -437,7 +444,6 @@ int path_reverse(struct path *p, struct path *r) /* General */ r->enabled = p->enabled; - r->cfg = p->cfg; struct path_destination *pd = alloc(sizeof(struct path_destination)); diff --git a/lib/plugin.c b/lib/plugin.c index 5bd37292e..cbeb5da00 100644 --- a/lib/plugin.c +++ b/lib/plugin.c @@ -38,13 +38,15 @@ int plugin_init(struct plugin *p) return 0; } -int plugin_parse(struct plugin *p, config_setting_t *cfg) +int plugin_parse(struct plugin *p, json_t *cfg) { const char *path; - path = config_setting_get_string(cfg); + path = json_string_value(cfg); if (!path) - cerror(cfg, "Setting 'plugin' must be a string."); + return -1; + + p->path = strdup(path); return 0; } @@ -103,6 +105,6 @@ void plugin_dump(enum plugin_type type) struct plugin *p = list_at(&plugins, i); if (p->type == type) - printf(" - %-12s: %s\n", p->name, p->description); + printf(" - %-13s: %s\n", p->name, p->description); } } diff --git a/lib/sample_io_json.c b/lib/sample_io_json.c deleted file mode 100644 index 033edd82c..000000000 --- a/lib/sample_io_json.c +++ /dev/null @@ -1,123 +0,0 @@ -/** JSON serializtion sample data. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include "sample_io_json.h" - -int sample_io_json_pack(json_t **j, struct sample *s, int flags) -{ - json_error_t err; - json_t *json_data = json_array(); - - for (int i = 0; i < s->length; i++) { - json_t *json_value = sample_get_data_format(s, i) - ? json_integer(s->data[i].i) - : json_real(s->data[i].f); - - json_array_append(json_data, json_value); - } - - *j = json_pack_ex(&err, 0, "{ s: { s: [ I, I ], s: [ I, I ], s: [ I, I ] }, s: I, s: o }", - "ts", - "origin", s->ts.origin.tv_sec, s->ts.origin.tv_nsec, - "received", s->ts.received.tv_sec, s->ts.received.tv_nsec, - "sent", s->ts.sent.tv_sec, s->ts.sent.tv_nsec, - "sequence", s->sequence, - "data", json_data); - - if (!*j) - return -1; - - return 0; -} - -int sample_io_json_unpack(json_t *j, struct sample *s, int *flags) -{ - int ret, i; - json_t *json_data, *json_value; - - ret = json_unpack(j, "{ s: { s: [ I, I ], s: [ I, I ], s: [ I, I ] }, s: I, s: o }", - "ts", - "origin", &s->ts.origin.tv_sec, &s->ts.origin.tv_nsec, - "received", &s->ts.received.tv_sec, &s->ts.received.tv_nsec, - "sent", &s->ts.sent.tv_sec, &s->ts.sent.tv_nsec, - "sequence", &s->sequence, - "data", &json_data); - - if (ret) - return ret; - - s->length = 0; - - json_array_foreach(json_data, i, json_value) { - switch (json_typeof(json_value)) { - case JSON_REAL: - s->data[i].f = json_real_value(json_value); - sample_set_data_format(s, i, SAMPLE_DATA_FORMAT_FLOAT); - break; - - case JSON_INTEGER: - s->data[i].f = json_integer_value(json_value); - sample_set_data_format(s, i, SAMPLE_DATA_FORMAT_INT); - break; - - default: - return -1; - } - - s->length++; - } - - return 0; -} - -int sample_io_json_fprint(FILE *f, struct sample *s, int flags) -{ - int ret; - json_t *json; - - ret = sample_io_json_pack(&json, s, flags); - if (ret) - return ret; - - ret = json_dumpf(json, f, 0); - - json_decref(json); - - return ret; -} - -int sample_io_json_fscan(FILE *f, struct sample *s, int *flags) -{ - int ret; - json_t *json; - json_error_t err; - - json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); - if (!json) - return -1; - - ret = sample_io_json_unpack(json, s, flags); - - json_decref(json); - - return ret; -} diff --git a/lib/stats.c b/lib/stats.c index bccfcac09..194fc5dc1 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -44,6 +44,18 @@ static struct stats_desc { { "owd", "seconds", "Histogram for one-way-delay (OWD) of received messages", 25 } }; +int stats_lookup_format(const char *str) +{ + if (!strcmp(str, "human")) + return STATS_FORMAT_HUMAN; + else if (!strcmp(str, "json")) + return STATS_FORMAT_JSON; + else if (!strcmp(str, "matlab")) + return STATS_FORMAT_MATLAB; + else + return -1; +} + int stats_init(struct stats *s, int buckets, int warmup) { for (int i = 0; i < STATS_COUNT; i++) @@ -112,7 +124,6 @@ void stats_collect(struct stats_delta *s, struct sample *smps[], size_t cnt) s->last = previous; } -#ifdef WITH_JSON json_t * stats_json(struct stats *s) { json_t *obj = json_object(); @@ -138,7 +149,6 @@ json_t * stats_json_periodic(struct stats *s, struct path *p) "skipped", s->histograms[STATS_SKIPPED].total ); } -#endif /* WITH_JSON */ void stats_reset(struct stats *s) { @@ -197,13 +207,11 @@ void stats_print_periodic(struct stats *s, FILE *f, enum stats_format fmt, int v ); break; -#ifdef WITH_JSON case STATS_FORMAT_JSON: { json_t *json_stats = stats_json_periodic(s, p); json_dumpf(json_stats, f, 0); break; } -#endif /* WITH_JSON */ default: { } } @@ -221,14 +229,12 @@ void stats_print(struct stats *s, FILE *f, enum stats_format fmt, int verbose) } break; -#ifdef WITH_JSON case STATS_FORMAT_JSON: { json_t *json_stats = stats_json(s); json_dumpf(json_stats, f, 0); fflush(f); break; } -#endif /* WITH_JSON */ default: { } } diff --git a/lib/super_node.c b/lib/super_node.c index 034f8ca67..3db5a7945 100644 --- a/lib/super_node.c +++ b/lib/super_node.c @@ -1,4 +1,4 @@ -/** Configuration parser. +/** The super node object holding the state of the application. * * @author Steffen Vogel * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC @@ -42,32 +42,10 @@ #include "kernel/rt.h" -static void config_dtor(void *data) -{ - if (data) - free(data); -} - -static int super_node_parse_global(struct super_node *sn, config_setting_t *cfg) -{ - if (!config_setting_is_group(cfg)) - cerror(cfg, "Global section must be a dictionary."); - - config_setting_lookup_int(cfg, "hugepages", &sn->hugepages); - config_setting_lookup_int(cfg, "affinity", &sn->affinity); - config_setting_lookup_int(cfg, "priority", &sn->priority); - config_setting_lookup_float(cfg, "stats", &sn->stats); - - return 0; -} - int super_node_init(struct super_node *sn) { assert(sn->state == STATE_DESTROYED); - config_init(&sn->cfg); - - log_init(&sn->log, V, LOG_ALL); #ifdef WITH_API api_init(&sn->api, sn); #endif /* WITH_API */ @@ -85,6 +63,9 @@ int super_node_init(struct super_node *sn) sn->stats = 0; sn->hugepages = DEFAULT_NR_HUGEPAGES; + sn->name = alloc(128); /** @todo missing free */ + gethostname(sn->name, 128); + sn->state = STATE_INITIALIZED; return 0; @@ -92,85 +73,82 @@ int super_node_init(struct super_node *sn) int super_node_parse_uri(struct super_node *sn, const char *uri) { - int ret = CONFIG_FALSE; + json_error_t err; + + info("Parsing configuration"); if (uri) { INDENT FILE *f; AFILE *af; - config_setting_t *cfg_root = NULL; /* Via stdin */ if (!strcmp("-", uri)) { + info("Reading configuration from stdin"); + af = NULL; f = stdin; - - info("Reading configuration from stdin"); } else { - /* Local file? */ - if (access(uri, F_OK) != -1) { - /* Setup libconfig include path. - * This is only supported for local files */ - char *uri_cpy = strdup(uri); - char *include_dir = dirname(uri_cpy); - - config_set_include_dir(&sn->cfg, include_dir); - - free(uri_cpy); - - info("Reading configuration from local file: %s", uri); - } - else - info("Reading configuration from URI: %s", uri); + info("Reading configuration from URI: %s", uri); af = afopen(uri, "r"); - f = af ? af->file : NULL; + if (!af) + error("Failed to open configuration from: %s", uri); + + f = af->file; } - /* Check if file could be loaded / opened */ - if (!f) - error("Failed to open configuration"); - - config_set_destructor(&sn->cfg, config_dtor); - config_set_auto_convert(&sn->cfg, 1); - - cfg_root = config_root_setting(&sn->cfg); - - /* Little hack to properly report configuration filename in error messages - * We add the uri as a "hook" object to the root setting. - * See cerror() on how this info is used. - */ - config_setting_set_hook(cfg_root, strdup(uri)); - /* Parse config */ - ret = config_read(&sn->cfg, f); - if (ret != CONFIG_TRUE) { -#ifdef WITH_JSON - /* This does not seem to be a valid libconfig configuration. - * Lets try to parse it as JSON instead. */ - json_error_t err; - json_t *json; + sn->cfg = json_loadf(f, 0, &err); + if (sn->cfg == NULL) { +#ifdef WITH_LIBCONFIG + int ret; - rewind(f); - json = json_loadf(f, 0, &err); - if (json) { - ret = json_to_config(json, cfg_root); - if (ret) - error("Failed t convert JSON to configuration file"); + config_t cfg; + config_setting_t *cfg_root = NULL; + + warn("Failed to parse JSON configuration. Re-trying with old libconfig format."); + { INDENT + warn("Please consider migrating to the new format using the 'conf2json' command."); } - else { + + config_init(&cfg); + config_set_auto_convert(&cfg, 1); + + /* Setup libconfig include path. + * This is only supported for local files */ + if (access(uri, F_OK) != -1) { + char *cpy = strdup(uri); + + config_set_include_dir(&cfg, dirname(cpy)); + + free(cpy); + } + + if (af) + arewind(af); + else + rewind(f); + + ret = config_read(&cfg, f); + if (ret != CONFIG_TRUE) { { INDENT - warn("conf: %s in %s:%d", config_error_text(&sn->cfg), uri, config_error_line(&sn->cfg)); + warn("conf: %s in %s:%d", config_error_text(&cfg), uri, config_error_line(&cfg)); warn("json: %s in %s:%d:%d", err.text, err.source, err.line, err.column); } error("Failed to parse configuration"); } + + cfg_root = config_root_setting(&cfg); + + sn->cfg = config_to_json(cfg_root); + if (sn->cfg == NULL) + error("Failed to convert JSON to configuration file"); + + config_destroy(&cfg); #else - { INDENT - warn("%s in %s:%d", config_error_text(&sn->cfg), uri, config_error_line(&sn->cfg)); - } - error("Failed to parse configuration"); -#endif + jerror(&err, "Failed to parse configuration file"); +#endif /* WITH_LIBCONFIG */ } /* Close configuration file */ @@ -181,7 +159,7 @@ int super_node_parse_uri(struct super_node *sn, const char *uri) sn->uri = strdup(uri); - return super_node_parse(sn, cfg_root); + return super_node_parse_json(sn, sn->cfg); } else { INDENT warn("No configuration file specified. Starting unconfigured. Use the API to configure this instance."); @@ -200,105 +178,120 @@ int super_node_parse_cli(struct super_node *sn, int argc, char *argv[]) return super_node_parse_uri(sn, uri); } -int super_node_parse(struct super_node *sn, config_setting_t *cfg) +int super_node_parse_json(struct super_node *sn, json_t *cfg) { int ret; + const char *name = NULL; assert(sn->state != STATE_STARTED); assert(sn->state != STATE_DESTROYED); - config_setting_t *cfg_nodes, *cfg_paths, *cfg_plugins, *cfg_logging; + json_t *cfg_nodes = NULL; + json_t *cfg_paths = NULL; + json_t *cfg_plugins = NULL; + json_t *cfg_logging = NULL; + json_t *cfg_web = NULL; - super_node_parse_global(sn, cfg); + json_error_t err; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: o, s?: o, s?: o, s?: o, s?: o, s?: i, s?: i, s?: i, s?: F, s?: s }", + "http", &cfg_web, + "logging", &cfg_logging, + "plugins", &cfg_plugins, + "nodes", &cfg_nodes, + "paths", &cfg_paths, + "hugepages", &sn->hugepages, + "affinity", &sn->affinity, + "priority", &sn->priority, + "stats", &sn->stats, + "name", &name + ); + if (ret) + jerror(&err, "Failed to parse global configuration"); + + if (name) + strncpy(sn->name, name, 128); #ifdef WITH_WEB - config_setting_t *cfg_web; - - cfg_web = config_setting_get_member(cfg, "http"); if (cfg_web) web_parse(&sn->web, cfg_web); #endif /* WITH_WEB */ - /* Parse logging settings */ - cfg_logging = config_setting_get_member(cfg, "logging"); if (cfg_logging) log_parse(&sn->log, cfg_logging); /* Parse plugins */ - cfg_plugins = config_setting_get_member(cfg, "plugins"); if (cfg_plugins) { - if (!config_setting_is_array(cfg_plugins)) - cerror(cfg_plugins, "Setting 'plugins' must be a list of strings"); - - for (int i = 0; i < config_setting_length(cfg_plugins); i++) { - struct config_setting_t *cfg_plugin = config_setting_get_elem(cfg_plugins, i); + if (!json_is_array(cfg_plugins)) + error("Setting 'plugins' must be a list of strings"); + size_t index; + json_t *cfg_plugin; + json_array_foreach(cfg_plugins, index, cfg_plugin) { struct plugin *p = alloc(sizeof(struct plugin)); ret = plugin_init(p); if (ret) - cerror(cfg_plugin, "Failed to initialize plugin"); + error("Failed to initialize plugin"); ret = plugin_parse(p, cfg_plugin); if (ret) - cerror(cfg_plugin, "Failed to parse plugin"); + error("Failed to parse plugin"); list_push(&sn->plugins, p); } } /* Parse nodes */ - cfg_nodes = config_setting_get_member(cfg, "nodes"); if (cfg_nodes) { - if (!config_setting_is_group(cfg_nodes)) - warn("Setting 'nodes' must be a group with node name => group mappings."); - - for (int i = 0; i < config_setting_length(cfg_nodes); i++) { - config_setting_t *cfg_node = config_setting_get_elem(cfg_nodes, i); + if (!json_is_object(cfg_nodes)) + error("Setting 'nodes' must be a group with node name => group mappings."); + const char *name; + json_t *cfg_node; + json_object_foreach(cfg_nodes, name, cfg_node) { struct plugin *p; const char *type; - /* Required settings */ - if (!config_setting_lookup_string(cfg_node, "type", &type)) - cerror(cfg_node, "Missing node type"); + ret = json_unpack_ex(cfg_node, &err, 0, "{ s: s }", "type", &type); + if (ret) + jerror(&err, "Failed to parse node"); p = plugin_lookup(PLUGIN_TYPE_NODE, type); if (!p) - cerror(cfg_node, "Invalid node type: %s", type); + error("Invalid node type: %s", type); struct node *n = alloc(sizeof(struct node)); ret = node_init(n, &p->node); if (ret) - cerror(cfg_node, "Failed to initialize node"); + error("Failed to initialize node"); - ret = node_parse(n, cfg_node); + ret = node_parse(n, cfg_node, name); if (ret) - cerror(cfg_node, "Failed to parse node"); + error("Failed to parse node"); list_push(&sn->nodes, n); } } /* Parse paths */ - cfg_paths = config_setting_get_member(cfg, "paths"); if (cfg_paths) { - if (!config_setting_is_list(cfg_paths)) + if (!json_is_array(cfg_paths)) warn("Setting 'paths' must be a list."); - for (int i = 0; i < config_setting_length(cfg_paths); i++) { - config_setting_t *cfg_path = config_setting_get_elem(cfg_paths, i); - + size_t index; + json_t *cfg_path; + json_array_foreach(cfg_paths, index, cfg_path) { struct path *p = alloc(sizeof(struct path)); ret = path_init(p, sn); if (ret) - cerror(cfg_path, "Failed to init path"); + error("Failed to initialize path"); ret = path_parse(p, cfg_path, &sn->nodes); if (ret) - cerror(cfg_path, "Failed to parse path"); + error("Failed to parse path"); list_push(&sn->paths, p); @@ -307,11 +300,11 @@ int super_node_parse(struct super_node *sn, config_setting_t *cfg) ret = path_init(r, sn); if (ret) - cerror(cfg_path, "Failed to init path"); + error("Failed to init path"); ret = path_reverse(p, r); if (ret) - cerror(cfg_path, "Failed to reverse path %s", path_name(p)); + error("Failed to reverse path %s", path_name(p)); list_push(&sn->paths, r); } @@ -456,16 +449,16 @@ int super_node_destroy(struct super_node *sn) #ifdef WITH_WEB web_destroy(&sn->web); -#endif +#endif /* WITH_WEB */ #ifdef WITH_API api_destroy(&sn->api); -#endif +#endif /* WITH_API */ + + json_decref(sn->cfg); log_destroy(&sn->log); - config_destroy(&sn->cfg); - - if (sn->uri) - free(sn->uri); + if (sn->name) + free(sn->name); sn->state = STATE_DESTROYED; diff --git a/lib/task.c b/lib/task.c new file mode 100644 index 000000000..d6e46ee16 --- /dev/null +++ b/lib/task.c @@ -0,0 +1,157 @@ +/** Run tasks periodically. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include "utils.h" + +#include "task.h" +#include "timing.h" + +#if PERIODIC_TASK_IMPL == TIMERFD + #include +#endif + +int task_init(struct task *t, double rate, int clock) +{ + t->clock = clock; + t->period = rate ? time_from_double(1.0 / rate) : (struct timespec) { 0, 0 }; + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP + struct timespec now; + + clock_gettime(t->clock, &now); + + t->next_period = time_add(&now, &t->period); +#elif PERIODIC_TASK_IMPL == TIMERFD + int ret; + + struct itimerspec its = { + .it_interval = t->period, + .it_value = t->period + }; + + t->fd = timerfd_create(t->clock, 0); + if (t->fd < 0) + return -1; + + ret = timerfd_settime(t->fd, 0, &its, NULL); + if (ret) + return ret; +#else + #error "Invalid period task implementation" +#endif + + return 0; +} + +int task_destroy(struct task *t) +{ +#if PERIODIC_TASK_IMPL == TIMERFD + return close(t->fd); +#endif + + return 0; +} + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP +static int time_lt(const struct timespec *lhs, const struct timespec *rhs) +{ + if (lhs->tv_sec == rhs->tv_sec) + return lhs->tv_nsec < rhs->tv_nsec; + else + return lhs->tv_sec < rhs->tv_sec; + + return 0; +} +#endif + +uint64_t task_wait_until_next_period(struct task *t) +{ + uint64_t runs; + int ret; + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP + ret = task_wait_until(t, &t->next_period); + + struct timespec now; + + ret = clock_gettime(t->clock, &now); + if (ret) + return 0; + + for (runs = 0; time_lt(&t->next_period, &now); runs++) + t->next_period = time_add(&t->next_period, &t->period); + +#elif PERIODIC_TASK_IMPL == TIMERFD + ret = read(t->fd, &runs, sizeof(runs)); + if (ret < 0) + return 0; +#else + #error "Invalid period task implementation" +#endif + + return runs; +} + + +int task_wait_until(struct task *t, const struct timespec *until) +{ + int ret; + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP +retry: ret = clock_nanosleep(t->clock, TIMER_ABSTIME, until, NULL); + if (ret == EINTR) + goto retry; +#elif PERIODIC_TASK_IMPL == NANOSLEEP + struct timespec now, delta; + + ret = clock_gettime(t->clock, &now); + if (ret) + return ret; + + delta = time_diff(&now, until); + + ret = nanosleep(&delta, NULL); +#elif PERIODIC_TASK_IMPL == TIMERFD + uint64_t runs; + + struct itimerspec its = { + .it_value = *until, + .it_interval = { 0, 0 } + }; + + ret = timerfd_settime(t->fd, TFD_TIMER_ABSTIME, &its, NULL); + if (ret) + return 0; + + ret = read(t->fd, &runs, sizeof(runs)); + if (ret < 0) + return ret; +#else + #error "Invalid period task implementation" +#endif + + return 0; +} \ No newline at end of file diff --git a/lib/timing.c b/lib/timing.c index ab06f9569..f637e3d91 100644 --- a/lib/timing.c +++ b/lib/timing.c @@ -24,54 +24,6 @@ #include "timing.h" -#ifdef __linux__ - -int timerfd_create_rate(double rate) -{ - int fd, ret; - - struct timespec ts = time_from_double(1 / rate); - - struct itimerspec its = { - .it_interval = ts, - .it_value = ts - }; - - fd = timerfd_create(CLOCK_MONOTONIC, 0); - if (fd < 0) - return fd; - - ret = timerfd_settime(fd, 0, &its, NULL); - if (ret) - return ret; - - return fd; -} - -uint64_t timerfd_wait(int fd) -{ - uint64_t runs; - - return read(fd, &runs, sizeof(runs)) < 0 ? 0 : runs; -} - -uint64_t timerfd_wait_until(int fd, const struct timespec *until) -{ - int ret; - struct itimerspec its = { - .it_value = *until, - .it_interval = { 0, 0 } - }; - - ret = timerfd_settime(fd, TFD_TIMER_ABSTIME, &its, NULL); - if (ret) - return 0; - - return timerfd_wait(fd); -} - -#endif /* __linux__ */ - struct timespec time_now() { struct timespec ts; diff --git a/lib/web.c b/lib/web.c index a6e8a7a8d..a92702a4f 100644 --- a/lib/web.c +++ b/lib/web.c @@ -20,7 +20,6 @@ * along with this program. If not, see . *********************************************************************************/ -#include #include #include @@ -52,6 +51,14 @@ static struct lws_protocols protocols[] = { .rx_buffer_size = 0 }, #endif /* WITH_API */ +#ifdef WITH_WEBSOCKET + { + .name = "live", + .callback = websocket_protocol_cb, + .per_session_data_size = sizeof(struct websocket_connection), + .rx_buffer_size = 0 + }, +#endif /* WITH_WEBSOCKET */ #if 0 /* not supported yet */ { .name = "log", @@ -66,14 +73,6 @@ static struct lws_protocols protocols[] = { .rx_buffer_size = 0 }, #endif -#ifdef WITH_WEBSOCKET - { - .name = "live", - .callback = websocket_protocol_cb, - .per_session_data_size = sizeof(struct websocket_connection), - .rx_buffer_size = 0 - }, -#endif /* WITH_WEBSOCKET */ { NULL /* terminator */ } }; @@ -126,6 +125,8 @@ static const struct lws_extension extensions[] = { { NULL /* terminator */ } }; +extern struct log *global_log; + static void logger(int level, const char *msg) { int len = strlen(msg); if (strchr(msg, '\n')) @@ -136,10 +137,10 @@ static void logger(int level, const char *msg) { level = LLL_WARN; switch (level) { - case LLL_ERR: warn("LWS: %.*s", len, msg); break; - case LLL_WARN: warn("LWS: %.*s", len, msg); break; - case LLL_INFO: info("LWS: %.*s", len, msg); break; - default: debug(LOG_WEBSOCKET | 1, "LWS: %.*s", len, msg); break; + case LLL_ERR: log_print(global_log, CLR_RED("Web"), "%.*s", len, msg); break; + case LLL_WARN: log_print(global_log, CLR_YEL("Web"), "%.*s", len, msg); break; + case LLL_INFO: log_print(global_log, CLR_WHT("Web"), "%.*s", len, msg); break; + default: log_print(global_log, "Web", "%.*s", len, msg); break; } } @@ -161,25 +162,43 @@ int web_init(struct web *w, struct api *a) /* Default values */ w->port = getuid() > 0 ? 8080 : 80; /**< @todo Use libcap to check if user can bind to ports < 1024 */ - w->htdocs = WEB_PATH; + w->htdocs = strdup(WEB_PATH); w->state = STATE_INITIALIZED; return 0; } -int web_parse(struct web *w, config_setting_t *cfg) +int web_parse(struct web *w, json_t *cfg) { - int enabled = true; + int ret, enabled = 1; + const char *ssl_cert = NULL; + const char *ssl_private_key = NULL; + const char *htdocs = NULL; + json_error_t err; - if (!config_setting_is_group(cfg)) - cerror(cfg, "Setting 'http' must be a group."); + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }", + "ssl_cert", &ssl_cert, + "ssl_private_key", &ssl_private_key, + "htdocs", &htdocs, + "port", &w->port, + "enabled", &enabled + ); + if (ret) + jerror(&err, "Failed to http section of configuration file"); - config_setting_lookup_string(cfg, "ssl_cert", &w->ssl_cert); - config_setting_lookup_string(cfg, "ssl_private_key", &w->ssl_private_key); - config_setting_lookup_int(cfg, "port", &w->port); - config_setting_lookup_string(cfg, "htdocs", &w->htdocs); - config_setting_lookup_bool(cfg, "enabled", &enabled); + if (ssl_cert) + w->ssl_cert = strdup(ssl_cert); + + if (ssl_private_key) + w->ssl_private_key = strdup(ssl_private_key); + + if (htdocs) { + if (w->htdocs) + free(w->htdocs); + + w->htdocs = strdup(htdocs); + } if (!enabled) w->port = CONTEXT_PORT_NO_LISTEN; @@ -227,7 +246,7 @@ int web_start(struct web *w) ret = pthread_create(&w->thread, NULL, worker, w); if (ret) - error("Failed to start Web worker"); + error("Failed to start Web worker thread"); w->state = STATE_STARTED; @@ -236,6 +255,8 @@ int web_start(struct web *w) int web_stop(struct web *w) { + int ret; + if (w->state != STATE_STARTED) return 0; @@ -246,11 +267,16 @@ int web_stop(struct web *w) /** @todo Wait for all connections to be closed */ - pthread_cancel(w->thread); - pthread_join(w->thread, NULL); + ret = pthread_cancel(w->thread); + if (ret) + serror("Failed to cancel Web worker thread"); - w->state = STATE_STOPPED; + ret = pthread_join(w->thread, NULL); + if (ret) + serror("Failed to join Web worker thread"); } + + w->state = STATE_STOPPED; return 0; } @@ -264,6 +290,15 @@ int web_destroy(struct web *w) lws_context_destroy(w->context); } + if (w->ssl_cert) + free(w->ssl_cert); + + if (w->ssl_private_key) + free(w->ssl_private_key); + + if (w->htdocs) + free(w->htdocs); + w->state = STATE_DESTROYED; return 0; diff --git a/lib/web/buffer.c b/lib/web/buffer.c deleted file mode 100644 index e6e8807d6..000000000 --- a/lib/web/buffer.c +++ /dev/null @@ -1,165 +0,0 @@ -/** API buffer for sending and receiving data from libwebsockets. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -#include "compat.h" -#include "assert.h" -#include "utils.h" -#include "web/buffer.h" - -int web_buffer_init(struct web_buffer *b, enum lws_write_protocol prot) -{ - assert(b->state == STATE_DESTROYED); - - b->protocol = prot; - b->size = 0; - b->len = 0; - b->buffer = NULL; - b->prefix = b->protocol == LWS_WRITE_TEXT || b->protocol == LWS_WRITE_BINARY ? LWS_PRE : 0; - - b->state = STATE_INITIALIZED; - - return 0; -} - -int web_buffer_destroy(struct web_buffer *b) -{ - if (b->state == STATE_DESTROYED) - return 0; - - if (b->buffer) - free(b->buffer); - - b->state = STATE_DESTROYED; - - return 0; -} - -int web_buffer_write(struct web_buffer *b, struct lws *w) -{ - int ret, len, sent = 0; - unsigned char *chunk; - - assert(b->state == STATE_INITIALIZED); - - if (b->len <= 0) - return 0; - - do { - chunk = (unsigned char *) b->buffer + b->prefix + sent; - len = strlen(b->buffer + b->prefix); - - ret = lws_write(w, chunk, len, b->protocol); - if (ret < 0) - break; - - sent += ret + 1; - } while (sent < b->len); - - web_buffer_read(b, NULL, sent); /* drop sent bytes from the buffer*/ - - return sent; -} - -int web_buffer_read(struct web_buffer *b, char *out, size_t len) -{ - assert(b->state == STATE_INITIALIZED); - - if (len > b->len) - len = b->len; - - if (out) - memcpy(out, b->buffer + b->prefix, len); - - memmove(b->buffer + b->prefix, b->buffer + b->prefix + len, b->len - len); - b->len -= len; - - return 0; -} - -#ifdef WITH_JSON -int web_buffer_read_json(struct web_buffer *b, json_t **req) -{ - json_error_t e; - - assert(b->state == STATE_INITIALIZED); - - if (b->len <= 0) - return -1; - - *req = json_loadb(b->buffer + b->prefix, b->len, JSON_DISABLE_EOF_CHECK, &e); - if (!*req) - return -2; - - web_buffer_read(b, NULL, e.position); - - return 1; -} -#endif /* WITH_JSON */ - -int web_buffer_append(struct web_buffer *b, const char *in, size_t len) -{ - assert(b->state == STATE_INITIALIZED); - - /* We append a \0 to split messages */ - len++; - - if (b->size < b->len + len) { - b->buffer = realloc(b->buffer, b->prefix + b->len + len); - if (!b->buffer) - return -1; - - b->size = b->len + len; - } - - memcpy(b->buffer + b->prefix + b->len, in, len); - b->buffer[b->prefix + b->len + len - 1] = 0; - b->len += len; - - return 0; -} - -#ifdef WITH_JSON -int web_buffer_append_json(struct web_buffer *b, json_t *res) -{ - size_t len; - - assert(b->state == STATE_INITIALIZED); - -retry: len = json_dumpb(res, b->buffer + b->prefix + b->len, b->size - b->len, 0) + 1; - if (b->size < b->len + len) { - b->buffer = realloc(b->buffer, b->prefix + b->len + len); - if (!b->buffer) - return -1; - - b->size = b->len + len; - goto retry; - } - - b->buffer[b->prefix + b->len + len - 1] = 0; - b->len += len; - - return 0; -} -#endif /* WITH_JSON */ diff --git a/lib/webmsg.c b/lib/webmsg.c deleted file mode 100644 index d0c1de283..000000000 --- a/lib/webmsg.c +++ /dev/null @@ -1,74 +0,0 @@ -/** Websocket message related functions. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#ifdef __MACH__ - #include "compat.h" -#elif defined(__linux__) - #include -#endif - -#include "webmsg.h" -#include "webmsg_format.h" - -void webmsg_ntoh(struct webmsg *m) -{ - webmsg_hdr_ntoh(m); - - for (int i = 0; i < m->length; i++) - m->data[i].i = le32toh(m->data[i].i); -} - -void webmsg_hton(struct webmsg *m) -{ - for (int i = 0; i < m->length; i++) - m->data[i].i = htole32(m->data[i].i); - - webmsg_hdr_hton(m); -} - -void webmsg_hdr_hton(struct webmsg *m) -{ - m->length = htole16(m->length); - m->sequence = htole32(m->sequence); - m->ts.sec = htole32(m->ts.sec); - m->ts.nsec = htole32(m->ts.nsec); -} - -void webmsg_hdr_ntoh(struct webmsg *m) -{ - m->length = le16toh(m->length); - m->sequence = le32toh(m->sequence); - m->ts.sec = le32toh(m->ts.sec); - m->ts.nsec = le32toh(m->ts.nsec); -} - -int webmsg_verify(struct webmsg *m) -{ - if (m->version != WEBMSG_VERSION) - return -1; - else if (m->type != WEBMSG_TYPE_DATA) - return -2; - else if (m->rsvd1 != 0) - return -3; - else - return 0; -} diff --git a/plugins/Makefile.inc b/plugins/Makefile.inc index 895241b83..ea1cecde8 100644 --- a/plugins/Makefile.inc +++ b/plugins/Makefile.inc @@ -43,7 +43,7 @@ $(PLUGINS): $(CC) $(PLUGIN_LDFLAGS) -o $@ $^ $(PLUGIN_LDLIBS) # Plugins are not installed to the system for now... -install-plugins: plugins +install-plugins: plugins | $(DESTDIR)$(PREFIX)/share/villas/node/plugins/ install -D -t $(DESTDIR)$(PREFIX)/share/villas/node/plugins $(PLUGINS) clean-plugins: diff --git a/src/hook.c b/src/hook.c index d2d35fbea..90eb9f78a 100644 --- a/src/hook.c +++ b/src/hook.c @@ -31,12 +31,13 @@ #include #include -#include +#include #include #include #include #include #include +#include #include @@ -44,12 +45,13 @@ int cnt; -struct sample **samples; +struct sample **smps; struct plugin *p; struct log l = { .state = STATE_DESTROYED }; struct pool q = { .state = STATE_DESTROYED }; struct hook h = { .state = STATE_DESTROYED }; +struct io io = { .state = STATE_DESTROYED }; static void quit(int signal, siginfo_t *sinfo, void *ctx) { @@ -63,7 +65,7 @@ static void quit(int signal, siginfo_t *sinfo, void *ctx) if (ret) error("Failed to destroy hook"); - sample_free(samples, cnt); + sample_free(smps, cnt); ret = pool_destroy(&q); if (ret) @@ -80,13 +82,20 @@ static void usage() printf(" NAME the name of the hook function\n"); printf(" PARAM* a string of configuration settings for the hook\n"); printf(" OPTIONS is one or more of the following options:\n"); + printf(" -f FMT the data format\n"); printf(" -h show this help\n"); printf(" -d LVL set debug level to LVL\n"); - printf(" -v CNT process CNT samples at once\n"); + printf(" -v CNT process CNT smps at once\n"); printf("\n"); + printf("The following hook functions are supported:\n"); plugin_dump(PLUGIN_TYPE_HOOK); printf("\n"); + + printf("Supported IO formats:\n"); + plugin_dump(PLUGIN_TYPE_IO); + printf("\n"); + printf("Example:"); printf(" villas-signal random | villas-hook skip_first seconds=10\n"); printf("\n"); @@ -96,27 +105,42 @@ static void usage() int main(int argc, char *argv[]) { - int ret; - - size_t recv; + int ret, recv; + char *format = "villas"; /* Default values */ cnt = 1; - char c; - while ((c = getopt(argc, argv, "hv:d:")) != -1) { + json_t *cfg_cli = json_object(); + + char c, *endptr; + while ((c = getopt(argc, argv, "hv:d:f:o:")) != -1) { switch (c) { + case 'f': + format = optarg; + break; case 'v': - cnt = atoi(optarg); - break; + cnt = strtoul(optarg, &endptr, 0); + goto check; case 'd': - l.level = atoi(optarg); + l.level = strtoul(optarg, &endptr, 0); + goto check; + case 'o': + ret = json_object_extend_str(cfg_cli, optarg); + if (ret) + error("Invalid option: %s", optarg); break; - case 'h': case '?': + case 'h': usage(); exit(c == '?' ? EXIT_FAILURE : EXIT_SUCCESS); } + + continue; + +check: if (optarg == endptr) + error("Failed to parse parse option argument '-%c %s'", c, optarg); + } if (argc < optind + 1) { @@ -124,39 +148,56 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - char *hookstr = argv[optind]; - - ret = signals_init(quit); - if (ret) - error("Failed to intialize signals"); + char *hook = argv[optind]; ret = log_init(&l, l.level, LOG_ALL); if (ret) error("Failed to initialize log"); - log_start(&l); + ret = log_start(&l); + if (ret) + error("Failed to start log"); + + ret = signals_init(quit); + if (ret) + error("Failed to intialize signals"); if (cnt < 1) error("Vectorize option must be greater than 0"); - memory_init(DEFAULT_NR_HUGEPAGES); + ret = memory_init(DEFAULT_NR_HUGEPAGES); + if (ret) + error("Failed to initialize memory"); - samples = alloc(cnt * sizeof(struct sample *)); + smps = alloc(cnt * sizeof(struct sample *)); ret = pool_init(&q, 10 * cnt, SAMPLE_LEN(DEFAULT_SAMPLELEN), &memtype_hugepage); if (ret) error("Failed to initilize memory pool"); - p = plugin_lookup(PLUGIN_TYPE_HOOK, hookstr); + /* Initialize IO */ + p = plugin_lookup(PLUGIN_TYPE_IO, format); if (!p) - error("Unknown hook function '%s'", hookstr); + error("Unknown IO format '%s'", format); + + ret = io_init(&io, &p->io, IO_FORMAT_ALL); + if (ret) + error("Failed to initialize IO"); + + ret = io_open(&io, NULL); + if (ret) + error("Failed to open IO"); + + /* Initialize hook */ + p = plugin_lookup(PLUGIN_TYPE_HOOK, hook); + if (!p) + error("Unknown hook function '%s'", hook); - /** @todo villas-hook does not use the path structure */ ret = hook_init(&h, &p->hook, NULL); if (ret) error("Failed to initialize hook"); - ret = hook_parse_cli(&h, argc - optind - 1, &argv[optind + 1]); + ret = hook_parse(&h, cfg_cli); if (ret) error("Failed to parse hook config"); @@ -165,35 +206,27 @@ int main(int argc, char *argv[]) error("Failed to start hook"); for (;;) { - if (feof(stdin)) { + if (io_eof(&io)) { killme(SIGTERM); pause(); } - ret = sample_alloc(&q, samples, cnt); + ret = sample_alloc(&q, smps, cnt); if (ret != cnt) - error("Failed to allocate %d samples from pool", cnt); + error("Failed to allocate %d smps from pool", cnt); - recv = 0; - for (int j = 0; j < cnt && !feof(stdin); j++) { - ret = sample_io_villas_fscan(stdin, samples[j], NULL); - if (ret < 0) - break; + recv = io_scan(&io, smps, cnt); + if (recv < 0) + killme(SIGTERM); - samples[j]->ts.received = time_now(); - recv++; - } + debug(15, "Read %u smps from stdin", recv); - debug(15, "Read %zu samples from stdin", recv); + hook_read(&h, smps, (unsigned *) &recv); + hook_write(&h, smps, (unsigned *) &recv); - hook_read(&h, samples, &recv); - hook_write(&h, samples, &recv); + io_print(&io, smps, recv); - for (int j = 0; j < recv; j++) - sample_io_villas_fprint(stdout, samples[j], SAMPLE_IO_ALL); - fflush(stdout); - - sample_free(samples, cnt); + sample_free(smps, cnt); } return 0; diff --git a/src/node.c b/src/node.c index 21157a6a2..f2b7c4544 100644 --- a/src/node.c +++ b/src/node.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include @@ -75,6 +75,7 @@ static void usage() printf(" This type of invocation is used by OPAL-RT Asynchronous processes.\n"); printf(" See in the RT-LAB User Guide for more information.\n\n"); #endif + printf("Supported node-types:\n"); plugin_dump(PLUGIN_TYPE_NODE); printf("\n"); @@ -87,6 +88,10 @@ static void usage() plugin_dump(PLUGIN_TYPE_API); printf("\n"); + printf("Supported IO formats:\n"); + plugin_dump(PLUGIN_TYPE_IO); + printf("\n"); + print_copyright(); exit(EXIT_FAILURE); @@ -94,6 +99,8 @@ static void usage() int main(int argc, char *argv[]) { + int ret; + /* Check arguments */ #ifdef ENABLE_OPAL_ASYNC if (argc != 4) @@ -109,7 +116,6 @@ int main(int argc, char *argv[]) else if (argc > 2) usage(); #endif - info("This is VILLASnode %s (built on %s, %s)", CLR_BLD(CLR_YEL(BUILDID)), CLR_BLD(CLR_MAG(__DATE__)), CLR_BLD(CLR_MAG(__TIME__))); @@ -120,24 +126,37 @@ int main(int argc, char *argv[]) error("Your kernel version is to old: required >= %u.%u", KERNEL_VERSION_MAJ, KERNEL_VERSION_MIN); #endif /* __linux__ */ - signals_init(quit); - log_init(&sn.log, V, LOG_ALL); - log_start(&sn.log); - - super_node_init(&sn); - super_node_parse_cli(&sn, argc, argv); - super_node_check(&sn); - super_node_start(&sn); - - if (sn.stats > 0) - stats_print_header(STATS_FORMAT_HUMAN); - + ret = signals_init(quit); + if (ret) + error("Failed to initialize signal subsystem"); + + ret = super_node_init(&sn); + if (ret) + error("Failed to initialize super node"); + + ret = super_node_parse_cli(&sn, argc, argv); + if (ret) + error("Failed to parse command line arguments"); + + ret = super_node_check(&sn); + if (ret) + error("Failed to verify configuration"); + + ret = super_node_start(&sn); + if (ret) + error("Failed to start super node"); if (sn.stats > 0) { - int tfd = timerfd_create_rate(1.0 / sn.stats); + stats_print_header(STATS_FORMAT_HUMAN); + + struct task t; + + ret = task_init(&t, 1.0 / sn.stats, CLOCK_REALTIME); + if (ret) + error("Failed to create stats timer"); for (;;) { - timerfd_wait(tfd); + task_wait_until_next_period(&t); for (size_t i = 0; i < list_length(&sn.paths); i++) { struct path *p = list_at(&sn.paths, i); diff --git a/src/pipe.c b/src/pipe.c index a53d21c6d..215affa74 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -33,19 +33,21 @@ #include #include #include -#include #include #include -#include +#include #include +#include +#include #include #include "config.h" static struct super_node sn = { .state = STATE_DESTROYED }; /**< The global configuration */ +static struct io io = { .state = STATE_DESTROYED }; -struct dir { +static struct dir { struct pool pool; pthread_t thread; bool enabled; @@ -84,20 +86,22 @@ static void usage() printf(" CONFIG path to a configuration file\n"); printf(" NODE the name of the node to which samples are sent and received from\n"); printf(" OPTIONS are:\n"); - printf(" -d LVL set debug log level to LVL\n"); - printf(" -x swap read / write endpoints\n"); - printf(" -s only read data from stdin and send it to node\n"); - printf(" -r only read data from node and write it to stdout\n"); - printf(" -t NUM terminate after NUM seconds\n"); - printf(" -L NUM terminate after NUM samples sent\n"); - printf(" -l NUM terminate after NUM samples received\n\n"); + printf(" -f FMT set the format\n"); + printf(" -d LVL set debug log level to LVL\n"); + printf(" -o OPTION=VALUE overwrite options in config file\n"); + printf(" -x swap read / write endpoints\n"); + printf(" -s only read data from stdin and send it to node\n"); + printf(" -r only read data from node and write it to stdout\n"); + printf(" -t NUM terminate after NUM seconds\n"); + printf(" -L NUM terminate after NUM samples sent\n"); + printf(" -l NUM terminate after NUM samples received\n\n"); print_copyright(); } static void * send_loop(void *ctx) { - int ret, cnt = 0; + int ret, len, sent, cnt = 0; struct sample *smps[node->vectorize]; /* Initialize memory */ @@ -109,42 +113,34 @@ static void * send_loop(void *ctx) if (ret < 0) error("Failed to get %u samples out of send pool (%d).", node->vectorize, ret); - while (!feof(stdin)) { - int len; - for (len = 0; len < node->vectorize; len++) { - struct sample *s = smps[len]; - int reason; + while (!io_eof(&io)) { + len = io_scan(&io, smps, node->vectorize); + if (len <= 0) + continue; - if (sendd.limit > 0 && cnt >= sendd.limit) - break; - -retry: reason = sample_io_villas_fscan(stdin, s, NULL); - if (reason < 0) { - if (feof(stdin)) - goto leave; - else { - warn("Skipped invalid message message: reason=%d", reason); - goto retry; - } - } + sent = node_write(node, smps, len); + if (sent < 0) { + warn("Failed to sent samples to node %s: reason=%d", node_name(node), sent); + continue; } - cnt += node_write(node, smps, len); - + cnt += sent; if (sendd.limit > 0 && cnt >= sendd.limit) - goto leave2; + goto leave; pthread_testcancel(); } -leave2: info("Reached send limit. Terminating..."); - killme(SIGTERM); - - return NULL; - - /* We reached EOF on stdin here. Lets kill the process */ -leave: if (recvv.limit < 0) { - info("Reached end-of-file. Terminating..."); +leave: if (io_eof(&io)) { + if (recvv.limit < 0) { + info("Reached end-of-file. Terminating..."); + killme(SIGTERM); + } + else + info("Reached end-of-file. Wait for receive side..."); + } + else { + info("Reached send limit. Terminating..."); killme(SIGTERM); } @@ -153,7 +149,7 @@ leave: if (recvv.limit < 0) { static void * recv_loop(void *ctx) { - int ret, cnt = 0; + int recv, ret, cnt = 0; struct sample *smps[node->vectorize]; /* Initialize memory */ @@ -163,26 +159,27 @@ static void * recv_loop(void *ctx) ret = sample_alloc(&recvv.pool, smps, node->vectorize); if (ret < 0) - error("Failed to get %u samples out of receive pool (%d).", node->vectorize, ret); - - /* Print header */ - fprintf(stdout, "# %-20s\t\t%s\n", "sec.nsec+offset", "data[]"); - fflush(stdout); + error("Failed to allocate %u samples from receive pool.", node->vectorize); for (;;) { - int recv = node_read(node, smps, node->vectorize); + recv = node_read(node, smps, node->vectorize); + if (recv < 0) { + warn("Failed to receive samples from node %s: reason=%d", node_name(node), recv); + continue; + } + struct timespec now = time_now(); + /* Fix timestamps */ for (int i = 0; i < recv; i++) { struct sample *s = smps[i]; if (s->ts.received.tv_sec == -1 || s->ts.received.tv_sec == 0) s->ts.received = now; - - sample_io_villas_fprint(stdout, s, SAMPLE_IO_ALL); - fflush(stdout); } + io_print(&io, smps, recv); + cnt += recv; if (recvv.limit > 0 && cnt >= recvv.limit) goto leave; @@ -200,15 +197,21 @@ int main(int argc, char *argv[]) { int ret, level = V, timeout = 0; bool reverse = false; + char *format = "villas"; sendd = recvv = (struct dir) { .enabled = true, .limit = -1 }; + json_t *cfg_cli = json_object(); + char c, *endptr; - while ((c = getopt(argc, argv, "hxrsd:l:L:t:")) != -1) { + while ((c = getopt(argc, argv, "hxrsd:l:L:t:f:o:")) != -1) { switch (c) { + case 'f': + format = optarg; + break; case 'x': reverse = true; break; @@ -230,6 +233,11 @@ int main(int argc, char *argv[]) case 't': timeout = strtoul(optarg, &endptr, 10); goto check; + case 'o': + ret = json_object_extend_str(cfg_cli, optarg); + if (ret) + error("Invalid option: %s", optarg); + break; case 'h': case '?': usage(); @@ -240,7 +248,6 @@ int main(int argc, char *argv[]) check: if (optarg == endptr) error("Failed to parse parse option argument '-%c %s'", c, optarg); - } if (argc != optind + 2) { @@ -250,23 +257,54 @@ check: if (optarg == endptr) char *configfile = argv[optind]; char *nodestr = argv[optind+1]; + struct plugin *p; - log_init(&sn.log, level, LOG_ALL); - log_start(&sn.log); + ret = log_init(&sn.log, level, LOG_ALL); + if (ret) + error("Failed to initialize log"); - super_node_init(&sn); - super_node_parse_uri(&sn, configfile); + ret = log_start(&sn.log); + if (ret) + error("Failed to start log"); - memory_init(sn.hugepages); - signals_init(quit); - rt_init(sn.priority, sn.affinity); + ret = signals_init(quit); + if (ret) + error("Failed to initialize signals"); + + ret = memory_init(sn.hugepages); + if (ret) + error("Failed to initialize memory"); + + ret = rt_init(sn.priority, sn.affinity); + if (ret) + error("Failed to initalize real-time"); + + p = plugin_lookup(PLUGIN_TYPE_IO, format); + if (!p) + error("Invalid format: %s", format); + + ret = io_init(&io, &p->io, IO_FORMAT_ALL); + if (ret) + error("Failed to initialize IO"); + + ret = io_open(&io, NULL); + if (ret) + error("Failed to open IO"); + + ret = super_node_init(&sn); + if (ret) + error("Failed to initialize super-node"); + + ret = super_node_parse_uri(&sn, configfile); + if (ret) + error("Failed to parse configuration"); - /* Initialize node */ node = list_lookup(&sn.nodes, nodestr); if (!node) error("Node '%s' does not exist!", nodestr); #ifdef WITH_WEBSOCKET + /* Only start web subsystem if villas-pipe is used with a websocket node */ if (node->_vt->start == websocket_start) web_start(&sn.web); #endif @@ -284,7 +322,7 @@ check: if (optarg == endptr) ret = node_start(node); if (ret) - error("Failed to start node: %s", node_name(node)); + error("Failed to start node %s: reason=%d", node_name(node), ret); /* Start threads */ if (recvv.enabled) diff --git a/src/signal.c b/src/signal.c index 23df871b7..3fa3a8836 100644 --- a/src/signal.c +++ b/src/signal.c @@ -30,7 +30,7 @@ #include #include -#include +#include #include #include #include @@ -39,6 +39,7 @@ /* Some default values */ struct node n; struct log l; +struct io io; struct sample *t; @@ -54,14 +55,15 @@ void usage() printf(" ramp\n"); printf("\n"); printf(" OPTIONS is one or more of the following options:\n"); - printf(" -d LVL set debug level\n"); - printf(" -v NUM specifies how many values a message should contain\n"); - printf(" -r HZ how many messages per second\n"); - printf(" -n non real-time mode. do not throttle output.\n"); - printf(" -f HZ the frequency of the signal\n"); - printf(" -a FLT the amplitude\n"); - printf(" -D FLT the standard deviation for 'random' signals\n"); - printf(" -l NUM only send LIMIT messages and stop\n\n"); + printf(" -d LVL set debug level\n"); + printf(" -f FMT set the format\n"); + printf(" -v NUM specifies how many values a message should contain\n"); + printf(" -r HZ how many messages per second\n"); + printf(" -n non real-time mode. do not throttle output.\n"); + printf(" -F HZ the frequency of the signal\n"); + printf(" -a FLT the amplitude\n"); + printf(" -D FLT the standard deviation for 'random' signals\n"); + printf(" -l NUM only send LIMIT messages and stop\n\n"); print_copyright(); } @@ -78,6 +80,14 @@ static void quit(int signal, siginfo_t *sinfo, void *ctx) if (ret) error("Failed to destroy node"); + ret = io_close(&io); + if (ret) + error("Failed to close output"); + + ret = io_destroy(&io); + if (ret) + error("Failed to destroy output"); + ret = log_stop(&l); if (ret) error("Failed to stop log"); @@ -92,7 +102,8 @@ int main(int argc, char *argv[]) { int ret; struct plugin *p; - struct signal *s; + + char *format = "villas"; /** @todo hardcoded for now */ ret = log_init(&l, l.level, LOG_ALL); if (ret) @@ -114,6 +125,18 @@ int main(int argc, char *argv[]) if (ret) error("Failed to initialize node"); + p = plugin_lookup(PLUGIN_TYPE_IO, format); + if (!p) + error("Invalid output format '%s'", format); + + ret = io_init(&io, &p->io, IO_FLUSH | (IO_FORMAT_ALL & ~IO_FORMAT_OFFSET)); + if (ret) + error("Failed to initialize output"); + + ret = io_open(&io, NULL); + if (ret) + error("Failed to open output"); + ret = node_parse_cli(&n, argc, argv); if (ret) error("Failed to parse command line options"); @@ -122,29 +145,20 @@ int main(int argc, char *argv[]) if (ret) error("Failed to verify node configuration"); - info("Starting signal generation: %s", node_name(&n)); - /* Allocate memory for message buffer */ - s = n._vd; + struct signal *s = n._vd; t = alloc(SAMPLE_LEN(s->values)); t->capacity = s->values; - /* Print header */ - printf("# VILLASnode signal params: type=%s, values=%u, rate=%f, limit=%d, amplitude=%f, freq=%f\n", - argv[optind], s->values, s->rate, s->limit, s->amplitude, s->frequency); - printf("# %-20s\t\t%s\n", "sec.nsec(seq)", "data[]"); - ret = node_start(&n); if (ret) serror("Failed to start node"); for (;;) { node_read(&n, &t, 1); - - sample_io_villas_fprint(stdout, t, SAMPLE_IO_ALL & ~SAMPLE_IO_OFFSET); - fflush(stdout); + io_print(&io, &t, 1); } return 0; diff --git a/src/test-cmp.c b/src/test-cmp.c index eb122bea6..7b21f3045 100644 --- a/src/test-cmp.c +++ b/src/test-cmp.c @@ -28,7 +28,9 @@ #include #include -#include +#include +#include +#include #include #include #include @@ -44,6 +46,9 @@ void usage() printf(" -h print this usage information\n"); printf(" -d LVL adjust the debug level\n"); printf(" -e EPS set epsilon for floating point comparisons to EPS\n"); + printf(" -v ignore data values\n"); + printf(" -t ignore timestamp\n"); + printf(" -s ignore sequence no\n"); printf("\n"); printf("Return codes:\n"); printf(" 0 files are equal\n"); @@ -60,7 +65,12 @@ void usage() int main(int argc, char *argv[]) { int ret; + + /* Default values */ double epsilon = 1e-9; + int timestamp = 1; + int sequence = 1; + int data = 1; struct log log; struct pool pool = { .state = STATE_DESTROYED }; @@ -75,7 +85,7 @@ int main(int argc, char *argv[]) /* Parse Arguments */ char c, *endptr; - while ((c = getopt (argc, argv, "hjmd:e:l:H:r:")) != -1) { + while ((c = getopt (argc, argv, "hjmd:e:l:H:r:vts")) != -1) { switch (c) { case 'd': log.level = strtoul(optarg, &endptr, 10); @@ -83,6 +93,15 @@ int main(int argc, char *argv[]) case 'e': epsilon = strtod(optarg, &endptr); goto check; + case 'v': + data = 0; + break; + case 't': + timestamp = 0; + break; + case 's': + sequence = 0; + break; case 'h': case '?': usage(); @@ -103,11 +122,21 @@ check: if (optarg == endptr) f1.path = argv[optind]; f2.path = argv[optind + 1]; - log_init(&log, V, LOG_ALL); - log_start(&log); + ret = log_init(&log, V, LOG_ALL); + if (ret) + error("Failed to initialize log"); - pool_init(&pool, 1024, SAMPLE_LEN(DEFAULT_SAMPLELEN), &memtype_heap); - sample_alloc(&pool, samples, 2); + ret = log_start(&log); + if (ret) + error("Failed to start log"); + + ret = pool_init(&pool, 1024, SAMPLE_LEN(DEFAULT_SAMPLELEN), &memtype_heap); + if (ret) + error("Failed to initialize pool"); + + ret = sample_alloc(&pool, samples, 2); + if (ret != 2) + error("Failed to allocate samples"); f1.sample = samples[0]; f2.sample = samples[1]; @@ -121,11 +150,11 @@ check: if (optarg == endptr) serror("Failed to open file: %s", f2.path); while (!feof(f1.handle) && !feof(f2.handle)) { - ret = sample_io_villas_fscan(f1.handle, f1.sample, &f1.flags); + ret = villas_fscan(f1.handle, &f1.sample, 1, &f1.flags); if (ret < 0 && !feof(f1.handle)) goto out; - ret = sample_io_villas_fscan(f2.handle, f2.sample, &f2.flags); + ret = villas_fscan(f2.handle, &f2.sample, 1, &f2.flags); if (ret < 0 && !feof(f2.handle)) goto out; @@ -139,7 +168,7 @@ check: if (optarg == endptr) } /* Compare sequence no */ - if ((f1.flags & SAMPLE_IO_SEQUENCE) && (f2.flags & SAMPLE_IO_SEQUENCE)) { + if (sequence && (f1.flags & IO_FORMAT_SEQUENCE) && (f2.flags & IO_FORMAT_SEQUENCE)) { if (f1.sample->sequence != f2.sample->sequence) { printf("sequence no: %d != %d\n", f1.sample->sequence, f2.sample->sequence); ret = 2; @@ -148,25 +177,29 @@ check: if (optarg == endptr) } /* Compare timestamp */ - if (time_delta(&f1.sample->ts.origin, &f2.sample->ts.origin) > epsilon) { - printf("ts.origin: %f != %f\n", time_to_double(&f1.sample->ts.origin), time_to_double(&f2.sample->ts.origin)); - ret = 3; - goto out; + if (timestamp) { + if (time_delta(&f1.sample->ts.origin, &f2.sample->ts.origin) > epsilon) { + printf("ts.origin: %f != %f\n", time_to_double(&f1.sample->ts.origin), time_to_double(&f2.sample->ts.origin)); + ret = 3; + goto out; + } } /* Compare data */ - if (f1.sample->length != f2.sample->length) { - printf("length: %d != %d\n", f1.sample->length, f2.sample->length); - ret = 4; - goto out; - } - - for (int i = 0; i < f1.sample->length; i++) { - if (fabs(f1.sample->data[i].f - f2.sample->data[i].f) > epsilon) { - printf("data[%d]: %f != %f\n", i, f1.sample->data[i].f, f2.sample->data[i].f); - ret = 5; + if (data) { + if (f1.sample->length != f2.sample->length) { + printf("length: %d != %d\n", f1.sample->length, f2.sample->length); + ret = 4; goto out; } + + for (int i = 0; i < f1.sample->length; i++) { + if (fabs(f1.sample->data[i].f - f2.sample->data[i].f) > epsilon) { + printf("data[%d]: %f != %f\n", i, f1.sample->data[i].f, f2.sample->data[i].f); + ret = 5; + goto out; + } + } } } diff --git a/tests/integration/api-config.sh b/tests/integration/api-config.sh index 3480464eb..dc4778d25 100755 --- a/tests/integration/api-config.sh +++ b/tests/integration/api-config.sh @@ -22,14 +22,17 @@ # along with this program. If not, see . ################################################################################## -set -e +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) -LOCAL_CONF=${SRCDIR}/etc/loopback.conf +LOCAL_CONF=${SCRIPTPATH}/../../etc/loopback.json FETCHED_CONF=$(mktemp) FETCHED_JSON_CONF=$(mktemp) LOCAL_JSON_CONF=$(mktemp) +ID=$(uuidgen) + # Start VILLASnode instance with local config (via advio) villas-node file://${LOCAL_CONF} & @@ -37,16 +40,13 @@ villas-node file://${LOCAL_CONF} & sleep 1 # Fetch config via API -curl -sX POST --data '{ "action" : "config", "id" : "5a786626-fbc6-4c04-98c2-48027e68c2fa" }' http://localhost/api/v1 > ${FETCHED_CONF} +curl -sX POST --data '{ "action" : "config", "id" : "'${ID}'" }' http://localhost/api/v1 > ${FETCHED_CONF} # Shutdown VILLASnode kill $! -conf2json < ${LOCAL_CONF} | jq -S . > ${LOCAL_JSON_CONF} -jq -S .response < ${FETCHED_CONF} > ${FETCHED_JSON_CONF} - # Compare local config with the fetched one -diff -u ${FETCHED_JSON_CONF} ${LOCAL_JSON_CONF} +diff -u <(jq -S .response < ${FETCHED_CONF}) <(jq -S . < ${LOCAL_CONF}) RC=$? rm -f ${FETCHED_CONF} ${FETCHED_JSON_CONF} ${LOCAL_JSON_CONF} diff --git a/tests/integration/api-nodes.sh b/tests/integration/api-nodes.sh index 5886b5822..f010b7b2e 100755 --- a/tests/integration/api-nodes.sh +++ b/tests/integration/api-nodes.sh @@ -28,17 +28,19 @@ CONFIG_FILE=$(mktemp) FETCHED_NODES=$(mktemp) cat > ${CONFIG_FILE} < ${FETCHED_CONF} diff --git a/tests/integration/hook-convert.sh b/tests/integration/hook-convert.sh index 8625cffab..81e3cef90 100755 --- a/tests/integration/hook-convert.sh +++ b/tests/integration/hook-convert.sh @@ -52,13 +52,13 @@ cat < ${EXPECT_FILE} 1490500400.676379108(9) -0.587785 -58 -58 -0.587785 EOF -villas-hook convert 'mode="fixed" scale=100 mask=0x6' < ${INPUT_FILE} > ${OUTPUT_FILE} +villas-hook convert -o mode=fixed -o scale=100 -o mask=0x6 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -diff -u <(cut -f2- ${OUTPUT_FILE}) <(cut -f2- ${EXPECT_FILE}) +villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/hook-decimate.sh b/tests/integration/hook-decimate.sh index 7a85cc42e..e92a6743d 100755 --- a/tests/integration/hook-decimate.sh +++ b/tests/integration/hook-decimate.sh @@ -46,13 +46,13 @@ cat < ${EXPECT_FILE} 1490500400.676379108(9) -0.587785 -0.587785 -0.587785 -0.587785 EOF -villas-hook decimate 'ratio=3' < ${INPUT_FILE} > ${OUTPUT_FILE} +villas-hook decimate -o ratio=3 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -diff -u <(cut -f2- ${OUTPUT_FILE}) <(cut -f2- ${EXPECT_FILE}) +villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/hook-drop.sh b/tests/integration/hook-drop.sh index 29959949e..2684ec9c5 100755 --- a/tests/integration/hook-drop.sh +++ b/tests/integration/hook-drop.sh @@ -55,10 +55,10 @@ EOF villas-hook drop < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -diff -u <(cut -f2- ${OUTPUT_FILE}) <(cut -f2- ${EXPECT_FILE}) +villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/hook-map.sh b/tests/integration/hook-map.sh index 1a52a8c20..2b6f64d72 100755 --- a/tests/integration/hook-map.sh +++ b/tests/integration/hook-map.sh @@ -52,13 +52,13 @@ cat < ${EXPECT_FILE} 1490500400.676379108(9) -0.587785 9 1490500400 676379108 -0.587785 -0.587785 EOF -villas-hook map 'mapping = [ "data[3]", "hdr.sequence", "ts.origin", "data[1-2]" ]' < ${INPUT_FILE} > ${OUTPUT_FILE} +villas-hook map -o map=data[3] -o map=hdr.sequence -o map=ts.origin -o map=data[1-2] < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -diff -u <(cut -f2- ${OUTPUT_FILE}) <(cut -f2- ${EXPECT_FILE}) +villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/hook-print.sh b/tests/integration/hook-print.sh new file mode 100755 index 000000000..283985f4e --- /dev/null +++ b/tests/integration/hook-print.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Integration test for print hook. +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################## + +INPUT_FILE=$(mktemp) +OUTPUT_FILE1=$(mktemp) +OUTPUT_FILE2=$(mktemp) + +NUM_SAMPLES=${NUM_SAMPLES:-100} + +# Prepare some test data +villas-signal random -v 1 -r 10 -l ${NUM_SAMPLES} -n > ${INPUT_FILE} + +villas-hook print -o format=villas -o output=${OUTPUT_FILE1} < ${INPUT_FILE} > ${OUTPUT_FILE2} + +# Compare only the data values +villas-test-cmp ${OUTPUT_FILE1} ${INPUT_FILE} && \ +villas-test-cmp ${OUTPUT_FILE2} ${INPUT_FILE} +RC=$? + +rm -f ${INPUT_FILE} ${OUTPUT_FILE1} ${OUTPUT_FILE2} + +exit $RC diff --git a/tests/integration/hook-shift_seq.sh b/tests/integration/hook-shift_seq.sh index 8aedc9e20..40dc842d9 100755 --- a/tests/integration/hook-shift_seq.sh +++ b/tests/integration/hook-shift_seq.sh @@ -23,18 +23,21 @@ ################################################################################## OUTPUT_FILE=$(mktemp) +INPUT_FILE=$(mktemp) OFFSET=100 -villas-signal random -l ${NUM_SAMPLES} -n | villas-hook shift_seq offset=${OFFSET} > ${OUTPUT_FILE} +villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} + +villas-hook shift_seq -o offset=${OFFSET} > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare shifted sequence no diff -u \ - <(sed -re 's/^[0-9]+\.[0-9]+([\+\-][0-9]+\.[0-9]+(e[\+\-][0-9]+)?)?\(([0-9]+)\).*/\3/g' ${OUTPUT_FILE}) \ + <(sed -re '/^#/d;s/^[0-9]+\.[0-9]+([\+\-][0-9]+\.[0-9]+(e[\+\-][0-9]+)?)?\(([0-9]+)\).*/\3/g' ${OUTPUT_FILE}) \ <(seq ${OFFSET} $((${NUM_SAMPLES}+${OFFSET}-1))) RC=$? -rm -f ${OUTPUT_FILE} +rm -f ${OUTPUT_FILE} ${INPUT_FILE} exit $RC diff --git a/tests/integration/hook-shift_ts.sh b/tests/integration/hook-shift_ts.sh index 03aacf002..082778013 100755 --- a/tests/integration/hook-shift_ts.sh +++ b/tests/integration/hook-shift_ts.sh @@ -26,10 +26,12 @@ STATS_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} -OFFSET=-10 +OFFSET=-10.0 EPSILON=0.05 -villas-signal random -l ${NUM_SAMPLES} -r 10 | villas-hook shift_ts offset=${OFFSET} | villas-hook stats format=\"json\" output=\"${STATS_FILE}\" > /dev/null +villas-signal random -l ${NUM_SAMPLES} -r 50 | \ +villas-hook shift_ts -o offset=${OFFSET} | \ +villas-hook stats -o format=json -o output="${STATS_FILE}" > /dev/null jq .owd ${STATS_FILE} diff --git a/tests/integration/hook-skip_first.sh b/tests/integration/hook-skip_first.sh index 7b0892c53..734824590 100755 --- a/tests/integration/hook-skip_first.sh +++ b/tests/integration/hook-skip_first.sh @@ -28,11 +28,11 @@ SKIP=10 echo ${OUTPUT_FILE} -villas-signal random -r 1 -l ${NUM_SAMPLES} -n | villas-hook skip_first seconds=${SKIP} > ${OUTPUT_FILE} +villas-signal random -r 1 -l ${NUM_SAMPLES} -n | villas-hook skip_first -o seconds=${SKIP} > ${OUTPUT_FILE} -LINES=$(wc -l < ${OUTPUT_FILE}) +LINES=$(sed -re '/^#/d' ${OUTPUT_FILE} | wc -l) rm ${OUTPUT_FILE} # Test condition -(( ${LINES} == ${NUM_SAMPLES} - ${SKIP} )) \ No newline at end of file +(( ${LINES} == ${NUM_SAMPLES} - ${SKIP} )) diff --git a/tests/integration/hook-skip_first2.sh b/tests/integration/hook-skip_first2.sh index 4ea001ffb..5d8e4f637 100755 --- a/tests/integration/hook-skip_first2.sh +++ b/tests/integration/hook-skip_first2.sh @@ -26,11 +26,11 @@ OUTPUT_FILE=$(mktemp) SKIP=50 -villas-signal random -r 1 -l ${NUM_SAMPLES} -n | villas-hook skip_first samples=${SKIP} > ${OUTPUT_FILE} +villas-signal random -r 1 -l ${NUM_SAMPLES} -n | villas-hook skip_first -o samples=${SKIP} > ${OUTPUT_FILE} -LINES=$(wc -l < ${OUTPUT_FILE}) +LINES=$(sed -re '/^#/d' ${OUTPUT_FILE} | wc -l) rm ${OUTPUT_FILE} # Test condition -(( ${LINES} == ${NUM_SAMPLES} - ${SKIP} )) \ No newline at end of file +(( ${LINES} == ${NUM_SAMPLES} - ${SKIP} )) diff --git a/tests/integration/node-loopback-socket.sh b/tests/integration/node-loopback-socket.sh index a2ca9f70c..d8b9315d7 100755 --- a/tests/integration/node-loopback-socket.sh +++ b/tests/integration/node-loopback-socket.sh @@ -22,6 +22,10 @@ # along with this program. If not, see . ################################################################################## +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + CONFIG_FILE=$(mktemp) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) @@ -29,42 +33,40 @@ OUTPUT_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} cat > ${CONFIG_FILE} < ${INPUT_FILE} # Start node +VILLAS_LOG_PREFIX=$(colorize "[Node] ") \ villas-node ${CONFIG_FILE} & # Wait for node to complete init sleep 1 # Send / Receive data to node +VILLAS_LOG_PREFIX=$(colorize "[Pipe] ") \ villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node2 > ${OUTPUT_FILE} < ${INPUT_FILE} # Wait for node to handle samples diff --git a/tests/integration/pipe-file-advio.sh b/tests/integration/pipe-file-advio.sh index 97b1885d7..4bd7cdbd1 100755 --- a/tests/integration/pipe-file-advio.sh +++ b/tests/integration/pipe-file-advio.sh @@ -22,6 +22,10 @@ # along with this program. If not, see . ################################################################################## +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + CONFIG_FILE=$(mktemp) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) @@ -30,37 +34,34 @@ NUM_SAMPLES=${NUM_SAMPLES:-10} URI=https://1Nrd46fZX8HbggT:badpass@rwth-aachen.sciebo.de/public.php/webdav/node/tests/pipe +# WebDav / OwnCloud / Sciebo do not support partial upload +# So we disable flusing the output cat > ${CONFIG_FILE} < /dev/null +VILLAS_LOG_PREFIX=$(colorize "[Signal] ") \ villas-signal random -n -l ${NUM_SAMPLES} > ${INPUT_FILE} -villas-pipe -s ${CONFIG_FILE} remote_file_out < ${INPUT_FILE} +VILLAS_LOG_PREFIX=$(colorize "[Send] ") \ +villas-pipe -s ${CONFIG_FILE} remote_file < ${INPUT_FILE} -villas-pipe -r ${CONFIG_FILE} remote_file_in > ${OUTPUT_FILE} +VILLAS_LOG_PREFIX=$(colorize "[Recv] ") \ +villas-pipe -r -l ${NUM_SAMPLES} ${CONFIG_FILE} remote_file > ${OUTPUT_FILE} villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} RC=$? diff --git a/tests/integration/pipe-loopback-file.sh b/tests/integration/pipe-loopback-file.sh index f8c63e752..ce961e5b3 100755 --- a/tests/integration/pipe-loopback-file.sh +++ b/tests/integration/pipe-loopback-file.sh @@ -30,21 +30,17 @@ NODE_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "file"; - - in = { - uri = "${NODE_FILE}", - mode = "w+", +{ + "nodes" : { + "node1" : { + "type" : "file", - epoch_mode = "original", - eof = "wait" - }, - out = { - uri = "${NODE_FILE}" - mode = "w+" - flush = true /* we need to flush / upload the new samples continously for a loopback */ + "uri" : "${NODE_FILE}", + "mode" : "w+", + + "epoch_mode" : "original", + "eof" : "wait", + "flush" : true } } } @@ -56,10 +52,14 @@ villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} # We delay EOF of the INPUT_FILE by 1 second in order to wait for incoming data to be received villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} -# Comapre data +# Compare data villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} RC=$? +cat ${OUTPUT_FILE} +echo +cat ${INPUT_FILE} + rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} ${NODE_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/pipe-loopback-loopback.sh b/tests/integration/pipe-loopback-loopback.sh index c1622e056..d057f640d 100755 --- a/tests/integration/pipe-loopback-loopback.sh +++ b/tests/integration/pipe-loopback-loopback.sh @@ -29,9 +29,11 @@ OUTPUT_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "loopback" +{ + "nodes" : { + "node1" : { + "type" : "loopback" + } } } EOF @@ -48,7 +50,7 @@ cat ${OUTPUT_FILE} # Comapre data villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} -RC=$? +RC:$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-multicast.sh b/tests/integration/pipe-loopback-multicast.sh index 91339e168..c7e82e815 100755 --- a/tests/integration/pipe-loopback-multicast.sh +++ b/tests/integration/pipe-loopback-multicast.sh @@ -29,19 +29,21 @@ OUTPUT_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "socket"; - layer = "udp"; +{ + "nodes" : { + "node1" : { + "type" : "socket"; + "layer" : "udp"; - local = "*:12000"; - remote = "224.1.2.3:12000"; - - multicast = { # IGMP multicast is only support for layer = (ip|udp) - enabled = true, - - group = "224.1.2.3", # The multicast group. Must be within 224.0.0.0/4 - loop = true # Whether or not to loopback outgoing multicast packets to the local host. + "local" : "*:12000"; + "remote" : "224.1.2.3:12000"; + + "multicast" : { + "enabled" : true, + + "group" : "224.1.2.3", + "loop" : true + } } } } @@ -55,8 +57,8 @@ villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FI # Comapre data villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} -RC=$? +RC:$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/pipe-loopback-nanomsg.sh b/tests/integration/pipe-loopback-nanomsg.sh index c63be12d1..34c9fc53d 100755 --- a/tests/integration/pipe-loopback-nanomsg.sh +++ b/tests/integration/pipe-loopback-nanomsg.sh @@ -22,37 +22,72 @@ # along with this program. If not, see . ################################################################################## +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + CONFIG_FILE=$(mktemp) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) -NUM_SAMPLES=${NUM_SAMPLES:-10} - -cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "nanomsg"; - - subscribe = "tcp://127.0.0.1:12000"; - publish = "tcp://127.0.0.1:12000" - } -} -EOF +NUM_SAMPLES=${NUM_SAMPLES:-100} # Generate test data villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} +for FORMAT in csv json villas csv msg gtnet-fake raw-flt32 gtnet-fake; do + +VECTORIZES="1" + +# The raw format does not support vectors +if villas_format_supports_vectorize ${FORMAT}; then + VECTORIZES="${VECTORIZES} 10" +fi + +for VECTORIZE in ${VECTORIZES}; do + +cat > ${CONFIG_FILE} << EOF +{ + "nodes" : { + "node1" : { + "type" : "nanomsg", + + "format" : "${FORMAT}", + "vectorize" : ${VECTORIZE}, + + "subscribe" : "tcp://127.0.0.1:12000", + "publish" : "tcp://127.0.0.1:12000" + } + } +} +EOF + # We delay EOF of the INPUT_FILE by 1 second in order to wait for incoming data to be received villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} -cat ${INPUT_FILE} -echo -cat ${OUTPUT_FILE} +# Ignore timestamp and seqeunce no if in raw format +if villas_format_supports_header ${FORMAT}; then + CMPFLAGS=-ts +fi -# Comapre data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +# Compare data +villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? +if (( ${RC} != 0 )); then + echo "=========== Sub-test failed for: format=${FORMAT}, vectorize=${VECTORIZE}" + cat ${CONFIG_FILE} + echo + cat ${INPUT_FILE} + echo + cat ${OUTPUT_FILE} + exit ${RC} +else + echo "=========== Sub-test succeeded for: format=${FORMAT}, vectorize=${VECTORIZE}" +fi + +done; done + rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/pipe-loopback-shmem.sh b/tests/integration/pipe-loopback-shmem.sh index d0cada19c..32545fc67 100755 --- a/tests/integration/pipe-loopback-shmem.sh +++ b/tests/integration/pipe-loopback-shmem.sh @@ -33,18 +33,20 @@ for POLLING in true false; do for VECTORIZE in 1 5 25; do cat > ${CONFIG_FILE} << EOF -logging = { - level = 2 -} -nodes = { - node1 = { - type = "shmem"; - out_name = "/villas-test"; - in_name = "/villas-test"; - samplelen = ${SAMPLELEN}; - queuelen = 1024, - polling = ${POLLING}; - vectorize = ${VECTORIZE}; +{ + "logging" : { + "level" : 2 + }, + "nodes" : { + "node1" : { + "type" : "shmem", + "out_name" : "/villas-test", + "in_name" : "/villas-test", + "samplelen" : ${SAMPLELEN}, + "queuelen" : 1024, + "polling" : ${POLLING}, + "vectorize" : ${VECTORIZE} + } } } EOF diff --git a/tests/integration/pipe-loopback-socket.sh b/tests/integration/pipe-loopback-socket.sh index 2df6c163f..25e1fd9e7 100755 --- a/tests/integration/pipe-loopback-socket.sh +++ b/tests/integration/pipe-loopback-socket.sh @@ -22,53 +22,70 @@ # along with this program. If not, see . ################################################################################## +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + CONFIG_FILE=$(mktemp) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) THEORIES=$(mktemp) -NUM_SAMPLES=${NUM_SAMPLES:-10} +NUM_SAMPLES=${NUM_SAMPLES:-100} # Generate test data villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} +for FORMAT in csv json villas csv msg gtnet-fake raw-flt32 gtnet-fake; do for LAYER in udp ip eth; do for HEADER in none default; do for ENDIAN in big little; do for VERIFY_SOURCE in true false; do + +VECTORIZES="1" + +# The raw format does not support vectors +if villas_format_supports_vectorize ${FORMAT}; then + VECTORIZES="${VECTORIZES} 10" +fi + +for VECTORIZE in ${VECTORIZES}; do case ${LAYER} in udp) LOCAL="*:12000" REMOTE="127.0.0.1:12000" ;; - + ip) # We use IP protocol number 253 which is reserved for experimentation and testing according to RFC 3692 LOCAL="127.0.0.1:254" REMOTE="127.0.0.1:254" ;; - + eth) # We use IP protocol number 253 which is reserved for experimentation and testing according to RFC 7042 LOCAL="00:00:00:00:00:00%lo:34997" REMOTE="00:00:00:00:00:00%lo:34997" ;; - esac - +esac cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "socket"; +{ + "nodes" : { + "node1" : { + "type" : "socket", + + "vectorize" : ${VECTORIZE}, + "format" : "${FORMAT}", + "layer" : "${LAYER}", + "header" : "${HEADER}", + "endian" : "${ENDIAN}", + "verify_source" : ${VERIFY_SOURCE}, - layer = "${LAYER}"; - header = "${HEADER}"; - endian = "${ENDIAN}"; - verify_source = ${VERIFY_SOURCE}; - - local = "${LOCAL}"; - remote = "${REMOTE}"; + "local" : "${LOCAL}", + "remote" : "${REMOTE}" + } } } EOF @@ -76,12 +93,17 @@ EOF # We delay EOF of the INPUT_FILE by 1 second in order to wait for incoming data to be received villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} +# Ignore timestamp and seqeunce no if in raw format +if ! villas_format_supports_header $FORMAT; then + CMPFLAGS=-ts +fi + # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? if (( ${RC} != 0 )); then - echo "=========== Sub-test failed for: layer=${LAYER}, header=${HEADER}, endian=${ENDIAN} verify_source=${VERIFY_SOURCE}" + echo "=========== Sub-test failed for: format=${FORMAT}, layer=${LAYER}, header=${HEADER}, endian=${ENDIAN}, verify_source=${VERIFY_SOURCE}, vectorize=${VECTORIZE}" cat ${CONFIG_FILE} echo cat ${INPUT_FILE} @@ -89,11 +111,11 @@ if (( ${RC} != 0 )); then cat ${OUTPUT_FILE} exit ${RC} else - echo "=========== Sub-test succeeded for: layer=${LAYER}, header=${HEADER}, endian=${ENDIAN} verify_source=${VERIFY_SOURCE}" + echo "=========== Sub-test succeeded for: format=${FORMAT}, layer=${LAYER}, header=${HEADER}, endian=${ENDIAN}, verify_source=${VERIFY_SOURCE}, vectorize=${VECTORIZE}" fi -done; done; done; done +done; done; done; done; done; done rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} ${THEORIES} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/integration/pipe-loopback-websocket.sh b/tests/integration/pipe-loopback-websocket.sh new file mode 100755 index 000000000..f42838c1e --- /dev/null +++ b/tests/integration/pipe-loopback-websocket.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Integration loopback test for villas-pipe. +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################## + +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + +CONFIG_FILE=$(mktemp) +CONFIG_FILE2=$(mktemp) +INPUT_FILE=$(mktemp) +OUTPUT_FILE=$(mktemp) + +NUM_SAMPLES=${NUM_SAMPLES:-100} + +cat > ${CONFIG_FILE} << EOF +{ + "nodes" : { + "node1" : { + "type" : "websocket", + + "destinations" : [ + "ws://127.0.0.1:81/node2" + ] + } + } +} +EOF + +cat > ${CONFIG_FILE2} << EOF +{ + "http" : { + "port" : 81 + }, + "nodes" : { + "node2" : { + "type" : "websocket" + } + } +} +EOF + +# Generate test data +VILLAS_LOG_PREFIX=$(colorize "[Signal]") \ +villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} + +VILLAS_LOG_PREFIX=$(colorize "[Recv] ") \ +villas-pipe -r -d 15 -l ${NUM_SAMPLES} ${CONFIG_FILE2} node2 > ${OUTPUT_FILE} & + +VILLAS_LOG_PREFIX=$(colorize "[Send] ") \ +villas-pipe -s -d 15 ${CONFIG_FILE} node1 < ${INPUT_FILE} + +wait $! + +# Compare data +villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +RC=$? + +rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} ${CONFIG_FILE2} + +exit $RC diff --git a/tests/integration/pipe-loopback-zeromq.sh b/tests/integration/pipe-loopback-zeromq.sh index 448077ae7..881663ba6 100755 --- a/tests/integration/pipe-loopback-zeromq.sh +++ b/tests/integration/pipe-loopback-zeromq.sh @@ -22,34 +22,72 @@ # along with this program. If not, see . ################################################################################## +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + CONFIG_FILE=$(mktemp) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} -cat > ${CONFIG_FILE} << EOF -nodes = { - node1 = { - type = "zeromq"; +# Generate test data +villas-signal random -l ${NUM_SAMPLES} -n -v 10 > ${INPUT_FILE} - pattern = "pubsub"; - subscribe = "tcp://127.0.0.1:12000"; - publish = "tcp://127.0.0.1:12000" +for FORMAT in csv json villas csv msg gtnet-fake raw-flt32 gtnet-fake; do + +VECTORIZES="1" + +# The raw format does not support vectors +if villas_format_supports_vectorize ${FORMAT}; then + VECTORIZES="${VECTORIZES} 10" +fi + +for VECTORIZE in ${VECTORIZES}; do + +cat > ${CONFIG_FILE} << EOF +{ + "nodes" : { + "node1" : { + "type" : "zeromq", + + "format" : "${FORMAT}", + "vectorize" : ${VECTORIZE}, + "pattern" : "pubsub", + "subscribe" : "tcp://127.0.0.1:12000", + "publish" : "tcp://127.0.0.1:12000" + } } } EOF -# Generate test data -villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} - # We delay EOF of the INPUT_FILE by 1 second in order to wait for incoming data to be received villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} -# Comapre data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +# Ignore timestamp and seqeunce no if in raw format +if villas_format_supports_header ${FORMAT}; then + CMPFLAGS=-ts +fi + +# Compare data +villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? +if (( ${RC} != 0 )); then + echo "=========== Sub-test failed for: format=${FORMAT}, vectorize=${VECTORIZE}" + cat ${CONFIG_FILE} + echo + cat ${INPUT_FILE} + echo + cat ${OUTPUT_FILE} + exit ${RC} +else + echo "=========== Sub-test succeeded for: format=${FORMAT}, vectorize=${VECTORIZE}" +fi + +done; done + rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} -exit $RC \ No newline at end of file +exit $RC diff --git a/tests/unit/advio.c b/tests/unit/advio.c index 96bbbd608..ec43aec22 100644 --- a/tests/unit/advio.c +++ b/tests/unit/advio.c @@ -28,24 +28,38 @@ #include #include #include -#include +#include /** This URI points to a Sciebo share which contains some test files. * The Sciebo share is read/write accessible via WebDAV. */ #define BASE_URI "https://1Nrd46fZX8HbggT:badpass@rwth-aachen.sciebo.de/public.php/webdav/node/tests" +Test(advio, islocal) +{ + int ret; + + ret = aislocal("/var/log/villas/dta.dat"); + cr_assert_eq(ret, 1); + + ret = aislocal("http://www.google.de"); + cr_assert_eq(ret, 0); + + ret = aislocal("torrent://www.google.de"); + cr_assert_eq(ret, -1); +} + Test(advio, local) { AFILE *af; int ret; char *buf = NULL; size_t buflen = 0; - + /* We open this file and check the first line */ af = afopen(__FILE__, "r"); cr_assert(af, "Failed to open local file"); - ret = getline(&buf, &buflen, af->file); + ret = getline(&buf, &buflen, af->file); cr_assert_gt(ret, 1); cr_assert_str_eq(buf, "/** Unit tests for advio\n"); } @@ -74,26 +88,26 @@ Test(advio, download_large) { AFILE *af; int ret, len = 16; - + struct sample *smp = alloc(SAMPLE_LEN(len)); smp->capacity = len; af = afopen(BASE_URI "/download-large" , "r"); cr_assert(af, "Failed to download file"); - ret = sample_io_villas_fscan(af->file, smp, NULL); - cr_assert_eq(ret, 0); - + ret = villas_fscan(af->file, &smp, 1, NULL); + cr_assert_eq(ret, 1); + cr_assert_eq(smp->sequence, 0); cr_assert_eq(smp->length, 4); cr_assert_eq(smp->ts.origin.tv_sec, 1497710378); cr_assert_eq(smp->ts.origin.tv_nsec, 863332240); - + float data[] = { 0.022245, 0.000000, -1.000000, 1.000000 }; - + for (int i = 0; i < smp->length; i++) cr_assert_float_eq(smp->data[i].f, data[i], 1e-5); - + ret = afclose(af); cr_assert_eq(ret, 0, "Failed to close file"); } @@ -106,44 +120,44 @@ Test(advio, resume) char line1[32]; char *line2 = NULL; size_t linelen = 0; - + mkdtemp(dir); ret = asprintf(&fn, "%s/file", dir); cr_assert_gt(ret, 0); - + af1 = afopen(fn, "w+"); cr_assert_not_null(af1); - + /* We flush once the empty file in order to upload an empty file. */ aupload(af1, 0); - + af2 = afopen(fn, "r"); cr_assert_not_null(af2); - + for (int i = 0; i < 100; i++) { snprintf(line1, sizeof(line1), "This is line %d\n", i); - + afputs(line1, af1); aupload(af1, 1); - + adownload(af2, 1); agetline(&line2, &linelen, af2); - + cr_assert_str_eq(line1, line2); } - + ret = afclose(af1); cr_assert_eq(ret, 0); - + ret = afclose(af2); cr_assert_eq(ret, 0); - + ret = unlink(fn); cr_assert_eq(ret, 0); - + ret = rmdir(dir); cr_assert_eq(ret, 0); - + free(line2); } @@ -236,4 +250,4 @@ Test(advio, append) cr_assert(ret == 0, "Failed to close file"); cr_assert_arr_eq(buffer, expect, sizeof(expect)); -} \ No newline at end of file +} diff --git a/tests/unit/config_json.c b/tests/unit/config_json.c index 8f372eb33..d1b86c6f7 100644 --- a/tests/unit/config_json.c +++ b/tests/unit/config_json.c @@ -22,6 +22,8 @@ #include +#ifdef WITH_LIBCONFIG + #include #include @@ -114,3 +116,5 @@ Test(utils, json_to_config) json_decref(json); } + +#endif diff --git a/tests/unit/fpga.c b/tests/unit/fpga.c index 1cd93b3ad..a7703eec3 100644 --- a/tests/unit/fpga.c +++ b/tests/unit/fpga.c @@ -40,7 +40,6 @@ static struct vfio_container vc; static void init() { int ret; - config_setting_t *cfg_root; ret = super_node_init(&sn); cr_assert_eq(ret, 0, "Failed to initialize Supernode"); @@ -51,9 +50,6 @@ static void init() ret = super_node_check(&sn); cr_assert_eq(ret, 0, "Failed to check configuration"); - cfg_root = config_root_setting(&sn.cfg); - cr_assert_neq(cfg_root, NULL); - ret = pci_init(&pci); cr_assert_eq(ret, 0, "Failed to initialize PCI sub-system"); @@ -62,7 +58,7 @@ static void init() /* Parse FPGA configuration */ list_init(&cards); - ret = fpga_card_parse_list(&cards, cfg_root); + ret = fpga_card_parse_list(&cards, sn.cfg); cr_assert_eq(ret, 0, "Failed to parse FPGA config"); card = list_lookup(&cards, "vc707"); @@ -120,7 +116,7 @@ Test(fpga, xsg, .description = "XSG: multiply_add") double factor, err = 0; struct fpga_ip *ip, *dma; - struct model_param *p; + struct model_parameter *p; struct dma_mem mem; ip = fpga_vlnv_lookup(&card->ips, &(struct fpga_vlnv) { NULL, "sysgen", "xsg_multiply", NULL }); @@ -135,7 +131,7 @@ Test(fpga, xsg, .description = "XSG: multiply_add") if (!p) error("Missing parameter 'factor' for model '%s'", ip->name); - ret = model_param_read(p, &factor); + ret = model_parameter_read(p, &factor); cr_assert_eq(ret, 0, "Failed to read parameter 'factor' from model '%s'", ip->name); info("Model param: factor = %f", factor); diff --git a/tests/unit/io.c b/tests/unit/io.c new file mode 100644 index 000000000..bb2834908 --- /dev/null +++ b/tests/unit/io.c @@ -0,0 +1,271 @@ +/** Unit tests for IO formats. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include +#include +#include + +#include "utils.h" +#include "timing.h" +#include "sample.h" +#include "plugin.h" +#include "pool.h" +#include "io.h" +#include "io/raw.h" + +#define NUM_SAMPLES 10 +#define NUM_VALUES 10 + +static char formats[][32] = { +#ifdef WITH_HDF5 + "hdf5", +#endif + "raw-int8", + "raw-int16-be", + "raw-int16-le", + "raw-int32-be", + "raw-int32-le", + "raw-int64-be", + "raw-int64-le", + "raw-flt32", + "raw-flt64", + "villas", + "csv", + "json", + "msg", + "gtnet", + "gtnet-fake" +}; + +void generate_samples(struct pool *p, struct sample *smps[], struct sample *smpt[], unsigned cnt, unsigned values) +{ + int ret; + + /* Prepare a sample with arbitrary data */ + ret = sample_alloc(p, smps, cnt); + cr_assert_eq(ret, NUM_SAMPLES); + + ret = sample_alloc(p, smpt, cnt); + cr_assert_eq(ret, cnt); + + for (int i = 0; i < cnt; i++) { + smpt[i]->capacity = values; + + smps[i]->length = values; + smps[i]->sequence = 235 + i; + smps[i]->format = 0; /* all float */ + smps[i]->ts.origin = time_now(); + + for (int j = 0; j < smps[i]->length; j++) + smps[i]->data[j].f = j * 0.1 + i * 100; + //smps[i]->data[j].i = -500 + j*100; + } +} + +void cr_assert_eq_samples(struct io_format *f, struct sample *smps[], struct sample *smpt[], unsigned cnt) +{ + /* The RAW format has certain limitations: + * - limited accuracy if smaller datatypes are used + * - no support for vectors / framing + * + * We need to take these limitations into account before comparing. + */ + if (f->sscan == raw_sscan) { + cr_assert_eq(cnt, 1); + cr_assert_eq(smpt[0]->length, smpt[0]->capacity, "Expected values: %d, Received values: %d", smpt[0]->capacity, smpt[0]->length); + + if (f->flags & RAW_FAKE) { + + } + else { + smpt[0]->sequence = smps[0]->sequence; + smpt[0]->ts.origin = smps[0]->ts.origin; + } + + int bits = 1 << (f->flags >> 24); + for (int j = 0; j < smpt[0]->length; j++) { + if (f->flags & RAW_FLT) { + switch (bits) { + case 32: smps[0]->data[j].f = (float) smps[0]->data[j].f; break; + case 64: smps[0]->data[j].f = (double) smps[0]->data[j].f; break; + } + } + else { + switch (bits) { + case 8: smps[0]->data[j].i = ( int8_t) smps[0]->data[j].i; break; + case 16: smps[0]->data[j].i = ( int16_t) smps[0]->data[j].i; break; + case 32: smps[0]->data[j].i = ( int32_t) smps[0]->data[j].i; break; + case 64: smps[0]->data[j].i = ( int64_t) smps[0]->data[j].i; break; + } + } + } + } + else + cr_assert_eq(cnt, NUM_SAMPLES, "Read only %d of %d samples back", cnt, NUM_SAMPLES); + + for (int i = 0; i < cnt; i++) { + cr_assert_eq(smps[i]->length, smpt[i]->length); + cr_assert_eq(smps[i]->sequence, smpt[i]->sequence); + + cr_assert_eq(smps[i]->ts.origin.tv_sec, smpt[i]->ts.origin.tv_sec); + cr_assert_eq(smps[i]->ts.origin.tv_nsec, smpt[i]->ts.origin.tv_nsec); + + for (int j = 0; j < MIN(smps[i]->length, smpt[i]->length); j++) { + switch (sample_get_data_format(smpt[i], j)) { + case SAMPLE_DATA_FORMAT_FLOAT: + cr_assert_float_eq(smps[i]->data[j].f, smpt[i]->data[j].f, 1e-3, "Sample data mismatch at index %d: %f != %f", j, smps[i]->data[j].f, smpt[i]->data[j].f); + break; + case SAMPLE_DATA_FORMAT_INT: + cr_assert_eq(smps[i]->data[j].i, smpt[i]->data[j].i); + break; + } + } + } +} + +ParameterizedTestParameters(io, lowlevel) +{ + return cr_make_param_array(char[32], formats, ARRAY_LEN(formats)); +} + +ParameterizedTest(char *fmt, io, lowlevel) +{ + int ret; + char buf[8192]; + size_t wbytes, rbytes; + + struct io_format *f; + + struct pool p = { .state = STATE_DESTROYED }; + struct sample *smps[NUM_SAMPLES]; + struct sample *smpt[NUM_SAMPLES]; + + ret = pool_init(&p, 2 * NUM_SAMPLES, SAMPLE_LEN(NUM_VALUES), &memtype_hugepage); + cr_assert_eq(ret, 0); + + generate_samples(&p, smps, smpt, NUM_SAMPLES, NUM_VALUES); + + f = io_format_lookup(fmt); + cr_assert_not_null(f, "Format '%s' does not exist", fmt); + + ret = io_format_sprint(f, buf, sizeof(buf), &wbytes, smps, NUM_SAMPLES, IO_FORMAT_ALL); + cr_assert_eq(ret, NUM_SAMPLES); + + ret = io_format_sscan(f, buf, wbytes, &rbytes, smpt, NUM_SAMPLES, NULL); + cr_assert_eq(rbytes, wbytes); + + cr_assert_eq_samples(f, smps, smpt, ret); + + sample_free(smps, NUM_SAMPLES); + sample_free(smpt, NUM_SAMPLES); + + ret = pool_destroy(&p); + cr_assert_eq(ret, 0); +} + +ParameterizedTestParameters(io, highlevel) +{ + return cr_make_param_array(char[32], formats, ARRAY_LEN(formats)); +} + +ParameterizedTest(char *fmt, io, highlevel) +{ + int ret, cnt; + + struct io io; + struct io_format *f; + + struct pool p = { .state = STATE_DESTROYED }; + struct sample *smps[NUM_SAMPLES]; + struct sample *smpt[NUM_SAMPLES]; + + ret = pool_init(&p, 2 * NUM_SAMPLES, SAMPLE_LEN(NUM_VALUES), &memtype_hugepage); + cr_assert_eq(ret, 0); + + generate_samples(&p, smps, smpt, NUM_SAMPLES, NUM_VALUES); + + /* Open a file for IO */ + char *fn, dir[64]; + strncpy(dir, "/tmp/villas.XXXXXX", sizeof(dir)); + + mkdtemp(dir); +// ret = asprintf(&fn, "file://%s/file", dir); + ret = asprintf(&fn, "%s/file", dir); + cr_assert_gt(ret, 0); + + f = io_format_lookup(fmt); + cr_assert_not_null(f, "Format '%s' does not exist", fmt); + + ret = io_init(&io, f, IO_FORMAT_ALL); + cr_assert_eq(ret, 0); + + ret = io_open(&io, fn); + cr_assert_eq(ret, 0); + + ret = io_print(&io, smps, NUM_SAMPLES); + cr_assert_eq(ret, NUM_SAMPLES); + + ret = io_flush(&io); + cr_assert_eq(ret, 0); + +#if 0 /* Show the file contents */ + char cmd[128]; + if (!strcmp(fmt, "json") || !strcmp(fmt, "villas")) + snprintf(cmd, sizeof(cmd), "cat %s", fn); + else + snprintf(cmd, sizeof(cmd), "hexdump -C %s", fn); + system(cmd); +#endif + + if (io.mode == IO_MODE_ADVIO) + adownload(io.advio.input, 0); + + io_rewind(&io); + + cnt = io_scan(&io, smpt, NUM_SAMPLES); + cr_assert_gt(cnt, 0, "Failed to read samples back"); + + cr_assert_eq_samples(f, smps, smpt, cnt); + + ret = io_close(&io); + cr_assert_eq(ret, 0); + + ret = io_destroy(&io); + cr_assert_eq(ret, 0); + + ret = unlink(fn); + cr_assert_eq(ret, 0); + + ret = rmdir(dir); + cr_assert_eq(ret, 0); + + free(fn); + + sample_free(smps, NUM_SAMPLES); + sample_free(smpt, NUM_SAMPLES); + + ret = pool_destroy(&p); + cr_assert_eq(ret, 0); +} diff --git a/tests/unit/json.c b/tests/unit/json.c new file mode 100644 index 000000000..6d9b6f9ab --- /dev/null +++ b/tests/unit/json.c @@ -0,0 +1,99 @@ +/** Unit tests for libjansson helpers + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + + +#include +#include + +#include "utils.h" +#include "config_helper.h" + +struct param { + const char *desc; + const char *json; + char *argv[32]; +}; + +ParameterizedTestParameters(json, json_load_cli) +{ + static struct param params[] = { + // Combined long option + { + .argv = { "dummy", "--option=value", NULL }, + .json = "{ \"option\" : \"value\" }" + }, + // Separated long option + { + .argv = { "dummy", "--option", "value", NULL }, + .json = "{ \"option\" : \"value\" }" + }, + // All kinds of data types + { + .argv = { "dummy", "--integer", "1", "--real", "1.1", "--bool", "true", "--null", "null", "--string", "hello world", NULL }, + .json = "{ \"integer\" : 1, \"real\" : 1.1, \"bool\" : true, \"null\" : null, \"string\" : \"hello world\" }" + }, + // Array generation + { + .argv = { "dummy", "--bool", "true", "--bool", "false", NULL }, + .json = "{ \"bool\" : [ true, false ] }" + }, + // Dots in the option name generate sub objects + { + .argv = { "dummy", "--sub.option", "value", NULL }, + .json = "{ \"sub\" : { \"option\" : \"value\" } }" + }, + // Nesting is possible + { + .argv = { "dummy", "--sub.sub.option", "value", NULL }, + .json = "{ \"sub\" : { \"sub\" : { \"option\" : \"value\" } } }" + }, + // Multiple subgroups are merged + { + .argv = { "dummy", "--sub.sub.option", "value1", "--sub.option", "value2", NULL }, + .json = "{ \"sub\" : { \"option\" : \"value2\", \"sub\" : { \"option\" : \"value1\" } } }" + } + }; + + return cr_make_param_array(struct param, params, ARRAY_LEN(params)); +} + +ParameterizedTest(struct param *p, json, json_load_cli) +{ + json_error_t err; + json_t *json, *cli; + + /* Calculate argc */ + int argc = 0; + while (p->argv[argc]) + argc++; + + json = json_loads(p->json, 0, &err); + cr_assert_not_null(json); + + cli = json_load_cli(argc, p->argv); + cr_assert_not_null(cli); + + //json_dumpf(json, stdout, JSON_INDENT(2)); putc('\n', stdout); + //json_dumpf(cli, stdout, JSON_INDENT(2)); putc('\n', stdout); + + cr_assert(json_equal(json, cli)); +} diff --git a/tests/unit/main.c b/tests/unit/main.c index adeec8a33..8d514869b 100644 --- a/tests/unit/main.c +++ b/tests/unit/main.c @@ -36,16 +36,16 @@ int main(int argc, char *argv[]) struct log log; ret = log_init(&log, V, LOG_ALL); - if (ret) { + if (ret) error("Failed to initialize logging sub-system"); - return ret; - } + + ret = log_start(&log); + if (ret) + error("Failed to start logging sub-system"); ret = memory_init(DEFAULT_NR_HUGEPAGES); - if (ret) { + if (ret) error("Failed to initialize memory sub-system"); - return ret; - } /* Run criterion tests */ tests = criterion_initialize(); diff --git a/tests/unit/sample_io.c b/tests/unit/sample_io.c deleted file mode 100644 index fee5a7b62..000000000 --- a/tests/unit/sample_io.c +++ /dev/null @@ -1,95 +0,0 @@ -/** Unit tests for the sample_io module. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include - -#include -#include - -#include "utils.h" -#include "timing.h" -#include "sample.h" -#include "sample_io.h" - -ParameterizedTestParameters(sample_io, read_write) -{ - static enum sample_io_format formats[] = { - SAMPLE_IO_FORMAT_VILLAS, - SAMPLE_IO_FORMAT_JSON, - }; - - return cr_make_param_array(enum sample_io_format, formats, ARRAY_LEN(formats)); -} - -ParameterizedTest(enum sample_io_format *fmt, sample_io, read_write) -{ - FILE *f; - int ret; - struct sample *s, *t; - - /* Prepare a sample with arbitrary data */ - s = malloc(SAMPLE_LEN(16)); - cr_assert_not_null(s); - - t = malloc(SAMPLE_LEN(16)); - cr_assert_not_null(s); - - t->capacity = - s->capacity = 16; - s->length = 12; - s->sequence = 235; - s->format = 0; - s->ts.origin = time_now(); - s->ts.received = (struct timespec) { s->ts.origin.tv_sec - 1, s->ts.origin.tv_nsec }; - - for (int i = 0; i < s->length; i++) - s->data[i].f = i * 1.2; - - /* Open a file for IO */ - f = tmpfile(); - cr_assert_not_null(f); - - ret = sample_io_fprint(f, s, *fmt, SAMPLE_IO_ALL); - cr_assert_eq(ret, 0); - - rewind(f); - - ret = sample_io_fscan(f, t, *fmt, NULL); - cr_assert_eq(ret, 0); - - cr_assert_eq(s->length, t->length); - cr_assert_eq(s->sequence, t->sequence); - cr_assert_eq(s->format, t->format); - - cr_assert_eq(s->ts.origin.tv_sec, t->ts.origin.tv_sec); - cr_assert_eq(s->ts.origin.tv_nsec, t->ts.origin.tv_nsec); - cr_assert_eq(s->ts.received.tv_sec, t->ts.received.tv_sec); - cr_assert_eq(s->ts.received.tv_nsec, t->ts.received.tv_nsec); - - for (int i = 0; i < MIN(s->length, t->length); i++) - cr_assert_float_eq(s->data[i].f, t->data[i].f, 1e-6); - - fclose(f); - - free(s); - free(t); -} \ No newline at end of file diff --git a/tests/unit/task.c b/tests/unit/task.c new file mode 100644 index 000000000..5bc9943f1 --- /dev/null +++ b/tests/unit/task.c @@ -0,0 +1,88 @@ +/** Unit tests for periodic tasks + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include + +#include "task.h" +#include "timing.h" + +Test(task, rate, .timeout = 10) +{ + int ret; + double rate = 5, waited; + struct timespec start, end; + struct task task; + + ret = task_init(&task, rate, CLOCK_MONOTONIC); + cr_assert_eq(ret, 0); + + int i; + for (i = 0; i < 10; i++) { + clock_gettime(CLOCK_MONOTONIC, &start); + + task_wait_until_next_period(&task); + + clock_gettime(CLOCK_MONOTONIC, &end); + + waited = time_delta(&start, &end); + + if (fabs(waited - 1.0 / rate) > 10e-3) + break; + } + + if (i < 10) + cr_assert_float_eq(waited, 1.0 / rate, 1e-2, "We slept for %f instead of %f secs in round %d", waited, 1.0 / rate, i); + + ret = task_destroy(&task); + cr_assert_eq(ret, 0); +} + +Test(task, wait_until, .timeout = 5) +{ + int ret; + struct task task; + struct timespec start, end, diff, future; + + ret = task_init(&task, 1, CLOCK_REALTIME); + cr_assert_eq(ret, 0); + + double waitfor = 3.423456789; + + start = time_now(); + diff = time_from_double(waitfor); + future = time_add(&start, &diff); + + ret = task_wait_until(&task, &future); + + end = time_now(); + + cr_assert_eq(ret, 0); + + double waited = time_delta(&start, &end); + + cr_assert_float_eq(waited, waitfor, 1e-2, "We slept for %f instead of %f secs", waited, waitfor); + + ret = task_destroy(&task); + cr_assert_eq(ret, 0); +} \ No newline at end of file diff --git a/tests/unit/timing.c b/tests/unit/timing.c index f6825ce54..69adfdc6e 100644 --- a/tests/unit/timing.c +++ b/tests/unit/timing.c @@ -89,58 +89,4 @@ Test(timing, time_to_from_double) double dbl = time_to_double(&ts); cr_assert_float_eq(dbl, ref, 1e-9); -} - -Test(timing, timerfd_create_rate, .timeout = 20) -{ - struct timespec start, end; - - double rate = 5, waited; - - int tfd = timerfd_create_rate(rate); - - cr_assert(tfd > 0); - - timerfd_wait(tfd); - - int i; - for (i = 0; i < 10; i++) { - start = time_now(); - - timerfd_wait(tfd); - - end = time_now(); - waited = time_delta(&start, &end); - - if (fabs(waited - 1.0 / rate) > 10e-3) - break; - } - - if (i < 10) - cr_assert_float_eq(waited, 1.0 / rate, 1e-2, "We slept for %f instead of %f secs in round %d", waited, 1.0 / rate, i); - - close(tfd); -} - -Test(timing, timerfd_wait_until, .timeout = 10) -{ - int tfd = timerfd_create(CLOCK_REALTIME, 0); - - cr_assert(tfd > 0); - - double waitfor = 2.423456789; - - struct timespec start = time_now(); - struct timespec diff = time_from_double(waitfor); - struct timespec future = time_add(&start, &diff); - - timerfd_wait_until(tfd, &future); - - struct timespec end = time_now(); - - double waited = time_delta(&start, &end); - - cr_assert_float_eq(waited, waitfor, 1e-2, "We slept for %f instead of %f secs", waited, waitfor); - - close(tfd); } \ No newline at end of file diff --git a/thirdparty/Makefile.inc b/thirdparty/Makefile.inc index f3cf47582..825f8c1c3 100644 --- a/thirdparty/Makefile.inc +++ b/thirdparty/Makefile.inc @@ -69,6 +69,7 @@ libconfig: | libconfig-fix libconfig-fix: rm -f $(SRCDIR)/thirdparty/libconfig/lib/scanner.[hc] +jansson: CMAKE_OPTS += -DJANSSON_BUILD_DOCS=OFF libwebsockets: CMAKE_OPTS += -DLWS_IPV6=1 -DLWS_WITH_STATIC=0 -DLWS_WITHOUT_TESTAPPS=1 -DLWS_WITH_HTTP2=1 libzmq: CONFIGURE_OPTS += --with-libsodium --with-pgm --enable-drafts diff --git a/thirdparty/libwebsockets b/thirdparty/libwebsockets index 2b9fff73f..c5d29ba5e 160000 --- a/thirdparty/libwebsockets +++ b/thirdparty/libwebsockets @@ -1 +1 @@ -Subproject commit 2b9fff73f92dfe1ea5fde9a11e75f3ef2a981f90 +Subproject commit c5d29ba5e9cbf897a0c921ccaeee4be1ddad0f08 diff --git a/tools/Makefile.inc b/tools/Makefile.inc index cc25934eb..11f8c6a76 100644 --- a/tools/Makefile.inc +++ b/tools/Makefile.inc @@ -20,9 +20,11 @@ # along with this program. If not, see . ################################################################################### -ifeq ($(WITH_JSON),1) +ifeq ($(WITH_CONFIG),1) +ifeq ($(shell $(PKGCONFIG) libconfig; echo $$?),0) TOOLS += $(BUILDDIR)/conf2json endif +endif TOOLS_CFLAGS = $(CFLAGS) TOOLS_LDLIBS = -lconfig -ljansson -lvillas diff --git a/tools/integration-tests-helper.sh b/tools/integration-tests-helper.sh new file mode 100755 index 000000000..424d7088a --- /dev/null +++ b/tools/integration-tests-helper.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Some helper functions for our integration test suite +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################## + +function villas_format_supports_vectorize() { + local FORMAT=$1 + + case $FORMAT in + raw-*) ;& + gtnet) ;& + gtnet-fake) return 1 ;; + esac + + return 0 +} + +function villas_format_supports_header() { + local FORMAT=$1 + + case $FORMAT in + raw-*) ;& + gtnet) return 1 ;; + esac + + return 0 +} + +function colorize() { + echo -e "\x1b[0;$((31 + $RANDOM % 7))m$1\x1b[0m" +} \ No newline at end of file diff --git a/tools/matlab/.gitkeep b/tools/matlab/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tools/python/.gitignore b/tools/python/.gitignore deleted file mode 100644 index 0d20b6487..000000000 --- a/tools/python/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc diff --git a/tools/python/README.md b/tools/python/README.md deleted file mode 100644 index 2e703ac5b..000000000 --- a/tools/python/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Python tools - -These python scripts are intended to manipulate measurements recorded by VILLASnode's `file` node-type. - -## Exampples - -##### Merge two files and filter the output based on timestamps - -``` -./file-merge.py testfile.dat testfile2.dat | ./file-filter.py 3 5 > output.dat -``` diff --git a/tools/python/file-filter.py b/tools/python/file-filter.py deleted file mode 100755 index de9e145d6..000000000 --- a/tools/python/file-filter.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python - -import sys - -import villas -from villas import * - -def main(): - if len(sys.argv) != 3: - print "Usage: %s from to" % (sys.argv[0]) - sys.exit(-1) - - start = villas.Timestamp.parse(sys.argv[1]) - end = villas.Timestamp.parse(sys.argv[2]) - - for line in sys.stdin: - msg = villas.Message.parse(line) - - if start <= msg.ts <= end: - print msg - -if __name__ == "__main__": - main() diff --git a/tools/python/file-merge.py b/tools/python/file-merge.py deleted file mode 100755 index 3b67b12b0..000000000 --- a/tools/python/file-merge.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -import sys - -import villas -from villas import * - -def main(): - files = sys.argv[1:] - - all = [ ] - last = { } - - for file in files: - handle = sys.stdin if file == '-' else open(file, "r") - - msgs = [ ] - for line in handle.xreadlines(): - msgs.append(villas.Message.parse(line, file)) - - all += msgs - last[file] = villas.Message(villas.Timestamp(), [0] * len(msgs[0].values), file) - - all.sort() - for msg in all: - last[msg.source] = msg - - values = [ ] - for file in files: - values += last[file].values - - print villas.Message(msg.ts, values, "") - -if __name__ == "__main__": - main() diff --git a/tools/python/villas/__init__.py b/tools/python/villas/__init__.py deleted file mode 100644 index bdf91102a..000000000 --- a/tools/python/villas/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from msg import Message -from ts import Timestamp - -__all__ = [ "msg", "ts" ] diff --git a/tools/python/villas/msg.py b/tools/python/villas/msg.py deleted file mode 100644 index 4d34e017a..000000000 --- a/tools/python/villas/msg.py +++ /dev/null @@ -1,24 +0,0 @@ -from . import ts - -class Message: - """Parsing a VILLASnode sample from a file (not a UDP package!!)""" - - def __init__(self, ts, values, source = None): - self.source = source - self.ts = ts - self.values = values - - @classmethod - def parse(self, line, source = None): - csv = line.split() - - t = ts.Timestamp.parse(csv[0]) - v = map(float, csv[1:]) - - return Message(t, v, source) - - def __str__(self): - return '%s %s' % (self.ts, " ".join(map(str, self.values))) - - def __cmp__(self, other): - return cmp(self.ts, other.ts) diff --git a/tools/python/villas/ocb.py b/tools/python/villas/ocb.py deleted file mode 100644 index c5983e83e..000000000 --- a/tools/python/villas/ocb.py +++ /dev/null @@ -1,38 +0,0 @@ -import re - -class OcbMapping: - def __init__(self, entityId, entityType, attributeName, attributeType): - self.entityId = entityId - self.entityType = entityType - self.attributeName = attributeName - self.attributeType = attributeType - - @classmethod - def parse(self, lines): - m = re.match('([a-z]+)\(([a-z]+)\)\.([a-z]+).\(([a-z]+)\)', re.I) - return OcbMapping(m.group(1), m.group(2), m.group(3), m.group(4)) - - @classMethod - def parseFile(self, file): - -class OcbEntity: - -class OcbAttribute: - -class OcbResponse: - def __init__(self, mapping): - self.mapping = mapping - - def getJson(self, msg): - json = { "contextResponses" : [ ] } - - for (value in msg.values): - json["contextResponses"].append({ - "statusCode" : { "code" : 200, "reasonPhrase" : "OK" } - "contextElement" : { - "attributes" = [ ], - "id" : "", - "isPattern" : false, - "type" - } - }) diff --git a/tools/python/villas/ts.py b/tools/python/villas/ts.py deleted file mode 100644 index 96df86812..000000000 --- a/tools/python/villas/ts.py +++ /dev/null @@ -1,46 +0,0 @@ -import re - -class Timestamp: - """Parsing the VILLASnode human-readable timestamp format""" - - def __init__(self, seconds = 0, nanoseconds = None, offset = None, sequence = None): - self.seconds = seconds - self.nanoseconds = nanoseconds - self.offset = offset - self.sequence = sequence - - @classmethod - def parse(self, ts): - m = re.match('(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?(?:e[+-]?\d+)?)?(?:\((\d+)\))?', ts) - - seconds = int(m.group(1)); # Mandatory - nanoseconds = int(m.group(2)) if m.group(2) else None - offset = float(m.group(3)) if m.group(3) else None - sequence = int(m.group(4)) if m.group(4) else None - - return Timestamp(seconds, nanoseconds, offset, sequence) - - def __str__(self): - str = "%u" % (self.seconds) - - if self.nanoseconds is not None: - str += ".%09u" % self.nanoseconds - if self.offset is not None: - str += "+%u" % self.offset - if self.sequence is not None: - str += "(%u)" % self.sequence - - return str - - def __float__(self): - sum = float(self.seconds) - - if self.nanoseconds is not None: - sum += self.nanoseconds * 1e-9 - if self.offset is not None: - sum += self.offset - - return sum - - def __cmp__(self, other): - return cmp(float(self), float(other)) diff --git a/web/Makefile.inc b/web/Makefile.inc index 04c416893..dbfd3e647 100644 --- a/web/Makefile.inc +++ b/web/Makefile.inc @@ -21,7 +21,8 @@ ################################################################################### install-web: - cp -R web $(DESTDIR)$(PREFIX)/share/villas/node + mkdir -p $(DESTDIR)$(PREFIX)/share/villas/node/web/ + cp -R $(SRCDIR)/web/socket/* $(DESTDIR)$(PREFIX)/share/villas/node/web/ clean-web: diff --git a/web/socket/app.js b/web/socket/app.js index 0f2c5388d..d29cbbb5e 100644 --- a/web/socket/app.js +++ b/web/socket/app.js @@ -273,8 +273,6 @@ function wsConnect(node) conn.onmessage = function(e) { var msgs = Msg.fromArrayBufferVector(e.data); - console.log('Received ' + msgs.length + ' messages with ' + msgs[0].data.length + ' values from id ' + msgs[0].id + ' with timestamp ' + msgs[0].timestamp); - for (var j = 0; j < plotData.length; j++) { // remove old while (plotData[j].length > 0 && plotData[j][0][0] < (Date.now() - xPast)) @@ -283,6 +281,8 @@ function wsConnect(node) for (var j = 0; j < msgs.length; j++) { var msg = msgs[j]; + + console.log('Received message with ' + msg.data.length + ' values from id ' + msg.id + ' with timestamp ' + new Date(msg.timestamp).toString()); if (msg.id !== currentNode.id) continue; diff --git a/web/socket/msg.js b/web/socket/msg.js index 8de38b4df..a15903167 100644 --- a/web/socket/msg.js +++ b/web/socket/msg.js @@ -47,13 +47,13 @@ function Msg(c) } /* Some constants for the binary protocol */ -Msg.prototype.VERSION = 1; +Msg.prototype.VERSION = 2; -Msg.prototype.TYPE_DATA = 0; /**< Message contains float values */ +Msg.prototype.TYPE_DATA = 0; /**< Message contains float values */ /* Some offsets in the binary message */ -Msg.prototype.OFFSET_TYPE = 2; -Msg.prototype.OFFSET_VERSION = 4; +Msg.prototype.OFFSET_TYPE = 2; +Msg.prototype.OFFSET_VERSION = 4; Msg.bytes = function(len) {