diff --git a/CMakeLists.txt b/CMakeLists.txt index ff1eb8e53..0a170f8cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) set(CMAKE_THREAD_PREFER_PTHREAD ON) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake;${PROJECT_SOURCE_DIR}/common/cmake") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) set(TOPLEVEL_PROJECT ON) diff --git a/include/villas/format.hpp b/include/villas/format.hpp new file mode 100644 index 000000000..13a2f7db0 --- /dev/null +++ b/include/villas/format.hpp @@ -0,0 +1,170 @@ +/** Read / write sample data in different formats. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include +#include + +namespace villas { +namespace node { + +class Format { + +protected: + + int flags; /**< A set of flags which is automatically used. */ + + int real_precision; /**< Number of digits used for floatint point numbers */ + + bool destroy_signals; + + struct { + char *buffer; + size_t buflen; + } in, out; + + struct vlist *signals; /**< Signal meta data for parsed samples by Format::scan() */ + +public: + Format(int fl); + + virtual bool isBinaryPayload() + { return false; } + + struct vlist * getSignals() const + { return signals; } + + int getFlags() const + { return flags; } + + virtual ~Format(); + + void start(struct vlist *sigs, int fl = (int) SampleFlags::HAS_ALL); + void start(const std::string &dtypes, int fl = (int) SampleFlags::HAS_ALL); + + virtual void start() + { } + + virtual void parse(json_t *json); + + virtual int print(FILE *f, const struct sample * const smps[], unsigned cnt); + virtual int scan(FILE *f, struct sample * const smps[], unsigned cnt); + + /** Print \p cnt samples from \p smps into buffer \p buf of length \p len. + * + * @param buf[out] The buffer which should be filled with serialized data. + * @param len[in] The length of the buffer \p buf. + * @param rbytes[out] The number of bytes which have been written to \p buf. Ignored if nullptr. + * @param smps[in] The array of pointers to samples. + * @param cnt[in] The number of pointers in the array \p smps. + * + * @retval >=0 The number of samples from \p smps which have been written into \p buf. + * @retval <0 Something went wrong. + */ + virtual int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) = 0; + + /** Parse samples from the buffer \p buf with a length of \p len bytes. + * + * @param buf[in] The buffer of data which should be parsed / de-serialized. + * @param len[in] The length of the buffer \p buf. + * @param rbytes[out] The number of bytes which have been read from \p buf. + * @param smps[out] The array of pointers to samples. + * @param cnt[in] The number of pointers in the array \p smps. + * + * @retval >=0 The number of samples which have been parsed from \p buf and written into \p smps. + * @retval <0 Something went wrong. + */ + virtual int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) = 0; + + /* Wrappers for sending a (un)parsing single samples */ + + int print(FILE *f, const struct sample *smp) + { + return print(f, &smp, 1); + } + + int scan(FILE *f, struct sample *smp) + { + return scan(f, &smp, 1); + } + + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample *smp) + { + return sprint(buf, len, wbytes, &smp, 1); + } + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample *smp) + { + return sscan(buf, len, rbytes, &smp, 1); + } +}; + +class BinaryFormat : public Format { + +public: + using Format::Format; + + virtual bool isBinaryPayload() + { return true; } +}; + +class FormatFactory : public plugin::Plugin { + +public: + using plugin::Plugin::Plugin; + + virtual Format * make() = 0; + + static + Format * make(json_t *json); + + static + Format * make(const std::string &format); +}; + +template +class FormatPlugin : public FormatFactory { + +public: + using FormatFactory::FormatFactory; + + virtual Format * make() + { + return new T(flags); + } + + /// Get plugin name + virtual std::string + getName() const + { return name; } + + /// Get plugin description + virtual std::string + getDescription() const + { return desc; } +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/format_type.h b/include/villas/format_type.h deleted file mode 100644 index f8ba6631d..000000000 --- a/include/villas/format_type.h +++ /dev/null @@ -1,104 +0,0 @@ -/** Read / write sample data in different formats. - * - * @file - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include - -/* Forward declarations */ -struct sample; -struct io; - -struct format_type { - int (*init)(struct io *io); - int (*destroy)(struct io *io); - - /** @{ - * High-level interface - */ - - /** Open an IO stream. - * - * @see fopen() - */ - int (*open)(struct io *io, const char *uri); - - /** Close an IO stream. - * - * @see fclose() - */ - int (*close)(struct io *io); - - /** Check if end-of-file was reached. - * - * @see feof() - */ - int (*eof)(struct io *io); - - /** Rewind an IO stream. - * - * @see rewind() - */ - void (*rewind)(struct io *io); - - /** Get a file descriptor which can be used with select / poll */ - int (*fd)(struct io *io); - - /** Flush buffered data to disk. - * - * @see fflush() - */ - int (*flush)(struct io *io); - - int (*print)(struct io *io, struct sample *smps[], unsigned cnt); - int (*scan)( struct io *io, struct sample *smps[], unsigned cnt); - - /** Print a header. */ - void (*header)(struct io *io, const struct sample *smp); - - /** Print a footer. */ - void (*footer)(struct io *io); - - /** @} */ - - /** @{ - * Low-level interface - */ - - /** @see format_type_sprint */ - int (*sprint)(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); - - /** @see format_type_sscan */ - int (*sscan)(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); - - /** @} */ - - size_t size; /**< Number of bytes to allocate for io::_vd */ - int flags; /**< A set of flags which is automatically used. */ - char delimiter; /**< Newline delimiter. */ - char separator; /**< Column separator (used by csv and villas.human formats only) */ -}; - -struct format_type * format_type_lookup(const char *name); - -const char * format_type_name(struct format_type *vt); diff --git a/include/villas/formats/column.hpp b/include/villas/formats/column.hpp new file mode 100644 index 000000000..b179379cc --- /dev/null +++ b/include/villas/formats/column.hpp @@ -0,0 +1,78 @@ +/** Comma-separated values. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +/* Forward declarations. */ +struct sample; + +namespace villas { +namespace node { + +class ColumnLineFormat : public LineFormat { + +protected: + virtual size_t sprintLine(char *buf, size_t len, const struct sample *smp); + virtual size_t sscanLine(const char *buf, size_t len, struct sample *smp); + + char separator; /**< Column separator */ + +public: + ColumnLineFormat(int fl, char delim, char sep) : + LineFormat(fl, delim), + separator(sep) + { } + + virtual void header(FILE *f, const struct vlist *sigs); + + virtual void parse(json_t *json); +}; + +template +class ColumnLineFormatPlugin : public FormatFactory { + +public: + using FormatFactory::FormatFactory; + + virtual Format * make() + { + return new ColumnLineFormat(flags, delimiter, separator); + } + + /// Get plugin name + virtual std::string + getName() const + { return name; } + + /// Get plugin description + virtual std::string + getDescription() const + { return desc; } +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/iotagent_ul.h b/include/villas/formats/iotagent_ul.hpp similarity index 71% rename from include/villas/formats/iotagent_ul.h rename to include/villas/formats/iotagent_ul.hpp index 3570cec4f..ee8373861 100644 --- a/include/villas/formats/iotagent_ul.h +++ b/include/villas/formats/iotagent_ul.hpp @@ -22,10 +22,23 @@ #pragma once -#include +#include /* Forward declarations */ struct sample; -struct io; -int iotagent_ul_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); +namespace villas { +namespace node { + +class IotAgentUltraLightFormat : public Format { + +protected: + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + +public: + using Format::Format; +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/json.h b/include/villas/formats/json.h deleted file mode 100644 index 991bb8566..000000000 --- a/include/villas/formats/json.h +++ /dev/null @@ -1,35 +0,0 @@ -/** JSON serializtion sample data. - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include - -/* Forward declarations */ -struct sample; - -int json_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); -int json_sscan(struct io *io, const char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); - -int json_print(struct io *io, struct sample *smps[], unsigned cnt); -int json_scan(struct io *io, struct sample *smps[], unsigned cnt); - diff --git a/include/villas/formats/json.hpp b/include/villas/formats/json.hpp new file mode 100644 index 000000000..8c073763c --- /dev/null +++ b/include/villas/formats/json.hpp @@ -0,0 +1,66 @@ +/** JSON serializtion sample data. + * + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +/* Forward declarations */ +struct sample; + +namespace villas { +namespace node { + +class JsonFormat : public Format { + +protected: + static enum SignalType detect(const json_t *val); + + json_t * packTimestamps(const struct sample *smp); + int unpackTimestamps(json_t *json_ts, struct sample *smp); + + virtual int packSample(json_t **j, const struct sample *smp); + virtual int packSamples(json_t **j, const struct sample * const smps[], unsigned cnt); + virtual int unpackSample(json_t *json_smp, struct sample *smp); + virtual int unpackSamples(json_t *json_smps, struct sample * const smps[], unsigned cnt); + + int dump_flags; + +public: + JsonFormat(int fl) : + Format(fl), + dump_flags(0) + { } + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); + + int print(FILE *f, const struct sample * const smps[], unsigned cnt); + int scan(FILE *f, struct sample * const smps[], unsigned cnt); + + virtual void parse(json_t *json); +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/json_kafka.h b/include/villas/formats/json_kafka.hpp similarity index 67% rename from include/villas/formats/json_kafka.h rename to include/villas/formats/json_kafka.hpp index 4f5bf9502..e3caf9010 100644 --- a/include/villas/formats/json_kafka.h +++ b/include/villas/formats/json_kafka.hpp @@ -22,14 +22,27 @@ #pragma once -#include +#include +#include -/* Forward declarations */ -struct sample; -struct io; +namespace villas { +namespace node { -int json_kafka_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); -int json_kafka_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); +class JsonKafkaFormat : public JsonFormat { -int json_kafka_print(struct io *io, struct sample *smps[], unsigned cnt); -int json_kafka_scan(struct io *io, struct sample *smps[], unsigned cnt); +protected: + int packSample(json_t **j, const struct sample *smp); + int unpackSample(json_t *json_smp, struct sample *smp); + + const char * villasToKafkaType(enum SignalType vt); + + json_t *json_schema; + +public: + JsonKafkaFormat(int fl); + + virtual void parse(json_t *json); +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/json_reserve.h b/include/villas/formats/json_reserve.h deleted file mode 100644 index 319128b4f..000000000 --- a/include/villas/formats/json_reserve.h +++ /dev/null @@ -1,35 +0,0 @@ -/** JSON serializtion for RESERVE project. - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include - -/* Forward declarations */ -struct sample; -struct io; - -int json_reserve_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); -int json_reserve_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); - -int json_reserve_print(struct io *io, struct sample *smps[], unsigned cnt); -int json_reserve_scan(struct io *io, struct sample *smps[], unsigned cnt); diff --git a/include/villas/formats/villas_human.h b/include/villas/formats/json_reserve.hpp similarity index 71% rename from include/villas/formats/villas_human.h rename to include/villas/formats/json_reserve.hpp index b62d5e251..0fbb06ea1 100644 --- a/include/villas/formats/villas_human.h +++ b/include/villas/formats/json_reserve.hpp @@ -1,6 +1,5 @@ -/** The VILLASframework sample format +/** JSON serializtion for RESERVE project. * - * @file * @author Steffen Vogel * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) @@ -23,13 +22,20 @@ #pragma once -#include +#include -/* Forward declarations */ -struct io; -struct sample; +namespace villas { +namespace node { -void villas_human_header(struct io *io, const struct sample *smp); +class JsonReserveFormat : public JsonFormat { -int villas_human_print(struct io *io, struct sample *smps[], unsigned cnt); -int villas_human_scan(struct io *io, struct sample *smps[], unsigned cnt); +protected: + int packSample(json_t **j, const struct sample *smp); + int unpackSample(json_t *json_smp, struct sample *smp); + +public: + using JsonFormat::JsonFormat; +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/line.hpp b/include/villas/formats/line.hpp new file mode 100644 index 000000000..362779d56 --- /dev/null +++ b/include/villas/formats/line.hpp @@ -0,0 +1,95 @@ +/** Line-based formats + * + * @file + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +namespace villas { +namespace node { + +class LineFormat : public Format { + +protected: + virtual size_t sprintLine(char *buf, size_t len, const struct sample *smp) = 0; + virtual size_t sscanLine(const char *buf, size_t len, struct sample *smp) = 0; + + char delimiter; /**< Newline delimiter. */ + char comment; /**< Prefix for comment lines. */ + + bool skip_first_line; /**< While reading, the first line is skipped (header) */ + bool print_header; /**< Before any data, a header line is printed */ + + bool first_line_skipped; + bool header_printed; + +public: + LineFormat(int fl, char delim = '\n', char com = '#') : + Format(fl), + delimiter(delim), + comment(com), + skip_first_line(false), + print_header(true), + first_line_skipped(false), + header_printed(false) + { } + + /** Print a header. */ + virtual void header(FILE *f, const struct vlist *sigs) + { + header_printed = true; + } + + virtual int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); + virtual int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + + virtual int scan(FILE *f, struct sample * const smps[], unsigned cnt); + virtual int print(FILE *f, const struct sample * const smps[], unsigned cnt); + + virtual void parse(json_t *json); +}; + +template +class LineFormatPlugin : public FormatFactory { + +public: + using FormatFactory::FormatFactory; + + virtual Format * make() + { + return new T(flags, delimiter); + } + + /// Get plugin name + virtual std::string + getName() const + { return name; } + + /// Get plugin description + virtual std::string + getDescription() const + { return desc; } +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/msg.h b/include/villas/formats/msg.hpp similarity index 88% rename from include/villas/formats/msg.h rename to include/villas/formats/msg.hpp index 2b75f870c..f7ded31c7 100644 --- a/include/villas/formats/msg.h +++ b/include/villas/formats/msg.hpp @@ -47,10 +47,10 @@ void msg_hdr_ntoh(struct msg *m); * @retval 0 The message header is valid. * @retval <0 The message header is invalid. */ -int msg_verify(struct msg *m); +int msg_verify(const struct msg *m); /** Copy fields from \p msg into \p smp. */ -int msg_to_sample(struct msg *msg, struct sample *smp, struct vlist *signals); +int msg_to_sample(const struct msg *msg, struct sample *smp, const struct vlist *sigs); /** Copy fields form \p smp into \p msg. */ -int msg_from_sample(struct msg *msg, struct sample *smp, struct vlist *signals); +int msg_from_sample(struct msg *msg, const struct sample *smp, const struct vlist *sigs); diff --git a/include/villas/formats/msg_format.h b/include/villas/formats/msg_format.hpp similarity index 100% rename from include/villas/formats/msg_format.h rename to include/villas/formats/msg_format.hpp diff --git a/include/villas/formats/villas_binary.h b/include/villas/formats/protobuf.hpp similarity index 64% rename from include/villas/formats/villas_binary.h rename to include/villas/formats/protobuf.hpp index d8297a9d5..02868a70a 100644 --- a/include/villas/formats/villas_binary.h +++ b/include/villas/formats/protobuf.hpp @@ -1,4 +1,4 @@ -/** Message related functions +/** Protobuf IO format * * @file * @author Steffen Vogel @@ -25,17 +25,28 @@ #include +#include + +/* Generated message descriptors by protoc */ +#include + /* Forward declarations. */ struct sample; -struct msg; -struct io; -enum villas_binary_flags { - VILLAS_BINARY_WEB = (1 << 16) /**< Use webmsg format (everying little endian) */ +namespace villas { +namespace node { + +class ProtobufFormat : public BinaryFormat { + +protected: + enum SignalType detect(const Villas__Node__Value *val); + +public: + using BinaryFormat::BinaryFormat; + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); }; -/** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ -int villas_binary_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); - -/** Read struct sample's from buffer \p buf into samples \p smps. */ -int villas_binary_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/raw.h b/include/villas/formats/raw.hpp similarity index 56% rename from include/villas/formats/raw.h rename to include/villas/formats/raw.hpp index 552a81712..2b5274ce1 100644 --- a/include/villas/formats/raw.h +++ b/include/villas/formats/raw.hpp @@ -23,8 +23,11 @@ #pragma once +#include #include +#include + /* float128 is currently not yet supported as htole128() functions a missing */ #if 0 && defined(__GNUC__) && defined(__linux__) #define HAS_128BIT @@ -33,23 +36,47 @@ /* Forward declarations */ struct sample; -enum raw_flags { - /** Treat the first three values as: sequenceno, seconds, nanoseconds */ - RAW_FAKE_HEADER = (1 << 16), - RAW_BIG_ENDIAN = (1 << 17), /**< Encode data in big-endian byte order */ +namespace villas { +namespace node { - RAW_BITS_8 = (3 << 24), /**< Pack each value as a byte. */ - RAW_BITS_16 = (4 << 24), /**< Pack each value as a word. */ - RAW_BITS_32 = (5 << 24), /**< Pack each value as a double word. */ - RAW_BITS_64 = (6 << 24), /**< Pack each value as a quad word. */ -#ifdef HAS_128BIT - RAW_128 = (7 << 24) /**< Pack each value as a double quad word. */ -#endif +class RawFormat : public BinaryFormat { + +public: + enum Endianess { + BIG, + LITTLE + }; + +protected: + + enum Endianess endianess; + int bits; + bool fake; + +public: + RawFormat(int fl, int b = 32, enum Endianess e = Endianess::LITTLE) : + BinaryFormat(fl), + endianess(e), + bits(b), + fake(false) + { + if (fake) + flags |= (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_TS_ORIGIN; + } + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); + + virtual void parse(json_t *json); }; -/** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ -int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); +class GtnetRawFormat : public RawFormat { -/** Read struct sample's from buffer \p buf into samples \p smps. */ -int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); +public: + GtnetRawFormat(int fl) : + RawFormat(fl, 32, Endianess::BIG) + { } +}; +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/csv.h b/include/villas/formats/value.hpp similarity index 63% rename from include/villas/formats/csv.h rename to include/villas/formats/value.hpp index 428a37573..d27f34e8e 100644 --- a/include/villas/formats/csv.h +++ b/include/villas/formats/value.hpp @@ -1,8 +1,8 @@ -/** Comma-separated values. +/** The VILLASframework sample format * * @file * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @copyright 2014-2021, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * * VILLASnode @@ -23,13 +23,21 @@ #pragma once -#include +#include -/* Forward declarations. */ -struct io; -struct sample; +#include -void csv_header(struct io *io, const struct sample *smp); +namespace villas { +namespace node { -int csv_sprint(struct io *io, char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); -int csv_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); +class ValueFormat : public Format { + +public: + using Format::Format; + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/villas_binary.hpp b/include/villas/formats/villas_binary.hpp new file mode 100644 index 000000000..ff0072cb5 --- /dev/null +++ b/include/villas/formats/villas_binary.hpp @@ -0,0 +1,90 @@ +/** Message related functions + * + * @file + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +/* Forward declarations. */ +struct sample; +struct msg; + +namespace villas { +namespace node { + +class VillasBinaryFormat : public BinaryFormat { + +public: + bool web; + + VillasBinaryFormat(int fl, bool w) : + BinaryFormat(fl), + web(w) + { } + + int sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt); + int sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt); +}; + + +template +class VillasBinaryFormatPlugin : public FormatFactory { + +public: + using FormatFactory::FormatFactory; + + virtual Format * make() + { + return new VillasBinaryFormat((int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA, web); + } + + /// Get plugin name + virtual std::string + getName() const + { + std::stringstream ss; + + ss << "villas." << (web ? "web" : "binary"); + + return ss.str(); + } + + /// Get plugin description + virtual std::string + getDescription() const + { + std::stringstream ss; + + ss << "VILLAS binary network format"; + + if (web) + ss << " for WebSockets"; + + return ss.str(); + } +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/formats/protobuf.h b/include/villas/formats/villas_human.hpp similarity index 67% rename from include/villas/formats/protobuf.h rename to include/villas/formats/villas_human.hpp index 28cc5926a..35f40042c 100644 --- a/include/villas/formats/protobuf.h +++ b/include/villas/formats/villas_human.hpp @@ -1,4 +1,4 @@ -/** Protobuf IO format +/** The VILLASframework sample format * * @file * @author Steffen Vogel @@ -23,14 +23,24 @@ #pragma once -#include +#include -/* Forward declarations */ -struct sample; +#include -/** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ -int protobuf_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); +namespace villas { +namespace node { -/** Read struct sample's from buffer \p buf into samples \p smps. */ -int protobuf_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); +class VILLASHumanFormat : public LineFormat { +protected: + size_t sprintLine(char *buf, size_t len, const struct sample *smp); + size_t sscanLine(const char *buf, size_t len, struct sample *smp); + +public: + using LineFormat::LineFormat; + + void header(FILE *f, const struct vlist *sigs); +}; + +} /* namespace node */ +} /* namespace villas */ diff --git a/include/villas/hook.hpp b/include/villas/hook.hpp index 3b2a4c1a4..355c1cf21 100644 --- a/include/villas/hook.hpp +++ b/include/villas/hook.hpp @@ -120,7 +120,7 @@ public: } /** Called whenever a sample is processed. */ - virtual Reason process(sample *smp) + virtual Reason process(struct sample *smp) { return Reason::OK; }; @@ -135,7 +135,7 @@ public: return flags; } - virtual struct vlist *getSignals() + virtual struct vlist * getSignals() { return &signals; } @@ -179,7 +179,7 @@ class HookFactory : public plugin::Plugin { public: using plugin::Plugin::Plugin; - virtual Hook *make(struct vpath *p, struct vnode *n) = 0; + virtual Hook * make(struct vpath *p, struct vnode *n) = 0; virtual int getFlags() const = 0; virtual int getPriority() const = 0; @@ -191,7 +191,7 @@ class HookPlugin : public HookFactory { public: using HookFactory::HookFactory; - virtual Hook *make(struct vpath *p, struct vnode *n) + virtual Hook * make(struct vpath *p, struct vnode *n) { return new T(p, n, getFlags(), getPriority()); } diff --git a/include/villas/hook_list.hpp b/include/villas/hook_list.hpp index 0cc2756cb..89a718238 100644 --- a/include/villas/hook_list.hpp +++ b/include/villas/hook_list.hpp @@ -65,7 +65,7 @@ int hook_list_prepare_signals(struct vlist *hs, struct vlist *signals); int hook_list_add(struct vlist *hs, int mask, struct vpath *p, struct vnode *n); -int hook_list_process(struct vlist *hs, struct sample *smps[], unsigned cnt); +int hook_list_process(struct vlist *hs, struct sample * smps[], unsigned cnt); void hook_list_periodic(struct vlist *hs); diff --git a/include/villas/io.h b/include/villas/io.h deleted file mode 100644 index 9bf9530bc..000000000 --- a/include/villas/io.h +++ /dev/null @@ -1,141 +0,0 @@ -/** Read / write sample data in different formats. - * - * @file - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include -#include -#include - -/* Forward declarations */ -struct sample; -struct format_type; - -enum class IOFlags { - /* Bits 0-7 are reserved for for flags defined by enum sample_flags */ - FLUSH = (1 << 8), /**< Flush the output stream after each chunk of samples. */ - NONBLOCK = (1 << 9), /**< Dont block io_read() while waiting for new samples. */ - NEWLINES = (1 << 10), /**< The samples of this format are newline delimited. */ - DESTROY_SIGNALS = (1 << 11), /**< Signal descriptors are managed by this IO instance. Destroy them in io_destoy() */ - HAS_BINARY_PAYLOAD = (1 << 12) /**< This IO instance en/decodes binary payloads. */ -}; - -enum class IOMode { - STDIO, - CUSTOM -}; - -struct io { - enum State state; - int flags; - char delimiter; /**< Newline delimiter. */ - char separator; /**< Column separator (used by csv and villas.human formats only) */ - - struct io_direction { - /** A format type can use this file handle or overwrite the - * format::{open,close,eof,rewind} functions and the private - * data in io::_vd. - */ - FILE *stream; - - char *buffer; - size_t buflen; - } in, out; - - struct vlist *signals; /**< Signal meta data for parsed samples by io_scan() */ - bool header_printed; - - enum IOMode mode; - - void *_vd; - const struct format_type *_vt; -}; - -int io_init(struct io *io, const struct format_type *fmt, struct vlist *signals, int flags) __attribute__ ((warn_unused_result)); - -int io_init2(struct io *io, const struct format_type *fmt, const char *dt, int flags) __attribute__ ((warn_unused_result)); - -int io_destroy(struct io *io) __attribute__ ((warn_unused_result)); - -int io_open(struct io *io, const char *uri); - -int io_close(struct io *io); - -void io_header(struct io *io, const struct sample *smp); - -void io_footer(struct io *io); - -int io_print(struct io *io, struct sample *smps[], unsigned cnt); - -int io_scan(struct io *io, struct sample *smps[], unsigned cnt); - -int io_eof(struct io *io); - -void io_rewind(struct io *io); - -int io_flush(struct io *io); - -int io_fd(struct io *io); - -const struct format_type * io_type(struct io *io); - -int io_stream_open(struct io *io, const char *uri); - -int io_stream_close(struct io *io); - -int io_stream_eof(struct io *io); - -void io_stream_rewind(struct io *io); - -int io_stream_fd(struct io *io); - -int io_stream_flush(struct io *io); - -FILE * io_stream_input(struct io *io); -FILE * io_stream_output(struct io *io); - -/** Parse samples from the buffer \p buf with a length of \p len bytes. - * - * @param buf[in] The buffer of data which should be parsed / de-serialized. - * @param len[in] The length of the buffer \p buf. - * @param rbytes[out] The number of bytes which have been read from \p buf. - * @param smps[out] The array of pointers to samples. - * @param cnt[in] The number of pointers in the array \p smps. - * - * @retval >=0 The number of samples which have been parsed from \p buf and written into \p smps. - * @retval <0 Something went wrong. - */ -int io_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt); - -/** Print \p cnt samples from \p smps into buffer \p buf of length \p len. - * - * @param buf[out] The buffer which should be filled with serialized data. - * @param len[in] The length of the buffer \p buf. - * @param rbytes[out] The number of bytes which have been written to \p buf. Ignored if nullptr. - * @param smps[in] The array of pointers to samples. - * @param cnt[in] The number of pointers in the array \p smps. - * - * @retval >=0 The number of samples from \p smps which have been written into \p buf. - * @retval <0 Something went wrong. - */ -int io_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt); diff --git a/include/villas/mapping.h b/include/villas/mapping.h index 5e7973085..2960bc21b 100644 --- a/include/villas/mapping.h +++ b/include/villas/mapping.h @@ -73,7 +73,7 @@ struct mapping_entry { * A value of 0 indicates that all remaining values starting from the offset of a sample should be mapped. */ int length; - unsigned offset; /**< Offset of this mapping entry within sample::data */ + unsigned offset; /**< Offset of this mapping entry within sample::data */ union { struct { diff --git a/include/villas/node.h b/include/villas/node.h index aff09fff4..11d327322 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -75,11 +75,6 @@ struct vnode { uint64_t sequence; /**< This is a counter of received samples, in case the node-type does not generate sequence numbers itself. */ - /** The path which uses this node as a destination. - * Usually every node should be used only by a single path as destination. - * Otherwise samples from different paths would be interleaved. - */ - struct vpath *output_path; std::shared_ptr stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ struct vnode_direction in, out; @@ -202,9 +197,9 @@ struct vlist * node_input_signals(struct vnode *n); */ int node_reverse(struct vnode *n); -int node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int node_read(struct vnode *n, struct sample * smps[], unsigned cnt); -int node_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int node_write(struct vnode *n, struct sample * smps[], unsigned cnt); int node_poll_fds(struct vnode *n, int fds[]); diff --git a/include/villas/node_direction.h b/include/villas/node_direction.h index ba6df1720..109b96081 100644 --- a/include/villas/node_direction.h +++ b/include/villas/node_direction.h @@ -35,6 +35,7 @@ /* Forward declarations */ struct vnode; +struct vpath; enum class NodeDir { IN, /**< VILLASnode is receiving/reading */ @@ -45,6 +46,13 @@ struct vnode_direction { enum State state; enum NodeDir direction; + /** The path which uses this node as a source/destination. + * + * Usually every node should be used only by a single path as destination. + * Otherwise samples from different paths would be interleaved. + */ + struct vpath *path; + int enabled; int builtin; /**< This node should use built-in hooks by default. */ unsigned vectorize; /**< Number of messages to send / recv at once (scatter / gather) */ diff --git a/include/villas/node_type.h b/include/villas/node_type.h index 331eb8858..ce8cce8a7 100644 --- a/include/villas/node_type.h +++ b/include/villas/node_type.h @@ -197,7 +197,7 @@ struct vnode_type { * @param release The number of samples that should be released after read is called. * @return The number of messages actually received. */ - int (*read)(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); + int (*read)(struct vnode *n, struct sample * const smps[], unsigned cnt); /** Send multiple messages in a single datagram / packet. * @@ -214,7 +214,7 @@ struct vnode_type { * @param release The number of samples that should be released after write is called * @return The number of messages actually sent. */ - int (*write)(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); + int (*write)(struct vnode *n, struct sample * const smps[], unsigned cnt); /** Reverse source and destination of a node. * diff --git a/include/villas/nodes/amqp.hpp b/include/villas/nodes/amqp.hpp index 35779b1fd..801fba4b5 100644 --- a/include/villas/nodes/amqp.hpp +++ b/include/villas/nodes/amqp.hpp @@ -33,10 +33,7 @@ #include #include -#include - -/* Forward declarations */ -struct format_type; +#include struct amqp_ssl_info { int verify_peer; @@ -59,8 +56,7 @@ struct amqp { amqp_connection_state_t producer; amqp_connection_state_t consumer; - struct format_type *format; - struct io io; + villas::node::Format *formatter; }; /** @see node_type::print */ @@ -76,9 +72,9 @@ int amqp_start(struct vnode *n); int amqp_stop(struct vnode *n); /** @see node_type::read */ -int amqp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int amqp_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int amqp_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int amqp_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/can.hpp b/include/villas/nodes/can.hpp index e9499361e..c9919a66c 100644 --- a/include/villas/nodes/can.hpp +++ b/include/villas/nodes/can.hpp @@ -82,10 +82,10 @@ int can_start(struct vnode *n); int can_stop(struct vnode *n); /** @see node_type::write */ -int can_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int can_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::read */ -int can_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int can_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::poll_fds */ int can_poll_fds(struct vnode *n, int fds[]); diff --git a/include/villas/nodes/comedi.hpp b/include/villas/nodes/comedi.hpp index 61dbf593b..48bb764ef 100644 --- a/include/villas/nodes/comedi.hpp +++ b/include/villas/nodes/comedi.hpp @@ -93,9 +93,9 @@ int comedi_start(struct vnode *n); int comedi_stop(struct vnode *n); /** @see node_type::read */ -int comedi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int comedi_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int comedi_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int comedi_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/ethercat.hpp b/include/villas/nodes/ethercat.hpp index 850a0165c..05f8b32b8 100644 --- a/include/villas/nodes/ethercat.hpp +++ b/include/villas/nodes/ethercat.hpp @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include /* Include hard-coded Ethercat Bus configuration */ @@ -109,9 +109,9 @@ int ethercat_start(struct vnode *n); int ethercat_stop(struct vnode *n); /** @see node_type::read */ -int ethercat_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ethercat_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int ethercat_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ethercat_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/example.hpp b/include/villas/nodes/example.hpp index 71cb5f849..ff96d3c2f 100644 --- a/include/villas/nodes/example.hpp +++ b/include/villas/nodes/example.hpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include struct example { @@ -82,10 +82,10 @@ int example_pause(struct vnode *n); int example_resume(struct vnode *n); /** @see node_type::write */ -int example_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int example_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::read */ -int example_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int example_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::reverse */ int example_reverse(struct vnode *n); diff --git a/include/villas/nodes/exec.hpp b/include/villas/nodes/exec.hpp index 9d27ef3d3..bc3bc91c4 100644 --- a/include/villas/nodes/exec.hpp +++ b/include/villas/nodes/exec.hpp @@ -30,12 +30,11 @@ #pragma once #include -#include +#include /* Forward declarations */ struct vnode; struct sample; -struct format_type; /** Node-type for signal generation. * @see node_type @@ -49,8 +48,8 @@ struct exec { std::string command; villas::utils::Popen::arg_list arguments; villas::utils::Popen::env_map environment; - struct format_type *format; - struct io io; + + villas::node::Format *formatter; }; /** @see node_type::print */ @@ -66,9 +65,9 @@ int exec_open(struct vnode *n); int exec_close(struct vnode *n); /** @see node_type::read */ -int exec_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int exec_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int exec_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int exec_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/file.hpp b/include/villas/nodes/file.hpp index 190def488..9577ea47e 100644 --- a/include/villas/nodes/file.hpp +++ b/include/villas/nodes/file.hpp @@ -29,15 +29,18 @@ #pragma once -#include +#include + +#include #include #include #define FILE_MAX_PATHLEN 512 struct file { - struct io io; /**< Format and file IO */ - struct format_type *format; + villas::node::Format *formatter; + FILE *stream_in; + FILE *stream_out; char *uri_tmpl; /**< Format string for file name. */ char *uri; /**< Real file name. */ @@ -82,9 +85,9 @@ int file_start(struct vnode *n); int file_stop(struct vnode *n); /** @see node_type::read */ -int file_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int file_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int file_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int file_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/fpga.hpp b/include/villas/nodes/fpga.hpp index e8296a9f0..2326234f8 100644 --- a/include/villas/nodes/fpga.hpp +++ b/include/villas/nodes/fpga.hpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include @@ -95,10 +95,10 @@ int fpga_start(struct vnode *n); int fpga_stop(struct vnode *n); /** @see node_type::write */ -int fpga_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int fpga_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::read */ -int fpga_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int fpga_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::poll_fds */ int fpga_poll_fds(struct vnode *n, int fds[]); diff --git a/include/villas/nodes/iec61850_sv.hpp b/include/villas/nodes/iec61850_sv.hpp index 126808e61..ec026cc18 100644 --- a/include/villas/nodes/iec61850_sv.hpp +++ b/include/villas/nodes/iec61850_sv.hpp @@ -96,10 +96,10 @@ int iec61850_sv_stop(struct vnode *n); int iec61850_sv_destroy(struct vnode *n); /** @see node_type::read */ -int iec61850_sv_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int iec61850_sv_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int iec61850_sv_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int iec61850_sv_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::fd */ int iec61850_sv_fd(struct vnode *n); diff --git a/include/villas/nodes/infiniband.hpp b/include/villas/nodes/infiniband.hpp index 243a30eb9..49eefe92e 100644 --- a/include/villas/nodes/infiniband.hpp +++ b/include/villas/nodes/infiniband.hpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include @@ -128,9 +128,9 @@ int ib_destroy(struct vnode *n); int ib_stop(struct vnode *n); /** @see node_type::read */ -int ib_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ib_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int ib_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ib_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/influxdb.hpp b/include/villas/nodes/influxdb.hpp index b92f7de59..cc8a7149f 100644 --- a/include/villas/nodes/influxdb.hpp +++ b/include/villas/nodes/influxdb.hpp @@ -61,6 +61,6 @@ int influxdb_open(struct vnode *n); int influxdb_close(struct vnode *n); /** @see node_type::write */ -int influxdb_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int influxdb_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/kafka.hpp b/include/villas/nodes/kafka.hpp index e375c9c5d..696a0c29d 100644 --- a/include/villas/nodes/kafka.hpp +++ b/include/villas/nodes/kafka.hpp @@ -33,13 +33,10 @@ #include #include -#include +#include #include -/* Forward declarations */ -struct format_type; - struct kafka { struct queue_signalled queue; struct pool pool; @@ -71,8 +68,7 @@ struct kafka { char *password; /**< SSL certificate. */ } sasl; - struct format_type *format; - struct io io; + villas::node::Format *formatter; }; /** @see node_type::reverse */ @@ -103,9 +99,9 @@ int kafka_type_start(villas::node::SuperNode *sn); int kafka_type_stop(); /** @see node_type::read */ -int kafka_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int kafka_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int kafka_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int kafka_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/loopback.hpp b/include/villas/nodes/loopback.hpp index 602a2f81e..90a56d98e 100644 --- a/include/villas/nodes/loopback.hpp +++ b/include/villas/nodes/loopback.hpp @@ -58,9 +58,9 @@ int loopback_open(struct vnode *n); int loopback_close(struct vnode *n); /** @see node_type::read */ -int loopback_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int loopback_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int loopback_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int loopback_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/loopback_internal.hpp b/include/villas/nodes/loopback_internal.hpp index 1fb1d872f..844087314 100644 --- a/include/villas/nodes/loopback_internal.hpp +++ b/include/villas/nodes/loopback_internal.hpp @@ -56,10 +56,10 @@ int loopback_internal_start(struct vnode *n); int loopback_internal_stop(struct vnode *n); /** @see node_type::read */ -int loopback_internal_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int loopback_internal_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int loopback_internal_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int loopback_internal_write(struct vnode *n, struct sample * const smps[], unsigned cnt); struct vnode * loopback_internal_create(struct vnode *orig); diff --git a/include/villas/nodes/mqtt.hpp b/include/villas/nodes/mqtt.hpp index 60e15be68..12bca2123 100644 --- a/include/villas/nodes/mqtt.hpp +++ b/include/villas/nodes/mqtt.hpp @@ -31,11 +31,10 @@ #include #include -#include +#include #include /* Forward declarations */ -struct format_type; struct mosquitto; struct mqtt { @@ -62,8 +61,7 @@ struct mqtt { char *keyfile; /**< SSL private key. */ } ssl; - struct format_type *format; - struct io io; + villas::node::Format *formatter; }; /** @see node_type::reverse */ @@ -94,9 +92,9 @@ int mqtt_type_start(villas::node::SuperNode *sn); int mqtt_type_stop(); /** @see node_type::read */ -int mqtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int mqtt_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int mqtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int mqtt_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/nanomsg.hpp b/include/villas/nodes/nanomsg.hpp index 3521f7bdf..4fb4d54db 100644 --- a/include/villas/nodes/nanomsg.hpp +++ b/include/villas/nodes/nanomsg.hpp @@ -31,22 +31,18 @@ #include #include -#include +#include /** The maximum length of a packet which contains stuct msg. */ #define NANOMSG_MAX_PACKET_LEN 1500 -/* Forward declarations */ -struct format_type; - struct nanomsg { struct { int socket; struct vlist endpoints; } in, out; - struct format_type *format; - struct io io; + villas::node::Format *formatter; }; /** @see node_type::print */ @@ -62,9 +58,9 @@ int nanomsg_start(struct vnode *n); int nanomsg_stop(struct vnode *n); /** @see node_type::read */ -int nanomsg_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int nanomsg_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int nanomsg_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int nanomsg_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/ngsi.hpp b/include/villas/nodes/ngsi.hpp index a90aa218b..7c00a633b 100644 --- a/include/villas/nodes/ngsi.hpp +++ b/include/villas/nodes/ngsi.hpp @@ -96,9 +96,9 @@ int ngsi_stop(struct vnode *n); int ngsi_reverse(struct vnode *n); /** @see node_type::read */ -int ngsi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ngsi_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int ngsi_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int ngsi_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/opal.hpp b/include/villas/nodes/opal.hpp index 09f80b4d8..3bd84333e 100644 --- a/include/villas/nodes/opal.hpp +++ b/include/villas/nodes/opal.hpp @@ -80,9 +80,9 @@ int opal_start(struct vnode *n); int opal_stop(struct vnode *n); /** @see node_type::read */ -int opal_read(struct vnode *n, struct sample *smps[], unsigned cnt); +int opal_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int opal_write(struct vnode *n, struct sample *smps[], unsigned cnt); +int opal_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/rtp.hpp b/include/villas/nodes/rtp.hpp index ce62ca1c4..e86a4b222 100644 --- a/include/villas/nodes/rtp.hpp +++ b/include/villas/nodes/rtp.hpp @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include @@ -48,9 +48,6 @@ extern "C" { #include } -/* Forward declarations */ -struct format_type; - /** The maximum length of a packet which contains rtp data. */ #define RTP_INITIAL_BUFFER_LEN 1500 #define RTP_PACKET_TYPE 21 @@ -69,8 +66,7 @@ struct rtp { struct sa saddr_rtcp; /**< Local/Remote address of the RTCP socket */ } in, out; - struct format_type *format; - struct io io; + villas::node::Format *formatter; struct { int enabled; @@ -116,9 +112,9 @@ int rtp_start(struct vnode *n); int rtp_stop(struct vnode *n); /** @see node_type::read */ -int rtp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int rtp_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int rtp_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int rtp_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/shmem.hpp b/include/villas/nodes/shmem.hpp index 6a1003f03..be66ac501 100644 --- a/include/villas/nodes/shmem.hpp +++ b/include/villas/nodes/shmem.hpp @@ -60,9 +60,9 @@ int shmem_start(struct vnode *n); int shmem_stop(struct vnode *n); /** @see node_type::read */ -int shmem_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int shmem_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int shmem_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int shmem_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/signal_generator.hpp b/include/villas/nodes/signal_generator.hpp index ce3a7df35..d66b7ea12 100644 --- a/include/villas/nodes/signal_generator.hpp +++ b/include/villas/nodes/signal_generator.hpp @@ -89,6 +89,6 @@ int signal_generator_start(struct vnode *n); int signal_generator_stop(struct vnode *n); /** @see node_type::read */ -int signal_generator_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int signal_generator_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/socket.hpp b/include/villas/nodes/socket.hpp index b646f326d..8d0990e94 100644 --- a/include/villas/nodes/socket.hpp +++ b/include/villas/nodes/socket.hpp @@ -32,10 +32,7 @@ #include #include #include -#include - -/* Forward declarations */ -struct format_type; +#include /** The maximum length of a packet which contains stuct msg. */ #define SOCKET_INITIAL_BUFFER_LEN (64*1024) @@ -46,8 +43,7 @@ struct socket { enum SocketLayer layer; /**< The OSI / IP layer which should be used for this socket */ - struct format_type *format; - struct io io; + villas::node::Format *formatter; /* Multicast options */ struct multicast { @@ -78,10 +74,10 @@ int socket_start(struct vnode *n); int socket_stop(struct vnode *n); /** @see node_type::write */ -int socket_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int socket_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::read */ -int socket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int socket_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::parse */ int socket_parse(struct vnode *n, json_t *json); diff --git a/include/villas/nodes/stats.hpp b/include/villas/nodes/stats.hpp index 7330c7f5a..e9c845c65 100644 --- a/include/villas/nodes/stats.hpp +++ b/include/villas/nodes/stats.hpp @@ -70,6 +70,6 @@ int stats_node_start(struct vnode *n); int stats_node_stop(struct vnode *n); /** @see node_type::read */ -int stats_node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int stats_node_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/test_rtt.hpp b/include/villas/nodes/test_rtt.hpp index 5a78f0d65..3b45af42a 100644 --- a/include/villas/nodes/test_rtt.hpp +++ b/include/villas/nodes/test_rtt.hpp @@ -30,7 +30,7 @@ #pragma once #include -#include +#include #include /* Forward declarations */ @@ -41,7 +41,7 @@ struct sample; struct test_rtt_case { double rate; unsigned values; - unsigned limit; /**< The number of samples we take per test. */ + unsigned limit; /**< The number of samples we take per test. */ char *filename; char *filename_formatted; @@ -50,19 +50,19 @@ struct test_rtt_case { }; struct test_rtt { - struct Task task; /**< The periodic task for test_rtt_read() */ - struct io io; /**< The format of the output file */ - struct format_type *format; + struct Task task; /**< The periodic task for test_rtt_read() */ + villas::node::Format *formatter;/**< The format of the output file */ + FILE *stream; - double cooldown; /**< Number of seconds to wait beween tests. */ + double cooldown; /**< Number of seconds to wait beween tests. */ - int current; /**< Index of current test in test_rtt::cases */ + int current; /**< Index of current test in test_rtt::cases */ int counter; - struct vlist cases; /**< List of test cases */ + struct vlist cases; /**< List of test cases */ - char *output; /**< The directory where we place the results. */ - char *prefix; /**< An optional prefix in the filename. */ + char *output; /**< The directory where we place the results. */ + char *prefix; /**< An optional prefix in the filename. */ }; /** @see node_type::print */ @@ -78,9 +78,9 @@ int test_rtt_start(struct vnode *n); int test_rtt_stop(struct vnode *n); /** @see node_type::read */ -int test_rtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int test_rtt_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int test_rtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int test_rtt_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/websocket.hpp b/include/villas/nodes/websocket.hpp index 8507f0236..be0a24b2e 100644 --- a/include/villas/nodes/websocket.hpp +++ b/include/villas/nodes/websocket.hpp @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #define DEFAULT_WEBSOCKET_QUEUE_LENGTH (DEFAULT_QUEUE_LENGTH * 64) @@ -70,10 +70,9 @@ struct websocket_connection { struct lws *wsi; struct vnode *node; - struct io io; + villas::node::Format *formatter; struct queue queue; /**< For samples which are sent to the Websocket */ - struct format_type *format; struct websocket_destination *destination; struct { @@ -107,9 +106,9 @@ int websocket_stop(struct vnode *n); int websocket_destroy(struct vnode *n); /** @see node_type::read */ -int websocket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int websocket_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int websocket_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int websocket_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/nodes/zeromq.hpp b/include/villas/nodes/zeromq.hpp index a3c06c9e1..7b124ecd5 100644 --- a/include/villas/nodes/zeromq.hpp +++ b/include/villas/nodes/zeromq.hpp @@ -32,23 +32,22 @@ #include #include +#include #include -#include +#include #if ZMQ_BUILD_DRAFT_API && (ZMQ_VERSION_MAJOR > 4 || (ZMQ_VERSION_MAJOR == 4 && ZMQ_VERSION_MINOR >= 2)) #define ZMQ_BUILD_DISH 1 #endif /* Forward declarations */ -struct format_type; struct vnode; struct sample; struct zeromq { int ipv6; - struct format_type *format; - struct io io; + villas::node::Format *formatter; struct Curve { int enabled; @@ -93,9 +92,9 @@ int zeromq_start(struct vnode *n); int zeromq_stop(struct vnode *n); /** @see node_type::read */ -int zeromq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int zeromq_read(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @see node_type::write */ -int zeromq_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release); +int zeromq_write(struct vnode *n, struct sample * const smps[], unsigned cnt); /** @} */ diff --git a/include/villas/path.h b/include/villas/path.h index 874d50b74..afe0bea01 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -79,7 +79,8 @@ struct vpath { struct Task timeout; double rate; /**< A timeout for */ - int enabled; /**< Is this path enabled. */ + int enabled; /**< Is this path enabled? */ + int muxed; /**< Is this path muxed? */ int affinity; /**< Thread affinity. */ int poll; /**< Weather or not to use poll(2). */ int reverse; /**< This path has a matching reverse path. */ @@ -163,6 +164,8 @@ void path_parse_mask(struct vpath *p, json_t *json_mask, struct vlist *nodes); bool path_is_simple(const struct vpath *p); +bool path_is_muxed(const struct vpath *p); + bool path_is_enabled(const struct vpath *p); bool path_is_reversed(const struct vpath *p); diff --git a/include/villas/path_destination.h b/include/villas/path_destination.h index 3e2415f26..b5aca6b82 100644 --- a/include/villas/path_destination.h +++ b/include/villas/path_destination.h @@ -49,7 +49,7 @@ int path_destination_prepare(struct vpath_destination *pd, int queuelen); void path_destination_check(struct vpath_destination *pd); -void path_destination_enqueue(struct vpath *p, struct sample *smps[], unsigned cnt); +void path_destination_enqueue(struct vpath *p, const struct sample * const smps[], unsigned cnt); void path_destination_write(struct vpath_destination *pd, struct vpath *p); diff --git a/include/villas/path_source.h b/include/villas/path_source.h index b2b77bc3f..c3cc237da 100644 --- a/include/villas/path_source.h +++ b/include/villas/path_source.h @@ -45,12 +45,12 @@ struct vpath_source { struct vnode *node; bool masked; - + enum PathSourceType type; struct pool pool; struct vlist mappings; /**< List of mappings (struct mapping_entry). */ - struct vlist secondaries; + struct vlist secondaries; /**< List of secondary path sources (struct path_sourced). */ }; int path_source_init_master(struct vpath_source *ps, struct vnode *n) __attribute__ ((warn_unused_result)); diff --git a/include/villas/plugin.h b/include/villas/plugin.h index 3d4d079bd..94e1170fe 100644 --- a/include/villas/plugin.h +++ b/include/villas/plugin.h @@ -26,13 +26,11 @@ #include #include #include -#include extern struct vlist plugins; enum PluginType { - NODE, - FORMAT, + NODE }; struct plugin { @@ -42,13 +40,12 @@ struct plugin { enum PluginType type; union { - struct format_type format; struct vnode_type node; }; }; /** Return a pointer to the plugin structure */ -#define plugin(vt) ((struct plugin *) ((char *) (vt) - offsetof(struct plugin, format))) +#define plugin(vt) ((struct plugin *) ((char *) (vt) - offsetof(struct plugin, node))) #define plugin_name(vt) plugin(vt)->name #define plugin_description(vt) plugin(vt)->description diff --git a/include/villas/sample.h b/include/villas/sample.h index 596e71261..d04684a05 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -120,7 +120,7 @@ int sample_incref(struct sample *s); /** Decrease reference count and release memory if last reference was held. */ int sample_decref(struct sample *s); -int sample_copy(struct sample *dst, struct sample *src); +int sample_copy(struct sample *dst, const struct sample *src); /** Dump all details about a sample to debug log */ void sample_dump(villas::Logger logger, struct sample *s); @@ -128,13 +128,12 @@ void sample_dump(villas::Logger logger, struct sample *s); /** Compare two samples */ int sample_cmp(struct sample *a, struct sample *b, double epsilon, int flags); -int sample_clone_many(struct sample *clones[], struct sample *origs[], int cnt); -int sample_copy_many(struct sample *dsts[], struct sample *srcs[], int cnt); -int sample_incref_many(struct sample *smps[], int cnt); -int sample_decref_many(struct sample *smps[], int cnt); +int sample_clone_many(struct sample *dsts[], const struct sample * const srcs[], int cnt); +int sample_copy_many(struct sample * const dsts[], const struct sample * const srcs[], int cnt); +int sample_incref_many(struct sample * const smps[], int cnt); +int sample_decref_many(struct sample * const smps[], int cnt); enum SignalType sample_format(const struct sample *s, unsigned idx); void sample_data_insert(struct sample *smp, const union signal_data *src, size_t offset, size_t len); - void sample_data_remove(struct sample *smp, size_t offset, size_t len); diff --git a/include/villas/shmem.h b/include/villas/shmem.h index 26635a6f7..2bb36715f 100644 --- a/include/villas/shmem.h +++ b/include/villas/shmem.h @@ -96,7 +96,7 @@ int shmem_int_close(struct shmem_int *shm); * @retval >=0 Number of samples that were read. Can be less than cnt (including 0) in case not enough samples were available. * @retval -1 The other process closed the interface; no samples can be read anymore. */ -int shmem_int_read(struct shmem_int *shm, struct sample *smps[], unsigned cnt); +int shmem_int_read(struct shmem_int *shm, struct sample * const smps[], unsigned cnt); /** Write samples to the interface. * @@ -106,7 +106,7 @@ int shmem_int_read(struct shmem_int *shm, struct sample *smps[], unsigned cnt); * @retval >=0 Number of samples that were successfully written. Can be less than cnt (including 0) in case of a full queue. * @retval -1 The write failed for some reason; no more samples can be written. */ -int shmem_int_write(struct shmem_int *shm, struct sample *smps[], unsigned cnt); +int shmem_int_write(struct shmem_int *shm, const struct sample * const smps[], unsigned cnt); /** Allocate samples to be written to the interface. * diff --git a/include/villas/signal_data.h b/include/villas/signal_data.h index 3c338cb99..5b4ea6a12 100644 --- a/include/villas/signal_data.h +++ b/include/villas/signal_data.h @@ -63,12 +63,12 @@ union signal_data { void signal_data_cast(union signal_data *data, enum SignalType from, enum SignalType to); /** Print value of a signal to a character buffer. */ -int signal_data_print_str(const union signal_data *data, enum SignalType type, char *buf, size_t len); +int signal_data_print_str(const union signal_data *data, enum SignalType type, char *buf, size_t len, int precision = 5); int signal_data_parse_str(union signal_data *data, enum SignalType type, const char *ptr, char **end); int signal_data_parse_json(union signal_data *data, enum SignalType type, json_t *json); -json_t * signal_data_to_json(union signal_data *data, enum SignalType type); +json_t * signal_data_to_json(const union signal_data *data, enum SignalType type); void signal_data_set(union signal_data *data, enum SignalType type, double val); diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index aa6479cf9..0590856b7 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -48,6 +48,7 @@ set(LIB_SRC config_helper.cpp config.cpp dumper.cpp + format.cpp mapping.cpp memory.cpp memory/heap.cpp @@ -65,15 +66,13 @@ set(LIB_SRC queue.cpp sample.cpp shmem.cpp - signal.cpp signal_data.cpp signal_list.cpp signal_type.cpp + signal.cpp + socket_addr.cpp stats.cpp super_node.cpp - socket_addr.cpp - io.cpp - format_type.cpp ) if(WITH_WEB) diff --git a/lib/api/requests/capabiltities.cpp b/lib/api/requests/capabiltities.cpp index c5501f443..8538dca49 100644 --- a/lib/api/requests/capabiltities.cpp +++ b/lib/api/requests/capabiltities.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -49,18 +50,31 @@ public: if (body != nullptr) throw BadRequest("Capabilities endpoint does not accept any body data"); - for (auto f : plugin::Registry::lookup()) { - json_name = json_string(f->getName().c_str()); + for (auto p : plugin::Registry::lookup()) { + json_name = json_string(p->getName().c_str()); json_array_append_new(json_apis, json_name); } - for (auto f : plugin::Registry::lookup()) { - json_name = json_string(f->getName().c_str()); + for (auto p : plugin::Registry::lookup()) { + json_name = json_string(p->getName().c_str()); json_array_append_new(json_hooks, json_name); } + for (auto p : plugin::Registry::lookup()) { + json_name = json_string(p->getName().c_str()); + + json_array_append_new(json_formats, json_name); + } + +#if 0 /* @todo Port to C++ */ + for (auto f : NodeFactory::lookup()) { + json_name = json_string(f->getName().c_str()); + + json_array_append_new(json_nodes, json_name); + } +#else for (size_t i = 0; i < vlist_length(&plugins); i++) { struct plugin *p = (struct plugin *) vlist_at(&plugins, i); @@ -70,26 +84,8 @@ public: json_name = json_string(p->name); json_array_append_new(json_nodes, json_name); break; - - case PluginType::FORMAT: - json_name = json_string(p->name); - json_array_append_new(json_formats, json_name); - break; } } - -#if 0 /* @todo Port to C++ */ - for (auto f : NodeFactory::lookup()) { - json_name = json_string(f->getName().c_str()); - - json_array_append_new(json_nodes, json_name); - } - - for (auto f : FormatFactory::lookup()) { - json_name = json_string(f->getName().c_str()); - - json_array_append_new(json_formats, json_name); - } #endif auto *json_capabilities = json_pack("{ s: o, s: o, s: o, s: o }", diff --git a/lib/api/requests/node_file.cpp b/lib/api/requests/node_file.cpp index 2799fce51..6bb3ae166 100644 --- a/lib/api/requests/node_file.cpp +++ b/lib/api/requests/node_file.cpp @@ -26,7 +26,7 @@ #include #include -#include +#include namespace villas { namespace node { @@ -55,7 +55,7 @@ public: struct file *f = (struct file *) node->_vd; if (matches[2] == "rewind") - io_rewind(&f->io); + rewind(f->stream_in); return new Response(session, HTTP_STATUS_OK); } diff --git a/lib/format.cpp b/lib/format.cpp new file mode 100644 index 000000000..0072a53f3 --- /dev/null +++ b/lib/format.cpp @@ -0,0 +1,196 @@ +/** Reading and writing simulation samples in various formats. + * + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace villas; +using namespace villas::node; +using namespace villas::utils; + +Format * FormatFactory::make(json_t *json) +{ + std::string type; + + FormatFactory *ff; + Format *f; + + if (json_is_string(json)) { + type = json_string_value(json); + + return FormatFactory::make(type); + } + else if (json_is_object(json)) { + json_t *json_type = json_object_get(json, "type"); + + type = json_string_value(json_type); + + ff = plugin::Registry::lookup(type); + if (!ff) + throw ConfigError(json, "Unknown format: {}", type); + + f = ff->make(); + f->parse(json); + + return f; + } + else + throw ConfigError(json, "Invalid format config"); +} + +Format * FormatFactory::make(const std::string &format) +{ + FormatFactory *ff = plugin::Registry::lookup(format); + if (!ff) + throw RuntimeError("Unknown format: {}", format); + + return ff->make(); +} + +Format::Format(int fl) : + flags(fl), + real_precision(17), + destroy_signals(false), + signals(nullptr) +{ + in.buflen = + out.buflen = 4096; + + in.buffer = new char[in.buflen]; + out.buffer = new char[out.buflen]; + + if (!in.buffer || !out.buffer) + throw MemoryAllocationError(); +} + +Format::~Format() +{ + int ret __attribute__((unused)); + + delete[] in.buffer; + delete[] out.buffer; + + if (signals && destroy_signals) + ret = vlist_destroy(signals, (dtor_cb_t) signal_decref, false); +} + +void Format::start(struct vlist *sigs, int fl) +{ + flags &= fl; + + signals = sigs; + + start(); +} + +void Format::start(const std::string &dtypes, int fl) +{ + int ret; + + flags |= fl; + + signals = new struct vlist; + if (!signals) + throw MemoryAllocationError(); + + ret = vlist_init(signals); + if (ret) + throw RuntimeError("Failed to initialize list"); + + ret = signal_list_generate2(signals, dtypes.c_str()); + if (ret) + throw RuntimeError("Failed to generate signal list"); + + destroy_signals = true; + + start(); +} + +int Format::print(FILE *f, const struct sample * const smps[], unsigned cnt) +{ + int ret; + size_t wbytes; + + ret = sprint(out.buffer, out.buflen, &wbytes, smps, cnt); + + fwrite(out.buffer, wbytes, 1, f); + + return ret; +} + +int Format::scan(FILE *f, struct sample * const smps[], unsigned cnt) +{ + size_t bytes, rbytes; + + bytes = fread(in.buffer, 1, in.buflen, f); + + return sscan(in.buffer, bytes, &rbytes, smps, cnt); +} + +void Format::parse(json_t *json) +{ + int ret; + json_error_t err; + + int ts_origin = -1; + int ts_received = -1; + int sequence = -1; + int data = -1; + int offset = -1; + + ret = json_unpack_ex(json, &err, 0, "{ s?: b, s?: b, s?: b, s?: b, s?: b, s?: i }", + "ts_origin", &ts_origin, + "ts_received", &ts_received, + "sequence", &sequence, + "data", &data, + "offset", &offset, + "real_precision", &real_precision + ); + if (ret) + throw ConfigError(json, err, "node-config-format", "Failed to parse format configuration"); + + if (real_precision < 0 || real_precision > 31) + throw ConfigError(json, err, "node-config-format-precision", "The valid range for the real_precision setting is between 0 and 31 (inclusive)"); + + if (ts_origin == 0) + flags &= ~ (int) SampleFlags::HAS_TS_ORIGIN; + + if (ts_received == 0) + flags &= ~ (int) SampleFlags::HAS_TS_RECEIVED; + + if (sequence == 0) + flags &= ~ (int) SampleFlags::HAS_SEQUENCE; + + if (data == 0) + flags &= ~ (int) SampleFlags::HAS_DATA; + + if (offset == 0) + flags &= ~ (int) SampleFlags::HAS_OFFSET; +} diff --git a/lib/format_type.cpp b/lib/format_type.cpp deleted file mode 100644 index 9f376c49d..000000000 --- a/lib/format_type.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/** Read / write sample data in different formats. - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -#include -#include - -struct format_type * format_type_lookup(const char *name) -{ - struct plugin *p; - - p = plugin_lookup(PluginType::FORMAT, name); - if (!p) - return nullptr; - - return &p->format; -} - -const char * format_type_name(struct format_type *vt) -{ - return plugin_name(vt); -} diff --git a/lib/formats/CMakeLists.txt b/lib/formats/CMakeLists.txt index e4b9deb05..8200305b2 100644 --- a/lib/formats/CMakeLists.txt +++ b/lib/formats/CMakeLists.txt @@ -60,16 +60,17 @@ if(DEFINED PROTOBUFC_COMPILER AND PROTOBUFC_FOUND) endif() list(APPEND FORMAT_SRC - json.cpp + column.cpp iotagent_ul.cpp - json_reserve.cpp json_kafka.cpp + json_reserve.cpp + json.cpp + line.cpp + msg.cpp + raw.cpp + value.cpp villas_binary.cpp villas_human.cpp - csv.cpp - raw.cpp - msg.cpp - value.cpp ) add_library(formats STATIC ${FORMAT_SRC}) diff --git a/lib/formats/column.cpp b/lib/formats/column.cpp new file mode 100644 index 000000000..a5baa8e04 --- /dev/null +++ b/lib/formats/column.cpp @@ -0,0 +1,207 @@ +/** Comma-separated values. + * + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace villas; +using namespace villas::node; + +size_t ColumnLineFormat::sprintLine(char *buf, size_t len, const struct sample *smp) +{ + size_t off = 0; + struct signal *sig; + + if (smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) + off += snprintf(buf + off, len - off, "%lld%c%09lld", (long long) smp->ts.origin.tv_sec, separator, + (long long) smp->ts.origin.tv_nsec); + else + off += snprintf(buf + off, len - off, "nan%cnan", separator); + + if (smp->flags & (int) SampleFlags::HAS_TS_RECEIVED) + off += snprintf(buf + off, len - off, "%c%.09f", separator, time_delta(&smp->ts.origin, &smp->ts.received)); + else + off += snprintf(buf + off, len - off, "%cnan", separator); + + if (smp->flags & (int) SampleFlags::HAS_SEQUENCE) + off += snprintf(buf + off, len - off, "%c%" PRIu64, separator, smp->sequence); + else + off += snprintf(buf + off, len - off, "%cnan", separator); + + for (unsigned i = 0; i < smp->length; i++) { + sig = (struct signal *) vlist_at_safe(smp->signals, i); + if (!sig) + break; + + off += snprintf(buf + off, len - off, "%c", separator); + off += signal_data_print_str(&smp->data[i], sig->type, buf + off, len - off, real_precision); + } + + off += snprintf(buf + off, len - off, "%c", delimiter); + + return off; +} + +size_t ColumnLineFormat::sscanLine(const char *buf, size_t len, struct sample *smp) +{ + int ret; + unsigned i = 0; + const char *ptr = buf; + char *end; + + struct timespec offset; + + smp->flags = 0; + smp->signals = signals; + + smp->ts.origin.tv_sec = strtoul(ptr, &end, 10); + if (end == ptr || *end == delimiter) + goto out; + + ptr = end + 1; + + smp->ts.origin.tv_nsec = strtoul(ptr, &end, 10); + if (end == ptr || *end == delimiter) + goto out; + + ptr = end + 1; + + smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; + + offset = time_from_double(strtof(ptr, &end)); + if (end == ptr || *end == delimiter) + goto out; + + smp->ts.received = time_add(&smp->ts.origin, &offset); + + smp->flags |= (int) SampleFlags::HAS_TS_RECEIVED; + + ptr = end + 1; + + smp->sequence = strtoul(ptr, &end, 10); + if (end == ptr || *end == delimiter) + goto out; + + smp->flags |= (int) SampleFlags::HAS_SEQUENCE; + + for (ptr = end + 1, i = 0; i < smp->capacity; ptr = end + 1, i++) { + if (*end == delimiter) + goto out; + + struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, i); + if (!sig) + goto out; + + ret = signal_data_parse_str(&smp->data[i], sig->type, ptr, &end); + if (ret || end == ptr) /* There are no valid values anymore. */ + goto out; + } + +out: if (*end == delimiter) + end++; + + smp->length = i; + if (smp->length > 0) + smp->flags |= (int) SampleFlags::HAS_DATA; + + return end - buf; +} + +void ColumnLineFormat::header(FILE *f, const struct vlist *sigs) +{ + /* Abort if we are not supposed to, or have already printed the header */ + if (!print_header || header_printed) + return; + + if (comment) + fprintf(f, "%c", comment); + + if (flags & (int) SampleFlags::HAS_TS_ORIGIN) + fprintf(f, "secs%cnsecs%c", separator, separator); + + if (flags & (int) SampleFlags::HAS_OFFSET) + fprintf(f, "offset%c", separator); + + if (flags & (int) SampleFlags::HAS_SEQUENCE) + fprintf(f, "sequence%c", separator); + + if (flags & (int) SampleFlags::HAS_DATA) { + for (unsigned i = 0; i < vlist_length(sigs); i++) { + struct signal *sig = (struct signal *) vlist_at_safe(sigs, i); + if (!sig) + break; + + if (sig->name) + fprintf(f, "%s", sig->name); + else + fprintf(f, "signal%u", i); + + if (sig->unit) + fprintf(f, "[%s]", sig->unit); + + if (i + 1 < vlist_length(sigs)) + fprintf(f, "%c", separator); + } + } + + fprintf(f, "%c", delimiter); + + LineFormat::header(f, sigs); +} + +void ColumnLineFormat::parse(json_t *json) +{ + int ret; + json_error_t err; + const char *sep = nullptr; + + ret = json_unpack_ex(json, &err, 0, "{ s?: s }", + "separator", &sep + ); + if (ret) + throw ConfigError(json, err, "node-config-format-column", "Failed to parse format configuration"); + + if (sep) { + if (strlen(sep) != 1) + throw ConfigError(json, "node-config-format-column-separator", "Column separator must be a single character!"); + + separator = sep[0]; + } + + LineFormat::parse(json); +} + + +static char n1[] = "csv"; +static char d1[] = "Comma-separated values"; +static ColumnLineFormatPlugin p1; + +static char n2[] = "tsv"; +static char d2[] = "Tabulator-separated values"; +static ColumnLineFormatPlugin p2; diff --git a/lib/formats/csv.cpp b/lib/formats/csv.cpp deleted file mode 100644 index 04c4de7fa..000000000 --- a/lib/formats/csv.cpp +++ /dev/null @@ -1,238 +0,0 @@ -/** Comma-separated values. - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -using namespace villas::utils; - -static size_t csv_sprint_single(struct io *io, char *buf, size_t len, const struct sample *smp) -{ - size_t off = 0; - struct signal *sig; - - if (smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) - off += snprintf(buf + off, len - off, "%lld%c%09lld", (long long) smp->ts.origin.tv_sec, io->separator, - (long long) smp->ts.origin.tv_nsec); - else - off += snprintf(buf + off, len - off, "nan%cnan", io->separator); - - if (smp->flags & (int) SampleFlags::HAS_TS_RECEIVED) - off += snprintf(buf + off, len - off, "%c%.09f", io->separator, time_delta(&smp->ts.origin, &smp->ts.received)); - else - off += snprintf(buf + off, len - off, "%cnan", io->separator); - - if (smp->flags & (int) SampleFlags::HAS_SEQUENCE) - off += snprintf(buf + off, len - off, "%c%" PRIu64, io->separator, smp->sequence); - else - off += snprintf(buf + off, len - off, "%cnan", io->separator); - - for (unsigned i = 0; i < smp->length; i++) { - sig = (struct signal *) vlist_at_safe(smp->signals, i); - if (!sig) - break; - - off += snprintf(buf + off, len - off, "%c", io->separator); - off += signal_data_print_str(&smp->data[i], sig->type, buf + off, len - off); - } - - off += snprintf(buf + off, len - off, "%c", io->delimiter); - - return off; -} - -static size_t csv_sscan_single(struct io *io, const char *buf, size_t len, struct sample *smp) -{ - int ret; - unsigned i = 0; - const char *ptr = buf; - char *end; - - struct timespec offset; - - smp->flags = 0; - smp->signals = io->signals; - - smp->ts.origin.tv_sec = strtoul(ptr, &end, 10); - if (end == ptr || *end == io->delimiter) - goto out; - - ptr = end + 1; - - smp->ts.origin.tv_nsec = strtoul(ptr, &end, 10); - if (end == ptr || *end == io->delimiter) - goto out; - - ptr = end + 1; - - smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; - - offset = time_from_double(strtof(ptr, &end)); - if (end == ptr || *end == io->delimiter) - goto out; - - smp->ts.received = time_add(&smp->ts.origin, &offset); - - smp->flags |= (int) SampleFlags::HAS_TS_RECEIVED; - - ptr = end + 1; - - smp->sequence = strtoul(ptr, &end, 10); - if (end == ptr || *end == io->delimiter) - goto out; - - smp->flags |= (int) SampleFlags::HAS_SEQUENCE; - - for (ptr = end + 1, i = 0; i < smp->capacity; ptr = end + 1, i++) { - if (*end == io->delimiter) - goto out; - - struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, i); - if (!sig) - goto out; - - ret = signal_data_parse_str(&smp->data[i], sig->type, ptr, &end); - if (ret || end == ptr) /* There are no valid values anymore. */ - goto out; - } - -out: if (*end == io->delimiter) - end++; - - smp->length = i; - if (smp->length > 0) - smp->flags |= (int) SampleFlags::HAS_DATA; - - return end - buf; -} - -int csv_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) -{ - unsigned i; - size_t off = 0; - - for (i = 0; i < cnt && off < len; i++) - off += csv_sprint_single(io, buf + off, len - off, smps[i]); - - if (wbytes) - *wbytes = off; - - return i; -} - -int csv_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) -{ - unsigned i; - size_t off = 0; - - for (i = 0; i < cnt && off < len; i++) - off += csv_sscan_single(io, buf + off, len - off, smps[i]); - - if (rbytes) - *rbytes = off; - - return i; -} - -void csv_header(struct io *io, const struct sample *smp) -{ - FILE *f = io_stream_output(io); - - fprintf(f, "# "); - if (io->flags & (int) SampleFlags::HAS_TS_ORIGIN) - fprintf(f, "secs%cnsecs%c", io->separator, io->separator); - - if (io->flags & (int) SampleFlags::HAS_OFFSET) - fprintf(f, "offset%c", io->separator); - - if (io->flags & (int) SampleFlags::HAS_SEQUENCE) - fprintf(f, "sequence%c", io->separator); - - if (io->flags & (int) SampleFlags::HAS_DATA) { - for (unsigned i = 0; i < smp->length; i++) { - struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, i); - if (!sig) - break; - - if (sig->name) - fprintf(f, "%s", sig->name); - else - fprintf(f, "signal%u", i); - - if (sig->unit) - fprintf(f, "[%s]", sig->unit); - - if (i + 1 < smp->length) - fprintf(f, "%c", io->separator); - } - } - - fprintf(f, "%c", io->delimiter); -} - -static struct plugin p1; -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p1.name = "tsv"; - p1.description = "Tabulator-separated values"; - p1.type = PluginType::FORMAT; - p1.format.header = csv_header; - p1.format.sprint = csv_sprint; - p1.format.sscan = csv_sscan; - p1.format.size = 0; - p1.format.flags = (int) IOFlags::NEWLINES | - (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - p1.format.separator = '\t'; - - vlist_push(&plugins, &p1); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p1); -} - -static struct plugin p2; -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p2.name = "csv"; - p2.description = "Comma-separated values"; - p2.type = PluginType::FORMAT; - p2.format.header = csv_header; - p2.format.sprint = csv_sprint; - p2.format.sscan = csv_sscan; - p2.format.size = 0; - p2.format.flags = (int) IOFlags::NEWLINES | - (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - p2.format.separator = ','; - - vlist_push(&plugins, &p2); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p2); -} diff --git a/lib/formats/iotagent_ul.cpp b/lib/formats/iotagent_ul.cpp index 4e22c8e66..826fb1551 100644 --- a/lib/formats/iotagent_ul.cpp +++ b/lib/formats/iotagent_ul.cpp @@ -24,20 +24,21 @@ #include -#include #include #include #include #include #include -#include -#include +#include -int iotagent_ul_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +using namespace villas; +using namespace villas::node; + +int IotAgentUltraLightFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { size_t printed = 0; struct signal *sig; - struct sample *smp = smps[0]; + const struct sample *smp = smps[0]; for (unsigned i = 0; (i < smp->length) && (printed < len); i++) { sig = (struct signal *) vlist_at_safe(smp->signals, i); @@ -59,18 +60,11 @@ int iotagent_ul_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, str return 0; } -static struct plugin p; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "iotagent_ul"; - p.description = "FIWARE IotAgent UltraLight format"; - p.type = PluginType::FORMAT; - p.format.sprint = iotagent_ul_sprint; - p.format.size = 0; - - vlist_init_and_push(&plugins, &p); +int IotAgentUltraLightFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) +{ + return -1; } -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} +static char n[] = "iotagent_ul"; +static char d[] = "FIWARE IotAgent UltraLight format"; +static FormatPlugin p; diff --git a/lib/formats/json.cpp b/lib/formats/json.cpp index 354a1bcfc..fee1998cd 100644 --- a/lib/formats/json.cpp +++ b/lib/formats/json.cpp @@ -20,17 +20,16 @@ * along with this program. If not, see . *********************************************************************************/ -#include #include #include #include -#include -#include +#include #include using namespace villas; +using namespace villas::node; -static enum SignalType json_detect_format(json_t *val) +enum SignalType JsonFormat::detect(const json_t *val) { int type = json_typeof(val); @@ -53,7 +52,7 @@ static enum SignalType json_detect_format(json_t *val) } } -static json_t * json_pack_timestamps(struct io *io, struct sample *smp) +json_t * JsonFormat::packTimestamps(const struct sample *smp) { json_t *json_ts = json_object(); @@ -63,12 +62,12 @@ static json_t * json_pack_timestamps(struct io *io, struct sample *smp) const char *fmt = "[ I, I ]"; #endif - if (io->flags & (int) SampleFlags::HAS_TS_ORIGIN) { + if (flags & (int) SampleFlags::HAS_TS_ORIGIN) { if (smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) json_object_set(json_ts, "origin", json_pack(fmt, smp->ts.origin.tv_sec, smp->ts.origin.tv_nsec)); } - if (io->flags & (int) SampleFlags::HAS_TS_RECEIVED) { + if (flags & (int) SampleFlags::HAS_TS_RECEIVED) { if (smp->flags & (int) SampleFlags::HAS_TS_RECEIVED) json_object_set(json_ts, "received", json_pack(fmt, smp->ts.received.tv_sec, smp->ts.received.tv_nsec)); } @@ -76,7 +75,7 @@ static json_t * json_pack_timestamps(struct io *io, struct sample *smp) return json_ts; } -static int json_unpack_timestamps(json_t *json_ts, struct sample *smp) +int JsonFormat::unpackTimestamps(json_t *json_ts, struct sample *smp) { int ret; json_error_t err; @@ -107,22 +106,22 @@ static int json_unpack_timestamps(json_t *json_ts, struct sample *smp) return 0; } -static int json_pack_sample(struct io *io, json_t **j, struct sample *smp) +int JsonFormat::packSample(json_t **json_smp, const struct sample *smp) { - json_t *json_smp; + json_t *json_root; json_error_t err; - json_smp = json_pack_ex(&err, 0, "{ s: o }", "ts", json_pack_timestamps(io, smp)); + json_root = json_pack_ex(&err, 0, "{ s: o }", "ts", packTimestamps(smp)); - if (io->flags & (int) SampleFlags::HAS_SEQUENCE) { + if (flags & (int) SampleFlags::HAS_SEQUENCE) { if (smp->flags & (int) SampleFlags::HAS_SEQUENCE) { json_t *json_sequence = json_integer(smp->sequence); - json_object_set(json_smp, "sequence", json_sequence); + json_object_set_new(json_root, "sequence", json_sequence); } } - if (io->flags & (int) SampleFlags::HAS_DATA) { + if (flags & (int) SampleFlags::HAS_DATA) { json_t *json_data = json_array(); for (unsigned i = 0; i < smp->length; i++) { @@ -132,38 +131,38 @@ static int json_pack_sample(struct io *io, json_t **j, struct sample *smp) json_t *json_value = signal_data_to_json(&smp->data[i], sig->type); - json_array_append(json_data, json_value); + json_array_append_new(json_data, json_value); } - json_object_set(json_smp, "data", json_data); + json_object_set_new(json_root, "data", json_data); } - *j = json_smp; + *json_smp = json_root; return 0; } -static int json_pack_samples(struct io *io, json_t **j, struct sample *smps[], unsigned cnt) +int JsonFormat::packSamples(json_t **json_smps, const struct sample * const smps[], unsigned cnt) { int ret; - json_t *json_smps = json_array(); + json_t *json_root = json_array(); for (unsigned i = 0; i < cnt; i++) { json_t *json_smp; - ret = json_pack_sample(io, &json_smp, smps[i]); + ret = packSample(&json_smp, smps[i]); if (ret) break; - json_array_append(json_smps, json_smp); + json_array_append_new(json_root, json_smp); } - *j = json_smps; + *json_smps = json_root; return cnt; } -static int json_unpack_sample(struct io *io, json_t *json_smp, struct sample *smp) +int JsonFormat::unpackSample(json_t *json_smp, struct sample *smp) { int ret; json_error_t err; @@ -171,7 +170,7 @@ static int json_unpack_sample(struct io *io, json_t *json_smp, struct sample *sm size_t i; int64_t sequence = -1; - smp->signals = io->signals; + smp->signals = signals; ret = json_unpack_ex(json_smp, &err, 0, "{ s?: o, s?: I, s: o }", "ts", &json_ts, @@ -184,7 +183,7 @@ static int json_unpack_sample(struct io *io, json_t *json_smp, struct sample *sm smp->length = 0; if (json_ts) { - ret = json_unpack_timestamps(json_ts, smp); + ret = unpackTimestamps(json_ts, smp); if (ret) return ret; } @@ -205,7 +204,7 @@ static int json_unpack_sample(struct io *io, json_t *json_smp, struct sample *sm if (!sig) return -1; - enum SignalType fmt = json_detect_format(json_value); + enum SignalType fmt = detect(json_value); if (sig->type != fmt) throw RuntimeError("Received invalid data type in JSON payload: Received {}, expected {} for signal {} (index {}).", signal_type_to_str(fmt), signal_type_to_str(sig->type), sig->name, i); @@ -223,7 +222,7 @@ static int json_unpack_sample(struct io *io, json_t *json_smp, struct sample *sm return 0; } -static int json_unpack_samples(struct io *io, json_t *json_smps, struct sample *smps[], unsigned cnt) +int JsonFormat::unpackSamples(json_t *json_smps, struct sample * const smps[], unsigned cnt) { int ret; json_t *json_smp; @@ -236,7 +235,7 @@ static int json_unpack_samples(struct io *io, json_t *json_smps, struct sample * if (i >= cnt) break; - ret = json_unpack_sample(io, json_smp, smps[i]); + ret = unpackSample(json_smp, smps[i]); if (ret < 0) break; } @@ -244,17 +243,17 @@ static int json_unpack_samples(struct io *io, json_t *json_smps, struct sample * return i; } -int json_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +int JsonFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { int ret; json_t *json; size_t wr; - ret = json_pack_samples(io, &json, smps, cnt); + ret = packSamples(&json, smps, cnt); if (ret < 0) return ret; - wr = json_dumpb(json, buf, len, 0); + wr = json_dumpb(json, buf, len, dump_flags); json_decref(json); @@ -264,7 +263,7 @@ int json_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sam return ret; } -int json_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) +int JsonFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) { int ret; json_t *json; @@ -274,7 +273,7 @@ int json_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struc if (!json) return -1; - ret = json_unpack_samples(io, json, smps, cnt); + ret = unpackSamples(json, smps, cnt); json_decref(json); @@ -287,21 +286,19 @@ int json_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struc return ret; } -int json_print(struct io *io, struct sample *smps[], unsigned cnt) +int JsonFormat::print(FILE *f, const struct sample * const smps[], unsigned cnt) { int ret; unsigned i; json_t *json; - FILE *f = io_stream_output(io); - for (i = 0; i < cnt; i++) { - ret = json_pack_sample(io, &json, smps[i]); + ret = packSample(&json, smps[i]); if (ret) return ret; - ret = json_dumpf(json, f, 0); - fputc(io->delimiter, f); + ret = json_dumpf(json, f, dump_flags); + fputc('\n', f); json_decref(json); @@ -312,15 +309,13 @@ int json_print(struct io *io, struct sample *smps[], unsigned cnt) return i; } -int json_scan(struct io *io, struct sample *smps[], unsigned cnt) +int JsonFormat::scan(FILE *f, struct sample * const smps[], unsigned cnt) { int ret; unsigned i; json_t *json; json_error_t err; - FILE *f = io_stream_input(io); - for (i = 0; i < cnt; i++) { if (feof(f)) return -1; @@ -329,7 +324,7 @@ skip: json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); if (!json) break; - ret = json_unpack_sample(io, json, smps[i]); + ret = unpackSample(json, smps[i]); if (ret) goto skip; @@ -339,23 +334,54 @@ skip: json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); return i; } -static struct plugin p; +void JsonFormat::parse(json_t *json) +{ + int ret; + json_error_t err; -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "json"; - p.description = "Javascript Object Notation"; - p.type = PluginType::FORMAT; - p.format.print = json_print; - p.format.scan = json_scan; - p.format.sprint = json_sprint; - p.format.sscan = json_sscan; - p.format.size = 0; - p.format.flags = (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - p.format.delimiter = '\n'; + int indent = 0; + int compact = 0; + int ensure_ascii = 0; + int escape_slash = 0; + int sort_keys = 0; - vlist_init_and_push(&plugins, &p); + ret = json_unpack_ex(json, &err, 0, "{ s?: i, s?: b, s?: b, s?: b, s?: b }", + "indent", &indent, + "compact", &compact, + "ensure_ascii", &ensure_ascii, + "escape_slash", &escape_slash, + "sort_keys", &sort_keys + + ); + if (ret) + throw ConfigError(json, err, "node-config-format-json", "Failed to parse format configuration"); + + if (indent > JSON_MAX_INDENT) + throw ConfigError(json, "node-config-format-json-indent", "The maximum indentation level is {}", JSON_MAX_INDENT); + + dump_flags = 0; + + if (indent) + dump_flags |= JSON_INDENT(indent); + + if (compact) + dump_flags |= JSON_COMPACT; + + if (ensure_ascii) + dump_flags |= JSON_ENSURE_ASCII; + + if (escape_slash) + dump_flags |= JSON_ESCAPE_SLASH; + + if (sort_keys) + dump_flags |= JSON_SORT_KEYS; + + Format::parse(json); + + if (real_precision) + dump_flags |= JSON_REAL_PRECISION(real_precision); } -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} +static char n[] = "json"; +static char d[] = "Javascript Object Notation"; +static FormatPlugin p; diff --git a/lib/formats/json_kafka.cpp b/lib/formats/json_kafka.cpp index 898399bbc..295a111f7 100644 --- a/lib/formats/json_kafka.cpp +++ b/lib/formats/json_kafka.cpp @@ -20,18 +20,15 @@ * along with this program. If not, see . *********************************************************************************/ -#include - -#include -#include -#include -#include -#include #include -#include -#include +#include +#include +#include -static const char * json_kafka_type_villas_to_kafka(enum SignalType vt) +using namespace villas; +using namespace villas::node; + +const char * JsonKafkaFormat::villasToKafkaType(enum SignalType vt) { switch (vt) { case SignalType::FLOAT: @@ -50,9 +47,9 @@ static const char * json_kafka_type_villas_to_kafka(enum SignalType vt) } } -static int json_kafka_pack_sample(struct io *io, json_t **json_smp, struct sample *smp) +int JsonKafkaFormat::packSample(json_t **json_smp, const struct sample *smp) { - json_t *json_root, *json_schema, *json_payload, *json_fields, *json_field, *json_value; + json_t *json_payload, *json_fields, *json_field, *json_value; json_fields = json_array(); json_payload = json_object(); @@ -66,8 +63,8 @@ static int json_kafka_pack_sample(struct io *io, json_t **json_smp, struct sampl ); uint64_t ts_origin_ms = smp->ts.origin.tv_sec * 1e3 + smp->ts.origin.tv_nsec / 1e6; - json_array_append(json_fields, json_field); - json_object_set(json_payload, "timestamp", json_integer(ts_origin_ms)); + json_array_append_new(json_fields, json_field); + json_object_set_new(json_payload, "timestamp", json_integer(ts_origin_ms)); } /* Include sample sequence no */ @@ -78,44 +75,41 @@ static int json_kafka_pack_sample(struct io *io, json_t **json_smp, struct sampl "field", "sequence" ); - json_array_append(json_fields, json_field); - json_object_set(json_payload, "sequence", json_integer(smp->sequence)); + json_array_append_new(json_fields, json_field); + json_object_set_new(json_payload, "sequence", json_integer(smp->sequence)); } /* Include sample data */ for (size_t i = 0; i < MIN(smp->length, vlist_length(smp->signals)); i++) { struct signal *sig = (struct signal *) vlist_at(smp->signals, i); - union signal_data *data = &smp->data[i]; + const union signal_data *data = &smp->data[i]; json_field = json_pack("{ s: s, s: b, s: s }", - "type", json_kafka_type_villas_to_kafka(sig->type), + "type", villasToKafkaType(sig->type), "optional", false, "field", sig->name ); json_value = signal_data_to_json(data, sig->type); - json_array_append(json_fields, json_field); - json_object_set(json_payload, sig->name, json_value); + json_array_append_new(json_fields, json_field); + json_object_set_new(json_payload, sig->name, json_value); } - json_schema = json_pack("{ s: s, s: s, s: o }", - "type", "struct", - "name", "villas-node.Value", - "fields", json_fields - ); + json_object_set_new(json_schema, "fields", json_fields); - json_root = json_pack("{ s: o, s: o }", + json_incref(json_schema); + *json_smp = json_pack("{ s: o, s: o }", "schema", json_schema, "payload", json_payload - ); - - *json_smp = json_root; + );; + if (*json_smp == nullptr) + return -1; return 0; } -static int json_kafka_unpack_sample(struct io *io, json_t *json_smp, struct sample *smp) +int JsonKafkaFormat::unpackSample(json_t *json_smp, struct sample *smp) { json_t *json_payload, *json_value; @@ -125,7 +119,7 @@ static int json_kafka_unpack_sample(struct io *io, json_t *json_smp, struct samp smp->length = 0; smp->flags = 0; - smp->signals = io->signals; + smp->signals = signals; /* Unpack timestamp */ json_value = json_object_get(json_payload, "timestamp"); @@ -145,8 +139,8 @@ static int json_kafka_unpack_sample(struct io *io, json_t *json_smp, struct samp } /* Unpack signal data */ - for (size_t i = 0; i < vlist_length(io->signals); i++) { - struct signal *sig = (struct signal *) vlist_at(io->signals, i); + for (size_t i = 0; i < vlist_length(signals); i++) { + struct signal *sig = (struct signal *) vlist_at(signals, i); json_value = json_object_get(json_payload, sig->name); if (!json_value) @@ -159,125 +153,44 @@ static int json_kafka_unpack_sample(struct io *io, json_t *json_smp, struct samp if (smp->length > 0) smp->flags |= (int) SampleFlags::HAS_DATA; - return 1; + return 0; } -/* - * Note: The following functions are the same as io/json.c !!! - */ - -int json_kafka_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +void JsonKafkaFormat::parse(json_t *json) { int ret; - json_t *json; - size_t wr; - assert(cnt == 1); - - ret = json_kafka_pack_sample(io, &json, smps[0]); - if (ret < 0) - return ret; - - wr = json_dumpb(json, buf, len, 0); - - json_decref(json); - - if (wbytes) - *wbytes = wr; - - return ret; -} - -int json_kafka_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) -{ - int ret; - json_t *json; json_error_t err; + json_t *json_schema_tmp = nullptr; - assert(cnt == 1); + ret = json_unpack_ex(json, &err, 0, "{ s?: o }", + "schema", &json_schema_tmp + ); + if (ret) + throw ConfigError(json, err, "node-config-format-json-kafka", "Failed to parse format configuration"); - json = json_loadb(buf, len, 0, &err); - if (!json) - return -1; + if (json_schema_tmp) { + if (!json_is_object(json_schema_tmp)) + throw ConfigError(json, "node-config-format-json-kafka-schema", "Kafka schema must be configured as a dictionary"); - ret = json_kafka_unpack_sample(io, json, smps[0]); + if (json_schema) + json_decref(json_schema); - json_decref(json); - - if (ret < 0) - return ret; - - if (rbytes) - *rbytes = err.position; - - return ret; -} - -int json_kafka_print(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - unsigned i; - json_t *json; - - FILE *f = io_stream_output(io); - - for (i = 0; i < cnt; i++) { - ret = json_kafka_pack_sample(io, &json, smps[i]); - if (ret) - return ret; - - ret = json_dumpf(json, f, 0); - fputc('\n', f); - - json_decref(json); - - if (ret) - return ret; + json_schema = json_schema_tmp; } - return i; + JsonFormat::parse(json); } -int json_kafka_scan(struct io *io, struct sample *smps[], unsigned cnt) +JsonKafkaFormat::JsonKafkaFormat(int fl) : + JsonFormat(fl) { - int ret; - unsigned i; - json_t *json; - json_error_t err; - - FILE *f = io_stream_input(io); - - for (i = 0; i < cnt; i++) { -skip: json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); - if (!json) - break; - - ret = json_kafka_unpack_sample(io, json, smps[i]); - - json_decref(json); - - if (ret < 0) - goto skip; - } - - return i; + json_schema = json_pack("{ s: s, s: s }", + "type", "struct", + "name", "villas-node.Value" + ); } -static struct plugin p; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "json.kafka"; - p.description = "JSON Kafka schema/payload messages"; - p.type = PluginType::FORMAT; - p.format.print = json_kafka_print; - p.format.scan = json_kafka_scan; - p.format.sprint = json_kafka_sprint; - p.format.sscan = json_kafka_sscan; - p.format.size = 0; - - vlist_init_and_push(&plugins, &p); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} +static char n[] = "json.kafka"; +static char d[] = "JSON Kafka schema/payload messages"; +static FormatPlugin p; diff --git a/lib/formats/json_reserve.cpp b/lib/formats/json_reserve.cpp index 5a6dd0081..e6f0ac240 100644 --- a/lib/formats/json_reserve.cpp +++ b/lib/formats/json_reserve.cpp @@ -22,21 +22,21 @@ #include -#include -#include -#include -#include -#include #include -#include -#include +#include + +using namespace villas::node; #define JSON_RESERVE_INTEGER_TARGET 1 -static int json_reserve_pack_sample(struct io *io, json_t **j, struct sample *smp) +int JsonReserveFormat::packSample(json_t **json_smp, const struct sample *smp) { json_error_t err; - json_t *json_data, *json_name, *json_unit, *json_value; + json_t *json_root; + json_t *json_data; + json_t *json_name; + json_t *json_unit; + json_t *json_value; json_t *json_created = nullptr; json_t *json_sequence = nullptr; @@ -75,24 +75,19 @@ static int json_reserve_pack_sample(struct io *io, json_t **j, struct sample *sm continue; if (json_unit) - json_object_set(json_value, "unit", json_unit); + json_object_set_new(json_value, "unit", json_unit); if (json_created) - json_object_set(json_value, "created", json_created); + json_object_set_new(json_value, "created", json_created); if (json_sequence) - json_object_set(json_value, "sequence", json_sequence); + json_object_set_new(json_value, "sequence", json_sequence); - json_array_append(json_data, json_value); + json_array_append_new(json_data, json_value); } - if (json_created) - json_decref(json_created); - if (json_sequence) - json_decref(json_sequence); - - *j = json_pack_ex(&err, 0, "{ s: o }", + json_root = json_pack_ex(&err, 0, "{ s: o }", "measurements", json_data ); - if (*j == nullptr) + if (json_root == nullptr) return -1; #if 0 #ifdef JSON_RESERVE_INTEGER_TARGET @@ -106,18 +101,20 @@ static int json_reserve_pack_sample(struct io *io, json_t **j, struct sample *sm if (endptr[0] != 0) return -1; - json_object_set_new(*j, "target", json_integer(id)); + json_object_set_new(json_root, "target", json_integer(id)); } #else if (io->out.node) - json_object_set_new(*j, "target", json_string(io->out.node->name)); + json_object_set_new(json_root, "target", json_string(io->out.node->name)); #endif #endif + *json_smp = json_root; + return 0; } -static int json_reserve_unpack_sample(struct io *io, json_t *json_smp, struct sample *smp) +int JsonReserveFormat::unpackSample(json_t *json_smp, struct sample *smp) { int ret, idx; double created = -1; @@ -185,12 +182,12 @@ static int json_reserve_unpack_sample(struct io *io, json_t *json_smp, struct sa struct signal *sig; - sig = vlist_lookup_name(io->signals, name); + sig = vlist_lookup_name(signals, name); if (sig) { if (!sig->enabled) continue; - idx = vlist_index(io->signals, sig); + idx = vlist_index(signals, sig); } else { ret = sscanf(name, "signal_%d", &idx); @@ -220,122 +217,7 @@ static int json_reserve_unpack_sample(struct io *io, json_t *json_smp, struct sa return smp->length > 0 ? 1 : 0; } -/* - * Note: The following functions are the same as io/json.c !!! - */ -int json_reserve_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) -{ - int ret; - json_t *json; - size_t wr; - - assert(cnt == 1); - - ret = json_reserve_pack_sample(io, &json, smps[0]); - if (ret < 0) - return ret; - - wr = json_dumpb(json, buf, len, 0); - - json_decref(json); - - if (wbytes) - *wbytes = wr; - - return ret; -} - -int json_reserve_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) -{ - int ret; - json_t *json; - json_error_t err; - - assert(cnt == 1); - - json = json_loadb(buf, len, 0, &err); - if (!json) - return -1; - - ret = json_reserve_unpack_sample(io, json, smps[0]); - - json_decref(json); - - if (ret < 0) - return ret; - - if (rbytes) - *rbytes = err.position; - - return ret; -} - -int json_reserve_print(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - unsigned i; - json_t *json; - - FILE *f = io_stream_output(io); - - for (i = 0; i < cnt; i++) { - ret = json_reserve_pack_sample(io, &json, smps[i]); - if (ret) - return ret; - - ret = json_dumpf(json, f, 0); - fputc('\n', f); - - json_decref(json); - - if (ret) - return ret; - } - - return i; -} - -int json_reserve_scan(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - unsigned i; - json_t *json; - json_error_t err; - - FILE *f = io_stream_input(io); - - for (i = 0; i < cnt; i++) { -skip: json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); - if (!json) - break; - - ret = json_reserve_unpack_sample(io, json, smps[i]); - - json_decref(json); - - if (ret < 0) - goto skip; - } - - return i; -} - -static struct plugin p; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "json.reserve"; - p.description = "RESERVE JSON format"; - p.type = PluginType::FORMAT; - p.format.print = json_reserve_print; - p.format.scan = json_reserve_scan; - p.format.sprint = json_reserve_sprint; - p.format.sscan = json_reserve_sscan; - p.format.size = 0; - - vlist_init_and_push(&plugins, &p); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} +static char n[] = "json.reserve"; +static char d[] = "RESERVE JSON format"; +static FormatPlugin p; diff --git a/lib/formats/line.cpp b/lib/formats/line.cpp new file mode 100644 index 000000000..0829878ae --- /dev/null +++ b/lib/formats/line.cpp @@ -0,0 +1,169 @@ +/** Line-based formats + * + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +using namespace villas; +using namespace villas::node; + +int LineFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) +{ + unsigned i; + size_t off = 0; + + for (i = 0; i < cnt && off < len; i++) + off += sprintLine(buf + off, len - off, smps[i]); + + if (wbytes) + *wbytes = off; + + return i; +} + +int LineFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) +{ + unsigned i; + size_t off = 0; + + if (skip_first_line) { + if (!first_line_skipped) { + while (off < len) { + if (buf[off++] == delimiter) + break; + } + + first_line_skipped = true; + } + } + + for (i = 0; i < cnt && off < len; i++) { + /* Skip comment lines */ + if (buf[off] == comment) { + while (off < len) { + if (buf[++off] == delimiter) + break; + } + + if (off == len) + break; + } + + off += sscanLine(buf + off, len - off, smps[i]); + } + + if (rbytes) + *rbytes = off; + + return i; +} + +int LineFormat::print(FILE *f, const struct sample * const smps[], unsigned cnt) +{ + int ret; + unsigned i; + + if (cnt > 0 && smps[0]->signals) + header(f, smps[0]->signals); + + for (i = 0; i < cnt; i++) { + size_t wbytes; + + ret = sprint(out.buffer, out.buflen, &wbytes, &smps[i], 1); + if (ret < 0) + return ret; + + fwrite(out.buffer, wbytes, 1, f); + } + + return i; +} + +int LineFormat::scan(FILE *f, struct sample * const smps[], unsigned cnt) +{ + int ret; + unsigned i; + ssize_t bytes; + + if (skip_first_line) { + if (!first_line_skipped) { + bytes = getdelim(&in.buffer, &in.buflen, delimiter, f); + if (bytes < 0) + return -1; /* An error or eof occured */ + + first_line_skipped = true; + } + } + + for (i = 0; i < cnt; i++) { + size_t rbytes; + char *ptr; + +skip: bytes = getdelim(&in.buffer, &in.buflen, delimiter, f); + if (bytes < 0) + return -1; /* An error or eof occured */ + + /* Skip whitespaces, empty and comment lines */ + for (ptr = in.buffer; isspace(*ptr); ptr++); + + if (ptr[0] == '\0' || ptr[0] == comment) + goto skip; + + ret = sscan(in.buffer, bytes, &rbytes, &smps[i], 1); + if (ret < 0) + return ret; + } + + return i; +} + +void LineFormat::parse(json_t *json) +{ + int ret; + json_error_t err; + const char *delim = nullptr; + int header = -1; + int skip = -1; + + ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: b, s?: b }", + "delimiter", &delim, + "header", &header, + "skip_first_line", &skip + ); + if (ret) + throw ConfigError(json, err, "node-config-format-line", "Failed to parse format configuration"); + + if (delim) { + if (strlen(delim) != 1) + throw ConfigError(json, "node-config-format-line-delimiter", "Line delimiter must be a single character!"); + + delimiter = delim[0]; + } + + if (header >= 0) + print_header = header; + + if (skip >= 0) + skip_first_line = skip; + + Format::parse(json); +} diff --git a/lib/formats/msg.cpp b/lib/formats/msg.cpp index e9811a260..8ac69fea0 100644 --- a/lib/formats/msg.cpp +++ b/lib/formats/msg.cpp @@ -22,8 +22,8 @@ #include -#include -#include +#include +#include #include #include #include @@ -61,7 +61,7 @@ void msg_hdr_ntoh(struct msg *m) m->ts.nsec = ntohl(m->ts.nsec); } -int msg_verify(struct msg *m) +int msg_verify(const struct msg *m) { if (m->version != MSG_VERSION) return -1; @@ -73,7 +73,7 @@ int msg_verify(struct msg *m) return 0; } -int msg_to_sample(struct msg *msg, struct sample *smp, struct vlist *signals) +int msg_to_sample(const struct msg *msg, struct sample *smp, const struct vlist *sigs) { int ret; unsigned i; @@ -83,8 +83,8 @@ int msg_to_sample(struct msg *msg, struct sample *smp, struct vlist *signals) return ret; unsigned len = MIN(msg->length, smp->capacity); - for (i = 0; i < MIN(len, vlist_length(signals)); i++) { - struct signal *sig = (struct signal *) vlist_at_safe(signals, i); + for (i = 0; i < MIN(len, vlist_length(sigs)); i++) { + struct signal *sig = (struct signal *) vlist_at_safe(sigs, i); if (!sig) return -1; @@ -110,7 +110,7 @@ int msg_to_sample(struct msg *msg, struct sample *smp, struct vlist *signals) return 0; } -int msg_from_sample(struct msg *msg_in, struct sample *smp, struct vlist *signals) +int msg_from_sample(struct msg *msg_in, const struct sample *smp, const struct vlist *sigs) { msg_in->type = MSG_TYPE_DATA; msg_in->version = MSG_VERSION; @@ -122,7 +122,7 @@ int msg_from_sample(struct msg *msg_in, struct sample *smp, struct vlist *signal msg_in->ts.nsec = smp->ts.origin.tv_nsec; for (unsigned i = 0; i < smp->length; i++) { - struct signal *sig = (struct signal *) vlist_at_safe(signals, i); + struct signal *sig = (struct signal *) vlist_at_safe(sigs, i); if (!sig) return -1; diff --git a/lib/formats/protobuf.cpp b/lib/formats/protobuf.cpp index a2a0cf286..860c2461e 100644 --- a/lib/formats/protobuf.cpp +++ b/lib/formats/protobuf.cpp @@ -20,20 +20,15 @@ * along with this program. If not, see . *********************************************************************************/ -/* Generated message descriptors by protoc */ -#include - #include #include -#include -#include -#include +#include #include +#include -using namespace villas; -using namespace villas::utils; +using namespace villas::node; -static enum SignalType protobuf_detect_format(Villas__Node__Value *val) +enum SignalType ProtobufFormat::detect(const Villas__Node__Value *val) { switch (val->value_case) { case VILLAS__NODE__VALUE__VALUE_F: @@ -54,7 +49,7 @@ static enum SignalType protobuf_detect_format(Villas__Node__Value *val) } } -int protobuf_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +int ProtobufFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { unsigned psz; @@ -76,16 +71,16 @@ int protobuf_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct villas__node__sample__init(pb_smp); - struct sample *smp = smps[i]; + const struct sample *smp = smps[i]; pb_smp->type = VILLAS__NODE__SAMPLE__TYPE__DATA; - if (io->flags & smp->flags & (int) SampleFlags::HAS_SEQUENCE) { + if (flags & smp->flags & (int) SampleFlags::HAS_SEQUENCE) { pb_smp->has_sequence = 1; pb_smp->sequence = smp->sequence; } - if (io->flags & smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) { + if (flags & smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) { pb_smp->timestamp = new Villas__Node__Timestamp; if (!pb_smp->timestamp) throw MemoryAllocationError(); @@ -162,7 +157,7 @@ out: return -1; } -int protobuf_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) +int ProtobufFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) { unsigned i, j; Villas__Node__Message *pb_msg; @@ -176,7 +171,7 @@ int protobuf_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, s Villas__Node__Sample *pb_smp = pb_msg->samples[i]; smp->flags = 0; - smp->signals = io->signals; + smp->signals = signals; if (pb_smp->type != VILLAS__NODE__SAMPLE__TYPE__DATA) throw RuntimeError("Parsed non supported message type. Skipping"); @@ -195,7 +190,7 @@ int protobuf_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, s for (j = 0; j < MIN(pb_smp->n_values, smp->capacity); j++) { Villas__Node__Value *pb_val = pb_smp->values[j]; - enum SignalType fmt = protobuf_detect_format(pb_val); + enum SignalType fmt = detect(pb_val); struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, j); if (!sig) @@ -240,21 +235,6 @@ int protobuf_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, s return i; } -static struct plugin p; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "protobuf"; - p.description = "Google Protobuf"; - p.type = PluginType::FORMAT; - p.format.sprint = protobuf_sprint; - p.format.sscan = protobuf_sscan; - p.format.flags = (int) IOFlags::HAS_BINARY_PAYLOAD | - (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - - vlist_init_and_push(&plugins, &p); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} - +static char n[] = "protobuf"; +static char d[] = "Google Protobuf"; +static FormatPlugin p; diff --git a/lib/formats/raw.cpp b/lib/formats/raw.cpp index 3b74120dc..857dd265e 100644 --- a/lib/formats/raw.cpp +++ b/lib/formats/raw.cpp @@ -21,16 +21,18 @@ *********************************************************************************/ #include -#include #include -#include -#include +#include #include +#include typedef float flt32_t; typedef double flt64_t; typedef long double flt128_t; /** @todo: check */ +using namespace villas; +using namespace villas::node; + /** Convert double to host byte order */ #define SWAP_FLOAT_XTOH(o, b, n) ({ \ union { flt ## b ## _t f; uint ## b ## _t i; } x = { .f = n }; \ @@ -51,7 +53,7 @@ typedef long double flt128_t; /** @todo: check */ /** Convert integer of varying width to big/little endian byte order */ #define SWAP_INT_HTOX(o, b, n) (o ? htobe ## b (n) : htole ## b (n)) -int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +int RawFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { int o = 0; size_t nlen; @@ -69,16 +71,14 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp __float128 *f128 = (__float128 *) vbuf; #endif - int bits = 1 << (io->flags >> 24); - for (unsigned i = 0; i < cnt; i++) { - struct sample *smp = smps[i]; + const struct sample *smp = smps[i]; /* First three values are sequence, seconds and nano-seconds timestamps * * These fields are always encoded as integers! */ - if (io->flags & RAW_FAKE_HEADER) { + if (fake) { /* Check length */ nlen = (o + 3) * (bits / 8); if (nlen >= len) @@ -92,28 +92,28 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp break; case 16: - i16[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 16, smp->sequence); - i16[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 16, smp->ts.origin.tv_sec); - i16[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 16, smp->ts.origin.tv_nsec); + i16[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 16, smp->sequence); + i16[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 16, smp->ts.origin.tv_sec); + i16[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 16, smp->ts.origin.tv_nsec); break; case 32: - i32[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, smp->sequence); - i32[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, smp->ts.origin.tv_sec); - i32[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, smp->ts.origin.tv_nsec); + i32[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 32, smp->sequence); + i32[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 32, smp->ts.origin.tv_sec); + i32[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 32, smp->ts.origin.tv_nsec); break; case 64: - i64[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, smp->sequence); - i64[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, smp->ts.origin.tv_sec); - i64[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, smp->ts.origin.tv_nsec); + i64[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 64, smp->sequence); + i64[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 64, smp->ts.origin.tv_sec); + i64[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 64, smp->ts.origin.tv_nsec); break; #ifdef HAS_128BIT case 128: - i128[o++] = SWAP_INT_TO_LE(io->flags & RAW_BIG_ENDIAN, 128, smp->sequence); - i128[o++] = SWAP_INT_TO_LE(io->flags & RAW_BIG_ENDIAN, 128, smp->ts.origin.tv_sec); - i128[o++] = SWAP_INT_TO_LE(io->flags & RAW_BIG_ENDIAN, 128, smp->ts.origin.tv_nsec); + i128[o++] = SWAP_INT_TO_LE(endianess == Endianess::BIG, 128, smp->sequence); + i128[o++] = SWAP_INT_TO_LE(endianess == Endianess::BIG, 128, smp->ts.origin.tv_sec); + i128[o++] = SWAP_INT_TO_LE(endianess == Endianess::BIG, 128, smp->ts.origin.tv_nsec); break; #endif } @@ -121,7 +121,7 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp for (unsigned j = 0; j < smp->length; j++) { enum SignalType fmt = sample_format(smp, j); - union signal_data *data = &smp->data[j]; + const union signal_data *data = &smp->data[j]; /* Check length */ nlen = (o + (fmt == SignalType::COMPLEX ? 2 : 1)) * (bits / 8); @@ -140,15 +140,15 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp break; /* Not supported */ case 32: - f32[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, (float) data->f); + f32[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 32, (float) data->f); break; case 64: - f64[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, data->f); + f64[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 64, data->f); break; #ifdef HAS_128BIT - case 128: f128[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 128, data->f); break; + case 128: f128[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 128, data->f); break; #endif } break; @@ -160,20 +160,20 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp break; case 16: - i16[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 16, data->i); + i16[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 16, data->i); break; case 32: - i32[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, data->i); + i32[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 32, data->i); break; case 64: - i64[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, data->i); + i64[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 64, data->i); break; #ifdef HAS_128BIT case 128: - i128[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 128, data->i); + i128[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 128, data->i); break; #endif } @@ -186,20 +186,20 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp break; case 16: - i16[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 16, data->b ? 1 : 0); + i16[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 16, data->b ? 1 : 0); break; case 32: - i32[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, data->b ? 1 : 0); + i32[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 32, data->b ? 1 : 0); break; case 64: - i64[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, data->b ? 1 : 0); + i64[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 64, data->b ? 1 : 0); break; #ifdef HAS_128BIT case 128: - i128[o++] = SWAP_INT_HTOX(io->flags & RAW_BIG_ENDIAN, 128, data->b ? 1 : 0); + i128[o++] = SWAP_INT_HTOX(endianess == Endianess::BIG, 128, data->b ? 1 : 0); break; #endif } @@ -218,18 +218,18 @@ int raw_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct samp break; case 32: - f32[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, (float) std::real(data->z)); - f32[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 32, (float) std::imag(data->z)); + f32[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 32, (float) std::real(data->z)); + f32[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 32, (float) std::imag(data->z)); break; case 64: - f64[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, std::real(data->z)); - f64[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 64, std::imag(data->z)); + f64[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 64, std::real(data->z)); + f64[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 64, std::imag(data->z)); break; #ifdef HAS_128BIT case 128: - f128[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 128, std::real(data->z)); - f128[o++] = SWAP_FLOAT_HTOX(io->flags & RAW_BIG_ENDIAN, 128, std::imag(data->z)); + f128[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 128, std::real(data->z)); + f128[o++] = SWAP_FLOAT_HTOX(endianess == Endianess::BIG, 128, std::imag(data->z)); break; #endif } @@ -247,7 +247,7 @@ out: if (wbytes) return cnt; } -int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) +int RawFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) { void *vbuf = (void *) buf; /* Avoid warning about invalid pointer cast */ @@ -266,7 +266,7 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct * as there is no support for framing. */ struct sample *smp = smps[0]; - int o = 0, bits = 1 << (io->flags >> 24); + int o = 0; int nlen = len / (bits / 8); if (cnt > 1) @@ -275,7 +275,7 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct if (len % (bits / 8)) return -1; /* Invalid RAW Payload length */ - if (io->flags & RAW_FAKE_HEADER) { + if (fake) { if (nlen < o + 3) return -1; /* Received a packet with no fake header. Skipping... */ @@ -287,28 +287,28 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct break; case 16: - smp->sequence = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 16, i16[o++]); - smp->ts.origin.tv_sec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 16, i16[o++]); - smp->ts.origin.tv_nsec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 16, i16[o++]); + smp->sequence = SWAP_INT_XTOH(endianess == Endianess::BIG, 16, i16[o++]); + smp->ts.origin.tv_sec = SWAP_INT_XTOH(endianess == Endianess::BIG, 16, i16[o++]); + smp->ts.origin.tv_nsec = SWAP_INT_XTOH(endianess == Endianess::BIG, 16, i16[o++]); break; case 32: - smp->sequence = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, i32[o++]); - smp->ts.origin.tv_sec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, i32[o++]); - smp->ts.origin.tv_nsec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, i32[o++]); + smp->sequence = SWAP_INT_XTOH(endianess == Endianess::BIG, 32, i32[o++]); + smp->ts.origin.tv_sec = SWAP_INT_XTOH(endianess == Endianess::BIG, 32, i32[o++]); + smp->ts.origin.tv_nsec = SWAP_INT_XTOH(endianess == Endianess::BIG, 32, i32[o++]); break; case 64: - smp->sequence = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, i64[o++]); - smp->ts.origin.tv_sec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, i64[o++]); - smp->ts.origin.tv_nsec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, i64[o++]); + smp->sequence = SWAP_INT_XTOH(endianess == Endianess::BIG, 64, i64[o++]); + smp->ts.origin.tv_sec = SWAP_INT_XTOH(endianess == Endianess::BIG, 64, i64[o++]); + smp->ts.origin.tv_nsec = SWAP_INT_XTOH(endianess == Endianess::BIG, 64, i64[o++]); break; #ifdef HAS_128BIT case 128: - smp->sequence = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, i128[o++]); - smp->ts.origin.tv_sec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, i128[o++]); - smp->ts.origin.tv_nsec = SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, i128[o++]); + smp->sequence = SWAP_INT_XTOH(endianess == Endianess::BIG, 128, i128[o++]); + smp->ts.origin.tv_sec = SWAP_INT_XTOH(endianess == Endianess::BIG, 128, i128[o++]); + smp->ts.origin.tv_nsec = SWAP_INT_XTOH(endianess == Endianess::BIG, 128, i128[o++]); break; #endif } @@ -322,7 +322,7 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct smp->ts.origin.tv_nsec = 0; } - smp->signals = io->signals; + smp->signals = signals; unsigned i; for (i = 0; i < smp->capacity && o < nlen; i++) { @@ -335,10 +335,10 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct case 8: data->f = -1; o++; break; /* Not supported */ case 16: data->f = -1; o++; break; /* Not supported */ - case 32: data->f = SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, f32[o++]); break; - case 64: data->f = SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, f64[o++]); break; + case 32: data->f = SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 32, f32[o++]); break; + case 64: data->f = SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 64, f64[o++]); break; #ifdef HAS_128BIT - case 128: data->f = SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, f128[o++]); break; + case 128: data->f = SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 128, f128[o++]); break; #endif } break; @@ -346,11 +346,11 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct case SignalType::INTEGER: switch (bits) { case 8: data->i = (int8_t) i8[o++]; break; - case 16: data->i = (int16_t) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 16, i16[o++]); break; - case 32: data->i = (int32_t) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, i32[o++]); break; - case 64: data->i = (int64_t) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, i64[o++]); break; + case 16: data->i = (int16_t) SWAP_INT_XTOH(endianess == Endianess::BIG, 16, i16[o++]); break; + case 32: data->i = (int32_t) SWAP_INT_XTOH(endianess == Endianess::BIG, 32, i32[o++]); break; + case 64: data->i = (int64_t) SWAP_INT_XTOH(endianess == Endianess::BIG, 64, i64[o++]); break; #ifdef HAS_128BIT - case 128: data->i = (__int128) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, i128[o++]); break; + case 128: data->i = (__int128) SWAP_INT_XTOH(endianess == Endianess::BIG, 128, i128[o++]); break; #endif } break; @@ -358,11 +358,11 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct case SignalType::BOOLEAN: switch (bits) { case 8: data->b = (bool) i8[o++]; break; - case 16: data->b = (bool) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 16, i16[o++]); break; - case 32: data->b = (bool) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, i32[o++]); break; - case 64: data->b = (bool) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, i64[o++]); break; + case 16: data->b = (bool) SWAP_INT_XTOH(endianess == Endianess::BIG, 16, i16[o++]); break; + case 32: data->b = (bool) SWAP_INT_XTOH(endianess == Endianess::BIG, 32, i32[o++]); break; + case 64: data->b = (bool) SWAP_INT_XTOH(endianess == Endianess::BIG, 64, i64[o++]); break; #ifdef HAS_128BIT - case 128: data->b = (bool) SWAP_INT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, i128[o++]); break; + case 128: data->b = (bool) SWAP_INT_XTOH(endianess == Endianess::BIG, 128, i128[o++]); break; #endif } break; @@ -373,20 +373,20 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct case 16: data->z = std::complex(-1, -1); o += 2; break; /* Not supported */ case 32: data->z = std::complex( - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, f32[o++]), - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 32, f32[o++])); + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 32, f32[o++]), + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 32, f32[o++])); break; case 64: data->z = std::complex( - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, f64[o++]), - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 64, f64[o++])); + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 64, f64[o++]), + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 64, f64[o++])); break; #if HAS_128BIT case 128: data->z = std::complex( - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, f128[o++]), - SWAP_FLOAT_XTOH(io->flags & RAW_BIG_ENDIAN, 128, f128[o++])); + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 128, f128[o++]), + SWAP_FLOAT_XTOH(endianess == Endianess::BIG, 128, f128[o++])); break; #endif } @@ -408,37 +408,50 @@ int raw_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct return 1; } -#define REGISTER_FORMAT_RAW(i, n, d, f) \ -static struct plugin i; \ -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { \ - i.name = n; \ - i.description = d; \ - i.type = PluginType::FORMAT; \ - i.format.sprint = raw_sprint; \ - i.format.sscan = raw_sscan; \ - i.format.flags = f | (int) IOFlags::HAS_BINARY_PAYLOAD | \ - (int) SampleFlags::HAS_DATA; \ - \ - vlist_push(&plugins, &i); \ -} \ - \ -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { \ - vlist_remove_all(&plugins, &i); \ +void RawFormat::parse(json_t *json) +{ + int ret; + json_error_t err; + + int fake_tmp = 0; + const char *end = nullptr; + + ret = json_unpack_ex(json, &err, 0, "{ }", + "endianess", &end, + "fake", &fake_tmp, + "bits", &bits + ); + if (ret) + throw ConfigError(json, err, "node-config-format-raw", "Failed to parse format configuration"); + + if (bits % 8 != 0 || bits > 128 || bits <= 0) + throw ConfigError(json, "node-config-format-raw-bits", "Failed to parse format configuration"); + + if (end) { + if (bits <= 8) + throw ConfigError(json, "node-config-format-raw-endianess", "An endianess settings must only provided for bits > 8"); + + if (!strcmp(end, "little")) + endianess = Endianess::LITTLE; + else if (!strcmp(end, "big")) + endianess = Endianess::BIG; + else + throw ConfigError(json, "node-config-format-raw-endianess", "Endianess must be either 'big' or 'little'"); + } + + fake = fake_tmp; + if (fake) + flags |= (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_TS_ORIGIN; + else + flags &= ~((int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_TS_ORIGIN); + + BinaryFormat::parse(json); } -/* Feel free to add additional format identifiers here to suit your needs */ -REGISTER_FORMAT_RAW(p_8, "raw.8", "Raw 8 bit", RAW_BITS_8) -REGISTER_FORMAT_RAW(p_16be, "raw.16.be", "Raw 16 bit, big endian byte-order", RAW_BITS_16 | RAW_BIG_ENDIAN) -REGISTER_FORMAT_RAW(p_32be, "raw.32.be", "Raw 32 bit, big endian byte-order", RAW_BITS_32 | RAW_BIG_ENDIAN) -REGISTER_FORMAT_RAW(p_64be, "raw.64.be", "Raw 64 bit, big endian byte-order", RAW_BITS_64 | RAW_BIG_ENDIAN) -REGISTER_FORMAT_RAW(p_16le, "raw.16.le", "Raw 16 bit, little endian byte-order", RAW_BITS_16) -REGISTER_FORMAT_RAW(p_32le, "raw.32.le", "Raw 32 bit, little endian byte-order", RAW_BITS_32) -REGISTER_FORMAT_RAW(p_64le, "raw.64.le", "Raw 64 bit, little endian byte-order", RAW_BITS_64) +static char n1[] = "raw"; +static char d1[] = "Raw binary data"; +static FormatPlugin p1; -#ifdef HAS_128BIT -REGISTER_FORMAT_RAW(p_128le, "raw.128.be", "Raw 128 bit, big endian byte-order", RAW_BITS_128 | RAW_BIG_ENDIAN) -REGISTER_FORMAT_RAW(p_128le, "raw.128.le", "Raw 128 bit, little endian byte-order", RAW_BITS_128) -#endif - -REGISTER_FORMAT_RAW(p_gtnet, "gtnet", "RTDS GTNET", RAW_BITS_32 | RAW_BIG_ENDIAN) -REGISTER_FORMAT_RAW(p_gtnef, "gtnet.fake", "RTDS GTNET with fake header", RAW_BITS_32 | RAW_BIG_ENDIAN | RAW_FAKE_HEADER) +static char n2[] = "gtnet"; +static char d2[] = "RTDS GTNET"; +static FormatPlugin p2; diff --git a/lib/formats/value.cpp b/lib/formats/value.cpp index ce57a7e8c..a6721535c 100644 --- a/lib/formats/value.cpp +++ b/lib/formats/value.cpp @@ -20,20 +20,18 @@ * along with this program. If not, see . *********************************************************************************/ -#include -#include -#include +#include #include #include -using namespace villas::utils; +using namespace villas::node; -int value_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +int ValueFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { unsigned i; size_t off = 0; struct signal *sig; - struct sample *smp = smps[0]; + const struct sample *smp = smps[0]; assert(cnt == 1); assert(smp->length <= 1); @@ -45,7 +43,7 @@ int value_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sa if (!sig) return -1; - off += signal_data_print_str(&smp->data[i], sig->type, buf, len); + off += signal_data_print_str(&smp->data[i], sig->type, buf, len, real_precision); off += snprintf(buf + off, len - off, "\n"); } @@ -55,7 +53,7 @@ int value_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sa return i; } -int value_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) +int ValueFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) { unsigned i = 0, ret; struct sample *smp = smps[0]; @@ -68,7 +66,7 @@ int value_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, stru printf("Reading: %s", buf); if (smp->capacity >= 1) { - struct signal *sig = (struct signal *) vlist_at_safe(io->signals, i); + struct signal *sig = (struct signal *) vlist_at_safe(signals, i); if (!sig) return -1; @@ -81,7 +79,7 @@ int value_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, stru } out: smp->flags = 0; - smp->signals = io->signals; + smp->signals = signals; smp->length = i; if (smp->length > 0) smp->flags |= (int) SampleFlags::HAS_DATA; @@ -92,21 +90,6 @@ out: smp->flags = 0; return i; } -static struct plugin p; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "value"; - p.description = "A bare text value without any headers"; - p.type = PluginType::FORMAT; - p.format.sprint = value_sprint; - p.format.sscan = value_sscan; - p.format.size = 0; - p.format.flags = (int) SampleFlags::HAS_DATA | (int) IOFlags::NEWLINES; - - vlist_init_and_push(&plugins, &p);; -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - if (plugins.state != State::DESTROYED) - vlist_remove_all(&plugins, &p); -} +static char n[] = "value"; +static char d[] = "A bare text value without any headers"; +static FormatPlugin p; diff --git a/lib/formats/villas_binary.cpp b/lib/formats/villas_binary.cpp index d21f9d31a..2dae9c85d 100644 --- a/lib/formats/villas_binary.cpp +++ b/lib/formats/villas_binary.cpp @@ -21,16 +21,17 @@ *********************************************************************************/ #include +#include -#include -#include -#include -#include +#include +#include +#include #include #include -#include -int villas_binary_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +using namespace villas::node; + +int VillasBinaryFormat::sprint(char *buf, size_t len, size_t *wbytes, const struct sample * const smps[], unsigned cnt) { int ret; unsigned i = 0; @@ -38,7 +39,7 @@ int villas_binary_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, s for (i = 0; i < cnt; i++) { struct msg *msg = (struct msg *) ptr; - struct sample *smp = smps[i]; + const struct sample *smp = smps[i]; if (ptr + MSG_LEN(smp->length) > buf + len) break; @@ -47,7 +48,7 @@ int villas_binary_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, s if (ret) return ret; - if (io->flags & VILLAS_BINARY_WEB) { + if (web) { /** @todo convert to little endian */ } else @@ -62,7 +63,7 @@ int villas_binary_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, s return i; } -int villas_binary_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) +int VillasBinaryFormat::sscan(const char *buf, size_t len, size_t *rbytes, struct sample * const smps[], unsigned cnt) { int ret, values; unsigned i = 0; @@ -75,7 +76,7 @@ int villas_binary_sscan(struct io *io, const char *buf, size_t len, size_t *rbyt struct msg *msg = (struct msg *) ptr; struct sample *smp = smps[i]; - smp->signals = io->signals; + smp->signals = signals; /* Complete buffer has been parsed */ if (ptr == buf + len) @@ -85,19 +86,19 @@ int villas_binary_sscan(struct io *io, const char *buf, size_t len, size_t *rbyt if (ptr + sizeof(struct msg) > buf + len) return -2; /* Invalid msg received */ - values = (io->flags & VILLAS_BINARY_WEB) ? msg->length : ntohs(msg->length); + values = web ? msg->length : ntohs(msg->length); /* Check if remainder of message is in buffer boundaries */ if (ptr + MSG_LEN(values) > buf + len) return -3; /*Invalid msg receive */ - if (io->flags & VILLAS_BINARY_WEB) { + if (web) { /** @todo convert from little endian */ } else msg_ntoh(msg); - ret = msg_to_sample(msg, smp, io->signals); + ret = msg_to_sample(msg, smp, signals); if (ret) return ret; /* Invalid msg received */ @@ -110,41 +111,5 @@ int villas_binary_sscan(struct io *io, const char *buf, size_t len, size_t *rbyt return i; } -static struct plugin p1; - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p1.name = "villas.binary"; - p1.description = "VILLAS binary network format"; - p1.type = PluginType::FORMAT; - p1.format.sprint = villas_binary_sprint; - p1.format.sscan = villas_binary_sscan; - p1.format.size = 0; - p1.format.flags = (int) IOFlags::HAS_BINARY_PAYLOAD | - (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - - vlist_push(&plugins, &p1); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p1); -} -/** The WebSocket node-type usually uses little endian byte order intead of network byte order */ -static struct plugin p2; - - -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p2.name = "villas.web"; - p2.description = "VILLAS binary network format for WebSockets"; - p2.type = PluginType::FORMAT; - p2.format.sprint = villas_binary_sprint; - p2.format.sscan = villas_binary_sscan; - p2.format.size = 0; - p2.format.flags = (int) IOFlags::HAS_BINARY_PAYLOAD | VILLAS_BINARY_WEB | - (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - - vlist_push(&plugins, &p2); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p2); -} +static VillasBinaryFormatPlugin p1; +static VillasBinaryFormatPlugin p2; diff --git a/lib/formats/villas_human.cpp b/lib/formats/villas_human.cpp index 1786af822..de0aab0b5 100644 --- a/lib/formats/villas_human.cpp +++ b/lib/formats/villas_human.cpp @@ -23,20 +23,20 @@ #include #include -#include -#include #include #include #include #include -#include +#include -static size_t villas_human_sprint_single(struct io *io, char *buf, size_t len, const struct sample *smp) +using namespace villas::node; + +size_t VILLASHumanFormat::sprintLine(char *buf, size_t len, const struct sample *smp) { size_t off = 0; struct signal *sig; - if (io->flags & (int) SampleFlags::HAS_TS_ORIGIN) { + if (flags & (int) SampleFlags::HAS_TS_ORIGIN) { if (smp->flags & (int) SampleFlags::HAS_TS_ORIGIN) { off += snprintf(buf + off, len - off, "%llu", (unsigned long long) smp->ts.origin.tv_sec); off += snprintf(buf + off, len - off, ".%09llu", (unsigned long long) smp->ts.origin.tv_nsec); @@ -45,33 +45,33 @@ static size_t villas_human_sprint_single(struct io *io, char *buf, size_t len, c off += snprintf(buf + off, len - off, "nan.nan"); } - if (io->flags & (int) SampleFlags::HAS_OFFSET) { + if (flags & (int) SampleFlags::HAS_OFFSET) { if (smp->flags & (int) SampleFlags::HAS_TS_RECEIVED) off += snprintf(buf + off, len - off, "%+e", time_delta(&smp->ts.origin, &smp->ts.received)); } - if (io->flags & (int) SampleFlags::HAS_SEQUENCE) { + if (flags & (int) SampleFlags::HAS_SEQUENCE) { if (smp->flags & (int) SampleFlags::HAS_SEQUENCE) off += snprintf(buf + off, len - off, "(%" PRIu64 ")", smp->sequence); } - if (io->flags & (int) SampleFlags::HAS_DATA) { + if (flags & (int) SampleFlags::HAS_DATA) { for (unsigned i = 0; i < smp->length; i++) { sig = (struct signal *) vlist_at_safe(smp->signals, i); if (!sig) break; - off += snprintf(buf + off, len - off, "%c", io->separator); - off += signal_data_print_str(&smp->data[i], sig->type, buf + off, len - off); + off += snprintf(buf + off, len - off, "\t"); + off += signal_data_print_str(&smp->data[i], sig->type, buf + off, len - off, real_precision); } } - off += snprintf(buf + off, len - off, "%c", io->delimiter); + off += snprintf(buf + off, len - off, "%c", delimiter); return off; } -static size_t villas_human_sscan_single(struct io *io, const char *buf, size_t len, struct sample *smp) +size_t VILLASHumanFormat::sscanLine(const char *buf, size_t len, struct sample *smp) { int ret; char *end; @@ -80,7 +80,7 @@ static size_t villas_human_sscan_single(struct io *io, const char *buf, size_t l double offset = 0; smp->flags = 0; - smp->signals = io->signals; + smp->signals = signals; /* Format: Seconds.NanoSeconds+Offset(SequenceNumber) Value1 Value2 ... * RegEx: (\d+(?:\.\d+)?)([-+]\d+(?:\.\d+)?(?:e[+-]?\d+)?)?(?:\((\d+)\))? @@ -90,7 +90,7 @@ static size_t villas_human_sscan_single(struct io *io, const char *buf, size_t l /* Mandatory: seconds */ smp->ts.origin.tv_sec = (uint32_t) strtoul(ptr, &end, 10); - if (ptr == end || *end == io->delimiter) + if (ptr == end || *end == delimiter) return -1; smp->flags |= (int) SampleFlags::HAS_TS_ORIGIN; @@ -133,10 +133,10 @@ static size_t villas_human_sscan_single(struct io *io, const char *buf, size_t l unsigned i; for (ptr = end + 1, i = 0; i < smp->capacity; ptr = end + 1, i++) { - if (*end == io->delimiter) + if (*end == delimiter) goto out; - struct signal *sig = (struct signal *) vlist_at_safe(io->signals, i); + struct signal *sig = (struct signal *) vlist_at_safe(signals, i); if (!sig) goto out; @@ -145,7 +145,7 @@ static size_t villas_human_sscan_single(struct io *io, const char *buf, size_t l goto out; } -out: if (*end == io->delimiter) +out: if (*end == delimiter) end++; smp->length = i; @@ -162,84 +162,44 @@ out: if (*end == io->delimiter) return end - buf; } -int villas_human_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) +void VILLASHumanFormat::header(FILE *f, const struct vlist *sigs) { - unsigned i; - size_t off = 0; - - for (i = 0; i < cnt && off < len; i++) - off += villas_human_sprint_single(io, buf + off, len - off, smps[i]); - - if (wbytes) - *wbytes = off; - - return i; -} - -int villas_human_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) -{ - unsigned i; - size_t off = 0; - - for (i = 0; i < cnt && off < len; i++) - off += villas_human_sscan_single(io, buf + off, len - off, smps[i]); - - if (rbytes) - *rbytes = off; - - return i; -} - -void villas_human_header(struct io *io, const struct sample *smp) -{ - FILE *f = io_stream_output(io); + /* Abort if we are not supposed to, or have already printed the header */ + if (!print_header || header_printed) + return; fprintf(f, "# "); - if (io->flags & (int) SampleFlags::HAS_TS_ORIGIN) + if (flags & (int) SampleFlags::HAS_TS_ORIGIN) fprintf(f, "seconds.nanoseconds"); - if (io->flags & (int) SampleFlags::HAS_OFFSET) + if (flags & (int) SampleFlags::HAS_OFFSET) fprintf(f, "+offset"); - if (io->flags & (int) SampleFlags::HAS_SEQUENCE) + if (flags & (int) SampleFlags::HAS_SEQUENCE) fprintf(f, "(sequence)"); - if (io->flags & (int) SampleFlags::HAS_DATA) { - for (unsigned i = 0; i < MIN(smp->length, vlist_length(smp->signals)); i++) { - struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, i); + if (flags & (int) SampleFlags::HAS_DATA) { + for (unsigned i = 0; i < vlist_length(sigs); i++) { + struct signal *sig = (struct signal *) vlist_at_safe(sigs, i); if (!sig) break; if (sig->name) - fprintf(f, "%c%s", io->separator, sig->name); + fprintf(f, "\t%s", sig->name); else - fprintf(f, "%csignal%u", io->separator, i); + fprintf(f, "\tsignal%u", i); if (sig->unit) fprintf(f, "[%s]", sig->unit); } } - fprintf(f, "%c", io->delimiter); + fprintf(f, "%c", delimiter); + + LineFormat::header(f, sigs); } -static struct plugin p; -__attribute__((constructor(110))) static void UNIQUE(__ctor)() { - p.name = "villas.human"; - p.description = "VILLAS human readable format"; - p.type = PluginType::FORMAT; - p.format.header = villas_human_header; - p.format.sprint = villas_human_sprint; - p.format.sscan = villas_human_sscan; - p.format.size = 0; - p.format.flags = (int) IOFlags::NEWLINES | (int) SampleFlags::HAS_TS_ORIGIN | (int) SampleFlags::HAS_SEQUENCE | (int) SampleFlags::HAS_DATA; - p.format.delimiter = '\n'; - p.format.separator = '\t'; - - vlist_init_and_push(&plugins, &p); -} - -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { - vlist_remove_all(&plugins, &p); -} +static char n[] = "villas.human"; +static char d[] = "VILLAS human readable format"; +static LineFormatPlugin p; diff --git a/lib/hook_list.cpp b/lib/hook_list.cpp index 5597a3e69..0296c610b 100644 --- a/lib/hook_list.cpp +++ b/lib/hook_list.cpp @@ -135,7 +135,7 @@ skip_add: } } -int hook_list_process(struct vlist *hs, struct sample *smps[], unsigned cnt) +int hook_list_process(struct vlist *hs, struct sample * smps[], unsigned cnt) { unsigned current, processed = 0; diff --git a/lib/hooks/dft.cpp b/lib/hooks/dft.cpp index ea2545d76..3881fe346 100644 --- a/lib/hooks/dft.cpp +++ b/lib/hooks/dft.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include namespace villas { @@ -63,8 +63,6 @@ protected: enum WindowType windowType; enum PaddingType paddingType; - struct format_type *format; - std::vector> smpMemory; std::vector>> dftMatrix; std::vector> dftResults; @@ -120,8 +118,6 @@ public: lastDftCal({0, 0}), signalIndex() { - format = format_type_lookup("villas.human"); - bool debug = false; if (debug) { origSigSync = std::make_shared("/tmp/plot/origSigSync"); diff --git a/lib/hooks/print.cpp b/lib/hooks/print.cpp index 06c390ef1..4df798764 100644 --- a/lib/hooks/print.cpp +++ b/lib/hooks/print.cpp @@ -24,12 +24,14 @@ * @{ */ +#include #include +#include #include #include #include -#include +#include #include namespace villas { @@ -38,85 +40,79 @@ namespace node { class PrintHook : public Hook { protected: - struct io io; - struct format_type *format; + Format *formatter; + FILE *stream; - char *prefix; - char *uri; + std::string prefix; + std::string uri; public: PrintHook(struct vpath *p, struct vnode *n, int fl, int prio, bool en = true) : Hook(p, n, fl, prio, en), - io(), - prefix(nullptr), - uri(nullptr) + formatter(nullptr), + stream(nullptr) + { } + + virtual ~PrintHook() { - format = format_type_lookup("villas.human"); + if (formatter) + delete formatter; } virtual void start() { - int ret; - assert(state == State::PREPARED || state == State::STOPPED); - ret = io_init(&io, format, &signals, (int) SampleFlags::HAS_ALL); - if (ret) - throw RuntimeError("Failed to initialze IO"); + formatter->start(&signals); - ret = io_open(&io, uri); - if (ret) - throw RuntimeError("Failed to open IO"); + stream = !uri.empty() ? fopen(uri.c_str(), "a+") : stdout; + if (!stream) + throw SystemError("Failed to open IO"); state = State::STARTED; } virtual void stop() { - int ret; - assert(state == State::STARTED); - ret = io_close(&io); - if (ret) - throw RuntimeError("Failed to close IO"); - - ret = io_destroy(&io); - if (ret) - throw RuntimeError("Failed to destroy IO"); + if (stream != stdout) + fclose(stream); state = State::STOPPED; } virtual void parse(json_t *json) { - const char *f = nullptr, *p = nullptr, *u = nullptr; + const char *p = nullptr, *u = nullptr; int ret; json_error_t err; + json_t *json_format = nullptr; assert(state != State::STARTED); Hook::parse(json); - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s }", + ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: o }", "output", &u, "prefix", &p, - "format", &f + "format", &json_format ); if (ret) throw ConfigError(json, err, "node-config-hook-print"); if (p) - prefix = strdup(p); + prefix = p; if (u) - uri = strdup(u); + uri = u; - if (f) { - format = format_type_lookup(f); - if (!format) - throw ConfigError(json, "node-config-hook-print-format", "Invalid IO format '{}'", f); - } + /* Format */ + formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.human"); + if (!formatter) + throw ConfigError(json_format, "node-config-hook-print-format", "Invalid format configuration"); state = State::PARSED; } @@ -125,26 +121,19 @@ public: { assert(state == State::STARTED); - if (prefix) - printf("%s", prefix); - else if (node) - printf("Node %s: ", node_name(node)); - else if (path) - printf("Path %s: ", path_name(path)); + if (stream == stdout) { + if (!prefix.empty()) + printf("%s", prefix.c_str()); + else if (node) + printf("Node %s: ", node_name(node)); + else if (path) + printf("Path %s: ", path_name(path)); + } - io_print(&io, &smp, 1); + formatter->print(stream, smp); return Reason::OK; } - - virtual ~PrintHook() - { - if (uri) - free(uri); - - if (prefix) - free(prefix); - } }; /* Register hook */ diff --git a/lib/io.cpp b/lib/io.cpp deleted file mode 100644 index 47f4f0402..000000000 --- a/lib/io.cpp +++ /dev/null @@ -1,476 +0,0 @@ -/** Reading and writing simulation samples in various formats. - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -using namespace villas; -using namespace villas::utils; - -static int io_print_lines(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - unsigned i; - - FILE *f = io_stream_output(io); - - for (i = 0; i < cnt; i++) { - size_t wbytes; - - ret = io_sprint(io, io->out.buffer, io->out.buflen, &wbytes, &smps[i], 1); - if (ret < 0) - return ret; - - fwrite(io->out.buffer, wbytes, 1, f); - } - - return i; -} - -static int io_scan_lines(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - unsigned i; - - FILE *f = io_stream_input(io); - - for (i = 0; i < cnt; i++) { - size_t rbytes; - ssize_t bytes; - char *ptr; - -skip: bytes = getdelim(&io->in.buffer, &io->in.buflen, io->delimiter, f); - if (bytes < 0) - return -1; /* An error or eof occured */ - - /* Skip whitespaces, empty and comment lines */ - for (ptr = io->in.buffer; isspace(*ptr); ptr++); - - if (ptr[0] == '\0' || ptr[0] == '#') - goto skip; - - ret = io_sscan(io, io->in.buffer, bytes, &rbytes, &smps[i], 1); - if (ret < 0) - return ret; - } - - return i; -} - -int io_init(struct io *io, const struct format_type *fmt, struct vlist *signals, int flags) -{ - int ret; - - io->_vt = fmt; - io->_vd = new char[fmt->size]; - if (!io->_vd) - throw MemoryAllocationError(); - - io->flags = flags | (io_type(io)->flags & ~(int) SampleFlags::HAS_ALL); - io->delimiter = io_type(io)->delimiter ? io_type(io)->delimiter : '\n'; - io->separator = io_type(io)->separator ? io_type(io)->separator : '\t'; - - io->in.buflen = - io->out.buflen = 4096; - - io->in.buffer = new char[io->in.buflen]; - io->out.buffer = new char[io->out.buflen]; - - if (!io->in.buffer || !io->out.buffer) - throw MemoryAllocationError(); - - io->signals = signals; - - ret = io_type(io)->init ? io_type(io)->init(io) : 0; - if (ret) - return ret; - - io->state = State::INITIALIZED; - - return 0; -} - -int io_init2(struct io *io, const struct format_type *fmt, const char *dt, int flags) -{ - int ret; - struct vlist *signals; - - signals = new struct vlist; - if (!signals) - throw MemoryAllocationError(); - - ret = vlist_init(signals); - if (ret) - return ret; - - ret = signal_list_generate2(signals, dt); - if (ret) - return ret; - - flags |= (int) IOFlags::DESTROY_SIGNALS; - - return io_init(io, fmt, signals, flags); -} - -int io_destroy(struct io *io) -{ - int ret; - - assert(io->state == State::CLOSED || io->state == State::INITIALIZED); - - ret = io_type(io)->destroy ? io_type(io)->destroy(io) : 0; - if (ret) - return ret; - - delete[] (char *) io->_vd; - delete[] io->in.buffer; - delete[] io->out.buffer; - - if (io->flags & (int) IOFlags::DESTROY_SIGNALS) { - ret = vlist_destroy(io->signals, (dtor_cb_t) signal_decref, false); - if (ret) - return ret; - } - - return 0; -} - -int io_stream_open(struct io *io, const char *uri) -{ - int ret; - - if (uri) { - if (!strcmp(uri, "-")) { - goto stdio; - } - else { - io->mode = IOMode::STDIO; - - io->out.stream = fopen(uri, "a+"); - if (io->out.stream == nullptr) - return -1; - - io->in.stream = fopen(uri, "r"); - if (io->in.stream == nullptr) - return -1; - } - } - else { -stdio: io->mode = IOMode::STDIO; - io->flags |= (int) IOFlags::FLUSH; - - io->in.stream = stdin; - io->out.stream = stdout; - } - - /* Make stream non-blocking if desired */ - if (io->flags & (int) IOFlags::NONBLOCK) { - int ret, fd, flags; - - fd = io_fd(io); - if (fd < 0) - return fd; - - flags = fcntl(fd, F_GETFL); - if (flags < 0) - return flags; - - flags |= O_NONBLOCK; - - ret = fcntl(fd, F_SETFL, flags); - if (ret) - return ret; - } - - /* Enable line buffering on stdio */ - if (io->mode == IOMode::STDIO) { - ret = setvbuf(io->in.stream, nullptr, _IOLBF, BUFSIZ); - if (ret) - return -1; - - ret = setvbuf(io->out.stream, nullptr, _IOLBF, BUFSIZ); - if (ret) - return -1; - } - - return 0; -} - -int io_stream_close(struct io *io) -{ - int ret; - - switch (io->mode) { - case IOMode::STDIO: - if (io->in.stream == stdin) - return 0; - - ret = fclose(io->in.stream); - if (ret) - return ret; - - ret = fclose(io->out.stream); - if (ret) - return ret; - - return 0; - - case IOMode::CUSTOM: - return 0; - } - - return -1; -} - -int io_stream_flush(struct io *io) -{ - switch (io->mode) { - case IOMode::STDIO: - return fflush(io->out.stream); - case IOMode::CUSTOM: - return 0; - } - - return -1; -} - -int io_stream_eof(struct io *io) -{ - switch (io->mode) { - case IOMode::STDIO: - return feof(io->in.stream); - case IOMode::CUSTOM: - return 0; - } - - return -1; -} - -void io_stream_rewind(struct io *io) -{ - switch (io->mode) { - case IOMode::STDIO: - rewind(io->in.stream); - break; - case IOMode::CUSTOM: { } - } -} - -int io_stream_fd(struct io *io) -{ - switch (io->mode) { - case IOMode::STDIO: - return fileno(io->in.stream); - case IOMode::CUSTOM: - return -1; - } - - return -1; -} - -int io_open(struct io *io, const char *uri) -{ - int ret; - - assert(io->state == State::INITIALIZED || io->state == State::CLOSED); - - ret = io_type(io)->open - ? io_type(io)->open(io, uri) - : io_stream_open(io, uri); - if (ret) - return ret; - - io->header_printed = false; - io->state = State::OPENED; - - return 0; -} - -int io_close(struct io *io) -{ - int ret; - - assert(io->state == State::OPENED); - - io_footer(io); - - ret = io_type(io)->close - ? io_type(io)->close(io) - : io_stream_close(io); - if (ret) - return ret; - - io->state = State::CLOSED; - - return 0; -} - -int io_flush(struct io *io) -{ - assert(io->state == State::OPENED); - - return io_type(io)->flush - ? io_type(io)->flush(io) - : io_stream_flush(io); -} - -int io_eof(struct io *io) -{ - assert(io->state == State::OPENED); - - return io_type(io)->eof - ? io_type(io)->eof(io) - : io_stream_eof(io); -} - -void io_rewind(struct io *io) -{ - assert(io->state == State::OPENED); - - if (io_type(io)->rewind) - io_type(io)->rewind(io); - else - io_stream_rewind(io); -} - -int io_fd(struct io *io) -{ - assert(io->state == State::OPENED); - - return io_type(io)->fd - ? io_type(io)->fd(io) - : io_stream_fd(io); -} - -const struct format_type * io_type(struct io *io) -{ - return io->_vt; -} - -void io_header(struct io *io, const struct sample *smp) -{ - assert(io->state == State::OPENED); - - if (io_type(io)->header) - io_type(io)->header(io, smp); - - io->header_printed = true; -} - -void io_footer(struct io *io) -{ - assert(io->state == State::OPENED); - - if (io_type(io)->footer) - io_type(io)->footer(io); -} - -int io_print(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - - assert(io->state == State::OPENED); - - if (!io->header_printed && cnt > 0) - io_header(io, smps[0]); - - if (io->flags & (int) IOFlags::NEWLINES) - ret = io_print_lines(io, smps, cnt); - else if (io_type(io)->print) - ret = io_type(io)->print(io, smps, cnt); - else if (io_type(io)->sprint) { - FILE *f = io_stream_output(io); - size_t wbytes; - - ret = io_sprint(io, io->out.buffer, io->out.buflen, &wbytes, smps, cnt); - - fwrite(io->out.buffer, wbytes, 1, f); - } - else - ret = -1; - - if (io->flags & (int) IOFlags::FLUSH) - io_flush(io); - - return ret; -} - -int io_scan(struct io *io, struct sample *smps[], unsigned cnt) -{ - int ret; - - assert(io->state == State::OPENED); - - if (io->flags & (int) IOFlags::NEWLINES) - ret = io_scan_lines(io, smps, cnt); - else if (io_type(io)->scan) - ret = io_type(io)->scan(io, smps, cnt); - else if (io_type(io)->sscan) { - FILE *f = io_stream_input(io); - size_t bytes, rbytes; - - bytes = fread(io->in.buffer, 1, io->in.buflen, f); - - ret = io_sscan(io, io->in.buffer, bytes, &rbytes, smps, cnt); - } - else - ret = -1; - - return ret; -} - -FILE * io_stream_output(struct io *io) { - if (io->state != State::OPENED) - return 0; - - return io->out.stream; -} - -FILE * io_stream_input(struct io *io) { - if (io->state != State::OPENED) - return 0; - - return io->in.stream; -} - -int io_sscan(struct io *io, const char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt) -{ - assert(io->state == State::INITIALIZED || io->state == State::OPENED || io->state == State::CLOSED); - - return io_type(io)->sscan ? io_type(io)->sscan(io, buf, len, rbytes, smps, cnt) : -1; -} - -int io_sprint(struct io *io, char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt) -{ - assert(io->state == State::INITIALIZED || io->state == State::OPENED || io->state == State::CLOSED); - - return io_type(io)->sprint ? io_type(io)->sprint(io, buf, len, wbytes, smps, cnt) : -1; -} diff --git a/lib/node.cpp b/lib/node.cpp index 7878dbeaf..0b8ea15e5 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -69,7 +69,6 @@ int node_init(struct vnode *n, struct vnode_type *vt) uuid_clear(n->uuid); - n->output_path = nullptr; n->name = nullptr; n->_name = nullptr; n->_name_long = nullptr; @@ -444,7 +443,7 @@ int node_destroy(struct vnode *n) return 0; } -int node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int node_read(struct vnode *n, struct sample * smps[], unsigned cnt) { int toread, readd, nread = 0; unsigned vect; @@ -462,7 +461,7 @@ int node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re while (cnt - nread > 0) { toread = MIN(cnt - nread, vect); - readd = node_type(n)->read(n, &smps[nread], toread, release); + readd = node_type(n)->read(n, &smps[nread], toread); if (readd < 0) return readd; @@ -494,7 +493,7 @@ int node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re #endif /* WITH_HOOKS */ } -int node_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int node_write(struct vnode *n, struct sample * smps[], unsigned cnt) { int tosend, sent, nsent = 0; unsigned vect; @@ -519,7 +518,7 @@ int node_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r while (cnt - nsent > 0) { tosend = MIN(cnt - nsent, vect); - sent = node_type(n)->write(n, &smps[nsent], tosend, release); + sent = node_type(n)->write(n, &smps[nsent], tosend); if (sent < 0) return sent; @@ -558,8 +557,8 @@ char * node_name_long(struct vnode *n) strcatf(&n->_name_long, ", fwmark=%d", n->fwmark); #endif /* WITH_NETEM */ - if (n->output_path) - strcatf(&n->_name_long, ", output_path=%s", path_name(n->output_path)); + if (n->out.path) + strcatf(&n->_name_long, ", out.path=%s", path_name(n->out.path)); if (node_type(n)->print) { struct vnode_type *vt = node_type(n); @@ -675,8 +674,8 @@ struct vlist * node_input_signals(struct vnode *n) struct vlist * node_output_signals(struct vnode *n) { - if (n->output_path) - return path_output_signals(n->output_path); + if (n->out.path) + return path_output_signals(n->out.path); return nullptr; } diff --git a/lib/node_direction.cpp b/lib/node_direction.cpp index 4fa51dd2d..9f34090cb 100644 --- a/lib/node_direction.cpp +++ b/lib/node_direction.cpp @@ -57,6 +57,7 @@ int node_direction_init(struct vnode_direction *nd, enum NodeDir dir, struct vno nd->enabled = 1; nd->vectorize = 1; nd->builtin = 1; + nd->path = nullptr; #ifdef WITH_HOOKS ret = hook_list_init(&nd->hooks); diff --git a/lib/nodes/amqp.cpp b/lib/nodes/amqp.cpp index 302e987e7..edfe4f7f0 100644 --- a/lib/nodes/amqp.cpp +++ b/lib/nodes/amqp.cpp @@ -28,10 +28,10 @@ #include #include #include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static void amqp_default_ssl_info(struct amqp_ssl_info *s) @@ -121,7 +121,6 @@ int amqp_parse(struct vnode *n, json_t *json) struct amqp *a = (struct amqp *) n->_vd; int port = 5672; - const char *format = "json"; const char *uri = nullptr; const char *host = "localhost"; const char *vhost = "/"; @@ -132,12 +131,13 @@ int amqp_parse(struct vnode *n, json_t *json) json_error_t err; json_t *json_ssl = nullptr; + json_t *json_format = nullptr; /* Default values */ amqp_default_ssl_info(&a->ssl_info); amqp_default_connection_info(&a->connection_info); - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: s, s?: s, s?: i, s: s, s: s, s?: s, s?: o }", + ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: s, s?: s, s?: i, s: s, s: s, s?: o, s?: o }", "uri", &uri, "host", &host, "vhost", &vhost, @@ -146,7 +146,7 @@ int amqp_parse(struct vnode *n, json_t *json) "port", &port, "exchange", &exchange, "routing_key", &routing_key, - "format", &format, + "format", &json_format, "ssl", &json_ssl ); if (ret) @@ -189,9 +189,12 @@ int amqp_parse(struct vnode *n, json_t *json) a->ssl_info.client_key = strdup(client_key); } - a->format = format_type_lookup(format); - if (!a->format) - throw ConfigError(json, "node-config-node-amqp-format", "Invalid format '{}'", format); + /* Format */ + a->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("json"); + if (!a->formatter) + throw ConfigError(json_format, "node-config-node-amqp-format", "Invalid format configuration"); return 0; } @@ -202,8 +205,7 @@ char * amqp_print(struct vnode *n) char *buf = nullptr; - strcatf(&buf, "format=%s, uri=%s://%s:%s@%s:%d%s, exchange=%s, routing_key=%s", - format_type_name(a->format), + strcatf(&buf, "uri=%s://%s:%s@%s:%d%s, exchange=%s, routing_key=%s", a->connection_info.ssl ? "amqps" : "amqp", a->connection_info.user, a->connection_info.password, @@ -235,16 +237,13 @@ char * amqp_print(struct vnode *n) int amqp_start(struct vnode *n) { - int ret; struct amqp *a = (struct amqp *) n->_vd; amqp_bytes_t queue; amqp_rpc_reply_t rep; amqp_queue_declare_ok_t *r; - ret = io_init(&a->io, a->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + a->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); /* Connect producer */ a->producer = amqp_connect(&a->connection_info, &a->ssl_info); @@ -302,14 +301,12 @@ int amqp_stop(struct vnode *n) if (ret) return ret; - ret = io_destroy(&a->io); - if (ret) - return ret; + delete a->formatter; return 0; } -int amqp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int amqp_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct amqp *a = (struct amqp *) n->_vd; @@ -320,21 +317,21 @@ int amqp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re if (rep.reply_type != AMQP_RESPONSE_NORMAL) return -1; - ret = io_sscan(&a->io, static_cast(env.message.body.bytes), env.message.body.len, nullptr, smps, cnt); + ret = a->formatter->sscan(static_cast(env.message.body.bytes), env.message.body.len, nullptr, smps, cnt); amqp_destroy_envelope(&env); return ret; } -int amqp_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int amqp_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct amqp *a = (struct amqp *) n->_vd; char data[1500]; size_t wbytes; - ret = io_sprint(&a->io, data, sizeof(data), &wbytes, smps, cnt); + ret = a->formatter->sprint(data, sizeof(data), &wbytes, smps, cnt); if (ret <= 0) return -1; diff --git a/lib/nodes/can.cpp b/lib/nodes/can.cpp index d647c776e..96099ab86 100644 --- a/lib/nodes/can.cpp +++ b/lib/nodes/can.cpp @@ -84,7 +84,7 @@ int can_destroy(struct vnode *n) return 0; } -int can_parse_signal(json_t *json, struct vlist *node_signals, struct can_signal *can_signals, size_t signal_index) +static int can_parse_signal(json_t *json, struct vlist *node_signals, struct can_signal *can_signals, size_t signal_index) { const char *name = nullptr; uint64_t can_id = 0; @@ -243,7 +243,7 @@ int can_stop(struct vnode *n) return 0; } -int can_conv_to_raw(union signal_data* sig, struct signal *from, void* to, int size) +static int can_convert_to_raw(const union signal_data *sig, const struct signal *from, void *to, int size) { if (size <= 0 || size > 8) throw RuntimeError("Signal size cannot be larger than 8!"); @@ -261,7 +261,6 @@ int can_conv_to_raw(union signal_data* sig, struct signal *from, void* to, int s case 2: *(int16_t*)to = (int16_t)sig->i; - sig->i = (int64_t)*(int16_t*)from; return 0; case 3: @@ -377,7 +376,7 @@ fail: return 1; } -int can_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int can_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret = 0; int nbytes; @@ -446,7 +445,7 @@ int can_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *rel return ret; } -int can_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int can_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int nbytes; unsigned nwrite; @@ -467,7 +466,7 @@ int can_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re frame[fsize].can_dlc = c->out[i].size; frame[fsize].can_id = c->out[i].id; - can_conv_to_raw( + can_convert_to_raw( &smps[nwrite]->data[i], (struct signal*)vlist_at(&(n->out.signals), i), &frame[fsize].data, @@ -486,7 +485,7 @@ int can_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re continue; frame[j].can_dlc += c->out[i].size; - can_conv_to_raw( + can_convert_to_raw( &smps[nwrite]->data[i], (struct signal*)vlist_at(&(n->out.signals), i), (uint8_t*)&frame[j].data + c->out[i].offset, diff --git a/lib/nodes/comedi.cpp b/lib/nodes/comedi.cpp index f2edfe884..a8a689a18 100644 --- a/lib/nodes/comedi.cpp +++ b/lib/nodes/comedi.cpp @@ -33,6 +33,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; /* Utility functions to dump a comedi_cmd graciously taken from comedilib demo */ @@ -510,7 +511,7 @@ int comedi_stop(struct vnode *n) #if COMEDI_USE_READ -int comedi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int comedi_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct comedi *c = (struct comedi *) n->_vd; @@ -622,7 +623,7 @@ int comedi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * #else -int comedi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int comedi_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct comedi *c = (struct comedi *) n->_vd; @@ -800,7 +801,7 @@ int comedi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * #endif -int comedi_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int comedi_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct comedi *c = (struct comedi *) n->_vd; @@ -858,7 +859,7 @@ int comedi_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned size_t villas_samples_written = 0; while (villas_samples_written < cnt) { - struct sample *sample = smps[villas_samples_written]; + const struct sample *sample = smps[villas_samples_written]; if (sample->length != d->chanlist_len) throw RuntimeError("Value count in sample ({}) != configured output channels ({})", sample->length, d->chanlist_len); diff --git a/lib/nodes/ethercat.cpp b/lib/nodes/ethercat.cpp index 08cd888dc..aa3144232 100644 --- a/lib/nodes/ethercat.cpp +++ b/lib/nodes/ethercat.cpp @@ -31,6 +31,7 @@ #include using namespace villas; +using namespace villas::node; /* Forward declartions */ static struct plugin p; @@ -320,7 +321,7 @@ int ethercat_stop(struct vnode *n) return 0; } -int ethercat_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ethercat_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct ethercat *w = (struct ethercat *) n->_vd; @@ -337,7 +338,7 @@ int ethercat_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned return avail; } -int ethercat_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ethercat_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct ethercat *w = (struct ethercat *) n->_vd; diff --git a/lib/nodes/example.cpp b/lib/nodes/example.cpp index d8ba59850..9fbe112b4 100644 --- a/lib/nodes/example.cpp +++ b/lib/nodes/example.cpp @@ -168,7 +168,7 @@ int example_resume(struct vnode *n) return 0; } -int example_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int example_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int read; struct example *s = (struct example *) n->_vd; @@ -192,7 +192,7 @@ int example_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned return read; } -int example_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int example_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int written; //struct example *s = (struct example *) n->_vd; diff --git a/lib/nodes/exec.cpp b/lib/nodes/exec.cpp index cafd7c363..45ab01fae 100644 --- a/lib/nodes/exec.cpp +++ b/lib/nodes/exec.cpp @@ -29,6 +29,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; int exec_parse(struct vnode *n, json_t *json) @@ -40,14 +41,14 @@ int exec_parse(struct vnode *n, json_t *json) json_t *json_exec; json_t *json_env = nullptr; + json_t *json_format = nullptr; const char *wd = nullptr; - const char *format = "villas.human"; int shell = -1; - ret = json_unpack_ex(json, &err, 0, "{ s: o, s?: s, s?: b, s?: o, s?: b, s?: s }", + ret = json_unpack_ex(json, &err, 0, "{ s: o, s?: o, s?: b, s?: o, s?: b, s?: s }", "exec", &json_exec, - "format", &format, + "format", &json_format, "flush", &flush, "environment", &json_env, "shell", &shell, @@ -101,26 +102,25 @@ int exec_parse(struct vnode *n, json_t *json) } } - json_t *json_format = json_object_get(json, "format"); - e->format = format_type_lookup(format); - if (!e->format) - throw ConfigError(json_format, "node-config-node-exec-format", "Invalid format: {)", format); + /* Format */ + e->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.human"); + if (!e->formatter) + throw ConfigError(json_format, "node-config-node-exec-format", "Invalid format configuration"); - if (!(e->format->flags & (int) IOFlags::NEWLINES)) - throw ConfigError(json_format, "node-config-node-exec-format", "Only line-delimited formats are currently supported"); + // if (!(e->format->flags & (int) Flags::NEWLINES)) + // throw ConfigError(json_format, "node-config-node-exec-format", "Only line-delimited formats are currently supported"); return 0; } int exec_prepare(struct vnode *n) { - int ret; struct exec *e = (struct exec *) n->_vd; /* Initialize IO */ - ret = io_init(&e->io, e->format, &n->in.signals, (int) SampleFlags::HAS_ALL); - if (ret) - return ret; + e->formatter->start(&n->in.signals); /* Start subprocess */ e->proc = std::make_unique(e->command, e->arguments, e->environment, e->working_dir, e->shell); @@ -144,14 +144,9 @@ int exec_init(struct vnode *n) int exec_destroy(struct vnode *n) { - int ret; struct exec *e = (struct exec *) n->_vd; - if (e->io.state == State::INITIALIZED) { - ret = io_destroy(&e->io); - if (ret) - return ret; - } + delete e->formatter; using uptr = std::unique_ptr; using str = std::string; @@ -182,7 +177,7 @@ int exec_stop(struct vnode *n) return 0; } -int exec_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int exec_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct exec *e = (struct exec *) n->_vd; @@ -192,14 +187,14 @@ int exec_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re std::getline(e->proc->cin(), line); - avail = io_sscan(&e->io, line.c_str(), line.length(), &rbytes, smps, cnt); + avail = e->formatter->sscan(line.c_str(), line.length(), &rbytes, smps, cnt); if (rbytes - 1 != line.length()) return -1; return avail; } -int exec_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int exec_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct exec *e = (struct exec *) n->_vd; @@ -209,7 +204,7 @@ int exec_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r if (!line) throw MemoryAllocationError(); - ret = io_sprint(&e->io, line, 1024, &wbytes, smps, cnt); + ret = e->formatter->sprint(line, 1024, &wbytes, smps, cnt); if (ret < 0) return ret; @@ -228,8 +223,7 @@ char * exec_print(struct vnode *n) struct exec *e = (struct exec *) n->_vd; char *buf = nullptr; - strcatf(&buf, "format=%s, exec=%s, shell=%s, flush=%s, #environment=%zu, #arguments=%zu, working_dir=%s", - format_type_name(e->format), + strcatf(&buf, "exec=%s, shell=%s, flush=%s, #environment=%zu, #arguments=%zu, working_dir=%s", e->command.c_str(), e->shell ? "yes" : "no", e->flush ? "yes" : "no", diff --git a/lib/nodes/file.cpp b/lib/nodes/file.cpp index 91e28190e..0917ac980 100644 --- a/lib/nodes/file.cpp +++ b/lib/nodes/file.cpp @@ -32,10 +32,11 @@ #include #include #include -#include +#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static char * file_format_name(const char *format, struct timespec *ts) @@ -86,16 +87,16 @@ int file_parse(struct vnode *n, json_t *json) int ret; json_error_t err; + json_t *json_format; const char *uri_tmpl = nullptr; - const char *format = "villas.human"; const char *eof = nullptr; const char *epoch = nullptr; double epoch_flt = 0; - ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: s, s?: { s?: s, s?: F, s?: s, s?: F, s?: i, s?: i }, s?: { s?: b, s?: i } }", + ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: o, s?: { s?: s, s?: F, s?: s, s?: F, s?: i, s?: i }, s?: { s?: b, s?: i } }", "uri", &uri_tmpl, - "format", &format, + "format", &json_format, "in", "eof", &eof, "rate", &f->rate, @@ -113,9 +114,12 @@ int file_parse(struct vnode *n, json_t *json) f->epoch = time_from_double(epoch_flt); f->uri_tmpl = uri_tmpl ? strdup(uri_tmpl) : nullptr; - f->format = format_type_lookup(format); - if (!f->format) - throw RuntimeError("Invalid format '{}'", format); + /* Format */ + f->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.human"); + if (!f->formatter) + throw ConfigError(json_format, "node-config-node-file-format", "Invalid format configuration"); if (eof) { if (!strcmp(eof, "exit") || !strcmp(eof, "stop")) @@ -200,9 +204,8 @@ char * file_print(struct vnode *n) break; } - strcatf(&buf, "uri=%s, format=%s, out.flush=%s, in.skip=%d, in.eof=%s, in.epoch=%s, in.epoch=%.2f", + strcatf(&buf, "uri=%s, out.flush=%s, in.skip=%d, in.eof=%s, in.epoch=%s, in.epoch=%.2f", f->uri ? f->uri : f->uri_tmpl, - format_type_name(f->format), f->flush ? "yes" : "no", f->skip_lines, eof_str, @@ -238,7 +241,7 @@ int file_start(struct vnode *n) struct file *f = (struct file *) n->_vd; struct timespec now = time_now(); - int ret, flags; + int ret; /* Prepare file name */ f->uri = file_format_name(f->uri_tmpl, &now); @@ -266,28 +269,26 @@ int file_start(struct vnode *n) free(cpy); - /* Open file */ - flags = (int) SampleFlags::HAS_ALL; - if (f->flush) - flags |= (int) IOFlags::FLUSH; - ret = io_init(&f->io, f->format, &n->in.signals, flags); - if (ret) - return ret; + f->formatter->start(&n->in.signals); - ret = io_open(&f->io, f->uri); - if (ret) - return ret; + f->stream_in = fopen(f->uri, "r"); + if (!f->stream_in) + return -1; + + f->stream_out = fopen(f->uri, "a+"); + if (!f->stream_out) + return -1; if (f->buffer_size_in) { - ret = setvbuf(f->io.in.stream, nullptr, _IOFBF, f->buffer_size_in); + ret = setvbuf(f->stream_in, nullptr, _IOFBF, f->buffer_size_in); if (ret) return ret; } if (f->buffer_size_out) { - ret = setvbuf(f->io.out.stream, nullptr, _IOFBF, f->buffer_size_out); + ret = setvbuf(f->stream_out, nullptr, _IOFBF, f->buffer_size_out); if (ret) return ret; } @@ -297,20 +298,19 @@ int file_start(struct vnode *n) /* Get timestamp of first line */ if (f->epoch_mode != file::EpochMode::ORIGINAL) { - io_rewind(&f->io); + rewind(f->stream_in); - if (io_eof(&f->io)) { + if (feof(f->stream_in)) { n->logger->warn("Empty file"); } else { - struct sample s; - struct sample *smps[] = { &s }; + struct sample smp; - s.capacity = 0; + smp.capacity = 0; - ret = io_scan(&f->io, smps, 1); + ret = f->formatter->scan(f->stream_in, &smp); if (ret == 1) { - f->first = s.ts.origin; + f->first = smp.ts.origin; f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); } else @@ -318,12 +318,12 @@ int file_start(struct vnode *n) } } - io_rewind(&f->io); + rewind(f->stream_in); /* Fast-forward */ struct sample *smp = sample_alloc_mem(vlist_length(&n->in.signals)); for (unsigned i = 0; i < f->skip_lines; i++) - io_scan(&f->io, &smp, 1); + f->formatter->scan(f->stream_in, smp); sample_free(smp); @@ -332,25 +332,20 @@ int file_start(struct vnode *n) int file_stop(struct vnode *n) { - int ret; struct file *f = (struct file *) n->_vd; f->task.stop(); - ret = io_close(&f->io); - if (ret) - return ret; - - ret = io_destroy(&f->io); - if (ret) - return ret; + fclose(f->stream_in); + fclose(f->stream_out); + delete f->formatter; delete f->uri; return 0; } -int file_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int file_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct file *f = (struct file *) n->_vd; int ret; @@ -358,15 +353,15 @@ int file_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re assert(cnt == 1); -retry: ret = io_scan(&f->io, smps, cnt); +retry: ret = f->formatter->scan(f->stream_in, smps, cnt); if (ret <= 0) { - if (io_eof(&f->io)) { + if (feof(f->stream_in)) { switch (f->eof_mode) { case file::EOFBehaviour::REWIND: n->logger->info("Rewind input file"); f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); - io_rewind(&f->io); + rewind(f->stream_in); goto retry; case file::EOFBehaviour::SUSPEND: @@ -374,15 +369,7 @@ retry: ret = io_scan(&f->io, smps, cnt); usleep(100000); /* Try to download more data if this is a remote file. */ - switch (f->io.mode) { - case IOMode::STDIO: - clearerr(f->io.in.stream); - break; - - case IOMode::CUSTOM: - break; - } - + clearerr(f->stream_in); goto retry; case file::EOFBehaviour::STOP: @@ -426,17 +413,20 @@ retry: ret = io_scan(&f->io, smps, cnt); return cnt; } -int file_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int file_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct file *f = (struct file *) n->_vd; assert(cnt == 1); - ret = io_print(&f->io, smps, cnt); + ret = f->formatter->print(f->stream_out, smps, cnt); if (ret < 0) return ret; + if (f->flush) + fflush(f->stream_out); + return cnt; } @@ -450,7 +440,7 @@ int file_poll_fds(struct vnode *n, int fds[]) return 1; } else if (f->epoch_mode == file::EpochMode::ORIGINAL) { - fds[0] = io_fd(&f->io); + fds[0] = fileno(f->stream_in); return 1; } diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp index bc334ac1c..f716c2c68 100644 --- a/lib/nodes/fpga.cpp +++ b/lib/nodes/fpga.cpp @@ -273,7 +273,7 @@ int fpga_stop(struct vnode *n) return 0; } -int fpga_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int fpga_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { unsigned read; struct fpga *f = (struct fpga *) n->_vd; @@ -293,7 +293,7 @@ int fpga_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re return read; } -int fpga_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int fpga_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int written; struct fpga *f = (struct fpga *) n->_vd; diff --git a/lib/nodes/iec61850.cpp b/lib/nodes/iec61850.cpp index a383160c9..ac2c2d4ac 100644 --- a/lib/nodes/iec61850.cpp +++ b/lib/nodes/iec61850.cpp @@ -35,6 +35,7 @@ #define CONFIG_SV_DEFAULT_VLAN_ID 0 using namespace villas; +using namespace villas::node; using namespace villas::utils; const struct iec61850_type_descriptor type_descriptors[] = { diff --git a/lib/nodes/iec61850_sv.cpp b/lib/nodes/iec61850_sv.cpp index d29d6dd22..e16a88179 100644 --- a/lib/nodes/iec61850_sv.cpp +++ b/lib/nodes/iec61850_sv.cpp @@ -34,7 +34,9 @@ #define CONFIG_SV_DEFAULT_PRIORITY 4 #define CONFIG_SV_DEFAULT_VLAN_ID 0 +using namespace villas; using namespace villas::utils; +using namespace villas::node; static void iec61850_sv_listener(SVSubscriber subscriber, void *ctx, SVSubscriber_ASDU asdu) { @@ -382,7 +384,7 @@ int iec61850_sv_destroy(struct vnode *n) return 0; } -int iec61850_sv_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int iec61850_sv_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int pulled; struct iec61850_sv *i = (struct iec61850_sv *) n->_vd; @@ -399,7 +401,7 @@ int iec61850_sv_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsig return pulled; } -int iec61850_sv_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int iec61850_sv_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct iec61850_sv *i = (struct iec61850_sv *) n->_vd; diff --git a/lib/nodes/infiniband.cpp b/lib/nodes/infiniband.cpp index 11d668b07..85809541e 100644 --- a/lib/nodes/infiniband.cpp +++ b/lib/nodes/infiniband.cpp @@ -29,11 +29,12 @@ #include #include #include -#include #include #include #include +using namespace villas; +using namespace villas::node; using namespace villas::utils; static int ib_disconnect(struct vnode *n) @@ -725,7 +726,7 @@ int ib_stop(struct vnode *n) return 0; } -int ib_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ib_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct infiniband *ib = (struct infiniband *) n->_vd; struct ibv_wc wc[cnt]; @@ -865,7 +866,7 @@ int ib_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *rele return read_values; } -int ib_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ib_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct infiniband *ib = (struct infiniband *) n->_vd; struct ibv_send_wr wr[cnt], *bad_wr = nullptr; diff --git a/lib/nodes/influxdb.cpp b/lib/nodes/influxdb.cpp index 9eefca3b1..1d7f5fad2 100644 --- a/lib/nodes/influxdb.cpp +++ b/lib/nodes/influxdb.cpp @@ -35,6 +35,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; int influxdb_parse(struct vnode *n, json_t *json) @@ -119,7 +120,7 @@ int influxdb_close(struct vnode *n) return 0; } -int influxdb_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int influxdb_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct influxdb *i = (struct influxdb *) n->_vd; @@ -127,7 +128,7 @@ int influxdb_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigne ssize_t sentlen, buflen; for (unsigned k = 0; k < cnt; k++) { - struct sample *smp = smps[k]; + const struct sample *smp = smps[k]; /* Key */ strcatf(&buf, "%s", i->key); @@ -135,7 +136,7 @@ int influxdb_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigne /* Fields */ for (unsigned j = 0; j < smp->length; j++) { struct signal *sig = (struct signal *) vlist_at(smp->signals, j); - union signal_data *data = &smp->data[j]; + const union signal_data *data = &smp->data[j]; if ( sig->type != SignalType::BOOLEAN && diff --git a/lib/nodes/kafka.cpp b/lib/nodes/kafka.cpp index 09178dcf2..8f0deff82 100644 --- a/lib/nodes/kafka.cpp +++ b/lib/nodes/kafka.cpp @@ -27,10 +27,10 @@ #include #include #include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; // Each process has a list of clients for which a thread invokes the kafka loop @@ -62,7 +62,6 @@ static void kafka_logger_cb(const rd_kafka_t *rk, int level, const char *fac, co default: logger->info("{}: {}", fac, buf); break; - } } @@ -81,7 +80,7 @@ static void kafka_message_cb(void *ctx, const rd_kafka_message_t *msg) return; } - ret = io_sscan(&k->io, (char *) msg->payload, msg->len, nullptr, smps, n->in.vectorize); + ret = k->formatter->sscan((char *) msg->payload, msg->len, nullptr, smps, n->in.vectorize); if (ret < 0) { n->logger->warn("Received an invalid message"); n->logger->warn(" Payload: {}", (char *) msg->payload); @@ -171,7 +170,6 @@ int kafka_parse(struct vnode *n, json_t *json) struct kafka *k = (struct kafka *) n->_vd; const char *server; - const char *format = "villas.binary"; const char *produce = nullptr; const char *consume = nullptr; const char *protocol; @@ -181,14 +179,15 @@ int kafka_parse(struct vnode *n, json_t *json) json_error_t err; json_t *json_ssl = nullptr; json_t *json_sasl = nullptr; + json_t *json_format = nullptr; - ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: s }, s?: { s?: s, s?: s }, s?: s, s: s, s?: F, s: s, s?: s, s?: o, s?: o }", + ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: s }, s?: { s?: s, s?: s }, s?: o, s: s, s?: F, s: s, s?: s, s?: o, s?: o }", "out", "produce", &produce, "in", "consume", &consume, "group_id", &group_id, - "format", &format, + "format", &json_format, "server", &server, "timeout", &k->timeout, "protocol", &protocol, @@ -245,10 +244,12 @@ int kafka_parse(struct vnode *n, json_t *json) k->sasl.password = strdup(password); } - k->format = format_type_lookup(format); - if (!k->format) - throw ConfigError(json_ssl, "node-config-node-kafka-format", "Invalid format '{}' for node {}", format, node_name(n)); - + /* Format */ + k->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.binary"); + if (!k->formatter) + throw ConfigError(json_format, "node-config-node-kafka-format", "Invalid format configuration"); return 0; } @@ -257,9 +258,7 @@ int kafka_prepare(struct vnode *n) int ret; struct kafka *k = (struct kafka *) n->_vd; - ret = io_init(&k->io, k->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + k->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); ret = pool_init(&k->pool, 1024, SAMPLE_LENGTH(vlist_length(&n->in.signals))); if (ret) @@ -278,7 +277,7 @@ char * kafka_print(struct vnode *n) char *buf = nullptr; - strcatf(&buf, "format=%s, bootstrap.server=%s, client.id=%s, security.protocol=%s", format_type_name(k->format), + strcatf(&buf, "bootstrap.server=%s, client.id=%s, security.protocol=%s", k->server, k->client_id, k->protocol @@ -305,9 +304,7 @@ int kafka_destroy(struct vnode *n) if (k->consumer.client) rd_kafka_destroy(k->consumer.client); - ret = io_destroy(&k->io); - if (ret) - return ret; + delete k->formatter; ret = pool_destroy(&k->pool); if (ret) @@ -528,7 +525,7 @@ kafka_error: return ret; } -int kafka_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int kafka_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int pulled; struct kafka *k = (struct kafka *) n->_vd; @@ -542,7 +539,7 @@ int kafka_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r return pulled; } -int kafka_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int kafka_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct kafka *k = (struct kafka *) n->_vd; @@ -551,7 +548,7 @@ int kafka_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * char data[4096]; - ret = io_sprint(&k->io, data, sizeof(data), &wbytes, smps, cnt); + ret = k->formatter->sprint(data, sizeof(data), &wbytes, smps, cnt); if (ret < 0) return ret; diff --git a/lib/nodes/loopback.cpp b/lib/nodes/loopback.cpp index c62b74995..3d53c0e17 100644 --- a/lib/nodes/loopback.cpp +++ b/lib/nodes/loopback.cpp @@ -30,6 +30,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static struct plugin p; @@ -95,7 +96,7 @@ int loopback_destroy(struct vnode *n) return queue_signalled_destroy(&l->queue); } -int loopback_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int loopback_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int avail; @@ -112,7 +113,7 @@ int loopback_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned return avail; } -int loopback_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int loopback_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct loopback *l = (struct loopback *) n->_vd; diff --git a/lib/nodes/loopback_internal.cpp b/lib/nodes/loopback_internal.cpp index 2ba8d5610..a4a526fc4 100644 --- a/lib/nodes/loopback_internal.cpp +++ b/lib/nodes/loopback_internal.cpp @@ -30,6 +30,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static struct plugin p; @@ -62,7 +63,7 @@ int loopback_internal_destroy(struct vnode *n) return queue_signalled_destroy(&l->queue); } -int loopback_internal_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int loopback_internal_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int avail; @@ -79,7 +80,7 @@ int loopback_internal_read(struct vnode *n, struct sample *smps[], unsigned cnt, return avail; } -int loopback_internal_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int loopback_internal_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct loopback_internal *l = (struct loopback_internal *) n->_vd; diff --git a/lib/nodes/mqtt.cpp b/lib/nodes/mqtt.cpp index 624403a2f..a4dd9bece 100644 --- a/lib/nodes/mqtt.cpp +++ b/lib/nodes/mqtt.cpp @@ -26,10 +26,10 @@ #include #include #include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; // Each process has a list of clients for which a thread invokes the mosquitto loop @@ -138,7 +138,7 @@ static void mqtt_message_cb(struct mosquitto *mosq, void *ctx, const struct mosq return; } - ret = io_sscan(&m->io, (char *) msg->payload, msg->payloadlen, nullptr, smps, n->in.vectorize); + ret = m->formatter->sscan((char *) msg->payload, msg->payloadlen, nullptr, smps, n->in.vectorize); if (ret < 0) { n->logger->warn("Received an invalid message"); n->logger->warn(" Payload: {}", (char *) msg->payload); @@ -225,7 +225,6 @@ int mqtt_parse(struct vnode *n, json_t *json) struct mqtt *m = (struct mqtt *) n->_vd; const char *host; - const char *format = "villas.binary"; const char *publish = nullptr; const char *subscribe = nullptr; const char *username = nullptr; @@ -233,13 +232,14 @@ int mqtt_parse(struct vnode *n, json_t *json) json_error_t err; json_t *json_ssl = nullptr; + json_t *json_format = nullptr; - ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: s }, s?: { s?: s }, s?: s, s: s, s?: i, s?: i, s?: i, s?: b, s?: s, s?: s, s?: o }", + ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: s }, s?: { s?: s }, s?: o, s: s, s?: i, s?: i, s?: i, s?: b, s?: s, s?: s, s?: o }", "out", "publish", &publish, "in", "subscribe", &subscribe, - "format", &format, + "format", &json_format, "host", &host, "port", &m->port, "qos", &m->qos, @@ -289,10 +289,12 @@ int mqtt_parse(struct vnode *n, json_t *json) m->ssl.keyfile = keyfile ? strdup(keyfile) : nullptr; } - m->format = format_type_lookup(format); - if (!m->format) - throw ConfigError(json_ssl, "node-config-node-mqtt-format", "Invalid format '{}' for node {}", format, node_name(n)); - + /* Format */ + m->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("json"); + if (!m->formatter) + throw ConfigError(json_format, "node-config-node-mqtt-format", "Invalid format configuration"); return 0; } @@ -317,9 +319,7 @@ int mqtt_prepare(struct vnode *n) int ret; struct mqtt *m = (struct mqtt *) n->_vd; - ret = io_init(&m->io, m->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + m->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); ret = pool_init(&m->pool, 1024, SAMPLE_LENGTH(vlist_length(&n->in.signals))); if (ret) @@ -338,7 +338,7 @@ char * mqtt_print(struct vnode *n) char *buf = nullptr; - strcatf(&buf, "format=%s, host=%s, port=%d, keepalive=%d, ssl=%s", format_type_name(m->format), + strcatf(&buf, "host=%s, port=%d, keepalive=%d, ssl=%s", m->host, m->port, m->keepalive, @@ -365,9 +365,7 @@ int mqtt_destroy(struct vnode *n) mosquitto_destroy(m->client); - ret = io_destroy(&m->io); - if (ret) - return ret; + delete m->formatter; ret = pool_destroy(&m->pool); if (ret) @@ -512,7 +510,7 @@ mosquitto_error: return ret; } -int mqtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int mqtt_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int pulled; struct mqtt *m = (struct mqtt *) n->_vd; @@ -526,7 +524,7 @@ int mqtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re return pulled; } -int mqtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int mqtt_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct mqtt *m = (struct mqtt *) n->_vd; @@ -535,7 +533,7 @@ int mqtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r char data[1500]; - ret = io_sprint(&m->io, data, sizeof(data), &wbytes, smps, cnt); + ret = m->formatter->sprint(data, sizeof(data), &wbytes, smps, cnt); if (ret < 0) return ret; diff --git a/lib/nodes/nanomsg.cpp b/lib/nodes/nanomsg.cpp index a4f38e366..6ab68f5d0 100644 --- a/lib/nodes/nanomsg.cpp +++ b/lib/nodes/nanomsg.cpp @@ -27,10 +27,10 @@ #include #include #include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; int nanomsg_reverse(struct vnode *n) @@ -86,10 +86,8 @@ int nanomsg_parse(struct vnode *n, json_t *json) int ret; struct nanomsg *m = (struct nanomsg *) n->_vd; - const char *format = "villas.binary"; - json_error_t err; - + json_t *json_format = nullptr; json_t *json_out_endpoints = nullptr; json_t *json_in_endpoints = nullptr; @@ -101,8 +99,8 @@ int nanomsg_parse(struct vnode *n, json_t *json) if (ret) return ret; - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: { s?: o }, s?: { s?: o } }", - "format", &format, + ret = json_unpack_ex(json, &err, 0, "{ s?: o, s?: { s?: o }, s?: { s?: o } }", + "format", &json_format, "out", "endpoints", &json_out_endpoints, "in", @@ -123,9 +121,12 @@ int nanomsg_parse(struct vnode *n, json_t *json) throw RuntimeError("Invalid type for 'subscribe' setting"); } - m->format = format_type_lookup(format); - if (!m->format) - throw RuntimeError("Invalid format '{}'", format); + /* Format */ + m->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("json"); + if (!m->formatter) + throw ConfigError(json_format, "node-config-node-nanomsg-format", "Invalid format configuration"); return 0; } @@ -136,7 +137,7 @@ char * nanomsg_print(struct vnode *n) char *buf = nullptr; - strcatf(&buf, "format=%s, in.endpoints=[ ", format_type_name(m->format)); + strcatf(&buf, "in.endpoints=[ "); for (size_t i = 0; i < vlist_length(&m->in.endpoints); i++) { char *ep = (char *) vlist_at(&m->in.endpoints, i); @@ -162,9 +163,7 @@ int nanomsg_start(struct vnode *n) int ret; struct nanomsg *m = (struct nanomsg *) n->_vd; - ret = io_init(&m->io, m->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + m->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); ret = m->in.socket = nn_socket(AF_SP, NN_SUB); if (ret < 0) @@ -213,9 +212,7 @@ int nanomsg_stop(struct vnode *n) if (ret < 0) return ret; - ret = io_destroy(&m->io); - if (ret) - return ret; + delete m->formatter; return 0; } @@ -227,7 +224,7 @@ int nanomsg_type_stop() return 0; } -int nanomsg_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int nanomsg_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct nanomsg *m = (struct nanomsg *) n->_vd; int bytes; @@ -238,10 +235,10 @@ int nanomsg_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned if (bytes < 0) return -1; - return io_sscan(&m->io, data, bytes, nullptr, smps, cnt); + return m->formatter->sscan(data, bytes, nullptr, smps, cnt); } -int nanomsg_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int nanomsg_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct nanomsg *m = (struct nanomsg *) n->_vd; @@ -250,7 +247,7 @@ int nanomsg_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned char data[NANOMSG_MAX_PACKET_LEN]; - ret = io_sprint(&m->io, data, sizeof(data), &wbytes, smps, cnt); + ret = m->formatter->sprint(data, sizeof(data), &wbytes, smps, cnt); if (ret <= 0) return -1; diff --git a/lib/nodes/ngsi.cpp b/lib/nodes/ngsi.cpp index 288d650f0..ca7d79393 100644 --- a/lib/nodes/ngsi.cpp +++ b/lib/nodes/ngsi.cpp @@ -40,6 +40,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; /* Some global settings */ @@ -181,7 +182,7 @@ public: metadata.emplace_back("index", "integer", fmt::format("{}", j)); } - json_t * build(struct sample *smps[], unsigned cnt, int flags) + json_t * build(const struct sample * const smps[], unsigned cnt, int flags) { json_t *json_attribute = json_pack("{ s: s, s: s }", "name", name.c_str(), @@ -206,10 +207,10 @@ public: )); } #else - struct sample *smp = smps[0]; + const struct sample *smp = smps[0]; - union signal_data *sd = &smp->data[index]; - struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, index); + const union signal_data *sd = &smp->data[index]; + const struct signal *sig = (struct signal *) vlist_at_safe(smp->signals, index); json_t *json_value = signal_data_to_json(sd, sig->type); #endif @@ -240,7 +241,7 @@ struct ngsi_response { size_t len; }; -static json_t* ngsi_build_entity(struct vnode *n, struct sample *smps[], unsigned cnt, int flags) +static json_t* ngsi_build_entity(struct vnode *n, const struct sample * const smps[], unsigned cnt, int flags) { struct ngsi *i = (struct ngsi *) n->_vd; @@ -279,7 +280,7 @@ static json_t* ngsi_build_entity(struct vnode *n, struct sample *smps[], unsigne return json_entity; } -static int ngsi_parse_entity(struct vnode *n, json_t *json_entity, struct sample *smps[], unsigned cnt) +static int ngsi_parse_entity(struct vnode *n, json_t *json_entity, struct sample * const smps[], unsigned cnt) { int ret, length = 0; const char *id, *name, *type; @@ -731,7 +732,7 @@ int ngsi_stop(struct vnode *n) return ret; } -int ngsi_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ngsi_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct ngsi *i = (struct ngsi *) n->_vd; int ret; @@ -756,7 +757,7 @@ out: json_decref(json_entity); return ret; } -int ngsi_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int ngsi_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct ngsi *i = (struct ngsi *) n->_vd; int ret; diff --git a/lib/nodes/opal.cpp b/lib/nodes/opal.cpp index bce879252..2412313f8 100644 --- a/lib/nodes/opal.cpp +++ b/lib/nodes/opal.cpp @@ -253,7 +253,7 @@ int opal_start(struct vnode *n) return 0; } -int opal_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int opal_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct opal *o = (struct opal *) n->_vd; @@ -318,7 +318,7 @@ int opal_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re return 1; } -int opal_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int opal_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct opal *o = (struct opal *) n->_vd; diff --git a/lib/nodes/rtp.cpp b/lib/nodes/rtp.cpp index 3d5c4cfd5..038566d48 100644 --- a/lib/nodes/rtp.cpp +++ b/lib/nodes/rtp.cpp @@ -44,7 +44,6 @@ extern "C" { #include #include #include -#include #include #ifdef WITH_NETEM @@ -130,16 +129,16 @@ int rtp_parse(struct vnode *n, json_t *json) struct rtp *r = (struct rtp *) n->_vd; const char *local, *remote; - const char *format = "villas.binary"; const char *log = nullptr; const char *hook_type = nullptr; uint16_t port; json_error_t err; json_t *json_aimd = nullptr; + json_t *json_format = nullptr; - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: b, s?: o, s: { s: s }, s: { s: s } }", - "format", &format, + ret = json_unpack_ex(json, &err, 0, "{ s?: o, s?: b, s?: o, s: { s: s }, s: { s: s } }", + "format", &json_format, "rtcp", &r->rtcp.enabled, "aimd", &json_aimd, "out", @@ -186,9 +185,11 @@ int rtp_parse(struct vnode *n, json_t *json) r->aimd.log_filename = strdup(log); /* Format */ - r->format = format_type_lookup(format); - if (!r->format) - throw RuntimeError("Invalid format '{}'", format); + r->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.binary"); + if (!r->formatter) + throw ConfigError(json_format, "node-config-node-rtp-format", "Invalid format configuration"); /* Remote address */ ret = sa_decode(&r->out.saddr_rtp, remote, strlen(remote)); @@ -225,8 +226,7 @@ char * rtp_print(struct vnode *n) char *local = socket_print_addr((struct sockaddr *) &r->in.saddr_rtp.u); char *remote = socket_print_addr((struct sockaddr *) &r->out.saddr_rtp.u); - buf = strf("format=%s, in.address=%s, out.address=%s, rtcp.enabled=%s", - format_type_name(r->format), + buf = strf("in.address=%s, out.address=%s, rtcp.enabled=%s", local, remote, r->rtcp.enabled ? "yes" : "no"); @@ -323,9 +323,7 @@ int rtp_start(struct vnode *n) return ret; /* Initialize IO */ - ret = io_init(&r->io, r->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + r->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); /* Initialize memory buffer for sending */ r->send_mb = mbuf_alloc(RTP_INITIAL_BUFFER_LEN); @@ -424,9 +422,7 @@ int rtp_stop(struct vnode *n) if (r->aimd.log) r->aimd.log->close(); - ret = io_destroy(&r->io); - if (ret) - return ret; + delete r->formatter; return 0; } @@ -505,7 +501,7 @@ int rtp_type_stop() return 0; } -int rtp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int rtp_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct rtp *r = (struct rtp *) n->_vd; @@ -517,14 +513,14 @@ int rtp_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *rel throw RuntimeError("Failed to pull from queue"); /* Unpack data */ - ret = io_sscan(&r->io, (char *) mb->buf + mb->pos, mbuf_get_left(mb), nullptr, smps, cnt); + ret = r->formatter->sscan((char *) mb->buf + mb->pos, mbuf_get_left(mb), nullptr, smps, cnt); mem_deref(mb); return ret; } -int rtp_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int rtp_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct rtp *r = (struct rtp *) n->_vd; @@ -536,7 +532,7 @@ int rtp_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *re retry: mbuf_set_pos(r->send_mb, RTP_HEADER_SIZE); avail = mbuf_get_space(r->send_mb); - cnt = io_sprint(&r->io, (char *) r->send_mb->buf + r->send_mb->pos, avail, &wbytes, smps, cnt); + cnt = r->formatter->sprint((char *) r->send_mb->buf + r->send_mb->pos, avail, &wbytes, smps, cnt); if (cnt < 0) return -1; diff --git a/lib/nodes/shmem.cpp b/lib/nodes/shmem.cpp index 83e44f301..307954de8 100644 --- a/lib/nodes/shmem.cpp +++ b/lib/nodes/shmem.cpp @@ -38,6 +38,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; int shmem_parse(struct vnode *n, json_t *json) @@ -129,7 +130,7 @@ int shmem_stop(struct vnode *n) return shmem_int_close(&shm->intf); } -int shmem_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int shmem_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct shmem *shm = (struct shmem *) n->_vd; int recv; @@ -160,7 +161,7 @@ int shmem_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *r return recv; } -int shmem_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int shmem_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct shmem *shm = (struct shmem *) n->_vd; struct sample *shared_smps[cnt]; /* Samples need to be copied to the shared pool first */ diff --git a/lib/nodes/signal_generator.cpp b/lib/nodes/signal_generator.cpp index 25a33eba0..49a95a0fa 100644 --- a/lib/nodes/signal_generator.cpp +++ b/lib/nodes/signal_generator.cpp @@ -31,6 +31,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static enum signal_generator::SignalType signal_generator_lookup_type(const char *type) @@ -323,7 +324,7 @@ int signal_generator_stop(struct vnode *n) return 0; } -int signal_generator_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int signal_generator_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct signal_generator *s = (struct signal_generator *) n->_vd; struct sample *t = smps[0]; diff --git a/lib/nodes/socket.cpp b/lib/nodes/socket.cpp index f84a6847e..0a472f04c 100644 --- a/lib/nodes/socket.cpp +++ b/lib/nodes/socket.cpp @@ -28,7 +28,6 @@ #include #include -#include #include #include #include @@ -100,7 +99,7 @@ char * socket_print(struct vnode *n) char *local = socket_print_addr((struct sockaddr *) &s->in.saddr); char *remote = socket_print_addr((struct sockaddr *) &s->out.saddr); - buf = strf("layer=%s, format=%s, in.address=%s, out.address=%s", layer, format_type_name(s->format), local, remote); + buf = strf("layer=%s, in.address=%s, out.address=%s", layer, local, remote); if (s->multicast.enabled) { char group[INET_ADDRSTRLEN]; @@ -164,9 +163,7 @@ int socket_start(struct vnode *n) int ret; /* Initialize IO */ - ret = io_init(&s->io, s->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + s->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); /* Create socket */ switch (s->layer) { @@ -311,17 +308,14 @@ int socket_stop(struct vnode *n) return ret; } - ret = io_destroy(&s->io); - if (ret) - return ret; - + delete s->formatter; delete[] s->in.buf; delete[] s->out.buf; return 0; } -int socket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int socket_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct socket *s = (struct socket *) n->_vd; @@ -372,14 +366,14 @@ int socket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * return 0; } - ret = io_sscan(&s->io, ptr, bytes, &rbytes, smps, cnt); + ret = s->formatter->sscan(ptr, bytes, &rbytes, smps, cnt); if (ret < 0 || (size_t) bytes != rbytes) n->logger->warn("Received invalid packet: ret={}, bytes={}, rbytes={}", ret, bytes, rbytes); return ret; } -int socket_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int socket_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct socket *s = (struct socket *) n->_vd; @@ -387,7 +381,7 @@ int socket_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned ssize_t bytes; size_t wbytes; -retry: ret = io_sprint(&s->io, s->out.buf, s->out.buflen, &wbytes, smps, cnt); +retry: ret = s->formatter->sprint(s->out.buf, s->out.buflen, &wbytes, smps, cnt); if (ret < 0) { n->logger->warn("Failed to format payload: reason={}", ret); return ret; @@ -453,24 +447,23 @@ retry2: bytes = sendto(s->sd, s->out.buf, wbytes, 0, (struct sockaddr *) &s->out int socket_parse(struct vnode *n, json_t *json) { + int ret; struct socket *s = (struct socket *) n->_vd; const char *local, *remote; const char *layer = nullptr; - const char *format = "villas.binary"; - int ret; - - json_t *json_multicast = nullptr; json_error_t err; + json_t *json_multicast = nullptr; + json_t *json_format = nullptr; /* Default values */ s->layer = SocketLayer::UDP; s->verify_source = 0; - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s: { s: s }, s: { s: s, s?: b, s?: o } }", + ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: o, s: { s: s }, s: { s: s, s?: b, s?: o } }", "layer", &layer, - "format", &format, + "format", &json_format, "out", "address", &remote, "in", @@ -482,9 +475,11 @@ int socket_parse(struct vnode *n, json_t *json) throw ConfigError(json, err, "node-config-node-socket"); /* Format */ - s->format = format_type_lookup(format); - if (!s->format) - throw RuntimeError("Invalid format '{}'", format); + s->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.binary"); + if (!s->formatter) + throw ConfigError(json_format, "node-config-node-socket-format", "Invalid format configuration"); /* IP layer */ if (layer) { diff --git a/lib/nodes/stats.cpp b/lib/nodes/stats.cpp index 0467ffc4d..9004521d0 100644 --- a/lib/nodes/stats.cpp +++ b/lib/nodes/stats.cpp @@ -216,7 +216,7 @@ int stats_node_parse(struct vnode *n, json_t *json) return 0; } -int stats_node_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int stats_node_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct stats_node *s = (struct stats_node *) n->_vd; diff --git a/lib/nodes/test_rtt.cpp b/lib/nodes/test_rtt.cpp index 96bb7472f..c8c8848db 100644 --- a/lib/nodes/test_rtt.cpp +++ b/lib/nodes/test_rtt.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -34,22 +35,22 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static struct plugin p; static int test_rtt_case_start(struct vnode *n, int id) { - int ret; struct test_rtt *t = (struct test_rtt *) n->_vd; struct test_rtt_case *c = (struct test_rtt_case *) vlist_at(&t->cases, id); n->logger->info("Starting case #{}: filename={}, rate={}, values={}, limit={}", t->current, c->filename_formatted, c->rate, c->values, c->limit); /* Open file */ - ret = io_open(&t->io, c->filename_formatted); - if (ret) - return ret; + t->stream = fopen(c->filename_formatted, "a+"); + if (!t->stream) + return -1; /* Start timer. */ t->task.setRate(c->rate); @@ -68,10 +69,9 @@ static int test_rtt_case_stop(struct vnode *n, int id) /* Stop timer */ t->task.stop(); - /* Close file */ - ret = io_close(&t->io); + ret = fclose(t->stream); if (ret) - return ret; + throw SystemError("Failed to close file"); n->logger->info("Stopping case #{}", id); @@ -126,7 +126,6 @@ int test_rtt_parse(struct vnode *n, json_t *json) int ret; struct test_rtt *t = (struct test_rtt *) n->_vd; - const char *format = "villas.binary"; const char *output = "."; const char *prefix = node_name_short(n); @@ -134,7 +133,7 @@ int test_rtt_parse(struct vnode *n, json_t *json) std::vector values; size_t i; - json_t *json_cases, *json_case, *json_val; + json_t *json_cases, *json_case, *json_val, *json_format = nullptr; json_t *json_rates = nullptr, *json_values = nullptr; json_error_t err; @@ -145,10 +144,10 @@ int test_rtt_parse(struct vnode *n, json_t *json) if (ret) return ret; - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: F, s: o }", + ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: o, s?: F, s: o }", "prefix", &prefix, "output", &output, - "format", &format, + "format", &json_format, "cooldown", &t->cooldown, "cases", &json_cases ); @@ -159,10 +158,12 @@ int test_rtt_parse(struct vnode *n, json_t *json) t->prefix = strdup(prefix); /* Initialize IO module */ - t->format = format_type_lookup(format); - if (!t->format) - throw ConfigError(json, "node-config-node-test-rtt-format", "Invalid value for setting 'format'"); + if (!json_format) + json_format = json_string("villas.binary"); + t->formatter = FormatFactory::make(json_format); + if (!t->formatter) + throw ConfigError(json, "node-config-node-test-rtt-format", "Invalid value for setting 'format'"); /* Construct vlist of test cases */ if (!json_is_array(json_cases)) @@ -296,9 +297,7 @@ int test_rtt_start(struct vnode *n) throw SystemError("Failed to create output directory: {}", t->output); } - ret = io_init(&t->io, t->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_DATA); - if (ret) - return ret; + t->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_DATA); t->task.setRate(c->rate); @@ -319,14 +318,12 @@ int test_rtt_stop(struct vnode *n) return ret; } - ret = io_destroy(&t->io); - if (ret) - return ret; + delete t->formatter; return 0; } -int test_rtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int test_rtt_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; unsigned i; @@ -398,7 +395,7 @@ int test_rtt_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned } } -int test_rtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int test_rtt_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct test_rtt *t = (struct test_rtt *) n->_vd; @@ -414,7 +411,7 @@ int test_rtt_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigne continue; } - io_print(&t->io, &smps[i], 1); + t->formatter->print(t->stream, smps[i]); } return i; diff --git a/lib/nodes/uldaq.cpp b/lib/nodes/uldaq.cpp index 015121620..13d447d2c 100644 --- a/lib/nodes/uldaq.cpp +++ b/lib/nodes/uldaq.cpp @@ -32,6 +32,7 @@ #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static unsigned num_devs = ULDAQ_MAX_DEV_COUNT; @@ -585,7 +586,7 @@ int uldaq_stop(struct vnode *n) return 0; } -int uldaq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int uldaq_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { struct uldaq *u = (struct uldaq *) n->_vd; diff --git a/lib/nodes/websocket.cpp b/lib/nodes/websocket.cpp index ffd05c80b..4968abfaa 100644 --- a/lib/nodes/websocket.cpp +++ b/lib/nodes/websocket.cpp @@ -32,11 +32,10 @@ #include #include #include -#include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; #define DEFAULT_WEBSOCKET_BUFFER_SIZE (1 << 12) @@ -91,9 +90,7 @@ static int websocket_connection_init(struct websocket_connection *c) if (ret) return ret; - ret = io_init(&c->io, c->format, &c->node->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + c->formatter->start(&c->node->in.signals, ~(int) SampleFlags::HAS_OFFSET); c->buffers.recv = new Buffer(DEFAULT_WEBSOCKET_BUFFER_SIZE); c->buffers.send = new Buffer(DEFAULT_WEBSOCKET_BUFFER_SIZE); @@ -125,10 +122,7 @@ static int websocket_connection_destroy(struct websocket_connection *c) if (ret) return ret; - ret = io_destroy(&c->io); - if (ret) - return ret; - + delete c->formatter; delete c->buffers.recv; delete c->buffers.send; @@ -140,7 +134,7 @@ static int websocket_connection_destroy(struct websocket_connection *c) return 0; } -static int websocket_connection_write(struct websocket_connection *c, struct sample *smps[], unsigned cnt) +static int websocket_connection_write(struct websocket_connection *c, struct sample * const smps[], unsigned cnt) { int pushed; @@ -224,8 +218,8 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi return -1; } - c->format = format_type_lookup(format); - if (!c->format) { + c->formatter = FormatFactory::make(format); + if (!c->formatter) { websocket_connection_close(c, wsi, LWS_CLOSE_STATUS_POLICY_VIOLATION, "Unknown format"); c->node->logger->warn("Failed to find format: format={}", format); return -1; @@ -277,9 +271,9 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi pulled = queue_pull_many(&c->queue, (void **) smps, cnt); if (pulled > 0) { size_t wbytes; - io_sprint(&c->io, c->buffers.send->data() + LWS_PRE, c->buffers.send->size() - LWS_PRE, &wbytes, smps, pulled); + c->formatter->sprint(c->buffers.send->data() + LWS_PRE, c->buffers.send->size() - LWS_PRE, &wbytes, smps, pulled); - ret = lws_write(wsi, (unsigned char *) c->buffers.send->data() + LWS_PRE, wbytes, c->io.flags & (int) IOFlags::HAS_BINARY_PAYLOAD ? LWS_WRITE_BINARY : LWS_WRITE_TEXT); + ret = lws_write(wsi, (unsigned char *) c->buffers.send->data() + LWS_PRE, wbytes, c->formatter->isBinaryPayload() ? LWS_WRITE_BINARY : LWS_WRITE_TEXT); sample_decref_many(smps, pulled); @@ -319,7 +313,7 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi if (avail < cnt) c->node->logger->warn("Pool underrun for connection: {}", websocket_connection_name(c)); - recvd = io_sscan(&c->io, c->buffers.recv->data(), c->buffers.recv->size(), nullptr, smps, avail); + recvd = c->formatter->sscan(c->buffers.recv->data(), c->buffers.recv->size(), nullptr, smps, avail); if (recvd < 0) { c->node->logger->warn("Failed to parse sample data received on connection: {}", websocket_connection_name(c)); break; @@ -401,8 +395,8 @@ int websocket_start(struct vnode *n) else format = "villas.web"; - c->format = format_type_lookup(format); - if (!c->format) + c->formatter = FormatFactory::make(format); + if (!c->formatter) return -1; c->node = n; @@ -474,7 +468,7 @@ int websocket_destroy(struct vnode *n) return 0; } -int websocket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int websocket_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int avail; @@ -491,7 +485,7 @@ int websocket_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigne return avail; } -int websocket_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int websocket_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int avail; diff --git a/lib/nodes/zeromq.cpp b/lib/nodes/zeromq.cpp index 899f3cf91..6dde1bc64 100644 --- a/lib/nodes/zeromq.cpp +++ b/lib/nodes/zeromq.cpp @@ -29,13 +29,14 @@ #include #include +#include #include #include #include -#include #include using namespace villas; +using namespace villas::node; using namespace villas::utils; static void *context; @@ -156,14 +157,14 @@ int zeromq_parse(struct vnode *n, json_t *json) const char *type = nullptr; const char *in_filter = nullptr; const char *out_filter = nullptr; - const char *format = "villas.binary"; + json_error_t err; json_t *json_in_ep = nullptr; json_t *json_out_ep = nullptr; json_t *json_curve = nullptr; - json_error_t err; + json_t *json_format = nullptr; - ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: o, s?: s, s?: b }, s?: { s?: o, s?: s, s?: b }, s?: o, s?: s, s?: b, s?: s }", + ret = json_unpack_ex(json, &err, 0, "{ s?: { s?: o, s?: s, s?: b }, s?: { s?: o, s?: s, s?: b }, s?: o, s?: s, s?: b, s?: o }", "in", "subscribe", &json_in_ep, "filter", &in_filter, @@ -175,7 +176,7 @@ int zeromq_parse(struct vnode *n, json_t *json) "curve", &json_curve, "pattern", &type, "ipv6", &z->ipv6, - "format", &format + "format", &json_format ); if (ret) throw ConfigError(json, err, "node-config-node-zeromq"); @@ -183,9 +184,12 @@ int zeromq_parse(struct vnode *n, json_t *json) z->in.filter = in_filter ? strdup(in_filter) : nullptr; z->out.filter = out_filter ? strdup(out_filter) : nullptr; - z->format = format_type_lookup(format); - if (!z->format) - throw ConfigError(json, "node-config-node-zeromq-format", "Invalid format '{}'", format); + /* Format */ + z->formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make("villas.binary"); + if (!z->formatter) + throw ConfigError(json_format, "node-config-node-zeromq-format", "Invalid format configuration"); if (json_out_ep) { ret = zeromq_parse_endpoints(json_out_ep, &z->out.endpoints); @@ -259,8 +263,7 @@ char * zeromq_print(struct vnode *n) #endif } - strcatf(&buf, "format=%s, pattern=%s, ipv6=%s, crypto=%s, in.bind=%s, out.bind=%s, in.subscribe=[ ", - format_type_name(z->format), + strcatf(&buf, "pattern=%s, ipv6=%s, crypto=%s, in.bind=%s, out.bind=%s, in.subscribe=[ ", pattern, z->ipv6 ? "yes" : "no", z->curve.enabled ? "yes" : "no", @@ -323,9 +326,7 @@ int zeromq_start(struct vnode *n) struct zeromq::Dir* dirs[] = { &z->out, &z->in }; - ret = io_init(&z->io, z->format, &n->in.signals, (int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET); - if (ret) - return ret; + z->formatter->start(&n->in.signals, ~(int) SampleFlags::HAS_OFFSET); switch (z->pattern) { #ifdef ZMQ_BUILD_DISH @@ -492,9 +493,7 @@ int zeromq_stop(struct vnode *n) return ret; } - ret = io_destroy(&z->io); - if (ret) - return ret; + delete z->formatter; return 0; } @@ -517,7 +516,7 @@ int zeromq_destroy(struct vnode *n) return 0; } -int zeromq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int zeromq_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { int recv, ret; struct zeromq *z = (struct zeromq *) n->_vd; @@ -544,7 +543,7 @@ int zeromq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * if (ret < 0) return ret; - recv = io_sscan(&z->io, (const char *) zmq_msg_data(&m), zmq_msg_size(&m), nullptr, smps, cnt); + recv = z->formatter->sscan((const char *) zmq_msg_data(&m), zmq_msg_size(&m), nullptr, smps, cnt); ret = zmq_msg_close(&m); if (ret) @@ -553,7 +552,7 @@ int zeromq_read(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned * return recv; } -int zeromq_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned *release) +int zeromq_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int ret; struct zeromq *z = (struct zeromq *) n->_vd; @@ -563,7 +562,7 @@ int zeromq_write(struct vnode *n, struct sample *smps[], unsigned cnt, unsigned char data[4096]; - ret = io_sprint(&z->io, data, sizeof(data), &wbytes, smps, cnt); + ret = z->formatter->sprint(data, sizeof(data), &wbytes, smps, cnt); if (ret <= 0) return -1; diff --git a/lib/path.cpp b/lib/path.cpp index a0a575cf6..84d892c5c 100644 --- a/lib/path.cpp +++ b/lib/path.cpp @@ -251,6 +251,8 @@ int path_prepare(struct vpath *p, struct vlist *nodes) if (ret) return ret; + p->muxed = path_is_muxed(p); + /* Create path sources */ std::map pss; for (size_t i = 0; i < vlist_length(&p->mappings); i++) { @@ -378,7 +380,7 @@ int path_prepare(struct vpath *p, struct vlist *nodes) hook_list_prepare(&p->hooks, &p->signals, m, p, nullptr); #endif /* WITH_HOOKS */ - p->logger->info("Prepared path {} with output signals:", path_name(p)); + p->logger->info("Prepared path {} with {} output signals", path_name(p), vlist_length(path_output_signals(p))); signal_list_dump(p->logger, path_output_signals(p)); p->state = State::PREPARED; @@ -459,10 +461,10 @@ int path_parse(struct vpath *p, json_t *json, struct vlist *nodes, const uuid_t for (size_t i = 0; i < vlist_length(&destinations); i++) { struct vnode *n = (struct vnode *) vlist_at(&destinations, i); - if (n->output_path) + if (n->out.path) throw ConfigError(json, "node-config-path", "Every node must only be used by a single path as destination"); - n->output_path = p; + n->out.path = p; auto *pd = new struct vpath_destination; if (!pd) @@ -779,6 +781,28 @@ bool path_is_simple(const struct vpath *p) return true; } +bool path_is_muxed(const struct vpath *p) +{ + if (vlist_length(&p->sources) > 0) + return true; + + if (vlist_length(&p->mappings) > 0) + return true; + + struct mapping_entry *me = (struct mapping_entry *) vlist_at_safe(&p->mappings, 0); + + if (me->type != MappingType::DATA) + return true; + + if (me->data.offset != 0) + return true; + + if (me->length != -1) + return true; + + return false; +} + bool path_is_enabled(const struct vpath *p) { return p->enabled; diff --git a/lib/path_destination.cpp b/lib/path_destination.cpp index a0a482d17..08fdad672 100644 --- a/lib/path_destination.cpp +++ b/lib/path_destination.cpp @@ -61,7 +61,7 @@ int path_destination_destroy(struct vpath_destination *pd) return 0; } -void path_destination_enqueue(struct vpath *p, struct sample *smps[], unsigned cnt) +void path_destination_enqueue(struct vpath *p, const struct sample * const smps[], unsigned cnt) { unsigned enqueued, cloned; @@ -91,9 +91,7 @@ void path_destination_write(struct vpath_destination *pd, struct vpath *p) { int cnt = pd->node->out.vectorize; int sent; - int released; int allocated; - unsigned release; struct sample *smps[cnt]; @@ -107,9 +105,7 @@ void path_destination_write(struct vpath_destination *pd, struct vpath *p) p->logger->debug("Dequeued {} samples from queue of node {} which is part of path {}", allocated, node_name(pd->node), path_name(p)); - release = allocated; - - sent = node_write(pd->node, smps, allocated, &release); + sent = node_write(pd->node, smps, allocated); if (sent < 0) { p->logger->error("Failed to sent {} samples to node {}: reason={}", cnt, node_name(pd->node), sent); return; @@ -117,7 +113,7 @@ void path_destination_write(struct vpath_destination *pd, struct vpath *p) else if (sent < allocated) p->logger->debug("Partial write to node {}: written={}, expected={}", node_name(pd->node), sent, allocated); - released = sample_decref_many(smps, release); + int released = sample_decref_many(smps, allocated); p->logger->debug("Released {} samples back to memory pool", released); } @@ -130,4 +126,4 @@ void path_destination_check(struct vpath_destination *pd) if (!node_type(pd->node)->write) throw RuntimeError("Destiation node {} is not supported as a sink for path ", node_name(pd->node)); -} \ No newline at end of file +} diff --git a/lib/path_source.cpp b/lib/path_source.cpp index 9753c6ca5..ffbe81292 100644 --- a/lib/path_source.cpp +++ b/lib/path_source.cpp @@ -118,7 +118,6 @@ int path_source_destroy(struct vpath_source *ps) int path_source_read(struct vpath_source *ps, struct vpath *p, int i) { int ret, recv, tomux, allocated, cnt, toenqueue, enqueued = 0; - unsigned release; cnt = ps->node->in.vectorize; @@ -132,9 +131,7 @@ int path_source_read(struct vpath_source *ps, struct vpath *p, int i) p->logger->warn("Pool underrun for path source {}", node_name(ps->node)); /* Read ready samples and store them to blocks pointed by smps[] */ - release = allocated; - - recv = node_read(ps->node, read_smps, allocated, &release); + recv = node_read(ps->node, read_smps, allocated); if (recv == 0) { enqueued = 0; goto out2; @@ -159,13 +156,12 @@ int path_source_read(struct vpath_source *ps, struct vpath *p, int i) auto *sps = (struct vpath_source *) vlist_at(&ps->secondaries, i); int sent; - unsigned release = recv; - sent = node_write(sps->node, read_smps, recv, &release); + sent = node_write(sps->node, read_smps, recv); if (sent < recv) p->logger->warn("Partial write to secondary path source {} of path {}", node_name(sps->node), path_name(p)); - sample_incref_many(read_smps+release, recv-release); + sample_incref_many(read_smps, recv); } p->received.set(i); @@ -243,7 +239,7 @@ int path_source_read(struct vpath_source *ps, struct vpath *p, int i) } sample_decref_many(muxed_smps, tomux); -out2: sample_decref_many(read_smps, release); +out2: sample_decref_many(read_smps, recv); return enqueued; } diff --git a/lib/queue.cpp b/lib/queue.cpp index c08097e81..52636afcc 100644 --- a/lib/queue.cpp +++ b/lib/queue.cpp @@ -105,7 +105,7 @@ int queue_push(struct queue *q, void *ptr) buffer = (struct queue_cell *) ((char *) q + q->buffer_off); pos = std::atomic_load_explicit(&q->tail, std::memory_order_relaxed); - for (;;) { + while (true) { cell = &buffer[pos & q->buffer_mask]; seq = std::atomic_load_explicit(&cell->sequence, std::memory_order_acquire); diff = (intptr_t) seq - (intptr_t) pos; @@ -137,7 +137,7 @@ int queue_pull(struct queue *q, void **ptr) buffer = (struct queue_cell *) ((char *) q + q->buffer_off); pos = std::atomic_load_explicit(&q->head, std::memory_order_relaxed); - for (;;) { + while (true) { cell = &buffer[pos & q->buffer_mask]; seq = std::atomic_load_explicit(&cell->sequence, std::memory_order_acquire); diff = (intptr_t) seq - (intptr_t) (pos + 1); diff --git a/lib/sample.cpp b/lib/sample.cpp index 3191692a2..97575e411 100644 --- a/lib/sample.cpp +++ b/lib/sample.cpp @@ -114,7 +114,7 @@ void sample_free_many(struct sample *smps[], int cnt) sample_free(smps[i]); } -int sample_decref_many(struct sample *smps[], int cnt) +int sample_decref_many(struct sample * const smps[], int cnt) { int released = 0; @@ -126,7 +126,7 @@ int sample_decref_many(struct sample *smps[], int cnt) return released; } -int sample_incref_many(struct sample *smps[], int cnt) +int sample_incref_many(struct sample * const smps[], int cnt) { for (int i = 0; i < cnt; i++) sample_incref(smps[i]); @@ -150,7 +150,7 @@ int sample_decref(struct sample *s) return prev - 1; } -int sample_copy(struct sample *dst, struct sample *src) +int sample_copy(struct sample *dst, const struct sample *src) { dst->length = MIN(src->length, dst->capacity); @@ -182,7 +182,7 @@ struct sample * sample_clone(struct sample *orig) return clone; } -int sample_clone_many(struct sample *clones[], struct sample *origs[], int cnt) +int sample_clone_many(struct sample *dsts[], const struct sample * const srcs[], int cnt) { int alloced, copied; struct pool *pool; @@ -190,18 +190,18 @@ int sample_clone_many(struct sample *clones[], struct sample *origs[], int cnt) if (cnt <= 0) return 0; - pool = sample_pool(origs[0]); + pool = sample_pool(srcs[0]); if (!pool) return 0; - alloced = sample_alloc_many(pool, clones, cnt); + alloced = sample_alloc_many(pool, dsts, cnt); - copied = sample_copy_many(clones, origs, alloced); + copied = sample_copy_many(dsts, srcs, alloced); return copied; } -int sample_copy_many(struct sample *dsts[], struct sample *srcs[], int cnt) +int sample_copy_many(struct sample * const dsts[], const struct sample * const srcs[], int cnt) { for (int i = 0; i < cnt; i++) sample_copy(dsts[i], srcs[i]); diff --git a/lib/shmem.cpp b/lib/shmem.cpp index c5d69b51e..f44bbb174 100644 --- a/lib/shmem.cpp +++ b/lib/shmem.cpp @@ -183,7 +183,7 @@ int shmem_int_close(struct shmem_int *shm) return 0; } -int shmem_int_read(struct shmem_int *shm, struct sample *smps[], unsigned cnt) +int shmem_int_read(struct shmem_int *shm, struct sample * const smps[], unsigned cnt) { int ret; @@ -197,7 +197,7 @@ int shmem_int_read(struct shmem_int *shm, struct sample *smps[], unsigned cnt) return ret; } -int shmem_int_write(struct shmem_int *shm, struct sample *smps[], unsigned cnt) +int shmem_int_write(struct shmem_int *shm, const struct sample * const smps[], unsigned cnt) { int ret; diff --git a/lib/signal_data.cpp b/lib/signal_data.cpp index 2f490b324..8da92622f 100644 --- a/lib/signal_data.cpp +++ b/lib/signal_data.cpp @@ -236,7 +236,7 @@ int signal_data_parse_json(union signal_data *data, enum SignalType type, json_t return 0; } -json_t * signal_data_to_json(union signal_data *data, enum SignalType type) +json_t * signal_data_to_json(const union signal_data *data, enum SignalType type) { switch (type) { case SignalType::INTEGER: @@ -261,11 +261,11 @@ json_t * signal_data_to_json(union signal_data *data, enum SignalType type) return nullptr; } -int signal_data_print_str(const union signal_data *data, enum SignalType type, char *buf, size_t len) +int signal_data_print_str(const union signal_data *data, enum SignalType type, char *buf, size_t len, int precision) { switch (type) { case SignalType::FLOAT: - return snprintf(buf, len, "%.6f", data->f); + return snprintf(buf, len, "%.*f", precision, data->f); case SignalType::INTEGER: return snprintf(buf, len, "%" PRIi64, data->i); @@ -274,7 +274,7 @@ int signal_data_print_str(const union signal_data *data, enum SignalType type, c return snprintf(buf, len, "%u", data->b); case SignalType::COMPLEX: - return snprintf(buf, len, "%.6f%+.6fi", std::real(data->z), std::imag(data->z)); + return snprintf(buf, len, "%.*f%+.*fi", precision, std::real(data->z), precision, std::imag(data->z)); default: return snprintf(buf, len, ""); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45eed5760..1f7a99078 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,13 +21,13 @@ ################################################################################### set(SRCS - villas-node - villas-test-config - villas-test-rtt - villas-test-cmp villas-convert + villas-node villas-pipe villas-signal + villas-compare + villas-test-config + villas-test-rtt ) add_executable(villas-node villas-node.cpp) @@ -39,8 +39,8 @@ target_link_libraries(villas-test-config PUBLIC villas) add_executable(villas-test-rtt villas-test-rtt.cpp) target_link_libraries(villas-test-rtt PUBLIC villas) -add_executable(villas-test-cmp villas-test-cmp.cpp) -target_link_libraries(villas-test-cmp PUBLIC villas) +add_executable(villas-compare villas-compare.cpp) +target_link_libraries(villas-compare PUBLIC villas) add_executable(villas-convert villas-convert.cpp) target_link_libraries(villas-convert PUBLIC villas) diff --git a/src/villas-test-cmp.cpp b/src/villas-compare.cpp similarity index 81% rename from src/villas-test-cmp.cpp rename to src/villas-compare.cpp index 6c8601131..bdab42ef1 100644 --- a/src/villas-test-cmp.cpp +++ b/src/villas-compare.cpp @@ -21,14 +21,13 @@ *********************************************************************************/ #include -#include +#include #include #include #include -#include -#include +#include #include #include #include @@ -41,52 +40,65 @@ namespace villas { namespace node { namespace tools { -class TestCmpSide { +class CompareSide { public: std::string path; std::string dtypes; + std::string format; struct sample *sample; - struct io io; - struct format_type *format; + Format *formatter; - TestCmpSide(const std::string &pth, struct format_type *fmt, const std::string &dt, struct pool *p) : + FILE *stream; + + CompareSide(const CompareSide&) = delete; + CompareSide & operator=(const CompareSide&) = delete; + + CompareSide(const std::string &pth, const std::string &fmt, const std::string &dt, struct pool *p) : path(pth), dtypes(dt), format(fmt) { - int ret; + json_t *json_format; + json_error_t err; - ret = io_init2(&io, format, dtypes.c_str(), 0); - if (ret) - throw RuntimeError("Failed to initialize IO"); + /* Try parsing format config as JSON */ + json_format = json_loads(format.c_str(), 0, &err); + formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make(format); + if (!formatter) + throw RuntimeError("Failed to initialize formatter"); - ret = io_open(&io, path.c_str()); - if (ret) - throw RuntimeError("Failed to open file: {}", path); + formatter->start(dtypes); + + stream = fopen(path.c_str(), "r"); + if (!stream) + throw SystemError("Failed to open file: {}", path); sample = sample_alloc(p); if (!sample) throw RuntimeError("Failed to allocate samples"); } - ~TestCmpSide() noexcept(false) + ~CompareSide() noexcept(false) { int ret __attribute((unused)); - ret = io_close(&io); - ret = io_destroy(&io); + ret = fclose(stream); + + delete formatter; sample_decref(sample); } }; -class TestCmp : public Tool { +class Compare : public Tool { public: - TestCmp(int argc, char *argv[]) : + Compare(int argc, char *argv[]) : Tool(argc, argv, "test-cmp"), pool(), epsilon(1e-9), @@ -113,7 +125,7 @@ protected: void usage() { - std::cout << "Usage: villas-test-cmp [OPTIONS] FILE1 FILE2 ... FILEn" << std::endl + std::cout << "Usage: villas-compare [OPTIONS] FILE1 FILE2 ... FILEn" << std::endl << " FILE a list of files to compare" << std::endl << " OPTIONS is one or more of the following options:" << std::endl << " -d LVL adjust the debug level" << std::endl @@ -202,18 +214,14 @@ check: if (optarg == endptr) int ret, rc = 0, line, failed; unsigned eofs; - struct format_type *fmt = format_type_lookup(format.c_str()); - if (!fmt) - throw RuntimeError("Invalid IO format: {}", format); - ret = pool_init(&pool, filenames.size(), SAMPLE_LENGTH(DEFAULT_SAMPLE_LENGTH), &memory_heap); if (ret) throw RuntimeError("Failed to initialize pool"); /* Open files */ - std::vector sides; + std::vector sides; for (auto filename : filenames) { - auto *s = new TestCmpSide(filename, fmt, dtypes, &pool); + auto *s = new CompareSide(filename, format, dtypes, &pool); if (!s) throw MemoryAllocationError(); @@ -221,11 +229,11 @@ check: if (optarg == endptr) } line = 0; - for (;;) { + while (true) { /* Read next sample from all files */ retry: eofs = 0; for (auto side : sides) { - ret = io_eof(&side->io); + ret = feof(side->stream); if (ret) eofs++; } @@ -243,7 +251,7 @@ retry: eofs = 0; failed = 0; for (auto side : sides) { - ret = io_scan(&side->io, &side->sample, 1); + ret = side->formatter->scan(side->stream, side->sample); if (ret <= 0) failed++; } @@ -279,7 +287,7 @@ out: for (auto side : sides) int main(int argc, char *argv[]) { - villas::node::tools::TestCmp t(argc, argv); + villas::node::tools::Compare t(argc, argv); return t.run(); } diff --git a/src/villas-conf2json.cpp b/src/villas-conf2json.cpp index 27a18b2a1..15d4e49bc 100644 --- a/src/villas-conf2json.cpp +++ b/src/villas-conf2json.cpp @@ -63,7 +63,7 @@ protected: } FILE *f = fopen(argv[1], "r"); - if(f == nullptr) + if (f == nullptr) return -1; const char *confdir = dirname(argv[1]); diff --git a/src/villas-convert.cpp b/src/villas-convert.cpp index 2871657b2..9515bb4e9 100644 --- a/src/villas-convert.cpp +++ b/src/villas-convert.cpp @@ -25,11 +25,12 @@ *********************************************************************************/ #include +#include #include #include #include -#include +#include #include #include #include @@ -65,7 +66,7 @@ protected: struct { std::string name; std::string format; - struct io io; + Format *formatter; } dirs[2]; void usage() @@ -126,42 +127,35 @@ protected: int ret; for (unsigned i = 0; i < ARRAY_LEN(dirs); i++) { - struct format_type *ft; + json_t *json_format; + json_error_t err; + std::string format = dirs[i].format; - ft = format_type_lookup(dirs[i].format.c_str()); - if (!ft) - throw RuntimeError("Invalid format: {}", dirs[i].format); + /* Try parsing format config as JSON */ + json_format = json_loads(format.c_str(), 0, &err); + dirs[i].formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make(format); + if (!dirs[i].formatter) + throw RuntimeError("Failed to initialize format: {}", dirs[i].name); - ret = io_init2(&dirs[i].io, ft, dtypes.c_str(), (int) SampleFlags::HAS_ALL); - if (ret) - throw RuntimeError("Failed to initialize IO: {}", dirs[i].name); - - ret = io_open(&dirs[i].io, nullptr); - if (ret) - throw RuntimeError("Failed to open IO"); + dirs[i].formatter->start(dtypes); } struct sample *smp = sample_alloc_mem(DEFAULT_SAMPLE_LENGTH); - for (;;) { - ret = io_scan(&dirs[0].io, &smp, 1); + while (true) { + ret = dirs[0].formatter->scan(stdin, smp); if (ret == 0) continue; - if (ret < 0) + else if (ret < 0) break; - io_print(&dirs[1].io, &smp, 1); + dirs[1].formatter->print(stdout, smp); } - for (unsigned i = 0; i < ARRAY_LEN(dirs); i++) { - ret = io_close(&dirs[i].io); - if (ret) - throw RuntimeError("Failed to close IO"); - - ret = io_destroy(&dirs[i].io); - if (ret) - throw RuntimeError("Failed to destroy IO"); - } + for (unsigned i = 0; i < ARRAY_LEN(dirs); i++) + delete dirs[i].formatter; return 0; } diff --git a/src/villas-hook.cpp b/src/villas-hook.cpp index 46df2bb68..e5b7a5a39 100644 --- a/src/villas-hook.cpp +++ b/src/villas-hook.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include #include @@ -93,8 +93,8 @@ protected: std::string dtypes; struct pool p; - struct io input; - struct io output; + Format *input; + Format *output; int cnt; @@ -113,22 +113,21 @@ protected: << " OPTIONS is one or more of the following options:" << std::endl << " -c CONFIG a JSON file containing just the hook configuration" << std::endl << " -f FMT the input data format" << std::endl - << " -F FMT the output data format" << std::endl + << " -F FMT the output data format (defaults to input format)" << std::endl << " -t DT the data-type format string" << std::endl << " -d LVL set debug level to LVL" << std::endl << " -v CNT process CNT smps at once" << std::endl << " -h show this help" << std::endl << " -V show the version of the tool" << std::endl << std::endl; -#ifdef WITH_HOOKS std::cout << "Supported hooks:" << std::endl; for (Plugin *p : Registry::lookup()) std::cout << " - " << p->getName() << ": " << p->getDescription() << std::endl; std::cout << std::endl; -#endif /* WITH_HOOKS */ std::cout << "Supported IO formats:" << std::endl; - plugin_dump(PluginType::FORMAT); + for (Plugin *p : Registry::lookup()) + std::cout << " - " << p->getName() << ": " << p->getDescription() << std::endl; std::cout << std::endl; std::cout << "Example:" << std::endl @@ -229,9 +228,9 @@ check: if (optarg == endptr) /* Initialize IO */ struct desc { - const char *dir; - const char *format; - struct io *io; + std::string dir; + std::string format; + Format **formatter; }; std::list descs = { { "in", input_format.c_str(), &input }, @@ -239,17 +238,18 @@ check: if (optarg == endptr) }; for (auto &d : descs) { - struct format_type *ft = format_type_lookup(d.format); - if (!ft) - throw RuntimeError("Unknown {} IO format '{}'", d.dir, d.format); + json_t *json_format; + json_error_t err; - ret = io_init2(d.io, ft, dtypes.c_str(), (int) SampleFlags::HAS_ALL); - if (ret) + /* Try parsing format config as JSON */ + json_format = json_loads(d.format.c_str(), 0, &err); + (*d.formatter) = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make(d.format); + if (!(*d.formatter)) throw RuntimeError("Failed to initialize {} IO", d.dir); - ret = io_open(d.io, nullptr); - if (ret) - throw RuntimeError("Failed to open {} IO", d.dir); + (*d.formatter)->start(dtypes, (int) SampleFlags::HAS_ALL); } /* Initialize hook */ @@ -263,7 +263,7 @@ check: if (optarg == endptr) h->parse(config); h->check(); - h->prepare(input.signals); + h->prepare(input->getSignals()); h->start(); while (!stop) { @@ -271,9 +271,9 @@ check: if (optarg == endptr) if (ret != cnt) throw RuntimeError("Failed to allocate {} smps from pool", cnt); - recv = io_scan(&input, smps, cnt); + recv = input->scan(stdin, smps, cnt); if (recv < 0) { - if (io_eof(&input)) + if (feof(stdin)) break; throw RuntimeError("Failed to read from stdin"); @@ -313,7 +313,7 @@ check: if (optarg == endptr) smp->signals = h->getSignals(); } -stop: sent = io_print(&output, smps, send); +stop: sent = output->print(stdout, smps, send); if (sent < 0) throw RuntimeError("Failed to write to stdout"); @@ -324,15 +324,8 @@ stop: sent = io_print(&output, smps, send); delete h; - for (auto &d : descs) { - ret = io_close(d.io); - if (ret) - throw RuntimeError("Failed to close {} IO", d.dir); - - ret = io_destroy(d.io); - if (ret) - throw RuntimeError("Failed to destroy {} IO", d.dir); - } + for (auto &d : descs) + delete (*d.formatter); sample_free_many(smps, cnt); diff --git a/src/villas-node.cpp b/src/villas-node.cpp index 5f91c4088..376f0bdd6 100644 --- a/src/villas-node.cpp +++ b/src/villas-node.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -106,7 +107,8 @@ protected: std::cout << std::endl; std::cout << "Supported IO formats:" << std::endl; - plugin_dump(PluginType::FORMAT); + for (Plugin *p : Registry::lookup()) + std::cout << " - " << p->getName() << ": " << p->getDescription() << std::endl; std::cout << std::endl; #ifdef WITH_HOOKS diff --git a/src/villas-pipe.cpp b/src/villas-pipe.cpp index 4ed784335..91cd381c6 100644 --- a/src/villas-pipe.cpp +++ b/src/villas-pipe.cpp @@ -43,10 +43,9 @@ #include #include #include -#include +#include #include #include -#include #include #include @@ -59,7 +58,7 @@ class PipeDirection { protected: struct pool pool; struct vnode *node; - struct io *io; + Format *formatter; std::thread thread; Logger logger; @@ -68,9 +67,9 @@ protected: bool enabled; int limit; public: - PipeDirection(struct vnode *n, struct io *i, bool en, int lim, const std::string &name) : + PipeDirection(struct vnode *n, Format *fmt, bool en, int lim, const std::string &name) : node(n), - io(i), + formatter(fmt), stop(false), enabled(en), limit(lim) @@ -120,25 +119,25 @@ public: class PipeSendDirection : public PipeDirection { public: - PipeSendDirection(struct vnode *n, struct io *i, bool en = true, int lim = -1) : + PipeSendDirection(struct vnode *n, Format *i, bool en = true, int lim = -1) : PipeDirection(n, i, en, lim, "send") { } virtual void run() { - unsigned last_sequenceno = 0, release; + unsigned last_sequenceno = 0; int scanned, sent, allocated, cnt = 0; struct sample *smps[node->out.vectorize]; - while (!stop && !io_eof(io)) { + while (!stop && !feof(stdin)) { allocated = sample_alloc_many(&pool, smps, node->out.vectorize); if (allocated < 0) throw RuntimeError("Failed to get {} samples out of send pool.", node->out.vectorize); else if (allocated < (int) node->out.vectorize) logger->warn("Send pool underrun"); - scanned = io_scan(io, smps, allocated); + scanned = formatter->scan(stdin, smps, allocated); if (scanned < 0) { if (stop) goto leave2; @@ -157,11 +156,9 @@ public: smps[i]->sequence = last_sequenceno++; } - release = allocated; + sent = node_write(node, smps, scanned); - sent = node_write(node, smps, scanned, &release); - - sample_decref_many(smps, release); + sample_decref_many(smps, scanned); cnt += sent; if (limit > 0 && cnt >= limit) @@ -172,7 +169,7 @@ leave2: logger->info("Send thread stopped"); return; -leave: if (io_eof(io)) { +leave: if (feof(stdin)) { if (limit < 0) { logger->info("Reached end-of-file. Terminating..."); raise(SIGINT); @@ -190,14 +187,13 @@ leave: if (io_eof(io)) { class PipeReceiveDirection : public PipeDirection { public: - PipeReceiveDirection(struct vnode *n, struct io *i, bool en = true, int lim = -1) : + PipeReceiveDirection(struct vnode *n, Format *i, bool en = true, int lim = -1) : PipeDirection(n, i, en, lim, "recv") { } virtual void run() { int recv, cnt = 0, allocated = 0; - unsigned release; struct sample *smps[node->in.vectorize]; while (!stop) { @@ -207,9 +203,7 @@ public: else if (allocated < (int) node->in.vectorize) logger->warn("Receive pool underrun: allocated only {} of {} samples", allocated, node->in.vectorize); - release = allocated; - - recv = node_read(node, smps, allocated, &release); + recv = node_read(node, smps, allocated); if (recv < 0) { if (node->state == State::STOPPING || stop) goto leave2; @@ -217,14 +211,14 @@ public: logger->warn("Failed to receive samples from node {}: reason={}", node_name(node), recv); } else { - io_print(io, smps, recv); + formatter->print(stdout, smps, recv); cnt += recv; if (limit > 0 && cnt >= limit) goto leave; } - sample_decref_many(smps, release); + sample_decref_many(smps, allocated); } return; @@ -243,7 +237,7 @@ public: Pipe(int argc, char *argv[]) : Tool(argc, argv, "pipe"), stop(false), - io(), + formatter(), timeout(0), reverse(false), format("villas.human"), @@ -270,7 +264,7 @@ protected: std::atomic stop; SuperNode sn; /**< The global configuration */ - struct io io; + Format *formatter; int timeout; bool reverse; @@ -402,9 +396,9 @@ check: if (optarg == endptr) int main() { int ret; - struct vnode *node; - struct format_type *ft; + json_t *json_format; + json_error_t err; logger->info("Logging level: {}", logging.getLevelName()); @@ -413,17 +407,16 @@ check: if (optarg == endptr) else logger->warn("No configuration file specified. Starting unconfigured. Use the API to configure this instance."); - ft = format_type_lookup(format.c_str()); - if (!ft) - throw RuntimeError("Invalid format: {}", format); - ret = io_init2(&io, ft, dtypes.c_str(), (int) SampleFlags::HAS_ALL); - if (ret) - throw RuntimeError("Failed to initialize IO"); + /* Try parsing format config as JSON */ + json_format = json_loads(format.c_str(), 0, &err); + formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make(format); + if (!formatter) + throw RuntimeError("Failed to initialize formatter"); - ret = io_open(&io, nullptr); - if (ret) - throw RuntimeError("Failed to open IO"); + formatter->start(dtypes); node = sn.getNode(nodestr); if (!node) @@ -464,8 +457,8 @@ check: if (optarg == endptr) if (ret) throw RuntimeError("Failed to start node {}: reason={}", node_name(node), ret); - PipeReceiveDirection recv_dir(node, &io, enable_recv, limit_recv); - PipeSendDirection send_dir(node, &io, enable_send, limit_send); + PipeReceiveDirection recv_dir(node, formatter, enable_recv, limit_recv); + PipeSendDirection send_dir(node, formatter, enable_send, limit_send); recv_dir.startThread(); send_dir.startThread(); @@ -496,13 +489,7 @@ check: if (optarg == endptr) } #endif /* WITH_NODE_WEBSOCKET */ - ret = io_close(&io); - if (ret) - throw RuntimeError("Failed to close IO"); - - ret = io_destroy(&io); - if (ret) - throw RuntimeError("Failed to destroy IO"); + delete formatter; return 0; } diff --git a/src/villas-signal.cpp b/src/villas-signal.cpp index 7e0ee42f8..f729c153a 100644 --- a/src/villas-signal.cpp +++ b/src/villas-signal.cpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include #include @@ -55,9 +55,9 @@ public: Signal(int argc, char *argv[]) : Tool(argc, argv, "signal"), stop(false), - n(), - io(), - q(), + node(), + formatter(nullptr), + pool(), format("villas.human") { int ret; @@ -70,9 +70,9 @@ public: protected: std::atomic stop; - struct vnode n; - struct io io; - struct pool q; + struct vnode node; + Format *formatter; + struct pool pool; std::string format; @@ -213,9 +213,9 @@ check: if (optarg == endptr) int main() { int ret; - json_t *json; + json_t *json, *json_format; + json_error_t err; struct vnode_type *nt; - struct format_type *ft; struct sample *t; @@ -223,7 +223,7 @@ check: if (optarg == endptr) if (!nt) throw RuntimeError("Signal generation is not supported."); - ret = node_init(&n, nt); + ret = node_init(&node, nt); if (ret) throw RuntimeError("Failed to initialize node"); @@ -236,78 +236,69 @@ check: if (optarg == endptr) uuid_t uuid; uuid_clear(uuid); - ret = node_parse(&n, json, uuid); + ret = node_parse(&node, json, uuid); if (ret) { usage(); exit(EXIT_FAILURE); } - ft = format_type_lookup(format.c_str()); - if (!ft) - throw RuntimeError("Invalid output format '{}'", format); - // nt == n._vt ret = node_type_start(nt, nullptr); if (ret) throw RuntimeError("Failed to initialize node type: {}", node_type_name(nt)); - ret = node_check(&n); + ret = node_check(&node); if (ret) throw RuntimeError("Failed to verify node configuration"); - ret = node_prepare(&n); + ret = node_prepare(&node); if (ret) - throw RuntimeError("Failed to prepare node {}: reason={}", node_name(&n), ret); + throw RuntimeError("Failed to prepare node {}: reason={}", node_name(&node), ret); - ret = io_init(&io, ft, &n.in.signals, (int) IOFlags::FLUSH | ((int) SampleFlags::HAS_ALL & ~(int) SampleFlags::HAS_OFFSET)); - if (ret) + /* Try parsing format config as JSON */ + json_format = json_loads(format.c_str(), 0, &err); + formatter = json_format + ? FormatFactory::make(json_format) + : FormatFactory::make(format); + if (!formatter) throw RuntimeError("Failed to initialize output"); - ret = pool_init(&q, 16, SAMPLE_LENGTH(vlist_length(&n.in.signals)), &memory_heap); + formatter->start(&node.in.signals, ~(int) SampleFlags::HAS_OFFSET); + + ret = pool_init(&pool, 16, SAMPLE_LENGTH(vlist_length(&node.in.signals)), &memory_heap); if (ret) throw RuntimeError("Failed to initialize pool"); - ret = io_open(&io, nullptr); + ret = node_start(&node); if (ret) - throw RuntimeError("Failed to open output"); + throw RuntimeError("Failed to start node {}: reason={}", node_name(&node), ret); - ret = node_start(&n); - if (ret) - throw RuntimeError("Failed to start node {}: reason={}", node_name(&n), ret); + while (!stop && node.state == State::STARTED) { + t = sample_alloc(&pool); - while (!stop && n.state == State::STARTED) { - t = sample_alloc(&q); - - unsigned release = 1; // release = allocated - -retry: ret = node_read(&n, &t, 1, &release); +retry: ret = node_read(&node, &t, 1); if (ret == 0) goto retry; else if (ret < 0) goto out; - io_print(&io, &t, 1); + formatter->print(stdout, t); + fflush(stdout); out: sample_decref(t); } - ret = node_stop(&n); + ret = node_stop(&node); if (ret) throw RuntimeError("Failed to stop node"); - ret = node_destroy(&n); + ret = node_destroy(&node); if (ret) throw RuntimeError("Failed to destroy node"); - ret = io_close(&io); - if (ret) - throw RuntimeError("Failed to close IO"); + delete formatter; - ret = io_destroy(&io); - if (ret) - throw RuntimeError("Failed to destroy IO"); - - ret = pool_destroy(&q); + ret = pool_destroy(&pool); if (ret) throw RuntimeError("Failed to destroy pool"); diff --git a/src/villas-test-config.cpp b/src/villas-test-config.cpp index 12101f302..c3d2b7bd7 100644 --- a/src/villas-test-config.cpp +++ b/src/villas-test-config.cpp @@ -21,6 +21,7 @@ *********************************************************************************/ #include +#include #include #include diff --git a/src/villas-test-rtt.cpp b/src/villas-test-rtt.cpp index b9b73b90c..fa806e2ae 100644 --- a/src/villas-test-rtt.cpp +++ b/src/villas-test-rtt.cpp @@ -206,13 +206,8 @@ check: if (optarg == endptr) while (!stop && (count < 0 || count--)) { clock_gettime(CLOCK_ID, &send); - unsigned release; - - release = 1; // release = allocated - node_write(node, &smp_send, 1, &release); /* Ping */ - - release = 1; // release = allocated - node_read(node, &smp_recv, 1, &release); /* Pong */ + node_write(node, &smp_send, 1); /* Ping */ + node_read(node, &smp_recv, 1); /* Pong */ clock_gettime(CLOCK_ID, &recv); diff --git a/test.conf b/test.conf new file mode 100644 index 000000000..6b6079852 --- /dev/null +++ b/test.conf @@ -0,0 +1,18 @@ +nodes = { + signal = { + type = "signal" + signal = "sine" + + in = { + hooks = ( + { type = "print" } + ) + } + } +} + +paths = ( + { + in = "signal" + } +) diff --git a/tests/integration/convert.sh b/tests/integration/convert.sh index f830f32b3..11d4300bf 100755 --- a/tests/integration/convert.sh +++ b/tests/integration/convert.sh @@ -36,5 +36,5 @@ for FORMAT in ${FORMATS}; do villas-convert -o ${FORMAT} < ${INPUT_FILE} | tee ${TEMP} | \ villas-convert -i ${FORMAT} > ${OUTPUT_FILE} - villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} + villas-compare ${INPUT_FILE} ${OUTPUT_FILE} done diff --git a/tests/integration/hook-average.sh b/tests/integration/hook-average.sh index 726890d6e..acc9de4cc 100755 --- a/tests/integration/hook-average.sh +++ b/tests/integration/hook-average.sh @@ -57,7 +57,7 @@ EOF villas-hook -o offset=0 -o signals=0,1,2,3,4 average < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-cast.sh b/tests/integration/hook-cast.sh index 321ff3fd2..1a504afba 100755 --- a/tests/integration/hook-cast.sh +++ b/tests/integration/hook-cast.sh @@ -60,7 +60,7 @@ EOF villas-hook cast -o new_name=test -o new_unit=V -o new_type=integer -o signal=1 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-decimate.sh b/tests/integration/hook-decimate.sh index 985a3f31e..973753903 100755 --- a/tests/integration/hook-decimate.sh +++ b/tests/integration/hook-decimate.sh @@ -49,7 +49,7 @@ EOF villas-hook -o ratio=3 decimate < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? diff --git a/tests/integration/hook-dp.sh b/tests/integration/hook-dp.sh index d940cf89d..46d33566a 100755 --- a/tests/integration/hook-dp.sh +++ b/tests/integration/hook-dp.sh @@ -45,7 +45,7 @@ villas-hook dp -o inverse=true ${OPTS} < ${OUTPUT_FILE} > ${RECON_FILE} exit 0 # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-drop.sh b/tests/integration/hook-drop.sh index fd46ae616..a35b58d36 100755 --- a/tests/integration/hook-drop.sh +++ b/tests/integration/hook-drop.sh @@ -55,7 +55,7 @@ EOF villas-hook drop < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? diff --git a/tests/integration/hook-gate.sh b/tests/integration/hook-gate.sh index 6773cbe65..12b49e0b5 100755 --- a/tests/integration/hook-gate.sh +++ b/tests/integration/hook-gate.sh @@ -52,7 +52,7 @@ EOF villas-hook gate -o signal=2 -o mode=above < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/hook-limit_rate.sh b/tests/integration/hook-limit_rate.sh index bead1e62f..2ae64a759 100755 --- a/tests/integration/hook-limit_rate.sh +++ b/tests/integration/hook-limit_rate.sh @@ -32,7 +32,7 @@ awk 'NR % 10 == 2' < ${INPUT_FILE} > ${EXPECT_FILE} villas-hook -o rate=100 -o mode=origin limit_rate < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? diff --git a/tests/integration/hook-lua.sh b/tests/integration/hook-lua.sh index 51772008e..8c4c768f7 100755 --- a/tests/integration/hook-lua.sh +++ b/tests/integration/hook-lua.sh @@ -74,7 +74,7 @@ echo cat ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/hook-lua_script.sh b/tests/integration/hook-lua_script.sh index aa8eec655..c2951143e 100755 --- a/tests/integration/hook-lua_script.sh +++ b/tests/integration/hook-lua_script.sh @@ -104,7 +104,7 @@ EOF villas-hook lua -c ${CONFIG_FILE} < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/hook-print.sh b/tests/integration/hook-print.sh index 407fbe3e6..9e87bf300 100755 --- a/tests/integration/hook-print.sh +++ b/tests/integration/hook-print.sh @@ -34,7 +34,7 @@ villas-signal -v 1 -r 10 -l ${NUM_SAMPLES} -n random > ${INPUT_FILE} villas-hook -o format=villas.human -o output=${OUTPUT_FILE1} print > ${OUTPUT_FILE2} < ${INPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE1} ${OUTPUT_FILE2} ${INPUT_FILE} +villas-compare ${OUTPUT_FILE1} ${OUTPUT_FILE2} ${INPUT_FILE} RC=$? rm -f ${OUTPUT_FILE1} ${OUTPUT_FILE2} ${INPUT_FILE} diff --git a/tests/integration/hook-scale.sh b/tests/integration/hook-scale.sh index a270e4092..febfdcff3 100755 --- a/tests/integration/hook-scale.sh +++ b/tests/integration/hook-scale.sh @@ -57,7 +57,7 @@ EOF villas-hook scale -o scale=100 -o offset=55 -o signal=signal4 < ${INPUT_FILE} > ${OUTPUT_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm -f ${INPUT_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/node-can.sh b/tests/integration/node-can.sh index cef1340ce..c4c06a985 100755 --- a/tests/integration/node-can.sh +++ b/tests/integration/node-can.sh @@ -144,7 +144,7 @@ sleep 1 kill %1 # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? #rm ${CAN_OUT_FILE} ${INPUT_FILE} ${OUTPUT_FILE} diff --git a/tests/integration/node-hook.sh b/tests/integration/node-hook.sh index 33ad1db36..7c58cacdf 100755 --- a/tests/integration/node-hook.sh +++ b/tests/integration/node-hook.sh @@ -88,7 +88,7 @@ EOF villas-node ${CONFIG_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? cat ${OUTPUT_FILE} diff --git a/tests/integration/node-infiniband.sh b/tests/integration/node-infiniband.sh index 06150195c..701546d8d 100755 --- a/tests/integration/node-infiniband.sh +++ b/tests/integration/node-infiniband.sh @@ -184,7 +184,7 @@ do kill $node_proc # Compare data - villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} + villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? # Exit, if an error occurs diff --git a/tests/integration/node-loopback-socket.sh b/tests/integration/node-loopback-socket.sh index 9d193f18b..c0b72a272 100755 --- a/tests/integration/node-loopback-socket.sh +++ b/tests/integration/node-loopback-socket.sh @@ -97,7 +97,7 @@ sleep 1 kill %1 # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${CONFIG_FILE} ${INPUT_FILE} ${OUTPUT_FILE} diff --git a/tests/integration/node-multiple_sources.sh b/tests/integration/node-multiple_sources.sh index 2f6f3cb3e..c2c7a4934 100755 --- a/tests/integration/node-multiple_sources.sh +++ b/tests/integration/node-multiple_sources.sh @@ -83,8 +83,8 @@ kill ${P1} wait ${P1} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE1} ${EXPECT_FILE} && \ -villas-test-cmp ${OUTPUT_FILE2} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE1} ${EXPECT_FILE} && \ +villas-compare ${OUTPUT_FILE2} ${EXPECT_FILE} RC=$? rm ${CONFIG_FILE} ${EXPECT_FILE} \ diff --git a/tests/integration/node-mux_demux.sh b/tests/integration/node-mux_demux.sh index dc19df24f..1e0a23445 100755 --- a/tests/integration/node-mux_demux.sh +++ b/tests/integration/node-mux_demux.sh @@ -96,7 +96,7 @@ VILLAS_LOG_PREFIX=$(colorize "[Node] ") \ villas-node ${CONFIG_FILE} # Compare only the data values -villas-test-cmp ${OUTPUT_FILE} ${EXPECT_FILE} +villas-compare ${OUTPUT_FILE} ${EXPECT_FILE} RC=$? rm ${CONFIG_FILE} ${OUTPUT_FILE} ${EXPECT_FILE} diff --git a/tests/integration/pipe-loopback-amqp.sh b/tests/integration/pipe-loopback-amqp.sh index 21fa6ed0a..0e0557a24 100755 --- a/tests/integration/pipe-loopback-amqp.sh +++ b/tests/integration/pipe-loopback-amqp.sh @@ -68,7 +68,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-exec.sh b/tests/integration/pipe-loopback-exec.sh index be4ea2ab9..cf03c9f4f 100755 --- a/tests/integration/pipe-loopback-exec.sh +++ b/tests/integration/pipe-loopback-exec.sh @@ -53,7 +53,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-file.sh b/tests/integration/pipe-loopback-file.sh index 8d15b0ba0..becea13de 100755 --- a/tests/integration/pipe-loopback-file.sh +++ b/tests/integration/pipe-loopback-file.sh @@ -55,7 +55,7 @@ villas-signal -l ${NUM_SAMPLES} -n random > ${INPUT_FILE} villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} ${NODE_FILE} diff --git a/tests/integration/pipe-loopback-iec61850-9-2.sh b/tests/integration/pipe-loopback-iec61850-9-2.sh index 32e250a78..648a7c360 100755 --- a/tests/integration/pipe-loopback-iec61850-9-2.sh +++ b/tests/integration/pipe-loopback-iec61850-9-2.sh @@ -68,7 +68,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp -T ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare -T ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-loopback.sh b/tests/integration/pipe-loopback-loopback.sh index f6ada7c6e..a76b5ab36 100755 --- a/tests/integration/pipe-loopback-loopback.sh +++ b/tests/integration/pipe-loopback-loopback.sh @@ -44,7 +44,7 @@ villas-signal -v 5 -l ${NUM_SAMPLES} -n mixed > ${INPUT_FILE} villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-mqtt.sh b/tests/integration/pipe-loopback-mqtt.sh index 307e54587..e465a89e1 100755 --- a/tests/integration/pipe-loopback-mqtt.sh +++ b/tests/integration/pipe-loopback-mqtt.sh @@ -65,7 +65,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-nanomsg.sh b/tests/integration/pipe-loopback-nanomsg.sh index dc15d6ae9..5403b71ca 100755 --- a/tests/integration/pipe-loopback-nanomsg.sh +++ b/tests/integration/pipe-loopback-nanomsg.sh @@ -66,7 +66,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-rtp-dual.sh b/tests/integration/pipe-loopback-rtp-dual.sh index e66df593f..855b47c44 100755 --- a/tests/integration/pipe-loopback-rtp-dual.sh +++ b/tests/integration/pipe-loopback-rtp-dual.sh @@ -126,7 +126,7 @@ VILLAS_LOG_PREFIX="[SRC] " \ villas-pipe ${CONFIG_FILE_SRC} rtp_node > ${OUTPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE_SRC} ${CONFIG_FILE_DEST} diff --git a/tests/integration/pipe-loopback-rtp-remote.sh b/tests/integration/pipe-loopback-rtp-remote.sh index d39543e1d..886e00925 100755 --- a/tests/integration/pipe-loopback-rtp-remote.sh +++ b/tests/integration/pipe-loopback-rtp-remote.sh @@ -137,7 +137,7 @@ villas-signal mixed -v 5 -r ${RATE} -l ${NUM_SAMPLES} | tee ${INPUT_FILE} | \ scp ${REMOTE_USER}@${REMOTE_ADDR}:${OUTPUT_FILE} ${OUTPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${INPUT_FILE} ${OUTPUT_FILE} ${CONFIG_FILE_DEST} ${CONFIG_FILE_SRC} diff --git a/tests/integration/pipe-loopback-rtp-tbf.sh b/tests/integration/pipe-loopback-rtp-tbf.sh index f8461a29f..ecf3b9a54 100755 --- a/tests/integration/pipe-loopback-rtp-tbf.sh +++ b/tests/integration/pipe-loopback-rtp-tbf.sh @@ -132,7 +132,7 @@ VILLAS_LOG_PREFIX="[SRC] " \ villas-pipe ${CONFIG_FILE_SRC} rtp_node > ${OUTPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE_SRC} ${CONFIG_FILE_DEST} diff --git a/tests/integration/pipe-loopback-rtp.sh b/tests/integration/pipe-loopback-rtp.sh index d06997479..5faa3fa03 100755 --- a/tests/integration/pipe-loopback-rtp.sh +++ b/tests/integration/pipe-loopback-rtp.sh @@ -85,7 +85,7 @@ villas-signal mixed -v 5 -r ${RATE} -l ${NUM_SAMPLES} | tee ${INPUT_FILE} | \ villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-shmem.sh b/tests/integration/pipe-loopback-shmem.sh index 4f5d51951..3dfd8370d 100755 --- a/tests/integration/pipe-loopback-shmem.sh +++ b/tests/integration/pipe-loopback-shmem.sh @@ -57,7 +57,7 @@ villas-signal -l ${NUM_SAMPLES} -v ${SIGNAL_COUNT} -n random > ${INPUT_FILE} villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? if (( ${RC} != 0 )); then diff --git a/tests/integration/pipe-loopback-socket-multicast.sh b/tests/integration/pipe-loopback-socket-multicast.sh index e271c7915..b679a5a9d 100755 --- a/tests/integration/pipe-loopback-socket-multicast.sh +++ b/tests/integration/pipe-loopback-socket-multicast.sh @@ -59,7 +59,7 @@ villas-signal -l ${NUM_SAMPLES} -n random > ${INPUT_FILE} villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-loopback-socket.sh b/tests/integration/pipe-loopback-socket.sh index ca0244972..7119c9617 100755 --- a/tests/integration/pipe-loopback-socket.sh +++ b/tests/integration/pipe-loopback-socket.sh @@ -105,7 +105,7 @@ if ! villas_format_supports_header $FORMAT; then fi # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? if (( ${RC} != 0 )); then diff --git a/tests/integration/pipe-loopback-websocket.sh b/tests/integration/pipe-loopback-websocket.sh index 0f2935f54..133678bff 100755 --- a/tests/integration/pipe-loopback-websocket.sh +++ b/tests/integration/pipe-loopback-websocket.sh @@ -78,7 +78,7 @@ villas-pipe -s ${CONFIG_FILE} node1 < <(sleep 1; cat ${INPUT_FILE}) wait $! # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} ${CONFIG_FILE2} diff --git a/tests/integration/pipe-loopback-zeromq.sh b/tests/integration/pipe-loopback-zeromq.sh index c91e494f7..88c400c24 100755 --- a/tests/integration/pipe-loopback-zeromq.sh +++ b/tests/integration/pipe-loopback-zeromq.sh @@ -65,7 +65,7 @@ EOF villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} node1 > ${OUTPUT_FILE} < ${INPUT_FILE} # Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${OUTPUT_FILE} ${INPUT_FILE} ${CONFIG_FILE} diff --git a/tests/integration/pipe-python-protobuf.sh b/tests/integration/pipe-python-protobuf.sh index 7df8bc683..49e099bba 100755 --- a/tests/integration/pipe-python-protobuf.sh +++ b/tests/integration/pipe-python-protobuf.sh @@ -95,7 +95,7 @@ kill ${CPID} wait ${CPID} # Compare data -villas-test-cmp ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} +villas-compare ${CMPFLAGS} ${INPUT_FILE} ${OUTPUT_FILE} RC=$? rm ${CONFIG_FILE} ${INPUT_FILE} ${OUTPUT_FILE} diff --git a/tests/integration/test-cmp.sh b/tests/integration/test-cmp.sh index 3c14d853a..29edb890e 100755 --- a/tests/integration/test-cmp.sh +++ b/tests/integration/test-cmp.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Integration test for villas-test-cmp. +# Integration test for villas-compare. # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC @@ -43,31 +43,31 @@ cat > ${INPUT_FILE} < ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 1 )) || fail 2 ( head -n-1 ${INPUT_FILE}; echo "1491095597.545159701(55) -0.587785" ) > ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 2 )) || fail 3 ( head -n-1 ${INPUT_FILE}; echo "1491095598.545159701(9) -0.587785" ) > ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 3 )) || fail 4 ( head -n-1 ${INPUT_FILE}; echo "1491095597.545159701(9) -0.587785 -0.587785" ) > ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 4 )) || fail 5 ( head -n-1 ${INPUT_FILE}; echo "1491095597.545159701(9) -1.587785" ) > ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 5 )) || fail 6 ( cat ${INPUT_FILE}; echo "1491095597.545159701(9) -0.587785" ) > ${TEMP_FILE} -villas-test-cmp ${INPUT_FILE} ${TEMP_FILE} +villas-compare ${INPUT_FILE} ${TEMP_FILE} (( $? == 1 )) || fail 7 rm ${INPUT_FILE} ${TEMP_FILE} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 6cba385b1..757109ea0 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -23,16 +23,16 @@ set(TEST_SRC config_json.cpp config.cpp + format.cpp + helpers.cpp + json.cpp main.cpp mapping.cpp memory.cpp - signal.cpp - helpers.cpp - json.cpp pool.cpp - queue.cpp queue_signalled.cpp - io.cpp + queue.cpp + signal.cpp ) add_executable(unit-tests ${TEST_SRC}) diff --git a/tests/unit/config.cpp b/tests/unit/config.cpp index 0c99e7c23..77e318089 100644 --- a/tests/unit/config.cpp +++ b/tests/unit/config.cpp @@ -59,18 +59,20 @@ Test(config, include) { const char *cfg_f2 = "magic = 1234\n"; - std::string f2_fn = std::tmpnam(nullptr); - std::FILE *f2 = std::fopen(f2_fn.c_str(), "w"); + char f2_fn_tpl[] = "/tmp/villas.unit-test.XXXXXX"; + int f2_fd = mkstemp(f2_fn_tpl); + + std::FILE *f2 = fdopen(f2_fd, "w"); std::fputs(cfg_f2, f2); std::rewind(f2); - auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn); + auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn_tpl); std::FILE *f1 = std::tmpfile(); std::fputs(cfg_f1.c_str(), f1); std::rewind(f1); - auto env = fmt::format("INCLUDE_FILE={}", f2_fn).c_str(); + auto env = fmt::format("INCLUDE_FILE={}", f2_fn_tpl).c_str(); putenv((char *) env); auto c = Config(); @@ -88,5 +90,5 @@ Test(config, include) cr_assert_eq(json_number_value(j2), 1234); std::fclose(f2); - std::remove(f2_fn.c_str()); + std::remove(f2_fn_tpl); } diff --git a/tests/unit/io.cpp b/tests/unit/format.cpp similarity index 65% rename from tests/unit/io.cpp rename to tests/unit/format.cpp index cbd764896..f4529c9b1 100644 --- a/tests/unit/io.cpp +++ b/tests/unit/format.cpp @@ -1,4 +1,4 @@ -/** Unit tests for IO formats. +/** Unit tests for formatters. * * @author Steffen Vogel * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC @@ -33,12 +33,13 @@ #include #include #include -#include +#include #include #include "helpers.hpp" using namespace villas; +using namespace villas::node; extern void init_memory(); @@ -107,7 +108,7 @@ void fill_sample_data(struct vlist *signals, struct sample *smps[], unsigned cnt void cr_assert_eq_sample(struct sample *a, struct sample *b, int flags) { - cr_assert_eq(a->length, b->length); + cr_assert_eq(a->length, b->length, "a->length=%d, b->length=%d", a->length, b->length); if (flags & (int) SampleFlags::HAS_SEQUENCE) cr_assert_eq(a->sequence, b->sequence); @@ -193,46 +194,47 @@ void cr_assert_eq_sample_raw(struct sample *a, struct sample *b, int flags, int } } -ParameterizedTestParameters(io, lowlevel) +ParameterizedTestParameters(format, lowlevel) { static criterion::parameters params; - params.emplace_back("gtnet", 1, 32); - params.emplace_back("gtnet.fake", 1, 32); - params.emplace_back("raw.8", 1, 8); - params.emplace_back("raw.16.be", 1, 16); - params.emplace_back("raw.16.le", 1, 16); - params.emplace_back("raw.32.be", 1, 32); - params.emplace_back("raw.32.le", 1, 32); - params.emplace_back("raw.64.be", 1, 64); - params.emplace_back("raw.64.le", 1, 64); - params.emplace_back("villas.human", 10, 0); - params.emplace_back("villas.binary", 10, 0); - params.emplace_back("csv", 10, 0); - params.emplace_back("json", 10, 0); + params.emplace_back("{ \"type\": \"gtnet\" }", 1, 32); + params.emplace_back("{ \"type\": \"gtnet\", \"fake\": true }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 8 }", 1, 8); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 16, \"endianess\": \"big\" }", 1, 16); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 16, \"endianess\": \"little\" }", 1, 16); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 32, \"endianess\": \"big\" }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 32, \"endianess\": \"little\" }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 64, \"endianess\": \"big\" }", 1, 64); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 64, \"endianess\": \"little\" }", 1, 64); + params.emplace_back("{ \"type\": \"villas.human\" }", 10, 0); + params.emplace_back("{ \"type\": \"villas.binary\" }", 10, 0); + params.emplace_back("{ \"type\": \"csv\" }", 10, 0); + params.emplace_back("{ \"type\": \"tsv\" }", 10, 0); + params.emplace_back("{ \"type\": \"json\" }", 10, 0); + // params.emplace_back("{ \"type\": \"json.kafka\" }", 10, 0); # broken due to signal names + // params.emplace_back("{ \"type\": \"json.reserve\" }", 10, 0); #ifdef PROTOBUF_FOUND - params.emplace_back("protobuf", 10, 0 ); + params.emplace_back("{ \"type\": \"protobuf\" }", 10, 0 ); #endif return params; } // cppcheck-suppress unknownMacro -ParameterizedTest(Param *p, io, lowlevel, .init = init_memory) +ParameterizedTest(Param *p, format, lowlevel, .init = init_memory) { int ret; unsigned cnt; char buf[8192]; size_t wbytes, rbytes; - Logger logger = logging.get("test:io:lowlevel"); + Logger logger = logging.get("test:format:lowlevel"); logger->info("Running test for format={}, cnt={}", p->fmt, p->cnt); - struct format_type *f; - struct pool pool; - struct io io; + Format *fmt; struct vlist signals; struct sample *smps[p->cnt]; struct sample *smpt[p->cnt]; @@ -252,25 +254,27 @@ ParameterizedTest(Param *p, io, lowlevel, .init = init_memory) fill_sample_data(&signals, smps, p->cnt); - f = format_type_lookup(p->fmt.c_str()); - cr_assert_not_null(f, "Format '%s' does not exist", p->fmt.c_str()); + json_t *json_format = json_loads(p->fmt.c_str(), 0, nullptr); + cr_assert_not_null(json_format); - ret = io_init(&io, f, &signals, (int) SampleFlags::HAS_ALL); - cr_assert_eq(ret, 0); + fmt = FormatFactory::make(json_format); + cr_assert_not_null(fmt, "Failed to create formatter of type '%s'", p->fmt.c_str()); - cnt = io_sprint(&io, buf, sizeof(buf), &wbytes, smps, p->cnt); - cr_assert_eq(cnt, p->cnt, "Written only %d of %d samples for format %s", cnt, p->cnt, format_type_name(f)); + fmt->start(&signals, (int) SampleFlags::HAS_ALL); - cnt = io_sscan(&io, buf, wbytes, &rbytes, smpt, p->cnt); - cr_assert_eq(cnt, p->cnt, "Read only %d of %d samples back for format %s", cnt, p->cnt, format_type_name(f)); + cnt = fmt->sprint(buf, sizeof(buf), &wbytes, smps, p->cnt); + cr_assert_eq(cnt, p->cnt, "Written only %d of %d samples", cnt, p->cnt); + + cnt = fmt->sscan(buf, wbytes, &rbytes, smpt, p->cnt); + cr_assert_eq(cnt, p->cnt, "Read only %d of %d samples back", cnt, p->cnt); cr_assert_eq(rbytes, wbytes, "rbytes != wbytes: %#zx != %#zx", rbytes, wbytes); for (unsigned i = 0; i < cnt; i++) { if (p->bits) - cr_assert_eq_sample_raw(smps[i], smpt[i], f->flags, p->bits); + cr_assert_eq_sample_raw(smps[i], smpt[i], fmt->getFlags(), p->bits); else - cr_assert_eq_sample(smps[i], smpt[i], f->flags); + cr_assert_eq_sample(smps[i], smpt[i], fmt->getFlags()); } sample_free_many(smps, p->cnt); @@ -280,49 +284,48 @@ ParameterizedTest(Param *p, io, lowlevel, .init = init_memory) cr_assert_eq(ret, 0); } -ParameterizedTestParameters(io, highlevel) +ParameterizedTestParameters(format, highlevel) { static criterion::parameters params; - params.emplace_back("gtnet", 1, 32); - params.emplace_back("gtnet.fake", 1, 32); - params.emplace_back("raw.8", 1, 8); - params.emplace_back("raw.16.be", 1, 16); - params.emplace_back("raw.16.le", 1, 16); - params.emplace_back("raw.32.be", 1, 32); - params.emplace_back("raw.32.le", 1, 32); - params.emplace_back("raw.64.be", 1, 64); - params.emplace_back("raw.64.le", 1, 64); - params.emplace_back("villas.human", 10, 0); - params.emplace_back("villas.binary", 10, 0); - params.emplace_back("csv", 10, 0); - params.emplace_back("json", 10, 0); + params.emplace_back("{ \"type\": \"gtnet\" }", 1, 32); + params.emplace_back("{ \"type\": \"gtnet\", \"fake\": true }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 8 }", 1, 8); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 16, \"endianess\": \"big\" }", 1, 16); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 16, \"endianess\": \"little\" }", 1, 16); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 32, \"endianess\": \"big\" }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 32, \"endianess\": \"little\" }", 1, 32); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 64, \"endianess\": \"big\" }", 1, 64); + params.emplace_back("{ \"type\": \"raw\", \"bits\": 64, \"endianess\": \"little\" }", 1, 64); + params.emplace_back("{ \"type\": \"villas.human\" }", 10, 0); + params.emplace_back("{ \"type\": \"villas.binary\" }", 10, 0); + params.emplace_back("{ \"type\": \"csv\" }", 10, 0); + params.emplace_back("{ \"type\": \"tsv\" }", 10, 0); + params.emplace_back("{ \"type\": \"json\" }", 10, 0); + // params.emplace_back("{ \"type\": \"json.kafka\" }", 10, 0); # broken due to signal names + // params.emplace_back("{ \"type\": \"json.reserve\" }", 10, 0); #ifdef PROTOBUF_FOUND - params.emplace_back("protobuf", 10, 0 ); + params.emplace_back("{ \"type\": \"protobuf\" }", 10, 0 ); #endif return params; } -ParameterizedTest(Param *p, io, highlevel, .init = init_memory) +ParameterizedTest(Param *p, format, highlevel, .init = init_memory) { int ret, cnt; char *retp; - Logger logger = logging.get("test:io:highlevel"); + Logger logger = logging.get("test:format:highlevel"); logger->info("Running test for format={}, cnt={}", p->fmt, p->cnt); - return; - - struct format_type *f; - struct sample *smps[p->cnt]; struct sample *smpt[p->cnt]; struct pool pool; struct vlist signals; - struct io io; + Format *fmt; ret = pool_init(&pool, 2 * p->cnt, SAMPLE_LENGTH(NUM_VALUES)); cr_assert_eq(ret, 0); @@ -339,7 +342,7 @@ ParameterizedTest(Param *p, io, highlevel, .init = init_memory) fill_sample_data(&signals, smps, p->cnt); - /* Open a file for IO */ + /* Open a file for testing the formatter */ char *fn, dir[64]; strncpy(dir, "/tmp/villas.XXXXXX", sizeof(dir)); @@ -349,19 +352,21 @@ ParameterizedTest(Param *p, io, highlevel, .init = init_memory) ret = asprintf(&fn, "%s/file", dir); cr_assert_gt(ret, 0); - f = format_type_lookup(p->fmt.c_str()); - cr_assert_not_null(f, "Format '%s' does not exist", p->fmt.c_str()); + json_t *json_format = json_loads(p->fmt.c_str(), 0, nullptr); + cr_assert_not_null(json_format); - ret = io_init(&io, f, &signals, (int) SampleFlags::HAS_ALL); - cr_assert_eq(ret, 0); + fmt = FormatFactory::make(json_format); + cr_assert_not_null(fmt, "Failed to create formatter of type '%s'", p->fmt.c_str()); - ret = io_open(&io, fn); - cr_assert_eq(ret, 0); + fmt->start(&signals, (int) SampleFlags::HAS_ALL); - cnt = io_print(&io, smps, p->cnt); - cr_assert_eq(cnt, p->cnt, "Written only %d of %d samples for format %s", cnt, p->cnt, format_type_name(f)); + auto *stream = fopen(fn, "w+"); + cr_assert_not_null(stream); - ret = io_flush(&io); + cnt = fmt->print(stream, smps, p->cnt); + cr_assert_eq(cnt, p->cnt, "Written only %d of %d samples", cnt, p->cnt); + + ret = fflush(stream); cr_assert_eq(ret, 0); #if 0 /* Show the file contents */ @@ -373,25 +378,22 @@ ParameterizedTest(Param *p, io, highlevel, .init = init_memory) system(cmd); #endif - io_rewind(&io); - - cnt = io_scan(&io, smpt, p->cnt); - cr_assert_gt(cnt, 0, "Failed to read samples back: cnt=%d", cnt); + rewind(stream); + cnt = fmt->scan(stream, smpt, p->cnt); cr_assert_eq(cnt, p->cnt, "Read only %d of %d samples back", cnt, p->cnt); for (int i = 0; i < cnt; i++) { if (p->bits) - cr_assert_eq_sample_raw(smps[i], smpt[i], f->flags, p->bits); + cr_assert_eq_sample_raw(smps[i], smpt[i], fmt->getFlags(), p->bits); else - cr_assert_eq_sample(smps[i], smpt[i], f->flags); + cr_assert_eq_sample(smps[i], smpt[i], fmt->getFlags()); } - ret = io_close(&io); + ret = fclose(stream); cr_assert_eq(ret, 0); - ret = io_destroy(&io); - cr_assert_eq(ret, 0); + delete fmt; ret = unlink(fn); cr_assert_eq(ret, 0);