diff --git a/CMakeLists.txt b/CMakeLists.txt index a51f53e45..e6f1bc0e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,7 @@ find_package(RDMACM) find_package(Etherlab) find_package(Lua) find_package(LibDataChannel) +find_package(HDF5 REQUIRED COMPONENTS CXX) # Check for tools find_program(PASTE NAMES paste) @@ -114,6 +115,8 @@ pkg_check_modules(CGRAPH IMPORTED_TARGET libcgraph>=2.30) pkg_check_modules(GVC IMPORTED_TARGET libgvc>=2.30) pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) +pkg_check_modules(HDF5 IMPORTED_TARGET hdf5>=1.14.3) +pkg_check_modules(HDF5_CPPfgs IMPORTED_TARGET hdf5_cppsgf>=1.14.3) # scheint nichts zu bringen if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) endif() @@ -187,6 +190,7 @@ cmake_dependent_option(WITH_NODE_EXAMPLE "Build with example node-type" cmake_dependent_option(WITH_NODE_EXEC "Build with exec node-type" "${WITH_DEFAULTS}" "" OFF) cmake_dependent_option(WITH_NODE_FILE "Build with file node-type" "${WITH_DEFAULTS}" "" OFF) cmake_dependent_option(WITH_NODE_FPGA "Build with fpga node-type" "${WITH_DEFAULTS}" "WITH_FPGA" OFF) +cmake_dependent_option(WITH_NODE_HDF5 "Build with hdf5 node-type" "${WITH_DEFAULTS}" "" OFF) cmake_dependent_option(WITH_NODE_IEC61850 "Build with iec61850 node-types" "${WITH_DEFAULTS}" "LIBIEC61850_FOUND; NOT WITHOUT_GPL" OFF) cmake_dependent_option(WITH_NODE_IEC60870 "Build with iec60870 node-types" "${WITH_DEFAULTS}" "LIB60870_FOUND; NOT WITHOUT_GPL" OFF) cmake_dependent_option(WITH_NODE_INFINIBAND "Build with infiniband node-type" "${WITH_DEFAULTS}" "IBVerbs_FOUND; RDMACM_FOUND" OFF) # Infiniband node-type is currenly broken @@ -293,6 +297,7 @@ add_feature_info(NODE_EXAMPLE WITH_NODE_EXAMPLE "Build with add_feature_info(NODE_EXEC WITH_NODE_EXEC "Build with exec node-type") add_feature_info(NODE_FILE WITH_NODE_FILE "Build with file node-type") add_feature_info(NODE_FPGA WITH_NODE_FPGA "Build with fpga node-type") +add_feature_info(NODE_HDF5 WITH_NODE_HDF5 "Build with hdf5 node-type") add_feature_info(NODE_IEC61850 WITH_NODE_IEC61850 "Build with iec61850 node-types") add_feature_info(NODE_IEC60870 WITH_NODE_IEC60870 "Build with iec60870 node-types") add_feature_info(NODE_INFINIBAND WITH_NODE_INFINIBAND "Build with infiniband node-type") diff --git a/include/villas/node.hpp b/include/villas/node.hpp index c920adc49..3ef8819a3 100644 --- a/include/villas/node.hpp +++ b/include/villas/node.hpp @@ -363,10 +363,10 @@ public: } // namespace node } // namespace villas -#ifndef FMT_LEGACY_OSTREAM_FORMATTER -template <> -class fmt::formatter : public fmt::ostream_formatter {}; -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#endif +// #ifndef FMT_LEGACY_OSTREAM_FORMATTER +// template <> +// class fmt::formatter : public fmt::ostream_formatter {}; +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #endif diff --git a/include/villas/nodes/hdf5.hpp b/include/villas/nodes/hdf5.hpp new file mode 100644 index 000000000..41313ca04 --- /dev/null +++ b/include/villas/nodes/hdf5.hpp @@ -0,0 +1,120 @@ +/* A HDF5 node-type. + * + * This node-type reads the samples and writes them to a HDF5 file. + * + * Author: Alexandra Bach + * SPDX-FileCopyrightText: 2014-2024 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +using namespace H5; + +namespace villas { +namespace node { + +// Forward declarations +class NodeCompat; +struct Sample; + +class HDF5node : public Node { + +protected: + Format *formatter; + FILE *stream_in; // File to read from. + FILE *stream_out; // File to write to. + + char *uri_tmpl; // Format string for file name. + char *uri; // Real file name. + + int flush; // Flush / upload file contents after each write. + size_t buffer_size_out; // Defines size of output stream buffer. No buffer is created if value is set to zero. + size_t buffer_size_in; // Defines size of input stream buffer. No buffer is created if value is set to zero. + + // names + const char *file_name; + const char *dataset_name; + const char *dataspace_name; + const char *location_name; + const char *description_name; + const char *project_name; + const char *author_name; + const char *unit; + + // HDF5 + // H5File file(H5std_string(uri), H5F_ACC_TRUNC); + // DataSet *dataset; + // DataSpace *dataspace; + // Attribute *timestamp_attr; + // Attribute *location_attr; + // Attribute *description_attr; + // Attribute *project_attr; + // Attribute *author_attr; + + struct attributes { + int *timestamp; + char *location; + char *description; + char *project; + char *author; + } attributes; + + // virtual int _read(struct Sample *smps[], unsigned cnt); + + virtual int _write(struct Sample *smps[], unsigned cnt); + +public: + HDF5node(const uuid_t &id = {}, const std::string &name = ""); + + /* All of the following virtual-declared functions are optional. + * Have a look at node.hpp/node.cpp for the default behaviour. + */ + + virtual ~HDF5node(); + + // virtual int prepare(); + + virtual int parse(json_t *json); + + // Validate node configuration + // virtual int check(); + + // virtual int start(); + + // virtual int stop(); + + // virtual + // int pause(); + + // virtual + // int resume(); + + // virtual + // int restart(); + + // virtual + // int reverse(); + + // virtual + // std::vector getPollFDs(); + + // virtual + // std::vector getNetemFDs(); + + // virtual + // struct villas::node::memory::Type * getMemoryType(); + + // virtual const std::string &getDetails(); +}; + +} // namespace node +} // namespace villas diff --git a/include/villas/nodes/redis_helpers.hpp b/include/villas/nodes/redis_helpers.hpp index 3cc5b2a39..bcf334294 100644 --- a/include/villas/nodes/redis_helpers.hpp +++ b/include/villas/nodes/redis_helpers.hpp @@ -153,16 +153,16 @@ inline sw::redis::ConnectionOptions make_redis_connection_options(char const *ur } // namespace node } // namespace villas -#ifndef FMT_LEGACY_OSTREAM_FORMATTER -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#ifdef REDISPP_WITH_TLS -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#endif -#endif +// #ifndef FMT_LEGACY_OSTREAM_FORMATTER +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #ifdef REDISPP_WITH_TLS +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #endif +// #endif diff --git a/include/villas/nodes/webrtc/peer_connection.hpp b/include/villas/nodes/webrtc/peer_connection.hpp index 26c180971..187e7828a 100644 --- a/include/villas/nodes/webrtc/peer_connection.hpp +++ b/include/villas/nodes/webrtc/peer_connection.hpp @@ -35,11 +35,11 @@ namespace rtc { using ::operator<<; } -#ifndef FMT_LEGACY_OSTREAM_FORMATTER -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#endif +// #ifndef FMT_LEGACY_OSTREAM_FORMATTER +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #endif namespace villas { namespace node { diff --git a/include/villas/nodes/websocket.hpp b/include/villas/nodes/websocket.hpp index 06698353d..a0e458f60 100644 --- a/include/villas/nodes/websocket.hpp +++ b/include/villas/nodes/websocket.hpp @@ -132,8 +132,8 @@ int websocket_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt); } // namespace node } // namespace villas -#ifndef FMT_LEGACY_OSTREAM_FORMATTER -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#endif +// #ifndef FMT_LEGACY_OSTREAM_FORMATTER +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #endif diff --git a/include/villas/path.hpp b/include/villas/path.hpp index 81d46c024..e2ae0631f 100644 --- a/include/villas/path.hpp +++ b/include/villas/path.hpp @@ -183,8 +183,8 @@ public: } // namespace node } // namespace villas -#ifndef FMT_LEGACY_OSTREAM_FORMATTER -template <> -class fmt::formatter - : public fmt::ostream_formatter {}; -#endif +// #ifndef FMT_LEGACY_OSTREAM_FORMATTER +// template <> +// class fmt::formatter +// : public fmt::ostream_formatter {}; +// #endif diff --git a/lib/node.cpp b/lib/node.cpp index 483573e9a..51b84a759 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -185,7 +185,7 @@ int Node::start() { assert(state == State::PREPARED); - logger->info("Starting node {}", getNameFull()); + // logger->info("Starting node {}", getNameFull()); ret = in.start(); if (ret) @@ -469,6 +469,7 @@ int NodeFactory::start(SuperNode *sn) { instances.size()); state = State::STARTED; + logger->debug("NodeFactory::start()"); return 0; } diff --git a/lib/nodes/CMakeLists.txt b/lib/nodes/CMakeLists.txt index 25340dda4..8878ce6b7 100644 --- a/lib/nodes/CMakeLists.txt +++ b/lib/nodes/CMakeLists.txt @@ -159,6 +159,12 @@ if(WITH_NODE_EXAMPLE) # list(APPEND LIBRARIES PkgConfig::EXAMPLELIB) endif() +# Enable hdf5 node type +if(WITH_NODE_HDF5) + list(APPEND NODE_SRC hdf5.cpp) + list(APPEND LIBRARIES HDF5::HDF5) +endif() + # Enable Ethercat support if(WITH_NODE_ETHERCAT) list(APPEND NODE_SRC ethercat.cpp) diff --git a/lib/nodes/file.cpp b/lib/nodes/file.cpp index 3fd6692e3..b7526c542 100644 --- a/lib/nodes/file.cpp +++ b/lib/nodes/file.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -212,6 +213,7 @@ char *villas::node::file_print(NodeCompat *n) { int villas::node::file_start(NodeCompat *n) { auto *f = n->getData(); + std::cout << "file_start" << std::endl; struct timespec now = time_now(); int ret; @@ -221,6 +223,7 @@ int villas::node::file_start(NodeCompat *n) { delete[] f->uri; f->uri = file_format_name(f->uri_tmpl, &now); + std::cout << "file_format_name" << std::endl; // Check if directory exists struct stat sb; @@ -240,19 +243,23 @@ int villas::node::file_start(NodeCompat *n) { if (ret) throw SystemError("Failed to create directory"); } - + std::cout << "stat" << std::endl; free(cpy); f->formatter->start(n->getInputSignals(false)); + std::cout << "formatter->start" << std::endl; // Open file f->stream_out = fopen(f->uri, "a+"); if (!f->stream_out) + std::cout << "fopen stream_out -1" << std::endl; return -1; + std::cout << "fopen stream_out" << std::endl; f->stream_in = fopen(f->uri, "r"); if (!f->stream_in) return -1; + std::cout << "fopen stream_in" << std::endl; if (f->buffer_size_in) { ret = setvbuf(f->stream_in, nullptr, _IOFBF, f->buffer_size_in); @@ -263,6 +270,7 @@ int villas::node::file_start(NodeCompat *n) { if (f->buffer_size_out) { ret = setvbuf(f->stream_out, nullptr, _IOFBF, f->buffer_size_out); if (ret) + std::cout << "ret: " << ret << std::endl; // return ret; return ret; } diff --git a/lib/nodes/hdf5.cpp b/lib/nodes/hdf5.cpp new file mode 100644 index 000000000..3909f2d84 --- /dev/null +++ b/lib/nodes/hdf5.cpp @@ -0,0 +1,231 @@ +/* A HDF5 node-type. + * + * This node-type reads the samples and writes them to a HDF5 file. + * + * Author: Alexandra Bach + * SPDX-FileCopyrightText: 2014-2024 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +using namespace H5; +using namespace villas; +using namespace villas::node; +using namespace villas::utils; + +HDF5node::HDF5node(const uuid_t &id, const std::string &name) + : Node(id, name) {} + +HDF5node::~HDF5node() {} + +// int HDF5node::prepare() { +// // state1 = setting1; + +// // if (setting2 == "double") +// // state1 *= 2; + +// // return 0; +// // assert(state == State::PREPARED;); +// state = State::PREPARED; +// logger->debug("HDF5node::prepare()"); +// return 0; +// } + +int HDF5node::parse(json_t *json) { + // default values + location_name = "RWTH Aachen University"; + description_name = "This is a test dataset"; + project_name = "VILLASframework"; + author_name = "none"; + + json_error_t err; + int ret = json_unpack_ex(json, &err, 0, "{ s: s, s: s, s?: s, s?: s, s?: s, s?: s, s: s }", + "file", &file_name, "dataset", &dataset_name, "location", &location_name, + "description", &description_name, "project", &project_name, "author", &author_name, + "unit", &unit); + if (ret) + throw ConfigError(json, err, "node-config-node-hdf5"); + + logger->debug("HDF5node::parse() file_name: {}", file_name); + + return 0; +} + +// int HDF5node::check() { +// // if (setting1 > 100 || setting1 < 0) +// // return -1; + +// // if (setting2.empty() || setting2.size() > 10) +// // return -1; + +// return 0; +// } + +// int HDF5node::start() { +// assert(state == State::PREPARED || +// state == State::PAUSED); +// logger->debug("HDF5node::start()"); +// return Node::start(); +// } + +// int HDF5node::stop() +// { +// fclose(stream_in); +// fclose(stream_out); + +// return 0; +// } + +// int HDF5node::pause() +// { +// // TODO add implementation here +// return 0; +// } + +// int HDF5node::resume() +// { +// // TODO add implementation here +// return 0; +// } + +// int HDF5node::restart() +// { +// // TODO add implementation here +// return 0; +// } + +// int HDF5node::reverse() +// { +// // TODO add implementation here +// return 0; +// } + +// std::vector HDF5node::getPollFDs() +// { +// // TODO add implementation here +// return {}; +// } + +// std::vector HDF5node::getNetemFDs() +// { +// // TODO add implementation here +// return {}; +// } + +// struct villas::node::memory::Type * HDF5node::getMemoryType() +// { +// // TODO add implementation here +// } + +// const std::string &HDF5node::getDetails() { +// details = fmt::format("setting1={}, setting2={}", setting1, setting2); +// return details; +// } + + +// int HDF5node::_read(struct Sample *smps[], unsigned cnt) { + +// } + +// similar to file node-type: takes the samples and writes them to a HDF5 file +int HDF5node::_write(struct Sample *smps[], unsigned cnt) { + // Create a new file using the default property lists. + H5File file(H5std_string(file_name), H5F_ACC_TRUNC); + + // Create the data space for the dataset. Defines the size and shape of the dataset. + logger->info("smps[0]->length {}", smps[0]->length); + // hsize_t dims[2] = {cnt, 1}; // cnt rowns, smps[0]->length columns + hsize_t dims[2] = {4, 6}; + DataSpace dataspace(2, dims); + + // Create the dataset. Composes a collection of data elements, raw data and metadata. + DataSet dataset = file.createDataSet(H5std_string(dataset_name), PredType::STD_I32BE, dataspace); + + // Write the data to the dataset. + // for (unsigned j = 0; j < cnt; j++) { + // struct Sample *smp = smps[j]; + // // ret = formatter->print(stream_out, smp, j+1); + // dataset.write(smp->data, PredType::NATIVE_DOUBLE); + // } + int data[4][6] = { {1, 2, 3, 4, 5, 6}, {7, 8, 9, 10, 11, 12}, {13, 14, 15, 16, 17, 18}, {19, 20, 21, 22, 23, 24}}; + dataset.write(data, H5::PredType::NATIVE_INT); + + // Create a dataspace for the attribute + DataSpace attr_dataspace(H5S_SCALAR); + + // Create a string datatype + StrType strdatatype(PredType::C_S1, H5T_VARIABLE); // for variable-length string + + // Create the timestamp attribute and write to it + Attribute timestampAttribute = dataset.createAttribute("timestamp", strdatatype, attr_dataspace); + + // Get the current time + auto now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + + // Convert the current time to a string + std::stringstream ss; + ss << std::put_time(std::localtime(&now_c), "%Y-%m-%dT%H:%M:%S"); + std::string timestamp = ss.str(); + + // Write the timestamp to the attribute + timestampAttribute.write(strdatatype, timestamp); + + // Create the location attribute and write to it + Attribute locationAttribute = dataset.createAttribute("location", strdatatype, attr_dataspace); + // convert location_name to sting + std::string location_name_string = location_name; + locationAttribute.write(strdatatype, location_name_string); + + // Create the description attribute and write to it + Attribute descriptionAttribute = dataset.createAttribute("description", strdatatype, attr_dataspace); + std::string description_name_string = description_name; + descriptionAttribute.write(strdatatype, description_name_string); + + // Create the project attribute and write to it + Attribute projectAttribute = dataset.createAttribute("project", strdatatype, attr_dataspace); + std::string project_name_string = project_name; + projectAttribute.write(strdatatype, project_name_string); + + // Create the author attribute and write to it + Attribute authorAttribute = dataset.createAttribute("author", strdatatype, attr_dataspace); + std::string author_name_string = author_name; + authorAttribute.write(strdatatype, author_name_string); + + logger->debug("attributes created and written"); + + timestampAttribute.close(); // put in extra destroy function? + locationAttribute.close(); + descriptionAttribute.close(); + projectAttribute.close(); + authorAttribute.close(); + dataset.close(); + file.close(); + + return cnt; +} + +// Register node +static char n[] = "hdf5"; +static char d[] = "This node-type writes samples to hdf5 file format"; +static NodePlugin + p; diff --git a/lib/signal_list.cpp b/lib/signal_list.cpp index dd1359973..95983152f 100644 --- a/lib/signal_list.cpp +++ b/lib/signal_list.cpp @@ -5,6 +5,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include #include #include @@ -145,9 +147,10 @@ Signal::Ptr SignalList::getByName(const std::string &name) { SignalList::Ptr SignalList::clone() { auto l = std::make_shared(); - - for (auto s : *this) - l->push_back(s); + if (!l->empty()){ // include check if list of pointers is empty + for (auto s : *this) + l->push_back(s); + } return l; } diff --git a/packaging/deps.sh b/packaging/deps.sh index 44ddebb81..2e56b5e8d 100644 --- a/packaging/deps.sh +++ b/packaging/deps.sh @@ -421,8 +421,7 @@ if ! pkg-config "libmodbus >= 3.1.0" && \ git clone ${GIT_OPTS} --recursive --branch v3.1.10 https://github.com/stephane/libmodbus.git mkdir -p libmodbus/build pushd libmodbus - autoreconf -i - ./configure ${CONFIGURE_OPTS} + cmake ${CMAKE_OPTS} .. make ${MAKE_OPTS} install popd fi diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index b9fcb9801..3147e8c10 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -51,7 +51,8 @@ RUN apt-get update && \ liblua5.3-dev \ libhiredis-dev \ libnice-dev \ - libmodbus-dev + libmodbus-dev \ + libhdf5-dev # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index d28802f2c..0456a3cda 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -64,7 +64,8 @@ RUN dnf -y install \ lua-devel \ hiredis-devel \ libnice-devel \ - libmodbus-devel + libmodbus-devel \ + hdf5-devel # Add local and 64-bit locations to linker paths RUN echo /usr/local/lib >> /etc/ld.so.conf && \ @@ -78,11 +79,11 @@ RUN ldconfig # Workaround for libnl3's search path for netem distributions RUN ln -s /usr/lib64/tc /usr/lib/tc -COPY ./python /python -RUN python -m venv /venv && \ - source /venv/bin/activate && \ - pip install /python[dev] && \ - rm -r /python +# COPY ./python /python +# RUN python -m venv /venv && \ +# source /venv/bin/activate && \ +# pip install /python[dev] && \ +# rm -r /python # Expose ports for HTTP and WebSocket frontend EXPOSE 80 diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index 38f660350..c15574f6a 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -53,7 +53,8 @@ RUN apt-get update && \ liblua5.3-dev \ libhiredis-dev \ libnice-dev \ - libmodbus-dev + libmodbus-dev \ + libhdf5-dev # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/tests/integration/node-hdf5.sh b/tests/integration/node-hdf5.sh new file mode 100755 index 000000000..c66158ff1 --- /dev/null +++ b/tests/integration/node-hdf5.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Test for the example node-type +# +# Author: Steffen Vogel +# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +set -e + +echo "The example node does nothing useful which we could test" +exit 99 + +DIR=$(mktemp -d) +pushd ${DIR} + +function finish { + popd + rm -rf ${DIR} +} +trap finish EXIT + +cat > expect.dat < config.json <