Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

first draft of hdf5 node-type

Browse source
This commit is contained in:
Alexandra 2024-02-28 11:00:35 +00:00
parent c3d991b23e
commit 33f64ee642
17 changed files with 481 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CMakeLists.txt
View file

@ -75,6 +75,7 @@ find_package(RDMACM)
find_package(Etherlab)
find_package(Lua)
find_package(LibDataChannel)
find_package(HDF5 REQUIRED COMPONENTS CXX)
# Check for tools
find_program(PASTE NAMES paste)
@ -114,6 +115,8 @@ pkg_check_modules(CGRAPH IMPORTED_TARGET libcgraph>=2.30)
pkg_check_modules(GVC IMPORTED_TARGET libgvc>=2.30)
pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23)
pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg)
pkg_check_modules(HDF5 IMPORTED_TARGET hdf5>=1.14.3)
pkg_check_modules(HDF5_CPPfgs IMPORTED_TARGET hdf5_cppsgf>=1.14.3) # scheint nichts zu bringen
if(NOT NANOMSG_FOUND)
pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0)
endif()
@ -187,6 +190,7 @@ cmake_dependent_option(WITH_NODE_EXAMPLE "Build with example node-type"
cmake_dependent_option(WITH_NODE_EXEC "Build with exec node-type" "${WITH_DEFAULTS}" "" OFF)
cmake_dependent_option(WITH_NODE_FILE "Build with file node-type" "${WITH_DEFAULTS}" "" OFF)
cmake_dependent_option(WITH_NODE_FPGA "Build with fpga node-type" "${WITH_DEFAULTS}" "WITH_FPGA" OFF)
cmake_dependent_option(WITH_NODE_HDF5 "Build with hdf5 node-type" "${WITH_DEFAULTS}" "" OFF)
cmake_dependent_option(WITH_NODE_IEC61850 "Build with iec61850 node-types" "${WITH_DEFAULTS}" "LIBIEC61850_FOUND; NOT WITHOUT_GPL" OFF)
cmake_dependent_option(WITH_NODE_IEC60870 "Build with iec60870 node-types" "${WITH_DEFAULTS}" "LIB60870_FOUND; NOT WITHOUT_GPL" OFF)
cmake_dependent_option(WITH_NODE_INFINIBAND "Build with infiniband node-type" "${WITH_DEFAULTS}" "IBVerbs_FOUND; RDMACM_FOUND" OFF) # Infiniband node-type is currenly broken
@ -293,6 +297,7 @@ add_feature_info(NODE_EXAMPLE WITH_NODE_EXAMPLE "Build with
add_feature_info(NODE_EXEC WITH_NODE_EXEC "Build with exec node-type")
add_feature_info(NODE_FILE WITH_NODE_FILE "Build with file node-type")
add_feature_info(NODE_FPGA WITH_NODE_FPGA "Build with fpga node-type")
add_feature_info(NODE_HDF5 WITH_NODE_HDF5 "Build with hdf5 node-type")
add_feature_info(NODE_IEC61850 WITH_NODE_IEC61850 "Build with iec61850 node-types")
add_feature_info(NODE_IEC60870 WITH_NODE_IEC60870 "Build with iec60870 node-types")
add_feature_info(NODE_INFINIBAND WITH_NODE_INFINIBAND "Build with infiniband node-type")

14
include/villas/node.hpp
View file

@ -363,10 +363,10 @@ public:
} // namespace node
} // namespace villas
#ifndef FMT_LEGACY_OSTREAM_FORMATTER
template <>
class fmt::formatter<villas::node::Node> : public fmt::ostream_formatter {};
template <>
class fmt::formatter<villas::node::NodeFactory>
: public fmt::ostream_formatter {};
#endif
// #ifndef FMT_LEGACY_OSTREAM_FORMATTER
// template <>
// class fmt::formatter<villas::node::Node> : public fmt::ostream_formatter {};
// template <>
// class fmt::formatter<villas::node::NodeFactory>
// : public fmt::ostream_formatter {};
// #endif

120
include/villas/nodes/hdf5.hpp Normal file
View file

@ -0,0 +1,120 @@
/* A HDF5 node-type.
*
* This node-type reads the samples and writes them to a HDF5 file.
*
* Author: Alexandra Bach <alexandra.bach@eonerc.rwth-aachen.de>
* SPDX-FileCopyrightText: 2014-2024 Institute for Automation of Complex Power Systems, RWTH Aachen University
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once
#include <H5Cpp.h>
#include <H5File.h>
#include <villas/format.hpp>
#include <villas/node.hpp>
#include <villas/node/config.hpp>
#include <villas/timing.hpp>
using namespace H5;
namespace villas {
namespace node {
// Forward declarations
class NodeCompat;
struct Sample;
class HDF5node : public Node {
protected:
Format *formatter;
FILE *stream_in; // File to read from.
FILE *stream_out; // File to write to.
char *uri_tmpl; // Format string for file name.
char *uri; // Real file name.
int flush; // Flush / upload file contents after each write.
size_t buffer_size_out; // Defines size of output stream buffer. No buffer is created if value is set to zero.
size_t buffer_size_in; // Defines size of input stream buffer. No buffer is created if value is set to zero.
// names
const char *file_name;
const char *dataset_name;
const char *dataspace_name;
const char *location_name;
const char *description_name;
const char *project_name;
const char *author_name;
const char *unit;
// HDF5
// H5File file(H5std_string(uri), H5F_ACC_TRUNC);
// DataSet *dataset;
// DataSpace *dataspace;
// Attribute *timestamp_attr;
// Attribute *location_attr;
// Attribute *description_attr;
// Attribute *project_attr;
// Attribute *author_attr;
struct attributes {
int *timestamp;
char *location;
char *description;
char *project;
char *author;
} attributes;
// virtual int _read(struct Sample *smps[], unsigned cnt);
virtual int _write(struct Sample *smps[], unsigned cnt);
public:
HDF5node(const uuid_t &id = {}, const std::string &name = "");
/* All of the following virtual-declared functions are optional.
* Have a look at node.hpp/node.cpp for the default behaviour.
*/
virtual ~HDF5node();
// virtual int prepare();
virtual int parse(json_t *json);
// Validate node configuration
// virtual int check();
// virtual int start();
// virtual int stop();
// virtual
// int pause();
// virtual
// int resume();
// virtual
// int restart();
// virtual
// int reverse();
// virtual
// std::vector<int> getPollFDs();
// virtual
// std::vector<int> getNetemFDs();
// virtual
// struct villas::node::memory::Type * getMemoryType();
// virtual const std::string &getDetails();
};
} // namespace node
} // namespace villas

26
include/villas/nodes/redis_helpers.hpp
View file

@ -153,16 +153,16 @@ inline sw::redis::ConnectionOptions make_redis_connection_options(char const *ur
} // namespace node
} // namespace villas
#ifndef FMT_LEGACY_OSTREAM_FORMATTER
template <>
class fmt::formatter<sw::redis::ConnectionType>
: public fmt::ostream_formatter {};
template <>
class fmt::formatter<sw::redis::ConnectionOptions>
: public fmt::ostream_formatter {};
#ifdef REDISPP_WITH_TLS
template <>
class fmt::formatter<sw::redis::tls::TlsOptions>
: public fmt::ostream_formatter {};
#endif
#endif
// #ifndef FMT_LEGACY_OSTREAM_FORMATTER
// template <>
// class fmt::formatter<sw::redis::ConnectionType>
// : public fmt::ostream_formatter {};
// template <>
// class fmt::formatter<sw::redis::ConnectionOptions>
// : public fmt::ostream_formatter {};
// #ifdef REDISPP_WITH_TLS
// template <>
// class fmt::formatter<sw::redis::tls::TlsOptions>
// : public fmt::ostream_formatter {};
// #endif
// #endif

10
include/villas/nodes/webrtc/peer_connection.hpp
View file

@ -35,11 +35,11 @@ namespace rtc {
using ::operator<<;
}
#ifndef FMT_LEGACY_OSTREAM_FORMATTER
template <>
class fmt::formatter<rtc::PeerConnection::State>
: public fmt::ostream_formatter {};
#endif
// #ifndef FMT_LEGACY_OSTREAM_FORMATTER
// template <>
// class fmt::formatter<rtc::PeerConnection::State>
// : public fmt::ostream_formatter {};
// #endif
namespace villas {
namespace node {

10
include/villas/nodes/websocket.hpp
View file

@ -132,8 +132,8 @@ int websocket_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt);
} // namespace node
} // namespace villas
#ifndef FMT_LEGACY_OSTREAM_FORMATTER
template <>
class fmt::formatter<villas::node::websocket_connection>
: public fmt::ostream_formatter {};
#endif
// #ifndef FMT_LEGACY_OSTREAM_FORMATTER
// template <>
// class fmt::formatter<villas::node::websocket_connection>
// : public fmt::ostream_formatter {};
// #endif

10
include/villas/path.hpp
View file

@ -183,8 +183,8 @@ public:
} // namespace node
} // namespace villas
#ifndef FMT_LEGACY_OSTREAM_FORMATTER
template <>
class fmt::formatter<villas::node::Path>
: public fmt::ostream_formatter {};
#endif
// #ifndef FMT_LEGACY_OSTREAM_FORMATTER
// template <>
// class fmt::formatter<villas::node::Path>
// : public fmt::ostream_formatter {};
// #endif

3
lib/node.cpp
View file

@ -185,7 +185,7 @@ int Node::start() {
assert(state == State::PREPARED);
logger->info("Starting node {}", getNameFull());
// logger->info("Starting node {}", getNameFull());
ret = in.start();
if (ret)
@ -469,6 +469,7 @@ int NodeFactory::start(SuperNode *sn) {
instances.size());
state = State::STARTED;
logger->debug("NodeFactory::start()");
return 0;
}

6
lib/nodes/CMakeLists.txt
View file

@ -159,6 +159,12 @@ if(WITH_NODE_EXAMPLE)
# list(APPEND LIBRARIES PkgConfig::EXAMPLELIB)
endif()
# Enable hdf5 node type
if(WITH_NODE_HDF5)
list(APPEND NODE_SRC hdf5.cpp)
list(APPEND LIBRARIES HDF5::HDF5)
endif()
# Enable Ethercat support
if(WITH_NODE_ETHERCAT)
list(APPEND NODE_SRC ethercat.cpp)

10
lib/nodes/file.cpp
View file

@ -8,6 +8,7 @@
#include <cerrno>
#include <cinttypes>
#include <cstring>
#include <iostream>
#include <libgen.h>
#include <sys/stat.h>
#include <unistd.h>
@ -212,6 +213,7 @@ char *villas::node::file_print(NodeCompat *n) {
int villas::node::file_start(NodeCompat *n) {
auto *f = n->getData<struct file>();
std::cout << "file_start" << std::endl;
struct timespec now = time_now();
int ret;
@ -221,6 +223,7 @@ int villas::node::file_start(NodeCompat *n) {
delete[] f->uri;
f->uri = file_format_name(f->uri_tmpl, &now);
std::cout << "file_format_name" << std::endl;
// Check if directory exists
struct stat sb;
@ -240,19 +243,23 @@ int villas::node::file_start(NodeCompat *n) {
if (ret)
throw SystemError("Failed to create directory");
}
std::cout << "stat" << std::endl;
free(cpy);
f->formatter->start(n->getInputSignals(false));
std::cout << "formatter->start" << std::endl;
// Open file
f->stream_out = fopen(f->uri, "a+");
if (!f->stream_out)
std::cout << "fopen stream_out -1" << std::endl;
return -1;
std::cout << "fopen stream_out" << std::endl;
f->stream_in = fopen(f->uri, "r");
if (!f->stream_in)
return -1;
std::cout << "fopen stream_in" << std::endl;
if (f->buffer_size_in) {
ret = setvbuf(f->stream_in, nullptr, _IOFBF, f->buffer_size_in);
@ -263,6 +270,7 @@ int villas::node::file_start(NodeCompat *n) {
if (f->buffer_size_out) {
ret = setvbuf(f->stream_out, nullptr, _IOFBF, f->buffer_size_out);
if (ret)
std::cout << "ret: " << ret << std::endl; // return ret;
return ret;
}

231
lib/nodes/hdf5.cpp Normal file
View file

@ -0,0 +1,231 @@
/* A HDF5 node-type.
*
* This node-type reads the samples and writes them to a HDF5 file.
*
* Author: Alexandra Bach <alexandra.bach@eonerc.rwth-aachen.de>
* SPDX-FileCopyrightText: 2014-2024 Institute for Automation of Complex Power Systems, RWTH Aachen University
* SPDX-License-Identifier: Apache-2.0
*/
#include <iostream>
#include <string>
#include <sys/stat.h>
#include <chrono>
#include <sstream>
#include <iomanip>
#include <H5Cpp.h>
#include <libgen.h>
#include <villas/exceptions.hpp>
#include <villas/node_compat.hpp>
#include <villas/nodes/hdf5.hpp>
#include <villas/sample.hpp>
#include <villas/super_node.hpp>
#include <villas/utils.hpp>
using namespace H5;
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
HDF5node::HDF5node(const uuid_t &id, const std::string &name)
: Node(id, name) {}
HDF5node::~HDF5node() {}
// int HDF5node::prepare() {
// // state1 = setting1;
// // if (setting2 == "double")
// // state1 *= 2;
// // return 0;
// // assert(state == State::PREPARED;);
// state = State::PREPARED;
// logger->debug("HDF5node::prepare()");
// return 0;
// }
int HDF5node::parse(json_t *json) {
// default values
location_name = "RWTH Aachen University";
description_name = "This is a test dataset";
project_name = "VILLASframework";
author_name = "none";
json_error_t err;
int ret = json_unpack_ex(json, &err, 0, "{ s: s, s: s, s?: s, s?: s, s?: s, s?: s, s: s }",
"file", &file_name, "dataset", &dataset_name, "location", &location_name,
"description", &description_name, "project", &project_name, "author", &author_name,
"unit", &unit);
if (ret)
throw ConfigError(json, err, "node-config-node-hdf5");
logger->debug("HDF5node::parse() file_name: {}", file_name);
return 0;
}
// int HDF5node::check() {
// // if (setting1 > 100 || setting1 < 0)
// // return -1;
// // if (setting2.empty() || setting2.size() > 10)
// // return -1;
// return 0;
// }
// int HDF5node::start() {
// assert(state == State::PREPARED ||
// state == State::PAUSED);
// logger->debug("HDF5node::start()");
// return Node::start();
// }
// int HDF5node::stop()
// {
// fclose(stream_in);
// fclose(stream_out);
// return 0;
// }
// int HDF5node::pause()
// {
// // TODO add implementation here
// return 0;
// }
// int HDF5node::resume()
// {
// // TODO add implementation here
// return 0;
// }
// int HDF5node::restart()
// {
// // TODO add implementation here
// return 0;
// }
// int HDF5node::reverse()
// {
// // TODO add implementation here
// return 0;
// }
// std::vector<int> HDF5node::getPollFDs()
// {
// // TODO add implementation here
// return {};
// }
// std::vector<int> HDF5node::getNetemFDs()
// {
// // TODO add implementation here
// return {};
// }
// struct villas::node::memory::Type * HDF5node::getMemoryType()
// {
// // TODO add implementation here
// }
// const std::string &HDF5node::getDetails() {
// details = fmt::format("setting1={}, setting2={}", setting1, setting2);
// return details;
// }
// int HDF5node::_read(struct Sample *smps[], unsigned cnt) {
// }
// similar to file node-type: takes the samples and writes them to a HDF5 file
int HDF5node::_write(struct Sample *smps[], unsigned cnt) {
// Create a new file using the default property lists.
H5File file(H5std_string(file_name), H5F_ACC_TRUNC);
// Create the data space for the dataset. Defines the size and shape of the dataset.
logger->info("smps[0]->length {}", smps[0]->length);
// hsize_t dims[2] = {cnt, 1}; // cnt rowns, smps[0]->length columns
hsize_t dims[2] = {4, 6};
DataSpace dataspace(2, dims);
// Create the dataset. Composes a collection of data elements, raw data and metadata.
DataSet dataset = file.createDataSet(H5std_string(dataset_name), PredType::STD_I32BE, dataspace);
// Write the data to the dataset.
// for (unsigned j = 0; j < cnt; j++) {
// struct Sample *smp = smps[j];
// // ret = formatter->print(stream_out, smp, j+1);
// dataset.write(smp->data, PredType::NATIVE_DOUBLE);
// }
int data[4][6] = { {1, 2, 3, 4, 5, 6}, {7, 8, 9, 10, 11, 12}, {13, 14, 15, 16, 17, 18}, {19, 20, 21, 22, 23, 24}};
dataset.write(data, H5::PredType::NATIVE_INT);
// Create a dataspace for the attribute
DataSpace attr_dataspace(H5S_SCALAR);
// Create a string datatype
StrType strdatatype(PredType::C_S1, H5T_VARIABLE); // for variable-length string
// Create the timestamp attribute and write to it
Attribute timestampAttribute = dataset.createAttribute("timestamp", strdatatype, attr_dataspace);
// Get the current time
auto now = std::chrono::system_clock::now();
std::time_t now_c = std::chrono::system_clock::to_time_t(now);
// Convert the current time to a string
std::stringstream ss;
ss << std::put_time(std::localtime(&now_c), "%Y-%m-%dT%H:%M:%S");
std::string timestamp = ss.str();
// Write the timestamp to the attribute
timestampAttribute.write(strdatatype, timestamp);
// Create the location attribute and write to it
Attribute locationAttribute = dataset.createAttribute("location", strdatatype, attr_dataspace);
// convert location_name to sting
std::string location_name_string = location_name;
locationAttribute.write(strdatatype, location_name_string);
// Create the description attribute and write to it
Attribute descriptionAttribute = dataset.createAttribute("description", strdatatype, attr_dataspace);
std::string description_name_string = description_name;
descriptionAttribute.write(strdatatype, description_name_string);
// Create the project attribute and write to it
Attribute projectAttribute = dataset.createAttribute("project", strdatatype, attr_dataspace);
std::string project_name_string = project_name;
projectAttribute.write(strdatatype, project_name_string);
// Create the author attribute and write to it
Attribute authorAttribute = dataset.createAttribute("author", strdatatype, attr_dataspace);
std::string author_name_string = author_name;
authorAttribute.write(strdatatype, author_name_string);
logger->debug("attributes created and written");
timestampAttribute.close(); // put in extra destroy function?
locationAttribute.close();
descriptionAttribute.close();
projectAttribute.close();
authorAttribute.close();
dataset.close();
file.close();
return cnt;
}
// Register node
static char n[] = "hdf5";
static char d[] = "This node-type writes samples to hdf5 file format";
static NodePlugin<HDF5node, n, d,
(int)NodeFactory::Flags::SUPPORTS_READ |
(int)NodeFactory::Flags::SUPPORTS_WRITE |
(int)NodeFactory::Flags::SUPPORTS_POLL >
p;

9
lib/signal_list.cpp
View file

@ -5,6 +5,8 @@
* SPDX-License-Identifier: Apache-2.0
*/
#include <iostream>
#include <villas/exceptions.hpp>
#include <villas/list.hpp>
#include <villas/signal.hpp>
@ -145,9 +147,10 @@ Signal::Ptr SignalList::getByName(const std::string &name) {
SignalList::Ptr SignalList::clone() {
auto l = std::make_shared<SignalList>();
for (auto s : *this)
l->push_back(s);
if (!l->empty()){ // include check if list of pointers is empty
for (auto s : *this)
l->push_back(s);
}
return l;
}

3
packaging/deps.sh
View file

@ -421,8 +421,7 @@ if ! pkg-config "libmodbus >= 3.1.0" && \
git clone ${GIT_OPTS} --recursive --branch v3.1.10 https://github.com/stephane/libmodbus.git
mkdir -p libmodbus/build
pushd libmodbus
autoreconf -i
./configure ${CONFIGURE_OPTS}
cmake ${CMAKE_OPTS} ..
make ${MAKE_OPTS} install
popd
fi

3
packaging/docker/Dockerfile.debian
View file

@ -51,7 +51,8 @@ RUN apt-get update && \
liblua5.3-dev \
libhiredis-dev \
libnice-dev \
libmodbus-dev
libmodbus-dev \
libhdf5-dev
# Add local and 64-bit locations to linker paths
ENV echo /usr/local/lib >> /etc/ld.so.conf && \

13
packaging/docker/Dockerfile.fedora
View file

@ -64,7 +64,8 @@ RUN dnf -y install \
lua-devel \
hiredis-devel \
libnice-devel \
libmodbus-devel
libmodbus-devel \
hdf5-devel
# Add local and 64-bit locations to linker paths
RUN echo /usr/local/lib >> /etc/ld.so.conf && \
@ -78,11 +79,11 @@ RUN ldconfig
# Workaround for libnl3's search path for netem distributions
RUN ln -s /usr/lib64/tc /usr/lib/tc
COPY ./python /python
RUN python -m venv /venv && \
source /venv/bin/activate && \
pip install /python[dev] && \
rm -r /python
# COPY ./python /python
# RUN python -m venv /venv && \
# source /venv/bin/activate && \
# pip install /python[dev] && \
# rm -r /python
# Expose ports for HTTP and WebSocket frontend
EXPOSE 80

3
packaging/docker/Dockerfile.ubuntu
View file

@ -53,7 +53,8 @@ RUN apt-get update && \
liblua5.3-dev \
libhiredis-dev \
libnice-dev \
libmodbus-dev
libmodbus-dev \
libhdf5-dev
# Add local and 64-bit locations to linker paths
ENV echo /usr/local/lib >> /etc/ld.so.conf && \

55
tests/integration/node-hdf5.sh Executable file
View file

@ -0,0 +1,55 @@
#!/bin/bash
#
# Test for the example node-type
#
# Author: Steffen Vogel <post@steffenvogel.de>
# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
# SPDX-License-Identifier: Apache-2.0
set -e
echo "The example node does nothing useful which we could test"
exit 99
DIR=$(mktemp -d)
pushd ${DIR}
function finish {
popd
rm -rf ${DIR}
}
trap finish EXIT
cat > expect.dat <<EOF
1553690684.041211800-4.313770e-02(5) 6.303265 0.492616 0.309017 -1.000000 0.800000 0.050000
1553690684.051211800-5.287260e-02(6) 5.673902 0.148827 0.368125 -1.000000 0.760000 0.060000
1553690684.061211800-6.266780e-02(7) 5.896198 0.232320 0.425779 -1.000000 0.720000 0.070000
1553690684.071211800-7.256350e-02(8) 5.788125 0.152309 0.481754 -1.000000 0.680000 0.080000
1553690684.081211800-8.251890e-02(9) 6.748635 0.608491 0.535827 -1.000000 0.640000 0.090000
EOF
cat > config.json <<EOF
{
"nodes": {
"example_node1": {
"type": "example",
"setting1": 66
},
"file": {
"type": "file",
"uri": "output.dat"
}
},
"paths": [
{
"in": "example_node1",
"out": "file",
}
]
}
EOF
villas node config.json
villas compare output.dat expect.dat