From 7db2b0a3804865fc38f87a569f5810e61a81969b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 09:33:35 +0100 Subject: [PATCH 1/8] added new example on using VILLASnode Python API --- python/examples/Shmem_CIGRE_MV.py | 86 +++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 python/examples/Shmem_CIGRE_MV.py diff --git a/python/examples/Shmem_CIGRE_MV.py b/python/examples/Shmem_CIGRE_MV.py new file mode 100644 index 000000000..78f1322f8 --- /dev/null +++ b/python/examples/Shmem_CIGRE_MV.py @@ -0,0 +1,86 @@ + + +import time +from villas.node.node import Node as VILLASnode + +# This could be moved to the DPsim Python code later +def get_dpsim_shmem_interface_config(): + return { + 'type': 'shmem', + 'in': { + 'name': '/dpsim1-villas', + 'hooks': [ + {'type': 'stats'} + ], + 'signals': { + 'count': 30, + 'type': 'float' + } + }, + 'out': { + 'name': '/villas-dpsim1' + } + } + +def get_villas_config(): + return { + 'nodes': { + 'broker1': { + 'type': 'mqtt', + 'format': 'json', + 'host': '172.17.0.1', + 'in': { + 'subscribe': '/powerflow-dpsim'}, + 'out': { + 'publish': '/dpsim-powerflow' + } + }, + 'dpsim1': get_dpsim_shmem_interface_config(), + }, + 'paths': [ + { + 'in': 'dpsim1', + 'out': 'broker1', + + 'hooks': [ + { + 'type': 'limit_rate', + 'rate': 50 + } + ] + } + ] + } + +def main(): + + node = VILLASnode( + config=get_villas_config() + ) + + node.start() # VILLASnode starts running in the background from here.. + + # Some infos from the running VILLASnode instance queried via its REST API + print('VILLASnode running?: ', node.is_running()) + print('VILLASnode status: ', node.status) + print('VILLASnode nodes: ', node.nodes) + print('VILLASnode paths: ', node.paths) + print('VILLASnode config: ', node.active_config) + print('VILLASnode version: ', node.get_version()) + + # Load a new config into the running VILLASnode instance (old config will be replaced) + new_config = node.active_config + new_config['paths'].append({ + 'out': 'dpsim1', + 'in': 'broker1' + }) + + node.load_config(new_config) + + time.sleep(100) + + node.stop() + + +if __name___ == 'main': + main() From 07d2ac98a49b77d091b2d8df90df6a7f08d455bc Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 09:52:06 +0100 Subject: [PATCH 2/8] Update Shmem_CIGRE_MV.py --- python/examples/Shmem_CIGRE_MV.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/python/examples/Shmem_CIGRE_MV.py b/python/examples/Shmem_CIGRE_MV.py index 78f1322f8..7648a93ba 100644 --- a/python/examples/Shmem_CIGRE_MV.py +++ b/python/examples/Shmem_CIGRE_MV.py @@ -1,21 +1,34 @@ - - import time from villas.node.node import Node as VILLASnode # This could be moved to the DPsim Python code later + +# It would be nice if the DPsim Shmem interface could build-up a list of actual +# signal descriptions (names, units, etc..) which attributes are exported. +# This would eliviate the user from manually configuring signal mappings +def get_dpsim_shmem_interface_signals(): + signals = [] + + for i in range(0, 30): + signals.append({ + 'name': f'signal_{i}', + 'type': 'float', + 'unit': 'volts' + }) + + return signals + def get_dpsim_shmem_interface_config(): return { 'type': 'shmem', 'in': { 'name': '/dpsim1-villas', 'hooks': [ - {'type': 'stats'} + { + 'type': 'stats' + } ], - 'signals': { - 'count': 30, - 'type': 'float' - } + 'signals': get_dpsim_shmem_interface_signals() }, 'out': { 'name': '/villas-dpsim1' From d06c388e56dcf332c1f0caac9371e53d73abc0a6 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 12:33:28 +0100 Subject: [PATCH 3/8] python: fix flake8 errors --- python/examples/Shmem_CIGRE_MV.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/python/examples/Shmem_CIGRE_MV.py b/python/examples/Shmem_CIGRE_MV.py index 7648a93ba..e87734354 100644 --- a/python/examples/Shmem_CIGRE_MV.py +++ b/python/examples/Shmem_CIGRE_MV.py @@ -3,10 +3,11 @@ from villas.node.node import Node as VILLASnode # This could be moved to the DPsim Python code later -# It would be nice if the DPsim Shmem interface could build-up a list of actual -# signal descriptions (names, units, etc..) which attributes are exported. -# This would eliviate the user from manually configuring signal mappings + def get_dpsim_shmem_interface_signals(): + """ It would be nice if the DPsim Shmem interface could build-up a list of actual + signal descriptions (names, units, etc..) which attributes are exported. + This would eliviate the user from manually configuring signal mappings """ signals = [] for i in range(0, 30): @@ -15,9 +16,10 @@ def get_dpsim_shmem_interface_signals(): 'type': 'float', 'unit': 'volts' }) - + return signals + def get_dpsim_shmem_interface_config(): return { 'type': 'shmem', @@ -35,6 +37,7 @@ def get_dpsim_shmem_interface_config(): } } + def get_villas_config(): return { 'nodes': { @@ -65,13 +68,14 @@ def get_villas_config(): ] } + def main(): node = VILLASnode( config=get_villas_config() ) - node.start() # VILLASnode starts running in the background from here.. + node.start() # VILLASnode starts running in the background from here.. # Some infos from the running VILLASnode instance queried via its REST API print('VILLASnode running?: ', node.is_running()) @@ -81,7 +85,8 @@ def main(): print('VILLASnode config: ', node.active_config) print('VILLASnode version: ', node.get_version()) - # Load a new config into the running VILLASnode instance (old config will be replaced) + # Load a new config into the running + # VILLASnode instance (old config will be replaced) new_config = node.active_config new_config['paths'].append({ 'out': 'dpsim1', @@ -95,5 +100,5 @@ def main(): node.stop() -if __name___ == 'main': +if __name__ == 'main': main() From f10708d771ca7eeb9401d1eadfdc93e0d0deb5a8 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 13:01:30 +0100 Subject: [PATCH 4/8] fix flake8 errors --- python/examples/Shmem_CIGRE_MV.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/examples/Shmem_CIGRE_MV.py b/python/examples/Shmem_CIGRE_MV.py index e87734354..c5cb8d713 100644 --- a/python/examples/Shmem_CIGRE_MV.py +++ b/python/examples/Shmem_CIGRE_MV.py @@ -5,9 +5,11 @@ from villas.node.node import Node as VILLASnode def get_dpsim_shmem_interface_signals(): - """ It would be nice if the DPsim Shmem interface could build-up a list of actual - signal descriptions (names, units, etc..) which attributes are exported. - This would eliviate the user from manually configuring signal mappings """ + """ It would be nice if the DPsim Shmem interface could + build-up a list of actual signal descriptions + (names, units, etc..) which attributes are exported. + This would eliviate the user from manually configuring + signal mappings """ signals = [] for i in range(0, 30): From 8b35978ddad7845d7ec4fd4040d622a75cda5668 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 15:57:32 +0100 Subject: [PATCH 5/8] ci: bump version of docekr image --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d66ff978b..9ca1de200 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ stages: # Build docker image which is used to build & test VILLASnode prepare:docker: stage: prepare - image: docker:19.03 + image: docker:20.10 script: - docker build ${DOCKER_OPTS} --file ${DOCKER_FILE} @@ -206,7 +206,7 @@ pkg:rpm: pkg:alpine: stage: packaging - image: docker:19.03 + image: docker:20.10 before_script: - mkdir -p ~/.docker/cli-plugins/ - wget -O ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.4.2/buildx-v0.4.2.linux-amd64 @@ -266,7 +266,7 @@ deploy:web: deploy:docker: stage: deploy - image: docker:19.03 + image: docker:20.10 variables: DOCKER_CLI_EXPERIMENTAL: enabled before_script: @@ -323,7 +323,7 @@ deploy:packages:tags: .latest:docker:latest: &deploy_latest_docker stage: latest - image: docker:19.03 + image: docker:20.10 before_script: - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} script: From 80b58e21c62e29da08af1edd9bd473ead5e25098 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 17 Feb 2021 16:04:47 +0100 Subject: [PATCH 6/8] ci: avoid seccomp issues --- .gitlab-ci.yml | 3 ++- packaging/docker/Dockerfile.alpine | 41 +++++++++++++++++------------- packaging/docker/Dockerfile.centos | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ca1de200..5491cc688 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -211,11 +211,12 @@ pkg:alpine: - mkdir -p ~/.docker/cli-plugins/ - wget -O ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.4.2/buildx-v0.4.2.linux-amd64 - chmod a+x ~/.docker/cli-plugins/docker-buildx - - docker buildx create --use --name cross-platform-build + - docker buildx create --use --name cross-platform-build --buildkitd-flags "--allow-insecure-entitlement security.insecure" - docker buildx inspect --bootstrap cross-platform-build - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} script: - docker buildx build ${DOCKER_OPTS} + --allow security.insecure --output type=docker --target app --build-arg ARCH=${ARCH} diff --git a/packaging/docker/Dockerfile.alpine b/packaging/docker/Dockerfile.alpine index 8688d829d..1d2e71ac3 100644 --- a/packaging/docker/Dockerfile.alpine +++ b/packaging/docker/Dockerfile.alpine @@ -1,3 +1,5 @@ +#syntax=docker/dockerfile:1.2-labs + # Dockerfile for VILLASnode development. # # This Dockerfile builds an image which contains all library dependencies @@ -87,7 +89,8 @@ ADD packaging/deps.sh / # https://github.com/creytiv/re/issues/256 # https://github.com/mz-automation/libiec61850/issues/279 -RUN export SKIP_CRITERION=1; \ +RUN --security=insecure \ + export SKIP_CRITERION=1; \ export SKIP_ETHERLAB=1; \ export SKIP_LIBRE=1; \ if [ "${ARCH}" == "armv6" -o "${ARCH}" == "armv7" ]; then \ @@ -103,7 +106,8 @@ COPY . /villas/ RUN mkdir -p /villas/build WORKDIR /villas/build -RUN cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ +RUN --security=insecure \ + cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DCMAKE_PREFIX_PATH=${PREFIX} .. && \ make -j$(nproc) install @@ -115,22 +119,23 @@ ARG TRIPLET=x86_64-alpine-linux-musl RUN echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN apk update RUN apk add \ - openssl \ - ossp-uuid@testing \ - libconfig \ - curl \ - jansson \ - spdlog \ - fmt \ - libnl3 \ - graphviz \ - protobuf \ - protobuf-c \ - zeromq \ - rabbitmq-c \ - mosquitto \ - libusb + openssl \ + libconfig \ + curl \ + jansson \ + spdlog \ + fmt \ + libnl3 \ + graphviz \ + protobuf \ + protobuf-c \ + zeromq \ + rabbitmq-c \ + mosquitto \ + libusb \ + ossp-uuid@testing RUN if [ "${ARCH}" != "armv6" -a "${ARCH}" != "armv7" ]; then \ apk add \ @@ -144,7 +149,7 @@ ENV LD_LIBRARY_PATH=/app/lib:/app/lib64 ENV PATH=/app/bin:${PATH} # Test if it runs -RUN villas node -h 2&>1 > /dev/null +RUN /app/bin/villas-node -h 2>&1 > /dev/null ARG GIT_REV=unknown ARG GIT_BRANCH=unknown diff --git a/packaging/docker/Dockerfile.centos b/packaging/docker/Dockerfile.centos index ecb1100a1..3a3870412 100644 --- a/packaging/docker/Dockerfile.centos +++ b/packaging/docker/Dockerfile.centos @@ -39,7 +39,7 @@ USER root # Enable Extra Packages for Enterprise Linux (EPEL) repo and PowerTools RUN dnf -y install epel-release dnf-plugins-core -RUN dnf config-manager --set-enabled PowerTools +RUN dnf config-manager --set-enabled powertools # Toolchain RUN dnf -y install \ From fb72d16a38525f8050cb1e8143cad0dda04de620 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 19 Feb 2021 00:04:13 +0100 Subject: [PATCH 7/8] ci: make cppcheck happy --- lib/nodes/websocket.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/nodes/websocket.cpp b/lib/nodes/websocket.cpp index 87fd29137..d7918fa7f 100644 --- a/lib/nodes/websocket.cpp +++ b/lib/nodes/websocket.cpp @@ -311,10 +311,6 @@ int websocket_protocol_cb(struct lws *wsi, enum lws_callback_reasons reason, voi int avail, enqueued; struct websocket *w = (struct websocket *) n->_vd; struct sample *smps[cnt]; - if (!smps) { - warning("Failed to allocate memory for connection: %s", websocket_connection_name(c)); - break; - } avail = sample_alloc_many(&w->pool, smps, cnt); if (avail < cnt) From 39442895321d2281e77469f963f2418ffe6509c3 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 8 Jan 2021 22:58:49 +0100 Subject: [PATCH 8/8] docker: update headers in Dockerfiles --- packaging/docker/Dockerfile.alpine | 13 ++----------- packaging/docker/Dockerfile.centos | 10 +--------- packaging/docker/Dockerfile.debian | 10 +--------- packaging/docker/Dockerfile.debian-multiarch | 10 +--------- packaging/docker/Dockerfile.fedora | 10 +--------- packaging/docker/Dockerfile.fedora-minimal | 2 +- packaging/docker/Dockerfile.ubuntu | 10 +--------- 7 files changed, 8 insertions(+), 57 deletions(-) diff --git a/packaging/docker/Dockerfile.alpine b/packaging/docker/Dockerfile.alpine index 1d2e71ac3..17ab454e9 100644 --- a/packaging/docker/Dockerfile.alpine +++ b/packaging/docker/Dockerfile.alpine @@ -1,14 +1,5 @@ #syntax=docker/dockerfile:1.2-labs - -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker-dev +# Alpine Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC @@ -109,7 +100,7 @@ WORKDIR /villas/build RUN --security=insecure \ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DCMAKE_PREFIX_PATH=${PREFIX} .. && \ - make -j$(nproc) install + make -j8 install FROM alpine:edge AS app diff --git a/packaging/docker/Dockerfile.centos b/packaging/docker/Dockerfile.centos index 3a3870412..bc60fce1d 100644 --- a/packaging/docker/Dockerfile.centos +++ b/packaging/docker/Dockerfile.centos @@ -1,12 +1,4 @@ -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker +# CentOS Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index 8bccba0ba..2d3c836ec 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -1,12 +1,4 @@ -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker +# Debian Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC diff --git a/packaging/docker/Dockerfile.debian-multiarch b/packaging/docker/Dockerfile.debian-multiarch index 5fd6cfae6..ef4fdffa4 100644 --- a/packaging/docker/Dockerfile.debian-multiarch +++ b/packaging/docker/Dockerfile.debian-multiarch @@ -1,12 +1,4 @@ -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker +# Debian Multiarch Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index 739a2d9f0..2c19c8f5d 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -1,12 +1,4 @@ -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker-dev +# Fedora Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC diff --git a/packaging/docker/Dockerfile.fedora-minimal b/packaging/docker/Dockerfile.fedora-minimal index 2d6d5810c..a7880ac29 100644 --- a/packaging/docker/Dockerfile.fedora-minimal +++ b/packaging/docker/Dockerfile.fedora-minimal @@ -1,4 +1,4 @@ -# Minimal Dockerfile for building VILLASnode +# Minimal Fedora Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index b34bf1e29..6ac0ff816 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -1,12 +1,4 @@ -# Dockerfile for VILLASnode development. -# -# This Dockerfile builds an image which contains all library dependencies -# and tools to build VILLASnode. -# However, VILLASnode itself it not part of the image. -# -# This image can be used for developing VILLASnode -# by running: -# make docker +# Ubuntu Dockerfile # # @author Steffen Vogel # @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC