diff --git a/lib/formats/villas.proto b/lib/formats/villas.proto index 8bc857b07..bbac30510 100644 --- a/lib/formats/villas.proto +++ b/lib/formats/villas.proto @@ -22,9 +22,10 @@ message Sample { required Type type = 1 [default = DATA]; optional uint64 sequence = 2; // The sequence number is incremented for consecutive samples. - optional Timestamp ts_origin = 4; - repeated Value values = 5; - optional bool new_frame = 6; + optional Timestamp ts_origin = 3; + optional Timestamp ts_received = 4; + optional bool new_frame = 5; + repeated Value values = 100; } message Timestamp { diff --git a/packaging/nix/python.nix b/packaging/nix/python.nix index 3d51314c8..bfaca0019 100644 --- a/packaging/nix/python.nix +++ b/packaging/nix/python.nix @@ -24,6 +24,8 @@ python3Packages.buildPythonPackage { mypy pytest types-requests + + pytestCheckHook ]; postPatch = '' diff --git a/python/.envrc b/python/.envrc index 58aee6e7d..ec3bd5350 100644 --- a/python/.envrc +++ b/python/.envrc @@ -1,36 +1,5 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 +use flake .#villas-python +watch_file ../flake.nix -export_or_unset() -{ - local var=$1 - - if [ -z "${!var+x}" ]; then - return - fi - - if [ -n "$2" ]; then - export $var="$2" - else - unset $var - fi - -} - -if direnv_version "2.30.0" \ -&& has nix \ -&& nix show-config experimental-features 2>/dev/null | grep -wqF flakes -then - local oldtmp="$TMP" - local oldtemp="$TEMP" - local oldtmpdir="$TMPDIR" - local oldtempdir="$TEMPDIR" - - watch_file ../packaging/nix/*.nix - use flake ../packaging/nix#villas-python - - export_or_unset TMP "$oldtmp" - export_or_unset TEMP "$oldtemp" - export_or_unset TMPDIR "$oldtmpdir" - export_or_unset TEMPDIR "$oldtempdir" -fi diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 07d184453..7f8fcdac6 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -9,7 +9,7 @@ if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND) OUTPUT villas_pb2.py COMMAND ${PROTOBUF_COMPILER} - --python_out=${CMAKE_CURRENT_BINARY_DIR} + --python_out=${CMAKE_CURRENT_BINARY_DIR}/villas/node villas.proto MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/lib/formats/villas.proto WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/lib/formats diff --git a/python/villas/node/formats.py b/python/villas/node/formats.py index 3371ba8de..a1aaaad11 100644 --- a/python/villas/node/formats.py +++ b/python/villas/node/formats.py @@ -10,6 +10,7 @@ from itertools import groupby from typing import Iterable from villas.node.sample import Sample, Signal, Timestamp +import villas.node.villas_pb2 as pb class SignalList(list[type]): @@ -239,3 +240,91 @@ class VillasHuman(Format): def _pack_complex(self, z: complex) -> str: return f"{z.real}+{z.imag}i" + + +@dataclass +class Protobuf(Format): + """ + The protobuf format in Python. + """ + + def loadb(self, b: bytes) -> list[Sample]: + msg = pb.Message() + msg.ParseFromString(b) + + return [self.load_sample(sample) for sample in msg.samples] + + def dumpb(self, samples: Iterable[Sample]) -> bytes: + msg = pb.Message() + + for sample in samples: + msg.samples.append(self.dump_sample(sample)) + + return msg.SerializeToString() + + def load_sample(self, pb_sample: pb.Sample) -> Sample: + sample = Sample() + + if pb_sample.HasField("ts_origin"): + sample.ts_origin = Timestamp( + pb_sample.ts_origin.sec, pb_sample.ts_origin.nsec + ) + + if pb_sample.HasField("ts_received"): + sample.ts_received = Timestamp( + pb_sample.ts_received.sec, pb_sample.ts_received.nsec + ) + + if pb_sample.HasField("new_frame"): + sample.new_frame = pb_sample.new_frame + + if pb_sample.HasField("sequence"): + sample.sequence = pb_sample.sequence + + for value in pb_sample.values: + if value.HasField("i"): + sample.data.append(value.i) + elif value.HasField("f"): + sample.data.append(value.f) + elif value.HasField("b"): + sample.data.append(value.b) + elif value.HasField("z"): + sample.data.append(complex(value.z.real, value.z.imag)) + else: + raise Exception("Missing value") + + return self._strip_sample(sample) + + def dump_sample(self, sample: Sample) -> pb.Sample: + pb_sample = pb.Sample() + pb_sample.type = pb.Sample.Type.DATA + + pb_sample.new_frame = sample.new_frame + + if sample.ts_origin: + pb_sample.ts_origin.sec = sample.ts_origin.seconds + pb_sample.ts_origin.nsec = sample.ts_origin.nanoseconds + + if sample.ts_received: + pb_sample.ts_received.sec = sample.ts_received.seconds + pb_sample.ts_received.nsec = sample.ts_received.nanoseconds + + if sample.sequence: + pb_sample.sequence = sample.sequence + + for value in sample.data: + pb_value = pb.Value() + + if isinstance(value, int): + pb_value.i = value + elif isinstance(value, float): + pb_value.f = value + elif isinstance(value, bool): + pb_value.b = value + elif isinstance(value, complex): + pb_value.z.real = value.real + pb_value.z.imag = value.imag + + pb_sample.values.append(pb_value) + + return pb_sample diff --git a/python/villas/node/test_formats.py b/python/villas/node/test_formats.py index 45b88b629..366281a24 100644 --- a/python/villas/node/test_formats.py +++ b/python/villas/node/test_formats.py @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 from cmath import sqrt -from villas.node.formats import SignalList, VillasHuman +from villas.node.formats import SignalList, VillasHuman, Protobuf from villas.node.sample import Sample, Timestamp @@ -25,29 +25,38 @@ def test_villas_human_repr(): assert villas_human == eval(repr(villas_human)) +smp1 = Sample( + ts_origin=Timestamp(123456780), + ts_received=Timestamp(123456781), + sequence=4, + new_frame=True, + data=[1.0, 2.0, 3.0, True, 42, sqrt(complex(-1))], +) + +smp2 = Sample( + ts_origin=Timestamp(123456789), + ts_received=Timestamp(123456790), + sequence=5, + new_frame=False, + data=[1.0, 2.0, 3.0, False, 42, sqrt(complex(-1))], +) + + def test_villas_human(): - smp1 = Sample( - ts_origin=Timestamp(123456780), - ts_received=Timestamp(123456781), - sequence=4, - new_frame=True, - data=[1.0, 2.0, 3.0, True, 42, sqrt(complex(-1))], - ) - - smp2 = Sample( - ts_origin=Timestamp(123456789), - ts_received=Timestamp(123456790), - sequence=5, - new_frame=False, - data=[1.0, 2.0, 3.0, False, 42, sqrt(complex(-1))], - ) - - villas_human = VillasHuman(signal_list=SignalList(smp1)) + vh = VillasHuman(signal_list=SignalList(smp1)) smp1_str = "123456780+1.0(4)F\t1.0\t2.0\t3.0\t1\t42\t0.0+1.0i\n" smp2_str = "123456789+1.0(5)\t1.0\t2.0\t3.0\t0\t42\t0.0+1.0i\n" - assert villas_human.dump_sample(smp1) == smp1_str - assert villas_human.dump_sample(smp2) == smp2_str - assert villas_human.dumps([smp1, smp2]) == smp1_str + smp2_str - assert villas_human.load_sample(smp1_str) == smp1 - assert villas_human.load_sample(smp2_str) == smp2 - assert villas_human.loads(smp1_str + smp2_str) == [smp1, smp2] + assert vh.dump_sample(smp1) == smp1_str + assert vh.dump_sample(smp2) == smp2_str + assert vh.dumps([smp1, smp2]) == smp1_str + smp2_str + assert vh.load_sample(smp1_str) == smp1 + assert vh.load_sample(smp2_str) == smp2 + assert vh.loads(smp1_str + smp2_str) == [smp1, smp2] + + +def test_protobuf(): + pb = Protobuf() + + raw = pb.dumpb([smp1, smp2]) + + assert pb.loadb(raw) == [smp1, smp2]