python: Add protobuf format and test

Signed-off-by: Steffen Vogel <post@steffenvogel.de>

lib/formats/villas.proto

@@ -22,9 +22,10 @@ message Sample {
 
   required Type type = 1 [default = DATA];
   optional uint64 sequence = 2; // The sequence number is incremented for consecutive samples.
-  optional Timestamp ts_origin = 4;
-  repeated Value values = 5;
-  optional bool new_frame = 6;
+  optional Timestamp ts_origin = 3;
+  optional Timestamp ts_received = 4;
+  optional bool new_frame = 5;
+  repeated Value values = 100;
 }
 
 message Timestamp {

packaging/nix/python.nix

@@ -24,6 +24,8 @@ python3Packages.buildPythonPackage {
     mypy
     pytest
     types-requests
+
+    pytestCheckHook
   ];
 
   postPatch = ''

python/.envrc

@@ -1,36 +1,5 @@
 # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
 # SPDX-License-Identifier: Apache-2.0
+use flake .#villas-python
+watch_file ../flake.nix
 
-export_or_unset()
-{
-    local var=$1
-
-    if [ -z "${!var+x}" ]; then
-        return
-    fi
-
-    if [ -n "$2" ]; then
-        export $var="$2"
-    else
-        unset $var
-    fi
-
-}
-
-if direnv_version "2.30.0" \
-&& has nix \
-&& nix show-config experimental-features 2>/dev/null | grep -wqF flakes
-then
-    local oldtmp="$TMP"
-    local oldtemp="$TEMP"
-    local oldtmpdir="$TMPDIR"
-    local oldtempdir="$TEMPDIR"
-
-    watch_file ../packaging/nix/*.nix
-    use flake ../packaging/nix#villas-python
-
-    export_or_unset TMP "$oldtmp"
-    export_or_unset TEMP "$oldtemp"
-    export_or_unset TMPDIR "$oldtmpdir"
-    export_or_unset TEMPDIR "$oldtempdir"
-fi

python/CMakeLists.txt

@@ -9,7 +9,7 @@ if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND)
         OUTPUT
             villas_pb2.py
         COMMAND ${PROTOBUF_COMPILER}
-            --python_out=${CMAKE_CURRENT_BINARY_DIR}
+            --python_out=${CMAKE_CURRENT_BINARY_DIR}/villas/node
             villas.proto
         MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/lib/formats/villas.proto
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/lib/formats

python/villas/node/formats.py

@@ -10,6 +10,7 @@ from itertools import groupby
 from typing import Iterable
 
 from villas.node.sample import Sample, Signal, Timestamp
+import villas.node.villas_pb2 as pb
 
 
 class SignalList(list[type]):
@@ -239,3 +240,91 @@ class VillasHuman(Format):
 
     def _pack_complex(self, z: complex) -> str:
         return f"{z.real}+{z.imag}i"
+
+
+@dataclass
+class Protobuf(Format):
+    """
+    The protobuf format in Python.
+    """
+
+    def loadb(self, b: bytes) -> list[Sample]:
+        msg = pb.Message()
+        msg.ParseFromString(b)
+
+        return [self.load_sample(sample) for sample in msg.samples]
+
+    def dumpb(self, samples: Iterable[Sample]) -> bytes:
+        msg = pb.Message()
+
+        for sample in samples:
+            msg.samples.append(self.dump_sample(sample))
+
+        return msg.SerializeToString()
+
+    def load_sample(self, pb_sample: pb.Sample) -> Sample:
+        sample = Sample()
+
+        if pb_sample.HasField("ts_origin"):
+            sample.ts_origin = Timestamp(
+                pb_sample.ts_origin.sec, pb_sample.ts_origin.nsec
+            )
+
+        if pb_sample.HasField("ts_received"):
+            sample.ts_received = Timestamp(
+                pb_sample.ts_received.sec, pb_sample.ts_received.nsec
+            )
+
+        if pb_sample.HasField("new_frame"):
+            sample.new_frame = pb_sample.new_frame
+
+        if pb_sample.HasField("sequence"):
+            sample.sequence = pb_sample.sequence
+
+        for value in pb_sample.values:
+            if value.HasField("i"):
+                sample.data.append(value.i)
+            elif value.HasField("f"):
+                sample.data.append(value.f)
+            elif value.HasField("b"):
+                sample.data.append(value.b)
+            elif value.HasField("z"):
+                sample.data.append(complex(value.z.real, value.z.imag))
+            else:
+                raise Exception("Missing value")
+
+        return self._strip_sample(sample)
+
+    def dump_sample(self, sample: Sample) -> pb.Sample:
+        pb_sample = pb.Sample()
+        pb_sample.type = pb.Sample.Type.DATA
+
+        pb_sample.new_frame = sample.new_frame
+
+        if sample.ts_origin:
+            pb_sample.ts_origin.sec = sample.ts_origin.seconds
+            pb_sample.ts_origin.nsec = sample.ts_origin.nanoseconds
+
+        if sample.ts_received:
+            pb_sample.ts_received.sec = sample.ts_received.seconds
+            pb_sample.ts_received.nsec = sample.ts_received.nanoseconds
+
+        if sample.sequence:
+            pb_sample.sequence = sample.sequence
+
+        for value in sample.data:
+            pb_value = pb.Value()
+
+            if isinstance(value, int):
+                pb_value.i = value
+            elif isinstance(value, float):
+                pb_value.f = value
+            elif isinstance(value, bool):
+                pb_value.b = value
+            elif isinstance(value, complex):
+                pb_value.z.real = value.real
+                pb_value.z.imag = value.imag
+
+            pb_sample.values.append(pb_value)
+
+        return pb_sample

python/villas/node/test_formats.py

@@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0
 
 from cmath import sqrt
 
-from villas.node.formats import SignalList, VillasHuman
+from villas.node.formats import SignalList, VillasHuman, Protobuf
 from villas.node.sample import Sample, Timestamp
 
 
@@ -25,29 +25,38 @@ def test_villas_human_repr():
     assert villas_human == eval(repr(villas_human))
 
 
+smp1 = Sample(
+    ts_origin=Timestamp(123456780),
+    ts_received=Timestamp(123456781),
+    sequence=4,
+    new_frame=True,
+    data=[1.0, 2.0, 3.0, True, 42, sqrt(complex(-1))],
+)
+
+smp2 = Sample(
+    ts_origin=Timestamp(123456789),
+    ts_received=Timestamp(123456790),
+    sequence=5,
+    new_frame=False,
+    data=[1.0, 2.0, 3.0, False, 42, sqrt(complex(-1))],
+)
+
+
 def test_villas_human():
-    smp1 = Sample(
-        ts_origin=Timestamp(123456780),
-        ts_received=Timestamp(123456781),
-        sequence=4,
-        new_frame=True,
-        data=[1.0, 2.0, 3.0, True, 42, sqrt(complex(-1))],
-    )
-
-    smp2 = Sample(
-        ts_origin=Timestamp(123456789),
-        ts_received=Timestamp(123456790),
-        sequence=5,
-        new_frame=False,
-        data=[1.0, 2.0, 3.0, False, 42, sqrt(complex(-1))],
-    )
-
-    villas_human = VillasHuman(signal_list=SignalList(smp1))
+    vh = VillasHuman(signal_list=SignalList(smp1))
     smp1_str = "123456780+1.0(4)F\t1.0\t2.0\t3.0\t1\t42\t0.0+1.0i\n"
     smp2_str = "123456789+1.0(5)\t1.0\t2.0\t3.0\t0\t42\t0.0+1.0i\n"
-    assert villas_human.dump_sample(smp1) == smp1_str
-    assert villas_human.dump_sample(smp2) == smp2_str
-    assert villas_human.dumps([smp1, smp2]) == smp1_str + smp2_str
-    assert villas_human.load_sample(smp1_str) == smp1
-    assert villas_human.load_sample(smp2_str) == smp2
-    assert villas_human.loads(smp1_str + smp2_str) == [smp1, smp2]
+    assert vh.dump_sample(smp1) == smp1_str
+    assert vh.dump_sample(smp2) == smp2_str
+    assert vh.dumps([smp1, smp2]) == smp1_str + smp2_str
+    assert vh.load_sample(smp1_str) == smp1
+    assert vh.load_sample(smp2_str) == smp2
+    assert vh.loads(smp1_str + smp2_str) == [smp1, smp2]
+
+
+def test_protobuf():
+    pb = Protobuf()
+
+    raw = pb.dumpb([smp1, smp2])
+
+    assert pb.loadb(raw) == [smp1, smp2]