diff --git a/include/villas/io/msg_format.h b/include/villas/io/msg_format.h index 3b544e09f..92746b548 100644 --- a/include/villas/io/msg_format.h +++ b/include/villas/io/msg_format.h @@ -29,7 +29,7 @@ #define MSG_VERSION 2 /** @todo Implement more message types */ -#define MSG_TYPE_DATA 0 /**< Message contains float values */ +#define MSG_TYPE_DATA 0 /**< Message contains float / integer values */ #define MSG_TYPE_START 1 /**< Message marks the beginning of a new simulation case */ #define MSG_TYPE_STOP 2 /**< Message marks the end of a simulation case */ @@ -75,7 +75,7 @@ struct msg #error Invalid byte-order #endif - uint8_t id; /**< An id which identifies the source of this sample */ + uint8_t id; /**< An id which identifies the source of this sample. */ uint16_t length; /**< The number of values in msg::data[]. */ uint32_t sequence; /**< The sequence number is incremented by one for consecutive messages. */ diff --git a/include/villas/io/protobuf.h b/include/villas/io/protobuf.h new file mode 100644 index 000000000..abba82533 --- /dev/null +++ b/include/villas/io/protobuf.h @@ -0,0 +1,35 @@ +/** Protobuf IO format + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +/* Forward declarations */ +struct sample; + +/** Copy / read struct msg's from buffer \p buf to / fram samples \p smps. */ +int protobuf_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags); + +/** Read struct sample's from buffer \p buf into samples \p smps. */ +int protobuf_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int flags); diff --git a/lib/io/.gitignore b/lib/io/.gitignore new file mode 100644 index 000000000..69330092e --- /dev/null +++ b/lib/io/.gitignore @@ -0,0 +1,3 @@ +# Generated code by Protobuf-c +*.pb-c.h +*.pb-c.c diff --git a/lib/io/Makefile.inc b/lib/io/Makefile.inc index 827679ed0..b08b4b684 100644 --- a/lib/io/Makefile.inc +++ b/lib/io/Makefile.inc @@ -21,3 +21,20 @@ ################################################################################### LIB_SRCS += $(addprefix lib/io/,json.c villas_binary.c villas_human.c csv.c raw.c msg.c) + +WITH_PROTOBUF ?= 1 + +# Enable Google Protobuf IO format +ifeq ($(WITH_PROTOBUF),1) +ifeq ($(shell $(PKGCONFIG) libprotobuf-c; echo $$?),0) + LIB_SRCS += lib/io/protobuf.c lib/io/villas.pb-c.c + LIB_PKGS += libprotobuf-c +endif +endif + +%.pb-c.h: %.proto + protoc-c --proto_path=$(SRCDIR) --c_out=$(SRCDIR) $(realpath $^) + +lib/io/villas.pb-c.c \ +lib/io/protobuf.c: | lib/io/villas.pb-c.h + diff --git a/lib/io/protobuf.c b/lib/io/protobuf.c new file mode 100644 index 000000000..3023ee230 --- /dev/null +++ b/lib/io/protobuf.c @@ -0,0 +1,152 @@ +/** Protobuf IO format + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +// Generated message descriptors by protoc +#include "villas.pb-c.h" + +#include "sample.h" +#include "plugin.h" +#include "io/protobuf.h" + +int protobuf_sprint(char *buf, size_t len, size_t *wbytes, struct sample *smps[], unsigned cnt, int flags) +{ + unsigned psz; + + Villas__Node__Message *pb_msg = alloc(sizeof(Villas__Node__Message)); + villas__node__message__init(pb_msg); + + pb_msg->n_samples = cnt; + pb_msg->samples = alloc(pb_msg->n_samples * sizeof(Villas__Node__Sample *)); + + for (unsigned i = 0; i < pb_msg->n_samples; i++) { + Villas__Node__Sample *pb_smp = pb_msg->samples[i] = alloc(sizeof(Villas__Node__Sample)); + villas__node__sample__init(pb_smp); + + struct sample *smp = smps[i]; + + pb_smp->type = VILLAS__NODE__SAMPLE__TYPE__DATA; + pb_smp->sequence = smp->sequence; + pb_smp->id = smp->id; + + pb_smp->timestamp = alloc(sizeof(Villas__Node__Timestamp)); + villas__node__timestamp__init(pb_smp->timestamp); + + pb_smp->timestamp->sec = smp->ts.origin.tv_sec; + pb_smp->timestamp->nsec = smp->ts.origin.tv_nsec; + + pb_smp->n_values = smp->length; + pb_smp->values = alloc(pb_smp->n_values * sizeof(Villas__Node__Value *)); + + for (unsigned j = 0; j < pb_smp->n_values; j++) { + Villas__Node__Value *pb_val = pb_smp->values[j] = alloc(sizeof(Villas__Node__Value)); + villas__node__value__init(pb_val); + + enum sample_data_format fmt = sample_get_data_format(smp, j); + + switch (fmt) { + case SAMPLE_DATA_FORMAT_FLOAT: pb_val->value_case = VILLAS__NODE__VALUE__VALUE_F; pb_val->f = smp->data[j].f; break; + case SAMPLE_DATA_FORMAT_INT: pb_val->value_case = VILLAS__NODE__VALUE__VALUE_I; pb_val->i = smp->data[j].i; break; + default: pb_val->value_case = VILLAS__NODE__VALUE__VALUE__NOT_SET; break; + } + } + } + + psz = villas__node__message__get_packed_size(pb_msg); + + if (psz > len) + goto out; + + villas__node__message__pack(pb_msg, (uint8_t *) buf); + villas__node__message__free_unpacked(pb_msg, NULL); + + *wbytes = psz; + + return cnt; + +out: + villas__node__message__free_unpacked(pb_msg, NULL); + + return -1; +} + +int protobuf_sscan(char *buf, size_t len, size_t *rbytes, struct sample *smps[], unsigned cnt, int flags) +{ + unsigned i, j; + Villas__Node__Message *pb_msg; + + pb_msg = villas__node__message__unpack(NULL, len, (uint8_t *) buf); + + for (i = 0; i < MIN(pb_msg->n_samples, cnt); i++) { + struct sample *smp = smps[i]; + Villas__Node__Sample *pb_smp = pb_msg->samples[i]; + + if (pb_smp->type != VILLAS__NODE__SAMPLE__TYPE__DATA) { + warn("Parsed non supported message type"); + break; + } + + smp->sequence = pb_smp->sequence; + + if (pb_smp->has_id) + smp->id = pb_smp->id; + + if (pb_smp->timestamp) { + smp->ts.origin.tv_sec = pb_smp->timestamp->sec; + smp->ts.origin.tv_nsec = pb_smp->timestamp->nsec; + } + + for (j = 0; j < MIN(pb_smp->n_values, smp->capacity); j++) { + Villas__Node__Value *pb_val = pb_smp->values[j]; + + enum sample_data_format fmt = pb_val->value_case == VILLAS__NODE__VALUE__VALUE_F + ? SAMPLE_DATA_FORMAT_FLOAT + : SAMPLE_DATA_FORMAT_INT; + + switch (fmt) { + case SAMPLE_DATA_FORMAT_FLOAT: smp->data[j].f = pb_val->f; break; + case SAMPLE_DATA_FORMAT_INT: smp->data[j].i = pb_val->i; break; + default: { } + } + + sample_set_data_format(smp, j, fmt); + } + + smp->length = j; + } + + *rbytes = villas__node__message__get_packed_size(pb_msg); + + villas__node__message__free_unpacked(pb_msg, NULL); + + return i; +} + +static struct plugin p = { + .name = "protobuf", + .description = "Google Protobuf", + .type = PLUGIN_TYPE_IO, + .io = { + .sprint = protobuf_sprint, + .sscan = protobuf_sscan + } +}; +REGISTER_PLUGIN(&p); diff --git a/lib/io/villas.proto b/lib/io/villas.proto new file mode 100644 index 000000000..e32539de7 --- /dev/null +++ b/lib/io/villas.proto @@ -0,0 +1,54 @@ +/// Protobuf schema based on msg_format.h +/// +/// @file +/// @author Steffen Vogel +/// @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +/// @license GNU General Public License (version 3) +/// +/// VILLASnode +/// +/// This program is free software: you can redistribute it and/or modify +/// it under the terms of the GNU General Public License as published by +/// the Free Software Foundation, either version 3 of the License, or +/// any later version. +/// +/// This program is distributed in the hope that it will be useful, +/// but WITHOUT ANY WARRANTY; without even the implied warranty of +/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +/// GNU General Public License for more details. +/// +/// You should have received a copy of the GNU General Public License +/// along with this program. If not, see . +//////////////////////////////////////////////////////////////////////////////////// + +package villas.node; + +message Message { + repeated Sample samples = 1; +} + +message Sample { + enum Type { + DATA = 1; // Message contains float / integer data values + START = 2; // Message marks the beginning of a new simulation case + STOP = 3; // Message marks the end of a simulation case + }; + + required Type type = 1 [default = DATA]; + required uint32 sequence = 2; // The sequence number is incremented by one for consecutive messages. + optional uint32 id = 3; // An id which identifies the source of this sample. + optional Timestamp timestamp = 4; + repeated Value values = 5; +} + +message Timestamp { + required uint32 sec = 1; // Seconds since 1970-01-01 00:00:00 + required uint32 nsec = 2; // Nanoseconds of the current second. +} + +message Value { + oneof value { + float f = 1; // Floating point values. + int32 i = 2; // Integer values. + } +} diff --git a/tests/integration/pipe-loopback-socket.sh b/tests/integration/pipe-loopback-socket.sh index abd1901b9..9396b118e 100755 --- a/tests/integration/pipe-loopback-socket.sh +++ b/tests/integration/pipe-loopback-socket.sh @@ -36,7 +36,7 @@ NUM_SAMPLES=${NUM_SAMPLES:-100} # Generate test data villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} -for FORMAT in villas-human villas-binary villas-web csv json gtnet-fake raw-flt32; do +for FORMAT in villas-human villas-binary villas-web csv json gtnet-fake raw-flt32 protobuf; do for LAYER in udp ip eth unix; do for VERIFY_SOURCE in true false; do