From bc83463831391d05094e757037a26c5d7e4bcdfa Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 28 Jul 2017 18:11:52 +0200 Subject: [PATCH] added new format subsystem --- include/villas/advio.h | 1 + include/villas/formats/csv.h | 35 ++++++ .../{sample_io_json.h => formats/json.h} | 8 +- include/villas/formats/villas.h | 36 ++++++ include/villas/io.h | 81 ++++++++++++++ include/villas/plugin.h | 3 + include/villas/sample_io.h | 89 --------------- include/villas/stats.h | 2 + lib/Makefile.inc | 1 + lib/Makefile.villas.inc | 6 +- lib/formats/Makefile.inc | 23 ++++ lib/formats/csv.c | 105 ++++++++++++++++++ lib/formats/hdf5.c | 90 +++++++++++++++ lib/{sample_io_json.c => formats/json.c} | 44 ++++++-- lib/{ => formats}/msg.c | 0 lib/{sample_io.c => formats/villas.c} | 76 ++++++------- lib/{ => formats}/webmsg.c | 0 lib/hooks/print.c | 21 +--- lib/hooks/stats_collect.c | 13 +-- lib/io.c | 102 +++++++++++++++++ lib/nodes/Makefile.inc | 7 ++ lib/nodes/file.c | 8 +- lib/stats.c | 12 ++ src/pipe.c | 7 +- 24 files changed, 591 insertions(+), 179 deletions(-) create mode 100644 include/villas/formats/csv.h rename include/villas/{sample_io_json.h => formats/json.h} (79%) create mode 100644 include/villas/formats/villas.h create mode 100644 include/villas/io.h delete mode 100644 include/villas/sample_io.h create mode 100644 lib/formats/Makefile.inc create mode 100644 lib/formats/csv.c create mode 100644 lib/formats/hdf5.c rename lib/{sample_io_json.c => formats/json.c} (71%) rename lib/{ => formats}/msg.c (100%) rename lib/{sample_io.c => formats/villas.c} (73%) rename lib/{ => formats}/webmsg.c (100%) create mode 100644 lib/io.c diff --git a/include/villas/advio.h b/include/villas/advio.h index b8b734d8f..415721aaa 100644 --- a/include/villas/advio.h +++ b/include/villas/advio.h @@ -47,6 +47,7 @@ typedef struct advio AFILE; /* The remaining functions from stdio are just replaced macros */ #define afeof(af) feof((af)->file) +#define afgets(ln, sz, af) fgets(ln, sz, (af)->file) #define aftell(af) ftell((af)->file) #define afileno(af) fileno((af)->file) #define afread(ptr, sz, nitems, af) fread(ptr, sz, nitems, (af)->file) diff --git a/include/villas/formats/csv.h b/include/villas/formats/csv.h new file mode 100644 index 000000000..36ccb502f --- /dev/null +++ b/include/villas/formats/csv.h @@ -0,0 +1,35 @@ +/** Comma-separated values. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include "advio.h" + +/* Forward declarations. */ +struct sample; + +#define CSV_SEPARATOR '\t' + +int io_format_csv_fprint(AFILE *f, struct sample *smp, int flags); + +int io_format_csv_fscan(AFILE *f, struct sample *smp, int *flags); diff --git a/include/villas/sample_io_json.h b/include/villas/formats/json.h similarity index 79% rename from include/villas/sample_io_json.h rename to include/villas/formats/json.h index 0f0e4e7ac..ea87d760f 100644 --- a/include/villas/sample_io_json.h +++ b/include/villas/formats/json.h @@ -26,10 +26,10 @@ #include "sample.h" -int sample_io_json_pack(json_t **j, struct sample *s, int flags); +int io_format_json_pack(json_t **j, struct sample *s, int flags); -int sample_io_json_unpack(json_t *j, struct sample *s, int *flags); +int io_format_json_unpack(json_t *j, struct sample *s, int *flags); -int sample_io_json_fprint(FILE *f, struct sample *s, int flags); +int io_format_json_fprint(AFILE *f, struct sample *s, int flags); -int sample_io_json_fscan(FILE *f, struct sample *s, int *flags); +int io_format_json_fscan(AFILE *f, struct sample *s, int *flags); diff --git a/include/villas/formats/villas.h b/include/villas/formats/villas.h new file mode 100644 index 000000000..5acb44888 --- /dev/null +++ b/include/villas/formats/villas.h @@ -0,0 +1,36 @@ +/** The VILLASframework sample format + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +/* VILLASnode human readable format */ + +int io_format_villas_print(char *buf, size_t len, struct sample *s, int flags); + +int io_format_villas_scan(const char *line, struct sample *s, int *fl); + +int io_format_villas_fprint(FILE *f, struct sample *s, int flags); + +int io_format_villas_fscan(FILE *f, struct sample *s, int *flags); diff --git a/include/villas/io.h b/include/villas/io.h new file mode 100644 index 000000000..4b0bed7a6 --- /dev/null +++ b/include/villas/io.h @@ -0,0 +1,81 @@ +/** Read / write sample data in different formats. + * + * @file + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include "advio.h" + +/* Forward declarations */ +struct sample; +struct io; + +/** These flags define the format which is used by io_fscan() and io_fprint(). */ +enum io_flags { + IO_FORMAT_NANOSECONDS = (1 << 0), + IO_FORMAT_OFFSET = (1 << 1), + IO_FORMAT_SEQUENCE = (1 << 2), + IO_FORMAT_VALUES = (1 << 3), + IO_FORMAT_ALL = 16-1 +}; + +struct io_format { + int (*init)(struct io *io); + int (*destroy)(struct io *io); + int (*open)(struct io *io, const char *uri, const char *mode); + int (*close)(struct io *io); + int (*eof)(struct io *io); + void (*rewind)(struct io *io); + int (*print)(struct io *io, struct sample *s, int fl); + int (*scan)(struct io *io, struct sample *s, int *fl); + + size_t size; /**< Number of bytes to allocate for io::_vd */ +}; + +struct io { + int flags; + + /** A format type can use this file handle or overwrite the + * io_format::{open,close,eof,rewind} functions and the private + * data in io::_vd. + */ + AFILE *file; + + void *_vd; + struct io_format *_vt; +}; + +int io_init(struct io *io, struct io_format *fmt, int flags); + +int io_destroy(struct io *io); + +int io_open(struct io *io, const char *uri, const char *mode); + +int io_close(struct io *io); + +int io_print(struct io *io, struct sample *smps[], size_t cnt); + +int io_scan(struct io *io, struct sample *smps[], size_t cnt); + +int io_eof(struct io *io); + +void io_rewind(struct io *io); diff --git a/include/villas/plugin.h b/include/villas/plugin.h index ff4792fa3..d4172f15c 100644 --- a/include/villas/plugin.h +++ b/include/villas/plugin.h @@ -23,6 +23,7 @@ #pragma once +#include "io.h" #include "hook.h" #include "api.h" #include "common.h" @@ -53,6 +54,7 @@ enum plugin_type { PLUGIN_TYPE_HOOK, PLUGIN_TYPE_NODE, PLUGIN_TYPE_API, + PLUGIN_TYPE_FORMAT, PLUGIN_TYPE_FPGA_IP, PLUGIN_TYPE_MODEL_CBUILDER }; @@ -71,6 +73,7 @@ struct plugin { int (*unload)(struct plugin *p); union { + struct io_format io; struct api_action api; struct node_type node; #ifdef WITH_FPGA diff --git a/include/villas/sample_io.h b/include/villas/sample_io.h deleted file mode 100644 index d900a0e05..000000000 --- a/include/villas/sample_io.h +++ /dev/null @@ -1,89 +0,0 @@ -/** Read / write sample data in different formats. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include -#include - -/* Forward declarations */ -struct sample; - -enum sample_io_format { - SAMPLE_IO_FORMAT_VILLAS, - SAMPLE_IO_FORMAT_JSON, - SAMPLE_IO_FORMAT_HDF5, - SAMPLE_IO_FORMAT_COMTRADE -}; - -/** These flags define the format which is used by sample_io_fscan() and sample_io_fprint(). */ -enum sample_flags { - SAMPLE_IO_NANOSECONDS = (1 << 0), - SAMPLE_IO_OFFSET = (1 << 1), - SAMPLE_IO_SEQUENCE = (1 << 2), - SAMPLE_IO_VALUES = (1 << 3), - SAMPLE_IO_ALL = 16-1 -}; - -/* Not implemented yet */ -#if 0 -struct sample_io_handle { - enum sample_io_format format; - int flags - - FILE *file; - - struct list fields; -}; - -int sample_io_init(struct sample_io *io, enum sample_io_format fmt, char *mode, int flags); - -int sample_io_destroy(struct sample_io *io); - -int sample_io_open(struct sample_io *io); - -int sample_io_close(struct sample_io *io); - -int sample_io_write(struct sample_io *io, struct sample *smps[], size_t cnt); - -int sample_io_read(struct sample_io *io, struct sample *smps[], size_t cnt); - -int sample_io_eof(struct sample_io *io); -int sample_io_rewind(struct sample_io *io); -#endif - -/* Lowlevel interface */ - -int sample_io_fprint(FILE *f, struct sample *s, enum sample_io_format fmt, int flags); - -int sample_io_fscan(FILE *f, struct sample *s, enum sample_io_format fmt, int *flags); - -/* VILLASnode human readable format */ - -int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags); - -int sample_io_villas_scan(const char *line, struct sample *s, int *fl); - -int sample_io_villas_fprint(FILE *f, struct sample *s, int flags); - -int sample_io_villas_fscan(FILE *f, struct sample *s, int *flags); diff --git a/include/villas/stats.h b/include/villas/stats.h index 643f7f3e2..5ea72ce23 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -61,6 +61,8 @@ struct stats { struct stats_delta *delta; }; +int stats_lookup_format(const char *str); + int stats_init(struct stats *s, int buckets, int warmup); int stats_destroy(struct stats *s); diff --git a/lib/Makefile.inc b/lib/Makefile.inc index de447f6c1..56ea6be64 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -28,6 +28,7 @@ LIB_CFLAGS = $(CFLAGS) -fPIC -include lib/hooks/Makefile.inc -include lib/nodes/Makefile.inc +-include lib/formats/Makefile.inc -include $(patsubst %, lib/Makefile.%.inc, $(SONAMES)) diff --git a/lib/Makefile.villas.inc b/lib/Makefile.villas.inc index 6f1a0ded3..bf5f00e05 100644 --- a/lib/Makefile.villas.inc +++ b/lib/Makefile.villas.inc @@ -33,8 +33,8 @@ LIB_SRCS += $(addprefix lib/kernel/, kernel.c rt.c) \ $(addprefix lib/, sample.c path.c node.c hook.c log.c log_config.c \ utils.c super_node.c hist.c timing.c pool.c list.c queue.c \ queue_signalled.c memory.c advio.c plugin.c node_type.c stats.c \ - mapping.c sample_io.c shmem.c config_helper.c crypt.c compat.c \ - log_table.c log_helper.c + mapping.c io.c shmem.c config_helper.c crypt.c compat.c \ + log_table.c log_helper.c \ ) LIB_LDFLAGS = -shared @@ -47,7 +47,7 @@ endif LIB_PKGS += openssl libcurl ifeq ($(WITH_JSON),1) - LIB_SRCS += lib/sample_io_json.c + LIB_SRCS += lib/formats/json.c LIB_PKGS += jansson LIB_CFLAGS += -DWITH_JSON endif diff --git a/lib/formats/Makefile.inc b/lib/formats/Makefile.inc new file mode 100644 index 000000000..e760b08ab --- /dev/null +++ b/lib/formats/Makefile.inc @@ -0,0 +1,23 @@ +# Makefile. +# +# @author Steffen Vogel +# @copyright 2017, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################### + +LIB_SRCS += $(addprefix lib/formats/,villas.c csv.c msg.c webmsg.c) diff --git a/lib/formats/csv.c b/lib/formats/csv.c new file mode 100644 index 000000000..b88da09d2 --- /dev/null +++ b/lib/formats/csv.c @@ -0,0 +1,105 @@ +/** Comma-separated values. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include "formats/csv.h" +#include "plugin.h" +#include "sample.h" + +int io_format_csv_fprint(AFILE *f, struct sample *s, int flags) +{ + afprintf(f, "%ld %09ld %d", s->ts.origin.tv_sec, s->ts.origin.tv_nsec, s->sequence); + + for (int i = 0; i < s->length; i++) { + switch ((s->format >> i) & 0x1) { + case SAMPLE_DATA_FORMAT_FLOAT: + afprintf(f, "%c%.6f", CSV_SEPARATOR, s->data[i].f); + break; + case SAMPLE_DATA_FORMAT_INT: + afprintf(f, "%c%d", CSV_SEPARATOR, s->data[i].i); + break; + } + } + + return 0; +} + +int io_format_csv_fscan(AFILE *f, struct sample *smp, int *flags) +{ + int ret, off; + char *ptr, line[4096]; + +skip: if (afgets(line, sizeof(line), f) == NULL) + return -1; /* An error occured */ + + /* Skip whitespaces, empty and comment lines */ + for (ptr = line; isspace(*ptr); ptr++); + if (*ptr == '\0' || *ptr == '#') + goto skip; + + ret = sscanf(line, "%ld %09ld %d %n", &smp->ts.origin.tv_sec, &smp->ts.origin.tv_nsec, &smp->sequence, &off); + if (ret != 4) + return -1; + + int i; + for (i = 0; i < smp->capacity; i++) { + switch (smp->format & (1 << i)) { + case SAMPLE_DATA_FORMAT_FLOAT: + ret = sscanf(line + off, "%f %n", &smp->data[i].f, &off); + break; + case SAMPLE_DATA_FORMAT_INT: + ret = sscanf(line + off, "%d %n", &smp->data[i].i, &off); + break; + } + + if (ret != 2) + break; + } + + smp->length = i; + + return ret; +} + +int io_format_csv_print(struct io *io, struct sample *smp, int flags) +{ + return io_format_csv_fprint(io->file, smp, flags); +} + +int io_format_csv_scan(struct io *io, struct sample *smp, int *flags) +{ + return io_format_csv_fscan(io->file, smp, flags); +} + +static struct plugin p = { + .name = "csv", + .description = "Tabulator-separated values", + .type = PLUGIN_TYPE_FORMAT, + .io = { + .scan = io_format_csv_scan, + .print = io_format_csv_print, + .size = 0 + } +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/formats/hdf5.c b/lib/formats/hdf5.c new file mode 100644 index 000000000..48c866a04 --- /dev/null +++ b/lib/formats/hdf5.c @@ -0,0 +1,90 @@ +#include "hdf5_hl.h" + +#include + +/*------------------------------------------------------------------------- + * Packet Table Fixed-Length Example + * + * Example program that creates a packet table and performs + * writes and reads. + * + *------------------------------------------------------------------------- + */ + +int main(void) +{ + hid_t fid; /* File identifier */ + hid_t ptable; /* Packet table identifier */ + + herr_t err; /* Function return status */ + hsize_t count; /* Number of records in the table */ + + int x; /* Loop variable */ + + /* Buffers to hold data */ + int writeBuffer[5]; + int readBuffer[5]; + + /* Initialize buffers */ + for(x=0; x<5; x++) + { + writeBuffer[x]=x; + readBuffer[x] = -1; + } + + /* Create a file using default properties */ + fid=H5Fcreate("packet_table_FLexample.h5",H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT); + + /* Create a fixed-length packet table within the file */ + /* This table's "packets" will be simple integers. */ + ptable = H5PTcreate_fl(fid, "Packet Test Dataset", H5T_NATIVE_INT, 1, 1); + if(ptable == H5I_INVALID_HID) + goto out; + + /* Write one packet to the packet table */ + err = H5PTappend(ptable, 1, &(writeBuffer[0]) ); + if(err < 0) + goto out; + + /* Write several packets to the packet table */ + err = H5PTappend(ptable, 4, &(writeBuffer[1]) ); + if(err < 0) + goto out; + + /* Get the number of packets in the packet table. This should be five. */ + err = H5PTget_num_packets(ptable, &count); + if(err < 0) + goto out; + + printf("Number of packets in packet table after five appends: %llu\n", count); + + /* Initialize packet table's "current record" */ + err = H5PTcreate_index(ptable); + if(err < 0) + goto out; + + /* Iterate through packets, read each one back */ + for(x=0; x<5; x++) + { + err = H5PTget_next(ptable, 1, &(readBuffer[x]) ); + if(err < 0) + goto out; + + printf("Packet %d's value is %d\n", x, readBuffer[x]); + } + + /* Close the packet table */ + err = H5PTclose(ptable); + if(err < 0) + goto out; + + /* Close the file */ + H5Fclose(fid); + + return 0; + + out: /* An error has occurred. Clean up and exit. */ + H5PTclose(ptable); + H5Fclose(fid); + return -1; +} diff --git a/lib/sample_io_json.c b/lib/formats/json.c similarity index 71% rename from lib/sample_io_json.c rename to lib/formats/json.c index 033edd82c..650b9a8fc 100644 --- a/lib/sample_io_json.c +++ b/lib/formats/json.c @@ -1,4 +1,4 @@ -/** JSON serializtion sample data. +/** JSON serializtion of sample data. * * @author Steffen Vogel * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC @@ -20,9 +20,10 @@ * along with this program. If not, see . *********************************************************************************/ -#include "sample_io_json.h" +#include "plugin.h" +#include "formats/json.h" -int sample_io_json_pack(json_t **j, struct sample *s, int flags) +int io_format_json_pack(json_t **j, struct sample *s, int flags) { json_error_t err; json_t *json_data = json_array(); @@ -49,7 +50,7 @@ int sample_io_json_pack(json_t **j, struct sample *s, int flags) return 0; } -int sample_io_json_unpack(json_t *j, struct sample *s, int *flags) +int io_format_json_unpack(json_t *j, struct sample *s, int *flags) { int ret, i; json_t *json_data, *json_value; @@ -89,35 +90,58 @@ int sample_io_json_unpack(json_t *j, struct sample *s, int *flags) return 0; } -int sample_io_json_fprint(FILE *f, struct sample *s, int flags) +int io_format_json_fprint(AFILE *f, struct sample *s, int flags) { int ret; json_t *json; - ret = sample_io_json_pack(&json, s, flags); + ret = io_format_json_pack(&json, s, flags); if (ret) return ret; - ret = json_dumpf(json, f, 0); + ret = json_dumpf(json, f->file, 0); json_decref(json); return ret; } -int sample_io_json_fscan(FILE *f, struct sample *s, int *flags) +int io_format_json_fscan(AFILE *f, struct sample *s, int *flags) { int ret; json_t *json; json_error_t err; - json = json_loadf(f, JSON_DISABLE_EOF_CHECK, &err); + json = json_loadf(f->file, JSON_DISABLE_EOF_CHECK, &err); if (!json) return -1; - ret = sample_io_json_unpack(json, s, flags); + ret = io_format_json_unpack(json, s, flags); json_decref(json); return ret; } + +int io_format_json_print(struct io *io, struct sample *smp, int flags) +{ + return io_format_json_fprint(io->file, smp, flags); +} + +int io_format_json_scan(struct io *io, struct sample *smp, int *flags) +{ + return io_format_json_fscan(io->file, smp, flags); +} + +static struct plugin p = { + .name = "json", + .description = "Javascript Object Notation", + .type = PLUGIN_TYPE_FORMAT, + .io = { + .print = io_format_json_print, + .scan = io_format_json_scan, + .size = 0 + }, +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/msg.c b/lib/formats/msg.c similarity index 100% rename from lib/msg.c rename to lib/formats/msg.c diff --git a/lib/sample_io.c b/lib/formats/villas.c similarity index 73% rename from lib/sample_io.c rename to lib/formats/villas.c index ea2d14b98..1c26f2609 100644 --- a/lib/sample_io.c +++ b/lib/formats/villas.c @@ -21,53 +21,28 @@ *********************************************************************************/ #include +#include +#include "io.h" +#include "plugin.h" +#include "utils.h" #include "timing.h" #include "sample.h" -#include "sample_io.h" -#ifdef WITH_JSON - #include "sample_io_json.h" -#endif - -int sample_io_fprint(FILE *f, struct sample *s, enum sample_io_format fmt, int flags) -{ - switch (fmt) { - case SAMPLE_IO_FORMAT_VILLAS: return sample_io_villas_fprint(f, s, flags); -#ifdef WITH_JSON - case SAMPLE_IO_FORMAT_JSON: return sample_io_json_fprint(f, s, flags); -#endif - default: - return -1; - } -} - -int sample_io_fscan(FILE *f, struct sample *s, enum sample_io_format fmt, int *flags) -{ - switch (fmt) { - case SAMPLE_IO_FORMAT_VILLAS: return sample_io_villas_fscan(f, s, flags); -#ifdef WITH_JSON - case SAMPLE_IO_FORMAT_JSON: return sample_io_json_fscan(f, s, flags); -#endif - default: - return -1; - } -} - -int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags) +int io_format_villas_print(char *buf, size_t len, struct sample *s, int flags) { size_t off = snprintf(buf, len, "%llu", (unsigned long long) s->ts.origin.tv_sec); - if (flags & SAMPLE_IO_NANOSECONDS) + if (flags & IO_FORMAT_NANOSECONDS) off += snprintf(buf + off, len - off, ".%09llu", (unsigned long long) s->ts.origin.tv_nsec); - if (flags & SAMPLE_IO_OFFSET) + if (flags & IO_FORMAT_OFFSET) off += snprintf(buf + off, len - off, "%+e", time_delta(&s->ts.origin, &s->ts.received)); - if (flags & SAMPLE_IO_SEQUENCE) + if (flags & IO_FORMAT_SEQUENCE) off += snprintf(buf + off, len - off, "(%u)", s->sequence); - if (flags & SAMPLE_IO_VALUES) { + if (flags & IO_FORMAT_VALUES) { for (int i = 0; i < s->length; i++) { switch ((s->format >> i) & 0x1) { case SAMPLE_DATA_FORMAT_FLOAT: @@ -85,7 +60,7 @@ int sample_io_villas_print(char *buf, size_t len, struct sample *s, int flags) return 0; /* trailing '\0' */ } -int sample_io_villas_scan(const char *line, struct sample *s, int *fl) +int io_format_villas_scan(const char *line, struct sample *s, int *fl) { char *end; const char *ptr = line; @@ -110,7 +85,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) s->ts.origin.tv_nsec = (uint32_t) strtoul(ptr, &end, 10); if (ptr != end) - flags |= SAMPLE_IO_NANOSECONDS; + flags |= IO_FORMAT_NANOSECONDS; else return -3; } @@ -123,7 +98,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) offset = strtof(ptr, &end); /* offset is ignored for now */ if (ptr != end) - flags |= SAMPLE_IO_OFFSET; + flags |= IO_FORMAT_OFFSET; else return -4; } @@ -134,7 +109,7 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) s->sequence = strtoul(ptr, &end, 10); if (ptr != end) - flags |= SAMPLE_IO_SEQUENCE; + flags |= IO_FORMAT_SEQUENCE; else return -5; @@ -160,11 +135,11 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) } if (s->length > 0) - flags |= SAMPLE_IO_VALUES; + flags |= IO_FORMAT_VALUES; if (fl) *fl = flags; - if (flags & SAMPLE_IO_OFFSET) { + if (flags & IO_FORMAT_OFFSET) { struct timespec off = time_from_double(offset); s->ts.received = time_add(&s->ts.origin, &off); } @@ -174,12 +149,12 @@ int sample_io_villas_scan(const char *line, struct sample *s, int *fl) return 0; } -int sample_io_villas_fprint(FILE *f, struct sample *s, int flags) +int io_format_villas_fprint(FILE *f, struct sample *s, int flags) { char line[4096]; int ret; - ret = sample_io_villas_print(line, sizeof(line), s, flags); + ret = io_format_villas_print(line, sizeof(line), s, flags); if (ret) return ret; @@ -188,7 +163,7 @@ int sample_io_villas_fprint(FILE *f, struct sample *s, int flags) return 0; } -int sample_io_villas_fscan(FILE *f, struct sample *s, int *fl) +int io_format_villas_fscan(FILE *f, struct sample *s, int *fl) { char *ptr, line[4096]; @@ -200,5 +175,18 @@ skip: if (fgets(line, sizeof(line), f) == NULL) if (*ptr == '\0' || *ptr == '#') goto skip; - return sample_io_villas_scan(line, s, fl); + return io_format_villas_scan(line, s, fl); } + +struct plugin p = { + .name = "villas", + .description = "Human readable VILLAS format", + .type = PLUGIN_TYPE_FORMAT, + .io = { + .fprint = io_format_villas_fprint, + .fscan = io_format_villas_fscan, + .size = 0 + } +}; + +REGISTER_PLUGIN(&p); diff --git a/lib/webmsg.c b/lib/formats/webmsg.c similarity index 100% rename from lib/webmsg.c rename to lib/formats/webmsg.c diff --git a/lib/hooks/print.c b/lib/hooks/print.c index a2e403d85..895437b0e 100644 --- a/lib/hooks/print.c +++ b/lib/hooks/print.c @@ -27,10 +27,10 @@ #include "hook.h" #include "plugin.h" #include "sample.h" -#include "sample_io.h" +#include "io.h" struct print { - FILE *output; + struct io output; const char *uri; }; @@ -38,7 +38,6 @@ static int print_init(struct hook *h) { struct print *p = h->_vd; - p->output = stdout; p->uri = NULL; return 0; @@ -48,23 +47,14 @@ static int print_start(struct hook *h) { struct print *p = h->_vd; - if (p->uri) { - p->output = fopen(p->uri, "w+"); - if (!p->output) - error("Failed to open file %s for writing", p->uri); - } - - return 0; + return io_open(&p->output, p->uri, "w+"); } static int print_stop(struct hook *h) { struct print *p = h->_vd; - if (p->uri) - fclose(p->output); - - return 0; + return io_close(&p->output); } static int print_parse(struct hook *h, config_setting_t *cfg) @@ -80,8 +70,7 @@ static int print_read(struct hook *h, struct sample *smps[], size_t *cnt) { struct print *p = h->_vd; - for (int i = 0; i < *cnt; i++) - sample_io_villas_fprint(p->output, smps[i], SAMPLE_IO_ALL); + io_print(&p->output, smps, *cnt); return 0; } diff --git a/lib/hooks/stats_collect.c b/lib/hooks/stats_collect.c index da55787e8..472f2481a 100644 --- a/lib/hooks/stats_collect.c +++ b/lib/hooks/stats_collect.c @@ -119,14 +119,13 @@ static int stats_collect_parse(struct hook *h, config_setting_t *cfg) const char *format; if (config_setting_lookup_string(cfg, "format", &format)) { - if (!strcmp(format, "human")) - p->format = STATS_FORMAT_HUMAN; - else if (!strcmp(format, "json")) - p->format = STATS_FORMAT_JSON; - else if (!strcmp(format, "matlab")) - p->format = STATS_FORMAT_MATLAB; - else + int fmt; + + fmt = stats_lookup_format(format); + if (fmt < 0) cerror(cfg, "Invalid statistic output format: %s", format); + + p->format = fmt; } config_setting_lookup_bool(cfg, "verbose", &p->verbose); diff --git a/lib/io.c b/lib/io.c new file mode 100644 index 000000000..91e79451d --- /dev/null +++ b/lib/io.c @@ -0,0 +1,102 @@ +/** Reading and writing simulation samples in various formats. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include "io.h" +#include "utils.h" + +int io_init(struct io *io, struct io_format *fmt, int flags) +{ + io->_vt = fmt; + io->_vd = alloc(fmt->size); + + io->flags = flags; + + return io->_vt->init ? io->_vt->init(io) : 0; +} + +int io_destroy(struct io *io) +{ + int ret; + + ret = io->_vt->destroy ? io->_vt->destroy(io) : 0; + if (ret) + return ret; + + free(io->_vd); + + return 0; +} + +int io_open(struct io *io, const char *uri, const char *mode) +{ + if (io->_vt->open) + return io->_vt->open(io, uri, mode); + else { + io->file = afopen(uri, mode); + if (!io->file) + return -1; + + return 0; + } +} + +int io_close(struct io *io) +{ + return io->_vt->close ? io->_vt->close(io) : afclose(io->file); +} + +int io_print(struct io *io, struct sample *smps[], size_t cnt) +{ + assert(io->_vt->print); + + for (int i = 0; i < cnt; i++) + io->_vt->print(io, smps[i], io->flags); + + return cnt; +} + +int io_scan(struct io *io, struct sample *smps[], size_t cnt) +{ + int ret; + + assert(io->_vt->scan); + + for (int i = 0; i < cnt && !io_eof(io); i++) { + ret = io->_vt->scan(io, smps[i], NULL); + if (ret < 0) + return i; + } + + return cnt; +} + +int io_eof(struct io *io) +{ + return io->_vt->eof ? io->_vt->eof(io) : afeof(io->file); +} + +void io_rewind(struct io *io) +{ + io->_vt->rewind ? io->_vt->rewind(io) : arewind(io->file); +} diff --git a/lib/nodes/Makefile.inc b/lib/nodes/Makefile.inc index 54c0ee57f..d612bc48e 100644 --- a/lib/nodes/Makefile.inc +++ b/lib/nodes/Makefile.inc @@ -27,6 +27,7 @@ ifeq ($(PLATFORM),Linux) WITH_SIGNAL ?= 1 WITH_NGSI ?= 1 WITH_FPGA ?= 1 + WITH_TEST_RTT ?= 1 endif WITH_WEBSOCKET ?= 1 @@ -53,6 +54,12 @@ ifeq ($(WITH_SIGNAL),1) LIB_CFLAGS += -DWITH_SIGNAL endif +# Enable RTT test node-tyoe +ifeq ($(WITH_TEST_RTT),1) + LIB_SRCS += lib/nodes/test_rtt.c + LIB_CFLAGS += -DWITH_TEST_RTT +endif + # Enable VILLASfpga support when libxil is available ifeq ($(WITH_FPGA),1) ifeq ($(shell $(PKGCONFIG) libxil; echo $$?),0) diff --git a/lib/nodes/file.c b/lib/nodes/file.c index db4a977b6..d9677666d 100644 --- a/lib/nodes/file.c +++ b/lib/nodes/file.c @@ -29,7 +29,7 @@ #include "timing.h" #include "queue.h" #include "plugin.h" -#include "sample_io.h" +#include "formats/villas.h" int file_reverse(struct node *n) { @@ -264,7 +264,7 @@ int file_start(struct node *n) struct sample s; s.capacity = 0; - ret = sample_io_villas_fscan(f->read.handle->file, &s, NULL); + ret = io_format_villas_fscan(f->read.handle->file, &s, NULL); if (ret < 0) error("Failed to read first timestamp of node %s", node_name(n)); @@ -314,7 +314,7 @@ int file_read(struct node *n, struct sample *smps[], unsigned cnt) assert(f->read.handle); assert(cnt == 1); -retry: values = sample_io_villas_fscan(f->read.handle->file, s, &flags); /* Get message and timestamp */ +retry: values = io_format_villas_fscan(f->read.handle->file, s, &flags); /* Get message and timestamp */ if (values < 0) { if (afeof(f->read.handle)) { switch (f->read_eof) { @@ -373,7 +373,7 @@ int file_write(struct node *n, struct sample *smps[], unsigned cnt) assert(f->write.handle); assert(cnt == 1); - sample_io_villas_fprint(f->write.handle->file, s, SAMPLE_IO_ALL & ~SAMPLE_IO_OFFSET); + io_format_villas_fprint(f->write.handle->file, s, IO_FORMAT_ALL & ~IO_FORMAT_OFFSET); if (f->flush) afflush(f->write.handle); diff --git a/lib/stats.c b/lib/stats.c index bccfcac09..ea156d26a 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -44,6 +44,18 @@ static struct stats_desc { { "owd", "seconds", "Histogram for one-way-delay (OWD) of received messages", 25 } }; +int stats_lookup_format(const char *str) +{ + if (!strcmp(str, "human")) + return STATS_FORMAT_HUMAN; + else if (!strcmp(str, "json")) + return STATS_FORMAT_JSON; + else if (!strcmp(str, "matlab")) + return STATS_FORMAT_MATLAB; + else + return -1; +} + int stats_init(struct stats *s, int buckets, int warmup) { for (int i = 0; i < STATS_COUNT; i++) diff --git a/src/pipe.c b/src/pipe.c index a53d21c6d..c00726032 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include @@ -84,6 +84,7 @@ static void usage() printf(" CONFIG path to a configuration file\n"); printf(" NODE the name of the node to which samples are sent and received from\n"); printf(" OPTIONS are:\n"); + printf(" -f FMT set the format\n") printf(" -d LVL set debug log level to LVL\n"); printf(" -x swap read / write endpoints\n"); printf(" -s only read data from stdin and send it to node\n"); @@ -207,8 +208,10 @@ int main(int argc, char *argv[]) }; char c, *endptr; - while ((c = getopt(argc, argv, "hxrsd:l:L:t:")) != -1) { + while ((c = getopt(argc, argv, "hxrsd:l:L:t:f:")) != -1) { switch (c) { + case 'f'; + case 'x': reverse = true; break;