From f6c204c4413aea5a5cce9e2086306b3695e0f32b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 29 Mar 2017 20:14:01 +0200 Subject: [PATCH] removed split and splitted settings from file node-type --- doc/nodes/File.md | 53 +++++++++++++--------------------- etc/example.conf | 4 --- include/villas/nodes/file.h | 8 +++--- lib/nodes/file.c | 57 ++++++++++++------------------------- 4 files changed, 41 insertions(+), 81 deletions(-) diff --git a/doc/nodes/File.md b/doc/nodes/File.md index 42839398d..659ed2bf2 100644 --- a/doc/nodes/File.md +++ b/doc/nodes/File.md @@ -5,7 +5,7 @@ The `file` node-type can be used to log or replay samples to / from disk. ## Configuration Every `file` node can be configured to only read or write or to do both at the same time. -The node configuration is splitted in to groups: `in` and `out`. +The node configuration is divided into two sub-groups: `in` and `out`. #### `uri` *(string: libcurl URI)* @@ -19,7 +19,9 @@ See [strftime(3)](http://man7.org/linux/man-pages/man3/strftime.3.html) for a li uri = "logs/measurements_%Y-%m-%d_%H-%M-%S.log" -will create a file called: *path_of_working_directory*/logs/measurements_2015-08-09_22-20-50.log +will create a file called: + + ./logs/measurements_2015-08-09_22-20-50.log See below for a description of the file format. @@ -48,12 +50,13 @@ Those variables will also been displayed during the startup phase of the server The supported values for `epoch_mode`: - | `epoch_mode` | `offset` | `start = first + offset` | - | -----------: | :-------------------: | :----------------------: | - | `direct` | `now - first + epoch` | `now + epoch` | - | `wait` | `now + epoch` | `now + first` | - | `relative` | `epoch` | `first + epoch` | - | `absolute` | `epoch - first` | `epoch` | +| `epoch_mode` | `offset` | `start = first + offset` | +| -----------: | :-------------------: | :----------------------: | +| `direct` | `now - first + epoch` | `now + epoch` | +| `wait` | `now + epoch` | `now + first` | +| `relative` | `epoch` | `first + epoch` | +| `absolute` | `epoch - first` | `epoch` | +| `original` | `0` | immeadiatly | #### `rate` *(float)* @@ -61,20 +64,6 @@ By default `send_rate` has the value `0` which means that the time between conse If this setting has a non-zero value, the default behaviour is overwritten with a fixed rate. -#### `split` *(integer)* - -Only valid for the `out` group. - -Splits the output file every `split` mega-byte. This setting will append the chunk number to the `uri` setting. - -Example: `data/my_measurements.log_001` - -#### `splitted` *(boolean)* - -Only valid for the `in` group. - -Expects the input data in splitted format. - ### Example nodes = { @@ -94,14 +83,10 @@ Expects the input data in splitted format. rate = 2.0 # A constant rate at which the lines of the input files should be read # A missing or zero value will use the timestamp in the first column # of the file to determine the pause between consecutive lines. - - splitted = false }, out = { - URI = "logs/output_%F_%T.log" # The output URI accepts all format tokens of (see strftime(3)) + uri = "logs/output_%F_%T.log" # The output URI accepts all format tokens of (see strftime(3)) mode = "a+" # You might want to use "a+" to append to a file - - split = 100, # Split output file every 100 MB } } } @@ -132,10 +117,10 @@ The columns are defined as follows: This example shows a dump with three values per sample: - 1438959964.162102394 6 3.489760 -1.882725 0.860070 - 1438959964.261677582 7 2.375948 -2.204084 0.907518 - 1438959964.361622787 8 3.620115 -1.359236 -0.622333 - 1438959964.461907066 9 5.844254 -0.966527 -0.628751 - 1438959964.561499526 10 6.317059 -1.716363 0.351925 - 1438959964.661578339 11 6.471288 -0.159862 0.123948 - 1438959964.761956859 12 7.365932 -1.488268 -0.780568 + 1438959964.162102394(6) 3.489760 -1.882725 0.860070 + 1438959964.261677582(7) 2.375948 -2.204084 0.907518 + 1438959964.361622787(8) 3.620115 -1.359236 -0.622333 + 1438959964.461907066(9) 5.844254 -0.966527 -0.628751 + 1438959964.561499526(10) 6.317059 -1.716363 0.351925 + 1438959964.661578339(11) 6.471288 -0.159862 0.123948 + 1438959964.761956859(12) 7.365932 -1.488268 -0.780568 diff --git a/etc/example.conf b/etc/example.conf index acab53b2d..e5dc28f83 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -94,14 +94,10 @@ nodes = { rate = 2.0 # A constant rate at which the lines of the input files should be read # A missing or zero value will use the timestamp in the first column # of the file to determine the pause between consecutive lines. - - splitted = false }, out = { uri = "logs/output_%F_%T.log" # The output path accepts all format tokens of (see strftime(3)) mode = "a+" # You might want to use "a+" to append to a file - - split = 100, # Split output file every 100 MB } }, gtfpga_node = { diff --git a/include/villas/nodes/file.h b/include/villas/nodes/file.h index d8355421b..54f00d91d 100644 --- a/include/villas/nodes/file.h +++ b/include/villas/nodes/file.h @@ -33,16 +33,16 @@ struct file { const char *fmt; /**< Format string for file name. */ char *uri; /**< Real file name */ - - int chunk; /**< Current chunk number. */ - int split; /**< Split file every file::split mega bytes. */ } read, write; + + int rewind; /**< Should we rewind the file when we reach EOF? */ enum read_epoch_mode { EPOCH_DIRECT, EPOCH_WAIT, EPOCH_RELATIVE, - EPOCH_ABSOLUTE + EPOCH_ABSOLUTE, + EPOCH_ORIGINAL } read_epoch_mode; /**< Specifies how file::offset is calculated. */ struct timespec read_first; /**< The first timestamp in the file file::{read,write}::uri */ diff --git a/lib/nodes/file.c b/lib/nodes/file.c index 0b540f486..dd0d28993 100644 --- a/lib/nodes/file.c +++ b/lib/nodes/file.c @@ -41,19 +41,10 @@ static char * file_format_name(const char *format, struct timespec *ts) static AFILE * file_reopen(struct file_direction *dir) { - char buf[FILE_MAX_PATHLEN]; - const char *uri = buf; - - /* Append chunk number to filename */ - if (dir->chunk >= 0) - snprintf(buf, FILE_MAX_PATHLEN, "%s_%03u", dir->uri, dir->chunk); - else - uri = dir->uri; - if (dir->handle) afclose(dir->handle); - return afopen(uri, dir->mode); + return afopen(dir->uri, dir->mode); } static int file_parse_direction(config_setting_t *cfg, struct file *f, int d) @@ -79,12 +70,6 @@ int file_parse(struct node *n, config_setting_t *cfg) if (cfg_out) { if (file_parse_direction(cfg_out, f, FILE_WRITE)) cerror(cfg_out, "Failed to parse output file for node %s", node_name(n)); - - /* More write specific settings */ - if (config_setting_lookup_int(cfg_out, "split", &f->write.split)) - f->write.split <<= 20; /* in MiB */ - else - f->write.split = -1; /* Save all samples in a single file */ } cfg_in = config_setting_get_member(cfg, "in"); @@ -93,8 +78,8 @@ int file_parse(struct node *n, config_setting_t *cfg) cerror(cfg_in, "Failed to parse input file for node %s", node_name(n)); /* More read specific settings */ - if (!config_setting_lookup_bool(cfg_in, "splitted", &f->read.split)) - f->read.split = 0; /* Input files are suffixed with split indizes (.000, .001) */ + if (!config_setting_lookup_bool(cfg_in, "rewind", &f->rewind)) + f->rewind = 0; if (!config_setting_lookup_float(cfg_in, "rate", &f->read_rate)) f->read_rate = 0; /* Disable fixed rate sending. Using timestamps of file instead */ @@ -116,6 +101,8 @@ int file_parse(struct node *n, config_setting_t *cfg) f->read_epoch_mode = EPOCH_RELATIVE; else if (!strcmp(epoch_mode, "absolute")) f->read_epoch_mode = EPOCH_ABSOLUTE; + else if (!strcmp(epoch_mode, "original")) + f->read_epoch_mode = EPOCH_ORIGINAL; else cerror(cfg_in, "Invalid value '%s' for setting 'epoch_mode'", epoch_mode); } @@ -137,6 +124,7 @@ char * file_print(struct node *n) case EPOCH_WAIT: epoch_str = "wait"; break; case EPOCH_RELATIVE: epoch_str = "relative"; break; case EPOCH_ABSOLUTE: epoch_str = "absolute"; break; + case EPOCH_ORIGINAL: epoch_str = "original"; break; } strcatf(&buf, "in=%s, epoch_mode=%s, epoch=%.2f, ", @@ -187,7 +175,6 @@ int file_start(struct node *n) if (f->read.fmt) { /* Prepare file name */ - f->read.chunk = f->read.split ? 0 : -1; f->read.uri = file_format_name(f->read.fmt, &now); /* Open file */ @@ -232,12 +219,13 @@ int file_start(struct node *n) case EPOCH_ABSOLUTE: /* read first value at f->read_epoch */ f->read_offset = time_diff(&f->read_first, &f->read_epoch); break; + + default: { } } } if (f->write.fmt) { /* Prepare file name */ - f->write.chunk = f->write.split ? 0 : -1; f->write.uri = file_format_name(f->write.fmt, &now); /* Open file */ @@ -278,27 +266,26 @@ int file_read(struct node *n, struct sample *smps[], unsigned cnt) retry: values = sample_io_villas_fscan(f->read.handle->file, s, &flags); /* Get message and timestamp */ if (values < 0) { if (afeof(f->read.handle)) { - if (f->read.split) { - f->read.chunk++; - f->read.handle = file_reopen(&f->read); - if (!f->read.handle) - return 0; - - info("Open new input chunk of node %s: %d", node_name(n), f->read.chunk); - } - else { + if (f->rewind) { info("Rewind input file of node %s", node_name(n)); arewind(f->read.handle); goto retry; } + else { + info("Reached end-of-file"); + exit(EXIT_SUCCESS); + } } else warn("Failed to read messages from node %s: reason=%d", node_name(n), values); return 0; } - - if (!f->read_rate || aftell(f->read.handle) == 0) { + + if (f->read_epoch_mode == EPOCH_ORIGINAL) { + return 1; + } + else if (!f->read_rate || aftell(f->read.handle) == 0) { s->ts.origin = time_add(&s->ts.origin, &f->read_offset); if (timerfd_wait_until(f->read_timer, &s->ts.origin) == 0) serror("Failed to wait for timer"); @@ -322,14 +309,6 @@ int file_write(struct node *n, struct sample *smps[], unsigned cnt) assert(f->write.handle); assert(cnt == 1); - /* Split file if requested */ - if (f->write.split > 0 && aftell(f->write.handle) > f->write.split) { - f->write.chunk++; - f->write.handle = file_reopen(&f->write); - - info("Splitted output node %s: chunk=%u", node_name(n), f->write.chunk); - } - sample_io_villas_fprint(f->write.handle->file, s, SAMPLE_IO_ALL & ~SAMPLE_IO_OFFSET); afflush(f->write.handle);