/* Node type: File. * * Author: Steffen Vogel * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace villas; using namespace villas::node; using namespace villas::utils; static char *file_format_name(const char *format, struct timespec *ts) { struct tm tm; char *buf = new char[FILE_MAX_PATHLEN]; if (!buf) throw MemoryAllocationError(); // Convert time gmtime_r(&ts->tv_sec, &tm); strftime(buf, FILE_MAX_PATHLEN, format, &tm); return buf; } static struct timespec file_calc_offset(const struct timespec *first, const struct timespec *epoch, enum file::EpochMode mode) { // Get current time struct timespec now = time_now(); struct timespec offset; // Set offset depending on epoch switch (mode) { case file::EpochMode::DIRECT: // read first value at now + epoch offset = time_diff(first, &now); return time_add(&offset, epoch); case file::EpochMode::WAIT: // read first value at now + first + epoch offset = now; return time_add(&now, epoch); case file::EpochMode::RELATIVE: // read first value at first + epoch return *epoch; case file::EpochMode::ABSOLUTE: // read first value at f->epoch return time_diff(first, epoch); default: return (struct timespec){0}; } } int villas::node::file_parse(NodeCompat *n, json_t *json) { auto *f = n->getData(); int ret; json_error_t err; json_t *json_format = nullptr; const char *uri_tmpl = nullptr; const char *eof = nullptr; const char *epoch = nullptr; double epoch_flt = 0; ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: o, s?: { s?: s, s?: F, s?: s, s?: F, s?: " "i, s?: i }, s?: { s?: b, s?: i } }", "uri", &uri_tmpl, "format", &json_format, "in", "eof", &eof, "rate", &f->rate, "epoch_mode", &epoch, "epoch", &epoch_flt, "buffer_size", &f->buffer_size_in, "skip", &f->skip_lines, "out", "flush", &f->flush, "buffer_size", &f->buffer_size_out); if (ret) throw ConfigError(json, err, "node-config-node-file"); f->epoch = time_from_double(epoch_flt); f->uri_tmpl = uri_tmpl ? strdup(uri_tmpl) : nullptr; // Format if (f->formatter) delete f->formatter; f->formatter = json_format ? FormatFactory::make(json_format) : FormatFactory::make("villas.human"); if (!f->formatter) throw ConfigError(json_format, "node-config-node-file-format", "Invalid format configuration"); if (eof) { if (!strcmp(eof, "exit") || !strcmp(eof, "stop")) f->eof_mode = file::EOFBehaviour::STOP; else if (!strcmp(eof, "rewind")) f->eof_mode = file::EOFBehaviour::REWIND; else if (!strcmp(eof, "wait")) f->eof_mode = file::EOFBehaviour::SUSPEND; else throw RuntimeError("Invalid mode '{}' for 'eof' setting", eof); } if (epoch) { if (!strcmp(epoch, "direct")) f->epoch_mode = file::EpochMode::DIRECT; else if (!strcmp(epoch, "wait")) f->epoch_mode = file::EpochMode::WAIT; else if (!strcmp(epoch, "relative")) f->epoch_mode = file::EpochMode::RELATIVE; else if (!strcmp(epoch, "absolute")) f->epoch_mode = file::EpochMode::ABSOLUTE; else if (!strcmp(epoch, "original")) f->epoch_mode = file::EpochMode::ORIGINAL; else throw RuntimeError("Invalid value '{}' for setting 'epoch'", epoch); } return 0; } char *villas::node::file_print(NodeCompat *n) { auto *f = n->getData(); char *buf = nullptr; const char *epoch_str = nullptr; const char *eof_str = nullptr; switch (f->epoch_mode) { case file::EpochMode::DIRECT: epoch_str = "direct"; break; case file::EpochMode::WAIT: epoch_str = "wait"; break; case file::EpochMode::RELATIVE: epoch_str = "relative"; break; case file::EpochMode::ABSOLUTE: epoch_str = "absolute"; break; case file::EpochMode::ORIGINAL: epoch_str = "original"; break; default: epoch_str = ""; break; } switch (f->eof_mode) { case file::EOFBehaviour::STOP: eof_str = "stop"; break; case file::EOFBehaviour::SUSPEND: eof_str = "wait"; break; case file::EOFBehaviour::REWIND: eof_str = "rewind"; break; default: eof_str = ""; break; } strcatf( &buf, "uri=%s, out.flush=%s, in.skip=%d, in.eof=%s, in.epoch=%s, in.epoch=%.2f", f->uri ? f->uri : f->uri_tmpl, f->flush ? "yes" : "no", f->skip_lines, eof_str, epoch_str, time_to_double(&f->epoch)); if (f->rate) strcatf(&buf, ", in.rate=%.1f", f->rate); if (f->first.tv_sec || f->first.tv_nsec) strcatf(&buf, ", first=%.2f", time_to_double(&f->first)); if (f->offset.tv_sec || f->offset.tv_nsec) strcatf(&buf, ", offset=%.2f", time_to_double(&f->offset)); if ((f->first.tv_sec || f->first.tv_nsec) && (f->offset.tv_sec || f->offset.tv_nsec)) { struct timespec eta, now = time_now(); eta = time_add(&f->first, &f->offset); eta = time_diff(&now, &eta); if (eta.tv_sec || eta.tv_nsec) strcatf(&buf, ", eta=%.2f sec", time_to_double(&eta)); } return buf; } int villas::node::file_start(NodeCompat *n) { auto *f = n->getData(); struct timespec now = time_now(); int ret; // Prepare file name if (f->uri) delete[] f->uri; f->uri = file_format_name(f->uri_tmpl, &now); // Check if directory exists struct stat sb; char *cpy = strdup(f->uri); char *dir = dirname(cpy); ret = stat(dir, &sb); if (ret) { if (errno == ENOENT || errno == ENOTDIR) { ret = mkdir(dir, 0755); if (ret) throw SystemError("Failed to create directory"); } else if (errno != EISDIR) throw SystemError("Failed to stat"); } else if (!S_ISDIR(sb.st_mode)) { ret = mkdir(dir, 0644); if (ret) throw SystemError("Failed to create directory"); } free(cpy); f->formatter->start(n->getInputSignals(false)); // Open file f->stream_out = fopen(f->uri, "a+"); if (!f->stream_out) return -1; f->stream_in = fopen(f->uri, "r"); if (!f->stream_in) return -1; if (f->buffer_size_in) { ret = setvbuf(f->stream_in, nullptr, _IOFBF, f->buffer_size_in); if (ret) return ret; } if (f->buffer_size_out) { ret = setvbuf(f->stream_out, nullptr, _IOFBF, f->buffer_size_out); if (ret) return ret; } // Create timer f->task.setRate(f->rate); // Get timestamp of first line if (f->epoch_mode != file::EpochMode::ORIGINAL) { rewind(f->stream_in); if (feof(f->stream_in)) { n->logger->warn("Empty file"); } else { struct Sample smp; smp.capacity = 0; ret = f->formatter->scan(f->stream_in, &smp); if (ret == 1) { f->first = smp.ts.origin; f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); } else n->logger->warn("Failed to read first timestamp"); } } rewind(f->stream_in); // Fast-forward struct Sample *smp = sample_alloc_mem(n->getInputSignals(false)->size()); for (unsigned i = 0; i < f->skip_lines; i++) f->formatter->scan(f->stream_in, smp); sample_free(smp); return 0; } int villas::node::file_stop(NodeCompat *n) { auto *f = n->getData(); f->task.stop(); fclose(f->stream_in); fclose(f->stream_out); return 0; } int villas::node::file_read(NodeCompat *n, struct Sample *const smps[], unsigned cnt) { auto *f = n->getData(); int ret; uint64_t steps; assert(cnt == 1); retry: ret = f->formatter->scan(f->stream_in, smps, cnt); if (ret <= 0) { if (feof(f->stream_in)) { switch (f->eof_mode) { case file::EOFBehaviour::REWIND: n->logger->info("Rewind input file"); f->offset = file_calc_offset(&f->first, &f->epoch, f->epoch_mode); rewind(f->stream_in); goto retry; case file::EOFBehaviour::SUSPEND: // We wait 10ms before fetching again. usleep(100000); // Try to download more data if this is a remote file. clearerr(f->stream_in); goto retry; case file::EOFBehaviour::STOP: n->logger->info("Reached end-of-file."); n->setState(State::STOPPING); return -1; default: { } } } else n->logger->warn("Failed to read messages: reason={}", ret); return 0; } // We dont wait in FILE_EPOCH_ORIGINAL mode if (f->epoch_mode == file::EpochMode::ORIGINAL) return cnt; if (f->rate) { steps = f->task.wait(); smps[0]->ts.origin = time_now(); } else { smps[0]->ts.origin = time_add(&smps[0]->ts.origin, &f->offset); f->task.setNext(&smps[0]->ts.origin); steps = f->task.wait(); } // Check for overruns if (steps == 0) throw SystemError("Failed to wait for timer"); else if (steps != 1) n->logger->warn("Missed steps: {}", steps - 1); return cnt; } int villas::node::file_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt) { int ret; auto *f = n->getData(); assert(cnt == 1); ret = f->formatter->print(f->stream_out, smps, cnt); if (ret < 0) return ret; if (f->flush) fflush(f->stream_out); return cnt; } int villas::node::file_poll_fds(NodeCompat *n, int fds[]) { auto *f = n->getData(); if (f->rate) { fds[0] = f->task.getFD(); return 1; } else if (f->epoch_mode == file::EpochMode::ORIGINAL) { fds[0] = fileno(f->stream_in); return 1; } return -1; // TODO: not supported yet } int villas::node::file_init(NodeCompat *n) { auto *f = n->getData(); new (&f->task) Task(CLOCK_REALTIME); // Default values f->rate = 0; f->eof_mode = file::EOFBehaviour::STOP; f->epoch_mode = file::EpochMode::DIRECT; f->flush = 0; f->buffer_size_in = 0; f->buffer_size_out = 0; f->skip_lines = 0; f->formatter = nullptr; return 0; } int villas::node::file_destroy(NodeCompat *n) { auto *f = n->getData(); f->task.~Task(); if (f->uri) delete[] f->uri; if (f->formatter) delete f->formatter; return 0; } static NodeCompatType p; __attribute__((constructor(110))) static void register_plugin() { p.name = "file"; p.description = "support for file log / replay node type"; p.vectorize = 1; p.size = sizeof(struct file); p.init = file_init; p.destroy = file_destroy; p.parse = file_parse; p.print = file_print; p.start = file_start; p.stop = file_stop; p.read = file_read; p.write = file_write; p.poll_fds = file_poll_fds; static NodeCompatFactory ncp(&p); }