From 26e3885fbcc556ecd7231165048e6a994b5a00d9 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 16 Oct 2016 02:33:36 -0400 Subject: [PATCH 01/57] mpmc_queue => queue --- include/villas/path.h | 2 +- include/villas/pool.h | 10 +++++----- include/villas/queue.h | 18 +++++++++--------- lib/path.c | 8 ++++---- lib/pool.c | 6 +++--- lib/queue.c | 22 +++++++++++----------- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/villas/path.h b/include/villas/path.h index 801acf51d..fb01e5d23 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -42,7 +42,7 @@ struct path struct node *in; /**< Pointer to the incoming node */ - struct mpmc_queue queue; /**< A ring buffer for all received messages (unmodified) */ + struct queue queue; /**< A ring buffer for all received messages (unmodified) */ struct pool pool; /**< Memory pool for messages / samples. */ struct list destinations; /**< List of all outgoing nodes */ diff --git a/include/villas/pool.h b/include/villas/pool.h index c6ffcc466..4c0fc70f4 100644 --- a/include/villas/pool.h +++ b/include/villas/pool.h @@ -28,7 +28,7 @@ struct pool { size_t blocksz; /**< Length of a block in bytes */ size_t alignment; /**< Alignment of a block in bytes */ - struct mpmc_queue queue; /**< The queue which is used to keep track of free blocks */ + struct queue queue; /**< The queue which is used to keep track of free blocks */ }; #define INLINE static inline __attribute__((unused)) @@ -42,26 +42,26 @@ int pool_destroy(struct pool *p); /** Pop cnt values from the stack an place them in the array blocks */ INLINE ssize_t pool_get_many(struct pool *p, void *blocks[], size_t cnt) { - return mpmc_queue_pull_many(&p->queue, blocks, cnt); + return queue_pull_many(&p->queue, blocks, cnt); } /** Push cnt values which are giving by the array values to the stack. */ INLINE ssize_t pool_put_many(struct pool *p, void *blocks[], size_t cnt) { - return mpmc_queue_push_many(&p->queue, blocks, cnt); + return queue_push_many(&p->queue, blocks, cnt); } /** Get a free memory block from pool. */ INLINE void * pool_get(struct pool *p) { void *ptr; - return mpmc_queue_pull(&p->queue, &ptr) == 1 ? ptr : NULL; + return queue_pull(&p->queue, &ptr) == 1 ? ptr : NULL; } /** Release a memory block back to the pool. */ INLINE int pool_put(struct pool *p, void *buf) { - return mpmc_queue_push(&p->queue, buf); + return queue_push(&p->queue, buf); } #endif /* _POOL_H_ */ \ No newline at end of file diff --git a/include/villas/queue.h b/include/villas/queue.h index bf2a6cac0..40baf1f17 100644 --- a/include/villas/queue.h +++ b/include/villas/queue.h @@ -42,12 +42,12 @@ #define CACHELINE_SIZE 64 typedef char cacheline_pad_t[CACHELINE_SIZE]; -struct mpmc_queue { +struct queue { cacheline_pad_t _pad0; /**< Shared area: all threads read */ struct memtype const * mem; size_t buffer_mask; - struct mpmc_queue_cell { + struct queue_cell { atomic_size_t sequence; void *data; } *buffer; @@ -64,24 +64,24 @@ struct mpmc_queue { }; /** Initialize MPMC queue */ -int mpmc_queue_init(struct mpmc_queue *q, size_t size, const struct memtype *mem); +int queue_init(struct queue *q, size_t size, const struct memtype *mem); /** Desroy MPMC queue and release memory */ -int mpmc_queue_destroy(struct mpmc_queue *q); +int queue_destroy(struct queue *q); /** Return estimation of current queue usage. * * Note: This is only an estimation and not accurate as long other * threads are performing operations. */ -size_t mpmc_queue_available(struct mpmc_queue *q); +size_t queue_available(struct queue *q); -int mpmc_queue_push(struct mpmc_queue *q, void *ptr); +int queue_push(struct queue *q, void *ptr); -int mpmc_queue_pull(struct mpmc_queue *q, void **ptr); +int queue_pull(struct queue *q, void **ptr); -int mpmc_queue_push_many(struct mpmc_queue *q, void *ptr[], size_t cnt); +int queue_push_many(struct queue *q, void *ptr[], size_t cnt); -int mpmc_queue_pull_many(struct mpmc_queue *q, void *ptr[], size_t cnt); +int queue_pull_many(struct queue *q, void *ptr[], size_t cnt); #endif /* _MPMC_QUEUE_H_ */ \ No newline at end of file diff --git a/lib/path.c b/lib/path.c index b783c816f..69667cbcc 100644 --- a/lib/path.c +++ b/lib/path.c @@ -26,7 +26,7 @@ static void path_write(struct path *p, bool resend) int sent, tosend, available, released; struct sample *smps[n->vectorize]; - available = mpmc_queue_pull_many(&p->queue, (void **) smps, cnt); + available = queue_pull_many(&p->queue, (void **) smps, cnt); if (available < cnt) warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); @@ -108,7 +108,7 @@ static void * path_run(void *arg) p->skipped += recv - enqueue; } - enqueued = mpmc_queue_push_many(&p->queue, (void **) smps, enqueue); + enqueued = queue_push_many(&p->queue, (void **) smps, enqueue); if (enqueue != enqueued) warn("Failed to enqueue %u samples for path %s", enqueue - enqueued, path_name(p)); @@ -219,7 +219,7 @@ int path_prepare(struct path *p) if (ret) error("Failed to allocate memory pool for path"); - ret = mpmc_queue_init(&p->queue, p->queuelen, &memtype_hugepage); + ret = queue_init(&p->queue, p->queuelen, &memtype_hugepage); if (ret) error("Failed to initialize queue for path"); @@ -233,7 +233,7 @@ void path_destroy(struct path *p) list_destroy(&p->destinations, NULL, false); list_destroy(&p->hooks, NULL, true); - mpmc_queue_destroy(&p->queue); + queue_destroy(&p->queue); pool_destroy(&p->pool); free(p->_name); diff --git a/lib/pool.c b/lib/pool.c index 90317bd9e..156200c5a 100644 --- a/lib/pool.c +++ b/lib/pool.c @@ -26,17 +26,17 @@ int pool_init(struct pool *p, size_t blocksz, size_t cnt, const struct memtype * else debug(DBG_POOL | 4, "Allocated %#zx bytes for memory pool", p->len); - mpmc_queue_init(&p->queue, cnt, m); + queue_init(&p->queue, cnt, m); for (int i = 0; i < cnt; i++) - mpmc_queue_push(&p->queue, (char *) p->buffer + i * p->blocksz); + queue_push(&p->queue, (char *) p->buffer + i * p->blocksz); return 0; } int pool_destroy(struct pool *p) { - mpmc_queue_destroy(&p->queue); + queue_destroy(&p->queue); return memory_free(p->mem, p->buffer, p->len); } \ No newline at end of file diff --git a/lib/queue.c b/lib/queue.c index 136a7d263..1d1b797bd 100644 --- a/lib/queue.c +++ b/lib/queue.c @@ -34,7 +34,7 @@ #include "queue.h" /** Initialize MPMC queue */ -int mpmc_queue_init(struct mpmc_queue *q, size_t size, const struct memtype *mem) +int queue_init(struct queue *q, size_t size, const struct memtype *mem) { /* Queue size must be 2 exponent */ if ((size < 2) || ((size & (size - 1)) != 0)) @@ -55,7 +55,7 @@ int mpmc_queue_init(struct mpmc_queue *q, size_t size, const struct memtype *mem return 0; } -int mpmc_queue_destroy(struct mpmc_queue *q) +int queue_destroy(struct queue *q) { return memory_free(q->mem, q->buffer, (q->buffer_mask + 1) * sizeof(sizeof(q->buffer[0]))); } @@ -65,15 +65,15 @@ int mpmc_queue_destroy(struct mpmc_queue *q) * Note: This is only an estimation and not accurate as long other * threads are performing operations. */ -size_t mpmc_queue_available(struct mpmc_queue *q) +size_t queue_available(struct queue *q) { return atomic_load_explicit(&q->tail, memory_order_relaxed) - atomic_load_explicit(&q->head, memory_order_relaxed); } -int mpmc_queue_push(struct mpmc_queue *q, void *ptr) +int queue_push(struct queue *q, void *ptr) { - struct mpmc_queue_cell *cell; + struct queue_cell *cell; size_t pos, seq; intptr_t diff; @@ -99,9 +99,9 @@ int mpmc_queue_push(struct mpmc_queue *q, void *ptr) return 1; } -int mpmc_queue_pull(struct mpmc_queue *q, void **ptr) +int queue_pull(struct queue *q, void **ptr) { - struct mpmc_queue_cell *cell; + struct queue_cell *cell; size_t pos, seq; intptr_t diff; @@ -128,13 +128,13 @@ int mpmc_queue_pull(struct mpmc_queue *q, void **ptr) return 1; } -int mpmc_queue_push_many(struct mpmc_queue *q, void *ptr[], size_t cnt) +int queue_push_many(struct queue *q, void *ptr[], size_t cnt) { int ret; size_t i; for (i = 0; i < cnt; i++) { - ret = mpmc_queue_push(q, ptr[i]); + ret = queue_push(q, ptr[i]); if (!ret) break; } @@ -142,13 +142,13 @@ int mpmc_queue_push_many(struct mpmc_queue *q, void *ptr[], size_t cnt) return i; } -int mpmc_queue_pull_many(struct mpmc_queue *q, void *ptr[], size_t cnt) +int queue_pull_many(struct queue *q, void *ptr[], size_t cnt) { int ret; size_t i; for (i = 0; i < cnt; i++) { - ret = mpmc_queue_pull(q, &ptr[i]); + ret = queue_pull(q, &ptr[i]); if (!ret) break; } From 07d4c623e80ee2fd8492c4d17c5861244bf6296f Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 16 Oct 2016 02:33:57 -0400 Subject: [PATCH 02/57] fix deployment of web site --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0ef8281f..449be5384 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,7 +101,7 @@ integration: website: stage: deploy script: - - rsync web/ $DEPLOY_PATH/ + - rsync -r web/ $DEPLOY_PATH/ only: - develop tags: From 626ddc2043b01becc64f2a8c6a948d8dda64f2d8 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 16 Oct 2016 13:12:08 -0400 Subject: [PATCH 03/57] added preliminary draft for new websocket msg format --- include/villas/webmsg_format.h | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 include/villas/webmsg_format.h diff --git a/include/villas/webmsg_format.h b/include/villas/webmsg_format.h new file mode 100644 index 000000000..3873d4470 --- /dev/null +++ b/include/villas/webmsg_format.h @@ -0,0 +1,101 @@ +/** Binary websocket message format + * + * @file + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#ifndef _WEBMSG_FORMAT_H_ +#define _WEBMSG_FORMAT_H_ + +#include + +#ifdef __linux__ + #define _BSD_SOURCE 1 + #include +#elif defined(__PPC__) /* Xilinx toolchain */ + #include +#endif + +/** The current version number for the message format */ +#define WEBMSG_VERSION 1 + +/** @todo Implement more message types */ +#define WEBMSG_TYPE_DATA 0 /**< Message contains float values */ +#define WEBMSG_TYPE_START 1 /**< Message marks the beginning of a new simulation case */ +#define WEBMSG_TYPE_STOP 2 /**< Message marks the end of a simulation case */ + +#define WEBMSG_ENDIAN_LITTLE 0 /**< Message values are in little endian format (float too!) */ +#define WEBMSG_ENDIAN_BIG 1 /**< Message values are in bit endian format */ + +#if BYTE_ORDER == LITTLE_ENDIAN + #define WEBMSG_ENDIAN_HOST MSG_ENDIAN_LITTLE +#elif BYTE_ORDER == BIG_ENDIAN + #define WEBMSG_ENDIAN_HOST MSG_ENDIAN_BIG +#else + #error "Unknown byte order!" +#endif + +/** The total size in bytes of a message */ +#define WEBMSG_LEN(values) (sizeof(struct msg) + MSG_DATA_LEN(values)) + +/** The length of \p values values in bytes. */ +#define WEBMSG_DATA_LEN(values) (sizeof(float) * (values)) + +/** The offset to the first data value in a message. */ +#define WEBMSG_DATA_OFFSET(msg) ((char *) (msg) + offsetof(struct msg, data)) + +/** Initialize a message with default values */ +#define WEBMSG_INIT(len, seq) (struct msg) {\ + .version = MSG_VERSION, \ + .type = MSG_TYPE_DATA, \ + .endian = MSG_ENDIAN_HOST, \ + .length = len, \ + .sequence = seq \ +} + +/** The timestamp of a message in struct timespec format */ +#define WEBMSG_TS(msg) (struct timespec) {\ + .tv_sec = (msg)->ts.sec, \ + .tv_nsec = (msg)->ts.nsec \ +} + +/** This message format is used by all clients + * + * @diafile msg_format.dia + **/ +struct webmsg +{ +#if BYTE_ORDER == BIG_ENDIAN + unsigned version: 4; /**< Specifies the format of the remaining message (see MGS_VERSION) */ + unsigned type : 2; /**< Data or control message (see MSG_TYPE_*) */ + unsigned endian : 1; /**< Specifies the byteorder of the message (see MSG_ENDIAN_*) */ + unsigned rsvd1 : 1; /**< Reserved bits */ +#elif BYTE_ORDER == LITTLE_ENDIAN + unsigned rsvd1 : 1; /**< Reserved bits */ + unsigned endian : 1; /**< Specifies the byteorder of the message (see MSG_ENDIAN_*) */ + unsigned type : 2; /**< Data or control message (see MSG_TYPE_*) */ + unsigned version: 4; /**< Specifies the format of the remaining message (see MGS_VERSION) */ +#endif + + uint8_t src_node; /**< Reserved bits */ + + uint16_t length; /**< The number of values in msg::data[]. Endianess is specified in msg::endian. */ + uint32_t sequence; /**< The sequence number is incremented by one for consecutive messages. Endianess is specified in msg::endian. */ + + /** A timestamp per message. Endianess is specified in msg::endian. */ + struct { + uint32_t sec; /**< Seconds since 1970-01-01 00:00:00 */ + uint32_t nsec; /**< Nanoseconds of the current second. */ + } ts; + + /** The message payload. Endianess is specified in msg::endian. */ + union { + float f; /**< Floating point values (note msg::endian) */ + uint32_t i; /**< Integer values (note msg::endian) */ + } data[]; +} __attribute__((packed)); + +#endif /* _WEBMSG_FORMAT_H_ */ From cf59a485abd95c3d856acac4b2b8948e05b55390 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:25:05 -0400 Subject: [PATCH 04/57] rdtscp => rdtsc --- include/villas/utils.h | 4 ++-- lib/utils.c | 5 +++-- src/fpga-bench-overruns.c | 4 ++-- src/fpga-bench.c | 16 ++++++++-------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/villas/utils.h b/include/villas/utils.h index 6664b2232..82b7cabb7 100644 --- a/include/villas/utils.h +++ b/include/villas/utils.h @@ -204,11 +204,11 @@ void printb(void *mem, size_t len); void printdw(void *mem, size_t len); /** Get CPU timestep counter */ -__attribute__((always_inline)) static inline uint64_t rdtscp() +__attribute__((always_inline)) static inline uint64_t rdtsc() { uint64_t tsc; - __asm__ ("rdtscp;" + __asm__ ("rdtsc;" "shl $32, %%rdx;" "or %%rdx,%%rax" : "=a" (tsc) diff --git a/lib/utils.c b/lib/utils.c index 66512c3bd..eff72a954 100644 --- a/lib/utils.c +++ b/lib/utils.c @@ -370,12 +370,13 @@ void rdtsc_sleep(uint64_t nanosecs, uint64_t start) { uint64_t cycles; + /** @todo Replace the hard coded CPU clock frequency */ cycles = (double) nanosecs / (1e9 / 3392389000); if (start == 0) - start = rdtscp(); + start = rdtsc(); do { __asm__("nop"); - } while (rdtscp() - start < cycles); + } while (rdtsc() - start < cycles); } \ No newline at end of file diff --git a/src/fpga-bench-overruns.c b/src/fpga-bench-overruns.c index f161c7696..73346b5e9 100644 --- a/src/fpga-bench-overruns.c +++ b/src/fpga-bench-overruns.c @@ -115,9 +115,9 @@ int fpga_benchmark_overruns(struct fpga *f) for (int i = 0; i < runs + BENCH_WARMUP; i++) { dma_read(dm, mem.base_phys, 0x200); - start = rdtscp(); + start = rdtsc(); lapack_workload(p, A); - stop = rdtscp(); + stop = rdtsc(); dma_read_complete(dm, NULL, NULL); diff --git a/src/fpga-bench.c b/src/fpga-bench.c index 2ab2523a0..1de87f1f3 100644 --- a/src/fpga-bench.c +++ b/src/fpga-bench.c @@ -108,7 +108,7 @@ int fpga_benchmark_jitter(struct fpga *f) XTmrCtr_SetResetValue(xtmr, 0, period * FPGA_AXI_HZ); XTmrCtr_Start(xtmr, 0); - uint64_t end, start = rdtscp(); + uint64_t end, start = rdtsc(); for (int i = 0; i < runs; i++) { uint64_t cnt = intc_wait(f->intc, tmr->irq); if (cnt != 1) @@ -117,7 +117,7 @@ int fpga_benchmark_jitter(struct fpga *f) /* Ackowledge IRQ */ XTmrCtr_WriteReg((uintptr_t) f->map + tmr->baseaddr, 0, XTC_TCSR_OFFSET, XTmrCtr_ReadReg((uintptr_t) f->map + tmr->baseaddr, 0, XTC_TCSR_OFFSET)); - end = rdtscp(); + end = rdtsc(); hist[i] = end - start; start = end; } @@ -157,11 +157,11 @@ int fpga_benchmark_latency(struct fpga *f) error("Failed to enable interrupts"); for (int i = 0; i < runs; i++) { - start = rdtscp(); + start = rdtsc(); XIntc_Out32((uintptr_t) f->map + f->intc->baseaddr + XIN_ISR_OFFSET, 0x100); intc_wait(f->intc, 8); - end = rdtscp(); + end = rdtsc(); hist[i] = end - start; } @@ -239,7 +239,7 @@ int fpga_benchmark_datamover(struct fpga *f) uint64_t runs = BENCH_RUNS >> exp; for (int i = 0; i < runs + BENCH_WARMUP; i++) { - start = rdtscp(); + start = rdtsc(); #if BENCH_DM == 1 ssize_t ret; @@ -255,7 +255,7 @@ int fpga_benchmark_datamover(struct fpga *f) if (ret) error("DMA ping pong failed"); #endif - stop = rdtscp(); + stop = rdtsc(); if (memcmp(src.base_virt, dst.base_virt, len)) warn("Compare failed"); @@ -304,13 +304,13 @@ int fpga_benchmark_memcpy(struct fpga *f) uint64_t runs = (BENCH_RUNS << 2) >> exp; for (int i = 0; i < runs + BENCH_WARMUP; i++) { - start = rdtscp(); + start = rdtsc(); for (int j = 0; j < len / 4; j++) // mapi[j] = j; // write dummy += mapi[j]; // read - end = rdtscp(); + end = rdtsc(); if (i > BENCH_WARMUP) total += end - start; From bb50cc843e04668cee9c894b4dd0de600adcc394 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:25:52 -0400 Subject: [PATCH 05/57] added unit test for MPMC queue --- tests/Makefile.inc | 2 +- tests/queue.c | 321 +++++++++++++++++++++++++++++++++++++++++-- thirdparty/criterion | 2 +- 3 files changed, 309 insertions(+), 16 deletions(-) diff --git a/tests/Makefile.inc b/tests/Makefile.inc index b701f5772..facbef13f 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -3,7 +3,7 @@ TEST_OBJS = $(patsubst %.c,$(BUILDDIR)/%.o,$(TEST_SRCS)) TEST_CFLAGS = $(CFLAGS) TEST_LDFLAGS = $(LDFLAGS) -Wl,-rpath,'$$ORIGIN' -TEST_LDLIBS = $(LDLIBS) -lcriterion -lvillas +TEST_LDLIBS = $(LDLIBS) -lcriterion -lvillas -pthread tests: $(BUILDDIR)/testsuite diff --git a/tests/queue.c b/tests/queue.c index 1c02306ba..9d30a1b4f 100644 --- a/tests/queue.c +++ b/tests/queue.c @@ -1,25 +1,318 @@ -/** Unit tests for MPMC queue - * - * @author Steffen Vogel - * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC - * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. - * Unauthorized copying of this file, via any medium is strictly prohibited. - *********************************************************************************/ +#include +#include +#include +#include +#include +#include #include +#include +#include +#include "utils.h" #include "queue.h" +#include "memory.h" -Test(queue, singleThreaded) +#define SIZE (1 << 10) + +static struct queue q; + +struct param { + int volatile start; + int thread_count; + int queue_size; + int iter_count; + int batch_size; + void * (*thread_func)(void *); + struct queue queue; + const struct memtype *memtype; +}; + +/** Get thread id as integer + * In contrast to pthread_t which is an opaque type */ +#ifdef __linux__ + #include +#endif + +uint64_t thread_get_id() { -/* struct queue q; +#ifdef __MACH__ + uint64_t id; + pthread_threadid_np(pthread_self(), &id); + return id; +#elif defined(SYS_gettid) + return (int) syscall(SYS_gettid); +#endif + return -1; +} + +/** Sleep, do nothing */ +__attribute__((always_inline)) static inline void nop() +{ + __asm__("rep nop;"); +} + +static void * producer(void *ctx) +{ + int ret; + struct param *p = (struct param *) ctx; - queue_init(&q, 100, &memtype_heap); + srand((unsigned) time(0) + thread_get_id()); + size_t nops = rand() % 1000; - srand(1337); + /** @todo Criterion cr_log() is broken for multi-threaded programs */ + //cr_log_info("producer: tid = %lu", thread_get_id()); + + /* Wait for global start signal */ + while (p->start == 0) + pthread_yield(); - for (int i = 0; i < 100; i++) - queue_push(&q, &) + //cr_log_info("producer: wait for %zd nops", nops); - queue_destroy(&q);*/ + /* Wait for a random time */ + for (size_t i = 0; i != nops; i += 1) + nop(); + + //cr_log_info("producer: start pushing"); + + /* Enqueue */ + for (unsigned long count = 0; count < p->iter_count; count++) { + do { + ret = queue_push(&p->queue, (void *) count); + pthread_yield(); + } while (ret != 1); + } + + //cr_log_info("producer: finished"); + + return NULL; +} + +static void * consumer(void *ctx) +{ + int ret; + struct param *p = (struct param *) ctx; + + srand((unsigned) time(0) + thread_get_id()); + size_t nops = rand() % 1000; + + //cr_log_info("consumer: tid = %lu", thread_get_id()); + + /* Wait for global start signal */ + while (p->start == 0) + pthread_yield(); + + //cr_log_info("consumer: wait for %zd nops", nops); + + /* Wait for a random time */ + for (size_t i = 0; i != nops; i += 1) + nop(); + + //cr_log_info("consumer: start pulling"); + + /* Dequeue */ + for (unsigned long count = 0; count < p->iter_count; count++) { + void *ptr; + + do { + ret = queue_pull(&p->queue, &ptr); + } while (ret != 1); + + //cr_log_info("consumer: %lu\n", count); + + //cr_assert_eq((intptr_t) ptr, count); + } + + //cr_log_info("consumer: finished"); + + return NULL; +} + +void * producer_consumer(void *ctx) +{ + struct param *p = (struct param *) ctx; + + srand((unsigned) time(0) + thread_get_id()); + size_t nops = rand() % 1000; + + /* Wait for global start signal */ + while (p->start == 0) + pthread_yield(); + + /* Wait for a random time */ + for (size_t i = 0; i != nops; i += 1) + nop(); + + for (int iter = 0; iter < p->iter_count; ++iter) { + for (size_t i = 0; i < p->batch_size; i++) { + void *ptr = (void *) (iter * p->batch_size + i); + while (!queue_push(&p->queue, ptr)) + pthread_yield(); /* queue full, let other threads proceed */ + } + + for (size_t i = 0; i < p->batch_size; i++) { + void *ptr; + while (!queue_pull(&p->queue, &ptr)) + pthread_yield(); /* queue empty, let other threads proceed */ + } + } + + return 0; +} + +void * producer_consumer_many(void *ctx) +{ + struct param *p = (struct param *) ctx; + + srand((unsigned) time(0) + thread_get_id()); + size_t nops = rand() % 1000; + + /* Wait for global start signal */ + while (p->start == 0) + pthread_yield(); + + /* Wait for a random time */ + for (size_t i = 0; i != nops; i += 1) + nop(); + + void *ptrs[p->batch_size]; + + for (int iter = 0; iter < p->iter_count; ++iter) { + for (size_t i = 0; i < p->batch_size; i++) + ptrs[i] = (void *) (iter * p->batch_size + i); + + int pushed = 0; + do { + pushed += queue_push_many(&p->queue, &ptrs[pushed], p->batch_size - pushed); + if (pushed != p->batch_size) + pthread_yield(); /* queue full, let other threads proceed */ + } while (pushed < p->batch_size); + + int pulled = 0; + do { + pulled += queue_pull_many(&p->queue, &ptrs[pulled], p->batch_size - pulled); + if (pulled != p->batch_size) + pthread_yield(); /* queue empty, let other threads proceed */ + } while (pulled < p->batch_size); + } + + return 0; +} + +Test(queue, single_threaded) +{ + int ret; + struct param p = { + .iter_count = 1 << 8, + .queue_size = 1 << 10, + .start = 1 /* we start immeadiatly */ + }; + + ret = queue_init(&p.queue, p.queue_size, &memtype_heap); + cr_assert_eq(ret, 0, "Failed to create queue"); + + producer(&p); + consumer(&p); + + cr_assert_eq(queue_available(&q), 0); + + ret = queue_destroy(&p.queue); + cr_assert_eq(ret, 0, "Failed to create queue"); +} + +ParameterizedTestParameters(queue, multi_threaded) +{ + static struct param params[] = { + { + .iter_count = 1 << 12, + .queue_size = 1 << 9, + .thread_count = 32, + .thread_func = producer_consumer_many, + .batch_size = 10, + .memtype = &memtype_heap + }, { + .iter_count = 1 << 8, + .queue_size = 1 << 9, + .thread_count = 4, + .thread_func = producer_consumer_many, + .batch_size = 100, + .memtype = &memtype_heap + }, { + .iter_count = 1 << 16, + .queue_size = 1 << 9, + .thread_count = 16, + .thread_func = producer_consumer_many, + .batch_size = 100, + .memtype = &memtype_heap + }, { + .iter_count = 1 << 8, + .queue_size = 1 << 9, + .thread_count = 4, + .thread_func = producer_consumer_many, + .batch_size = 10, + .memtype = &memtype_heap + }, { + .iter_count = 1 << 16, + .queue_size = 1 << 9, + .thread_count = 16, + .thread_func = producer_consumer, + .batch_size = 10, + .memtype = &memtype_hugepage + } + }; + + return cr_make_param_array(struct param, params, ARRAY_LEN(params)); +} + +ParameterizedTest(struct param *p, queue, multi_threaded, .timeout = 10) +{ + int ret, cycpop; + + pthread_t threads[p->thread_count]; + + p->start = 0; + + ret = queue_init(&p->queue, p->queue_size, &memtype_heap); + cr_assert_eq(ret, 0, "Failed to create queue"); + + uint64_t start_tsc_time, end_tsc_time; + + for (int i = 0; i < p->thread_count; ++i) + pthread_create(&threads[i], NULL, p->thread_func, &p); + + sleep(1); + + start_tsc_time = rdtsc(); + p->start = 1; + + for (int i = 0; i < p->thread_count; ++i) + pthread_join(threads[i], NULL); + + end_tsc_time = rdtsc(); + cycpop = (end_tsc_time - start_tsc_time) / p->iter_count; + + if (cycpop < 400) + cr_log_info("cycles/op: %u\n", cycpop); + else + cr_log_warn("cycles/op are very high (%u). Are you running on a a hypervisor?\n", cycpop); + + cr_assert_eq(queue_available(&q), 0); + + ret = queue_destroy(&p->queue); + cr_assert_eq(ret, 0, "Failed to create queue"); +} + +Test(queue, init_destroy) +{ + int ret; + struct queue q; + + ret = queue_init(&q, 100, &memtype_heap); + cr_assert_eq(ret, -1); /* Should fail as size is not 2^x */ + + ret = queue_init(&q, 1024, &memtype_heap); + cr_assert_eq(ret, 0); /* Should succeed */ + + ret = queue_destroy(&q); + cr_assert_eq(ret, 0); /* Should succeed */ } \ No newline at end of file diff --git a/thirdparty/criterion b/thirdparty/criterion index 5b0f2b129..1b687a9f6 160000 --- a/thirdparty/criterion +++ b/thirdparty/criterion @@ -1 +1 @@ -Subproject commit 5b0f2b129046955c004ff56ab2caada5b55ba8e3 +Subproject commit 1b687a9f6a0e51c9e9b0a047e1fcd6c94de7a080 From 8e88493fb86532a7c4c3936c946ee4e7357ba6cb Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:33:18 -0400 Subject: [PATCH 06/57] fix continuous integration --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 408766244..54ec74dc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ COPY thirdparty/libwebsockets /tmp/libwebsockets RUN mkdir -p /tmp/libwebsockets/build && cd /tmp/libwebsockets/build && cmake .. && make install # Cleanup intermediate files from builds -RUN rm -rf /tmp +RUN rm -rf /tmp/* WORKDIR /villas From 6e13d5a895b2aa6d66e4d12c812bc0288bdbb45d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:34:27 -0400 Subject: [PATCH 07/57] add debug statements to memory functions --- include/villas/log.h | 1 + lib/memory.c | 15 ++++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/villas/log.h b/include/villas/log.h index b94ccb891..c57d686bf 100644 --- a/include/villas/log.h +++ b/include/villas/log.h @@ -38,6 +38,7 @@ enum debug_facilities { DBG_CONFIG = (1 << 10), DBG_HOOK = (1 << 11), DBG_PATH = (1 << 12), + DBG_MEM = (1 << 13), /* Node-types */ DBG_SOCKET = (1 << 16), diff --git a/lib/memory.c b/lib/memory.c index e6501bf01..df4eccd7e 100644 --- a/lib/memory.c +++ b/lib/memory.c @@ -19,23 +19,20 @@ void * memory_alloc(const struct memtype *m, size_t len) { + debug(DBG_MEM | 2, "Allocating %zu byte of %s memory", len, m->name); return m->alloc(len); } void * memory_alloc_aligned(const struct memtype *m, size_t len, size_t alignment) { - warn("memory_alloc_aligned: not implemented yet!"); - return memory_alloc(m, len); -} - -void * memory_aligned_alloc(const struct memtype *m, size_t len, size_t align) -{ - warn("memory_aligned_alloc: not implemented yet. Falling back to unaligned version."); + debug(DBG_MEM | 2, "Allocating %zu byte of %zu-byte-aligned %s memory", len, alignment, m->name); + warn("%s: not implemented yet!", __FUNCTION__); return memory_alloc(m, len); } int memory_free(const struct memtype *m, void *ptr, size_t len) { + debug(DBG_MEM | 2, "Releasing %zu bytes of %s memory", len, m->name); return m->free(ptr, len); } @@ -85,7 +82,7 @@ const struct memtype memtype_hugepage = { .flags = MEMORY_MMAP | MEMORY_HUGEPAGE, .alloc = memory_hugepage_alloc, .free = memory_hugepage_free, - .alignment = 1 << 21 /* 2 MiB hugepage */ + .alignment = 21 /* 2 MiB hugepage */ }; /** @todo */ @@ -93,5 +90,5 @@ const struct memtype memtype_dma = { .name = "dma", .flags = MEMORY_DMA | MEMORY_MMAP, .alloc = NULL, .free = NULL, - .alignment = 1 << 12 + .alignment = 12 }; From 09f486bc322f944e6be13db0b5ce351d0d6bbe69 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:35:08 -0400 Subject: [PATCH 08/57] timout for timing unit tests --- tests/timing.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/timing.c b/tests/timing.c index b4ef56813..82fb8bf6f 100644 --- a/tests/timing.c +++ b/tests/timing.c @@ -76,7 +76,7 @@ Test(timing, time_to_from_double) cr_assert_float_eq(dbl, ref, 1e-9); } -Test(timing, timerfd_create_rate) +Test(timing, timerfd_create_rate, .timeout = 20) { struct timespec start, end; @@ -100,7 +100,7 @@ Test(timing, timerfd_create_rate) close(tfd); } -Test(timing, timerfd_wait_until) +Test(timing, timerfd_wait_until, .timeout = 1) { int tfd = timerfd_create(CLOCK_REALTIME, 0); From 7a7b36e67811452000b6fbe8cd60c592f898aac4 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Wed, 19 Oct 2016 01:35:41 -0400 Subject: [PATCH 09/57] some fixes for memory pool --- include/villas/pool.h | 5 ++--- lib/pool.c | 12 ++++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/villas/pool.h b/include/villas/pool.h index 4c0fc70f4..49f2f1c9a 100644 --- a/include/villas/pool.h +++ b/include/villas/pool.h @@ -15,8 +15,7 @@ #include #include "queue.h" - -struct memtype; +#include "memory.h" /** A thread-safe memory pool */ struct pool { @@ -34,7 +33,7 @@ struct pool { #define INLINE static inline __attribute__((unused)) /** Initiazlize a pool */ -int pool_init(struct pool *p, size_t blocksz, size_t alignment, const struct memtype *mem); +int pool_init(struct pool *p, size_t cnt, size_t blocksz, const struct memtype *mem); /** Destroy and release memory used by pool. */ int pool_destroy(struct pool *p); diff --git a/lib/pool.c b/lib/pool.c index 156200c5a..955b8b77a 100644 --- a/lib/pool.c +++ b/lib/pool.c @@ -12,8 +12,10 @@ #include "memory.h" #include "kernel/kernel.h" -int pool_init(struct pool *p, size_t blocksz, size_t cnt, const struct memtype *m) +int pool_init(struct pool *p, size_t cnt, size_t blocksz, const struct memtype *m) { + int ret; + /* Make sure that we use a block size that is aligned to the size of a cache line */ p->alignment = kernel_get_cacheline_size(); p->blocksz = blocksz * CEIL(blocksz, p->alignment); @@ -25,12 +27,14 @@ int pool_init(struct pool *p, size_t blocksz, size_t cnt, const struct memtype * serror("Failed to allocate memory for memory pool"); else debug(DBG_POOL | 4, "Allocated %#zx bytes for memory pool", p->len); - - queue_init(&p->queue, cnt, m); + + ret = queue_init(&p->queue, cnt, m); + if (ret) + return ret; for (int i = 0; i < cnt; i++) queue_push(&p->queue, (char *) p->buffer + i * p->blocksz); - + return 0; } From 64ee734dc20253ca24f029d9f7c82c2542aba03d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 18:01:06 -0400 Subject: [PATCH 10/57] remove fixed-rate sending (will be replaced by new register node-type) --- include/villas/path.h | 8 ++----- lib/cfg.c | 2 -- lib/path.c | 53 ++++--------------------------------------- 3 files changed, 7 insertions(+), 56 deletions(-) diff --git a/include/villas/path.h b/include/villas/path.h index fb01e5d23..654131e2e 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -42,7 +42,7 @@ struct path struct node *in; /**< Pointer to the incoming node */ - struct queue queue; /**< A ring buffer for all received messages (unmodified) */ + struct queue queue; /**< A ring buffer for all received messages (unmodified) */ struct pool pool; /**< Memory pool for messages / samples. */ struct list destinations; /**< List of all outgoing nodes */ @@ -51,11 +51,8 @@ struct path int samplelen; /**< Maximum number of values per sample for this path. */ int queuelen; /**< Size of sample queue for this path. */ int enabled; /**< Is this path enabled */ - int tfd; /**< Timer file descriptor for fixed rate sending */ - double rate; /**< Send messages with a fixed rate over this path */ - pthread_t recv_tid; /**< The thread id for this path */ - pthread_t sent_tid; /**< A second thread id for fixed rate sending thread */ + pthread_t tid; /**< The thread id for this path */ config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path */ @@ -74,7 +71,6 @@ struct path uintmax_t invalid; /**< Counter for invalid messages */ uintmax_t skipped; /**< Counter for skipped messages due to hooks */ uintmax_t dropped; /**< Counter for dropped messages due to reordering */ - uintmax_t overrun; /**< Counter of overruns for fixed-rate sending */ /** @} */ }; diff --git a/lib/cfg.c b/lib/cfg.c index 33fe11ed6..84fbfd35f 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -201,8 +201,6 @@ int cfg_parse_path(config_setting_t *cfg, reverse = 0; if (!config_setting_lookup_bool(cfg, "enabled", &p->enabled)) p->enabled = 1; - if (!config_setting_lookup_float(cfg, "rate", &p->rate)) - p->rate = 0; /* disabled */ p->cfg = cfg; diff --git a/lib/path.c b/lib/path.c index 69667cbcc..ec8e100ba 100644 --- a/lib/path.c +++ b/lib/path.c @@ -51,32 +51,7 @@ static void path_write(struct path *p, bool resend) } } -/** Send messages asynchronously */ -static void * path_run_async(void *arg) -{ - struct path *p = arg; - - /* Block until 1/p->rate seconds elapsed */ - for (;;) { - /* Check for overruns */ - uint64_t expir = timerfd_wait(p->tfd); - if (expir == 0) - perror("Failed to wait for timer"); - else if (expir > 1) { - p->overrun += expir; - warn("Overrun detected for path: overruns=%" PRIu64, expir); - } - - if (hook_run(p, NULL, 0, HOOK_ASYNC)) - continue; - - path_write(p, true); - } - - return NULL; -} - -/** Receive messages */ +/** Main thread function per path: receive -> sent messages */ static void * path_run(void *arg) { struct path *p = arg; @@ -116,9 +91,7 @@ static void * path_run(void *arg) debug(DBG_PATH | 3, "Enqueuing %u samples to queue of path %s", enqueue, path_name(p)); - /* At fixed rate mode, messages are send by another (asynchronous) thread */ - if (p->rate == 0) - path_write(p, false); + path_write(p); } return NULL; @@ -128,25 +101,16 @@ int path_start(struct path *p) { int ret; - info("Starting path: %s (#hooks=%zu, rate=%.1f)", - path_name(p), list_length(&p->hooks), p->rate); + info("Starting path: %s (#hooks=%zu)", + path_name(p), list_length(&p->hooks)); ret = hook_run(p, NULL, 0, HOOK_PATH_START); if (ret) return -1; - /* At fixed rate mode, we start another thread for sending */ - if (p->rate) { - p->tfd = timerfd_create_rate(p->rate); - if (p->tfd < 0) - serror("Failed to create timer"); - - pthread_create(&p->sent_tid, NULL, &path_run_async, p); - } - p->state = PATH_RUNNING; - return pthread_create(&p->recv_tid, NULL, &path_run, p); + return pthread_create(&p->tid, NULL, &path_run, p); } int path_stop(struct path *p) @@ -156,13 +120,6 @@ int path_stop(struct path *p) pthread_cancel(p->recv_tid); pthread_join(p->recv_tid, NULL); - if (p->rate) { - pthread_cancel(p->sent_tid); - pthread_join(p->sent_tid, NULL); - - close(p->tfd); - } - p->state = PATH_STOPPED; if (hook_run(p, NULL, 0, HOOK_PATH_STOP)) From bad1dc953adda594b00d7c16c3cb1686e74aa61b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 18:01:42 -0400 Subject: [PATCH 11/57] fix Doxygen commenting style --- include/villas/kernel/pci.h | 2 +- include/villas/nodes/socket.h | 4 ++-- include/villas/queue.h | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/villas/kernel/pci.h b/include/villas/kernel/pci.h index 747f5c488..82906b9d2 100644 --- a/include/villas/kernel/pci.h +++ b/include/villas/kernel/pci.h @@ -31,7 +31,7 @@ struct pci_dev { }; struct pci { - struct list devices; /**> List of available PCI devices in the system (struct pci_dev) */ + struct list devices; /**< List of available PCI devices in the system (struct pci_dev) */ }; /** Initialize Linux PCI handle. diff --git a/include/villas/nodes/socket.h b/include/villas/nodes/socket.h index 64c1dd82e..044e76de9 100644 --- a/include/villas/nodes/socket.h +++ b/include/villas/nodes/socket.h @@ -29,8 +29,8 @@ enum socket_layer { }; enum socket_header { - SOCKET_HEADER_DEFAULT, /**> Default header in the payload, (see msg_format.h) */ - SOCKET_HEADER_GTNET_SKT /**> No header in the payload, same as HDR_NONE*/ + SOCKET_HEADER_DEFAULT, /**< Default header in the payload, (see msg_format.h) */ + SOCKET_HEADER_GTNET_SKT /**< No header in the payload, same as HDR_NONE*/ }; union sockaddr_union { diff --git a/include/villas/queue.h b/include/villas/queue.h index 40baf1f17..50fa5aff9 100644 --- a/include/villas/queue.h +++ b/include/villas/queue.h @@ -52,15 +52,15 @@ struct queue { void *data; } *buffer; - cacheline_pad_t _pad1; /**> Producer area: only producers read & write */ + cacheline_pad_t _pad1; /**< Producer area: only producers read & write */ - atomic_size_t tail; /**> Queue tail pointer */ + atomic_size_t tail; /**< Queue tail pointer */ - cacheline_pad_t _pad2; /**> Consumer area: only consumers read & write */ + cacheline_pad_t _pad2; /**< Consumer area: only consumers read & write */ - atomic_size_t head; /**> Queue head pointer */ + atomic_size_t head; /**< Queue head pointer */ - cacheline_pad_t _pad3; /**> @todo Why needed? */ + cacheline_pad_t _pad3; /**< @todo Why needed? */ }; /** Initialize MPMC queue */ From 69a16b2ac70e0816f3663b8b4f2584d56c25e727 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 18:04:18 -0400 Subject: [PATCH 12/57] added a couple of states to the path FSM --- include/villas/path.h | 6 ++++-- lib/path.c | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/villas/path.h b/include/villas/path.h index 654131e2e..abd1dfbc2 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -35,9 +35,11 @@ struct path { enum { PATH_INVALID, /**< Path is invalid. */ - PATH_CREATED, /**< Path has been created. */ + PATH_CREATED, /**< Path has been created: lists initialized */ + PATH_INITIALIZED, /**< Path queues, memory pools & hook system initialized. */ PATH_RUNNING, /**< Path is currently running. */ - PATH_STOPPED /**< Path has been stopped. */ + PATH_STOPPED, /**< Path has been stopped. */ + PATH_DESTROYED /**< Path is destroyed. */ } state; /**< Path state */ struct node *in; /**< Pointer to the incoming node */ diff --git a/lib/path.c b/lib/path.c index ec8e100ba..dde5b0e0e 100644 --- a/lib/path.c +++ b/lib/path.c @@ -120,11 +120,11 @@ int path_stop(struct path *p) pthread_cancel(p->recv_tid); pthread_join(p->recv_tid, NULL); - p->state = PATH_STOPPED; - if (hook_run(p, NULL, 0, HOOK_PATH_STOP)) return -1; + p->state = PATH_STOPPED; + return 0; } @@ -179,6 +179,8 @@ int path_prepare(struct path *p) ret = queue_init(&p->queue, p->queuelen, &memtype_hugepage); if (ret) error("Failed to initialize queue for path"); + + p->state = PATH_INITIALIZED; return 0; } @@ -192,6 +194,8 @@ void path_destroy(struct path *p) queue_destroy(&p->queue); pool_destroy(&p->pool); + + p->state = PATH_DESTROYED; free(p->_name); } From de1058cbd66cab9483b7c5fef9dde4c2f7b0d4e5 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 18:05:56 -0400 Subject: [PATCH 13/57] merging path_write() into path_run() --- lib/path.c | 61 +++++++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/lib/path.c b/lib/path.c index dde5b0e0e..f94feeea3 100644 --- a/lib/path.c +++ b/lib/path.c @@ -19,38 +19,6 @@ #include "pool.h" #include "queue.h" -static void path_write(struct path *p, bool resend) -{ - list_foreach(struct node *n, &p->destinations) { - int cnt = n->vectorize; - int sent, tosend, available, released; - struct sample *smps[n->vectorize]; - - available = queue_pull_many(&p->queue, (void **) smps, cnt); - if (available < cnt) - warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); - - if (available == 0) - continue; - - tosend = hook_run(p, smps, available, HOOK_WRITE); - if (tosend == 0) - continue; - - sent = node_write(n, smps, tosend); - if (sent < 0) - error("Failed to sent %u samples to node %s", cnt, node_name(n)); - else if (sent < tosend) - warn("Partial write to node %s", node_name(n)); - - debug(DBG_PATH | 15, "Sent %u messages to node %s", sent, node_name(n)); - - released = pool_put_many(&p->pool, (void **) smps, sent); - if (sent != released) - warn("Failed to release %u samples to pool for path %s", sent - released, path_name(p)); - } -} - /** Main thread function per path: receive -> sent messages */ static void * path_run(void *arg) { @@ -91,7 +59,34 @@ static void * path_run(void *arg) debug(DBG_PATH | 3, "Enqueuing %u samples to queue of path %s", enqueue, path_name(p)); - path_write(p); + list_foreach(struct node *n, &p->destinations) { + int cnt = n->vectorize; + int sent, tosend, available, released; + struct sample *smps[n->vectorize]; + + available = queue_pull_many(&p->queue, (void **) smps, cnt); + if (available < cnt) + warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); + + if (available == 0) + continue; + + tosend = hook_run(p, smps, available, HOOK_WRITE); + if (tosend == 0) + continue; + + sent = node_write(n, smps, tosend); + if (sent < 0) + error("Failed to sent %u samples to node %s", cnt, node_name(n)); + else if (sent < tosend) + warn("Partial write to node %s", node_name(n)); + + debug(DBG_PATH | 15, "Sent %u messages to node %s", sent, node_name(n)); + + released = pool_put_many(&p->pool, (void **) smps, sent); + if (sent != released) + warn("Failed to release %u samples to pool for path %s", sent - released, path_name(p)); + } } return NULL; From 982cd91d638ece928c90539519ae938311f0bf90 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 21:16:01 -0400 Subject: [PATCH 14/57] simplified path_init / prepare logic --- include/villas/path.h | 10 ++-------- lib/cfg.c | 4 +++- lib/path.c | 26 +++++++++----------------- src/node.c | 2 +- 4 files changed, 15 insertions(+), 27 deletions(-) diff --git a/include/villas/path.h b/include/villas/path.h index abd1dfbc2..0bcb4cf56 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -77,8 +77,8 @@ struct path /** @} */ }; -/** Create a path by allocating dynamic memory. */ -void path_init(struct path *p); +/** Initialize internal data structures. */ +int path_init(struct path *p); /** Destroy path by freeing dynamically allocated memory. * @@ -86,12 +86,6 @@ void path_init(struct path *p); */ void path_destroy(struct path *p); -/** Initialize pool queue and hooks. - * - * Should be called after path_init() and before path_start(). - */ -int path_prepare(struct path *p); - /** Start a path. * * Start a new pthread for receiving/sending messages over this path. diff --git a/lib/cfg.c b/lib/cfg.c index 84fbfd35f..f8f4b963d 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -154,7 +154,9 @@ int cfg_parse_path(config_setting_t *cfg, /* Allocate memory and intialize path structure */ p = alloc(sizeof(struct path)); - path_init(p); + + list_init(&p->destinations); + list_init(&p->hooks); /* Input node */ if (!config_setting_lookup_string(cfg, "in", &in) && diff --git a/lib/path.c b/lib/path.c index f94feeea3..6399bb6e1 100644 --- a/lib/path.c +++ b/lib/path.c @@ -135,28 +135,20 @@ const char * path_name(struct path *p) return p->_name; } -void path_init(struct path *p) -{ - list_init(&p->destinations); - list_init(&p->hooks); - - /* Initialize hook system */ - list_foreach(struct hook *h, &hooks) { - if (h->type & HOOK_INTERNAL) - list_push(&p->hooks, memdup(h, sizeof(*h))); - } - - p->state = PATH_CREATED; -} - -int path_prepare(struct path *p) +int path_init(struct path *p) { int ret; - + + /* Add internal hooks if they are not already in the list*/ + list_foreach(struct hook *h, &hooks) { + if ((h->type & HOOK_INTERNAL) && (list_lookup(&p->hooks, h->name) == NULL)) + list_push(&p->hooks, memdup(h, sizeof(struct hook))); + } + /* We sort the hooks according to their priority before starting the path */ list_sort(&p->hooks, hooks_sort_priority); - /* Allocate hook private memory */ + /* Allocate hook private memory for storing state / parameters */ ret = hook_run(p, NULL, 0, HOOK_INIT); if (ret) error("Failed to initialize hooks of path: %s", path_name(p)); diff --git a/src/node.c b/src/node.c index b4cf53cd7..1a4305b90 100644 --- a/src/node.c +++ b/src/node.c @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) info("Starting paths"); list_foreach(struct path *p, &paths) { INDENT if (p->enabled) { - path_prepare(p); + path_init(p); path_start(p); } else From d4e6204e8e6d279690746bde95f08c0fcb042476 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Thu, 20 Oct 2016 21:16:47 -0400 Subject: [PATCH 15/57] fixed for rate --- lib/hooks/hooks-stats.c | 10 ++++------ lib/path.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index 0d1ce43af..ff5fbd268 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -18,13 +18,12 @@ void hook_stats_header() { #define UNIT(u) "(" YEL(u) ")" - stats("%-40s|%19s|%19s|%19s|%19s|%19s|%10s|", "Source " MAG("=>") " Destination", + stats("%-40s|%19s|%19s|%19s|%19s|%19s|", "Source " MAG("=>") " Destination", "OWD" UNIT("S") " ", "Rate" UNIT("p/S") " ", "Drop" UNIT("p") " ", "Skip" UNIT("p") " ", - "Inval" UNIT("p") " ", - "Overuns " + "Inval" UNIT("p") " " ); line(); } @@ -83,8 +82,8 @@ int hook_stats(struct path *p, struct hook *h, int when, struct sample *smps[], break; case HOOK_PERIODIC: - stats("%-40.40s|%10s|%10s|%10ju|%10ju|%10ju|%10ju|", path_name(p), "", "", - p->dropped, p->skipped, p->invalid, p->overrun); + stats("%-40.40s|%10s|%10s|%10ju|%10ju|%10ju|", path_name(p), "", "", + p->dropped, p->skipped, p->invalid); break; } @@ -124,7 +123,6 @@ int hook_stats_send(struct path *p, struct hook *h, int when, struct sample *smp smp->data[i++].f = p->invalid; smp->data[i++].f = p->skipped; smp->data[i++].f = p->dropped; - smp->data[i++].f = p->overrun; smp->data[i++].f = p->hist.owd.last, smp->data[i++].f = 1.0 / p->hist.gap_msg.last; smp->data[i++].f = 1.0 / p->hist.gap_recv.last; diff --git a/lib/path.c b/lib/path.c index 6399bb6e1..7d22f768e 100644 --- a/lib/path.c +++ b/lib/path.c @@ -112,8 +112,8 @@ int path_stop(struct path *p) { info("Stopping path: %s", path_name(p)); - pthread_cancel(p->recv_tid); - pthread_join(p->recv_tid, NULL); + pthread_cancel(p->tid); + pthread_join(p->tid, NULL); if (hook_run(p, NULL, 0, HOOK_PATH_STOP)) return -1; From 1ca719ac6f1f7af41bb87ce51147dbd77dd6cc1e Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 12:11:34 -0400 Subject: [PATCH 16/57] use a better name for the new field and added a comment explaining its pupose --- include/villas/webmsg_format.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/villas/webmsg_format.h b/include/villas/webmsg_format.h index 3873d4470..55f885980 100644 --- a/include/villas/webmsg_format.h +++ b/include/villas/webmsg_format.h @@ -80,7 +80,8 @@ struct webmsg unsigned version: 4; /**< Specifies the format of the remaining message (see MGS_VERSION) */ #endif - uint8_t src_node; /**< Reserved bits */ + uint8_t node_id; /**< The node index from / to which this sample received / sent to. + * Corresponds to the index of the node in the http://localhost/nodes.json array. */ uint16_t length; /**< The number of values in msg::data[]. Endianess is specified in msg::endian. */ uint32_t sequence; /**< The sequence number is incremented by one for consecutive messages. Endianess is specified in msg::endian. */ From 9813d2a6c258e94f04a1c7bf58be0fc26d8725bb Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:37:02 -0400 Subject: [PATCH 17/57] use hard coded names for usage info --- src/fpga.c | 10 +++++----- src/node.c | 6 +++--- src/pipe.c | 12 ++++++------ src/signal.c | 8 ++++---- src/test.c | 8 ++++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/fpga.c b/src/fpga.c index 46895c0f6..da8205076 100644 --- a/src/fpga.c +++ b/src/fpga.c @@ -31,9 +31,9 @@ int fpga_tests(int argc, char *argv[], struct fpga *f); struct settings settings; -void usage(char *name) +void usage() { - printf("Usage: %s CONFIGFILE CMD [OPTIONS]\n", name); + printf("Usage: villas-fpga CONFIGFILE CMD [OPTIONS]\n"); printf(" Commands:\n"); printf(" tests Test functionality of VILLASfpga card\n"); printf(" benchmarks Do benchmarks\n\n"); @@ -57,13 +57,13 @@ int main(int argc, char *argv[]) } subcommand; if (argc < 3) - usage(argv[0]); + usage(); if (strcmp(argv[2], "tests") == 0) subcommand = FPGA_TESTS; else if (strcmp(argv[2], "benchmarks") == 0) subcommand = FPGA_BENCH; else - usage(argv[0]); + usage(); /* Parse arguments */ char c, *endptr; @@ -75,7 +75,7 @@ int main(int argc, char *argv[]) case '?': default: - usage(argv[0]); + usage(); } } diff --git a/src/node.c b/src/node.c index 1a4305b90..4f89f6bbf 100644 --- a/src/node.c +++ b/src/node.c @@ -71,9 +71,9 @@ static void signals_init() sigaction(SIGTERM, &sa_quit, NULL); } -static void usage(const char *name) +static void usage() { - printf("Usage: %s CONFIG\n", name); + printf("Usage: villas-node CONFIG\n"); printf(" CONFIG is a required path to a configuration file\n\n"); #ifdef ENABLE_OPAL_ASYNC printf("Usage: %s OPAL_ASYNC_SHMEM_NAME OPAL_ASYNC_SHMEM_SIZE OPAL_PRINT_SHMEM_NAME\n", name); @@ -103,7 +103,7 @@ int main(int argc, char *argv[]) #else if (argc != 2) #endif - usage(argv[0]); + usage(); char *configfile = (argc == 2) ? argv[1] : "opal-shmem.conf"; diff --git a/src/pipe.c b/src/pipe.c index 334fa0302..626536b08 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -69,9 +69,9 @@ static void quit(int signal, siginfo_t *sinfo, void *ctx) exit(EXIT_SUCCESS); } -static void usage(char *name) +static void usage() { - printf("Usage: %s CONFIG NODE [OPTIONS]\n", name); + printf("Usage: villas-pipe CONFIG NODE [OPTIONS]\n"); printf(" CONFIG path to a configuration file\n"); printf(" NODE the name of the node to which samples are sent and received from\n"); printf(" OPTIONS are:\n"); @@ -100,7 +100,7 @@ static void * send_loop(void *ctx) if (ret < 0) error("Failed to allocate memory for receive pool."); - ret = sample_get_many(&sendd.pool, smps, node->vectorize); + ret = sample_alloc(&sendd.pool, smps, node->vectorize); if (ret < 0) error("Failed to get %u samples out of send pool (%d).", node->vectorize, ret); @@ -144,7 +144,7 @@ static void * recv_loop(void *ctx) if (ret < 0) error("Failed to allocate memory for receive pool."); - ret = sample_get_many(&recvv.pool, smps, node->vectorize); + ret = sample_alloc(&recvv.pool, smps, node->vectorize); if (ret < 0) error("Failed to get %u samples out of receive pool (%d).", node->vectorize, ret); @@ -180,7 +180,7 @@ int main(int argc, char *argv[]) /* Parse command line arguments */ if (argc < 3) - usage(argv[0]); + usage(); /* Default values */ sendd.enabled = true; @@ -202,7 +202,7 @@ int main(int argc, char *argv[]) break; case 'h': case '?': - usage(argv[0]); + usage(); } } diff --git a/src/signal.c b/src/signal.c index 82e6b9e27..bce67fadd 100644 --- a/src/signal.c +++ b/src/signal.c @@ -30,9 +30,9 @@ enum SIGNAL_TYPE { TYPE_MIXED }; -void usage(char *name) +void usage() { - printf("Usage: %s SIGNAL [OPTIONS]\n", name); + printf("Usage: villas-signal SIGNAL [OPTIONS]\n"); printf(" SIGNAL is on of: 'mixed', 'random', 'sine', 'triangle', 'square', 'ramp'\n"); printf(" -v NUM specifies how many values a message should contain\n"); printf(" -r HZ how many messages per second\n"); @@ -59,7 +59,7 @@ int main(int argc, char *argv[]) log_init(); if (argc < 2) { - usage(argv[0]); + usage(); exit(EXIT_FAILURE); } @@ -101,7 +101,7 @@ int main(int argc, char *argv[]) goto check; case 'h': case '?': - usage(argv[0]); + usage(); } continue; diff --git a/src/test.c b/src/test.c index ef95b2c65..4f9e351b9 100644 --- a/src/test.c +++ b/src/test.c @@ -53,9 +53,9 @@ void quit() running = 0; } -void usage(char *name) +void usage() { - printf("Usage: %s CONFIG TEST NODE [ARGS]\n", name); + printf("Usage: villas-test CONFIG TEST NODE [ARGS]\n"); printf(" CONFIG path to a configuration file\n"); printf(" TEST the name of the test to execute: 'rtt'\n"); printf(" NODE name of the node which shoud be used\n\n"); @@ -68,7 +68,7 @@ int main(int argc, char *argv[]) config_t config; if (argc < 4) { - usage(argv[0]); + usage(); exit(EXIT_FAILURE); } @@ -184,7 +184,7 @@ void test_rtt() { struct stat st; if (!fstat(fd, &st)) { FILE *f = fdopen(fd, "w"); - hist_matlab(&hist, f); + hist_dump_matlab(&hist, f); } else error("Invalid file descriptor: %u", fd); From d2347102f83dde75b02c5f55424ce371edb72549 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:37:29 -0400 Subject: [PATCH 18/57] fix path to hotdogs --- etc/global.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/global.conf b/etc/global.conf index 163eddb0d..17d51c86f 100644 --- a/etc/global.conf +++ b/etc/global.conf @@ -34,6 +34,6 @@ name = "villas-acs" # The name of this VILLASnode. Might by used by node-typ # to identify themselves (default is the hostname). http = { - htdocs = "/villas/contrib/websocket", # Root directory of internal webserver + htdocs = "/villas/web/socket/", # Root directory of internal webserver port = 80 # Port for HTTP connections } \ No newline at end of file From bee825ae305be8e740901c3a2f2d484e84e9eaf1 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:38:31 -0400 Subject: [PATCH 19/57] generate json object from histogram --- include/villas/hist.h | 12 +++++++++- lib/hist.c | 53 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/include/villas/hist.h b/include/villas/hist.h index 72b232a21..11be82119 100644 --- a/include/villas/hist.h +++ b/include/villas/hist.h @@ -14,6 +14,10 @@ #include "config.h" +#ifdef WITH_JANSSON + #include +#endif + #define HIST_HEIGHT (LOG_WIDTH - 55) #define HIST_SEQ 17 @@ -75,6 +79,12 @@ void hist_plot(struct hist *h); char * hist_dump(struct hist *h); /** Prints Matlab struct containing all infos to file. */ -void hist_matlab(struct hist *h, FILE *f); +int hist_dump_matlab(struct hist *h, FILE *f); + +#ifdef WITH_JANSSON +int hist_dump_json(struct hist *h, FILE *f); + +json_t * hist_json(struct hist *h); +#endif #endif /* _HIST_H_ */ diff --git a/lib/hist.c b/lib/hist.c index 8f29f6597..33bbce263 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -157,19 +157,60 @@ char * hist_dump(struct hist *h) return buf; } -void hist_matlab(struct hist *h, FILE *f) +#ifdef WITH_JANSSON +json_t * hist_json(struct hist *h) +{ + json_t *b = json_array(); + + for (int i = 0; i < h->length; i++) + json_array_append(b, json_integer(h->data[i])); + + return json_pack("{ s: f, s: f, s: i, s: i, s: i, s: f, s: f, s: f, s: f, s: f, s: o }", + "low", h->low, + "high", h->high, + "total", h->total, + "higher", h->higher, + "lower", h->lower, + "highest", h->highest, + "lowest", h->lowest, + "mean", hist_mean(h), + "variance", hist_var(h), + "stddev", hist_stddev(h), + "buckets", b + ); +} + +int hist_dump_json(struct hist *h, FILE *f) +{ + json_t *j = hist_json(h); + + int ret = json_dumpf(j, f, 0); + + json_decref(j); + + return ret; +} +#endif /* WITH_JANNSON */ + +int hist_dump_matlab(struct hist *h, FILE *f) { char *buf = hist_dump(h); fprintf(f, "%lu = struct( ", time(NULL)); - fprintf(f, "'min', %f, 'max', %f, ", h->low, h->high); - fprintf(f, "'total', %u, higher', %u, 'lower', %u, ", h->total, h->higher, h->lower); - fprintf(f, "'highest', %f, 'lowest', %f, ", h->highest, h->lowest); + fprintf(f, "'low', %f, ", h->low); + fprintf(f, "'high', %f, ", h->high); + fprintf(f, "'total', %u, ", h->total); + fprintf(f, "'higher', %u, ", h->higher); + fprintf(f, "'lower', %u, ", h->lower); + fprintf(f, "'highest', %f, ", h->highest); + fprintf(f, "'lowest', %f, ", h->lowest); fprintf(f, "'mean', %f, ", hist_mean(h)); - fprintf(f, "'var', %f, ", hist_var(h)); + fprintf(f, "'variance', %f, ", hist_var(h)); fprintf(f, "'stddev', %f, ", hist_stddev(h)); - fprintf(f, "'hist', %s ", buf); + fprintf(f, "'buckets', %s ", buf); fprintf(f, "),\n"); free(buf); + + return 0; } From a9363dad44df9f2277217ba1898ea5baf68c4e35 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:42:05 -0400 Subject: [PATCH 20/57] entangle hook and statistic collection system --- include/villas/hooks.h | 58 +++++++++------ include/villas/stats.h | 63 ++++++++++++++++ lib/Makefile.inc | 2 +- lib/hooks.c | 44 +++++++++--- lib/hooks/hooks-internal.c | 63 ++++++++-------- lib/hooks/hooks-other.c | 78 +++++++++++--------- lib/hooks/hooks-stats.c | 95 ++++++------------------- lib/stats.c | 142 +++++++++++++++++++++++++++++++++++++ 8 files changed, 376 insertions(+), 169 deletions(-) create mode 100644 include/villas/stats.h create mode 100644 lib/stats.c diff --git a/include/villas/hooks.h b/include/villas/hooks.h index 9eae7bb6b..7c69b0125 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -50,17 +50,33 @@ struct settings; /** This is a list of hooks which can be used in the configuration file. */ extern struct list hooks; +/** Optional parameters to hook callbacks */ +struct hook_info { + struct node *node; + struct path *path; + + struct sample **smps; + size_t cnt; + + struct list *paths; + struct list *nodes; + struct settings *settings; +}; + /** Callback type of hook function * - * @param p The path which is processing this message. * @param h The hook datastructure which contains parameter, name and private context for the hook. - * @param m A pointer to the first message which should be processed by the hook. - * @param cnt The number of messages which should be processed by the hook. * @param when Provides the type of hook for which this occurence of the callback function was executed. See hook_type for possible values. + * @param i The hook_info structure contains references to the current node, path or samples. Some fields of this structure can be NULL. * @retval 0 Success. Continue processing and forwarding the message. * @retval <0 Error. Drop the message. */ -typedef int (*hook_cb_t)(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); +typedef int (*hook_cb_t)(struct hook *h, int when, struct hook_info *i); + +enum hook_state { + HOOK_DESTROYED, + HOOK_INITIALIZED +}; /** The type of a hook defines when a hook will be exectuted. This is used as a bitmask. */ enum hook_type { @@ -74,15 +90,15 @@ enum hook_type { HOOK_ASYNC = 1 << 7, /**< Called asynchronously with fixed rate (see path::rate). */ HOOK_PERIODIC = 1 << 8, /**< Called periodically. Period is set by global 'stats' option in the configuration file. */ - HOOK_INIT = 1 << 9, /**< Called before path is started to parse parameters. */ - HOOK_DEINIT = 1 << 10, /**< Called after path has been stopped to release memory allocated by HOOK_INIT */ + HOOK_INIT = 1 << 9, /**< Called before path is started to parseHOOK_DESTROYs. */ + HOOK_DESTROY = 1 << 10, /**< Called after path has been stopped to release memory allocated by HOOK_INIT */ HOOK_INTERNAL = 1 << 11, /**< Internal hooks are added to every path implicitely. */ HOOK_PARSE = 1 << 12, /**< Called for parsing hook arguments. */ /** @{ Classes of hooks */ /** Hooks which are using private data must allocate and free them propery. */ - HOOK_STORAGE = HOOK_INIT | HOOK_DEINIT, + HOOK_STORAGE = HOOK_INIT | HOOK_DESTROY, /** All path related actions */ HOOK_PATH = HOOK_PATH_START | HOOK_PATH_STOP | HOOK_PATH_RESTART, /** Hooks which are used to collect statistics. */ @@ -111,8 +127,11 @@ struct hook { hook_cb_t cb; /**< The hook callback function as a function pointer. */ }; -/** Save references to global nodes, paths and settings */ -void hook_init(struct list *nodes, struct list *paths, struct settings *set); +int hook_init(struct hook *h, struct list *nodes, struct list *paths, struct settings *settings); + +void hook_destroy(struct hook *h); + +int hook_copy(struct hook *h, struct hook *c); /** Sort hook list according to the their priority. See hook::priority. */ int hooks_sort_priority(const void *a, const void *b); @@ -139,18 +158,17 @@ int hook_run(struct path *p, struct sample *smps[], size_t cnt, int when); */ void * hook_storage(struct hook *h, int when, size_t len); -int hook_print(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_ts(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_convert(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_decimate(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_skip_first(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); +int hook_print(struct hook *h, int when, struct hook_info *j); +int hook_ts(struct hook *h, int when, struct hook_info *j); +int hook_convert(struct hook *h, int when, struct hook_info *j); +int hook_decimate(struct hook *h, int when, struct hook_info *j); +int hook_skip_first(struct hook *h, int when, struct hook_info *j); -int hook_stats_send(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_stats(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -void hook_stats_header(); +int hook_stats_send(struct hook *h, int when, struct hook_info *j); +int hook_stats(struct hook *h, int when, struct hook_info *j); -int hook_fix_ts(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_restart(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); -int hook_drop(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt); +int hook_fix_ts(struct hook *h, int when, struct hook_info *j); +int hook_restart(struct hook *h, int when, struct hook_info *j); +int hook_drop(struct hook *h, int when, struct hook_info *j); #endif /** _HOOKS_H_ @} */ diff --git a/include/villas/stats.h b/include/villas/stats.h new file mode 100644 index 000000000..e6260833b --- /dev/null +++ b/include/villas/stats.h @@ -0,0 +1,63 @@ +/** Statistic collection. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + */ + +#ifndef _STATS_H_ +#define _STATS_H_ + +#include + +#ifdef WITH_JANSSON + #include +#endif + +#include "hist.h" + +/* Forward declarations */ +struct sample; +struct path; +struct node; + +struct stats { + struct { + uintmax_t invalid; /**< Counter for invalid messages */ + uintmax_t skipped; /**< Counter for skipped messages due to hooks */ + uintmax_t dropped; /**< Counter for dropped messages due to reordering */ + } counter; + + struct { + struct hist owd; /**< Histogram for one-way-delay (OWD) of received messages */ + struct hist gap_msg; /**< Histogram for inter message timestamps (as sent by remote) */ + struct hist gap_recv; /**< Histogram for inter message arrival time (as seen by this instance) */ + struct hist gap_seq; /**< Histogram of sequence number displacement of received messages */ + } histogram; + + struct sample *last; +}; + +int stats_init(struct stats *s); + +void stats_destroy(struct stats *s); + +void stats_collect(struct stats *s, struct sample *smps[], size_t cnt); + +#ifdef WITH_JANSSON +json_t * stats_json(struct stats *s); +#endif + +void stats_reset(struct stats *s); + +void stats_print_header(); + +void stats_print_periodic(struct stats *s, struct path *p); + +void stats_print(struct stats *s); + +void stats_send(struct stats *s, struct node *n); + +#endif /* _STATS_H_ */ \ No newline at end of file diff --git a/lib/Makefile.inc b/lib/Makefile.inc index 21b221772..258c50d14 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -6,7 +6,7 @@ LIB_SRCS = $(addprefix lib/nodes/, file.c cbuilder.c) \ $(addprefix lib/kernel/, kernel.c rt.c) \ $(addprefix lib/, sample.c path.c node.c hooks.c \ log.c utils.c cfg.c hist.c timing.c pool.c list.c \ - queue.c memory.c \ + queue.c memory.c stats.c \ ) \ $(wildcard lib/hooks/*.c) \ diff --git a/lib/hooks.c b/lib/hooks.c index ea4745e80..5ffcc9044 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -19,16 +19,32 @@ struct list hooks; -/* Those references can be used inside the hook callbacks after initializing them with hook_init() */ -struct list *hook_nodes = NULL; -struct list *hook_paths = NULL; -struct settings *hook_settings = NULL; - -void hook_init(struct list *nodes, struct list *paths, struct settings *set) +int hook_init(struct hook *h, struct list *nodes, struct list *paths, struct settings *settings) { - hook_nodes = nodes; - hook_paths = paths; - hook_settings = set; + struct hook_info i = { + .paths = paths, + .nodes = nodes, + .settings = settings + }; + + return h->cb(h, HOOK_INIT, &i); +} + +void hook_destroy(struct hook *h) +{ + struct hook_info i = { NULL }; + h->cb(h, HOOK_DESTROY, &i); +} + +int hook_copy(struct hook *h, struct hook *c) +{ + memcpy(c, h, sizeof(struct hook)); + + c->_vd = + c->prev = + c->last = NULL; + + return 0; } int hooks_sort_priority(const void *a, const void *b) { @@ -40,11 +56,17 @@ int hooks_sort_priority(const void *a, const void *b) { int hook_run(struct path *p, struct sample *smps[], size_t cnt, int when) { + struct hook_info i = { + .path = p, + .smps = smps, + .cnt = cnt + }; + list_foreach(struct hook *h, &p->hooks) { if (h->type & when) { debug(DBG_HOOK | 22, "Running hook when=%u '%s' prio=%u, cnt=%zu", when, h->name, h->priority, cnt); - cnt = h->cb(p, h, when, smps, cnt); + cnt = h->cb(h, when, &i); if (cnt == 0) break; } @@ -60,7 +82,7 @@ void * hook_storage(struct hook *h, int when, size_t len) h->_vd = alloc(len); break; - case HOOK_DEINIT: + case HOOK_DESTROY: free(h->_vd); h->_vd = NULL; break; diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index 26958703c..070e68359 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -13,75 +13,78 @@ #include "utils.h" REGISTER_HOOK("fix_ts", "Update timestamps of sample if not set", 0, 0, hook_fix_ts, HOOK_INTERNAL | HOOK_READ) -int hook_fix_ts(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_fix_ts(struct hook *h, int when, struct hook_info *j) { struct timespec now = time_now(); + + assert(j->smps); - for (int i = 0; i < cnt; i++) { + for (int i = 0; i < j->cnt; i++) { /* Check for missing receive timestamp * Usually node_type::read() should update the receive timestamp. * An example would be to use hardware timestamp capabilities of * modern NICs. */ - if ((smps[i]->ts.received.tv_sec == 0 && smps[i]->ts.received.tv_nsec == 0) || - (smps[i]->ts.received.tv_sec == -1 && smps[i]->ts.received.tv_nsec == -1)) - smps[i]->ts.received = now; + if ((j->smps[i]->ts.received.tv_sec == 0 && j->smps[i]->ts.received.tv_nsec == 0) || + (j->smps[i]->ts.received.tv_sec == -1 && j->smps[i]->ts.received.tv_nsec == -1)) + j->smps[i]->ts.received = now; /* Check for missing origin timestamp */ - if ((smps[i]->ts.origin.tv_sec == 0 && smps[i]->ts.origin.tv_nsec == 0) || - (smps[i]->ts.origin.tv_sec == -1 && smps[i]->ts.origin.tv_nsec == -1)) - smps[i]->ts.origin = now; + if ((j->smps[i]->ts.origin.tv_sec == 0 && j->smps[i]->ts.origin.tv_nsec == 0) || + (j->smps[i]->ts.origin.tv_sec == -1 && j->smps[i]->ts.origin.tv_nsec == -1)) + j->smps[i]->ts.origin = now; } - return cnt; + return j->cnt; } REGISTER_HOOK("restart", "Call restart hooks for current path", 1, 1, hook_restart, HOOK_INTERNAL | HOOK_READ) -int hook_restart(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_restart(struct hook *h, int when, struct hook_info *j) { - for (int i = 0; i < cnt; i++) { - h->last = smps[i]; + assert(j->smps); + assert(j->path); + + for (int i = 0; i < j->cnt; i++) { + h->last = j->smps[i]; if (h->prev) { if (h->last->sequence == 0 && h->prev->sequence <= UINT32_MAX - 32) { warn("Simulation for path %s restarted (prev->seq=%u, current->seq=%u)", - path_name(p), h->prev->sequence, h->last->sequence); + path_name(j->path), h->prev->sequence, h->last->sequence); - p->invalid = - p->skipped = - p->dropped = 0; - - hook_run(p, &smps[i], cnt - i, HOOK_PATH_RESTART); + hook_run(j->path, &j->smps[i], j->cnt - i, HOOK_PATH_RESTART); } } h->prev = h->last; } - return cnt; + return j->cnt; } REGISTER_HOOK("drop", "Drop messages with reordered sequence numbers", 3, 1, hook_drop, HOOK_INTERNAL | HOOK_READ) -int hook_drop(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_drop(struct hook *h, int when, struct hook_info *j) { int i, ok, dist; + + assert(j->smps); - for (i = 0, ok = 0; i < cnt; i++) { - h->last = smps[i]; + for (i = 0, ok = 0; i < j->cnt; i++) { + h->last = j->smps[i]; if (h->prev) { dist = h->last->sequence - (int32_t) h->prev->sequence; if (dist <= 0) { - p->dropped++; + j->path->stats->counter.dropped++; warn("Dropped sample: dist = %d, i = %d", dist, i); } else { struct sample *tmp; - tmp = smps[i]; - smps[i] = smps[ok]; - smps[ok++] = tmp; + tmp = j->smps[i]; + j->smps[i] = j->smps[ok]; + j->smps[ok++] = tmp; } /* To discard the first X samples in 'smps[]' we must @@ -93,11 +96,11 @@ int hook_drop(struct path *p, struct hook *h, int when, struct sample *smps[], s else { struct sample *tmp; - tmp = smps[i]; - smps[i] = smps[ok]; - smps[ok++] = tmp; + tmp = j->smps[i]; + j->smps[i] = j->smps[ok]; + j->smps[ok++] = tmp; } - + h->prev = h->last; } diff --git a/lib/hooks/hooks-other.c b/lib/hooks/hooks-other.c index 21814ef29..86733cb19 100644 --- a/lib/hooks/hooks-other.c +++ b/lib/hooks/hooks-other.c @@ -17,25 +17,29 @@ #include "sample.h" REGISTER_HOOK("print", "Print the message to stdout", 99, 0, hook_print, HOOK_READ) -int hook_print(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_print(struct hook *h, int when, struct hook_info *j) { - for (int i = 0; i < cnt; i++) - sample_fprint(stdout, smps[i], SAMPLE_ALL); + assert(j->smps); + + for (int i = 0; i < j->cnt; i++) + sample_fprint(stdout, j->smps[i], SAMPLE_ALL); - return cnt; + return j->cnt; } REGISTER_HOOK("ts", "Update timestamp of message with current time", 99, 0, hook_ts, HOOK_READ) -int hook_ts(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_ts(struct hook *h, int when, struct hook_info *j) { - for (int i = 0; i < cnt; i++) - smps[i]->ts.origin = smps[i]->ts.received; + assert(j->smps); - return cnt; + for (int i = 0; i < j->cnt; i++) + j->smps[i]->ts.origin = j->smps[i]->ts.received; + + return j->cnt; } -REGISTER_HOOK("convert", "Convert message from / to floating-point / integer", 99, 0, hook_convert, HOOK_STORAGE | HOOK_PARSE | HOOK_READ) -int hook_convert(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +REGISTER_HOOK("convert", "Convert message from / to floating-point / integer", 99, 0, hook_convert, HOOK_STORAGE | HOOK_DESTROY | HOOK_READ) +int hook_convert(struct hook *h, int when, struct hook_info *k) { struct { enum { @@ -45,7 +49,7 @@ int hook_convert(struct path *p, struct hook *h, int when, struct sample *smps[] } *private = hook_storage(h, when, sizeof(*private)); switch (when) { - case HOOK_PARSE: + case HOOK_DESTROY: if (!h->parameter) error("Missing parameter for hook: '%s'", h->name); @@ -58,30 +62,29 @@ int hook_convert(struct path *p, struct hook *h, int when, struct sample *smps[] break; case HOOK_READ: - for (int i = 0; i < cnt; i++) { - for (int j = 0; j < smps[0]->length; j++) { + for (int i = 0; i < k->cnt; i++) { + for (int j = 0; j < k->smps[0]->length; j++) { switch (private->mode) { - case TO_FIXED: smps[i]->data[j].i = smps[i]->data[j].f * 1e3; break; - case TO_FLOAT: smps[i]->data[j].f = smps[i]->data[j].i; break; + case TO_FIXED: k->smps[i]->data[j].i = k->smps[i]->data[j].f * 1e3; break; + case TO_FLOAT: k->smps[i]->data[j].f = k->smps[i]->data[j].i; break; } } } - break; + + return k->cnt; } return 0; } -REGISTER_HOOK("decimate", "Downsamping by integer factor", 99, 0, hook_decimate, HOOK_STORAGE | HOOK_PARSE | HOOK_READ) -int hook_decimate(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +REGISTER_HOOK("decimate", "Downsamping by integer factor", 99, 0, hook_decimate, HOOK_STORAGE | HOOK_DESTROY | HOOK_READ) +int hook_decimate(struct hook *h, int when, struct hook_info *j) { struct { unsigned ratio; unsigned counter; } *private = hook_storage(h, when, sizeof(*private)); - int ok; - switch (when) { case HOOK_PARSE: if (!h->parameter) @@ -95,24 +98,27 @@ int hook_decimate(struct path *p, struct hook *h, int when, struct sample *smps[ break; case HOOK_READ: - ok = 0; - for (int i = 0; i < cnt; i++) { + assert(j->smps); + + int i, ok; + for (i = 0, ok = 0; i < j->cnt; i++) { if (private->counter++ % private->ratio == 0) { struct sample *tmp; - tmp = smps[ok]; - smps[ok++] = smps[i]; - smps[i] = tmp; + tmp = j->smps[ok]; + j->smps[ok++] = j->smps[i]; + j->smps[i] = tmp; } } + return ok; } - return cnt; + return 0; } REGISTER_HOOK("skip_first", "Skip the first samples", 99, 0, hook_skip_first, HOOK_STORAGE | HOOK_PARSE | HOOK_READ | HOOK_PATH) -int hook_skip_first(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_skip_first(struct hook *h, int when, struct hook_info *j) { struct { struct timespec skip; /**< Time to wait until first message is not skipped */ @@ -121,7 +127,6 @@ int hook_skip_first(struct path *p, struct hook *h, int when, struct sample *smp char *endptr; double wait; - int i, ok; switch (when) { case HOOK_PARSE: @@ -135,19 +140,22 @@ int hook_skip_first(struct path *p, struct hook *h, int when, struct sample *smp private->skip = time_from_double(wait); break; + case HOOK_PATH_START: case HOOK_PATH_RESTART: - case HOOK_PATH_STOP: - private->until = time_add(&smps[0]->ts.received, &private->skip); + private->until = time_add(&j->smps[0]->ts.received, &private->skip); break; case HOOK_READ: - for (i = 0, ok = 0; i < cnt; i++) { - if (time_delta(&private->until, &smps[i]->ts.received) > 0) { + assert(j->smps); + + int i, ok; + for (i = 0, ok = 0; i < j->cnt; i++) { + if (time_delta(&private->until, &j->smps[i]->ts.received) > 0) { struct sample *tmp; - tmp = smps[i]; - smps[i] = smps[ok]; - smps[ok++] = tmp; + tmp = j->smps[i]; + j->smps[i] = j->smps[ok]; + j->smps[ok++] = tmp; } diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index ff5fbd268..4b787b78f 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -10,88 +10,51 @@ #include "sample.h" #include "path.h" #include "utils.h" -#include "timing.h" +#include "stats.h" extern struct list *hook_nodes; -void hook_stats_header() -{ - #define UNIT(u) "(" YEL(u) ")" - - stats("%-40s|%19s|%19s|%19s|%19s|%19s|", "Source " MAG("=>") " Destination", - "OWD" UNIT("S") " ", - "Rate" UNIT("p/S") " ", - "Drop" UNIT("p") " ", - "Skip" UNIT("p") " ", - "Inval" UNIT("p") " " - ); - line(); -} - REGISTER_HOOK("stats", "Collect statistics for the current path", 2, 1, hook_stats, HOOK_STATS) -int hook_stats(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +int hook_stats(struct hook *h, int when, struct hook_info *j) { + struct stats *s = hook_storage(h, when, sizeof(struct stats)); + switch (when) { case HOOK_INIT: - /** @todo Allow configurable bounds for histograms */ - hist_create(&p->hist.owd, 0, 1, 100e-3); - hist_create(&p->hist.gap_msg, 90e-3, 110e-3, 1e-3); - hist_create(&p->hist.gap_recv, 90e-3, 110e-3, 1e-3); - hist_create(&p->hist.gap_seq, -HIST_SEQ, +HIST_SEQ, 1); + stats_init(s); break; case HOOK_READ: - for (int i = 0; i < cnt; i++) { - h->last = smps[i]; - - if (h->prev) { - int gap_seq = h->last->sequence - (int32_t) h->prev->sequence; - double owd = time_delta(&h->last->ts.origin, &h->last->ts.received); - double gap = time_delta(&h->prev->ts.origin, &h->last->ts.origin); - double gap_recv = time_delta(&h->prev->ts.received, &h->last->ts.received); - - hist_put(&p->hist.gap_msg, gap); - hist_put(&p->hist.gap_seq, gap_seq); - hist_put(&p->hist.owd, owd); - hist_put(&p->hist.gap_recv, gap_recv); - } - - h->prev = h->last; - } + assert(j->smps); + + stats_collect(s, j->smps, j->cnt); break; case HOOK_PATH_STOP: - if (p->hist.owd.total) { info("One-way delay:"); hist_print(&p->hist.owd); } - if (p->hist.gap_recv.total){ info("Inter-message arrival time:"); hist_print(&p->hist.gap_recv); } - if (p->hist.gap_msg.total) { info("Inter-message ts gap:"); hist_print(&p->hist.gap_msg); } - if (p->hist.gap_seq.total) { info("Inter-message sequence number gaps:"); hist_print(&p->hist.gap_seq); } + stats_print(s); break; - case HOOK_DEINIT: - hist_destroy(&p->hist.owd); - hist_destroy(&p->hist.gap_msg); - hist_destroy(&p->hist.gap_recv); - hist_destroy(&p->hist.gap_seq); + case HOOK_DESTROY: + stats_destroy(s); break; case HOOK_PATH_RESTART: - hist_reset(&p->hist.owd); - hist_reset(&p->hist.gap_seq); - hist_reset(&p->hist.gap_msg); - hist_reset(&p->hist.gap_recv); + stats_reset(s); break; case HOOK_PERIODIC: - stats("%-40.40s|%10s|%10s|%10ju|%10ju|%10ju|", path_name(p), "", "", - p->dropped, p->skipped, p->invalid); + assert(j->path); + + stats_print_periodic(s, j->path); break; } - return cnt; + return j->cnt; } -REGISTER_HOOK("stats_send", "Send path statistics to another node", 99, 0, hook_stats_send, HOOK_STORAGE | HOOK_PARSE | HOOK_PERIODIC | HOOK_PATH) -int hook_stats_send(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +#if 0 /* currently broken */ +REGISTER_HOOK("stats_send", "Send path statistics to another node", 99, 0, hook_stats_send, HOOK_STORAGE | HOOK_DESTROY | HOOK_PERIODIC | HOOK_PATH) +int hook_stats_send(struct hook *h, int when, struct hook_info *i) { struct private { struct node *dest; @@ -99,7 +62,7 @@ int hook_stats_send(struct path *p, struct hook *h, int when, struct sample *smp } *private = hook_storage(h, when, sizeof(*private)); switch (when) { - case HOOK_PARSE: + case HOOK_DESTROY: if (!h->parameter) error("Missing parameter for hook '%s'", h->name); @@ -115,23 +78,11 @@ int hook_stats_send(struct path *p, struct hook *h, int when, struct sample *smp break; case HOOK_PERIODIC: { - int i; - char buf[SAMPLE_LEN(16)]; - struct sample *smp = (struct sample *) buf; - - i = 0; - smp->data[i++].f = p->invalid; - smp->data[i++].f = p->skipped; - smp->data[i++].f = p->dropped; - smp->data[i++].f = p->hist.owd.last, - smp->data[i++].f = 1.0 / p->hist.gap_msg.last; - smp->data[i++].f = 1.0 / p->hist.gap_recv.last; - smp->length = i; - - node_write(private->dest, &smp, 1); /* Send single message with statistics to destination node */ + stats_send(s, node); break; } } return 0; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/lib/stats.c b/lib/stats.c new file mode 100644 index 000000000..f25675d1d --- /dev/null +++ b/lib/stats.c @@ -0,0 +1,142 @@ +/** Statistic collection. + * + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + */ + +#include "stats.h" +#include "hist.h" +#include "timing.h" +#include "path.h" +#include "sample.h" +#include "log.h" + +int stats_init(struct stats *s) +{ + /** @todo Allow configurable bounds for histograms */ + hist_create(&s->histogram.owd, 0, 1, 100e-3); + hist_create(&s->histogram.gap_msg, 90e-3, 110e-3, 1e-3); + hist_create(&s->histogram.gap_recv, 90e-3, 110e-3, 1e-3); + hist_create(&s->histogram.gap_seq, -HIST_SEQ, +HIST_SEQ, 1); + + return 0; +} + +void stats_destroy(struct stats *s) +{ + hist_destroy(&s->histogram.owd); + hist_destroy(&s->histogram.gap_msg); + hist_destroy(&s->histogram.gap_recv); + hist_destroy(&s->histogram.gap_seq); +} + +void stats_collect(struct stats *s, struct sample *smps[], size_t cnt) +{ + for (int i = 0; i < cnt; i++) { + if (s->last) { + int gap_seq = smps[i]->sequence - (int32_t) s->last->sequence; + double owd = time_delta(&smps[i]->ts.origin, &smps[i]->ts.received); + double gap = time_delta(&s->last->ts.origin, &smps[i]->ts.origin); + double gap_recv = time_delta(&s->last->ts.received, &smps[i]->ts.received); + + hist_put(&s->histogram.owd, owd); + hist_put(&s->histogram.gap_msg, gap); + hist_put(&s->histogram.gap_seq, gap_seq); + hist_put(&s->histogram.gap_recv, gap_recv); + } + + if (i == 0 && s->last) + sample_put(s->last); + if (i == cnt - 1) + sample_get(smps[i]); + + s->last = smps[i]; + } +} + +#ifdef WITH_JANSSON +json_t * stats_json(struct stats *s) +{ + return json_pack("{ s: { s: i, s: i, s: i }, s: { s: o, s: o, s: o } }", + "counter", + "dropped", s->counter.dropped, + "invalid", s->counter.invalid, + "skipped", s->counter.skipped, + "histogram", + "owd", hist_json(&s->histogram.owd), + "gap_msg", hist_json(&s->histogram.gap_msg), + "gap_recv",hist_json(&s->histogram.gap_recv), + "gap_seq", hist_json(&s->histogram.gap_seq) + ); +} +#endif + +void stats_reset(struct stats *s) +{ + s->counter.invalid = + s->counter.skipped = + s->counter.dropped = 0; + + hist_reset(&s->histogram.owd); + hist_reset(&s->histogram.gap_seq); + hist_reset(&s->histogram.gap_msg); + hist_reset(&s->histogram.gap_recv); +} + +void stats_print_header() +{ + #define UNIT(u) "(" YEL(u) ")" + + stats("%-40s|%19s|%19s|%19s|%19s|%19s|", "Source " MAG("=>") " Destination", + "OWD" UNIT("S") " ", + "Rate" UNIT("p/S") " ", + "Drop" UNIT("p") " ", + "Skip" UNIT("p") " ", + "Inval" UNIT("p") " " + ); + line(); +} + +void stats_print_periodic(struct stats *s, struct path *p) +{ + stats("%-40.40s|%10s|%10s|%10ju|%10ju|%10ju|", path_name(p), "", "", + s->counter.dropped, s->counter.skipped, s->counter.invalid); +} + +void stats_print(struct stats *s) +{ + stats("Dropped samples: %ju", s->counter.dropped); + stats("Skipped samples: %ju", s->counter.skipped); + stats("Invalid samples: %ju", s->counter.invalid); + + stats("One-way delay:"); + hist_print(&s->histogram.owd); + + stats("Inter-message arrival time:"); + hist_print(&s->histogram.gap_recv); + + stats("Inter-message ts gap:"); + hist_print(&s->histogram.gap_msg); + + stats("Inter-message sequence number gaps:"); + hist_print(&s->histogram.gap_seq); +} + +void stats_send(struct stats *s, struct node *n) +{ + char buf[SAMPLE_LEN(16)]; + struct sample *smp = (struct sample *) buf; + + int i = 0; + smp->data[i++].f = s->counter.invalid; /**< Use integer here? */ + smp->data[i++].f = s->counter.skipped; + smp->data[i++].f = s->counter.dropped; + smp->data[i++].f = s->histogram.owd.last, + smp->data[i++].f = 1.0 / s->histogram.gap_msg.last; + smp->data[i++].f = 1.0 / s->histogram.gap_recv.last; + smp->length = i; + + node_write(n, &smp, 1); /* Send single message with statistics to destination node */ +} \ No newline at end of file From 5ba5164c1ac5a2406f21a7e435924483c5bd5ba1 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:43:36 -0400 Subject: [PATCH 21/57] added new tool to run hook functions with villas-pipe and villas-signal --- plugins/hooks/example_hook.c | 2 +- src/Makefile.inc | 3 +- src/hook.c | 121 +++++++++++++++++++++++++++++++++++ src/node.c | 9 +-- 4 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 src/hook.c diff --git a/plugins/hooks/example_hook.c b/plugins/hooks/example_hook.c index 843b2734c..22e1b48dc 100644 --- a/plugins/hooks/example_hook.c +++ b/plugins/hooks/example_hook.c @@ -1,7 +1,7 @@ #include #include -static int hook_example(struct path *p, struct hook *h, int when, struct sample *smps[], size_t cnt) +static int hook_example(struct hook *h, int when, struct hook_info *j) { info("Hello world from example hook!"); diff --git a/src/Makefile.inc b/src/Makefile.inc index 280efdbe2..94cd8fcdb 100644 --- a/src/Makefile.inc +++ b/src/Makefile.inc @@ -2,7 +2,8 @@ TARGETS = $(BUILDDIR)/villas-node \ $(BUILDDIR)/villas-pipe \ $(BUILDDIR)/villas-signal \ - $(BUILDDIR)/villas-test + $(BUILDDIR)/villas-test \ + $(BUILDDIR)/villas-hook SRC_LDLIBS = $(LDLIBS) -pthread -lm -lvillas SRC_CFLAGS = $(CFLAGS) diff --git a/src/hook.c b/src/hook.c new file mode 100644 index 000000000..a68a8bfae --- /dev/null +++ b/src/hook.c @@ -0,0 +1,121 @@ +/** Receive messages from server snd print them on stdout. + * + * @file + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @addtogroup tools Test and debug tools + * @{ + *********************************************************************************/ + +#include +#include + +#include +#include +#include +#include +#include +#include + +static void usage() +{ + printf("Usage: villas-hook [OPTIONS] NAME [PARAMETER] \n"); + printf(" NAME the name of the hook function to run\n"); + printf(" PARAM the name of the node to which samples are sent and received from\n\n"); + printf(" OPTIONS are:\n"); + printf(" -h show this help\n"); + printf(" -d lvl set debug level\n"); + printf(" -v process multiple samples at once\n\n"); + + print_copyright(); +} + +int main(int argc, char *argv[]) +{ + int j, ret, cnt = 1; + + char *name, *parameter; + + struct hook *h; + struct hook_info *i = alloc(sizeof(struct hook_info)); + + char c; + while ((c = getopt(argc, argv, "hv:d:")) != -1) { + switch (c) { + case 'v': + cnt = atoi(optarg); + break; + case 'd': + log_setlevel(atoi(optarg), -1); + break; + case 'h': + case '?': + usage(argv[0]); + exit(EXIT_SUCCESS); + } + } + + if (argc <= optind) + + name = argc > optind ? argv[optind ] : NULL; + parameter = argc > optind + 1 ? argv[optind+1] : NULL; + + if (argc > optind) + name = argv[optind]; + else { + usage(argv[0]); + exit(EXIT_FAILURE); + } + + if (argc > optind + 1) + parameter = argv[optind + 1]; + + h = list_lookup(&hooks, name); + if (!h) + error("Unknown hook function '%s'", argv[optind]); + + if (cnt < 1) + error("Vectorize option must be greater than 0"); + + struct pool pool; + struct sample *smps[cnt]; + + ret = pool_init(&pool, 10 * cnt, SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); + if (ret) + error("Failed to initilize memory pool"); + + ret = sample_alloc(&pool, smps, cnt); + if (ret) + error("Failed to allocate %u samples from pool", cnt); + + h->parameter = parameter; + i->smps = smps; + + h->cb(h, HOOK_INIT, i); + h->cb(h, HOOK_PARSE, i); + h->cb(h, HOOK_PATH_START, i); + + while (!feof(stdin)) { + for (j = 0; j < cnt && !feof(stdin); j++) + sample_fscan(stdin, i->smps[j], NULL); + + i->cnt = j; + i->cnt = h->cb(h, HOOK_READ, i); + i->cnt = h->cb(h, HOOK_WRITE, i); + + for (j = 0; j < i->cnt; j++) + sample_fprint(stdout, i->smps[j], SAMPLE_ALL); + } + + h->cb(h, HOOK_PATH_STOP, i); + h->cb(h, HOOK_DESTROY, i); + + sample_free(smps, cnt); + + pool_destroy(&pool); + + return 0; +} \ No newline at end of file diff --git a/src/node.c b/src/node.c index 4f89f6bbf..93c42b1fc 100644 --- a/src/node.c +++ b/src/node.c @@ -128,9 +128,6 @@ int main(int argc, char *argv[]) info("Initialize signals"); signals_init(); - info("Initialize hook sub-system"); - hook_init(&nodes, &paths, &settings); - info("Initialize node types"); list_foreach(struct node_type *vt, &node_types) { INDENT int refs = list_length(&vt->instances); @@ -151,6 +148,10 @@ int main(int argc, char *argv[]) list_foreach(struct path *p, &paths) { INDENT if (p->enabled) { path_init(p); + + list_foreach(struct hook *h, &p->hooks) + hook_init(h, &nodes, &paths, &settings); + path_start(p); } else @@ -159,7 +160,7 @@ int main(int argc, char *argv[]) /* Run! */ if (settings.stats > 0) { - hook_stats_header(); + stats_print_header(); for (;;) { list_foreach(struct path *p, &paths) From f82fc64990d0bceed27271d2b121787e7f857b55 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:43:55 -0400 Subject: [PATCH 22/57] removed two unused functions --- include/villas/utils.h | 6 ------ lib/utils.c | 37 ------------------------------------- 2 files changed, 43 deletions(-) diff --git a/include/villas/utils.h b/include/villas/utils.h index 82b7cabb7..87f2a6c69 100644 --- a/include/villas/utils.h +++ b/include/villas/utils.h @@ -197,12 +197,6 @@ int version_parse(const char *s, struct version *v); /** Fill buffer with random data */ int read_random(char *buf, size_t len); -/** Hexdump bytes */ -void printb(void *mem, size_t len); - -/** Hexdump 32-bit dwords */ -void printdw(void *mem, size_t len); - /** Get CPU timestep counter */ __attribute__((always_inline)) static inline uint64_t rdtsc() { diff --git a/lib/utils.c b/lib/utils.c index eff72a954..2c7fb04fd 100644 --- a/lib/utils.c +++ b/lib/utils.c @@ -329,43 +329,6 @@ out: return -1; } -void printb(void *mem, size_t len) -{ - uint8_t *mem8 = (uint8_t *) mem; - - for (int i = 0; i < len; i++) { - printf("%02hx ", mem8[i]); - - if (i % 16 == 15) - printf("\n"); - } -} - -void printdw(void *mem, size_t len) -{ - int columns = 4; - - uint32_t *mem32 = (uint32_t *) mem; - - for (int i = 0; i < len; i++) { - if (i % columns == 0) - printf("%#x: ", i * 4); - - printf("%08x ", mem32[i]); - - char *memc = (char *) &mem32[i]; - printf("%c%c%c%c ", - isprint(memc[0]) ? memc[0] : ' ', - isprint(memc[1]) ? memc[1] : ' ', - isprint(memc[2]) ? memc[2] : ' ', - isprint(memc[3]) ? memc[3] : ' ' - ); - - if ((i+1) % columns == 0) - printf("\n"); - } -} - void rdtsc_sleep(uint64_t nanosecs, uint64_t start) { uint64_t cycles; From d5c844f18d1dd811877d4837ad306f4839483158 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:44:18 -0400 Subject: [PATCH 23/57] added LOG2_CEIL --- include/villas/utils.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/include/villas/utils.h b/include/villas/utils.h index 87f2a6c69..4207d3626 100644 --- a/include/villas/utils.h +++ b/include/villas/utils.h @@ -55,7 +55,11 @@ #define ALIGN_MASK(x, m) (((uintptr_t) (x) + (m)) & ~(m)) #define IS_ALIGNED(x, a) (ALIGN(x, a) == (uintptr_t) x) -#define CEIL(x, y) ((x + y - 1) / y) +/** Round-up integer division */ +#define CEIL(x, y) (((x) + (y) - 1) / (y)) + +/** Get nearest up-rounded power of 2 */ +#define LOG2_CEIL(x) (1 << log2i((x) - 1) + 1) /** Calculate the number of elements in an array. */ #define ARRAY_LEN(a) ( sizeof (a) / sizeof (a)[0] ) @@ -212,6 +216,13 @@ __attribute__((always_inline)) static inline uint64_t rdtsc() return tsc; } +/** Get log2 of long long integers */ +static inline int log2i(long long x) { + assert(x > 0); + + return sizeof(x) * 8 - __builtin_clzll(x) - 1; +} + /** Sleep with rdtsc */ void rdtsc_sleep(uint64_t nanosecs, uint64_t start); From efdc5cdff3497df222d47e81b28fae4185426c5a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 22 Oct 2016 20:47:36 -0400 Subject: [PATCH 24/57] added sample_{alloc,free} --- include/villas/sample.h | 9 +++++++-- lib/sample.c | 12 ++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/include/villas/sample.h b/include/villas/sample.h index 5d2db263d..7870d3cd9 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -60,8 +60,13 @@ struct sample { } data[]; }; -/** Request \p cnt samples from memory pool \p p and initialize them. */ -int sample_get_many(struct pool *p, struct sample *smps[], int cnt); +/** Request \p cnt samples from memory pool \p p and initialize them. + * This will leave the reference count of the sample to zero. + * Use the sample_get() function to increase it. */ +int sample_alloc(struct pool *p, struct sample *smps[], int cnt); + +/** Release an array of samples back to their pools */ +void sample_free(struct sample *smps[], int cnt); /** Increase reference count of sample */ int sample_get(struct sample *s); diff --git a/lib/sample.c b/lib/sample.c index f3040d3a0..058a2ad30 100644 --- a/lib/sample.c +++ b/lib/sample.c @@ -12,19 +12,27 @@ #include "sample.h" #include "timing.h" -int sample_get_many(struct pool *p, struct sample *smps[], int cnt) { +int sample_alloc(struct pool *p, struct sample *smps[], int cnt) { int ret; ret = pool_get_many(p, (void **) smps, cnt); if (ret < 0) return ret; - for (int i = 0; i < ret; i++) + for (int i = 0; i < ret; i++) { smps[i]->capacity = (p->blocksz - sizeof(**smps)) / sizeof(smps[0]->data[0]); + smps[i]->pool = p; + } return ret; } +void sample_free(struct sample *smps[], int cnt) +{ + for (int i = 0; i < cnt; i++) + pool_put(smps[i]->pool, smps[i]); +} + int sample_get(struct sample *s) { return atomic_fetch_add(&s->refcnt, 1) + 1; From 800e71bb96b891e1b245d6cd5c908dbfbb47eaae Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:05:29 -0400 Subject: [PATCH 25/57] proper return codes for incorrect CLI arguments --- src/hook.c | 4 ++-- src/pipe.c | 1 + src/signal.c | 1 + src/test.c | 11 ++--------- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/hook.c b/src/hook.c index a68a8bfae..7b04cd27e 100644 --- a/src/hook.c +++ b/src/hook.c @@ -53,8 +53,8 @@ int main(int argc, char *argv[]) break; case 'h': case '?': - usage(argv[0]); - exit(EXIT_SUCCESS); + usage(); + exit(c == '?' ? EXIT_FAILURE : EXIT_SUCCESS); } } diff --git a/src/pipe.c b/src/pipe.c index 626536b08..9cdb1cac1 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -203,6 +203,7 @@ int main(int argc, char *argv[]) case 'h': case '?': usage(); + exit(c == '?' ? EXIT_FAILURE : EXIT_SUCCESS); } } diff --git a/src/signal.c b/src/signal.c index bce67fadd..2d47dfa95 100644 --- a/src/signal.c +++ b/src/signal.c @@ -102,6 +102,7 @@ int main(int argc, char *argv[]) case 'h': case '?': usage(); + exit(c == '?' ? EXIT_FAILURE : EXIT_SUCCESS); } continue; diff --git a/src/test.c b/src/test.c index 4f9e351b9..a05e4b9ea 100644 --- a/src/test.c +++ b/src/test.c @@ -114,15 +114,8 @@ int main(int argc, char *argv[]) res = strtod(optarg, &endptr); goto check; case '?': - if (optopt == 'c') - error("Option -%c requires an argument.", optopt); - else if (isprint(optopt)) - error("Unknown option '-%c'.", optopt); - else - error("Unknown option character '\\x%x'.", optopt); - exit(EXIT_FAILURE); - default: - abort(); + usage(); + exit(c == '?' ? EXIT_FAILURE : EXIT_SUCCESS); } continue; From 510aec76309ba29cdb24b126930ff7946fda05ac Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:52:06 -0400 Subject: [PATCH 26/57] added constructor and destructor methods to hook_storage() management --- include/villas/hooks.h | 11 ++++++++++- lib/hooks.c | 9 ++++++++- lib/hooks/hooks-other.c | 8 ++++---- lib/hooks/hooks-stats.c | 12 ++---------- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/include/villas/hooks.h b/include/villas/hooks.h index 8dbd4c09c..110e01ba6 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -74,6 +74,15 @@ struct hook_info { */ typedef int (*hook_cb_t)(struct hook *h, int when, struct hook_info *i); +/** Destructor callback for hook_storage() + * + * @param data A pointer to the data which should be destroyed. + */ +typedef void (*dtor_cb_t)(void *); + +/** Constructor callback for hook_storage() */ +typedef int (*ctor_cb_t)(void *); + enum hook_state { HOOK_DESTROYED, HOOK_INITIALIZED @@ -157,7 +166,7 @@ int hook_run(struct path *p, struct sample *smps[], size_t cnt, int when); * @param len The size of hook prvate memory allocation. * @return A pointer to the allocated memory region or NULL after it was released. */ -void * hook_storage(struct hook *h, int when, size_t len); +void * hook_storage(struct hook *h, int when, size_t len, ctor_cb_t ctor, dtor_cb_t dtor); int hook_print(struct hook *h, int when, struct hook_info *j); int hook_ts(struct hook *h, int when, struct hook_info *j); diff --git a/lib/hooks.c b/lib/hooks.c index 5ffcc9044..daad04c42 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -75,14 +75,21 @@ int hook_run(struct path *p, struct sample *smps[], size_t cnt, int when) return cnt; } -void * hook_storage(struct hook *h, int when, size_t len) +void * hook_storage(struct hook *h, int when, size_t len, ctor_cb_t ctor, dtor_cb_t dtor) { switch (when) { case HOOK_INIT: h->_vd = alloc(len); + + if (ctor) + ctor(h->_vd); + break; case HOOK_DESTROY: + if (dtor) + dtor(h->_vd); + free(h->_vd); h->_vd = NULL; break; diff --git a/lib/hooks/hooks-other.c b/lib/hooks/hooks-other.c index 86733cb19..0a63ea5fa 100644 --- a/lib/hooks/hooks-other.c +++ b/lib/hooks/hooks-other.c @@ -46,10 +46,10 @@ int hook_convert(struct hook *h, int when, struct hook_info *k) TO_FIXED, TO_FLOAT } mode; - } *private = hook_storage(h, when, sizeof(*private)); + } *private = hook_storage(h, when, sizeof(*private), NULL, NULL); switch (when) { - case HOOK_DESTROY: + case HOOK_PARSE: if (!h->parameter) error("Missing parameter for hook: '%s'", h->name); @@ -83,7 +83,7 @@ int hook_decimate(struct hook *h, int when, struct hook_info *j) struct { unsigned ratio; unsigned counter; - } *private = hook_storage(h, when, sizeof(*private)); + } *private = hook_storage(h, when, sizeof(*private), NULL, NULL); switch (when) { case HOOK_PARSE: @@ -123,7 +123,7 @@ int hook_skip_first(struct hook *h, int when, struct hook_info *j) struct { struct timespec skip; /**< Time to wait until first message is not skipped */ struct timespec until; /**< Absolute point in time from where we accept samples. */ - } *private = hook_storage(h, when, sizeof(*private)); + } *private = hook_storage(h, when, sizeof(*private), NULL, NULL); char *endptr; double wait; diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index 4b787b78f..e0139efff 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -17,13 +17,9 @@ extern struct list *hook_nodes; REGISTER_HOOK("stats", "Collect statistics for the current path", 2, 1, hook_stats, HOOK_STATS) int hook_stats(struct hook *h, int when, struct hook_info *j) { - struct stats *s = hook_storage(h, when, sizeof(struct stats)); + struct stats *s = hook_storage(h, when, sizeof(struct stats), (ctor_cb_t) stats_init, (dtor_cb_t) stats_destroy); switch (when) { - case HOOK_INIT: - stats_init(s); - break; - case HOOK_READ: assert(j->smps); @@ -31,11 +27,7 @@ int hook_stats(struct hook *h, int when, struct hook_info *j) break; case HOOK_PATH_STOP: - stats_print(s); - break; - - case HOOK_DESTROY: - stats_destroy(s); + stats_print(s, 1); break; case HOOK_PATH_RESTART: From 693329cb68303456ebd5dce1e5b4ad475d18d78f Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:54:20 -0400 Subject: [PATCH 27/57] use hist_cnt_t throughout the histogram code --- include/villas/hist.h | 2 +- lib/hist.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/villas/hist.h b/include/villas/hist.h index 11be82119..a410e3649 100644 --- a/include/villas/hist.h +++ b/include/villas/hist.h @@ -21,7 +21,7 @@ #define HIST_HEIGHT (LOG_WIDTH - 55) #define HIST_SEQ 17 -typedef unsigned hist_cnt_t; +typedef uintmax_t hist_cnt_t; /** Histogram structure used to collect statistics. */ struct hist { diff --git a/lib/hist.c b/lib/hist.c index 33bbce263..a1ff40aac 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -103,7 +103,7 @@ double hist_stddev(struct hist *h) void hist_print(struct hist *h) { INDENT - stats("Counted values: %u (%u between %f and %f)", h->total, h->total-h->higher-h->lower, h->high, h->low); + stats("Counted values: %ju (%ju between %f and %f)", h->total, h->total-h->higher-h->lower, h->high, h->low); stats("Highest: %f Lowest: %f", h->highest, h->lowest); stats("Mu: %f Sigma2: %f Sigma: %f", hist_mean(h), hist_var(h), hist_stddev(h)); @@ -135,11 +135,11 @@ void hist_plot(struct hist *h) for (int i = 0; i < h->length; i++) { double value = VAL(h, i); - int cnt = h->data[i]; + hist_cnt_t cnt = h->data[i]; int bar = HIST_HEIGHT * ((double) cnt / max); if (value > h->lowest || value < h->highest) - stats("%+9g | " "%5u" " | %.*s", value, cnt, bar, buf); + stats("%+9g | %5ju | %.*s", value, cnt, bar, buf); } } @@ -150,7 +150,7 @@ char * hist_dump(struct hist *h) strcatf(&buf, "[ "); for (int i = 0; i < h->length; i++) - strcatf(&buf, "%u ", h->data[i]); + strcatf(&buf, "%ju ", h->data[i]); strcatf(&buf, "]"); @@ -199,9 +199,9 @@ int hist_dump_matlab(struct hist *h, FILE *f) fprintf(f, "%lu = struct( ", time(NULL)); fprintf(f, "'low', %f, ", h->low); fprintf(f, "'high', %f, ", h->high); - fprintf(f, "'total', %u, ", h->total); - fprintf(f, "'higher', %u, ", h->higher); - fprintf(f, "'lower', %u, ", h->lower); + fprintf(f, "'total', %ju, ", h->total); + fprintf(f, "'higher', %ju, ", h->higher); + fprintf(f, "'lower', %ju, ", h->lower); fprintf(f, "'highest', %f, ", h->highest); fprintf(f, "'lowest', %f, ", h->lowest); fprintf(f, "'mean', %f, ", hist_mean(h)); From c0d5bbb47b8f26914fa256bb835caa8369a13a67 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:55:12 -0400 Subject: [PATCH 28/57] add a couple of NULL pointer checks for bucket-less histograms --- lib/hist.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/hist.c b/lib/hist.c index a1ff40aac..b57eaacb0 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -24,15 +24,25 @@ void hist_create(struct hist *h, double low, double high, double resolution) h->low = low; h->high = high; h->resolution = resolution; - h->length = (high - low) / resolution; - h->data = alloc(h->length * sizeof(unsigned)); + + if (resolution > 0) { + h->length = (high - low) / resolution; + h->data = alloc(h->length * sizeof(unsigned)); + } + else { + h->length = 0; + h->data = NULL; + } hist_reset(h); } void hist_destroy(struct hist *h) { - free(h->data); + if (h->data) { + free(h->data); + h->data = NULL; + } } void hist_put(struct hist *h, double value) @@ -52,7 +62,7 @@ void hist_put(struct hist *h, double value) h->higher++; else if (idx < 0) h->lower++; - else + else if (h->data != NULL) h->data[idx]++; h->total++; @@ -83,7 +93,8 @@ void hist_reset(struct hist *h) h->highest = DBL_MIN; h->lowest = DBL_MAX; - memset(h->data, 0, h->length * sizeof(unsigned)); + if (h->data) + memset(h->data, 0, h->length * sizeof(unsigned)); } double hist_mean(struct hist *h) From 3d694cf4d921b8e202dd68480e8521835f607faf Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:55:56 -0400 Subject: [PATCH 29/57] add possibility to print histogram in different verbosity levels --- include/villas/hist.h | 2 +- lib/hist.c | 4 ++-- src/test.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/villas/hist.h b/include/villas/hist.h index a410e3649..abb3ea091 100644 --- a/include/villas/hist.h +++ b/include/villas/hist.h @@ -67,7 +67,7 @@ double hist_mean(struct hist *h); double hist_stddev(struct hist *h); /** Print all statistical properties of distribution including a graphilcal plot of the histogram. */ -void hist_print(struct hist *h); +void hist_print(struct hist *h, int details); /** Print ASCII style plot of histogram */ void hist_plot(struct hist *h); diff --git a/lib/hist.c b/lib/hist.c index b57eaacb0..21baa6784 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -112,13 +112,13 @@ double hist_stddev(struct hist *h) return sqrt(hist_var(h)); } -void hist_print(struct hist *h) +void hist_print(struct hist *h, int details) { INDENT stats("Counted values: %ju (%ju between %f and %f)", h->total, h->total-h->higher-h->lower, h->high, h->low); stats("Highest: %f Lowest: %f", h->highest, h->lowest); stats("Mu: %f Sigma2: %f Sigma: %f", hist_mean(h), hist_var(h), hist_stddev(h)); - if (h->total - h->higher - h->lower > 0) { + if (details > 0 && h->total - h->higher - h->lower > 0) { char *buf = hist_dump(h); stats("Matlab: %s", buf); free(buf); diff --git a/src/test.c b/src/test.c index a05e4b9ea..bb31796d0 100644 --- a/src/test.c +++ b/src/test.c @@ -182,7 +182,7 @@ void test_rtt() { else error("Invalid file descriptor: %u", fd); - hist_print(&hist); + hist_print(&hist, 1); hist_destroy(&hist); } From 0d58df66e75f36192bb14a06000dddf04754d0e7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:57:05 -0400 Subject: [PATCH 30/57] fixed segmentation fault in villas-pipe --- src/pipe.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pipe.c b/src/pipe.c index afd57e7b6..2226c917d 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -49,15 +49,14 @@ static void quit(int signal, siginfo_t *sinfo, void *ctx) if (recvv.started) { pthread_cancel(recvv.thread); pthread_join(recvv.thread, NULL); + pool_destroy(&recvv.pool); } if (sendd.started) { pthread_cancel(sendd.thread); pthread_join(sendd.thread, NULL); + pool_destroy(&sendd.pool); } - - pool_destroy(&recvv.pool); - pool_destroy(&sendd.pool); node_stop(node); node_deinit(node->_vt); @@ -96,7 +95,7 @@ static void * send_loop(void *ctx) sendd.started = true; /* Initialize memory */ - ret = pool_init(&sendd.pool, node->vectorize, SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); + ret = pool_init(&sendd.pool, LOG2_CEIL(node->vectorize), SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); if (ret < 0) error("Failed to allocate memory for receive pool."); @@ -140,7 +139,7 @@ static void * recv_loop(void *ctx) recvv.started = true; /* Initialize memory */ - ret = pool_init(&recvv.pool, node->vectorize, SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); + ret = pool_init(&recvv.pool, LOG2_CEIL(node->vectorize), SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); if (ret < 0) error("Failed to allocate memory for receive pool."); From 248a6638da6848a74a5349d52b9f1c9cf6dba17a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 22:57:58 -0400 Subject: [PATCH 31/57] these hooks do not use HOOK_{INIT,DEINIT} to lets return early to not fail the assertion --- lib/hooks/hooks-internal.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index 070e68359..d10efb92e 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -16,6 +16,9 @@ REGISTER_HOOK("fix_ts", "Update timestamps of sample if not set", 0, 0, hook_fix int hook_fix_ts(struct hook *h, int when, struct hook_info *j) { struct timespec now = time_now(); + + if (when != HOOK_READ) + return 0; assert(j->smps); @@ -41,6 +44,9 @@ int hook_fix_ts(struct hook *h, int when, struct hook_info *j) REGISTER_HOOK("restart", "Call restart hooks for current path", 1, 1, hook_restart, HOOK_INTERNAL | HOOK_READ) int hook_restart(struct hook *h, int when, struct hook_info *j) { + if (when != HOOK_READ) + return 0; + assert(j->smps); assert(j->path); @@ -68,6 +74,9 @@ int hook_drop(struct hook *h, int when, struct hook_info *j) { int i, ok, dist; + if (when != HOOK_READ) + return 0; + assert(j->smps); for (i = 0, ok = 0; i < j->cnt; i++) { From 287e5727fba5dde8ad085a49f39f8ddaee4b0f43 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 23:00:17 -0400 Subject: [PATCH 32/57] fix block size calculation for memory pool --- lib/pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pool.c b/lib/pool.c index 955b8b77a..7db22d59c 100644 --- a/lib/pool.c +++ b/lib/pool.c @@ -18,7 +18,7 @@ int pool_init(struct pool *p, size_t cnt, size_t blocksz, const struct memtype * /* Make sure that we use a block size that is aligned to the size of a cache line */ p->alignment = kernel_get_cacheline_size(); - p->blocksz = blocksz * CEIL(blocksz, p->alignment); + p->blocksz = p->alignment * CEIL(blocksz, p->alignment); p->len = cnt * p->blocksz; p->mem = m; From 3d22238976174ea2d2ac43ba549f19fb8198e1cd Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 30 Oct 2016 23:01:14 -0400 Subject: [PATCH 33/57] make pool initialisation more robust by allowing non-pow2 pool sizes (we will use the next bigger power-of-2 for the internal queue) --- lib/pool.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pool.c b/lib/pool.c index 7db22d59c..df187790b 100644 --- a/lib/pool.c +++ b/lib/pool.c @@ -25,10 +25,8 @@ int pool_init(struct pool *p, size_t cnt, size_t blocksz, const struct memtype * p->buffer = memory_alloc_aligned(m, p->len, p->alignment); if (!p->buffer) serror("Failed to allocate memory for memory pool"); - else - debug(DBG_POOL | 4, "Allocated %#zx bytes for memory pool", p->len); - ret = queue_init(&p->queue, cnt, m); + ret = queue_init(&p->queue, LOG2_CEIL(cnt), m); if (ret) return ret; From dd7e0443ced2d1bae4303751cf176c30fd50cc0a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 21:30:28 -0500 Subject: [PATCH 34/57] add node id to web mockup javascript --- include/villas/webmsg_format.h | 8 ++++---- web/socket/app.js | 2 +- web/socket/msg.js | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/villas/webmsg_format.h b/include/villas/webmsg_format.h index 55f885980..c6b0e0d4d 100644 --- a/include/villas/webmsg_format.h +++ b/include/villas/webmsg_format.h @@ -49,9 +49,9 @@ /** Initialize a message with default values */ #define WEBMSG_INIT(len, seq) (struct msg) {\ - .version = MSG_VERSION, \ - .type = MSG_TYPE_DATA, \ - .endian = MSG_ENDIAN_HOST, \ + .version = WEBMSG_VERSION, \ + .type = WEBMSG_TYPE_DATA, \ + .endian = WEBMSG_ENDIAN_HOST, \ .length = len, \ .sequence = seq \ } @@ -80,7 +80,7 @@ struct webmsg unsigned version: 4; /**< Specifies the format of the remaining message (see MGS_VERSION) */ #endif - uint8_t node_id; /**< The node index from / to which this sample received / sent to. + uint8_t id; /**< The node index from / to which this sample received / sent to. * Corresponds to the index of the node in the http://localhost/nodes.json array. */ uint16_t length; /**< The number of values in msg::data[]. Endianess is specified in msg::endian. */ diff --git a/web/socket/app.js b/web/socket/app.js index a1b584dca..8bf88f94b 100644 --- a/web/socket/app.js +++ b/web/socket/app.js @@ -174,7 +174,7 @@ function wsConnect(url, protocol) { connection.onmessage = function(e) { var msgs = Msg.fromArrayBufferVector(e.data); - console.log('Received ' + msgs.length + ' messages with ' + msgs[0].data.length + ' values: ' + msgs[0].timestamp); + console.log('Received ' + msgs.length + ' messages with ' + msgs[0].data.length + ' values from id ' + msgs[0].id + ' with timestamp ' + msgs[0].timestamp); for (var j = 0; j < plotData.length; j++) { // remove old diff --git a/web/socket/msg.js b/web/socket/msg.js index a23795197..75436e396 100644 --- a/web/socket/msg.js +++ b/web/socket/msg.js @@ -20,6 +20,7 @@ function Msg(c, d) this.endian = typeof c.endian === 'undefined' ? Msg.prototype.ENDIAN_LITTLE : c.endian; this.version = typeof c.version === 'undefined' ? Msg.prototype.VERSION : c.version; this.type = typeof c.type === 'undefined' ? Msg.prototype.TYPE_DATA : c.type; + this.id = typeof c.id === 'undefined' ? -1 : c.id; this.timestamp = typeof c.timestamp === 'undefined' ? Date.now() : c.timestamp; if (Array.isArray(d)) { @@ -55,6 +56,7 @@ Msg.fromArrayBuffer = function(data) endian: (bits >> Msg.prototype.OFFSET_ENDIAN) & 0x1, version: (bits >> Msg.prototype.OFFSET_VERSION) & 0xF, type: (bits >> Msg.prototype.OFFSET_TYPE) & 0x3, + id: data.getUint8( 0x01, endian), length: data.getUint16(0x02, endian), sequence: data.getUint32(0x04, endian), timestamp: data.getUint32(0x08, endian) * 1e3 + @@ -109,6 +111,7 @@ Msg.prototype.toArrayBuffer = function() var nsec = (this.timestamp - sec * 1e3) * 1e6; view.setUint8( 0x00, bits, true); + view.setUint8( 0x01, this.id, true); view.setUint16(0x02, this.length, true); view.setUint32(0x04, this.sequence, true); view.setUint32(0x08, sec, true); From 88e36f272b54c6dacc4d267ee322c8e1ee3129cc Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 22:15:00 -0500 Subject: [PATCH 35/57] fix calculation of log2i(1) --- include/villas/utils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/villas/utils.h b/include/villas/utils.h index 0e29fba8e..e6ef0e469 100644 --- a/include/villas/utils.h +++ b/include/villas/utils.h @@ -221,7 +221,8 @@ __attribute__((always_inline)) static inline uint64_t rdtsc() /** Get log2 of long long integers */ static inline int log2i(long long x) { - assert(x > 0); + if (x == 0) + return 1; return sizeof(x) * 8 - __builtin_clzll(x) - 1; } From b23484550f01455a602388ac16c27bdfd5eff321 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 22:17:45 -0500 Subject: [PATCH 36/57] major rewrite with MPMC queues --- include/villas/path.h | 54 +++++---- lib/cfg.c | 78 +++++++------ lib/path.c | 265 ++++++++++++++++++++++++++++-------------- 3 files changed, 248 insertions(+), 149 deletions(-) diff --git a/include/villas/path.h b/include/villas/path.h index 0bcb4cf56..7bf71a003 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -26,55 +26,50 @@ #include "hooks.h" #include "queue.h" #include "pool.h" +#include "stats.h" -/** The datastructure for a path. - * - * @todo Add support for multiple outgoing nodes - */ +struct path_source +{ + struct node *node; + struct pool pool; + int samplelen; + pthread_t tid; +}; + +struct path_destination +{ + struct node *node; + struct queue queue; + int queuelen; + pthread_t tid; +}; + +/** The datastructure for a path. */ struct path { enum { PATH_INVALID, /**< Path is invalid. */ - PATH_CREATED, /**< Path has been created: lists initialized */ PATH_INITIALIZED, /**< Path queues, memory pools & hook system initialized. */ PATH_RUNNING, /**< Path is currently running. */ PATH_STOPPED, /**< Path has been stopped. */ PATH_DESTROYED /**< Path is destroyed. */ } state; /**< Path state */ - struct node *in; /**< Pointer to the incoming node */ - - struct queue queue; /**< A ring buffer for all received messages (unmodified) */ - struct pool pool; /**< Memory pool for messages / samples. */ + /* Each path has a single source and multiple destinations */ + struct path_source *source; /**< Pointer to the incoming node */ + struct list destinations; /**< List of all outgoing nodes (struct path_destination). */ - struct list destinations; /**< List of all outgoing nodes */ struct list hooks; /**< List of function pointers to hooks */ - int samplelen; /**< Maximum number of values per sample for this path. */ - int queuelen; /**< Size of sample queue for this path. */ int enabled; /**< Is this path enabled */ pthread_t tid; /**< The thread id for this path */ - - config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path */ char *_name; /**< Singleton: A string which is used to print this path to screen. */ - /** The following fields are mostly managed by hook_ functions @{ */ + struct stats *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ - struct { - struct hist owd; /**< Histogram for one-way-delay (OWD) of received messages */ - struct hist gap_msg; /**< Histogram for inter message timestamps (as sent by remote) */ - struct hist gap_recv; /**< Histogram for inter message arrival time (as seen by this instance) */ - struct hist gap_seq; /**< Histogram of sequence number displacement of received messages */ - } hist; - - /* Statistic counters */ - uintmax_t invalid; /**< Counter for invalid messages */ - uintmax_t skipped; /**< Counter for skipped messages due to hooks */ - uintmax_t dropped; /**< Counter for dropped messages due to reordering */ - - /** @} */ + config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path */ }; /** Initialize internal data structures. */ @@ -119,6 +114,9 @@ void path_print_stats(struct path *p); */ const char * path_name(struct path *p); +/** Reverse a path */ +int path_reverse(struct path *p, struct path *r); + /** Check if node is used as source or destination of a path. */ int path_uses_node(struct path *p, struct node *n); diff --git a/lib/cfg.c b/lib/cfg.c index a541de4c1..038744dc9 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -149,14 +149,14 @@ int cfg_parse_path(config_setting_t *cfg, { config_setting_t *cfg_out, *cfg_hook; const char *in; - int ret, reverse; + int ret, reverse, samplelen, queuelen; struct path *p; + + struct node *source; + struct list destinations; /* Allocate memory and intialize path structure */ p = alloc(sizeof(struct path)); - - list_init(&p->destinations); - list_init(&p->hooks); /* Input node */ if (!config_setting_lookup_string(cfg, "in", &in) && @@ -165,8 +165,8 @@ int cfg_parse_path(config_setting_t *cfg, !config_setting_lookup_string(cfg, "source", &in)) cerror(cfg, "Missing input node for path"); - p->in = list_lookup(nodes, in); - if (!p->in) + source = list_lookup(nodes, in); + if (!source) cerror(cfg, "Invalid input node '%s'", in); /* Output node(s) */ @@ -177,58 +177,66 @@ int cfg_parse_path(config_setting_t *cfg, !(cfg_out = config_setting_get_member(cfg, "sink"))) cerror(cfg, "Missing output nodes for path"); - ret = cfg_parse_nodelist(cfg_out, &p->destinations, nodes); + list_init(&destinations); + ret = cfg_parse_nodelist(cfg_out, &destinations, nodes); if (ret <= 0) cerror(cfg_out, "Invalid output nodes"); - - /* Check if nodes are suitable */ - if (p->in->_vt->read == NULL) - cerror(cfg, "Input node '%s' is not supported as a source.", node_name(p->in)); - - list_foreach(struct node *n, &p->destinations) { - if (n->_vt->write == NULL) - cerror(cfg_out, "Output node '%s' is not supported as a destination.", node_name(n)); - } /* Optional settings */ + list_init(&p->hooks); cfg_hook = config_setting_get_member(cfg, "hook"); if (cfg_hook) cfg_parse_hooklist(cfg_hook, &p->hooks); - if (!config_setting_lookup_int(cfg, "values", &p->samplelen)) - p->samplelen = DEFAULT_VALUES; - if (!config_setting_lookup_int(cfg, "queuelen", &p->queuelen)) - p->queuelen = DEFAULT_QUEUELEN; if (!config_setting_lookup_bool(cfg, "reverse", &reverse)) reverse = 0; if (!config_setting_lookup_bool(cfg, "enabled", &p->enabled)) p->enabled = 1; + if (!config_setting_lookup_int(cfg, "values", &samplelen)) + samplelen = DEFAULT_VALUES; + if (!config_setting_lookup_int(cfg, "queuelen", &queuelen)) + queuelen = DEFAULT_QUEUELEN; - if (!IS_POW2(p->queuelen)) { - p->queuelen = LOG2_CEIL(p->queuelen); - warn("Queue length should always be a power of 2. Adjusting to %d", p->queuelen); + if (!IS_POW2(queuelen)) { + queuelen = LOG2_CEIL(queuelen); + warn("Queue length should always be a power of 2. Adjusting to %d", queuelen); } p->cfg = cfg; + + /* Check if nodes are suitable */ + if (source->_vt->read == NULL) + cerror(cfg, "Input node '%s' is not supported as a source.", node_name(source)); + + p->source = alloc(sizeof(struct path_source)); + p->source->node = source; + p->source->samplelen = samplelen; + + list_foreach(struct node *n, &destinations) { + if (n->_vt->write == NULL) + cerror(cfg_out, "Output node '%s' is not supported as a destination.", node_name(n)); + + struct path_destination *pd = alloc(sizeof(struct path_destination)); + + pd->node = n; + pd->queuelen = queuelen; + + list_push(&p->destinations, pd); + } list_push(paths, p); if (reverse) { - if (list_length(&p->destinations) > 1) - cerror(cfg, "Can't reverse path with multiple destination nodes"); - - struct path *r = memdup(p, sizeof(struct path)); - path_init(r); - - /* Swap source and destination node */ - r->in = list_first(&p->destinations); - list_push(&r->destinations, p->in); - - if (cfg_hook) - cfg_parse_hooklist(cfg_hook, &r->hooks); + struct path *r = alloc(sizeof(struct path)); + + ret = path_reverse(p, r); + if (ret) + cerror(cfg, "Failed to reverse path %s", path_name(p)); list_push(paths, r); } + + list_destroy(&destinations, NULL, false); return 0; } diff --git a/lib/path.c b/lib/path.c index 7d22f768e..4a9045072 100644 --- a/lib/path.c +++ b/lib/path.c @@ -19,74 +19,103 @@ #include "pool.h" #include "queue.h" +static void path_read(struct path *p) +{ + int recv; + int enqueue; + int enqueued; + int ready = 0; /**< Number of blocks in smps[] which are allocated and ready to be used by node_read(). */ + + struct path_source *ps = p->source; + + int cnt = ps->node->vectorize; + + struct sample *smps[cnt]; + + /* Fill smps[] free sample blocks from the pool */ + ready += sample_alloc(&ps->pool, smps, cnt - ready); + if (ready != cnt) + warn("Pool underrun for path %s", path_name(p)); + + /* Read ready samples and store them to blocks pointed by smps[] */ + recv = node_read(ps->node, smps, ready); + if (recv < 0) + error("Failed to receive message from node %s", node_name(ps->node)); + else if (recv < ready) + warn("Partial read for path %s: read=%u expected=%u", path_name(p), recv, ready); + + debug(DBG_PATH | 15, "Received %u messages from node %s", recv, node_name(ps->node)); + + /* Run preprocessing hooks for vector of samples */ + enqueue = hook_run(p, smps, recv, HOOK_READ); + if (enqueue != recv) { + info("Hooks skipped %u out of %u samples for path %s", recv - enqueue, recv, path_name(p)); + + stats_update(p->stats, STATS_SKIPPED, recv - enqueue); + } + + list_foreach(struct path_destination *pd, &p->destinations) { + enqueued = queue_push_many(&pd->queue, (void **) smps, enqueue); + if (enqueue != enqueued) + warn("Queue overrun for path %s", path_name(p)); + + for (int i = 0; i < enqueued; i++) + sample_get(smps[i]); /* increase reference count */ + + debug(DBG_PATH | 15, "Enqueued %u samples to queue of path %s", enqueued, path_name(p)); + } +} + +static void path_write(struct path *p) +{ + list_foreach(struct path_destination *pd, &p->destinations) { + int cnt = pd->node->vectorize; + int sent; + int tosend; + int available; + int released; + + struct sample *smps[cnt]; + + /* As long as there are still samples in the queue */ + while (1) { + available = queue_pull_many(&pd->queue, (void **) smps, cnt); + if (available == 0) + break; + else if (available < cnt) + warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); + + debug(DBG_PATH | 15, "Dequeued %u samples from queue of node %s which is part of path %s", available, node_name(pd->node), path_name(p)); + + tosend = hook_run(p, smps, available, HOOK_WRITE); + if (tosend == 0) + continue; + + sent = node_write(pd->node, smps, tosend); + if (sent < 0) + error("Failed to sent %u samples to node %s", cnt, node_name(pd->node)); + else if (sent < tosend) + warn("Partial write to node %s", node_name(pd->node)); + + debug(DBG_PATH | 15, "Sent %u messages to node %s", sent, node_name(pd->node)); + + released = 0; + for (int i = 0; i < sent; i++) + released += sample_put(smps[i]); + + debug(DBG_PATH | 15, "Released %d samples back to memory pool", released); + } + } +} + /** Main thread function per path: receive -> sent messages */ static void * path_run(void *arg) { struct path *p = arg; - unsigned cnt = p->in->vectorize; - int recv, enqueue, enqueued; - int ready = 0; /**< Number of blocks in smps[] which are allocated and ready to be used by node_read(). */ - struct sample *smps[cnt]; - /* Main thread loop */ for (;;) { - /* Fill smps[] free sample blocks from the pool */ - ready += sample_get_many(&p->pool, smps, cnt - ready); - if (ready != cnt) - warn("Pool underrun for path %s", path_name(p)); - - /* Read ready samples and store them to blocks pointed by smps[] */ - recv = p->in->_vt->read(p->in, smps, ready); - if (recv < 0) - error("Failed to receive message from node %s", node_name(p->in)); - else if (recv < ready) - warn("Partial read for path %s: read=%u expected=%u", path_name(p), recv, ready); - - debug(DBG_PATH | 15, "Received %u messages from node %s", recv, node_name(p->in)); - - /* Run preprocessing hooks for vector of samples */ - enqueue = hook_run(p, smps, recv, HOOK_READ); - if (enqueue != recv) { - info("Hooks skipped %u out of %u samples for path %s", recv - enqueue, recv, path_name(p)); - p->skipped += recv - enqueue; - } - - enqueued = queue_push_many(&p->queue, (void **) smps, enqueue); - if (enqueue != enqueued) - warn("Failed to enqueue %u samples for path %s", enqueue - enqueued, path_name(p)); - - ready -= enqueued; - - debug(DBG_PATH | 3, "Enqueuing %u samples to queue of path %s", enqueue, path_name(p)); - - list_foreach(struct node *n, &p->destinations) { - int cnt = n->vectorize; - int sent, tosend, available, released; - struct sample *smps[n->vectorize]; - - available = queue_pull_many(&p->queue, (void **) smps, cnt); - if (available < cnt) - warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); - - if (available == 0) - continue; - - tosend = hook_run(p, smps, available, HOOK_WRITE); - if (tosend == 0) - continue; - - sent = node_write(n, smps, tosend); - if (sent < 0) - error("Failed to sent %u samples to node %s", cnt, node_name(n)); - else if (sent < tosend) - warn("Partial write to node %s", node_name(n)); - - debug(DBG_PATH | 15, "Sent %u messages to node %s", sent, node_name(n)); - - released = pool_put_many(&p->pool, (void **) smps, sent); - if (sent != released) - warn("Failed to release %u samples to pool for path %s", sent - released, path_name(p)); - } + path_read(p); + path_write(p); } return NULL; @@ -126,10 +155,10 @@ int path_stop(struct path *p) const char * path_name(struct path *p) { if (!p->_name) { - strcatf(&p->_name, "%s " MAG("=>"), node_name_short(p->in)); + strcatf(&p->_name, "%s " MAG("=>"), node_name_short(p->source->node)); - list_foreach(struct node *n, &p->destinations) - strcatf(&p->_name, " %s", node_name_short(n)); + list_foreach(struct path_destination *pd, &p->destinations) + strcatf(&p->_name, " %s", node_name_short(pd->node)); } return p->_name; @@ -137,18 +166,20 @@ const char * path_name(struct path *p) int path_init(struct path *p) { - int ret; - + int ret, max_queuelen = 0; + /* Add internal hooks if they are not already in the list*/ list_foreach(struct hook *h, &hooks) { - if ((h->type & HOOK_INTERNAL) && (list_lookup(&p->hooks, h->name) == NULL)) + if ( + (h->type & HOOK_INTERNAL) && /* is internal hook? */ + (list_lookup(&p->hooks, h->name) == NULL) /* is not already in list? */ + ) list_push(&p->hooks, memdup(h, sizeof(struct hook))); } - + /* We sort the hooks according to their priority before starting the path */ list_sort(&p->hooks, hooks_sort_priority); - /* Allocate hook private memory for storing state / parameters */ ret = hook_run(p, NULL, 0, HOOK_INIT); if (ret) error("Failed to initialize hooks of path: %s", path_name(p)); @@ -157,36 +188,98 @@ int path_init(struct path *p) ret = hook_run(p, NULL, 0, HOOK_PARSE); if (ret) error("Failed to parse arguments for hooks of path: %s", path_name(p)); - - /* Initialize queue */ - ret = pool_init(&p->pool, SAMPLE_LEN(p->samplelen), p->queuelen, &memtype_hugepage); + + /* Initialize destinations */ + list_foreach(struct path_destination *pd, &p->destinations) { + ret = queue_init(&pd->queue, pd->queuelen, &memtype_hugepage); + if (ret) + error("Failed to initialize queue for path"); + + if (pd->queuelen > max_queuelen) + max_queuelen = pd->queuelen; + } + + /* Initialize source */ + ret = pool_init(&p->source->pool, max_queuelen, SAMPLE_LEN(p->source->samplelen), &memtype_hugepage); if (ret) error("Failed to allocate memory pool for path"); - ret = queue_init(&p->queue, p->queuelen, &memtype_hugepage); - if (ret) - error("Failed to initialize queue for path"); - p->state = PATH_INITIALIZED; return 0; } +void path_source_destroy(struct path_source *ps) +{ + pool_destroy(&ps->pool); +} + +void path_destination_destroy(struct path_destination *pd) +{ + queue_destroy(&pd->queue); +} + void path_destroy(struct path *p) { - hook_run(p, NULL, 0, HOOK_DEINIT); /* Release memory */ + list_destroy(&p->hooks, (dtor_cb_t) hook_destroy, true); + list_destroy(&p->destinations, (dtor_cb_t) path_destination_destroy, true); - list_destroy(&p->destinations, NULL, false); - list_destroy(&p->hooks, NULL, true); - - queue_destroy(&p->queue); - pool_destroy(&p->pool); - - p->state = PATH_DESTROYED; + path_source_destroy(p->source); free(p->_name); + free(p->source); + + p->state = PATH_DESTROYED; } int path_uses_node(struct path *p, struct node *n) { - return (p->in == n) || list_contains(&p->destinations, n) ? 0 : 1; + list_foreach(struct path_destination *pd, &p->destinations) { + if (pd->node == n) + return 0; + } + + return p->source->node == n ? 0 : -1; } + +int path_reverse(struct path *p, struct path *r) +{ + int ret; + + if (list_length(&p->destinations) > 1) + return -1; + + struct path_destination *first_pd = list_first(&p->destinations); + + list_init(&r->destinations); + list_init(&r->hooks); + + /* General */ + r->enabled = p->enabled; + r->cfg = p->cfg; + + struct path_destination *pd = alloc(sizeof(struct path_destination)); + + pd->node = p->source->node; + pd->queuelen = first_pd->queuelen; + + list_push(&r->destinations, pd); + + struct path_source *ps = alloc(sizeof(struct path_source)); + + ps->node = first_pd->node; + ps->samplelen = p->source->samplelen; + + r->source = ps; + + list_foreach(struct hook *h, &p->hooks) { + struct hook *hc = alloc(sizeof(struct hook)); + + ret = hook_copy(h, hc); + if (ret) + return ret; + + list_push(&r->hooks, hc); + } + + return 0; +} \ No newline at end of file From 735f08f55121026662e57b2ec748e029f52f20af Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 22:18:26 -0500 Subject: [PATCH 37/57] more improvements to make statistic subsystem extensible --- include/villas/stats.h | 38 +++++---- lib/hooks/hooks-internal.c | 2 +- lib/stats.c | 162 ++++++++++++++++++++++--------------- 3 files changed, 124 insertions(+), 78 deletions(-) diff --git a/include/villas/stats.h b/include/villas/stats.h index e6260833b..49d6e11ef 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -23,20 +23,24 @@ struct sample; struct path; struct node; -struct stats { - struct { - uintmax_t invalid; /**< Counter for invalid messages */ - uintmax_t skipped; /**< Counter for skipped messages due to hooks */ - uintmax_t dropped; /**< Counter for dropped messages due to reordering */ - } counter; +enum stats_id { + STATS_INVALID, /**< Counter for invalid messages */ + STATS_SKIPPED, /**< Counter for skipped messages due to hooks */ + STATS_DROPPED, /**< Counter for dropped messages due to reordering */ + STATS_GAP_SEQUENCE, /**< Histogram of sequence number displacement of received messages */ + STATS_GAP_SAMPLE, /**< Histogram for inter message timestamps (as sent by remote) */ + STATS_GAP_RECEIVED, /**< Histogram for inter message arrival time (as seen by this instance) */ + STATS_OWD, /**< Histogram for one-way-delay (OWD) of received messages */ + STATS_COUNT /**< Just here to have an updated number of statistics */ +}; + +struct stats_delta { + double vals[STATS_COUNT]; +}; + +struct stats { + struct hist histograms[STATS_COUNT]; - struct { - struct hist owd; /**< Histogram for one-way-delay (OWD) of received messages */ - struct hist gap_msg; /**< Histogram for inter message timestamps (as sent by remote) */ - struct hist gap_recv; /**< Histogram for inter message arrival time (as seen by this instance) */ - struct hist gap_seq; /**< Histogram of sequence number displacement of received messages */ - } histogram; - struct sample *last; }; @@ -44,8 +48,14 @@ int stats_init(struct stats *s); void stats_destroy(struct stats *s); +void stats_update(struct stats *s, enum stats_id id, double val); + +int stats_commit(struct stats *s, struct stats_delta *d); + void stats_collect(struct stats *s, struct sample *smps[], size_t cnt); +void stats_decollect(struct stats *s, struct sample *smps[], size_t cnt); + #ifdef WITH_JANSSON json_t * stats_json(struct stats *s); #endif @@ -56,7 +66,7 @@ void stats_print_header(); void stats_print_periodic(struct stats *s, struct path *p); -void stats_print(struct stats *s); +void stats_print(struct stats *s, int details); void stats_send(struct stats *s, struct node *n); diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index d10efb92e..cb91505fd 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -85,7 +85,7 @@ int hook_drop(struct hook *h, int when, struct hook_info *j) if (h->prev) { dist = h->last->sequence - (int32_t) h->prev->sequence; if (dist <= 0) { - j->path->stats->counter.dropped++; + stats_update(j->path->stats, STATS_DROPPED, dist); warn("Dropped sample: dist = %d, i = %d", dist, i); } else { diff --git a/lib/stats.c b/lib/stats.c index f25675d1d..5bcceee21 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -13,38 +13,81 @@ #include "sample.h" #include "log.h" +static struct stats_desc { + const char *name; + const char *unit; + const char *desc; + struct { + double min; + double max; + double resolution; + } hist; +} stats_table[] = { + { "invalid", "", "invalid messages", {0, 0, -1, }}, + { "skipped", "", "skipped messages by hooks", {0, 0, -1, }}, + { "dropped", "", "dropped messages because of reordering", {0, 0, -1, }}, + { "gap_sequence", "", "sequence number displacement of received messages", {-10, 10, 20, }}, + { "gap_sample", "", "inter message timestamps (as sent by remote)", {90e-3, 110e-3, 1e-3, }}, + { "gap_received", "", "Histogram for inter message arrival time (as seen by this instance)", {90e-3, 110e-3, 1e-3, }}, + { "owd", "s", "Histogram for one-way-delay (OWD) of received messages", {0, 1, 100e-3, }} +}; + int stats_init(struct stats *s) { - /** @todo Allow configurable bounds for histograms */ - hist_create(&s->histogram.owd, 0, 1, 100e-3); - hist_create(&s->histogram.gap_msg, 90e-3, 110e-3, 1e-3); - hist_create(&s->histogram.gap_recv, 90e-3, 110e-3, 1e-3); - hist_create(&s->histogram.gap_seq, -HIST_SEQ, +HIST_SEQ, 1); - + for (int i = 0; i < STATS_COUNT; i++) { + struct stats_desc *desc = &stats_table[i]; + hist_create(&s->histograms[i], desc->hist.min, desc->hist.max, desc->hist.resolution); + } + return 0; } void stats_destroy(struct stats *s) { - hist_destroy(&s->histogram.owd); - hist_destroy(&s->histogram.gap_msg); - hist_destroy(&s->histogram.gap_recv); - hist_destroy(&s->histogram.gap_seq); + for (int i = 0; i < STATS_COUNT; i++) { + hist_destroy(&s->histograms[i]); + } +} + +void stats_update(struct stats *s, enum stats_id id, double val) +{ + if (!s) + return; + + hist_put(&s->histograms[id], val); +} + +#if 0 +int stats_delta(struct stats_delta *d, struct sample *s, struct sample *p) +{ + d->histogram.owd = time_delta(&smps[i]->ts.origin, &smps[i]->ts.received); + d->histogram.gap = time_delta(&s->last->ts.origin, &smps[i]->ts.origin); + d->histogram.gap_seq = s->sequence - (int32_t) p->sequence; + d->histogram.gap_recv = time_delta(&s->last->ts.received, &smps[i]->ts.received); + + d->counter.dropped = d->histogram.gap_seq <= 0 ? 1 : 0; + d->counter.invalid = 0; + + return 0; +} +#endif + +int stats_commit(struct stats *s, struct stats_delta *d) +{ + for (int i = 0; i < STATS_COUNT; i++) { + hist_put(&s->histograms[i], d->vals[i]); + } + + return 0; } void stats_collect(struct stats *s, struct sample *smps[], size_t cnt) { for (int i = 0; i < cnt; i++) { if (s->last) { - int gap_seq = smps[i]->sequence - (int32_t) s->last->sequence; - double owd = time_delta(&smps[i]->ts.origin, &smps[i]->ts.received); - double gap = time_delta(&s->last->ts.origin, &smps[i]->ts.origin); - double gap_recv = time_delta(&s->last->ts.received, &smps[i]->ts.received); - - hist_put(&s->histogram.owd, owd); - hist_put(&s->histogram.gap_msg, gap); - hist_put(&s->histogram.gap_seq, gap_seq); - hist_put(&s->histogram.gap_recv, gap_recv); +// struct stats_delta d; +// stats_get_delta(&d, smps[i], s->last); +// stats_commit(s, &d); } if (i == 0 && s->last) @@ -59,30 +102,25 @@ void stats_collect(struct stats *s, struct sample *smps[], size_t cnt) #ifdef WITH_JANSSON json_t * stats_json(struct stats *s) { - return json_pack("{ s: { s: i, s: i, s: i }, s: { s: o, s: o, s: o } }", - "counter", - "dropped", s->counter.dropped, - "invalid", s->counter.invalid, - "skipped", s->counter.skipped, - "histogram", - "owd", hist_json(&s->histogram.owd), - "gap_msg", hist_json(&s->histogram.gap_msg), - "gap_recv",hist_json(&s->histogram.gap_recv), - "gap_seq", hist_json(&s->histogram.gap_seq) - ); + json_t *obj = json_object(); + + for (int i = 0; i < STATS_COUNT; i++) { + struct stats_desc *desc = &stats_table[i]; + + json_t *stats = hist_json(&s->histograms[i]); + + json_object_set(obj, desc->name, stats); + } + + return obj; } #endif void stats_reset(struct stats *s) { - s->counter.invalid = - s->counter.skipped = - s->counter.dropped = 0; - - hist_reset(&s->histogram.owd); - hist_reset(&s->histogram.gap_seq); - hist_reset(&s->histogram.gap_msg); - hist_reset(&s->histogram.gap_recv); + for (int i = 0; i < STATS_COUNT; i++) { + hist_reset(&s->histograms[i]); + } } void stats_print_header() @@ -101,41 +139,39 @@ void stats_print_header() void stats_print_periodic(struct stats *s, struct path *p) { - stats("%-40.40s|%10s|%10s|%10ju|%10ju|%10ju|", path_name(p), "", "", - s->counter.dropped, s->counter.skipped, s->counter.invalid); + stats("%-40.40s|%10f|%10f|%10ju|%10ju|%10ju|", path_name(p), + s->histograms[STATS_OWD].last, + 1.0 / s->histograms[STATS_GAP_SAMPLE].last, + s->histograms[STATS_DROPPED].total, + s->histograms[STATS_SKIPPED].total, + s->histograms[STATS_INVALID].total + ); } -void stats_print(struct stats *s) +void stats_print(struct stats *s, int details) { - stats("Dropped samples: %ju", s->counter.dropped); - stats("Skipped samples: %ju", s->counter.skipped); - stats("Invalid samples: %ju", s->counter.invalid); - - stats("One-way delay:"); - hist_print(&s->histogram.owd); - - stats("Inter-message arrival time:"); - hist_print(&s->histogram.gap_recv); - - stats("Inter-message ts gap:"); - hist_print(&s->histogram.gap_msg); - - stats("Inter-message sequence number gaps:"); - hist_print(&s->histogram.gap_seq); + for (int i = 0; i < STATS_COUNT; i++) { + struct stats_desc *desc = &stats_table[i]; + + stats("%s: %s", desc->name, desc->desc); + hist_print(&s->histograms[i], details); + } } void stats_send(struct stats *s, struct node *n) { - char buf[SAMPLE_LEN(16)]; + char buf[SAMPLE_LEN(STATS_COUNT * 5)]; struct sample *smp = (struct sample *) buf; int i = 0; - smp->data[i++].f = s->counter.invalid; /**< Use integer here? */ - smp->data[i++].f = s->counter.skipped; - smp->data[i++].f = s->counter.dropped; - smp->data[i++].f = s->histogram.owd.last, - smp->data[i++].f = 1.0 / s->histogram.gap_msg.last; - smp->data[i++].f = 1.0 / s->histogram.gap_recv.last; + + for (int j = 0; j < STATS_COUNT; j++) { + smp->data[i++].f = s->histograms[j].last; + smp->data[i++].f = s->histograms[j].highest; + smp->data[i++].f = s->histograms[j].lowest; + smp->data[i++].f = hist_mean(&s->histograms[j]); + smp->data[i++].f = hist_var(&s->histograms[j]); + } smp->length = i; node_write(n, &smp, 1); /* Send single message with statistics to destination node */ From 9e13b06a59e6a4fad18f4292bb62945afcea047d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 22:19:30 -0500 Subject: [PATCH 38/57] get first working version of web socket node-type with MPMC queues --- etc/websocket.conf | 3 +- include/villas/nodes/websocket.h | 17 +- include/villas/sample.h | 8 +- lib/nodes/websocket.c | 337 ++++++++++++++++++------------- thirdparty/libwebsockets | 2 +- 5 files changed, 214 insertions(+), 153 deletions(-) diff --git a/etc/websocket.conf b/etc/websocket.conf index 21e2513b8..2a012a0a3 100644 --- a/etc/websocket.conf +++ b/etc/websocket.conf @@ -33,7 +33,8 @@ nodes = { series = ( { label = "Random walk" }, { label = "Sine" }, - { label = "Rect" } + { label = "Rect" }, + { label = "Ramp" } ) } }; diff --git a/include/villas/nodes/websocket.h b/include/villas/nodes/websocket.h index 4f3149534..9d4679d51 100644 --- a/include/villas/nodes/websocket.h +++ b/include/villas/nodes/websocket.h @@ -29,38 +29,35 @@ struct lws; /** Internal data per websocket node */ struct websocket { struct list connections; /**< List of active libwebsocket connections in server mode (struct websocket_connection) */ + struct list destinations; /**< List of struct lws_client_connect_info to connect to in client mode. */ struct pool pool; - struct queue queue_tx; /**< For samples which are sent to WebSockets */ - struct queue queue_rx; /**< For samples which are received from WebSockets */ - - qptr_t sent; - qptr_t received; - - int shutdown; + struct queue queue; /**< For samples which are received from WebSockets a */ + + int id; /**< The index of this node */ }; struct websocket_connection { enum { WEBSOCKET_ESTABLISHED, WEBSOCKET_ACTIVE, + WEBSOCKET_SHUTDOWN, WEBSOCKET_CLOSED } state; struct node *node; struct path *path; + struct queue queue; /**< For samples which are sent to the WebSocket */ + struct lws *wsi; struct { char name[64]; char ip[64]; } peer; - - qptr_t sent; - qptr_t received; }; /** @see node_vtable::init */ diff --git a/include/villas/sample.h b/include/villas/sample.h index c412f7127..4caf3f927 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -45,8 +45,6 @@ struct sample { atomic_int refcnt; /**< Reference counter. */ struct pool *pool; /**< This sample is belong to this memory pool. */ - - int endian; /**< Endianess of data in the sample. */ /** All timestamps are seconds / nano seconds after 1.1.1970 UTC */ struct { @@ -57,9 +55,9 @@ struct sample { /** The values. */ union { - float f; /**< Floating point values (note msg::endian) */ - uint32_t i; /**< Integer values (note msg::endian) */ - } data[]; + float f; /**< Floating point values. */ + uint32_t i; /**< Integer values. */ + } data[]; /**< Data is in host endianess! */ }; /** Request \p cnt samples from memory pool \p p and initialize them. diff --git a/lib/nodes/websocket.c b/lib/nodes/websocket.c index d7b51733c..901d1c8ce 100644 --- a/lib/nodes/websocket.c +++ b/lib/nodes/websocket.c @@ -16,6 +16,7 @@ #include #include "nodes/websocket.h" +#include "webmsg_format.h" #include "timing.h" #include "utils.h" #include "msg.h" @@ -33,43 +34,21 @@ static const char *ssl_cert; /**< Path to the SSL certitifcate for HTTPS / WSS static const char *ssl_private_key; /**< Path to the SSL private key for HTTPS / WSS */ static const char *htdocs; /**< Path to the directory which should be served by build in HTTP server */ +static int id = 0; + +struct list connections; /**< List of active libwebsocket connections which receive samples from all nodes (catch all) */ + /* Forward declarations */ static struct node_type vt; static int protocol_cb_http(struct lws *, enum lws_callback_reasons, void *, void *, size_t); static int protocol_cb_live(struct lws *, enum lws_callback_reasons, void *, void *, size_t); static struct lws_protocols protocols[] = { - { - "http-only", - protocol_cb_http, - 0, - 0 - }, - { - "live", - protocol_cb_live, - sizeof(struct websocket_connection), - 0 - }, - { 0 /* terminator */ } + { "http-only", protocol_cb_http, 0, 0 }, + { "live", protocol_cb_live, sizeof(struct websocket_connection), 0 }, + { NULL } }; -#if 0 -static const struct lws_extension exts[] = { - { - "permessage-deflate", - lws_extension_callback_pm_deflate, - "permessage-deflate" - }, - { - "deflate-frame", - lws_extension_callback_pm_deflate, - "deflate_frame" - }, - { NULL, NULL, NULL /* terminator */ } -}; -#endif - static void logger(int level, const char *msg) { int len = strlen(msg); if (strchr(msg, '\n')) @@ -80,7 +59,7 @@ static void logger(int level, const char *msg) { level = LLL_WARN; switch (level) { - case LLL_ERR: error("LWS: %.*s", len, msg); break; + case LLL_ERR: warn("LWS: %.*s", len, msg); break; case LLL_WARN: warn("LWS: %.*s", len, msg); break; case LLL_INFO: info("LWS: %.*s", len, msg); break; default: debug(DBG_WEBSOCKET | 1, "LWS: %.*s", len, msg); break; @@ -160,8 +139,9 @@ int protocol_cb_http(struct lws *wsi, enum lws_callback_reasons reason, void *us list_foreach(struct node *n, &vt.instances) { struct websocket *w = n->_vd; - json_t *json_node = json_pack("{ s: s, s: i, s: i, s: i, s: i }", + json_t *json_node = json_pack("{ s: s, s: i, s: i, s: i, s: i, s: i }", "name", node_name_short(n), + "id", w->id, "connections", list_length(&w->connections), "state", n->state, "vectorize", n->vectorize, @@ -228,7 +208,7 @@ int protocol_cb_http(struct lws *wsi, enum lws_callback_reasons reason, void *us } default: - break; + return 0; } return 0; @@ -242,6 +222,7 @@ try_to_reuse: int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { + int ret; struct websocket_connection *c = user; struct websocket *w; @@ -256,36 +237,63 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us return -1; } - /* Search for node whose name matches the URI. */ - c->node = list_lookup(&vt.instances, uri + 1); - if (c->node == NULL) { - warn("LWS: Closing Connection for non-existent node: %s", uri + 1); - return -1; + if ((uri[0] == '/' && uri[1] == 0) || uri[0] == 0){ + /* Catch all connection */ + c->node = NULL; } + else { + char *node = uri + 1; - /* Check if node is running */ - if (c->node->state != NODE_RUNNING) - return -1; + /* Search for node whose name matches the URI. */ + c->node = list_lookup(&vt.instances, node); + if (c->node == NULL) { + warn("LWS: Closing Connection for non-existent node: %s", uri + 1); + return -1; + } + + /* Check if node is running */ + if (c->node->state != NODE_RUNNING) + return -1; + } c->state = WEBSOCKET_ESTABLISHED; c->wsi = wsi; + + ret = queue_init(&c->queue, DEFAULT_QUEUELEN, &memtype_hugepage); + if (ret) { + warn("Failed to create queue for incoming websocket connection. Closing.."); + return -1; + } /* Lookup peer address for debug output */ lws_get_peer_addresses(wsi, lws_get_socket_fd(wsi), c->peer.name, sizeof(c->peer.name), c->peer.ip, sizeof(c->peer.ip)); - info("LWS: New Connection for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); + if (c->node != NULL) + info("LWS: New connection for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); + else + info("LWS: New connection from %s (%s)", c->peer.name, c->peer.ip); - struct websocket *w = (struct websocket *) c->node->_vd; - list_push(&w->connections, c); + if (c->node != NULL) { + struct websocket *w = (struct websocket *) c->node->_vd; + list_push(&w->connections, c); + } + else { + list_push(&connections, c); + } return 0; } case LWS_CALLBACK_CLOSED: - info("LWS: Connection closed for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); + if (c->node != NULL) + info("LWS: Connection closed for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); + else + info("LWS: Connection closed from %s (%s)", c->peer.name, c->peer.ip); c->state = WEBSOCKET_CLOSED; c->wsi = NULL; + + queue_destroy(&c->queue); return 0; @@ -293,36 +301,30 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us case LWS_CALLBACK_SERVER_WRITEABLE: { w = (struct websocket *) c->node->_vd; - if (c->node->state != NODE_RUNNING) + if (c->node && c->node->state != NODE_RUNNING) return -1; - if (w->shutdown) { - lws_close_reason(wsi, LWS_CLOSE_STATUS_GOINGAWAY, (unsigned char *) "Bye", 4); + if (w->state == WEBSOCKET_SHUTDOWN) { + lws_close_reason(wsi, LWS_CLOSE_STATUS_GOINGAWAY, (unsigned char *) "Node stopped", 4); return -1; } - - - int cnt, sent, ret; - unsigned char *bufs[DEFAULT_QUEUELEN]; - - cnt = queue_get_many(&w->queue_tx, (void **) bufs, DEFAULT_QUEUELEN, c->sent); - for (sent = 0; sent < cnt; sent++) { - struct msg *msg = (struct msg *) (bufs[sent] + LWS_PRE); + char *buf; + int cnt; + while ((cnt = queue_pull(&c->queue, (void **) &buf))) { + struct webmsg *msg = (struct webmsg *) (buf + LWS_PRE); - ret = lws_write(wsi, (unsigned char *) msg, MSG_LEN(msg->length), LWS_WRITE_BINARY); - if (ret < MSG_LEN(msg->length)) + pool_put(&w->pool, (void *) buf); + + ret = lws_write(wsi, (unsigned char *) msg, WEBMSG_LEN(msg->length), LWS_WRITE_BINARY); + if (ret < WEBMSG_LEN(msg->length)) error("Failed lws_write()"); if (lws_send_pipe_choked(wsi)) - break; + break; } - - queue_pull_many(&w->queue_tx, (void **) bufs, sent, &c->sent); - - pool_put_many(&w->pool, (void **) bufs, sent); - if (sent < cnt) + if (queue_available(&c->queue) > 0) lws_callback_on_writable(wsi); return 0; @@ -335,27 +337,30 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us if (c->node->state != NODE_RUNNING) return -1; - if (!lws_frame_is_binary(wsi) || len < MSG_LEN(0)) + if (!lws_frame_is_binary(wsi) || len < WEBMSG_LEN(0)) warn("LWS: Received invalid packet for node: %s", node_name(c->node)); - struct msg *msg = (struct msg *) in; + struct webmsg *msg = (struct webmsg *) in; - while ((char *) msg + MSG_LEN(msg->length) <= (char *) in + len) { - struct msg *msg2 = pool_get(&w->pool); + while ((char *) msg + WEBMSG_LEN(msg->length) <= (char *) in + len) { + struct webmsg *msg2 = pool_get(&w->pool); if (!msg2) { warn("Pool underrun for node: %s", node_name(c->node)); - return -1; + break; } - memcpy(msg2, msg, MSG_LEN(msg->length)); + memcpy(msg2, msg, WEBMSG_LEN(msg->length)); - queue_push(&w->queue_rx, msg2, &c->received); + ret = queue_push(&w->queue, msg2); + if (ret != 1) { + warn("Queue overrun for node: %s", node_name(c->node)); + break; + } /* Next message */ - msg = (struct msg *) ((char *) msg + MSG_LEN(msg->length)); + msg = (struct webmsg *) ((char *) msg + WEBMSG_LEN(msg->length)); } - /** @todo Implement */ return 0; } @@ -421,29 +426,24 @@ int websocket_deinit() int websocket_open(struct node *n) { - struct websocket *w = n->_vd; - int ret; + struct websocket *w = n->_vd; + + w->id = id++; list_init(&w->connections); list_init(&w->destinations); - size_t blocklen = LWS_PRE + MSG_LEN(DEFAULT_VALUES); + size_t blocklen = LWS_PRE + WEBMSG_LEN(DEFAULT_VALUES); - ret = pool_init_mmap(&w->pool, blocklen, 2 * DEFAULT_QUEUELEN); + ret = pool_init(&w->pool, 64 * DEFAULT_QUEUELEN, blocklen, &memtype_hugepage); if (ret) return ret; - ret = queue_init(&w->queue_tx, DEFAULT_QUEUELEN); + ret = queue_init(&w->queue, DEFAULT_QUEUELEN, &memtype_hugepage); if (ret) return ret; - - ret = queue_init(&w->queue_rx, DEFAULT_QUEUELEN); - if (ret) - return ret; - - queue_reader_add(&w->queue_rx, 0, 0); - + return 0; } @@ -451,13 +451,13 @@ int websocket_close(struct node *n) { struct websocket *w = n->_vd; - w->shutdown = 1; - - list_foreach(struct lws *wsi, &w->connections) - lws_callback_on_writable(wsi); + list_foreach(struct websocket_connection *c, &w->connections) { + c->state = WEBSOCKET_SHUTDOWN; + lws_callback_on_writable(c->wsi); + } pool_destroy(&w->pool); - queue_destroy(&w->queue_tx); + queue_destroy(&w->queue); list_destroy(&w->connections, NULL, false); @@ -475,17 +475,21 @@ int websocket_read(struct node *n, struct sample *smps[], unsigned cnt) { struct websocket *w = n->_vd; - struct msg *msgs[cnt]; + struct webmsg *msgs[cnt]; int got; - got = queue_pull_many(&w->queue_rx, (void **) msgs, cnt, &w->received); + do { + got = queue_pull_many(&w->queue, (void **) msgs, cnt); + pthread_yield(); + } while (got == 0); + for (int i = 0; i < got; i++) { smps[i]->sequence = msgs[i]->sequence; smps[i]->length = msgs[i]->length; - smps[i]->ts.origin = MSG_TS(msgs[i]); + smps[i]->ts.origin = WEBMSG_TS(msgs[i]); - memcpy(&smps[i]->data, &msgs[i]->data, MSG_DATA_LEN(msgs[i]->length)); + memcpy(&smps[i]->data, &msgs[i]->data, WEBMSG_DATA_LEN(msgs[i]->length)); } pool_put_many(&w->pool, (void **) msgs, got); @@ -493,59 +497,118 @@ int websocket_read(struct node *n, struct sample *smps[], unsigned cnt) return got; } +static int websocket_connection_init(struct websocket_connection *c) +{ + /** @todo */ + return -1; +} + +static void websocket_connection_destroy(struct websocket_connection *c) +{ + /** @todo */ +} + +static char * websocket_connection_name(struct websocket_connection *c) +{ + /** @todo */ + return "(todo)"; +} + +static int websocket_connection_write(struct websocket_connection *c, struct sample *smps[], unsigned cnt) +{ + int blocks, enqueued; + char *bufs[cnt]; + + switch (c->state) { + case WEBSOCKET_SHUTDOWN: + return -1; + case WEBSOCKET_CLOSED: + if (c->node) { + struct websocket *w = (struct websocket *) c->node->_vd; + list_remove(&w->connections, c); + } + else + list_remove(&connections, c); + break; + + case WEBSOCKET_ESTABLISHED: + c->state = WEBSOCKET_ACTIVE; + /* fall through */ + + case WEBSOCKET_ACTIVE: + blocks = pool_get_many(&w->pool, (void **) bufs, cnt); + if (blocks != cnt) + warn("Pool underrun in websocket connection: %s", websocket_connection_name(c)); + + for (int i = 0; i < blocks; i++) { + struct webmsg *msg = (struct webmsg *) (bufs[i] + LWS_PRE); + + msg->version = WEBMSG_VERSION; + msg->type = WEBMSG_TYPE_DATA; + msg->endian = WEBMSG_ENDIAN_HOST; + msg->length = smps[i]->length; + msg->sequence = smps[i]->sequence; + msg->id = w->id; + msg->ts.sec = smps[i]->ts.origin.tv_sec; + msg->ts.nsec = smps[i]->ts.origin.tv_nsec; + + memcpy(&msg->data, &smps[i]->data, smps[i]->length * 4); + } + + enqueued = queue_push_many(&c->queue, (void **) bufs, cnt); + if (enqueued != blocks) + warn("Queue overrun in websocket connection: %s", websocket_connection_name(c)); + + lws_callback_on_writable(c->wsi); + break; + } + + return 0; +} + int websocket_write(struct node *n, struct sample *smps[], unsigned cnt) { struct websocket *w = n->_vd; - int blocks, enqueued; - char *bufs[cnt]; - - /* Copy samples to websocket queue */ - blocks = pool_get_many(&w->pool, (void **) bufs, cnt); - if (blocks != cnt) - warn("Pool underrun in websocket node: %s", node_name(n)); - - for (int i = 0; i < blocks; i++) { - struct msg *msg = (struct msg *) (bufs[i] + LWS_PRE); - - msg->version = MSG_VERSION; - msg->type = MSG_TYPE_DATA; - msg->endian = MSG_ENDIAN_HOST; - msg->length = smps[i]->length; - msg->sequence = smps[i]->sequence; - msg->ts.sec = smps[i]->ts.origin.tv_sec; - msg->ts.nsec = smps[i]->ts.origin.tv_nsec; - - memcpy(&msg->data, &smps[i]->data, smps[i]->length * 4); + list_foreach(struct websocket_connection *c, &w->connections) { + websocket_write_connection(c, smps, cnt); } - enqueued = queue_push_many(&w->queue_tx, (void **) bufs, cnt, &w->sent); - if (enqueued != blocks) - warn("Queue overrun in websocket node: %s", node_name(n)); - - /* Notify all active websocket connections to send new data */ - list_foreach(struct websocket_connection *c, &w->connections) { - switch (c->state) { - case WEBSOCKET_CLOSED: - queue_reader_remove(&w->queue_tx, c->sent, w->sent); - list_remove(&w->connections, c); - break; - - case WEBSOCKET_ESTABLISHED: - c->sent = w->sent; - c->state = WEBSOCKET_ACTIVE; - - queue_reader_add(&w->queue_tx, c->sent, w->sent); - - case WEBSOCKET_ACTIVE: - lws_callback_on_writable(c->wsi); - break; - } + list_foreach(struct websocket_connection *c, &connections) { + websocket_write_connection(c, smps, cnt); } return cnt; } +int websocket_parse(struct node *n, config_setting_t *cfg) +{ + config_setting_t *cfg_dests; + + cfg_dests = config_setting_get_member(cfg, "destinations"); + + if (!config_setting_is_array(cfg_dests)) + cerror(dests, "The 'destinations' setting must be an array of URLs"); + + for (int i = 0; i < config_setting_length(cfg_dests); i++) { + config_setting_t *cfg_dest; + const char *url; + struct lws_client_connect_info *i; + + url = config_setting_get_string_elem(cfg_dests, i); + if (!url) + cerror(dests, "The 'destinations' setting must be an array of URLs"); + + i = alloc(sizeof()) + + } +} + +char * websocket_print(struct node *n) +{ + +} + static struct node_type vt = { .name = "websocket", .description = "Send and receive samples of a WebSocket connection (libwebsockets)", @@ -557,7 +620,9 @@ static struct node_type vt = { .read = websocket_read, .write = websocket_write, .init = websocket_init, - .deinit = websocket_deinit + .deinit = websocket_deinit, + .print = websocket_print, + .parse = websocket_parse }; REGISTER_NODE_TYPE(&vt) \ No newline at end of file diff --git a/thirdparty/libwebsockets b/thirdparty/libwebsockets index 0c984014f..5fb327754 160000 --- a/thirdparty/libwebsockets +++ b/thirdparty/libwebsockets @@ -1 +1 @@ -Subproject commit 0c984014f0a82e184af2ff18f97b45e2cbccd0bd +Subproject commit 5fb327754ab4d202fca903dd5bd6b546b340eecb From 15e7e78e04a8b5e1d96c71164600ff69863817f9 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 7 Nov 2016 22:20:09 -0500 Subject: [PATCH 39/57] mark samples with source node --- include/villas/sample.h | 1 + lib/node.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/include/villas/sample.h b/include/villas/sample.h index 4caf3f927..53230a7cf 100644 --- a/include/villas/sample.h +++ b/include/villas/sample.h @@ -45,6 +45,7 @@ struct sample { atomic_int refcnt; /**< Reference counter. */ struct pool *pool; /**< This sample is belong to this memory pool. */ + struct node *source; /**< The node from which this sample originates. */ /** All timestamps are seconds / nano seconds after 1.1.1970 UTC */ struct { diff --git a/lib/node.c b/lib/node.c index b4aa9dc02..9bfe0a4c6 100644 --- a/lib/node.c +++ b/lib/node.c @@ -38,6 +38,9 @@ int node_read(struct node *n, struct sample *smps[], unsigned cnt) nread = n->_vt->read(n, smps, cnt); } + for (int i = 0; i < nread; i++) + smps[i]->source = n; + return nread; } From fab53b0302fda003333e5affe5609eee4168e2f6 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 8 Nov 2016 00:24:57 -0500 Subject: [PATCH 40/57] further steps to implement websocket client functionality --- include/villas/nodes/websocket.h | 29 +--- lib/nodes/websocket.c | 269 +++++++++++++++++++------------ 2 files changed, 173 insertions(+), 125 deletions(-) diff --git a/include/villas/nodes/websocket.h b/include/villas/nodes/websocket.h index 9d4679d51..3fb1bef79 100644 --- a/include/villas/nodes/websocket.h +++ b/include/villas/nodes/websocket.h @@ -19,6 +19,8 @@ #ifndef _WEBSOCKET_H_ #define _WEBSOCKET_H_ +#include + #include "node.h" #include "pool.h" #include "queue.h" @@ -28,38 +30,15 @@ struct lws; /** Internal data per websocket node */ struct websocket { - struct list connections; /**< List of active libwebsocket connections in server mode (struct websocket_connection) */ - - struct list destinations; /**< List of struct lws_client_connect_info to connect to in client mode. */ + struct list connections; /**< List of active libwebsocket connections in server mode (struct websocket_connection). */ + struct list destinations; /**< List of websocket servers connect to in client mode (struct websocket_destination). */ struct pool pool; - struct queue queue; /**< For samples which are received from WebSockets a */ int id; /**< The index of this node */ }; -struct websocket_connection { - enum { - WEBSOCKET_ESTABLISHED, - WEBSOCKET_ACTIVE, - WEBSOCKET_SHUTDOWN, - WEBSOCKET_CLOSED - } state; - - struct node *node; - struct path *path; - - struct queue queue; /**< For samples which are sent to the WebSocket */ - - struct lws *wsi; - - struct { - char name[64]; - char ip[64]; - } peer; -}; - /** @see node_vtable::init */ int websocket_init(int argc, char * argv[], config_setting_t *cfg); diff --git a/lib/nodes/websocket.c b/lib/nodes/websocket.c index 901d1c8ce..1809b4761 100644 --- a/lib/nodes/websocket.c +++ b/lib/nodes/websocket.c @@ -12,7 +12,6 @@ #include #include -#include #include #include "nodes/websocket.h" @@ -23,6 +22,35 @@ #include "cfg.h" #include "config.h" +/* Internal datastructures */ +struct connection { + enum { + WEBSOCKET_ESTABLISHED, + WEBSOCKET_ACTIVE, + WEBSOCKET_SHUTDOWN, + WEBSOCKET_CLOSED + } state; + + struct node *node; + struct path *path; + + struct queue queue; /**< For samples which are sent to the WebSocket */ + + struct lws *wsi; + + struct { + char name[64]; + char ip[64]; + } peer; + + char *_name; +}; + +struct destination { + char *uri; + struct lws_client_connect_info info; +}; + /* Private static storage */ static config_setting_t *cfg_root; /**< Root config */ static pthread_t thread; /**< All nodes are served by a single websocket server. This server is running in a dedicated thread. */ @@ -45,10 +73,93 @@ static int protocol_cb_live(struct lws *, enum lws_callback_reasons, void *, voi static struct lws_protocols protocols[] = { { "http-only", protocol_cb_http, 0, 0 }, - { "live", protocol_cb_live, sizeof(struct websocket_connection), 0 }, + { "live", protocol_cb_live, sizeof(struct connection), 0 }, { NULL } }; +__attribute__((unused)) static int connection_init(struct connection *c) +{ + /** @todo */ + return -1; +} + +__attribute__((unused)) static void connection_destroy(struct connection *c) +{ + if (c->_name) + free(c->_name); +} + +static char * connection_name(struct connection *c) +{ + if (!c->_name) { + if (c->node) + asprintf(&c->_name, "%s (%s) for node %s", c->peer.name, c->peer.ip, node_name(c->node)); + else + asprintf(&c->_name, "%s (%s) for all nodes", c->peer.name, c->peer.ip); + } + + return c->_name; +} + +static void destination_destroy(struct destination *d) +{ + free(d->uri); +} + +static int connection_write(struct connection *c, struct sample *smps[], unsigned cnt) +{ + int blocks, enqueued; + char *bufs[cnt]; + + struct websocket *w = c->node->_vd; + + switch (c->state) { + case WEBSOCKET_SHUTDOWN: + return -1; + case WEBSOCKET_CLOSED: + if (c->node) { + struct websocket *w = (struct websocket *) c->node->_vd; + list_remove(&w->connections, c); + } + else + list_remove(&connections, c); + break; + + case WEBSOCKET_ESTABLISHED: + c->state = WEBSOCKET_ACTIVE; + /* fall through */ + + case WEBSOCKET_ACTIVE: + blocks = pool_get_many(&w->pool, (void **) bufs, cnt); + if (blocks != cnt) + warn("Pool underrun in websocket connection: %s", connection_name(c)); + + for (int i = 0; i < blocks; i++) { + struct webmsg *msg = (struct webmsg *) (bufs[i] + LWS_PRE); + + msg->version = WEBMSG_VERSION; + msg->type = WEBMSG_TYPE_DATA; + msg->endian = WEBMSG_ENDIAN_HOST; + msg->length = smps[i]->length; + msg->sequence = smps[i]->sequence; + msg->id = w->id; + msg->ts.sec = smps[i]->ts.origin.tv_sec; + msg->ts.nsec = smps[i]->ts.origin.tv_nsec; + + memcpy(&msg->data, &smps[i]->data, smps[i]->length * 4); + } + + enqueued = queue_push_many(&c->queue, (void **) bufs, cnt); + if (enqueued != blocks) + warn("Queue overrun in websocket connection: %s", connection_name(c)); + + lws_callback_on_writable(c->wsi); + break; + } + + return 0; +} + static void logger(int level, const char *msg) { int len = strlen(msg); if (strchr(msg, '\n')) @@ -102,7 +213,7 @@ static char * get_mimetype(const char *resource_path) return "text/plain"; } -int protocol_cb_http(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) +static int protocol_cb_http(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { switch (reason) { case LWS_CALLBACK_HTTP: @@ -220,10 +331,10 @@ try_to_reuse: return 0; } -int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) +static int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len) { int ret; - struct websocket_connection *c = user; + struct connection *c = user; struct websocket *w; switch (reason) { @@ -268,10 +379,7 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us /* Lookup peer address for debug output */ lws_get_peer_addresses(wsi, lws_get_socket_fd(wsi), c->peer.name, sizeof(c->peer.name), c->peer.ip, sizeof(c->peer.ip)); - if (c->node != NULL) - info("LWS: New connection for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); - else - info("LWS: New connection from %s (%s)", c->peer.name, c->peer.ip); + info("LWS: New connection %s", connection_name(c)); if (c->node != NULL) { struct websocket *w = (struct websocket *) c->node->_vd; @@ -285,10 +393,7 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us } case LWS_CALLBACK_CLOSED: - if (c->node != NULL) - info("LWS: Connection closed for node %s from %s (%s)", node_name(c->node), c->peer.name, c->peer.ip); - else - info("LWS: Connection closed from %s (%s)", c->peer.name, c->peer.ip); + info("LWS: Connection %s closed", connection_name(c)); c->state = WEBSOCKET_CLOSED; c->wsi = NULL; @@ -304,7 +409,7 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us if (c->node && c->node->state != NODE_RUNNING) return -1; - if (w->state == WEBSOCKET_SHUTDOWN) { + if (c->state == WEBSOCKET_SHUTDOWN) { lws_close_reason(wsi, LWS_CLOSE_STATUS_GOINGAWAY, (unsigned char *) "Node stopped", 4); return -1; } @@ -338,14 +443,14 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us return -1; if (!lws_frame_is_binary(wsi) || len < WEBMSG_LEN(0)) - warn("LWS: Received invalid packet for node: %s", node_name(c->node)); + warn("LWS: Received invalid packet for connection %s", connection_name(c)); struct webmsg *msg = (struct webmsg *) in; while ((char *) msg + WEBMSG_LEN(msg->length) <= (char *) in + len) { struct webmsg *msg2 = pool_get(&w->pool); if (!msg2) { - warn("Pool underrun for node: %s", node_name(c->node)); + warn("Pool underrun for connection %s", connection_name(c)); break; } @@ -353,7 +458,7 @@ int protocol_cb_live(struct lws *wsi, enum lws_callback_reasons reason, void *us ret = queue_push(&w->queue, msg2); if (ret != 1) { - warn("Queue overrun for node: %s", node_name(c->node)); + warn("Queue overrun for connection %s", connection_name(c)); break; } @@ -451,7 +556,7 @@ int websocket_close(struct node *n) { struct websocket *w = n->_vd; - list_foreach(struct websocket_connection *c, &w->connections) { + list_foreach(struct connection *c, &w->connections) { c->state = WEBSOCKET_SHUTDOWN; lws_callback_on_writable(c->wsi); } @@ -466,7 +571,9 @@ int websocket_close(struct node *n) int websocket_destroy(struct node *n) { -// struct websocket *w = n->_vd; + struct websocket *w = n->_vd; + + list_destroy(&w->destinations, (dtor_cb_t) destination_destroy, true); return 0; } @@ -497,85 +604,16 @@ int websocket_read(struct node *n, struct sample *smps[], unsigned cnt) return got; } -static int websocket_connection_init(struct websocket_connection *c) -{ - /** @todo */ - return -1; -} - -static void websocket_connection_destroy(struct websocket_connection *c) -{ - /** @todo */ -} - -static char * websocket_connection_name(struct websocket_connection *c) -{ - /** @todo */ - return "(todo)"; -} - -static int websocket_connection_write(struct websocket_connection *c, struct sample *smps[], unsigned cnt) -{ - int blocks, enqueued; - char *bufs[cnt]; - - switch (c->state) { - case WEBSOCKET_SHUTDOWN: - return -1; - case WEBSOCKET_CLOSED: - if (c->node) { - struct websocket *w = (struct websocket *) c->node->_vd; - list_remove(&w->connections, c); - } - else - list_remove(&connections, c); - break; - - case WEBSOCKET_ESTABLISHED: - c->state = WEBSOCKET_ACTIVE; - /* fall through */ - - case WEBSOCKET_ACTIVE: - blocks = pool_get_many(&w->pool, (void **) bufs, cnt); - if (blocks != cnt) - warn("Pool underrun in websocket connection: %s", websocket_connection_name(c)); - - for (int i = 0; i < blocks; i++) { - struct webmsg *msg = (struct webmsg *) (bufs[i] + LWS_PRE); - - msg->version = WEBMSG_VERSION; - msg->type = WEBMSG_TYPE_DATA; - msg->endian = WEBMSG_ENDIAN_HOST; - msg->length = smps[i]->length; - msg->sequence = smps[i]->sequence; - msg->id = w->id; - msg->ts.sec = smps[i]->ts.origin.tv_sec; - msg->ts.nsec = smps[i]->ts.origin.tv_nsec; - - memcpy(&msg->data, &smps[i]->data, smps[i]->length * 4); - } - - enqueued = queue_push_many(&c->queue, (void **) bufs, cnt); - if (enqueued != blocks) - warn("Queue overrun in websocket connection: %s", websocket_connection_name(c)); - - lws_callback_on_writable(c->wsi); - break; - } - - return 0; -} - int websocket_write(struct node *n, struct sample *smps[], unsigned cnt) { struct websocket *w = n->_vd; - list_foreach(struct websocket_connection *c, &w->connections) { - websocket_write_connection(c, smps, cnt); + list_foreach(struct connection *c, &w->connections) { + connection_write(c, smps, cnt); } - list_foreach(struct websocket_connection *c, &connections) { - websocket_write_connection(c, smps, cnt); + list_foreach(struct connection *c, &connections) { + connection_write(c, smps, cnt); } return cnt; @@ -583,30 +621,61 @@ int websocket_write(struct node *n, struct sample *smps[], unsigned cnt) int websocket_parse(struct node *n, config_setting_t *cfg) { + struct websocket *w = n->_vd; config_setting_t *cfg_dests; + int ret; cfg_dests = config_setting_get_member(cfg, "destinations"); if (!config_setting_is_array(cfg_dests)) - cerror(dests, "The 'destinations' setting must be an array of URLs"); + cerror(cfg_dests, "The 'destinations' setting must be an array of URLs"); for (int i = 0; i < config_setting_length(cfg_dests); i++) { - config_setting_t *cfg_dest; - const char *url; - struct lws_client_connect_info *i; + struct destination *d; + const char *uri, *prot, *ads, *path; - url = config_setting_get_string_elem(cfg_dests, i); - if (!url) - cerror(dests, "The 'destinations' setting must be an array of URLs"); + uri = config_setting_get_string_elem(cfg_dests, i); + if (!uri) + cerror(cfg_dests, "The 'destinations' setting must be an array of URLs"); - i = alloc(sizeof()) + d = alloc(sizeof(struct destination)); + d->uri = strdup(uri); + if (!d->uri) + serror("Failed to allocate memory"); + + ret = lws_parse_uri(d->uri, &prot, &ads, &d->info.port, &path); + if (ret) + cerror(cfg_dests, "Failed to parse websocket URI: '%s'", uri); + + d->info.ssl_connection = !strcmp(prot, "https"); + d->info.address = ads; + d->info.path = path; + d->info.protocol = prot; + d->info.ietf_version_or_minus_one = -1; + + list_push(&w->destinations, d); } + + return 0; } char * websocket_print(struct node *n) { + struct websocket *w = n->_vd; + + char *buf = NULL; + list_foreach(struct lws_client_connect_info *in, &w->destinations) { + buf = strcatf(&buf, "%s://%s:%d/%s", + in->ssl_connection ? "https" : "http", + in->address, + in->port, + in->path + ); + } + + return buf; } static struct node_type vt = { From 1d3bb7c730624e911a20164ff8e8ebf5cd119c21 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 8 Nov 2016 00:26:06 -0500 Subject: [PATCH 41/57] some cleanups in the hook system --- include/villas/hooks.h | 5 +---- lib/hooks/hooks-internal.c | 15 +++------------ lib/path.c | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/include/villas/hooks.h b/include/villas/hooks.h index 110e01ba6..f87f228d1 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -103,7 +103,7 @@ enum hook_type { HOOK_INIT = 1 << 9, /**< Called before path is started to parseHOOK_DESTROYs. */ HOOK_DESTROY = 1 << 10, /**< Called after path has been stopped to release memory allocated by HOOK_INIT */ - HOOK_INTERNAL = 1 << 11, /**< Internal hooks are added to every path implicitely. */ + HOOK_AUTO = 1 << 11, /**< Internal hooks are added to every path implicitely. */ HOOK_PARSE = 1 << 12, /**< Called for parsing hook arguments. */ /** @{ Classes of hooks */ @@ -113,9 +113,6 @@ enum hook_type { HOOK_PATH = HOOK_PATH_START | HOOK_PATH_STOP | HOOK_PATH_RESTART, /** Hooks which are used to collect statistics. */ HOOK_STATS = HOOK_INTERNAL | HOOK_STORAGE | HOOK_PATH | HOOK_READ | HOOK_PERIODIC, - - /** All hooks */ - HOOK_ALL = HOOK_INTERNAL - 1 /** @} */ }; diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index cb91505fd..5ce5cf705 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -12,14 +12,11 @@ #include "path.h" #include "utils.h" -REGISTER_HOOK("fix_ts", "Update timestamps of sample if not set", 0, 0, hook_fix_ts, HOOK_INTERNAL | HOOK_READ) +REGISTER_HOOK("fix_ts", "Update timestamps of sample if not set", 0, 0, hook_fix_ts, HOOK_AUTO | HOOK_READ) int hook_fix_ts(struct hook *h, int when, struct hook_info *j) { struct timespec now = time_now(); - if (when != HOOK_READ) - return 0; - assert(j->smps); for (int i = 0; i < j->cnt; i++) { @@ -41,12 +38,9 @@ int hook_fix_ts(struct hook *h, int when, struct hook_info *j) return j->cnt; } -REGISTER_HOOK("restart", "Call restart hooks for current path", 1, 1, hook_restart, HOOK_INTERNAL | HOOK_READ) +REGISTER_HOOK("restart", "Call restart hooks for current path", 1, 1, hook_restart, HOOK_AUTO | HOOK_READ) int hook_restart(struct hook *h, int when, struct hook_info *j) { - if (when != HOOK_READ) - return 0; - assert(j->smps); assert(j->path); @@ -69,14 +63,11 @@ int hook_restart(struct hook *h, int when, struct hook_info *j) return j->cnt; } -REGISTER_HOOK("drop", "Drop messages with reordered sequence numbers", 3, 1, hook_drop, HOOK_INTERNAL | HOOK_READ) +REGISTER_HOOK("drop", "Drop messages with reordered sequence numbers", 3, 1, hook_drop, HOOK_AUTO | HOOK_READ) int hook_drop(struct hook *h, int when, struct hook_info *j) { int i, ok, dist; - if (when != HOOK_READ) - return 0; - assert(j->smps); for (i = 0, ok = 0; i < j->cnt; i++) { diff --git a/lib/path.c b/lib/path.c index 4a9045072..18ab94b53 100644 --- a/lib/path.c +++ b/lib/path.c @@ -171,7 +171,7 @@ int path_init(struct path *p) /* Add internal hooks if they are not already in the list*/ list_foreach(struct hook *h, &hooks) { if ( - (h->type & HOOK_INTERNAL) && /* is internal hook? */ + (h->type & HOOK_AUTO) && /* should this hook be added implicitely? */ (list_lookup(&p->hooks, h->name) == NULL) /* is not already in list? */ ) list_push(&p->hooks, memdup(h, sizeof(struct hook))); From baaa809b90caaa908fc0076a4b5e8e06a1eed2cc Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 8 Nov 2016 00:26:56 -0500 Subject: [PATCH 42/57] remove STATS_INVALID as it is too specific for the socket node-type --- include/villas/stats.h | 1 - lib/stats.c | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/villas/stats.h b/include/villas/stats.h index 49d6e11ef..f5c4178c5 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -24,7 +24,6 @@ struct path; struct node; enum stats_id { - STATS_INVALID, /**< Counter for invalid messages */ STATS_SKIPPED, /**< Counter for skipped messages due to hooks */ STATS_DROPPED, /**< Counter for dropped messages due to reordering */ STATS_GAP_SEQUENCE, /**< Histogram of sequence number displacement of received messages */ diff --git a/lib/stats.c b/lib/stats.c index 5bcceee21..e62dda812 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -23,7 +23,6 @@ static struct stats_desc { double resolution; } hist; } stats_table[] = { - { "invalid", "", "invalid messages", {0, 0, -1, }}, { "skipped", "", "skipped messages by hooks", {0, 0, -1, }}, { "dropped", "", "dropped messages because of reordering", {0, 0, -1, }}, { "gap_sequence", "", "sequence number displacement of received messages", {-10, 10, 20, }}, @@ -127,24 +126,22 @@ void stats_print_header() { #define UNIT(u) "(" YEL(u) ")" - stats("%-40s|%19s|%19s|%19s|%19s|%19s|", "Source " MAG("=>") " Destination", + stats("%-40s|%19s|%19s|%19s|%19s|", "Source " MAG("=>") " Destination", "OWD" UNIT("S") " ", "Rate" UNIT("p/S") " ", "Drop" UNIT("p") " ", - "Skip" UNIT("p") " ", - "Inval" UNIT("p") " " + "Skip" UNIT("p") " " ); line(); } void stats_print_periodic(struct stats *s, struct path *p) { - stats("%-40.40s|%10f|%10f|%10ju|%10ju|%10ju|", path_name(p), + stats("%-40.40s|%10f|%10f|%10ju|%10ju|", path_name(p), s->histograms[STATS_OWD].last, 1.0 / s->histograms[STATS_GAP_SAMPLE].last, s->histograms[STATS_DROPPED].total, - s->histograms[STATS_SKIPPED].total, - s->histograms[STATS_INVALID].total + s->histograms[STATS_SKIPPED].total ); } From f50e44937348491d51bd6603fcd9dd1244196e62 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 8 Nov 2016 00:27:28 -0500 Subject: [PATCH 43/57] refactoring in the stats system to fix some compiler errors --- include/villas/hooks.h | 2 +- include/villas/path.h | 2 +- include/villas/stats.h | 12 ++++---- lib/hooks/hooks-internal.c | 3 +- lib/hooks/hooks-stats.c | 3 +- lib/stats.c | 56 +++++++++++++++----------------------- 6 files changed, 34 insertions(+), 44 deletions(-) diff --git a/include/villas/hooks.h b/include/villas/hooks.h index f87f228d1..f88635a21 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -112,7 +112,7 @@ enum hook_type { /** All path related actions */ HOOK_PATH = HOOK_PATH_START | HOOK_PATH_STOP | HOOK_PATH_RESTART, /** Hooks which are used to collect statistics. */ - HOOK_STATS = HOOK_INTERNAL | HOOK_STORAGE | HOOK_PATH | HOOK_READ | HOOK_PERIODIC, + HOOK_STATS = HOOK_STORAGE | HOOK_PATH | HOOK_READ | HOOK_PERIODIC, /** @} */ }; diff --git a/include/villas/path.h b/include/villas/path.h index 7bf71a003..c9e5104aa 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -67,7 +67,7 @@ struct path char *_name; /**< Singleton: A string which is used to print this path to screen. */ - struct stats *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ + struct stats_delta *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path */ }; diff --git a/include/villas/stats.h b/include/villas/stats.h index f5c4178c5..dc69beb41 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -34,7 +34,9 @@ enum stats_id { }; struct stats_delta { - double vals[STATS_COUNT]; + double values[STATS_COUNT]; + + struct sample *last; }; struct stats { @@ -47,14 +49,12 @@ int stats_init(struct stats *s); void stats_destroy(struct stats *s); -void stats_update(struct stats *s, enum stats_id id, double val); +void stats_update(struct stats_delta *s, enum stats_id id, double val); + +void stats_collect(struct stats_delta *s, struct sample *smps[], size_t cnt); int stats_commit(struct stats *s, struct stats_delta *d); -void stats_collect(struct stats *s, struct sample *smps[], size_t cnt); - -void stats_decollect(struct stats *s, struct sample *smps[], size_t cnt); - #ifdef WITH_JANSSON json_t * stats_json(struct stats *s); #endif diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index 5ce5cf705..81673a3c3 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -76,8 +76,9 @@ int hook_drop(struct hook *h, int when, struct hook_info *j) if (h->prev) { dist = h->last->sequence - (int32_t) h->prev->sequence; if (dist <= 0) { - stats_update(j->path->stats, STATS_DROPPED, dist); warn("Dropped sample: dist = %d, i = %d", dist, i); + if (j->path) + stats_update(j->path->stats, STATS_DROPPED, dist); } else { struct sample *tmp; diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index e0139efff..f9fb97b7f 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -22,8 +22,9 @@ int hook_stats(struct hook *h, int when, struct hook_info *j) switch (when) { case HOOK_READ: assert(j->smps); + assert(j->path); - stats_collect(s, j->smps, j->cnt); + stats_collect(j->path->stats, j->smps, j->cnt); break; case HOOK_PATH_STOP: diff --git a/lib/stats.c b/lib/stats.c index e62dda812..6441edf85 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -48,54 +48,42 @@ void stats_destroy(struct stats *s) } } -void stats_update(struct stats *s, enum stats_id id, double val) +void stats_update(struct stats_delta *d, enum stats_id id, double val) { - if (!s) - return; - - hist_put(&s->histograms[id], val); + if (d && id >= 0 && id < STATS_COUNT) + d->values[id] = val; } -#if 0 -int stats_delta(struct stats_delta *d, struct sample *s, struct sample *p) -{ - d->histogram.owd = time_delta(&smps[i]->ts.origin, &smps[i]->ts.received); - d->histogram.gap = time_delta(&s->last->ts.origin, &smps[i]->ts.origin); - d->histogram.gap_seq = s->sequence - (int32_t) p->sequence; - d->histogram.gap_recv = time_delta(&s->last->ts.received, &smps[i]->ts.received); - - d->counter.dropped = d->histogram.gap_seq <= 0 ? 1 : 0; - d->counter.invalid = 0; - - return 0; -} -#endif - int stats_commit(struct stats *s, struct stats_delta *d) { for (int i = 0; i < STATS_COUNT; i++) { - hist_put(&s->histograms[i], d->vals[i]); + hist_put(&s->histograms[i], d->values[i]); } return 0; } -void stats_collect(struct stats *s, struct sample *smps[], size_t cnt) +void stats_collect(struct stats_delta *s, struct sample *smps[], size_t cnt) { - for (int i = 0; i < cnt; i++) { - if (s->last) { -// struct stats_delta d; -// stats_get_delta(&d, smps[i], s->last); -// stats_commit(s, &d); + struct sample *previous = s->last; + + if (previous) { + sample_put(previous); + + for (int i = 0; i < cnt; i++) { + stats_update(s, STATS_GAP_RECEIVED, time_delta(&previous->ts.received, &smps[i]->ts.received)); + stats_update(s, STATS_GAP_SAMPLE, time_delta(&previous->ts.origin, &smps[i]->ts.origin)); + stats_update(s, STATS_OWD, time_delta(&smps[i]->ts.origin, &smps[i]->ts.received)); + stats_update(s, STATS_GAP_SEQUENCE, smps[i]->sequence - (int32_t) previous->sequence); + + /* Make sure there is always a reference to the previous sample */ + + previous = smps[i]; } - - if (i == 0 && s->last) - sample_put(s->last); - if (i == cnt - 1) - sample_get(smps[i]); - - s->last = smps[i]; } + + sample_get(previous); + s->last = previous; } #ifdef WITH_JANSSON From 90e973ce37e5b4bfbf57cd354ea095c0cf75269d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 8 Nov 2016 00:36:04 -0500 Subject: [PATCH 44/57] fix memory allocation bug discovered by CI :-) --- include/villas/hist.h | 3 ++- lib/hist.c | 6 ++++-- tests/hist.c | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/villas/hist.h b/include/villas/hist.h index abb3ea091..9779ffef6 100644 --- a/include/villas/hist.h +++ b/include/villas/hist.h @@ -11,6 +11,7 @@ #define _HIST_H_ #include +#include #include "config.h" @@ -46,7 +47,7 @@ struct hist { }; /** Initialize struct hist with supplied values and allocate memory for buckets. */ -void hist_create(struct hist *h, double start, double end, double resolution); +int hist_create(struct hist *h, double start, double end, double resolution); /** Free the dynamically allocated memory. */ void hist_destroy(struct hist *h); diff --git a/lib/hist.c b/lib/hist.c index 21baa6784..2cda30811 100644 --- a/lib/hist.c +++ b/lib/hist.c @@ -19,7 +19,7 @@ #define VAL(h, i) ((h)->low + (i) * (h)->resolution) #define INDEX(h, v) round((v - (h)->low) / (h)->resolution) -void hist_create(struct hist *h, double low, double high, double resolution) +int hist_create(struct hist *h, double low, double high, double resolution) { h->low = low; h->high = high; @@ -27,7 +27,7 @@ void hist_create(struct hist *h, double low, double high, double resolution) if (resolution > 0) { h->length = (high - low) / resolution; - h->data = alloc(h->length * sizeof(unsigned)); + h->data = alloc(h->length * sizeof(hist_cnt_t)); } else { h->length = 0; @@ -35,6 +35,8 @@ void hist_create(struct hist *h, double low, double high, double resolution) } hist_reset(h); + + return 0; } void hist_destroy(struct hist *h) diff --git a/tests/hist.c b/tests/hist.c index df9c655e8..aeca35701 100644 --- a/tests/hist.c +++ b/tests/hist.c @@ -18,8 +18,10 @@ const int hist_result[] = {}; Test(hist, simple) { struct hist h; + int ret; - hist_create(&h, -100, 100, 1); + ret = hist_create(&h, -100, 100, 1); + cr_assert_eq(ret, 0); for (int i = 0; i < ARRAY_LEN(test_data); i++) hist_put(&h, test_data[i]); From 4bb0aa76c9b25a3a3723e9dde94ab7f118a177e7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 02:45:16 -0500 Subject: [PATCH 45/57] increase number of reserved huge pages automatically if required --- config.h | 4 ++++ include/villas/kernel/kernel.h | 10 +++++++++- include/villas/memory.h | 3 +++ lib/kernel/kernel.c | 32 ++++++++++++++++++++++++++++++++ lib/memory.c | 15 +++++++++++++++ src/node.c | 4 ++++ src/pipe.c | 3 +++ src/test.c | 3 +++ tests/Makefile.inc | 1 - 9 files changed, 73 insertions(+), 2 deletions(-) diff --git a/config.h b/config.h index 087f7b3f9..f40f333a9 100644 --- a/config.h +++ b/config.h @@ -24,6 +24,10 @@ #define DEFAULT_VALUES 64 #define DEFAULT_QUEUELEN 1024 +/** Number of hugepages which are requested from the the kernel. + * @see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt */ +#define DEFAULT_NR_HUGEPAGES 25 + /** Width of log output in characters */ #define LOG_WIDTH 132 diff --git a/include/villas/kernel/kernel.h b/include/villas/kernel/kernel.h index 10e750468..300866e1c 100644 --- a/include/villas/kernel/kernel.h +++ b/include/villas/kernel/kernel.h @@ -12,6 +12,8 @@ #include +#include "config.h" + //#include /** Check if current process has capability \p cap. @@ -19,7 +21,13 @@ * @retval 0 If capabilty is present. * @retval <0 If capability is not present. */ -//int kernel_check_cap(cap_value_t cap): +//int kernel_check_cap(cap_value_t cap); + +/** Get number of reserved hugepages. */ +int kernel_get_nr_hugepages(); + +/** Set number of reserved hugepages. */ +int kernel_set_nr_hugepages(int nr); /** Checks for realtime (PREEMPT_RT) patched kernel. * diff --git a/include/villas/memory.h b/include/villas/memory.h index f6e3064f5..e546c106b 100644 --- a/include/villas/memory.h +++ b/include/villas/memory.h @@ -44,6 +44,9 @@ struct memzone { size_t len; }; +/** Initilialize memory subsystem */ +int memory_init(); + /** Allocate \p len bytes memory of type \p m. * * @retval NULL If allocation failed. diff --git a/lib/kernel/kernel.c b/lib/kernel/kernel.c index b2c9b94cc..00a0818e7 100644 --- a/lib/kernel/kernel.c +++ b/lib/kernel/kernel.c @@ -152,6 +152,38 @@ int kernel_get_cacheline_size() return sysconf(_SC_LEVEL1_ICACHE_LINESIZE); } +int kernel_get_nr_hugepages() +{ + FILE *f; + int nr, ret; + + f = fopen(PROCFS_PATH "/sys/vm/nr_hugepages", "r"); + if (!f) + serror("Failed to open %s", PROCFS_PATH "/sys/vm/nr_hugepages"); + + ret = fscanf(f, "%d", &nr); + if (ret != 1) + nr = -1; + + fclose(f); + + return nr; +} + +int kernel_set_nr_hugepages(int nr) +{ + FILE *f; + + f = fopen(PROCFS_PATH "/sys/vm/nr_hugepages", "w"); + if (!f) + serror("Failed to open %s", PROCFS_PATH "/sys/vm/nr_hugepages"); + + fprintf(f, "%d\n", nr); + fclose(f); + + return 0; +} + #if 0 int kernel_check_cap(cap_value_t cap) { diff --git a/lib/memory.c b/lib/memory.c index 2f46161b3..2e3120450 100644 --- a/lib/memory.c +++ b/lib/memory.c @@ -12,11 +12,26 @@ /* Required to allocate hugepages on Apple OS X */ #ifdef __MACH__ #include +#elif defined(__linux__) + #include "kernel/kernel.h" #endif #include "log.h" #include "memory.h" +int memory_init() +{ +#ifdef __linux__ + int nr = kernel_get_nr_hugepages(); + + debug(DBG_MEM | 2, "System has %d reserved hugepages", nr); + + if (nr < DEFAULT_NR_HUGEPAGES) + kernel_set_nr_hugepages(DEFAULT_NR_HUGEPAGES); +#endif + return 0; +} + void * memory_alloc(const struct memtype *m, size_t len) { debug(DBG_MEM | 2, "Allocating %#zx bytes of %s memory", len, m->name); diff --git a/src/node.c b/src/node.c index 93c42b1fc..a578f507a 100644 --- a/src/node.c +++ b/src/node.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -124,6 +125,9 @@ int main(int argc, char *argv[]) info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); + + info("Initialize memory system"); + memory_init(); info("Initialize signals"); signals_init(); diff --git a/src/pipe.c b/src/pipe.c index 2226c917d..16c4e0a35 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -225,6 +225,9 @@ int main(int argc, char *argv[]) info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); + info("Initialize memory system"); + memory_init(); + /* Initialize node */ node = list_lookup(&nodes, argv[2]); if (!node) diff --git a/src/test.c b/src/test.c index bb31796d0..18d8e8eef 100644 --- a/src/test.c +++ b/src/test.c @@ -87,6 +87,9 @@ int main(int argc, char *argv[]) log_init(); cfg_parse(argv[1], &config, &settings, &nodes, NULL); + info("Initialize memory system"); + memory_init(); + node = list_lookup(&nodes, argv[3]); if (!node) error("There's no node with the name '%s'", argv[3]); diff --git a/tests/Makefile.inc b/tests/Makefile.inc index 9bac51525..277851453 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -8,7 +8,6 @@ TEST_LDLIBS = $(LDLIBS) -lcriterion -lvillas -pthread tests: $(BUILDDIR)/testsuite run-tests: tests - echo 25 > /proc/sys/vm/nr_hugepages $(BUILDDIR)/testsuite # Compile From eda6c66f0eaa9547f80cc3af920897914b34484b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 02:45:39 -0500 Subject: [PATCH 46/57] use proper style for #include --- src/hook.c | 3 ++- src/node.c | 6 +++--- src/signal.c | 7 ++++--- src/test.c | 20 +++++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/hook.c b/src/hook.c index 7b04cd27e..d30133410 100644 --- a/src/hook.c +++ b/src/hook.c @@ -18,7 +18,8 @@ #include #include #include -#include + +#include "config.h" static void usage() { diff --git a/src/node.c b/src/node.c index a578f507a..4b8d9f681 100644 --- a/src/node.c +++ b/src/node.c @@ -12,8 +12,6 @@ #include #include -#include "config.h" - #include #include #include @@ -23,9 +21,11 @@ #include #ifdef ENABLE_OPAL_ASYNC - #include "opal.h" + #include #endif +#include "config.h" + struct list paths; /**< List of paths */ struct list nodes; /**< List of nodes */ diff --git a/src/signal.c b/src/signal.c index 2d47dfa95..601c350d3 100644 --- a/src/signal.c +++ b/src/signal.c @@ -14,10 +14,11 @@ #include #include +#include +#include +#include + #include "config.h" -#include "utils.h" -#include "sample.h" -#include "timing.h" #define CLOCKID CLOCK_REALTIME diff --git a/src/test.c b/src/test.c index 18d8e8eef..8e4f56ae6 100644 --- a/src/test.c +++ b/src/test.c @@ -13,14 +13,15 @@ #include #include +#include +#include +#include +#include +#include +#include +#include + #include "config.h" -#include "cfg.h" -#include "msg.h" -#include "node.h" -#include "utils.h" -#include "hist.h" -#include "timing.h" -#include "pool.h" struct settings settings; /** Date: Sun, 20 Nov 2016 02:50:12 -0500 Subject: [PATCH 47/57] fix: initialize hooks only if required --- lib/hooks.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/hooks.c b/lib/hooks.c index daad04c42..5eb56c7ee 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -27,7 +27,10 @@ int hook_init(struct hook *h, struct list *nodes, struct list *paths, struct set .settings = settings }; - return h->cb(h, HOOK_INIT, &i); + if (h->type & HOOK_INIT) + return h->cb(h, HOOK_INIT, &i); + else + return 0; } void hook_destroy(struct hook *h) From b8a8476a923c022c6be2b66fdddf2207fabf7bca Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 02:50:49 -0500 Subject: [PATCH 48/57] add custom main() for criterion tests in order to initiatize huge pages --- tests/main.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/main.c diff --git a/tests/main.c b/tests/main.c new file mode 100644 index 000000000..370ba501f --- /dev/null +++ b/tests/main.c @@ -0,0 +1,26 @@ +/** Custom main() for Criterion + * + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#include + +#include +#include + +int main(int argc, char *argv[]) { + struct criterion_test_set *tests = criterion_initialize(); + + info("Initialize memory system"); + memory_init(); + + int result = 0; + if (criterion_handle_args(argc, argv, true)) + result = !criterion_run_all_tests(tests); + + criterion_finalize(tests); + return result; +} \ No newline at end of file From bfec0b2f0091d72e800f2a64342f53868ce76ed3 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 03:02:19 -0500 Subject: [PATCH 49/57] added missing implementation for aligned memory allocator --- include/villas/memory.h | 2 +- lib/memory.c | 29 +++++++++++++++++++++-------- tests/memory.c | 5 ++--- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/include/villas/memory.h b/include/villas/memory.h index e546c106b..0390a0b92 100644 --- a/include/villas/memory.h +++ b/include/villas/memory.h @@ -15,7 +15,7 @@ #define HUGEPAGESIZE (1 << 21) -typedef void *(*memzone_allocator_t)(size_t len); +typedef void *(*memzone_allocator_t)(size_t len, size_t alignment); typedef int (*memzone_deallocator_t)(void *ptr, size_t len); enum memtype_flags { diff --git a/lib/memory.c b/lib/memory.c index 2e3120450..85ef52d7f 100644 --- a/lib/memory.c +++ b/lib/memory.c @@ -34,15 +34,20 @@ int memory_init() void * memory_alloc(const struct memtype *m, size_t len) { - debug(DBG_MEM | 2, "Allocating %#zx bytes of %s memory", len, m->name); - return m->alloc(len); + void *ptr = m->alloc(len, sizeof(void *)); + + debug(DBG_MEM | 2, "Allocated %#zx bytes of %s memory: %p", len, m->name, ptr); + + return ptr; } void * memory_alloc_aligned(const struct memtype *m, size_t len, size_t alignment) { - debug(DBG_MEM | 2, "Allocating %#zx bytes of %#zx-byte-aligned %s memory", len, alignment, m->name); - warn("%s: not implemented yet!", __FUNCTION__); - return memory_alloc(m, len); + void *ptr = m->alloc(len, alignment); + + debug(DBG_MEM | 2, "Allocated %#zx bytes of %#zx-byte-aligned %s memory: %p", len, alignment, m->name, ptr); + + return ptr; } int memory_free(const struct memtype *m, void *ptr, size_t len) @@ -51,9 +56,17 @@ int memory_free(const struct memtype *m, void *ptr, size_t len) return m->free(ptr, len); } -static void * memory_heap_alloc(size_t len) +static void * memory_heap_alloc(size_t len, size_t alignment) { - return malloc(len); + void *ptr; + int ret; + + if (alignment < sizeof(void *)) + alignment = sizeof(void *); + + ret = posix_memalign(&ptr, alignment, len); + + return ret ? NULL : ptr; } int memory_heap_free(void *ptr, size_t len) @@ -64,7 +77,7 @@ int memory_heap_free(void *ptr, size_t len) } /** Allocate memory backed by hugepages with malloc() like interface */ -static void * memory_hugepage_alloc(size_t len) +static void * memory_hugepage_alloc(size_t len, size_t alignment) { int prot = PROT_READ | PROT_WRITE; int flags = MAP_PRIVATE | MAP_ANONYMOUS; diff --git a/tests/memory.c b/tests/memory.c index 234e6defd..6c56d1be8 100644 --- a/tests/memory.c +++ b/tests/memory.c @@ -15,8 +15,7 @@ #include "utils.h" TheoryDataPoints(memory, aligned) = { -// DataPoints(size_t, 1, 32, 55, 1 << 10, 1 << 20), - DataPoints(size_t, 1<<12), + DataPoints(size_t, 1, 32, 55, 1 << 10, 1 << 20), DataPoints(size_t, 1, 8, 1 << 12), DataPoints(const struct memtype *, &memtype_heap, &memtype_hugepage) }; @@ -28,7 +27,7 @@ Theory((size_t len, size_t align, const struct memtype *m), memory, aligned) { ptr = memory_alloc_aligned(m, len, align); cr_assert_neq(ptr, NULL, "Failed to allocate memory"); - //cr_assert(IS_ALIGNED(ptr, align)); + cr_assert(IS_ALIGNED(ptr, align)); if (m == &memtype_hugepage) { cr_assert(IS_ALIGNED(ptr, HUGEPAGESIZE)); From b9d5bccb570eee82199bc9befa1be70bed98b5bd Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 03:40:23 -0500 Subject: [PATCH 50/57] improved debug outputs for paths --- lib/path.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/path.c b/lib/path.c index 18ab94b53..271de36e2 100644 --- a/lib/path.c +++ b/lib/path.c @@ -62,7 +62,7 @@ static void path_read(struct path *p) for (int i = 0; i < enqueued; i++) sample_get(smps[i]); /* increase reference count */ - debug(DBG_PATH | 15, "Enqueued %u samples to queue of path %s", enqueued, path_name(p)); + debug(DBG_PATH | 15, "Enqueued %u samples from %s to queue of %s", enqueued, node_name(ps->node), node_name(pd->node)); } } @@ -154,11 +154,22 @@ int path_stop(struct path *p) const char * path_name(struct path *p) { - if (!p->_name) { - strcatf(&p->_name, "%s " MAG("=>"), node_name_short(p->source->node)); - - list_foreach(struct path_destination *pd, &p->destinations) - strcatf(&p->_name, " %s", node_name_short(pd->node)); + if (!p->_name) { + if (list_length(&p->destinations) == 1) { + struct path_destination *pd = (struct path_destination *) list_first(&p->destinations); + + strcatf(&p->_name, "%s " MAG("=>") " %s", + node_name_short(p->source->node), + node_name_short(pd->node)); + } + else { + strcatf(&p->_name, "%s " MAG("=>") " [", node_name_short(p->source->node)); + + list_foreach(struct path_destination *pd, &p->destinations) + strcatf(&p->_name, " %s", node_name_short(pd->node)); + + strcatf(&p->_name, " ]"); + } } return p->_name; From 8240bf52c96ffb865d22aa3f8a144d958ed33b1d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 03:44:51 -0500 Subject: [PATCH 51/57] fixed debug output: incorrect number of released samples --- lib/path.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/path.c b/lib/path.c index 271de36e2..e5c7728b9 100644 --- a/lib/path.c +++ b/lib/path.c @@ -100,8 +100,10 @@ static void path_write(struct path *p) debug(DBG_PATH | 15, "Sent %u messages to node %s", sent, node_name(pd->node)); released = 0; - for (int i = 0; i < sent; i++) - released += sample_put(smps[i]); + for (int i = 0; i < sent; i++) { + if (sample_put(smps[i]) == 0) + released++; /* we had the last reference (0 remaining) */ + } debug(DBG_PATH | 15, "Released %d samples back to memory pool", released); } From 671e964d4309f6db1ecc592986a98133873a1404 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 03:46:53 -0500 Subject: [PATCH 52/57] only call HOOK_DESTROY callback if hook supports it --- lib/hooks.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/hooks.c b/lib/hooks.c index 5eb56c7ee..15492a52d 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -36,7 +36,9 @@ int hook_init(struct hook *h, struct list *nodes, struct list *paths, struct set void hook_destroy(struct hook *h) { struct hook_info i = { NULL }; - h->cb(h, HOOK_DESTROY, &i); + + if (h->type & HOOK_DESTROY) + h->cb(h, HOOK_DESTROY, &i); } int hook_copy(struct hook *h, struct hook *c) From cbaf8d40b3f3939d2bc5495261ea1db1e840cd01 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 03:47:41 -0500 Subject: [PATCH 53/57] remove HOOK_STATS as it is only used once in the codebase --- include/villas/hooks.h | 4 +--- lib/hooks/hooks-stats.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/villas/hooks.h b/include/villas/hooks.h index f88635a21..66501985b 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -110,9 +110,7 @@ enum hook_type { /** Hooks which are using private data must allocate and free them propery. */ HOOK_STORAGE = HOOK_INIT | HOOK_DESTROY, /** All path related actions */ - HOOK_PATH = HOOK_PATH_START | HOOK_PATH_STOP | HOOK_PATH_RESTART, - /** Hooks which are used to collect statistics. */ - HOOK_STATS = HOOK_STORAGE | HOOK_PATH | HOOK_READ | HOOK_PERIODIC, + HOOK_PATH = HOOK_PATH_START | HOOK_PATH_STOP | HOOK_PATH_RESTART /** @} */ }; diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index f9fb97b7f..bd78d3c87 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -14,7 +14,7 @@ extern struct list *hook_nodes; -REGISTER_HOOK("stats", "Collect statistics for the current path", 2, 1, hook_stats, HOOK_STATS) +REGISTER_HOOK("stats", "Collect statistics for the current path", 2, 1, hook_stats, HOOK_STORAGE | HOOK_PATH | HOOK_READ | HOOK_PERIODIC) int hook_stats(struct hook *h, int when, struct hook_info *j) { struct stats *s = hook_storage(h, when, sizeof(struct stats), (ctor_cb_t) stats_init, (dtor_cb_t) stats_destroy); From b37d3df332014b0f3c7169db359656b43a66a2e8 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 12:59:37 -0500 Subject: [PATCH 54/57] moved cfg_parse_{hook, node, path} to {hooks, node, path}.c --- config.h | 7 +- etc/global.conf | 21 ++- include/villas/cfg.h | 60 ------- include/villas/hooks.h | 18 ++ include/villas/log.h | 17 +- include/villas/node.h | 31 +++- include/villas/path.h | 20 ++- lib/cfg.c | 400 +++++++++++++---------------------------- lib/hooks.c | 50 ++++++ lib/log.c | 48 ++++- lib/node.c | 103 ++++++++++- lib/path.c | 86 +++++++++ src/fpga.c | 7 +- src/hook.c | 7 +- src/node.c | 4 +- src/pipe.c | 8 +- src/signal.c | 4 +- src/test.c | 5 +- 18 files changed, 507 insertions(+), 389 deletions(-) diff --git a/config.h b/config.h index f40f333a9..3f4203fae 100644 --- a/config.h +++ b/config.h @@ -69,8 +69,13 @@ struct settings { int priority; /**< Process priority (lower is better) */ int affinity; /**< Process affinity of the server and all created threads */ - int debug; /**< Debug log level */ double stats; /**< Interval for path statistics. Set to 0 to disable themo disable them. */ + + struct { + int level; /**< Log level */ + int facilities; /**< Debug facilities */ + const char *file; /**< Log file */ + } log; }; #endif /* _CONFIG_H_ */ diff --git a/etc/global.conf b/etc/global.conf index 17d51c86f..fb81612fc 100644 --- a/etc/global.conf +++ b/etc/global.conf @@ -1,9 +1,5 @@ # Global configuration file for VILLASnode # -# This example includes all valid configuration options for the server. -# Please note, that using all options at the same time does not really -# makes sense. The purpose of this example is to serve as a reference. -# # The syntax of this file is similar to JSON. # A detailed description of the format can be found here: # http://www.hyperrealm.com/libconfig/libconfig_manual.html#Configuration-Files @@ -17,22 +13,31 @@ affinity = 0x01; # Mask of cores the server should run on # This also maps the NIC interrupts to those cores! -#priority = 50; # Priority for the server tasks. +priority = 50; # Priority for the server tasks. # Usually the server is using a real-time FIFO # scheduling algorithm # See: https://github.com/docker/docker/issues/22380 # on why we cant use real-time scheduling in Docker -debug = 5; # The level of verbosity for debug messages - # Higher number => increased verbosity - stats = 3; # The interval in seconds to print path statistics. # A value of 0 disables the statistics. name = "villas-acs" # The name of this VILLASnode. Might by used by node-types # to identify themselves (default is the hostname). + +log = { + level = 5; # The level of verbosity for debug messages + # Higher number => increased verbosity + + faciltities = [ "path", "socket" ]; # The list of enabled debug faciltities. + # If omitted, all faciltities are enabled + # For a full list of available faciltities, check lib/log.c + + file = "/var/log/villas-node.log"; # File for logs +}; + http = { htdocs = "/villas/web/socket/", # Root directory of internal webserver port = 80 # Port for HTTP connections diff --git a/include/villas/cfg.h b/include/villas/cfg.h index 9d8dcea75..d9640abd0 100644 --- a/include/villas/cfg.h +++ b/include/villas/cfg.h @@ -48,64 +48,4 @@ void cfg_destroy(config_t *cfg); int cfg_parse(const char *filename, config_t *cfg, struct settings *set, struct list *nodes, struct list *paths); -/** Parse the global section of a configuration file. - * - * @param cfg A libconfig object pointing to the root of the file - * @param set The global configuration file - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int cfg_parse_global(config_setting_t *cfg, struct settings *set); - -/** Parse a single path and add it to the global configuration. - * - * @param cfg A libconfig object pointing to the path - * @param paths Add new paths to this linked list - * @param nodes A linked list of all existing nodes - * @param set The global configuration structure - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int cfg_parse_path(config_setting_t *cfg, - struct list *paths, struct list *nodes, struct settings *set); - -/** Parse an array or single node and checks if they exist in the "nodes" section. - * - * Examples: - * out = [ "sintef", "scedu" ] - * out = "acs" - * - * @param cfg The libconfig object handle for "out". - * @param nodes The nodes will be added to this list. - * @param all This list contains all valid nodes. - */ -int cfg_parse_nodelist(config_setting_t *cfg, struct list *nodes, struct list *all); - -/** Parse an array or single hook function. - * - * Examples: - * hooks = [ "print", "fir" ] - * hooks = "log" - **/ -int cfg_parse_hooklist(config_setting_t *cfg, struct list *hooks); - -/** Parse a single hook and append it to the list. - * A hook definition is composed of the hook name and optional parameters - * seperated by a colon. - * - * Examples: - * "print:stdout" - */ -int cfg_parse_hook(config_setting_t *cfg, struct list *list); - -/** Parse a single node and add it to the global configuration. - * - * @param cfg A libconfig object pointing to the node. - * @param nodes Add new nodes to this linked list. - * @param set The global configuration structure - * @retval 0 Success. Everything went well. - * @retval <0 Error. Something went wrong. - */ -int cfg_parse_node(config_setting_t *cfg, struct list *nodes, struct settings *set); - #endif /* _CFG_H_ */ diff --git a/include/villas/hooks.h b/include/villas/hooks.h index 66501985b..cf693305b 100644 --- a/include/villas/hooks.h +++ b/include/villas/hooks.h @@ -47,6 +47,7 @@ struct path; struct hook; struct sample; struct settings; +typedef struct config_setting_t config_setting_t; /** This is a list of hooks which can be used in the configuration file. */ extern struct list hooks; @@ -176,4 +177,21 @@ int hook_fix_ts(struct hook *h, int when, struct hook_info *j); int hook_restart(struct hook *h, int when, struct hook_info *j); int hook_drop(struct hook *h, int when, struct hook_info *j); +/** Parse an array or single hook function. + * + * Examples: + * hooks = [ "print", "fir" ] + * hooks = "log" + */ +int hook_parse_list(struct list *list, config_setting_t *cfg); + +/** Parse a single hook and append it to the list. + * A hook definition is composed of the hook name and optional parameters + * seperated by a colon. + * + * Examples: + * "print:stdout" + */ +int hook_parse(config_setting_t *cfg, struct list *list); + #endif /** _HOOKS_H_ @} */ diff --git a/include/villas/log.h b/include/villas/log.h index c57d686bf..c5a8e1a56 100644 --- a/include/villas/log.h +++ b/include/villas/log.h @@ -42,12 +42,12 @@ enum debug_facilities { /* Node-types */ DBG_SOCKET = (1 << 16), - DBG_FILE = (1 << 17), - DBG_FPGA = (1 << 18), - DBG_NGSI = (1 << 19), - DBG_WEBSOCKET = (1 << 20), + DBG_WEBSOCKET = (1 << 17), + DBG_FILE = (1 << 18), + DBG_FPGA = (1 << 19), + DBG_NGSI = (1 << 20), DBG_OPAL = (1 << 21), - DBG_NODE = (0xFF << 16) + DBG_NODES = (0xFF << 16) }; /** Change log indention for current thread. @@ -63,15 +63,14 @@ int log_indent(int levels); */ void log_outdent(int *); -/** Set the verbosity level of debug messages. +/** Reset the wallclock of debug messages. * * @param lvl The new debug level. * @param fac The new mask for debug facilities. */ -void log_setlevel(int lvl, int fac); +void log_init(int lvl, int fac, const char *path); -/** Reset the wallclock of debug messages. */ -void log_init(); +int log_lookup_facility(const char *facility_name); /** Logs variadic messages to stdout. * diff --git a/include/villas/node.h b/include/villas/node.h index d57553402..defbe2b17 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -30,8 +30,9 @@ __attribute__((constructor)) static void __register() { \ extern struct list node_types; /**< Vtable for virtual node sub types */ /* Forward declarations */ -struct config_setting_t cfg; struct node_type; +struct settings; +typedef struct config_setting_t config_setting_t; /** The data structure for a node. * @@ -219,12 +220,6 @@ int node_start(struct node *n); */ int node_stop(struct node *n); -/** Parse node connection details. - * - * @see node_type::parse - */ -int node_parse(struct node *n, config_setting_t *cfg); - /** Return a pointer to a string which should be used to print this node. * * @see node::_name‚ @@ -256,4 +251,26 @@ int node_read(struct node *n, struct sample *smps[], unsigned cnt); int node_write(struct node *n, struct sample *smps[], unsigned cnt); +/** Parse an array or single node and checks if they exist in the "nodes" section. + * + * Examples: + * out = [ "sintef", "scedu" ] + * out = "acs" + * + * @param cfg The libconfig object handle for "out". + * @param nodes The nodes will be added to this list. + * @param all This list contains all valid nodes. + */ +int node_parse_list(struct list *list, config_setting_t *cfg, struct list *all); + +/** Parse a single node and add it to the global configuration. + * + * @param cfg A libconfig object pointing to the node. + * @param nodes Add new nodes to this linked list. + * @param set The global configuration structure + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int node_parse(struct node *n, config_setting_t *cfg, struct settings *set); + #endif /** _NODE_H_ @} */ diff --git a/include/villas/path.h b/include/villas/path.h index c9e5104aa..f3913265d 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -28,6 +28,9 @@ #include "pool.h" #include "stats.h" +/* Forward declarations */ +typedef struct config_setting_t config_setting_t; + struct path_source { struct node *node; @@ -62,16 +65,20 @@ struct path struct list hooks; /**< List of function pointers to hooks */ int enabled; /**< Is this path enabled */ + int reverse; /**< This path as a matching reverse path */ pthread_t tid; /**< The thread id for this path */ char *_name; /**< Singleton: A string which is used to print this path to screen. */ - struct stats_delta *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ + struct stats *stats; /**< Statistic counters. This is a pointer to the statistic hooks private data. */ config_setting_t *cfg; /**< A pointer to the libconfig object which instantiated this path */ }; +/** Allocate memory for a new path */ +struct path * path_create(); + /** Initialize internal data structures. */ int path_init(struct path *p); @@ -120,4 +127,15 @@ int path_reverse(struct path *p, struct path *r); /** Check if node is used as source or destination of a path. */ int path_uses_node(struct path *p, struct node *n); +/** Parse a single path and add it to the global configuration. + * + * @param cfg A libconfig object pointing to the path + * @param p Pointer to the allocated memory for this path + * @param nodes A linked list of all existing nodes + * @param set The global configuration structure + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes, struct settings *set); + #endif /** _PATH_H_ @} */ diff --git a/lib/cfg.c b/lib/cfg.c index 038744dc9..6f1328bc2 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -19,6 +19,88 @@ #include "path.h" #include "hooks.h" +static int cfg_parse_log(config_setting_t *cfg, struct settings *set) +{ + config_setting_t *cfg_facilities; + + if (!config_setting_lookup_int(cfg, "level", &set->log.level)) + set->log.level = V; + + if (!config_setting_lookup_string(cfg, "file", &set->log.file)) + set->log.file = NULL; + + cfg_facilities = config_setting_get_member(cfg, "facilities"); + if (cfg_facilities) { + set->log.facilities = 0; + + for (int i = 0; i < config_setting_length(cfg); i++) { + int facility; + const char *str; + config_setting_t *elm = config_setting_get_elem(cfg, i); + + str = config_setting_get_string(elm); + if (!str) + cerror(elm, "Facilties must be configured as an array of strings"); + + facility = log_lookup_facility(str); + if (!facility) + cerror(elm, "%s is an unknown debug facility", str); + + set->log.facilities |= facility; + } + } + else /* By default we enable all faciltities */ + set->log.facilities = ~0; + + return 0; +} + +static int cfg_parse_plugins(config_setting_t *cfg) +{ + if (!config_setting_is_array(cfg)) + cerror(cfg, "Setting 'plugins' must be a list of strings"); + + for (int i = 0; i < config_setting_length(cfg); i++) { + void *handle; + const char *path; + + path = config_setting_get_string_elem(cfg, i); + if (!path) + cerror(cfg, "Setting 'plugins' must be a list of strings"); + + handle = dlopen(path, RTLD_NOW); + if (!handle) + error("Failed to load plugin %s", dlerror()); + } + + return 0; +} + +static int cfg_parse_global(config_setting_t *cfg, struct settings *set) +{ + config_setting_t *cfg_plugins, *cfg_log; + + if (!config_setting_lookup_int(cfg, "affinity", &set->affinity)) + set->affinity = 0; + + if (!config_setting_lookup_int(cfg, "priority", &set->priority)) + set->priority = 0; + + if (!config_setting_lookup_float(cfg, "stats", &set->stats)) + set->stats = 0; + + + cfg_log = config_setting_get_member(cfg, "log"); + if (cfg_log) + cfg_parse_log(cfg_log, set); + + cfg_plugins = config_setting_get_member(cfg, "plugins"); + if (cfg_plugins) + cfg_parse_plugins(cfg_plugins); + + return 0; +} + void cfg_init(config_t *cfg) { config_init(cfg); @@ -79,7 +161,25 @@ int cfg_parse(const char *filename, config_t *cfg, struct settings *set, for (int i = 0; i < config_setting_length(cfg_nodes); i++) { config_setting_t *cfg_node = config_setting_get_elem(cfg_nodes, i); - cfg_parse_node(cfg_node, nodes, set); + + struct node_type *vt; + const char *type; + + /* Required settings */ + if (!config_setting_lookup_string(cfg_node, "type", &type)) + cerror(cfg_node, "Missing node type"); + + vt = list_lookup(&node_types, type); + if (!vt) + cerror(cfg_node, "Invalid node type: %s", type); + + struct node *n = node_create(vt); + + ret = node_parse(n, cfg_node, set); + if (ret) + cerror(cfg_node, "Failed to parse node"); + + list_push(nodes, n); } } @@ -91,286 +191,26 @@ int cfg_parse(const char *filename, config_t *cfg, struct settings *set, for (int i = 0; i < config_setting_length(cfg_paths); i++) { config_setting_t *cfg_path = config_setting_get_elem(cfg_paths, i); - cfg_parse_path(cfg_path, paths, nodes, set); + + struct path *p = path_create(); + + ret = path_parse(p, cfg_path, nodes, set); + if (ret) + cerror(cfg_path, "Failed to parse path"); + + list_push(paths, p); + + if (p->reverse) { + struct path *r = path_create(); + + ret = path_reverse(p, r); + if (ret) + cerror(cfg_path, "Failed to reverse path %s", path_name(p)); + + list_push(paths, r); + } } } return 0; -} - -int cfg_parse_plugins(config_setting_t *cfg) -{ - if (!config_setting_is_array(cfg)) - cerror(cfg, "Setting 'plugins' must be a list of strings"); - - for (int i = 0; i < config_setting_length(cfg); i++) { - void *handle; - const char *path; - - path = config_setting_get_string_elem(cfg, i); - if (!path) - cerror(cfg, "Setting 'plugins' must be a list of strings"); - - handle = dlopen(path, RTLD_NOW); - if (!handle) - error("Failed to load plugin %s", dlerror()); - } - - return 0; -} - -int cfg_parse_global(config_setting_t *cfg, struct settings *set) -{ - config_setting_t *cfg_plugins; - - if (!config_setting_lookup_int(cfg, "affinity", &set->affinity)) - set->affinity = 0; - - if (!config_setting_lookup_int(cfg, "priority", &set->priority)) - set->priority = 0; - - if (!config_setting_lookup_int(cfg, "debug", &set->debug)) - set->debug = V; - - if (!config_setting_lookup_float(cfg, "stats", &set->stats)) - set->stats = 0; - - cfg_plugins = config_setting_get_member(cfg, "plugins"); - if (cfg_plugins) - cfg_parse_plugins(cfg_plugins); - - log_setlevel(set->debug, -1); - - return 0; -} - -int cfg_parse_path(config_setting_t *cfg, - struct list *paths, struct list *nodes, struct settings *set) -{ - config_setting_t *cfg_out, *cfg_hook; - const char *in; - int ret, reverse, samplelen, queuelen; - struct path *p; - - struct node *source; - struct list destinations; - - /* Allocate memory and intialize path structure */ - p = alloc(sizeof(struct path)); - - /* Input node */ - if (!config_setting_lookup_string(cfg, "in", &in) && - !config_setting_lookup_string(cfg, "from", &in) && - !config_setting_lookup_string(cfg, "src", &in) && - !config_setting_lookup_string(cfg, "source", &in)) - cerror(cfg, "Missing input node for path"); - - source = list_lookup(nodes, in); - if (!source) - cerror(cfg, "Invalid input node '%s'", in); - - /* Output node(s) */ - if (!(cfg_out = config_setting_get_member(cfg, "out")) && - !(cfg_out = config_setting_get_member(cfg, "to")) && - !(cfg_out = config_setting_get_member(cfg, "dst")) && - !(cfg_out = config_setting_get_member(cfg, "dest")) && - !(cfg_out = config_setting_get_member(cfg, "sink"))) - cerror(cfg, "Missing output nodes for path"); - - list_init(&destinations); - ret = cfg_parse_nodelist(cfg_out, &destinations, nodes); - if (ret <= 0) - cerror(cfg_out, "Invalid output nodes"); - - /* Optional settings */ - list_init(&p->hooks); - cfg_hook = config_setting_get_member(cfg, "hook"); - if (cfg_hook) - cfg_parse_hooklist(cfg_hook, &p->hooks); - - if (!config_setting_lookup_bool(cfg, "reverse", &reverse)) - reverse = 0; - if (!config_setting_lookup_bool(cfg, "enabled", &p->enabled)) - p->enabled = 1; - if (!config_setting_lookup_int(cfg, "values", &samplelen)) - samplelen = DEFAULT_VALUES; - if (!config_setting_lookup_int(cfg, "queuelen", &queuelen)) - queuelen = DEFAULT_QUEUELEN; - - if (!IS_POW2(queuelen)) { - queuelen = LOG2_CEIL(queuelen); - warn("Queue length should always be a power of 2. Adjusting to %d", queuelen); - } - - p->cfg = cfg; - - /* Check if nodes are suitable */ - if (source->_vt->read == NULL) - cerror(cfg, "Input node '%s' is not supported as a source.", node_name(source)); - - p->source = alloc(sizeof(struct path_source)); - p->source->node = source; - p->source->samplelen = samplelen; - - list_foreach(struct node *n, &destinations) { - if (n->_vt->write == NULL) - cerror(cfg_out, "Output node '%s' is not supported as a destination.", node_name(n)); - - struct path_destination *pd = alloc(sizeof(struct path_destination)); - - pd->node = n; - pd->queuelen = queuelen; - - list_push(&p->destinations, pd); - } - - list_push(paths, p); - - if (reverse) { - struct path *r = alloc(sizeof(struct path)); - - ret = path_reverse(p, r); - if (ret) - cerror(cfg, "Failed to reverse path %s", path_name(p)); - - list_push(paths, r); - } - - list_destroy(&destinations, NULL, false); - - return 0; -} - -int cfg_parse_nodelist(config_setting_t *cfg, struct list *list, struct list *all) { - const char *str; - struct node *node; - - switch (config_setting_type(cfg)) { - case CONFIG_TYPE_STRING: - str = config_setting_get_string(cfg); - if (str) { - node = list_lookup(all, str); - if (node) - list_push(list, node); - else - cerror(cfg, "Unknown outgoing node '%s'", str); - } - else - cerror(cfg, "Invalid outgoing node"); - break; - - case CONFIG_TYPE_ARRAY: - for (int i = 0; i < config_setting_length(cfg); i++) { - config_setting_t *elm = config_setting_get_elem(cfg, i); - - str = config_setting_get_string(elm); - if (str) { - node = list_lookup(all, str); - if (!node) - cerror(elm, "Unknown outgoing node '%s'", str); - else if (node->_vt->write == NULL) - cerror(cfg, "Output node '%s' is not supported as a sink.", node_name(node)); - - list_push(list, node); - } - else - cerror(cfg, "Invalid outgoing node"); - } - break; - - default: - cerror(cfg, "Invalid output node(s)"); - } - - return list_length(list); -} - -int cfg_parse_node(config_setting_t *cfg, struct list *nodes, struct settings *set) -{ - const char *type, *name; - int ret; - - struct node *n; - struct node_type *vt; - - /* Required settings */ - if (!config_setting_lookup_string(cfg, "type", &type)) - cerror(cfg, "Missing node type"); - - name = config_setting_name(cfg); - - vt = list_lookup(&node_types, type); - if (!vt) - cerror(cfg, "Invalid type for node '%s'", config_setting_name(cfg)); - - n = node_create(vt); - - n->name = name; - n->cfg = cfg; - - ret = node_parse(n, cfg); - if (ret) - cerror(cfg, "Failed to parse node '%s'", node_name(n)); - - if (config_setting_lookup_int(cfg, "vectorize", &n->vectorize)) { - config_setting_t *cfg_vectorize = config_setting_lookup(cfg, "vectorize"); - - if (n->vectorize <= 0) - cerror(cfg_vectorize, "Invalid value for `vectorize` %d. Must be natural number!", n->vectorize); - if (vt->vectorize && vt->vectorize < n->vectorize) - cerror(cfg_vectorize, "Invalid value for `vectorize`. Node type %s requires a number smaller than %d!", - node_name_type(n), vt->vectorize); - } - else - n->vectorize = 1; - - if (!config_setting_lookup_int(cfg, "affinity", &n->affinity)) - n->affinity = set->affinity; - - list_push(nodes, n); - - return ret; -} - -int cfg_parse_hooklist(config_setting_t *cfg, struct list *list) { - switch (config_setting_type(cfg)) { - case CONFIG_TYPE_STRING: - cfg_parse_hook(cfg, list); - break; - - case CONFIG_TYPE_ARRAY: - for (int i = 0; i < config_setting_length(cfg); i++) - cfg_parse_hook(config_setting_get_elem(cfg, i), list); - break; - - default: - cerror(cfg, "Invalid hook functions"); - } - - return list_length(list); -} - -int cfg_parse_hook(config_setting_t *cfg, struct list *list) -{ - struct hook *hook, *copy; - char *name, *param; - const char *hookline = config_setting_get_string(cfg); - if (!hookline) - cerror(cfg, "Invalid hook function"); - - name = strtok((char *) hookline, ":"); - param = strtok(NULL, ""); - - debug(3, "Hook: %s => %s", name, param); - - hook = list_lookup(&hooks, name); - if (!hook) - cerror(cfg, "Unknown hook function '%s'", name); - - copy = memdup(hook, sizeof(struct hook)); - copy->parameter = param; - - list_push(list, copy); - - return 0; -} +} \ No newline at end of file diff --git a/lib/hooks.c b/lib/hooks.c index 15492a52d..c6f9e4196 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -8,6 +8,7 @@ #include #include +#include #include "timing.h" #include "config.h" @@ -101,4 +102,53 @@ void * hook_storage(struct hook *h, int when, size_t len, ctor_cb_t ctor, dtor_c } return h->_vd; +} + +/** Parse an array or single hook function. + * + * Examples: + * hooks = [ "print", "fir" ] + * hooks = "log" + */ +int hook_parse_list(struct list *list, config_setting_t *cfg) { + switch (config_setting_type(cfg)) { + case CONFIG_TYPE_STRING: + hook_parse(cfg, list); + break; + + case CONFIG_TYPE_ARRAY: + for (int i = 0; i < config_setting_length(cfg); i++) + hook_parse(config_setting_get_elem(cfg, i), list); + break; + + default: + cerror(cfg, "Invalid hook functions"); + } + + return list_length(list); +} + +int hook_parse(config_setting_t *cfg, struct list *list) +{ + struct hook *hook, *copy; + char *name, *param; + const char *hookline = config_setting_get_string(cfg); + if (!hookline) + cerror(cfg, "Invalid hook function"); + + name = strtok((char *) hookline, ":"); + param = strtok(NULL, ""); + + debug(3, "Hook: %s => %s", name, param); + + hook = list_lookup(&hooks, name); + if (!hook) + cerror(cfg, "Unknown hook function '%s'", name); + + copy = memdup(hook, sizeof(struct hook)); + copy->parameter = param; + + list_push(list, copy); + + return 0; } \ No newline at end of file diff --git a/lib/log.c b/lib/log.c index 74e8d1694..144a985c6 100644 --- a/lib/log.c +++ b/lib/log.c @@ -34,6 +34,27 @@ static unsigned facilities = ~0; /** A global clock used to prefix the log messages. */ static struct timespec epoch; +/** Handle to log file */ +static FILE *file; + +static const char *facility_names[] = { + "pool", // DBG_POOL + "queue", // DBG_QUEUE + "config", // DBG_CONFIG + "hook", // DBG_HOOK + "path", // DBG_PATH + "mem", // DBG_MEM + + /* Node-types */ + "socket", // DBG_SOCKET + "websocket", // DBG_WEBSOCKET + "file", // DBG_FILE + "fpga", // DBG_FPGA + "ngsi", // DBG_NGSI + "opal", // DBG_OPAL + NULL +}; + #ifdef __GNUC__ /** The current log indention level (per thread!). */ static __thread int indent = 0; @@ -51,16 +72,29 @@ void log_outdent(int *old) } #endif -void log_setlevel(int lvl, int fac) +void log_init(int lvl, int fac, const char *path) { + char *fac_str = NULL; + + /* Initialize global variables */ + epoch = time_now(); level = lvl; - debug(10, "Switched to debug level %u", level); + facilities = fac; + + if (path) + file = fopen(path, "a"); + + debug(10, "Started logging: level = %u, faciltities = %s", level, fac_str); } -void log_init() +int log_lookup_facility(const char *facility_name) { - epoch = time_now(); - debug(10, "Debug clock resetted"); + for (int i = 0; facility_names[i]; i++) { + if (!strcmp(facility_name, facility_names[i])) + return 1 << i; + } + + return 0; } void log_print(const char *lvl, const char *fmt, ...) @@ -99,6 +133,10 @@ void log_vprint(const char *lvl, const char *fmt, va_list ap) OpalPrint("VILLASnode: %s\n", buf); #endif fprintf(stderr, "\r%s\n", buf); + + if (file) + fprintf(file, "%s\n", buf); + free(buf); } diff --git a/lib/node.c b/lib/node.c index 9bfe0a4c6..194b72357 100644 --- a/lib/node.c +++ b/lib/node.c @@ -7,20 +7,18 @@ *********************************************************************************/ #include +#include + #include "sample.h" #include "node.h" #include "cfg.h" #include "utils.h" +#include "config.h" /** List of registered node-types */ struct list node_types = LIST_INIT(); -int node_parse(struct node *n, config_setting_t *cfg) -{ - return n->_vt->parse ? n->_vt->parse(n, cfg) : 0; -} - int node_read(struct node *n, struct sample *smps[], unsigned cnt) { int nread = 0; @@ -207,3 +205,98 @@ void node_destroy(struct node *n) free(n->_name); free(n); } + +/** Parse an array or single node and checks if they exist in the "nodes" section. + * + * Examples: + * out = [ "sintef", "scedu" ] + * out = "acs" + * + * @param cfg The libconfig object handle for "out". + * @param nodes The nodes will be added to this list. + * @param all This list contains all valid nodes. + */ +int node_parse_list(struct list *list, config_setting_t *cfg, struct list *all) { + const char *str; + struct node *node; + + switch (config_setting_type(cfg)) { + case CONFIG_TYPE_STRING: + str = config_setting_get_string(cfg); + if (str) { + node = list_lookup(all, str); + if (node) + list_push(list, node); + else + cerror(cfg, "Unknown outgoing node '%s'", str); + } + else + cerror(cfg, "Invalid outgoing node"); + break; + + case CONFIG_TYPE_ARRAY: + for (int i = 0; i < config_setting_length(cfg); i++) { + config_setting_t *elm = config_setting_get_elem(cfg, i); + + str = config_setting_get_string(elm); + if (str) { + node = list_lookup(all, str); + if (!node) + cerror(elm, "Unknown outgoing node '%s'", str); + else if (node->_vt->write == NULL) + cerror(cfg, "Output node '%s' is not supported as a sink.", node_name(node)); + + list_push(list, node); + } + else + cerror(cfg, "Invalid outgoing node"); + } + break; + + default: + cerror(cfg, "Invalid output node(s)"); + } + + return list_length(list); +} + +int node_parse(struct node *n, config_setting_t *cfg, struct settings *set) +{ + struct node_type *vt; + const char *type, *name; + int ret; + + name = config_setting_name(cfg); + + if (!config_setting_lookup_string(cfg, "type", &type)) + cerror(cfg, "Missing node type"); + + vt = list_lookup(&node_types, type); + if (!vt) + cerror(cfg, "Invalid type for node '%s'", config_setting_name(cfg)); + + n->name = name; + n->cfg = cfg; + + ret = n->_vt->parse ? n->_vt->parse(n, cfg) : 0; + if (ret) + cerror(cfg, "Failed to parse node '%s'", node_name(n)); + + if (config_setting_lookup_int(cfg, "vectorize", &n->vectorize)) { + config_setting_t *cfg_vectorize = config_setting_lookup(cfg, "vectorize"); + + if (n->vectorize <= 0) + cerror(cfg_vectorize, "Invalid value for `vectorize` %d. Must be natural number!", n->vectorize); + if (vt->vectorize && vt->vectorize < n->vectorize) + cerror(cfg_vectorize, "Invalid value for `vectorize`. Node type %s requires a number smaller than %d!", + node_name_type(n), vt->vectorize); + } + else + n->vectorize = 1; + + if (!config_setting_lookup_int(cfg, "affinity", &n->affinity)) + n->affinity = set->affinity; + + return ret; +} + diff --git a/lib/path.c b/lib/path.c index e5c7728b9..4a6c7825b 100644 --- a/lib/path.c +++ b/lib/path.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "config.h" #include "utils.h" @@ -177,6 +178,11 @@ const char * path_name(struct path *p) return p->_name; } +struct path * path_create() +{ + return (struct path *) alloc(sizeof(struct path)); +} + int path_init(struct path *p) { int ret, max_queuelen = 0; @@ -294,5 +300,85 @@ int path_reverse(struct path *p, struct path *r) list_push(&r->hooks, hc); } + return 0; +} + +int path_parse(struct path *p, config_setting_t *cfg, struct list *nodes, struct settings *set) +{ + config_setting_t *cfg_out, *cfg_hook; + const char *in; + int ret, samplelen, queuelen; + + struct node *source; + struct list destinations; + + /* Input node */ + if (!config_setting_lookup_string(cfg, "in", &in) && + !config_setting_lookup_string(cfg, "from", &in) && + !config_setting_lookup_string(cfg, "src", &in) && + !config_setting_lookup_string(cfg, "source", &in)) + cerror(cfg, "Missing input node for path"); + + source = list_lookup(nodes, in); + if (!source) + cerror(cfg, "Invalid input node '%s'", in); + + /* Output node(s) */ + if (!(cfg_out = config_setting_get_member(cfg, "out")) && + !(cfg_out = config_setting_get_member(cfg, "to")) && + !(cfg_out = config_setting_get_member(cfg, "dst")) && + !(cfg_out = config_setting_get_member(cfg, "dest")) && + !(cfg_out = config_setting_get_member(cfg, "sink"))) + cerror(cfg, "Missing output nodes for path"); + + list_init(&destinations); + ret = node_parse_list(&destinations, cfg_out, nodes); + if (ret <= 0) + cerror(cfg_out, "Invalid output nodes"); + + /* Optional settings */ + list_init(&p->hooks); + cfg_hook = config_setting_get_member(cfg, "hook"); + if (cfg_hook) + hook_parse_list(&p->hooks, cfg_hook); + + if (!config_setting_lookup_bool(cfg, "reverse", &p->reverse)) + p->reverse = 0; + if (!config_setting_lookup_bool(cfg, "enabled", &p->enabled)) + p->enabled = 1; + if (!config_setting_lookup_int(cfg, "values", &samplelen)) + samplelen = DEFAULT_VALUES; + if (!config_setting_lookup_int(cfg, "queuelen", &queuelen)) + queuelen = DEFAULT_QUEUELEN; + + if (!IS_POW2(queuelen)) { + queuelen = LOG2_CEIL(queuelen); + warn("Queue length should always be a power of 2. Adjusting to %d", queuelen); + } + + p->cfg = cfg; + + /* Check if nodes are suitable */ + if (source->_vt->read == NULL) + cerror(cfg, "Input node '%s' is not supported as a source.", node_name(source)); + + p->source = alloc(sizeof(struct path_source)); + p->source->node = source; + p->source->samplelen = samplelen; + + list_foreach(struct node *n, &destinations) { + if (n->_vt->write == NULL) + cerror(cfg_out, "Output node '%s' is not supported as a destination.", node_name(n)); + + struct path_destination *pd = alloc(sizeof(struct path_destination)); + + pd->node = n; + pd->queuelen = queuelen; + + list_push(&p->destinations, pd); + } + + list_destroy(&destinations, NULL, false); + return 0; } \ No newline at end of file diff --git a/src/fpga.c b/src/fpga.c index da8205076..4c8617551 100644 --- a/src/fpga.c +++ b/src/fpga.c @@ -47,7 +47,7 @@ void usage() int main(int argc, char *argv[]) { - int ret; + int ret, level = -1; struct fpga *fpga; config_t config; @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) while ((c = getopt(argc-1, argv+1, "d:")) != -1) { switch (c) { case 'd': - log_setlevel(strtoul(optarg, &endptr, 10), ~0); + level = strtoul(optarg, &endptr, 10); break; case '?': @@ -82,6 +82,9 @@ int main(int argc, char *argv[]) info("Parsing configuration"); cfg_parse(argv[1], &config, &settings, NULL, NULL); + info("Initialize logging system"); + log_init(level > 0 ? level : settings.log.level, settings.log.facilities, settings.log.file); + info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); diff --git a/src/hook.c b/src/hook.c index d30133410..87e961444 100644 --- a/src/hook.c +++ b/src/hook.c @@ -36,7 +36,7 @@ static void usage() int main(int argc, char *argv[]) { - int j, ret, cnt = 1; + int j, ret, cnt = 1, level = -1; char *name, *parameter; @@ -50,7 +50,7 @@ int main(int argc, char *argv[]) cnt = atoi(optarg); break; case 'd': - log_setlevel(atoi(optarg), -1); + level = atoi(optarg); break; case 'h': case '?': @@ -83,6 +83,9 @@ int main(int argc, char *argv[]) struct pool pool; struct sample *smps[cnt]; + + info("Initialize logging system"); + log_init(level, -1, NULL); ret = pool_init(&pool, 10 * cnt, SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); if (ret) diff --git a/src/node.c b/src/node.c index 4b8d9f681..99e73a25c 100644 --- a/src/node.c +++ b/src/node.c @@ -108,7 +108,6 @@ int main(int argc, char *argv[]) char *configfile = (argc == 2) ? argv[1] : "opal-shmem.conf"; - log_init(); info("This is VILLASnode %s (built on %s, %s)", BLD(YEL(VERSION)), BLD(MAG(__DATE__)), BLD(MAG(__TIME__))); @@ -122,6 +121,9 @@ int main(int argc, char *argv[]) info("Parsing configuration"); cfg_parse(configfile, &config, &settings, &nodes, &paths); + + info("Initialize logging system"); + log_init(settings.log.level, settings.log.facilities, settings.log.file); info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); diff --git a/src/pipe.c b/src/pipe.c index 16c4e0a35..26b18683f 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -171,11 +171,10 @@ static void * recv_loop(void *ctx) int main(int argc, char *argv[]) { - int ret; + int ret, level = -1; char c; ptid = pthread_self(); - log_init(); /* Parse command line arguments */ if (argc < 3) @@ -197,7 +196,7 @@ int main(int argc, char *argv[]) sendd.enabled = false; // receive only break; case 'd': - log_setlevel(atoi(optarg), -1); + level = atoi(optarg); break; case 'h': case '?': @@ -221,6 +220,9 @@ int main(int argc, char *argv[]) info("Parsing configuration"); cfg_parse(argv[1], &config, &settings, &nodes, NULL); + + info("Initialize logging system"); + log_init(level > 0 ? level : settings.log.level, settings.log.facilities, settings.log.file); info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); diff --git a/src/signal.c b/src/signal.c index 601c350d3..05b06e0ea 100644 --- a/src/signal.c +++ b/src/signal.c @@ -56,8 +56,6 @@ int main(int argc, char *argv[]) int values = 1; int limit = -1; int counter; - - log_init(); if (argc < 2) { usage(); @@ -111,7 +109,7 @@ int main(int argc, char *argv[]) check: if (optarg == endptr) error("Failed to parse parse option argument '-%c %s'", c, optarg); } - + /* Allocate memory for message buffer */ struct sample *s = alloc(SAMPLE_LEN(values)); diff --git a/src/test.c b/src/test.c index 8e4f56ae6..eb7424c73 100644 --- a/src/test.c +++ b/src/test.c @@ -84,11 +84,12 @@ int main(int argc, char *argv[]) sigaction(SIGINT, &sa_quit, NULL); list_init(&nodes); - - log_init(); info("Parsing configuration"); cfg_parse(argv[1], &config, &settings, &nodes, NULL); + + info("Initialize logging system"); + log_init(settings.log.level, settings.log.facilities, settings.log.file); info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); From d1d43692673203218fea71698f3d51f18c112939 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 13:00:23 -0500 Subject: [PATCH 55/57] properly initialize other applications apart from villas-node --- src/fpga.c | 3 +++ src/hook.c | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/fpga.c b/src/fpga.c index 4c8617551..bd7926ad8 100644 --- a/src/fpga.c +++ b/src/fpga.c @@ -87,6 +87,9 @@ int main(int argc, char *argv[]) info("Initialize real-time system"); rt_init(settings.affinity, settings.priority); + + info("Initialize memory system"); + memory_init(); /* Initialize VILLASfpga card */ config_setting_t *cfg_root = config_root_setting(&config); diff --git a/src/hook.c b/src/hook.c index 87e961444..75bda2904 100644 --- a/src/hook.c +++ b/src/hook.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "config.h" @@ -86,6 +87,12 @@ int main(int argc, char *argv[]) info("Initialize logging system"); log_init(level, -1, NULL); + + info("Initialize real-time system"); + rt_init(-1, 50); + + info("Initialize memory system"); + memory_init(); ret = pool_init(&pool, 10 * cnt, SAMPLE_LEN(DEFAULT_VALUES), &memtype_hugepage); if (ret) From a93952aaae30ecc568380bff2a1395ed8d0a41b5 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 13:01:17 -0500 Subject: [PATCH 56/57] statistics are almost working again --- include/villas/stats.h | 3 ++- lib/hooks.c | 5 +++-- lib/hooks/hooks-internal.c | 4 ++-- lib/hooks/hooks-stats.c | 43 +++++++++++++++++++++++--------------- lib/path.c | 4 ++-- lib/stats.c | 32 +++++++++++++++++----------- 6 files changed, 55 insertions(+), 36 deletions(-) diff --git a/include/villas/stats.h b/include/villas/stats.h index dc69beb41..833823f78 100644 --- a/include/villas/stats.h +++ b/include/villas/stats.h @@ -36,13 +36,14 @@ enum stats_id { struct stats_delta { double values[STATS_COUNT]; + int update; /**< Bitmask of stats_id. Only those which are masked will be updated */ struct sample *last; }; struct stats { struct hist histograms[STATS_COUNT]; - struct sample *last; + struct stats_delta *delta; }; int stats_init(struct stats *s); diff --git a/lib/hooks.c b/lib/hooks.c index c6f9e4196..cc74110eb 100644 --- a/lib/hooks.c +++ b/lib/hooks.c @@ -70,9 +70,10 @@ int hook_run(struct path *p, struct sample *smps[], size_t cnt, int when) list_foreach(struct hook *h, &p->hooks) { if (h->type & when) { - debug(DBG_HOOK | 22, "Running hook when=%u '%s' prio=%u, cnt=%zu", when, h->name, h->priority, cnt); - cnt = h->cb(h, when, &i); + + debug(DBG_HOOK | 22, "Ran hook '%s' when=%u prio=%u, cnt=%zu", h->name, when, h->priority, cnt); + if (cnt == 0) break; } diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/hooks-internal.c index 81673a3c3..735fa50df 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/hooks-internal.c @@ -77,8 +77,8 @@ int hook_drop(struct hook *h, int when, struct hook_info *j) dist = h->last->sequence - (int32_t) h->prev->sequence; if (dist <= 0) { warn("Dropped sample: dist = %d, i = %d", dist, i); - if (j->path) - stats_update(j->path->stats, STATS_DROPPED, dist); + if (j->path && j->path->stats) + stats_update(j->path->stats->delta, STATS_DROPPED, dist); } else { struct sample *tmp; diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/hooks-stats.c index bd78d3c87..38f02916f 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/hooks-stats.c @@ -20,11 +20,16 @@ int hook_stats(struct hook *h, int when, struct hook_info *j) struct stats *s = hook_storage(h, when, sizeof(struct stats), (ctor_cb_t) stats_init, (dtor_cb_t) stats_destroy); switch (when) { + case HOOK_INIT: + if (j->path) + j->path->stats = s; + break; + case HOOK_READ: assert(j->smps); - assert(j->path); - stats_collect(j->path->stats, j->smps, j->cnt); + stats_collect(s->delta, j->smps, j->cnt); + stats_commit(s, s->delta); break; case HOOK_PATH_STOP: @@ -45,37 +50,41 @@ int hook_stats(struct hook *h, int when, struct hook_info *j) return j->cnt; } -#if 0 /* currently broken */ -REGISTER_HOOK("stats_send", "Send path statistics to another node", 99, 0, hook_stats_send, HOOK_STORAGE | HOOK_DESTROY | HOOK_PERIODIC | HOOK_PATH) -int hook_stats_send(struct hook *h, int when, struct hook_info *i) +/** @todo This is untested */ +REGISTER_HOOK("stats_send", "Send path statistics to another node", 99, 0, hook_stats_send, HOOK_STORAGE | HOOK_PATH | HOOK_READ) +int hook_stats_send(struct hook *h, int when, struct hook_info *j) { struct private { struct node *dest; + struct stats *stats; int ratio; - } *private = hook_storage(h, when, sizeof(*private)); + } *private = hook_storage(h, when, sizeof(*private), NULL, NULL); switch (when) { - case HOOK_DESTROY: + case HOOK_INIT: + assert(j->nodes); + assert(j->path); + if (!h->parameter) error("Missing parameter for hook '%s'", h->name); - if (!hook_nodes) - error("Missing reference to node list for hook '%s", h->name); - - private->dest = list_lookup(hook_nodes, h->parameter); + private->dest = list_lookup(j->nodes, h->parameter); if (!private->dest) error("Invalid destination node '%s' for hook '%s'", h->parameter, h->name); - - node_start(private->dest); + break; + case HOOK_PATH_START: + node_start(private->dest); break; - case HOOK_PERIODIC: { - stats_send(s, node); + case HOOK_PATH_STOP: + node_stop(private->dest); + break; + + case HOOK_READ: + stats_send(private->stats, private->dest); break; - } } return 0; } -#endif \ No newline at end of file diff --git a/lib/path.c b/lib/path.c index 4a6c7825b..863378380 100644 --- a/lib/path.c +++ b/lib/path.c @@ -52,7 +52,7 @@ static void path_read(struct path *p) if (enqueue != recv) { info("Hooks skipped %u out of %u samples for path %s", recv - enqueue, recv, path_name(p)); - stats_update(p->stats, STATS_SKIPPED, recv - enqueue); + stats_update(p->stats->delta, STATS_SKIPPED, recv - enqueue); } list_foreach(struct path_destination *pd, &p->destinations) { @@ -84,7 +84,7 @@ static void path_write(struct path *p) if (available == 0) break; else if (available < cnt) - warn("Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); + debug(DBG_PATH | 5, "Queue underrun for path %s: available=%u expected=%u", path_name(p), available, cnt); debug(DBG_PATH | 15, "Dequeued %u samples from queue of node %s which is part of path %s", available, node_name(pd->node), path_name(p)); diff --git a/lib/stats.c b/lib/stats.c index 6441edf85..fd137f8ca 100644 --- a/lib/stats.c +++ b/lib/stats.c @@ -37,6 +37,8 @@ int stats_init(struct stats *s) struct stats_desc *desc = &stats_table[i]; hist_create(&s->histograms[i], desc->hist.min, desc->hist.max, desc->hist.resolution); } + + s->delta = alloc(sizeof(struct stats_delta)); return 0; } @@ -46,18 +48,23 @@ void stats_destroy(struct stats *s) for (int i = 0; i < STATS_COUNT; i++) { hist_destroy(&s->histograms[i]); } + + free(s->delta); } void stats_update(struct stats_delta *d, enum stats_id id, double val) { - if (d && id >= 0 && id < STATS_COUNT) - d->values[id] = val; + assert(id >= 0 && id < STATS_COUNT); + + d->values[id] = val; + d->update |= 1 << id; } int stats_commit(struct stats *s, struct stats_delta *d) { for (int i = 0; i < STATS_COUNT; i++) { - hist_put(&s->histograms[i], d->values[i]); + if (d->update & 1 << i) + hist_put(&s->histograms[i], d->values[i]); } return 0; @@ -67,22 +74,23 @@ void stats_collect(struct stats_delta *s, struct sample *smps[], size_t cnt) { struct sample *previous = s->last; - if (previous) { - sample_put(previous); - - for (int i = 0; i < cnt; i++) { + for (int i = 0; i < cnt; i++) { + if (previous) { stats_update(s, STATS_GAP_RECEIVED, time_delta(&previous->ts.received, &smps[i]->ts.received)); stats_update(s, STATS_GAP_SAMPLE, time_delta(&previous->ts.origin, &smps[i]->ts.origin)); stats_update(s, STATS_OWD, time_delta(&smps[i]->ts.origin, &smps[i]->ts.received)); stats_update(s, STATS_GAP_SEQUENCE, smps[i]->sequence - (int32_t) previous->sequence); - - /* Make sure there is always a reference to the previous sample */ - - previous = smps[i]; } + + previous = smps[i]; } - sample_get(previous); + if (s->last) + sample_put(s->last); + + if (previous) + sample_get(previous); + s->last = previous; } From c84df390bca1ba0dc611d698ece61ed9fd4535c7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 20 Nov 2016 13:11:37 -0500 Subject: [PATCH 57/57] renamed hook related files --- include/villas/{hooks.h => hook.h} | 0 include/villas/path.h | 1 - lib/Makefile.inc | 2 +- lib/cfg.c | 2 +- lib/{hooks.c => hook.c} | 2 +- lib/hooks/{hooks-internal.c => internal.c} | 2 +- lib/hooks/{hooks-other.c => other.c} | 2 +- lib/hooks/{hooks-stats.c => stats.c} | 2 +- lib/path.c | 5 +---- plugins/hooks/example_hook.c | 2 +- src/hook.c | 2 +- src/node.c | 1 + 12 files changed, 10 insertions(+), 13 deletions(-) rename include/villas/{hooks.h => hook.h} (100%) rename lib/{hooks.c => hook.c} (99%) rename lib/hooks/{hooks-internal.c => internal.c} (99%) rename lib/hooks/{hooks-other.c => other.c} (99%) rename lib/hooks/{hooks-stats.c => stats.c} (99%) diff --git a/include/villas/hooks.h b/include/villas/hook.h similarity index 100% rename from include/villas/hooks.h rename to include/villas/hook.h diff --git a/include/villas/path.h b/include/villas/path.h index f3913265d..6d3750683 100644 --- a/include/villas/path.h +++ b/include/villas/path.h @@ -23,7 +23,6 @@ #include "hist.h" #include "node.h" #include "msg.h" -#include "hooks.h" #include "queue.h" #include "pool.h" #include "stats.h" diff --git a/lib/Makefile.inc b/lib/Makefile.inc index 258c50d14..a36c1bf9f 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -4,7 +4,7 @@ LIBS = $(BUILDDIR)/libvillas.so # Object files for libvillas LIB_SRCS = $(addprefix lib/nodes/, file.c cbuilder.c) \ $(addprefix lib/kernel/, kernel.c rt.c) \ - $(addprefix lib/, sample.c path.c node.c hooks.c \ + $(addprefix lib/, sample.c path.c node.c hook.c \ log.c utils.c cfg.c hist.c timing.c pool.c list.c \ queue.c memory.c stats.c \ ) \ diff --git a/lib/cfg.c b/lib/cfg.c index 6f1328bc2..b8e99ff33 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -17,7 +17,7 @@ #include "cfg.h" #include "node.h" #include "path.h" -#include "hooks.h" +#include "hook.h" static int cfg_parse_log(config_setting_t *cfg, struct settings *set) { diff --git a/lib/hooks.c b/lib/hook.c similarity index 99% rename from lib/hooks.c rename to lib/hook.c index cc74110eb..c1dd12d97 100644 --- a/lib/hooks.c +++ b/lib/hook.c @@ -13,7 +13,7 @@ #include "timing.h" #include "config.h" #include "msg.h" -#include "hooks.h" +#include "hook.h" #include "path.h" #include "utils.h" #include "node.h" diff --git a/lib/hooks/hooks-internal.c b/lib/hooks/internal.c similarity index 99% rename from lib/hooks/hooks-internal.c rename to lib/hooks/internal.c index 735fa50df..3b7eece90 100644 --- a/lib/hooks/hooks-internal.c +++ b/lib/hooks/internal.c @@ -6,7 +6,7 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ -#include "hooks.h" +#include "hook.h" #include "timing.h" #include "sample.h" #include "path.h" diff --git a/lib/hooks/hooks-other.c b/lib/hooks/other.c similarity index 99% rename from lib/hooks/hooks-other.c rename to lib/hooks/other.c index 0a63ea5fa..f63de05d8 100644 --- a/lib/hooks/hooks-other.c +++ b/lib/hooks/other.c @@ -11,7 +11,7 @@ #include #include -#include "hooks.h" +#include "hook.h" #include "timing.h" #include "utils.h" #include "sample.h" diff --git a/lib/hooks/hooks-stats.c b/lib/hooks/stats.c similarity index 99% rename from lib/hooks/hooks-stats.c rename to lib/hooks/stats.c index 38f02916f..9dcb9ffb5 100644 --- a/lib/hooks/hooks-stats.c +++ b/lib/hooks/stats.c @@ -6,7 +6,7 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ -#include "hooks.h" +#include "hook.h" #include "sample.h" #include "path.h" #include "utils.h" diff --git a/lib/path.c b/lib/path.c index 863378380..f585dcce0 100644 --- a/lib/path.c +++ b/lib/path.c @@ -19,6 +19,7 @@ #include "timing.h" #include "pool.h" #include "queue.h" +#include "hook.h" static void path_read(struct path *p) { @@ -199,10 +200,6 @@ int path_init(struct path *p) /* We sort the hooks according to their priority before starting the path */ list_sort(&p->hooks, hooks_sort_priority); - ret = hook_run(p, NULL, 0, HOOK_INIT); - if (ret) - error("Failed to initialize hooks of path: %s", path_name(p)); - /* Parse hook arguments */ ret = hook_run(p, NULL, 0, HOOK_PARSE); if (ret) diff --git a/plugins/hooks/example_hook.c b/plugins/hooks/example_hook.c index 22e1b48dc..827c33b54 100644 --- a/plugins/hooks/example_hook.c +++ b/plugins/hooks/example_hook.c @@ -1,4 +1,4 @@ -#include +#include #include static int hook_example(struct hook *h, int when, struct hook_info *j) diff --git a/src/hook.c b/src/hook.c index 75bda2904..648be8e0a 100644 --- a/src/hook.c +++ b/src/hook.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/node.c b/src/node.c index 99e73a25c..6db1e1d52 100644 --- a/src/node.c +++ b/src/node.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifdef ENABLE_OPAL_ASYNC #include