diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 356c9010f..6fa5c6dee 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -29,11 +29,16 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror") set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) +add_definitions(-D__STDC_FORMAT_MACROS -D_GNU_SOURCE) + +# Check packages +find_package(OpenSSL 1.0.0 REQUIRED) +find_package(CURL 7.29 REQUIRED) +find_package(Criterion) + include(FindPkgConfig) pkg_check_modules(JANSSON IMPORTED_TARGET jansson) -find_package(Criterion) - add_subdirectory(lib) add_subdirectory(tests) diff --git a/common/include/villas/advio.h b/common/include/villas/advio.h new file mode 100644 index 000000000..96bb4a0da --- /dev/null +++ b/common/include/villas/advio.h @@ -0,0 +1,91 @@ +/** libcurl based advanced IO aka ADVIO. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct advio { + CURL *curl; + FILE *file; + + long uploaded; /**< Upload progress. How much has already been uploaded to the remote file. */ + long downloaded; /**< Download progress. How much has already been downloaded from the remote file. */ + + int completed:1; /**< Was the upload completd */ + + unsigned char hash[SHA_DIGEST_LENGTH]; + + char mode[3]; + char *uri; +}; + +typedef struct advio AFILE; + +/* The remaining functions from stdio are just replaced macros */ +#define afeof(af) feof((af)->file) +#define afgets(ln, sz, af) fgets(ln, sz, (af)->file) +#define aftell(af) ftell((af)->file) +#define afileno(af) fileno((af)->file) +#define afread(ptr, sz, nitems, af) fread(ptr, sz, nitems, (af)->file) +#define afwrite(ptr, sz, nitems, af) fwrite(ptr, sz, nitems, (af)->file) +#define afputs(ptr, af) fputs(ptr, (af)->file) +#define afprintf(af, fmt, ...) fprintf((af)->file, fmt, __VA_ARGS__) +#define afscanf(af, fmt, ...) fprintf((af)->file, fmt, __VA_ARGS__) +#define agetline(linep, linecapp, af) getline(linep, linecapp, (af)->file) + +/* Extensions */ +#define auri(af) ((af)->uri) +#define ahash(af) ((af)->hash) + +/** Check if a URI is pointing to a local file. */ +int aislocal(const char *uri); + +AFILE *afopen(const char *url, const char *mode); + +int afclose(AFILE *file); + +int afflush(AFILE *file); + +int afseek(AFILE *file, long offset, int origin); + +void arewind(AFILE *file); + +/** Download contens from remote file + * + * @param resume Do a partial download and append to the local file + */ +int adownload(AFILE *af, int resume); + +int aupload(AFILE *af, int resume); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/atomic.h b/common/include/villas/atomic.h new file mode 100644 index 000000000..8d2f76430 --- /dev/null +++ b/common/include/villas/atomic.h @@ -0,0 +1,42 @@ +/** Workaround for differently named atomic types in C/C++ + * + * @file + * @author Georg Reinke + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#ifdef __cplusplus + +#include + +typedef std::atomic_int atomic_int; +typedef std::atomic_size_t atomic_size_t; +typedef std::atomic atomic_state; + +#else + +#include + +typedef _Atomic enum state atomic_state; + +#endif /* __cplusplus */ diff --git a/common/include/villas/bitset.h b/common/include/villas/bitset.h new file mode 100644 index 000000000..4887fec88 --- /dev/null +++ b/common/include/villas/bitset.h @@ -0,0 +1,73 @@ +/** A datastructure storing bitsets of arbitrary dimensions. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct bitset { + char *set; + size_t dimension; +}; + +/** Allocate memory for a new betset */ +int bitset_init(struct bitset *b, size_t dim); + +/** Release memory of bit set */ +int bitset_destroy(struct bitset *b); + +void bitset_set_value(struct bitset *b, uintmax_t val); +uintmax_t bitset_get_value(struct bitset *b); + +/** Return the number of bits int the set which are set (1) */ +size_t bitset_count(struct bitset *b); + +/** Set a single bit in the set */ +int bitset_set(struct bitset *b, size_t bit); + +/** Clear a single bit in the set */ +int bitset_clear(struct bitset *b, size_t bit); + +/** Set all bits in the set */ +void bitset_set_all(struct bitset *b); + +/** Clear all bits in the set */ +void bitset_clear_all(struct bitset *b); + +/** Test if a single bit in the set is set */ +int bitset_test(struct bitset *b, size_t bit); + +/** Compare two bit sets bit-by-bit */ +int bitset_cmp(struct bitset *a, struct bitset *b); + +/** Return an human readable representation of the bit set */ +char *bitset_dump(struct bitset *b); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/buffer.h b/common/include/villas/buffer.h new file mode 100644 index 000000000..f97138db4 --- /dev/null +++ b/common/include/villas/buffer.h @@ -0,0 +1,58 @@ +/** A simple growing buffer. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct buffer { + enum state state; + + char *buf; + size_t len; + size_t size; +}; + +int buffer_init(struct buffer *b, size_t size); + +int buffer_destroy(struct buffer *b); + +void buffer_clear(struct buffer *b); + +int buffer_append(struct buffer *b, const char *data, size_t len); + +int buffer_parse_json(struct buffer *b, json_t **j); + +int buffer_append_json(struct buffer *b, json_t *j); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/common.h b/common/include/villas/common.h index 2f4e7acdc..7b8f566bd 100644 --- a/common/include/villas/common.h +++ b/common/include/villas/common.h @@ -23,14 +23,32 @@ #pragma once -/* Common states for most objects in VILLASfpga (paths, nodes, hooks, plugins) */ +#ifdef __cplusplus +extern "C" { +#endif + +/* Common states for most objects in VILLAScommon (paths, nodes, hooks, plugins) */ enum state { STATE_DESTROYED = 0, STATE_INITIALIZED = 1, STATE_PARSED = 2, STATE_CHECKED = 3, STATE_STARTED = 4, - STATE_LOADED = 4, /* alias for STATE_STARTED used by plugins */ + STATE_LOADED = 4, /* alias for STATE_STARTED used by struct plugin */ + STATE_OPENED = 4, /* alias for STATE_STARTED used by struct io */ STATE_STOPPED = 5, - STATE_UNLOADED = 5 /* alias for STATE_STARTED used by plugins */ + STATE_UNLOADED = 5, /* alias for STATE_STARTED used by struct plugin */ + STATE_CLOSED = 5, /* alias for STATE_STARTED used by struct io */ + STATE_PENDING_CONNECT = 6, + STATE_CONNECTED = 7 }; + +/** Callback to destroy list elements. + * + * @param data A pointer to the data which should be freed. + */ +typedef int (*dtor_cb_t)(void *); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/compat.h b/common/include/villas/compat.h new file mode 100644 index 000000000..1f3b71ab3 --- /dev/null +++ b/common/include/villas/compat.h @@ -0,0 +1,61 @@ +/** Compatability for different library versions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if JANSSON_VERSION_HEX < 0x020A00 +size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags); +#endif + +#if (LIBCONFIG_VER_MAJOR <= 1) && (LIBCONFIG_VER_MINOR < 5) + #define config_setting_lookup config_lookup_from +#endif + + +#ifdef __MACH__ + #include + + #define le16toh(x) OSSwapLittleToHostInt16(x) + #define le32toh(x) OSSwapLittleToHostInt32(x) + #define le64toh(x) OSSwapLittleToHostInt64(x) + #define be16toh(x) OSSwapBigToHostInt16(x) + #define be32toh(x) OSSwapBigToHostInt32(x) + #define be64toh(x) OSSwapBigToHostInt64(x) + + #define htole16(x) OSSwapHostToLittleInt16(x) + #define htole32(x) OSSwapHostToLittleInt32(x) + #define htole64(x) OSSwapHostToLittleInt64(x) + #define htobe16(x) OSSwapHostToBigInt16(x) + #define htobe32(x) OSSwapHostToBigInt32(x) + #define htobe64(x) OSSwapHostToBigInt64(x) +#endif /* __MACH__ */ + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/crypt.h b/common/include/villas/crypt.h new file mode 100644 index 000000000..fcac5b1ae --- /dev/null +++ b/common/include/villas/crypt.h @@ -0,0 +1,41 @@ +/** Crypto helpers. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Calculate SHA1 hash of complete file \p f and place it into \p sha1. + * + * @param sha1[out] Must be SHA_DIGEST_LENGTH (20) in size. + * @retval 0 Everything was okay. + */ +int sha1sum(FILE *f, unsigned char *sha1); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/common/include/villas/hash_table.h b/common/include/villas/hash_table.h new file mode 100644 index 000000000..9b97066f8 --- /dev/null +++ b/common/include/villas/hash_table.h @@ -0,0 +1,85 @@ +/** A generic hash table + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct hash_table_entry { + void *key; + void *data; + + struct hash_table_entry *next; +}; + +/** A thread-safe hash table using separate chaing with linked lists. */ +struct hash_table { + enum state state; + + struct hash_table_entry **table; + + size_t size; + pthread_mutex_t lock; +}; + +/** Initialize a new hash table. + * + */ +int hash_table_init(struct hash_table *ht, size_t size); + +/** Destroy a hash table. + * + * + */ +int hash_table_destroy(struct hash_table *ht, dtor_cb_t dtor, bool release); + +/** Insert a new key/value pair into the hash table. + * + */ +int hash_table_insert(struct hash_table *ht, void *key, void *data); + +/** Delete a key from the hash table. + * + * + */ +int hash_table_delete(struct hash_table *ht, void *key); + +/** Perform a lookup in the hash table. + * + * @retval != NULL The value for the given key. + * @retval NULL The given key is not stored in the hash table. + */ +void * hash_table_lookup(struct hash_table *ht, void *key); + +/** Dump the contents of the hash table in a human readable format to stdout. */ +void hash_table_dump(struct hash_table *ht); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/hist.h b/common/include/villas/hist.h new file mode 100644 index 000000000..24a917afd --- /dev/null +++ b/common/include/villas/hist.h @@ -0,0 +1,113 @@ +/** Histogram functions. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define HIST_HEIGHT (LOG_WIDTH - 55) +#define HIST_SEQ 17 + +typedef uintmax_t hist_cnt_t; + +/** Histogram structure used to collect statistics. */ +struct hist { + double resolution; /**< The distance between two adjacent buckets. */ + + double high; /**< The value of the highest bucket. */ + double low; /**< The value of the lowest bucket. */ + + double highest; /**< The highest value observed (may be higher than #high). */ + double lowest; /**< The lowest value observed (may be lower than #low). */ + double last; /**< The last value which has been put into the buckets */ + + int length; /**< The number of buckets in #data. */ + + hist_cnt_t total; /**< Total number of counted values. */ + hist_cnt_t warmup; /**< Number of values which are used during warmup phase. */ + + hist_cnt_t higher; /**< The number of values which are higher than #high. */ + hist_cnt_t lower; /**< The number of values which are lower than #low. */ + + hist_cnt_t *data; /**< Pointer to dynamically allocated array of size length. */ + + double _m[2], _s[2]; /**< Private variables for online variance calculation */ +}; + +#define hist_last(h) ((h)->last) +#define hist_highest(h) ((h)->highest) +#define hist_lowest(h) ((h)->lowest) +#define hist_total(h) ((h)->total) + +/** Initialize struct hist with supplied values and allocate memory for buckets. */ +int hist_init(struct hist *h, int buckets, hist_cnt_t warmup); + +/** Free the dynamically allocated memory. */ +int hist_destroy(struct hist *h); + +/** Reset all counters and values back to zero. */ +void hist_reset(struct hist *h); + +/** Count a value within its corresponding bucket. */ +void hist_put(struct hist *h, double value); + +/** Calcluate the variance of all counted values. */ +double hist_var(const struct hist *h); + +/** Calculate the mean average of all counted values. */ +double hist_mean(const struct hist *h); + +/** Calculate the standard derivation of all counted values. */ +double hist_stddev(const struct hist *h); + +/** Print all statistical properties of distribution including a graphilcal plot of the histogram. */ +void hist_print(const struct hist *h, int details); + +/** Print ASCII style plot of histogram */ +void hist_plot(const struct hist *h); + +/** Dump histogram data in Matlab format. + * + * @return The string containing the dump. The caller is responsible to free() the buffer. + */ +char * hist_dump(const struct hist *h); + +/** Prints Matlab struct containing all infos to file. */ +int hist_dump_matlab(const struct hist *h, FILE *f); + +/** Write the histogram in JSON format to fiel \p f. */ +int hist_dump_json(const struct hist *h, FILE *f); + +/** Build a libjansson / JSON object of the histogram. */ +json_t * hist_json(const struct hist *h); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/kernel/kernel.h b/common/include/villas/kernel/kernel.h index e6f087bb2..1f75e968a 100644 --- a/common/include/villas/kernel/kernel.h +++ b/common/include/villas/kernel/kernel.h @@ -21,7 +21,9 @@ * along with this program. If not, see . *********************************************************************************/ -/** @addtogroup fpga Kernel @{ */ +/** @addtogroup kernel Kernel + * @{ + */ #pragma once @@ -35,14 +37,16 @@ extern "C" { /* Forward declarations */ struct version; -//#include +#if WITH_CAP +#include /** Check if current process has capability \p cap. * * @retval 0 If capabilty is present. * @retval <0 If capability is not present. */ -//int kernel_check_cap(cap_value_t cap); +int kernel_check_cap(cap_value_t cap); +#endif /** Get number of reserved hugepages. */ int kernel_get_nr_hugepages(); @@ -88,8 +92,11 @@ int kernel_get_page_size(); /** Get the size of a huge page in bytes. */ int kernel_get_hugepage_size(); +/** Get CPU base frequency */ +int kernel_get_cpu_frequency(uint64_t *freq); + /** Set SMP affinity of IRQ */ -int kernel_irq_setaffinity(unsigned irq, uintmax_t affinity, uintmax_t *old); +int kernel_irq_setaffinity(unsigned irq, uintmax_t aff , uintmax_t *old); #ifdef __cplusplus } diff --git a/common/include/villas/kernel/pci.h b/common/include/villas/kernel/pci.h index 8f253306b..2a4fec56a 100644 --- a/common/include/villas/kernel/pci.h +++ b/common/include/villas/kernel/pci.h @@ -2,8 +2,24 @@ * * @file * @author Steffen Vogel - * @copyright 2017-2018, Steffen Vogel - **********************************************************************************/ + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ /** @addtogroup fpga Kernel @{ */ diff --git a/common/include/villas/kernel/rt.h b/common/include/villas/kernel/rt.h new file mode 100644 index 000000000..da9ea7767 --- /dev/null +++ b/common/include/villas/kernel/rt.h @@ -0,0 +1,56 @@ +/** Linux specific real-time optimizations + * + * @see: https://wiki.linuxfoundation.org/realtime/documentation/howto/applications/application_base + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup kernel Kernel + * @{ + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +int rt_init(int priority, int affinity); + +int rt_set_affinity(int affinity); + +int rt_set_priority(int priority); + +int rt_lock_memory(); + +/** Checks for realtime (PREEMPT_RT) patched kernel. + * + * See https://rt.wiki.kernel.org + * + * @retval 0 Kernel is patched. + * @reval <>0 Kernel is not patched. + */ +int rt_is_preemptible(); + +#ifdef __cplusplus +} +#endif + +/** @} */ diff --git a/common/include/villas/list.h b/common/include/villas/list.h index b206da037..4be9b6eef 100644 --- a/common/include/villas/list.h +++ b/common/include/villas/list.h @@ -9,15 +9,35 @@ * @file * @author Steffen Vogel * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC +* + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . *********************************************************************************/ #pragma once #include +#include #include #include +#ifdef __cplusplus +extern "C" { +#endif + #define LIST_CHUNKSIZE 16 /** Static list initialization */ @@ -45,16 +65,6 @@ __attribute__((destructor(105))) static void UNIQUE(__dtor)() { \ #define list_first(list) list_at(list, 0) #define list_last(list) list_at(list, (list)->length-1) -#ifdef __cplusplus -extern "C" { -#endif - -/** Callback to destroy list elements. - * - * @param data A pointer to the data which should be freed. - */ -typedef int (*dtor_cb_t)(void *); - /** Callback to search or sort a list. */ typedef int (*cmp_cb_t)(const void *, const void *); @@ -100,6 +110,8 @@ void list_remove(struct list *l, void *p); */ void * list_lookup(struct list *l, const char *name); +ssize_t list_lookup_index(struct list *l, const char *name); + /** Return the first element of the list for which cmp returns zero */ void * list_search(struct list *l, cmp_cb_t cmp, void *ctx); @@ -115,6 +127,16 @@ void list_sort(struct list *l, cmp_cb_t cmp); /** Set single element in list */ int list_set(struct list *l, int index, void *value); +/** Return index in list for value. + * + * @retval <0 No list entry matching \p value was found. + * @retval >=0 Entry \p value was found at returned index. + */ +ssize_t list_index(struct list *l, void *value); + +/** Extend the list to the given length by filling new slots with given value. */ +void list_extend(struct list *l, size_t len, void *val); + #ifdef __cplusplus } #endif diff --git a/common/include/villas/log.h b/common/include/villas/log.h index f37b806cf..334bd1c36 100644 --- a/common/include/villas/log.h +++ b/common/include/villas/log.h @@ -28,20 +28,13 @@ extern "C" { #endif #include +#include #include #include #include #include -#ifdef __GNUC__ - #define INDENT int __attribute__ ((__cleanup__(log_outdent), unused)) _old_indent = log_indent(1); - #define NOINDENT int __attribute__ ((__cleanup__(log_outdent), unused)) _old_indent = log_noindent(); -#else - #define INDENT ; - #define NOINDENT ; -#endif - /* The log level which is passed as first argument to print() */ #define LOG_LVL_DEBUG CLR_GRY("Debug") #define LOG_LVL_INFO CLR_WHT("Info ") @@ -54,35 +47,38 @@ extern "C" { * To be or-ed with the debug level */ enum log_facilities { - LOG_POOL = (1L << 8), - LOG_QUEUE = (1L << 9), - LOG_CONFIG = (1L << 10), - LOG_HOOK = (1L << 11), - LOG_PATH = (1L << 12), - LOG_NODE = (1L << 13), - LOG_MEM = (1L << 14), - LOG_WEB = (1L << 15), - LOG_API = (1L << 16), - LOG_LOG = (1L << 17), - LOG_VFIO = (1L << 18), - LOG_PCI = (1L << 19), - LOG_XIL = (1L << 20), - LOG_TC = (1L << 21), - LOG_IF = (1L << 22), - LOG_ADVIO = (1L << 23), + LOG_POOL = (1L << 8), + LOG_QUEUE = (1L << 9), + LOG_CONFIG = (1L << 10), + LOG_HOOK = (1L << 11), + LOG_PATH = (1L << 12), + LOG_NODE = (1L << 13), + LOG_MEM = (1L << 14), + LOG_WEB = (1L << 15), + LOG_API = (1L << 16), + LOG_LOG = (1L << 17), + LOG_VFIO = (1L << 18), + LOG_PCI = (1L << 19), + LOG_XIL = (1L << 20), + LOG_TC = (1L << 21), + LOG_IF = (1L << 22), + LOG_ADVIO = (1L << 23), + LOG_IO = (1L << 24), /* Node-types */ - LOG_SOCKET = (1L << 24), - LOG_FILE = (1L << 25), - LOG_FPGA = (1L << 26), - LOG_NGSI = (1L << 27), - LOG_WEBSOCKET = (1L << 28), - LOG_OPAL = (1L << 30), + LOG_SOCKET = (1L << 25), + LOG_FILE = (1L << 26), + LOG_FPGA = (1L << 27), + LOG_NGSI = (1L << 28), + LOG_WEBSOCKET = (1L << 29), + LOG_OPAL = (1L << 30), + LOG_COMEDI = (1L << 31), + LOG_IB = (1L << 32), /* Classes */ - LOG_NODES = LOG_NODE | LOG_SOCKET | LOG_FILE | LOG_FPGA | LOG_NGSI | LOG_WEBSOCKET | LOG_OPAL, - LOG_KERNEL = LOG_VFIO | LOG_PCI | LOG_TC | LOG_IF, - LOG_ALL = ~0xFF + LOG_NODES = LOG_NODE | LOG_SOCKET | LOG_FILE | LOG_FPGA | LOG_NGSI | LOG_WEBSOCKET | LOG_OPAL | LOG_COMEDI | LOG_IB, + LOG_KERNEL = LOG_VFIO | LOG_PCI | LOG_TC | LOG_IF, + LOG_ALL = ~0xFF }; struct log { @@ -101,40 +97,25 @@ struct log { const char *path; /**< Path of the log file. */ char *prefix; /**< Prefix each line with this string. */ int syslog; /**< Whether or not to log to syslogd. */ + bool tty; /**< Is the log file a tty? */ FILE *file; /**< Send all log output to this file / stdout / stderr. */ }; /** The global log instance. */ -extern struct log *global_log; -extern struct log default_log; +struct log *global_log; +struct log default_log; /** Initialize log object */ int log_init(struct log *l, int level, long faciltities); -int log_start(struct log *l); +int log_open(struct log *l); -int log_stop(struct log *l); +int log_close(struct log *l); /** Destroy log object */ int log_destroy(struct log *l); -/** Change log indention for current thread. - * - * The argument level can be negative! - */ -int log_indent(int levels); - -/** Disable log indention of current thread. */ -int log_noindent(); - -/** A helper function the restore the previous log indention level. - * - * This function is usually called by a __cleanup__ handler (GCC C Extension). - * See INDENT macro. - */ -void log_outdent(int *); - /** Set logging facilities based on expression. * * Currently we support two types of expressions: diff --git a/common/include/villas/table.h b/common/include/villas/table.h new file mode 100644 index 000000000..ba1d96532 --- /dev/null +++ b/common/include/villas/table.h @@ -0,0 +1,67 @@ +/** Print fancy tables + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup table Print fancy tables + * @{ + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +struct table_column { + int width; /**< Width of the column. */ + char *title; /**< The title as shown in the table header. */ + char *format; /**< The format which is used to print the table rows. */ + char *unit; /**< An optional unit which will be shown in the table header. */ + + enum { + TABLE_ALIGN_LEFT, + TABLE_ALIGN_RIGHT + } align; + + int _width; /**< The real width of this column. Calculated by table_header() */ +}; + +struct table { + int ncols; + int width; + struct table_column *cols; +}; + +/** Print a table header consisting of \p n columns. */ +void table_header(struct table *t); + +/** Print table rows. */ +void table_row(struct table *t, ...); + +/** Print the table footer. */ +void table_footer(struct table *t); + +/** @} */ + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/task.h b/common/include/villas/task.h new file mode 100644 index 000000000..4fe9a1a8b --- /dev/null +++ b/common/include/villas/task.h @@ -0,0 +1,92 @@ +/** Run tasks periodically. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** We can choose between two periodic task implementations */ +//#define PERIODIC_TASK_IMPL NANOSLEEP +#define TIMERFD 1 +#define CLOCK_NANOSLEEP 2 +#define NANOSLEEP 3 +#define RDTSC 4 + +#if defined(__MACH__) + #define PERIODIC_TASK_IMPL NANOSLEEP +#elif defined(__linux__) + #define PERIODIC_TASK_IMPL TIMERFD +#else + #error "Platform not supported" +#endif + +struct task { + int clock; /**< CLOCK_{MONOTONIC,REALTIME} */ + +#if PERIODIC_TASK_IMPL == RDTSC /* We use cycle counts in RDTSC mode */ + uint64_t period; + uint64_t next; +#else + struct timespec period; /**< The period of periodic invations of this task */ + struct timespec next; /**< The timer value for the next invocation */ +#endif + +#if PERIODIC_TASK_IMPL == TIMERFD + int fd; /**< The timerfd_create(2) file descriptior. */ +#elif PERIODIC_TASK_IMPL == RDTSC + struct rdtsc tsc; /**< Initialized by tsc_init(). */ +#endif +}; + +/** Create a new task with the given rate. */ +int task_init(struct task *t, double rate, int clock); + +int task_destroy(struct task *t); + +/** Wait until task elapsed + * + * @retval 0 An error occured. Maybe the task was stopped. + * @retval >0 The nummer of runs this task already fired. + */ +uint64_t task_wait(struct task *t); + +int task_set_next(struct task *t, struct timespec *next); +int task_set_timeout(struct task *t, double to); +int task_set_rate(struct task *t, double rate); + +/** Returns a poll'able file descriptor which becomes readable when the timer expires. + * + * Note: currently not supported on all platforms. + */ +int task_fd(struct task *t); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/timing.h b/common/include/villas/timing.h new file mode 100644 index 000000000..8a887ec5f --- /dev/null +++ b/common/include/villas/timing.h @@ -0,0 +1,55 @@ +/** Time related functions. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Get delta between two timespec structs */ +struct timespec time_diff(const struct timespec *start, const struct timespec *end); + +/** Get sum of two timespec structs */ +struct timespec time_add(const struct timespec *start, const struct timespec *end); + +/** Return current time as a struct timespec. */ +struct timespec time_now(); + +/** Return the diffrence off two timestamps as double value in seconds. */ +double time_delta(const struct timespec *start, const struct timespec *end); + +/** Convert timespec to double value representing seconds */ +double time_to_double(const struct timespec *ts); + +/** Convert double containing seconds after 1970 to timespec. */ +struct timespec time_from_double(double secs); + +#ifdef __cplusplus +} +#endif diff --git a/common/include/villas/tsc.h b/common/include/villas/tsc.h new file mode 100644 index 000000000..5db9c2cc6 --- /dev/null +++ b/common/include/villas/tsc.h @@ -0,0 +1,95 @@ +/** Measure time and sleep with IA-32 time-stamp counter. + * + * @file + * @author Steffen Vogel + * @copyright 2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#pragma once + +#include +#include +#include + +#include + +#ifdef __APPLE__ + #include + #include +#endif + +#ifndef bit_TSC + #define bit_TSC (1 << 4) +#endif + +#define bit_TSC_INVARIANT (1 << 8) +#define bit_RDTSCP (1 << 27) + +#if !(__x86_64__ || __i386__) + #error this header is for x86 only +#endif + +struct tsc { + uint64_t frequency; + + bool rdtscp_supported; + bool is_invariant; +}; + +/** Get CPU timestep counter */ +__attribute__((unused,always_inline)) +static inline uint64_t rdtscp() +{ + uint64_t tsc; + + __asm__ __volatile__( + "rdtscp;" + "shl $32, %%rdx;" + "or %%rdx,%%rax" + : "=a" (tsc) + : + : "%rcx", "%rdx", "memory" + ); + + return tsc; +} + +__attribute__((unused,always_inline)) +static inline uint64_t rdtsc() +{ + uint64_t tsc; + + __asm__ __volatile__( + "lfence;" + "rdtsc;" + "shl $32, %%rdx;" + "or %%rdx,%%rax" + : "=a" (tsc) + : + : "%rcx", "%rdx", "memory" + ); + + return tsc; +} + +int tsc_init(struct tsc *t); + +uint64_t tsc_rate_to_cyles(struct tsc *t, double rate); + +uint64_t tsc_now(struct tsc *t); diff --git a/common/include/villas/utils.h b/common/include/villas/utils.h index 1b21f5edd..96547713a 100644 --- a/common/include/villas/utils.h +++ b/common/include/villas/utils.h @@ -30,6 +30,7 @@ #include #include +#include #include #ifdef __cplusplus @@ -240,22 +241,7 @@ int version_parse(const char *s, struct version *v); #endif /** Fill buffer with random data */ -size_t read_random(char *buf, size_t len); - -/** Get CPU timestep counter */ -__attribute__((always_inline)) static inline uint64_t rdtsc() -{ - uint64_t tsc; - - __asm__ ("rdtsc;" - "shl $32, %%rdx;" - "or %%rdx,%%rax" - : "=a" (tsc) - : - : "%rcx", "%rdx", "memory"); - - return tsc; -} +ssize_t read_random(char *buf, size_t len); /** Get log2 of long long integers */ static inline int log2i(long long x) { @@ -265,9 +251,6 @@ static inline int log2i(long long x) { return sizeof(x) * 8 - __builtin_clzll(x) - 1; } -/** Sleep with rdtsc */ -void rdtsc_sleep(uint64_t nanosecs, uint64_t start); - /** Register a exit callback for program termination: SIGINT, SIGKILL & SIGALRM. */ int signals_init(void (*cb)(int signal, siginfo_t *sinfo, void *ctx)); @@ -279,6 +262,10 @@ pid_t spawn(const char *name, char *const argv[]); /** Determines the string length as printed on the screen (ignores escable sequences). */ size_t strlenp(const char *str); +/** Remove ANSI control sequences for colored output. */ +char * decolor(char *str); + + #ifdef __cplusplus } #endif diff --git a/common/lib/CMakeLists.txt b/common/lib/CMakeLists.txt index 876dd3c5e..8988ee405 100644 --- a/common/lib/CMakeLists.txt +++ b/common/lib/CMakeLists.txt @@ -21,33 +21,49 @@ ############################################################################## add_library(villas-common SHARED - plugin.cpp - utils.cpp - memory.cpp - memory_manager.cpp - - utils.c + advio.c + bitset.c + buffer.c + compat.c + crypt.c + hash_table.c + hist.c + kernel/kernel.c + kernel/rt.c list.c log.c log_config.c log_helper.c + memory.cpp + memory_manager.cpp + plugin.cpp + table.c + task.c + timing.c + tsc.c + utils.c + utils.cpp ) -if(${CMAKE_SYSTEM_NAME} STREQUAL Linux) - target_sources(villas-common - kernel/kernel.c +if(CMAKE_SYSTEM_NAME STREQUAL Linux) + target_sources(villas-common PRIVATE kernel/pci.c kernel/vfio.cpp ) endif() target_include_directories(villas-common PUBLIC - ${villas-common_SOURCE_DIR}/include - ${villas-common_SOURCE_DIR}/thirdparty/spdlog/include + ${OPENSSL_INCLUDE_DIR} + ${CURL_INCLUDE_DIRS} + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/thirdparty/spdlog/include ) target_link_libraries(villas-common PUBLIC PkgConfig::JANSSON + ${OPENSSL_LIBRARIES} + ${CURL_LIBRARIES} ${CMAKE_DL_LIBS} ) diff --git a/common/lib/advio.c b/common/lib/advio.c new file mode 100644 index 000000000..1fcc248f2 --- /dev/null +++ b/common/lib/advio.c @@ -0,0 +1,496 @@ +/** libcurl based advanced IO aka ADVIO. + * + * This example requires libcurl 7.9.7 or later. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#define BAR_WIDTH 60 /**< How wide you want the progress meter to be. */ + +static int advio_trace(CURL *handle, curl_infotype type, char *data, size_t size, void *userp) +{ + const char *text; + + switch (type) { + case CURLINFO_TEXT: + text = "info"; + break; + + case CURLINFO_HEADER_OUT: + text = "send header"; + break; + + case CURLINFO_DATA_OUT: + text = "send data"; + break; + + case CURLINFO_HEADER_IN: + text = "recv header"; + break; + + case CURLINFO_DATA_IN: + text = "recv data"; + break; + + case CURLINFO_SSL_DATA_IN: + text = "recv SSL data"; + break; + + case CURLINFO_SSL_DATA_OUT: + text = "send SSL data"; + break; + + default: /* in case a new one is introduced to shock us */ + return 0; + } + + debug(LOG_ADVIO | 5, "CURL: %s: %.*s", text, (int) size-1, data); + + return 0; +} + +static char * advio_human_time(double t, char *buf, size_t len) +{ + int i = 0; + const char *units[] = { "secs", "mins", "hrs", "days", "weeks", "months", "years" }; + int divs[] = { 60, 60, 24, 7, 4, 12 }; + + while (t > divs[i] && i < ARRAY_LEN(divs)) { + t /= divs[i]; + i++; + } + + snprintf(buf, len, "%.2f %s", t, units[i]); + + return buf; +} + +static char * advio_human_size(double s, char *buf, size_t len) +{ + int i = 0; + const char *units[] = { "B", "kiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB" }; + + while (s > 1024 && i < ARRAY_LEN(units)) { + s /= 1024; + i++; + } + + snprintf(buf, len, "%.*f %s", i ? 2 : 0, s, units[i]); + + return buf; +} + +#if LIBCURL_VERSION_NUM >= 0x072000 +static int advio_xferinfo(void *p, curl_off_t dl_total_bytes, curl_off_t dl_bytes, curl_off_t ul_total_bytes, curl_off_t ul_bytes) +{ + struct advio *af = (struct advio *) p; + double cur_time, eta_time, estimated_time, frac; + + curl_easy_getinfo(af->curl, CURLINFO_TOTAL_TIME, &cur_time); + + /* Is this transaction an upload? */ + int upload = ul_total_bytes > 0; + + curl_off_t total_bytes = upload ? ul_total_bytes : dl_total_bytes; + curl_off_t bytes = upload ? ul_bytes : dl_bytes; + + /* Are we finished? */ + if (bytes == 0) + af->completed = 0; + + if (af->completed) + return 0; + + /* Ensure that the file to be downloaded is not empty + * because that would cause a division by zero error later on */ + if (total_bytes <= 0) + return 0; + + frac = (double) bytes / total_bytes; + + estimated_time = cur_time * (1.0 / frac); + eta_time = estimated_time - cur_time; + + /* Print file sizes in human readable format */ + char buf[4][32]; + + char *bytes_human = advio_human_size(bytes, buf[0], sizeof(buf[0])); + char *total_bytes_human = advio_human_size(total_bytes, buf[1], sizeof(buf[1])); + char *eta_time_human = advio_human_time(eta_time, buf[2], sizeof(buf[2])); + + /* Part of the progressmeter that's already "full" */ + int dotz = round(frac * BAR_WIDTH); + + /* Progress bar */ + fprintf(stderr, "\r["); + + for (int i = 0 ; i < BAR_WIDTH; i++) { + if (upload) + fputc(BAR_WIDTH - i > dotz ? ' ' : '<', stderr); + else + fputc(i > dotz ? ' ' : '>', stderr); + } + + fprintf(stderr, "] "); + + /* Details */ + fprintf(stderr, "eta %-12s %12s of %-12s", eta_time_human, bytes_human, total_bytes_human); + fflush(stderr); + + if (bytes == total_bytes) { + af->completed = 1; + fprintf(stderr, "\33[2K\r"); + } + + return 0; +} +#endif /* LIBCURL_VERSION_NUM >= 0x072000 */ + +int aislocal(const char *uri) +{ + char *sep; + const char *supported_schemas[] = { "file", "http", "https", "tftp", "ftp", "scp", "sftp", "smb", "smbs" }; + + sep = strstr(uri, "://"); + if (!sep) + return 1; /* no schema, we assume its a local file */ + + for (int i = 0; i < ARRAY_LEN(supported_schemas); i++) { + if (!strncmp(supported_schemas[i], uri, sep - uri)) + return 0; + } + + return -1; /* none of the supported schemas match. this is an invalid uri */ +} + +AFILE * afopen(const char *uri, const char *mode) +{ + int ret; + char *sep, *cwd; + + AFILE *af = alloc(sizeof(AFILE)); + + snprintf(af->mode, sizeof(af->mode), "%s", mode); + + sep = strstr(uri, "://"); + if (sep) { + af->uri = strdup(uri); + if (!af->uri) + goto out2; + } + else { /* Open local file by prepending file:// schema. */ + if (strlen(uri) <= 1) + return NULL; + + /* Handle relative paths */ + if (uri[0] != '/') { + cwd = getcwd(NULL, 0); + + af->uri = strf("file://%s/%s", cwd, uri); + + free(cwd); + } + else + af->uri = strf("file://%s", uri); + } + + af->file = tmpfile(); + if (!af->file) + goto out2; + + af->curl = curl_easy_init(); + if (!af->curl) + goto out1; + + /* Setup libcurl handle */ + curl_easy_setopt(af->curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(af->curl, CURLOPT_UPLOAD, 0L); + curl_easy_setopt(af->curl, CURLOPT_USERAGENT, USER_AGENT); + curl_easy_setopt(af->curl, CURLOPT_URL, af->uri); + curl_easy_setopt(af->curl, CURLOPT_WRITEDATA, af->file); + curl_easy_setopt(af->curl, CURLOPT_READDATA, af->file); + + curl_easy_setopt(af->curl, CURLOPT_DEBUGFUNCTION, advio_trace); + curl_easy_setopt(af->curl, CURLOPT_VERBOSE, 1); + +/* CURLOPT_XFERINFOFUNCTION is only supported on libcurl >= 7.32.0 */ +#if LIBCURL_VERSION_NUM >= 0x072000 + curl_easy_setopt(af->curl, CURLOPT_XFERINFOFUNCTION, advio_xferinfo); + curl_easy_setopt(af->curl, CURLOPT_XFERINFODATA, af); +#endif + + ret = adownload(af, 0); + if (ret) + goto out0; + + af->uploaded = 0; + af->downloaded = 0; + + return af; + +out0: curl_easy_cleanup(af->curl); +out1: fclose(af->file); +out2: free(af->uri); + free(af); + + return NULL; +} + +int afclose(AFILE *af) +{ + int ret; + + ret = afflush(af); + if (ret) + return ret; + + curl_easy_cleanup(af->curl); + + ret = fclose(af->file); + if (ret) + return ret; + + free(af->uri); + free(af); + + return 0; +} + +int afseek(AFILE *af, long offset, int origin) +{ + long new, cur = aftell(af); + + switch (origin) { + case SEEK_SET: + new = offset; + break; + + case SEEK_END: + fseek(af->file, 0, SEEK_END); + new = aftell(af); + fseek(af->file, cur, SEEK_SET); + break; + + case SEEK_CUR: + new = cur + offset; + break; + + default: + return -1; + } + + if (new < af->uploaded) + af->uploaded = new; + + return fseek(af->file, offset, origin); +} + +void arewind(AFILE *af) +{ + af->uploaded = 0; + + return rewind(af->file); +} + +int afflush(AFILE *af) +{ + bool dirty; + unsigned char hash[SHA_DIGEST_LENGTH]; + + /* Check if fle was modified on disk by comparing hashes */ + sha1sum(af->file, hash); + dirty = memcmp(hash, af->hash, sizeof(hash)); + + if (dirty) + return aupload(af, 1); + + return 0; +} + +int aupload(AFILE *af, int resume) +{ + CURLcode res; + + long pos, end; + + double total_bytes = 0, total_time = 0; + char buf[2][32]; + + pos = aftell(af); + fseek(af->file, 0, SEEK_END); + end = aftell(af); + fseek(af->file, 0, SEEK_SET); + + if (resume) { + if (end == af->uploaded) + return 0; + + char *size_human = advio_human_size(end - af->uploaded, buf[0], sizeof(buf[0])); + + info("Resume upload of %s of %s from offset %lu", af->uri, size_human, af->uploaded); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, af->uploaded); + } + else { + char *size_human = advio_human_size(end, buf[0], sizeof(buf[0])); + + info("Start upload of %s of %s", af->uri, size_human); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, 0); + } + + curl_easy_setopt(af->curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(af->curl, CURLOPT_INFILESIZE, end - af->uploaded); + curl_easy_setopt(af->curl, CURLOPT_NOPROGRESS, !isatty(fileno(stderr))); + + res = curl_easy_perform(af->curl); + + fseek(af->file, pos, SEEK_SET); /* Restore old stream pointer */ + + if (res != CURLE_OK) + return -1; + + sha1sum(af->file, af->hash); + + curl_easy_getinfo(af->curl, CURLINFO_SIZE_UPLOAD, &total_bytes); + curl_easy_getinfo(af->curl, CURLINFO_TOTAL_TIME, &total_time); + + char *total_bytes_human = advio_human_size(total_bytes, buf[0], sizeof(buf[0])); + char *total_time_human = advio_human_time(total_time, buf[1], sizeof(buf[1])); + + info("Finished upload of %s in %s", total_bytes_human, total_time_human); + + af->uploaded += total_bytes; + + return 0; +} + +int adownload(AFILE *af, int resume) +{ + CURLcode res; + long code, pos; + int ret; + + double total_bytes = 0, total_time = 0; + char buf[2][32]; + + pos = aftell(af); + + if (resume) { + info("Resume download of %s from offset %lu", af->uri, af->downloaded); + + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, af->downloaded); + } + else { + info("Start download of %s", af->uri); + + rewind(af->file); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, 0); + } + + curl_easy_setopt(af->curl, CURLOPT_UPLOAD, 0L); + curl_easy_setopt(af->curl, CURLOPT_NOPROGRESS, !isatty(fileno(stderr))); + + res = curl_easy_perform(af->curl); + + switch (res) { + case CURLE_OK: + curl_easy_getinfo(af->curl, CURLINFO_SIZE_DOWNLOAD, &total_bytes); + curl_easy_getinfo(af->curl, CURLINFO_TOTAL_TIME, &total_time); + + char *total_bytes_human = advio_human_size(total_bytes, buf[0], sizeof(buf[0])); + char *total_time_human = advio_human_time(total_time, buf[1], sizeof(buf[1])); + + info("Finished download of %s in %s", total_bytes_human, total_time_human); + + af->downloaded += total_bytes; + af->uploaded = af->downloaded; + + res = curl_easy_getinfo(af->curl, CURLINFO_RESPONSE_CODE, &code); + if (res) + return -1; + + switch (code) { + case 0: + case 200: goto exist; + case 404: goto notexist; + default: return -1; + } + + /* The following error codes indicate that the file does not exist + * Check the fopen mode to see if we should continue with an emoty file */ + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_TFTP_NOTFOUND: + case CURLE_REMOTE_FILE_NOT_FOUND: + info("File does not exist."); + goto notexist; + + /* If libcurl does not know the protocol, we will try it with the stdio */ + case CURLE_UNSUPPORTED_PROTOCOL: + af->file = fopen(af->uri, af->mode); + if (!af->file) + return -1; + + default: + error("Failed to download file: %s: %s", af->uri, curl_easy_strerror(res)); + return -1; + } + + +notexist: /* File does not exist */ + + /* According to mode the file must exist! */ + if (af->mode[1] != '+' || (af->mode[0] != 'w' && af->mode[0] != 'a')) { + errno = ENOENT; + return -1; + } + + /* If we receive a 404, we discard the already received error page + * and start with an empty file. */ + fflush(af->file); + ret = ftruncate(fileno(af->file), 0); + if (ret) + return ret; + +exist: /* File exists */ + if (resume) + afseek(af, pos, SEEK_SET); + else if (af->mode[0] == 'a') + afseek(af, 0, SEEK_END); + else if (af->mode[0] == 'r' || af->mode[0] == 'w') + afseek(af, 0, SEEK_SET); + + sha1sum(af->file, af->hash); + + return 0; +} diff --git a/common/lib/bitset.c b/common/lib/bitset.c new file mode 100644 index 000000000..01df9edbb --- /dev/null +++ b/common/lib/bitset.c @@ -0,0 +1,170 @@ +/** A datastructure storing bitsets of arbitrary dimensions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include +#include + +#define bitset_mask(b) (1 << ((b) % CHAR_BIT)) +#define bitset_slot(b) ((b) / CHAR_BIT) +#define bitset_nslots(nb) ((nb + CHAR_BIT - 1) / CHAR_BIT) + +int bitset_init(struct bitset *b, size_t dim) +{ + int s = bitset_nslots(dim); + + b->set = alloc(s * CHAR_BIT); + b->dimension = dim; + + return 0; +} + +int bitset_destroy(struct bitset *b) +{ + free(b->set); + + return 0; +} + +void bitset_set_value(struct bitset *b, uintmax_t val) +{ + bitset_clear_all(b); + + for (size_t i = 0; i < b->dimension; i++) { + if (val & (1 << i)) + bitset_set(b, i); + } +} + +uintmax_t bitset_get_value(struct bitset *b) +{ + uintmax_t v = 0; + + for (size_t i = 0; i < b->dimension; i++) + v += bitset_test(b, i) << i; + + return v; +} + +size_t bitset_count(struct bitset *b) +{ + size_t cnt = 0; + + for (size_t i = 0; i < b->dimension; i++) + cnt += bitset_test(b, i); + + return cnt; +} + +int bitset_set(struct bitset *b, size_t bit) +{ + int s = bitset_slot(bit); + char m = bitset_mask(bit); + + if (bit >= b->dimension) + return -1; + + b->set[s] |= m; + + return 0; +} + +int bitset_clear(struct bitset *b, size_t bit) +{ + int s = bitset_slot(bit); + char m = bitset_mask(bit); + + if (bit >= b->dimension) + return -1; + + b->set[s] &= ~m; + + return 0; +} + +int bitset_test(struct bitset *b, size_t bit) +{ + int s = bitset_slot(bit); + char m = bitset_mask(bit); + + if (bit >= b->dimension) + return -1; + + return b->set[s] & m ? 1 : 0; +} + +void bitset_set_all(struct bitset *b) +{ + int s = b->dimension / CHAR_BIT; + + if (b->dimension % CHAR_BIT) { + char m = (1 << (b->dimension % CHAR_BIT)) - 1; + + b->set[s] |= m; + } + + memset(b->set, ~0, s); +} + +void bitset_clear_all(struct bitset *b) +{ + int s = b->dimension / CHAR_BIT; + + /* Clear last byte */ + if (b->dimension % CHAR_BIT) { + char m = (1 << (b->dimension % CHAR_BIT)) - 1; + + b->set[s] &= ~m; + } + + memset(b->set, 0x00, s); +} + +int bitset_cmp(struct bitset *a, struct bitset *b) +{ + int s = a->dimension / CHAR_BIT; + + if (a->dimension != b->dimension) + return -1; + + /* Compare last byte with mask */ + if (a->dimension % CHAR_BIT) { + char m = (1 << (a->dimension % CHAR_BIT)) - 1; + + if ((a->set[s] & m) != (b->set[s] & m)) + return -1; + } + + return memcmp(a->set, b->set, s); +} + +char * bitset_dump(struct bitset *b) +{ + char *str = NULL; + + for (int i = 0; i < b->dimension; i++) + strcatf(&str, "%d", bitset_test(b, i)); + + return str; +} diff --git a/common/lib/buffer.c b/common/lib/buffer.c new file mode 100644 index 000000000..2069971a5 --- /dev/null +++ b/common/lib/buffer.c @@ -0,0 +1,100 @@ +/** A simple growing buffer. + * + * @file + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include +#include + +int buffer_init(struct buffer *b, size_t size) +{ + b->len = 0; + b->size = size; + b->buf = malloc(size); + if (!b->buf) + return -1; + + b->state = STATE_INITIALIZED; + + return 0; +} + +int buffer_destroy(struct buffer *b) +{ + if (b->buf) + free(b->buf); + + b->state = STATE_DESTROYED; + + return 0; +} + +void buffer_clear(struct buffer *b) +{ + b->len = 0; +} + +int buffer_append(struct buffer *b, const char *data, size_t len) +{ + if (b->len + len > b->size) { + b->size = b->len + len; + b->buf = realloc(b->buf, b->size); + if (!b->buf) + return -1; + } + + memcpy(b->buf + b->len, data, len); + + b->len += len; + + return 0; +} + +int buffer_parse_json(struct buffer *b, json_t **j) +{ + *j = json_loadb(b->buf, b->len, 0, NULL); + if (!*j) + return -1; + + return 0; +} + +int buffer_append_json(struct buffer *b, json_t *j) +{ + size_t len; + +retry: len = json_dumpb(j, b->buf + b->len, b->size - b->len, 0); + if (b->size < b->len + len) { + b->buf = realloc(b->buf, b->len + len); + if (!b->buf) + return -1; + + b->size = b->len + len; + goto retry; + } + + b->len += len; + + return 0; +} diff --git a/common/lib/compat.c b/common/lib/compat.c new file mode 100644 index 000000000..fd7de9324 --- /dev/null +++ b/common/lib/compat.c @@ -0,0 +1,46 @@ +/** Compatability for different library versions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include + +#if JANSSON_VERSION_HEX < 0x020A00 +size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags) +{ + char *str; + size_t len; + + str = json_dumps(json, flags); + if (!str) + return 0; + + len = strlen(str); // not \0 terminated + if (buffer && len <= size) + memcpy(buffer, str, len); + + free(str); + + return len; +} +#endif diff --git a/common/lib/crypt.c b/common/lib/crypt.c new file mode 100644 index 000000000..2df0c9ad7 --- /dev/null +++ b/common/lib/crypt.c @@ -0,0 +1,49 @@ +/** Crypto helpers. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +int sha1sum(FILE *f, unsigned char *sha1) +{ + SHA_CTX c; + char buf[512]; + ssize_t bytes; + long seek; + + seek = ftell(f); + fseek(f, 0, SEEK_SET); + + SHA1_Init(&c); + + bytes = fread(buf, 1, 512, f); + while (bytes > 0) { + SHA1_Update(&c, buf, bytes); + bytes = fread(buf, 1, 512, f); + } + + SHA1_Final(sha1, &c); + + fseek(f, seek, SEEK_SET); + + return 0; +} + diff --git a/common/lib/hash_table.c b/common/lib/hash_table.c new file mode 100644 index 000000000..d38aa2f15 --- /dev/null +++ b/common/lib/hash_table.c @@ -0,0 +1,205 @@ +/** A generic hash table + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +static int hash_table_hash(struct hash_table *ht, void *key) +{ + uintptr_t ptr = (uintptr_t) key; + + return ptr % ht->size; +} + +int hash_table_init(struct hash_table *ht, size_t size) +{ + int ret; + size_t len = sizeof(struct hash_table_entry *) * size; + + assert(ht->state == STATE_DESTROYED); + + ret = pthread_mutex_init(&ht->lock, NULL); + if (ret) + return ret; + + ht->table = alloc(len); + + memset(ht->table, 0, len); + + ht->size = size; + ht->state = STATE_INITIALIZED; + + return 0; +} + +int hash_table_destroy(struct hash_table *ht, dtor_cb_t dtor, bool release) +{ + int ret; + struct hash_table_entry *cur, *next; + + assert(ht->state == STATE_INITIALIZED); + + pthread_mutex_lock(&ht->lock); + + for (int i = 0; i < ht->size; i++) { + for (cur = ht->table[i]; cur; cur = next) { + if (dtor) + dtor(cur->data); + + if (release) + free(cur->data); + + next = cur->next; + + free(cur); + } + } + + pthread_mutex_unlock(&ht->lock); + + ret = pthread_mutex_destroy(&ht->lock); + if (ret) + return ret; + + free(ht->table); + + ht->state = STATE_DESTROYED; + + return 0; +} + +int hash_table_insert(struct hash_table *ht, void *key, void *data) +{ + int ret, ikey = hash_table_hash(ht, key); + struct hash_table_entry *hte, *cur; + + assert(ht->state == STATE_INITIALIZED); + + pthread_mutex_lock(&ht->lock); + + /* Check that the key is not already in the table */ + for (cur = ht->table[ikey]; + cur && cur->key != key; + cur = cur->next); + + if (cur) + ret = -1; + else { + hte = alloc(sizeof(struct hash_table_entry)); + if (hte) { + hte->key = key; + hte->data = data; + hte->next = NULL; + + if ((cur = ht->table[ikey])) + hte->next = ht->table[ikey]; + + ht->table[ikey] = hte; + + ret = 0; + } + else + ret = -1; + } + + pthread_mutex_unlock(&ht->lock); + + return ret; +} + +#include + +int hash_table_delete(struct hash_table *ht, void *key) +{ + int ret, ikey = hash_table_hash(ht, key); + struct hash_table_entry *cur, *prev; + + assert(ht->state == STATE_INITIALIZED); + + pthread_mutex_lock(&ht->lock); + + for (prev = NULL, + cur = ht->table[ikey]; + cur && cur->key != key; + prev = cur, + cur = cur->next); + + if (cur) { + if (prev) + prev->next = cur->next; + else + ht->table[ikey] = cur->next; + + free(cur); + + ret = 0; + } + else + ret = -1; /* not found */ + + pthread_mutex_unlock(&ht->lock); + + return ret; +} + +void * hash_table_lookup(struct hash_table *ht, void *key) +{ + int ikey = hash_table_hash(ht, key); + struct hash_table_entry *hte; + + assert(ht->state == STATE_INITIALIZED); + + pthread_mutex_lock(&ht->lock); + + for (hte = ht->table[ikey]; + hte && hte->key != key; + hte = hte->next); + + void *data = hte ? hte->data : NULL; + + pthread_mutex_unlock(&ht->lock); + + return data; +} + +void hash_table_dump(struct hash_table *ht) +{ + struct hash_table_entry *hte; + + assert(ht->state == STATE_INITIALIZED); + + pthread_mutex_lock(&ht->lock); + + for (int i = 0; i < ht->size; i++) { + char *strlst = NULL; + + for (hte = ht->table[i]; hte; hte = hte->next) + strcatf(&strlst, "%p->%p ", hte->key, hte->data); + + info("%i: %s", i, strlst); + free(strlst); + } + + pthread_mutex_unlock(&ht->lock); +} diff --git a/common/lib/hist.c b/common/lib/hist.c new file mode 100644 index 000000000..565bd8db0 --- /dev/null +++ b/common/lib/hist.c @@ -0,0 +1,287 @@ +/** Histogram functions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define VAL(h, i) ((h)->low + (i) * (h)->resolution) +#define INDEX(h, v) round((v - (h)->low) / (h)->resolution) + +int hist_init(struct hist *h, int buckets, hist_cnt_t warmup) +{ + h->length = buckets; + h->warmup = warmup; + + h->data = buckets ? alloc(h->length * sizeof(hist_cnt_t)) : NULL; + + hist_reset(h); + + return 0; +} + +int hist_destroy(struct hist *h) +{ + if (h->data) { + free(h->data); + h->data = NULL; + } + + return 0; +} + +void hist_put(struct hist *h, double value) +{ + h->last = value; + + /* Update min/max */ + if (value > h->highest) + h->highest = value; + if (value < h->lowest) + h->lowest = value; + + if (h->total < h->warmup) { + + } + else if (h->total == h->warmup) { + h->low = hist_mean(h) - 3 * hist_stddev(h); + h->high = hist_mean(h) + 3 * hist_stddev(h); + h->resolution = (h->high - h->low) / h->length; + } + else { + int idx = INDEX(h, value); + + /* Check bounds and increment */ + if (idx >= h->length) + h->higher++; + else if (idx < 0) + h->lower++; + else if (h->data != NULL) + h->data[idx]++; + } + + h->total++; + + /* Online / running calculation of variance and mean + * by Donald Knuthโ€™s Art of Computer Programming, Vol 2, page 232, 3rd edition */ + if (h->total == 1) { + h->_m[1] = h->_m[0] = value; + h->_s[1] = 0.0; + } + else { + h->_m[0] = h->_m[1] + (value - h->_m[1]) / h->total; + h->_s[0] = h->_s[1] + (value - h->_m[1]) * (value - h->_m[0]); + + // set up for next iteration + h->_m[1] = h->_m[0]; + h->_s[1] = h->_s[0]; + } + +} + +void hist_reset(struct hist *h) +{ + h->total = 0; + h->higher = 0; + h->lower = 0; + + h->highest = DBL_MIN; + h->lowest = DBL_MAX; + + if (h->data) + memset(h->data, 0, h->length * sizeof(unsigned)); +} + +double hist_mean(const struct hist *h) +{ + return (h->total > 0) ? h->_m[0] : NAN; +} + +double hist_var(const struct hist *h) +{ + return (h->total > 1) ? h->_s[0] / (h->total - 1) : NAN; +} + +double hist_stddev(const struct hist *h) +{ + return sqrt(hist_var(h)); +} + +void hist_print(const struct hist *h, int details) +{ + if (h->total > 0) { + hist_cnt_t missed = h->total - h->higher - h->lower; + + stats("Counted values: %ju (%ju between %f and %f)", h->total, missed, h->low, h->high); + stats("Highest: %g", h->highest); + stats("Lowest: %g", h->lowest); + stats("Mu: %g", hist_mean(h)); + stats("1/Mu: %g", 1.0/hist_mean(h)); + stats("Variance: %g", hist_var(h)); + stats("Stddev: %g", hist_stddev(h)); + + if (details > 0 && h->total - h->higher - h->lower > 0) { + char *buf = hist_dump(h); + stats("Matlab: %s", buf); + free(buf); + + hist_plot(h); + } + } + else + stats("Counted values: %ju", h->total); +} + +void hist_plot(const struct hist *h) +{ + hist_cnt_t max = 1; + + /* Get highest bar */ + for (int i = 0; i < h->length; i++) { + if (h->data[i] > max) + max = h->data[i]; + } + + struct table_column cols[] = { + { -9, "Value", "%+9.3g", NULL, TABLE_ALIGN_RIGHT }, + { -6, "Count", "%6ju", NULL, TABLE_ALIGN_RIGHT }, + { 0, "Plot", "%s", "occurences", TABLE_ALIGN_LEFT } + }; + + struct table table = { + .ncols = ARRAY_LEN(cols), + .cols = cols + }; + + /* Print plot */ + table_header(&table); + + for (int i = 0; i < h->length; i++) { + double value = VAL(h, i); + hist_cnt_t cnt = h->data[i]; + int bar = cols[2]._width * ((double) cnt / max); + + char *buf = strf("%s", ""); + for (int i = 0; i < bar; i++) + buf = strcatf(&buf, "\u2588"); + + table_row(&table, value, cnt, buf); + + free(buf); + } + + table_footer(&table); +} + +char * hist_dump(const struct hist *h) +{ + char *buf = alloc(128); + + strcatf(&buf, "[ "); + + for (int i = 0; i < h->length; i++) + strcatf(&buf, "%ju ", h->data[i]); + + strcatf(&buf, "]"); + + return buf; +} + +json_t * hist_json(const struct hist *h) +{ + json_t *json_buckets, *json_hist; + + json_hist = json_pack("{ s: f, s: f, s: i }", + "low", h->low, + "high", h->high, + "total", h->total + ); + + if (h->total > 0) { + json_object_update(json_hist, json_pack("{ s: i, s: i, s: f, s: f, s: f, s: f, s: f }", + "higher", h->higher, + "lower", h->lower, + "highest", h->highest, + "lowest", h->lowest, + "mean", hist_mean(h), + "variance", hist_var(h), + "stddev", hist_stddev(h) + )); + } + + if (h->total - h->lower - h->higher > 0) { + json_buckets = json_array(); + + for (int i = 0; i < h->length; i++) + json_array_append(json_buckets, json_integer(h->data[i])); + + json_object_set(json_hist, "buckets", json_buckets); + } + + return json_hist; +} + +int hist_dump_json(const struct hist *h, FILE *f) +{ + json_t *j = hist_json(h); + + int ret = json_dumpf(j, f, 0); + + json_decref(j); + + return ret; +} + +int hist_dump_matlab(const struct hist *h, FILE *f) +{ + fprintf(f, "struct("); + fprintf(f, "'low', %f, ", h->low); + fprintf(f, "'high', %f, ", h->high); + fprintf(f, "'total', %ju, ", h->total); + fprintf(f, "'higher', %ju, ", h->higher); + fprintf(f, "'lower', %ju, ", h->lower); + fprintf(f, "'highest', %f, ", h->highest); + fprintf(f, "'lowest', %f, ", h->lowest); + fprintf(f, "'mean', %f, ", hist_mean(h)); + fprintf(f, "'variance', %f, ", hist_var(h)); + fprintf(f, "'stddev', %f, ", hist_stddev(h)); + + if (h->total - h->lower - h->higher > 0) { + char *buf = hist_dump(h); + fprintf(f, "'buckets', %s", buf); + free(buf); + } + else + fprintf(f, "'buckets', zeros(1, %d)", h->length); + + fprintf(f, ")"); + + return 0; +} diff --git a/common/lib/kernel/kernel.c b/common/lib/kernel/kernel.c index 64d004c1b..1d29d3acd 100644 --- a/common/lib/kernel/kernel.c +++ b/common/lib/kernel/kernel.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include @@ -45,6 +45,13 @@ int kernel_get_cacheline_size() #endif } +#if defined(__linux__) || defined(__APPLE__) +int kernel_get_page_size() +{ + return sysconf(_SC_PAGESIZE); +} +#endif + #ifdef __linux__ int kernel_module_set_param(const char *module, const char *param, const char *value) @@ -152,7 +159,7 @@ int kernel_get_cmdline_param(const char *param, char *buf, size_t len) if (strcmp(param, key) == 0) { if (ret >= 2 && buf) - strncpy(buf, value, len); + snprintf(buf, len, "%s", value); return 0; /* found */ } @@ -165,11 +172,6 @@ out: return -1; /* not found or error */ } -int kernel_get_page_size() -{ - return sysconf(_SC_PAGESIZE); -} - /* There is no sysconf interface to get the hugepage size */ int kernel_get_hugepage_size() { @@ -259,7 +261,7 @@ int kernel_has_cap(cap_value_t cap) } #endif -int kernel_irq_setaffinity(unsigned irq, uintmax_t affinity, uintmax_t *old) +int kernel_irq_setaffinity(unsigned irq, uintmax_t aff, uintmax_t *old) { char fn[64]; FILE *f; @@ -269,15 +271,75 @@ int kernel_irq_setaffinity(unsigned irq, uintmax_t affinity, uintmax_t *old) f = fopen(fn, "w+"); if (!f) - return errno; + return -1; /* IRQ does not exist */ if (old) ret = fscanf(f, "%jx", old); - fprintf(f, "%jx", affinity); + fprintf(f, "%jx", aff); fclose(f); return ret; } +int kernel_get_cpu_frequency(uint64_t *freq) +{ + char *line = NULL, *sep, *end; + size_t len = 0; + double dfreq; + int ret; + FILE *f; + + /* Try to get CPU frequency from cpufreq module */ + f = fopen(SYSFS_PATH "/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", "r"); + if (!f) + goto cpuinfo; + + ret = fscanf(f, "%" PRIu64, freq); + fclose(f); + if (ret != 1) + return -1; + + /* cpufreq reports kHz */ + *freq = *freq * 1000; + + return 0; + +cpuinfo: + /* Try to read CPU frequency from /proc/cpuinfo */ + f = fopen(PROCFS_PATH "/cpuinfo", "r"); + if (!f) + return -1; /* We give up here */ + + ret = -1; + while (getline(&line, &len, f) >= 0) { + if (strstr(line, "cpu MHz") == line) { + ret = 0; + break; + } + } + if (ret) + goto out; + + sep = strchr(line, ':'); + if (!sep) { + ret = -1; + goto out; + } + + dfreq = strtod(sep+1, &end); + + if (end == sep+1) { + ret = -1; + goto out; + } + + /* Frequency is given in MHz */ + *freq = dfreq * 1e6; + +out: fclose(f); + free(line); + + return ret; +} #endif /* __linux__ */ diff --git a/common/lib/kernel/pci.c b/common/lib/kernel/pci.c index 6e035bd00..07a78dd5e 100644 --- a/common/lib/kernel/pci.c +++ b/common/lib/kernel/pci.c @@ -1,7 +1,7 @@ /** Linux PCI helpers * * @author Steffen Vogel - * @copyright 2017-2018, Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * * VILLAScommon @@ -28,9 +28,8 @@ #include #include - -#include #include +#include int pci_init(struct pci *p) { @@ -373,7 +372,6 @@ int pci_get_iommu_group(const struct pci_device *d) { int ret; char *group, link[1024], sysfs[1024]; - memset(link, 0, sizeof(link)); snprintf(sysfs, sizeof(sysfs), "%s/bus/pci/devices/%04x:%02x:%02x.%x/iommu_group", SYSFS_PATH, d->slot.domain, d->slot.bus, d->slot.device, d->slot.function); diff --git a/common/lib/kernel/rt.c b/common/lib/kernel/rt.c new file mode 100644 index 000000000..01180f297 --- /dev/null +++ b/common/lib/kernel/rt.c @@ -0,0 +1,145 @@ +/** Linux specific real-time optimizations + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include +#include + +#include +#include + +int rt_init(int priority, int affinity) +{ + info("Initialize real-time sub-system"); + + { +#ifdef __linux__ + int is_rt; + + /* Use FIFO scheduler with real time priority */ + is_rt = rt_is_preemptible(); + if (is_rt) + warn("We recommend to use an PREEMPT_RT patched kernel!"); + + if (priority) + rt_set_priority(priority); + else + warn("You might want to use the 'priority' setting to increase VILLASnode's process priority"); + + if (affinity) + rt_set_affinity(affinity); + else + warn("You might want to use the 'affinity' setting to pin VILLASnode to dedicate CPU cores"); + + rt_lock_memory(); +#else + warn("This platform is not optimized for real-time execution"); +#endif + } + + return 0; +} + +#ifdef __linux__ + +int rt_lock_memory() +{ + int ret; + +#ifdef _POSIX_MEMLOCK + ret = mlockall(MCL_CURRENT | MCL_FUTURE); + if (ret) + error("Failed to lock memory"); +#endif + + return 0; +} + +int rt_set_affinity(int affinity) +{ + char isolcpus[255]; + int is_isol, ret; + + /* Pin threads to CPUs by setting the affinity */ + cpu_set_t cset_pin, cset_isol, cset_non_isol; + + cpuset_from_integer(affinity, &cset_pin); + + is_isol = kernel_get_cmdline_param("isolcpus", isolcpus, sizeof(isolcpus)); + if (is_isol) { + warn("You should reserve some cores for VILLASnode (see 'isolcpus')"); + + CPU_ZERO(&cset_isol); + } + else { + ret = cpulist_parse(isolcpus, &cset_isol, 0); + if (ret) + error("Invalid isolcpus cmdline parameter: %s", isolcpus); + + CPU_XOR(&cset_non_isol, &cset_isol, &cset_pin); + if (CPU_COUNT(&cset_non_isol) > 0) { + char isol[128], pin[128]; + + cpulist_create(isol, sizeof(isol), &cset_isol); + cpulist_create(pin, sizeof(pin), &cset_pin); + + warn("Affinity setting includes cores which are not isolated: affinity=%s isolcpus=%s", pin, isol); + } + } + + char list[128]; + cpulist_create(list, sizeof(list), &cset_pin); + + ret = sched_setaffinity(0, sizeof(cpu_set_t), &cset_pin); + if (ret) + serror("Failed to set CPU affinity to %s", list); + + debug(LOG_KERNEL | 3, "Set affinity to %s", list); + + return 0; +} + +int rt_set_priority(int priority) +{ + int ret; + struct sched_param param = { + .sched_priority = priority + }; + + ret = sched_setscheduler(0, SCHED_FIFO, ¶m); + if (ret) + serror("Failed to set real time priority"); + + debug(LOG_KERNEL | 3, "Task priority set to %u", priority); + + return 0; +} + +int rt_is_preemptible() +{ + return access(SYSFS_PATH "/kernel/realtime", R_OK); +} + +#endif /* __linux__ */ diff --git a/common/lib/kernel/vfio.c b/common/lib/kernel/vfio.c new file mode 100644 index 000000000..3e0cfcab9 --- /dev/null +++ b/common/lib/kernel/vfio.c @@ -0,0 +1,622 @@ +/** Virtual Function IO wrapper around kernel API + * + * @author Steffen Vogel + * @copyright 2017, Steffen Vogel + **********************************************************************************/ + +#define _DEFAULT_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +static const char *vfio_pci_region_names[] = { + "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX, + "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX, + "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX, + "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX, + "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX, + "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX, + "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX, + "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX, + "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX, +}; + +static const char *vfio_pci_irq_names[] = { + "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX, + "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX, + "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX, + "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX, + "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX, +}; + +/* Helpers */ +int vfio_get_iommu_name(int index, char *buf, size_t len) +{ + FILE *f; + char fn[256]; + + snprintf(fn, sizeof(fn), "/sys/kernel/iommu_groups/%d/name", index); + + f = fopen(fn, "r"); + if (!f) + return -1; + + int ret = fgets(buf, len, f) == buf ? 0 : -1; + + /* Remove trailing newline */ + char *c = strrchr(buf, '\n'); + if (c) + *c = 0; + + fclose(f); + + return ret; +} + +/* Destructors */ +int vfio_destroy(struct vfio_container *v) +{ + int ret; + + /* Release memory and close fds */ + list_destroy(&v->groups, (dtor_cb_t) vfio_group_destroy, true); + + /* Close container */ + ret = close(v->fd); + if (ret < 0) + return -1; + + debug(5, "VFIO: closed container: fd=%d", v->fd); + + return 0; +} + +int vfio_group_destroy(struct vfio_group *g) +{ + int ret; + + list_destroy(&g->devices, (dtor_cb_t) vfio_device_destroy, false); + + ret = ioctl(g->fd, VFIO_GROUP_UNSET_CONTAINER); + if (ret) + return ret; + + debug(5, "VFIO: released group from container: group=%u", g->index); + + ret = close(g->fd); + if (ret) + return ret; + + debug(5, "VFIO: closed group: group=%u, fd=%d", g->index, g->fd); + + return 0; +} + +int vfio_device_destroy(struct vfio_device *d) +{ + int ret; + + for (int i = 0; i < d->info.num_regions; i++) + vfio_unmap_region(d, i); + + ret = close(d->fd); + if (ret) + return ret; + + debug(5, "VFIO: closed device: name=%s, fd=%d", d->name, d->fd); + + free(d->mappings); + free(d->name); + + return 0; +} + +/* Constructors */ +int vfio_init(struct vfio_container *v) +{ + int ret; + + /* Initialize datastructures */ + memset(v, 0, sizeof(*v)); + + list_init(&v->groups); + + /* Load VFIO kernel module */ + if (kernel_module_load("vfio")) + error("Failed to load kernel module: %s", "vfio"); + + /* Open VFIO API */ + v->fd = open(VFIO_DEV("vfio"), O_RDWR); + if (v->fd < 0) + error("Failed to open VFIO container"); + + /* Check VFIO API version */ + v->version = ioctl(v->fd, VFIO_GET_API_VERSION); + if (v->version < 0 || v->version != VFIO_API_VERSION) + error("Failed to get VFIO version"); + + /* Check available VFIO extensions (IOMMU types) */ + v->extensions = 0; + for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { + ret = ioctl(v->fd, VFIO_CHECK_EXTENSION, i); + if (ret < 0) + error("Failed to get VFIO extensions"); + else if (ret > 0) + v->extensions |= (1 << i); + } + + return 0; +} + +int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index) +{ + int ret; + char buf[128]; + + g->index = index; + g->container = c; + + list_init(&g->devices); + + /* Open group fd */ + snprintf(buf, sizeof(buf), VFIO_DEV("%u"), g->index); + g->fd = open(buf, O_RDWR); + if (g->fd < 0) + serror("Failed to open VFIO group: %u", g->index); + + /* Claim group ownership */ + ret = ioctl(g->fd, VFIO_GROUP_SET_CONTAINER, &c->fd); + if (ret < 0) + serror("Failed to attach VFIO group to container"); + + /* Set IOMMU type */ + ret = ioctl(c->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret < 0) + serror("Failed to set IOMMU type of container"); + + /* Check group viability and features */ + g->status.argsz = sizeof(g->status); + + ret = ioctl(g->fd, VFIO_GROUP_GET_STATUS, &g->status); + if (ret < 0) + serror("Failed to get VFIO group status"); + + if (!(g->status.flags & VFIO_GROUP_FLAGS_VIABLE)) + error("VFIO group is not available: bind all devices to the VFIO driver!"); + + list_push(&c->groups, g); + + return 0; +} + +int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev) +{ + char name[32]; + int ret; + + /* Load PCI bus driver for VFIO */ + if (kernel_module_load("vfio_pci")) + error("Failed to load kernel driver: %s", "vfio_pci"); + + /* Bind PCI card to vfio-pci driver*/ + ret = pci_attach_driver(pdev, "vfio-pci"); + if (ret) + error("Failed to attach device to driver"); + + /* Get IOMMU group of device */ + int index = pci_get_iommu_group(pdev); + if (index < 0) + error("Failed to get IOMMU group of device"); + + /* VFIO device name consists of PCI BDF */ + snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, pdev->slot.bus, pdev->slot.device, pdev->slot.function); + + ret = vfio_device_attach(d, c, name, index); + if (ret < 0) + return ret; + + /* Check if this is really a vfio-pci device */ + if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) { + vfio_device_destroy(d); + return -1; + } + + d->pci_device = pdev; + + return 0; +} + +int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index) +{ + int ret; + struct vfio_group *g = NULL; + + /* Check if group already exists */ + for (size_t i = 0; i < list_length(&c->groups); i++) { + struct vfio_group *h = (struct vfio_group *) list_at(&c->groups, i); + + if (h->index == index) + g = h; + } + + if (!g) { + g = alloc(sizeof(struct vfio_group)); + + /* Aquire group ownership */ + ret = vfio_group_attach(g, c, index); + if (ret) + error("Failed to attach to IOMMU group: %u", index); + + info("Attached new group %u to VFIO container", g->index); + } + + d->group = g; + d->name = strdup(name); + + /* Open device fd */ + d->fd = ioctl(g->fd, VFIO_GROUP_GET_DEVICE_FD, d->name); + if (d->fd < 0) + serror("Failed to open VFIO device: %s", d->name); + + /* Get device info */ + d->info.argsz = sizeof(d->info); + + ret = ioctl(d->fd, VFIO_DEVICE_GET_INFO, &d->info); + if (ret < 0) + serror("Failed to get VFIO device info for: %s", d->name); + + d->irqs = alloc(d->info.num_irqs * sizeof(struct vfio_irq_info)); + d->regions = alloc(d->info.num_regions * sizeof(struct vfio_region_info)); + d->mappings = alloc(d->info.num_regions * sizeof(void *)); + + /* Get device regions */ + for (int i = 0; i < d->info.num_regions && i < 8; i++) { + struct vfio_region_info *region = &d->regions[i]; + + region->argsz = sizeof(*region); + region->index = i; + + ret = ioctl(d->fd, VFIO_DEVICE_GET_REGION_INFO, region); + if (ret < 0) + serror("Failed to get regions of VFIO device: %s", d->name); + } + + /* Get device irqs */ + for (int i = 0; i < d->info.num_irqs; i++) { + struct vfio_irq_info *irq = &d->irqs[i]; + + irq->argsz = sizeof(*irq); + irq->index = i; + + ret = ioctl(d->fd, VFIO_DEVICE_GET_IRQ_INFO, irq); + if (ret < 0) + serror("Failed to get IRQs of VFIO device: %s", d->name); + } + + list_push(&d->group->devices, d); + + return 0; +} + +int vfio_pci_reset(struct vfio_device *d) +{ + int ret; + + /* Check if this is really a vfio-pci device */ + if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return -1; + + size_t reset_infolen = sizeof(struct vfio_pci_hot_reset_info) + sizeof(struct vfio_pci_dependent_device) * 64; + size_t resetlen = sizeof(struct vfio_pci_hot_reset) + sizeof(int32_t) * 1; + + struct vfio_pci_hot_reset_info *reset_info = (struct vfio_pci_hot_reset_info *) alloc(reset_infolen); + struct vfio_pci_hot_reset *reset = (struct vfio_pci_hot_reset *) alloc(resetlen); + + reset_info->argsz = reset_infolen; + reset->argsz = resetlen; + + ret = ioctl(d->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info); + if (ret) + return ret; + + debug(5, "VFIO: dependent devices for hot-reset:"); + for (int i = 0; i < reset_info->count; i++) { + struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; + debug(5, "%04x:%02x:%02x.%01x: iommu_group=%u", dd->segment, dd->bus, PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); + + if (dd->group_id != d->group->index) + return -3; + } + + reset->count = 1; + reset->group_fds[0] = d->group->fd; + + ret = ioctl(d->fd, VFIO_DEVICE_PCI_HOT_RESET, reset); + + free(reset_info); + + return ret; +} + +int vfio_pci_msi_find(struct vfio_device *d, int nos[32]) +{ + int ret, idx, irq; + char *end, *col, *last, line[1024], name[13]; + FILE *f; + + f = fopen("/proc/interrupts", "r"); + if (!f) + return -1; + + for (int i = 0; i < 32; i++) + nos[i] = -1; + + /* For each line in /proc/interruipts */ + while (fgets(line, sizeof(line), f)) { + col = strtok(line, " "); + + /* IRQ number is in first column */ + irq = strtol(col, &end, 10); + if (col == end) + continue; + + /* Find last column of line */ + do { + last = col; + } while ((col = strtok(NULL, " "))); + + + ret = sscanf(last, "vfio-msi[%u](%12[0-9:])", &idx, name); + if (ret == 2) { + if (strstr(d->name, name) == d->name) + nos[idx] = irq; + } + } + + fclose(f); + + return 0; +} + +int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]) +{ + int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + struct vfio_irq_set *irq_set; + + /* Check if this is really a vfio-pci device */ + if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return -1; + + irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; + irq_set = alloc(irq_setlen); + + irq_set->argsz = irq_setlen; + irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; + irq_set->count = irq_count; + irq_set->start = 0; + + for (int i = 0; i < irq_count; i++) { + close(efds[i]); + efds[i] = -1; + } + + memcpy(irq_set->data, efds, sizeof(int) * irq_count); + + ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); + if (ret) + return -4; + + free(irq_set); + + return irq_count; +} + +int vfio_pci_msi_init(struct vfio_device *d, int efds[32]) +{ + int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + struct vfio_irq_set *irq_set; + + /* Check if this is really a vfio-pci device */ + if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return -1; + + irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; + irq_set = alloc(irq_setlen); + + irq_set->argsz = irq_setlen; + irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; + irq_set->start = 0; + irq_set->count = irq_count; + + /* Now set the new eventfds */ + for (int i = 0; i < irq_count; i++) { + efds[i] = eventfd(0, 0); + if (efds[i] < 0) + return -3; + } + memcpy(irq_set->data, efds, sizeof(int) * irq_count); + + ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); + if (ret) + return -4; + + free(irq_set); + + return irq_count; +} + +int vfio_pci_enable(struct vfio_device *d) +{ + int ret; + uint32_t reg; + off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; + + /* Check if this is really a vfio-pci device */ + if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return -1; + + ret = pread(d->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return -1; + + /* Enable memory access and PCI bus mastering which is required for DMA */ + reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + + ret = pwrite(d->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return -1; + + return 0; +} + +int vfio_device_reset(struct vfio_device *d) +{ + if (d->info.flags & VFIO_DEVICE_FLAGS_RESET) + return ioctl(d->fd, VFIO_DEVICE_RESET); + else + return -1; /* not supported by this device */ +} + +void vfio_dump(struct vfio_container *v) +{ + info("VFIO Version: %u", v->version); + info("VFIO Extensions: %#x", v->extensions); + + for (size_t i = 0; i < list_length(&v->groups); i++) { + struct vfio_group *g = (struct vfio_group *) list_at(&v->groups, i); + + info("VFIO Group %u, viable=%u, container=%d", g->index, + (g->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0, + (g->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0 + ); + + + for (size_t i = 0; i < list_length(&g->devices); i++) { + struct vfio_device *d = (struct vfio_device *) list_at(&g->devices, i); + + info("Device %s: regions=%u, irqs=%u, flags=%#x", d->name, + d->info.num_regions, + d->info.num_irqs, + d->info.flags + ); + + for (int i = 0; i < d->info.num_regions && i < 8; i++) { + struct vfio_region_info *region = &d->regions[i]; + + if (region->size > 0) + info("Region %u %s: size=%#llx, offset=%#llx, flags=%u", + region->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI) ? vfio_pci_region_names[i] : "", + region->size, + region->offset, + region->flags + ); + } + + for (int i = 0; i < d->info.num_irqs; i++) { + struct vfio_irq_info *irq = &d->irqs[i]; + + if (irq->count > 0) + info("IRQ %u %s: count=%u, flags=%u", + irq->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI ) ? vfio_pci_irq_names[i] : "", + irq->count, + irq->flags + ); + } + } + } +} + +void * vfio_map_region(struct vfio_device *d, int idx) +{ + struct vfio_region_info *r = &d->regions[idx]; + + if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP)) + return MAP_FAILED; + + d->mappings[idx] = mmap(NULL, r->size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_32BIT, d->fd, r->offset); + + return d->mappings[idx]; +} + +int vfio_unmap_region(struct vfio_device *d, int idx) +{ + int ret; + struct vfio_region_info *r = &d->regions[idx]; + + if (!d->mappings[idx]) + return -1; /* was not mapped */ + + debug(3, "VFIO: unmap region %u from device", idx); + + ret = munmap(d->mappings[idx], r->size); + if (ret) + return -2; + + d->mappings[idx] = NULL; + + return 0; +} + +int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) +{ + int ret; + + if (len & 0xFFF) { + len += 0x1000; + len &= ~0xFFF; + } + + /* Super stupid allocator */ + if (phys == -1) { + phys = c->iova_next; + c->iova_next += len; + } + + struct vfio_iommu_type1_dma_map dma_map = { + .argsz = sizeof(struct vfio_iommu_type1_dma_map), + .vaddr = virt, + .iova = phys, + .size = len, + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE + }; + + ret = ioctl(c->fd, VFIO_IOMMU_MAP_DMA, &dma_map); + if (ret) + serror("Failed to create DMA mapping"); + + info("DMA map size=%#llx, iova=%#llx, vaddr=%#llx", dma_map.size, dma_map.iova, dma_map.vaddr); + + return 0; +} + +int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) +{ + int ret; + + struct vfio_iommu_type1_dma_unmap dma_unmap = { + .argsz = sizeof(struct vfio_iommu_type1_dma_unmap), + .flags = 0, + .iova = phys, + .size = len, + }; + + ret = ioctl(c->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); + if (ret) + serror("Failed to unmap DMA mapping"); + + return 0; +} diff --git a/common/lib/list.c b/common/lib/list.c index 10d688e9c..57897ad53 100644 --- a/common/lib/list.c +++ b/common/lib/list.c @@ -56,7 +56,6 @@ int list_init(struct list *l) l->length = 0; l->capacity = 0; l->array = NULL; - l->state = STATE_INITIALIZED; return 0; @@ -69,26 +68,24 @@ int list_destroy(struct list *l, dtor_cb_t destructor, bool release) assert(l->state != STATE_DESTROYED); for (size_t i = 0; i < list_length(l); i++) { - void *p = list_at(l, i); + void *e = list_at(l, i); if (destructor) - destructor(p); + destructor(e); if (release) - free(p); + free(e); } free(l->array); - l->array = NULL; - l->length = -1; l->capacity = 0; + l->array = NULL; + l->state = STATE_DESTROYED; pthread_mutex_unlock(&l->lock); pthread_mutex_destroy(&l->lock); - l->state = STATE_DESTROYED; - return 0; } @@ -119,10 +116,10 @@ void list_remove(struct list *l, void *p) assert(l->state == STATE_INITIALIZED); for (size_t i = 0; i < list_length(l); i++) { - if (l->array[i] == p) + if (list_at(l, i) == p) removed++; else - l->array[i - removed] = l->array[i]; + l->array[i - removed] = list_at(l, i); } l->length -= removed; @@ -135,6 +132,15 @@ void * list_lookup(struct list *l, const char *name) return list_search(l, cmp_lookup, (void *) name); } +ssize_t list_lookup_index(struct list *l, const char *name) +{ + void *ptr = list_lookup(l, name); + if (!ptr) + return -1; + + return list_index(l, ptr); +} + int list_contains(struct list *l, void *p) { return list_count(l, cmp_contains, p); @@ -143,14 +149,14 @@ int list_contains(struct list *l, void *p) int list_count(struct list *l, cmp_cb_t cmp, void *ctx) { int c = 0; + void *e; pthread_mutex_lock(&l->lock); assert(l->state == STATE_INITIALIZED); for (size_t i = 0; i < list_length(l); i++) { - void *e = list_at(l, i); - + e = list_at(l, i); if (cmp(e, ctx) == 0) c++; } @@ -201,3 +207,33 @@ int list_set(struct list *l, int index, void *value) return 0; } + +ssize_t list_index(struct list *l, void *p) +{ + void *e; + ssize_t f; + + pthread_mutex_lock(&l->lock); + + assert(l->state == STATE_INITIALIZED); + + for (size_t i = 0; i < list_length(l); i++) { + e = list_at(l, i); + if (e == p) { + f = i; + goto found; + } + } + + f = -1; + +found: pthread_mutex_unlock(&l->lock); + + return f; +} + +void list_extend(struct list *l, size_t len, void *val) +{ + while (list_length(l) < len) + list_push(l, val); +} diff --git a/common/lib/log.c b/common/lib/log.c index 679cf147b..ad3d2c694 100644 --- a/common/lib/log.c +++ b/common/lib/log.c @@ -24,12 +24,16 @@ #include #include #include +#include #include #include +#include #include #include #include +#include +#include #ifdef ENABLE_OPAL_ASYNC /* Define RTLAB before including OpalPrint.h for messages to be sent @@ -39,7 +43,6 @@ #endif struct log *global_log; -struct log default_log; /* We register a default log instance */ __attribute__((constructor)) @@ -52,7 +55,7 @@ void register_default_log() if (ret) error("Failed to initalize log"); - ret = log_start(&default_log); + ret = log_open(&default_log); if (ret) error("Failed to start log"); } @@ -75,6 +78,7 @@ static const char *facilities_strs[] = { "tc", /* LOG_TC */ "if", /* LOG_IF */ "advio", /* LOG_ADVIO */ + "io", /* LOG_IO */ /* Node-types */ "socket", /* LOG_SOCKET */ @@ -83,32 +87,10 @@ static const char *facilities_strs[] = { "ngsi", /* LOG_NGSI */ "websocket", /* LOG_WEBSOCKET */ "opal", /* LOG_OPAL */ + "comedi", /* LOG_COMEDI */ + "ib", /* LOG_IB */ }; -#ifdef __GNUC__ -/** The current log indention level (per thread!). */ -static __thread int indent = 0; - -int log_indent(int levels) -{ - int old = indent; - indent += levels; - return old; -} - -int log_noindent() -{ - int old = indent; - indent = 0; - return old; -} - -void log_outdent(int *old) -{ - indent = *old; -} -#endif - static void log_resize(int signal, siginfo_t *sinfo, void *ctx) { int ret; @@ -137,6 +119,7 @@ int log_init(struct log *l, int level, long facilitites) l->file = stderr; l->path = NULL; + l->epoch = time_now(); l->prefix = getenv("VILLAS_LOG_PREFIX"); /* Register signal handler which is called whenever the @@ -154,7 +137,13 @@ int log_init(struct log *l, int level, long facilitites) return ret; /* Try to get initial window size */ - ioctl(STDERR_FILENO, TIOCGWINSZ, &global_log->window); + ioctl(STDERR_FILENO, TIOCGWINSZ, &l->window); + + /* Fallback if for some reason we can not determine a prober window size */ + if (l->window.ws_col == 0) + l->window.ws_col = 150; + if (l->window.ws_row == 0) + l->window.ws_row = 50; } else { l->window.ws_col = LOG_WIDTH; @@ -170,7 +159,7 @@ int log_init(struct log *l, int level, long facilitites) return 0; } -int log_start(struct log *l) +int log_open(struct log *l) { if (l->path) { l->file = fopen(l->path, "a+");; @@ -182,20 +171,22 @@ int log_start(struct log *l) else l->file = stderr; - l->state = STATE_STARTED; + l->tty = isatty(fileno(l->file)); if (l->syslog) { openlog(NULL, LOG_PID, LOG_DAEMON); } + l->state = STATE_OPENED; + debug(LOG_LOG | 5, "Log sub-system started: level=%d, faciltities=%#lx, path=%s", l->level, l->facilities, l->path); return 0; } -int log_stop(struct log *l) +int log_close(struct log *l) { - if (l->state != STATE_STARTED) + if (l->state != STATE_OPENED) return 0; if (l->file != stderr && l->file != stdout) { @@ -206,7 +197,7 @@ int log_stop(struct log *l) closelog(); } - l->state = STATE_STOPPED; + l->state = STATE_CLOSED; return 0; } @@ -283,28 +274,37 @@ void log_print(struct log *l, const char *lvl, const char *fmt, ...) void log_vprint(struct log *l, const char *lvl, const char *fmt, va_list ap) { - char *buf = alloc(512); + struct timespec ts = time_now(); + static __thread char buf[1024]; + + int off = 0; + int len = sizeof(buf); /* Optional prefix */ if (l->prefix) - strcatf(&buf, "%s", l->prefix); + off += snprintf(buf + off, len - off, "%s", l->prefix); - /* Indention */ -#ifdef __GNUC__ - for (int i = 0; i < indent; i++) - strcatf(&buf, "%s ", BOX_UD); - - strcatf(&buf, "%s ", BOX_UDR); -#endif + /* Timestamp & Severity */ + off += snprintf(buf + off, len - off, "%10.3f %-5s ", time_delta(&l->epoch, &ts), lvl); /* Format String */ - vstrcatf(&buf, fmt, ap); + off += vsnprintf(buf + off, len - off, fmt, ap); /* Output */ #ifdef ENABLE_OPAL_ASYNC - OpalPrint("VILLASfpga: %s\n", buf); + OpalPrint("VILLASnode: %s\n", buf); #endif - fprintf(l->file ? l->file : stderr, "%s\n", buf); + if (l->file) { + if (l->tty == false) + decolor(buf); - free(buf); + fprintf(l->file, "%s\n", buf); + } + + if (l->syslog) { + if (l->tty == true) // Only decolor if not done before + decolor(buf); + + vsyslog(LOG_INFO, fmt, ap); + } } diff --git a/common/lib/log_config.c b/common/lib/log_config.c index 167e74d19..ead230296 100644 --- a/common/lib/log_config.c +++ b/common/lib/log_config.c @@ -4,7 +4,7 @@ * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * - * VILLAScommon + * VILLASconfig * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -31,6 +31,26 @@ #include #include +int log_parse_wrapper(struct log *l, json_t *cfg) +{ + int ret; + json_t *json_logging = NULL; + json_error_t err; + + if (cfg) { + ret = json_unpack_ex(cfg, &err, 0, "{s?: o}", + "logging", &json_logging + ); + if (ret) + jerror(&err, "Failed to parse logging from global configuration"); + + if (json_logging) + log_parse(l, json_logging); + } + + return 0; +} + int log_parse(struct log *l, json_t *cfg) { const char *facilities = NULL; @@ -71,12 +91,7 @@ void jerror(json_error_t *err, const char *fmt, ...) va_end(ap); log_print(l, LOG_LVL_ERROR, "%s:", buf); - { INDENT - log_print(l, LOG_LVL_ERROR, "%s in %s:%d:%d", err->text, err->source, err->line, err->column); - - if (l->syslog) - syslog(LOG_ERR, "%s in %s:%d:%d", err->text, err->source, err->line, err->column); - } + log_print(l, LOG_LVL_ERROR, " %s in %s:%d:%d", err->text, err->source, err->line, err->column); free(buf); diff --git a/common/lib/table.c b/common/lib/table.c new file mode 100644 index 000000000..f9b65c036 --- /dev/null +++ b/common/lib/table.c @@ -0,0 +1,178 @@ +/** Print fancy tables. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include +#include +#include + +static int table_resize(struct table *t, int width) +{ + int norm, flex, fixed, total; + + t->width = width; + + norm = 0; + flex = 0; + fixed = 0; + total = t->width - t->ncols * 2; + + /* Normalize width */ + for (int i = 0; i < t->ncols; i++) { + if (t->cols[i].width > 0) + norm += t->cols[i].width; + if (t->cols[i].width == 0) + flex++; + if (t->cols[i].width < 0) + fixed += -1 * t->cols[i].width; + } + + for (int i = 0; i < t->ncols; i++) { + if (t->cols[i].width > 0) + t->cols[i]._width = t->cols[i].width * (float) (total - fixed) / norm; + if (t->cols[i].width == 0) + t->cols[i]._width = (float) (total - fixed) / flex; + if (t->cols[i].width < 0) + t->cols[i]._width = -1 * t->cols[i].width; + } + + return 0; +} + +void table_header(struct table *t) +{ + struct log *l = global_log ? global_log : &default_log; + + if (t->width != l->width) + table_resize(t, l->width); + + char *line0 = strf("\b"); + char *line1 = strf("\b\b" BOX_UD); + char *line2 = strf("\b\b" BOX_UD); + char *line3 = strf("\b"); + + for (int i = 0; i < t->ncols; i++) { + int w, u; + char *col, *unit; + + col = strf(CLR_BLD("%s"), t->cols[i].title); + unit = t->cols[i].unit ? strf(CLR_YEL("%s"), t->cols[i].unit) : ""; + + w = t->cols[i]._width + strlen(col) - strlenp(col); + u = t->cols[i]._width + strlen(unit) - strlenp(unit); + + if (t->cols[i].align == TABLE_ALIGN_LEFT) { + strcatf(&line1, " %-*.*s\e[0m " BOX_UD, w, w, col); + strcatf(&line2, " %-*.*s\e[0m " BOX_UD, u, u, unit); + } + else { + strcatf(&line1, " %*.*s\e[0m " BOX_UD, w, w, col); + strcatf(&line2, " %*.*s\e[0m " BOX_UD, u, u, unit); + } + + for (int j = 0; j < t->cols[i]._width + 2; j++) { + strcatf(&line0, "%s", BOX_LR); + strcatf(&line3, "%s", BOX_LR); + } + + if (i == t->ncols - 1) { + strcatf(&line0, "%s", BOX_DL); + strcatf(&line3, "%s", BOX_UDL); + } + else { + strcatf(&line0, "%s", BOX_DLR); + strcatf(&line3, "%s", BOX_UDLR); + } + + free(col); + } + + stats("%s", line0); + stats("%s", line1); + stats("%s", line2); + stats("%s", line3); + + free(line0); + free(line1); + free(line2); + free(line3); +} + +void table_row(struct table *t, ...) +{ + struct log *l = global_log ? global_log : &default_log; + + if (t->width != l->width) { + table_resize(t, l->width); + table_header(t); + } + + va_list args; + va_start(args, t); + + char *line = strf("\b\b" BOX_UD); + + for (int i = 0; i < t->ncols; ++i) { + char *col = vstrf(t->cols[i].format, args); + + int l = strlenp(col); + int r = strlen(col); + int w = t->cols[i]._width + r - l; + + if (t->cols[i].align == TABLE_ALIGN_LEFT) + strcatf(&line, " %-*.*s\e[0m " BOX_UD, w, w, col); + else + strcatf(&line, " %*.*s\e[0m " BOX_UD, w, w, col); + + free(col); + } + + va_end(args); + + stats("%s", line); + free(line); +} + +void table_footer(struct table *t) +{ + struct log *l = global_log ? global_log : &default_log; + + if (t->width != l->width) + table_resize(t, l->width); + + char *line = strf("\b"); + + for (int i = 0; i < t->ncols; i++) { + for (int j = 0; j < t->cols[i]._width + 2; j++) + strcatf(&line, BOX_LR); + + if (i == t->ncols - 1) + strcatf(&line, "%s", BOX_UL); + else + strcatf(&line, "%s", BOX_ULR); + } + + stats("%s", line); + free(line); +} diff --git a/common/lib/task.c b/common/lib/task.c new file mode 100644 index 000000000..af70cb2b1 --- /dev/null +++ b/common/lib/task.c @@ -0,0 +1,212 @@ +/** Run tasks periodically. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include +#include +#include + +#if PERIODIC_TASK_IMPL == TIMERFD + #include +#elif PERIODIC_TASK_IMPL == RDTSC + #include +#endif + +int task_init(struct task *t, double rate, int clock) +{ + int ret; + + t->clock = clock; + +#if PERIODIC_TASK_IMPL == TIMERFD + t->fd = timerfd_create(t->clock, 0); + if (t->fd < 0) + return -1; +#elif PERIODIC_TASK_IMPL == RDTSC + ret = tsc_init(&t->tsc); + if (ret) + return ret; +#endif + + ret = task_set_rate(t, rate); + if (ret) + return ret; + + return 0; +} + +int task_set_timeout(struct task *t, double to) +{ + struct timespec now; + + clock_gettime(t->clock, &now); + + struct timespec timeout = time_from_double(to); + struct timespec next = time_add(&now, &timeout); + + return task_set_next(t, &next); +} + +int task_set_next(struct task *t, struct timespec *next) +{ + +#if PERIODIC_TASK_IMPL == RDTSC +#else + t->next = *next; + + #if PERIODIC_TASK_IMPL == TIMERFD + int ret; + struct itimerspec its = { + .it_interval = (struct timespec) { 0, 0 }, + .it_value = t->next + }; + + ret = timerfd_settime(t->fd, TFD_TIMER_ABSTIME, &its, NULL); + if (ret) + return ret; + #endif +#endif + + return 0; +} + +int task_set_rate(struct task *t, double rate) +{ + +#if PERIODIC_TASK_IMPL == RDTSC + t->period = tsc_rate_to_cycles(&t->tsc, rate); + t->next = tsc_now(&t->tsc) + t->period; +#else + /* A rate of 0 will disarm the timer */ + t->period = rate ? time_from_double(1.0 / rate) : (struct timespec) { 0, 0 }; + + #if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP + struct timespec now, next; + + clock_gettime(t->clock, &now); + + next = time_add(&now, &t->period); + + return task_set_next(t, &next); + #elif PERIODIC_TASK_IMPL == TIMERFD + int ret; + struct itimerspec its = { + .it_interval = t->period, + .it_value = t->period + }; + + ret = timerfd_settime(t->fd, 0, &its, NULL); + if (ret) + return ret; + #endif +#endif + + return 0; +} + +int task_destroy(struct task *t) +{ +#if PERIODIC_TASK_IMPL == TIMERFD + return close(t->fd); +#endif + + return 0; +} + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP +static int time_lt(const struct timespec *lhs, const struct timespec *rhs) +{ + if (lhs->tv_sec == rhs->tv_sec) + return lhs->tv_nsec < rhs->tv_nsec; + else + return lhs->tv_sec < rhs->tv_sec; + + return 0; +} +#endif + +uint64_t task_wait(struct task *t) +{ + uint64_t runs; + +#if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP || PERIODIC_TASK_IMPL == NANOSLEEP + int ret; + struct timespec now; + + #if PERIODIC_TASK_IMPL == CLOCK_NANOSLEEP + do { + ret = clock_nanosleep(t->clock, TIMER_ABSTIME, &t->next, NULL); + } while (ret == EINTR); + #elif PERIODIC_TASK_IMPL == NANOSLEEP + struct timespec delta; + + ret = clock_gettime(t->clock, &now); + if (ret) + return ret; + + delta = time_diff(&now, &t->next); + + ret = nanosleep(&delta, NULL); + #endif + if (ret < 0) + return 0; + + ret = clock_gettime(t->clock, &now); + if (ret) + return 0; + + for (runs = 0; time_lt(&t->next, &now); runs++) + t->next = time_add(&t->next, &t->period); +#elif PERIODIC_TASK_IMPL == TIMERFD + int ret; + + ret = read(t->fd, &runs, sizeof(runs)); + if (ret < 0) + return 0; +#elif PERIODIC_TASK_IMPL == RDTSC + uint64_t now; + + do { + now = rdtscp(); + } while (now < t->next); + + + for (runs = 0; t->next < now; runs++) + t->next += t->period; +#else + #error "Invalid period task implementation" +#endif + + return runs; +} + +int task_fd(struct task *t) +{ +#if PERIODIC_TASK_IMPL == TIMERFD + return t->fd; +#else + return -1; +#endif +} diff --git a/common/lib/timing.c b/common/lib/timing.c new file mode 100644 index 000000000..9324c895a --- /dev/null +++ b/common/lib/timing.c @@ -0,0 +1,86 @@ +/** Time related functions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include + +struct timespec time_now() +{ + struct timespec ts; + + clock_gettime(CLOCK_REALTIME, &ts); + + return ts; +} + +struct timespec time_add(const struct timespec *start, const struct timespec *end) +{ + struct timespec sum = { + .tv_sec = end->tv_sec + start->tv_sec, + .tv_nsec = end->tv_nsec + start->tv_nsec + }; + + if (sum.tv_nsec >= 1000000000) { + sum.tv_sec += 1; + sum.tv_nsec -= 1000000000; + } + + return sum; +} + +struct timespec time_diff(const struct timespec *start, const struct timespec *end) +{ + struct timespec diff = { + .tv_sec = end->tv_sec - start->tv_sec, + .tv_nsec = end->tv_nsec - start->tv_nsec + }; + + if (diff.tv_nsec < 0) { + diff.tv_sec -= 1; + diff.tv_nsec += 1000000000; + } + + return diff; +} + +struct timespec time_from_double(double secs) +{ + struct timespec ts; + + ts.tv_sec = secs; + ts.tv_nsec = 1.0e9 * (secs - ts.tv_sec); + + return ts; +} + +double time_to_double(const struct timespec *ts) +{ + return ts->tv_sec + ts->tv_nsec * 1e-9; +} + +double time_delta(const struct timespec *start, const struct timespec *end) +{ + struct timespec diff = time_diff(start, end); + + return time_to_double(&diff); +} diff --git a/common/lib/tsc.c b/common/lib/tsc.c new file mode 100644 index 000000000..9244c1013 --- /dev/null +++ b/common/lib/tsc.c @@ -0,0 +1,84 @@ +/** Measure time and sleep with IA-32 time-stamp counter. + * + * @file + * @author Steffen Vogel + * @copyright 2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +int tsc_init(struct tsc *t) +{ + uint32_t eax, ebx, ecx, edx; + + /** Check if TSC is supported */ + __get_cpuid(0x1, &eax, &ebx, &ecx, &edx); + if (!(edx & bit_TSC)) + return -2; + + /** Check if RDTSCP instruction is supported */ + __get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx); + t->rdtscp_supported = edx & bit_RDTSCP; + + /** Check if TSC is invariant */ + __get_cpuid(0x80000007, &eax, &ebx, &ecx, &edx); + t->is_invariant = edx & bit_TSC_INVARIANT; + + /** Intel SDM Vol 3, Section 18.7.3: + * Nominal TSC frequency = CPUID.15H.ECX[31:0] * CPUID.15H.EBX[31:0] ) รท CPUID.15H.EAX[31:0] + */ + __get_cpuid(0x15, &eax, &ebx, &ecx, &edx); + + if (ecx != 0) + t->frequency = ecx * ebx / eax; + else { + int ret; +#ifdef __linux__ + ret = kernel_get_cpu_frequency(&t->frequency); + if (ret) + return ret; +#elif defined(__APPLE__) + int64_t frequency; + size_t lenp = sizeof(frequency); + + /** @todo: machdep.tsc.frequency seems to be a measured frequency (based on local APIC? + * We should figure out which frequency is more accurate */ +// ret = sysctlbyname("hw.cpufrequency", &frequency, &lenp, NULL, 0); + ret = sysctlbyname("machdep.tsc.frequency", &frequency, &lenp, NULL, 0); + if (ret) + return ret; + + t->frequency = frequency; +#endif + } + + return 0; +} + +uint64_t tsc_rate_to_cyles(struct tsc *t, double rate) +{ + return t->frequency / rate; +} + +uint64_t tsc_now(struct tsc *t) +{ + return t->rdtscp_supported + ? rdtscp() + : rdtsc(); +} diff --git a/common/lib/utils.c b/common/lib/utils.c index 08a1531d3..9ce0dff03 100644 --- a/common/lib/utils.c +++ b/common/lib/utils.c @@ -260,14 +260,14 @@ void * memdup(const void *src, size_t bytes) return dst; } -size_t read_random(char *buf, size_t len) +ssize_t read_random(char *buf, size_t len) { int fd; ssize_t bytes, total; fd = open("/dev/urandom", O_RDONLY); if (fd < 0) - return 0; + return -1; bytes = 0; total = 0; @@ -284,21 +284,6 @@ size_t read_random(char *buf, size_t len) return bytes; } -void rdtsc_sleep(uint64_t nanosecs, uint64_t start) -{ - uint64_t cycles; - - /** @todo Replace the hard coded CPU clock frequency */ - cycles = (double) nanosecs / (1e9 / 3392389000); - - if (start == 0) - start = rdtsc(); - - do { - __asm__("nop"); - } while (rdtsc() - start < cycles); -} - /* Setup exit handler */ int signals_init(void (*cb)(int signal, siginfo_t *sinfo, void *ctx)) { @@ -399,3 +384,36 @@ size_t strlenp(const char *str) return sz; } + +char * decolor(char *str) +{ + char *p, *q; + bool inseq = false; + + for (p = q = str; *p; p++) { + switch (*p) { + case 0x1b: + if (*(++p) == '[') { + inseq = true; + continue; + } + break; + + case 'm': + if (inseq) { + inseq = false; + continue; + } + break; + } + + if (!inseq) { + *q = *p; + q++; + } + } + + *q = '\0'; + + return str; +} diff --git a/common/tests/CMakeLists.txt b/common/tests/CMakeLists.txt index a9c724b06..b78165cf0 100644 --- a/common/tests/CMakeLists.txt +++ b/common/tests/CMakeLists.txt @@ -23,7 +23,20 @@ set(SOURCES main.cpp logging.cpp + graph.cpp + + advio.c + bitset.c + hash_table.c + hist.c + kernel.c + list.c + log.c + task.c + timing.c + tsc.c + utils.c ) add_executable(unit-tests-common ${SOURCES}) diff --git a/common/tests/advio.c b/common/tests/advio.c new file mode 100644 index 000000000..15947c76d --- /dev/null +++ b/common/tests/advio.c @@ -0,0 +1,255 @@ +/** Unit tests for advio + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +#include +#include + +/** This URI points to a Sciebo share which contains some test files. + * The Sciebo share is read/write accessible via WebDAV. */ +#define BASE_URI "https://1Nrd46fZX8HbggT:badpass@rwth-aachen.sciebo.de/public.php/webdav/node/tests" + +Test(advio, islocal) +{ + int ret; + + ret = aislocal("/var/log/villas/dta.dat"); + cr_assert_eq(ret, 1); + + ret = aislocal("http://www.google.de"); + cr_assert_eq(ret, 0); + + ret = aislocal("torrent://www.google.de"); + cr_assert_eq(ret, -1); +} + +Test(advio, local) +{ + AFILE *af; + int ret; + char *buf = NULL; + size_t buflen = 0; + + /* We open this file and check the first line */ + af = afopen(__FILE__, "r"); + cr_assert(af, "Failed to open local file"); + + ret = getline(&buf, &buflen, af->file); + cr_assert_gt(ret, 1); + cr_assert_str_eq(buf, "/** Unit tests for advio\n"); +} + +Test(advio, download) +{ + AFILE *af; + int ret; + size_t len; + char buffer[64]; + char expect[64] = "ook4iekohC2Teegoghu6ayoo1OThooregheebaet8Zod1angah0che7quai4ID7A"; + + af = afopen(BASE_URI "/download" , "r"); + cr_assert(af, "Failed to download file"); + + len = afread(buffer, 1, sizeof(buffer), af); + cr_assert_gt(len, 0, "len=%zu, feof=%u", len, afeof(af)); + + cr_assert_arr_eq(buffer, expect, sizeof(expect)); + + ret = afclose(af); + cr_assert_eq(ret, 0, "Failed to close file"); +} + +Test(advio, download_large) +{ + AFILE *af; + int ret; + + af = afopen(BASE_URI "/download-large" , "r"); + cr_assert(af, "Failed to download file"); + + char line[4096]; + + char *f; + + f = afgets(line, 4096, af); + cr_assert_not_null(f); + + /* Check first line */ + cr_assert_str_eq(line, "# VILLASnode signal params: type=mixed, values=4, rate=1000.000000, limit=100000, amplitude=1.000000, freq=1.000000\n"); + + while(afgets(line, 4096, af)); + + /* Check last line */ + cr_assert_str_eq(line, "1497710478.862332239(99999) 0.752074 -0.006283 1.000000 0.996000\n"); + + ret = afclose(af); + cr_assert_eq(ret, 0, "Failed to close file"); +} + +Test(advio, resume) +{ + int ret; + char *retp; + AFILE *af1, *af2; + char *fn, dir[] = "/tmp/temp.XXXXXX"; + char line1[32]; + char *line2 = NULL; + size_t linelen = 0; + + retp = mkdtemp(dir); + cr_assert_not_null(retp); + + ret = asprintf(&fn, "%s/file", dir); + cr_assert_gt(ret, 0); + + af1 = afopen(fn, "w+"); + cr_assert_not_null(af1); + + /* We flush once the empty file in order to upload an empty file. */ + aupload(af1, 0); + + af2 = afopen(fn, "r"); + cr_assert_not_null(af2); + + for (int i = 0; i < 100; i++) { + snprintf(line1, sizeof(line1), "This is line %d\n", i); + + afputs(line1, af1); + aupload(af1, 1); + + adownload(af2, 1); + + ret = agetline(&line2, &linelen, af2); + cr_assert_gt(ret, 0); + + cr_assert_str_eq(line1, line2); + } + + ret = afclose(af1); + cr_assert_eq(ret, 0); + + ret = afclose(af2); + cr_assert_eq(ret, 0); + + ret = unlink(fn); + cr_assert_eq(ret, 0); + + ret = rmdir(dir); + cr_assert_eq(ret, 0); + + free(line2); +} + +Test(advio, upload) +{ + AFILE *af; + int ret; + size_t len; + + char upload[64]; + char buffer[64]; + + /* Get some random data to upload */ + len = read_random(upload, sizeof(upload)); + cr_assert_eq(len, sizeof(upload)); + + /* Open file for writing */ + af = afopen(BASE_URI "/upload", "w+"); + cr_assert(af, "Failed to download file"); + + len = afwrite(upload, 1, sizeof(upload), af); + cr_assert_eq(len, sizeof(upload)); + + ret = afclose(af); + cr_assert_eq(ret, 0, "Failed to close/upload file"); + + /* Open for reading and comparison */ + af = afopen(BASE_URI "/upload", "r"); + cr_assert(af, "Failed to download file"); + + len = afread(buffer, 1, sizeof(upload), af); + cr_assert_eq(len, sizeof(upload)); + + cr_assert_arr_eq(buffer, upload, len); + + ret = afclose(af); + cr_assert(ret == 0, "Failed to close file"); +} + +Test(advio, append) +{ + AFILE *af; + int ret; + size_t len; + + char append1[64] = "xa5gieTohlei9iu1uVaePae6Iboh3eeheeme5iejue5sheshae4uzisha9Faesei"; + char append2[64] = "bitheeRae7igee2miepahJaefoGad1Ooxeif0Mooch4eojoumueYahn4ohc9poo2"; + char expect[128] = "xa5gieTohlei9iu1uVaePae6Iboh3eeheeme5iejue5sheshae4uzisha9FaeseibitheeRae7igee2miepahJaefoGad1Ooxeif0Mooch4eojoumueYahn4ohc9poo2"; + char buffer[128]; + + /* Open file for writing first chunk */ + af = afopen(BASE_URI "/append", "w+"); + cr_assert(af, "Failed to download file"); + + /* The append file might already exist and be not empty from a previous run. */ + ret = ftruncate(afileno(af), 0); + cr_assert_eq(ret, 0); + + char c; + fseek(af->file, 0, SEEK_SET); + if (af->file) { + while ((c = getc(af->file)) != EOF) + putchar(c); + } + + len = afwrite(append1, 1, sizeof(append1), af); + cr_assert_eq(len, sizeof(append1)); + + ret = afclose(af); + cr_assert_eq(ret, 0, "Failed to close/upload file"); + + /* Open file for writing second chunk */ + af = afopen(BASE_URI "/append", "a"); + cr_assert(af, "Failed to download file"); + + len = afwrite(append2, 1, sizeof(append2), af); + cr_assert_eq(len, sizeof(append2)); + + ret = afclose(af); + cr_assert_eq(ret, 0, "Failed to close/upload file"); + + /* Open for reading and comparison */ + af = afopen(BASE_URI "/append", "r"); + cr_assert(af, "Failed to download file"); + + len = afread(buffer, 1, sizeof(buffer), af); + cr_assert_eq(len, sizeof(buffer)); + + ret = afclose(af); + cr_assert(ret == 0, "Failed to close file"); + + cr_assert_arr_eq(buffer, expect, sizeof(expect)); +} diff --git a/common/tests/bitset.c b/common/tests/bitset.c new file mode 100644 index 000000000..41c6abd8e --- /dev/null +++ b/common/tests/bitset.c @@ -0,0 +1,130 @@ +/** Unit tests for advio + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +#define LEN 1027 + +Test(bitset, simple) +{ + int ret; + struct bitset bs; + + int bits[] = { 23, 223, 25, 111, 252, 86, 222, 454, LEN-1 }; + + ret = bitset_init(&bs, LEN); + cr_assert_eq(ret, 0); + + for (int i = 0; i < ARRAY_LEN(bits); i++) { + bitset_set(&bs, bits[i]); + cr_assert_eq(ret, 0); + } + + for (int i = 0; i < ARRAY_LEN(bits); i++) { + ret = bitset_test(&bs, bits[i]); + cr_assert_eq(ret, 1, "Failed at bit %d", i); + } + + for (int i = 0; i < ARRAY_LEN(bits); i++) { + ret = bitset_clear(&bs, bits[i]); + cr_assert_eq(ret, 0, "Failed at bit %d", i); + } + + for (int i = 0; i < LEN; i++) { + ret = bitset_test(&bs, i); + cr_assert_eq(ret, 0); + } + + ret = bitset_destroy(&bs); + cr_assert_eq(ret, 0); +} + +Test(bitset, outofbounds) +{ + int ret; + struct bitset bs; + + ret = bitset_init(&bs, LEN); + cr_assert_eq(ret, 0); + + ret = bitset_set(&bs, LEN+1); + cr_assert_eq(ret, -1); + + ret = bitset_test(&bs, LEN+1); + cr_assert_eq(ret, -1); + + ret = bitset_destroy(&bs); + cr_assert_eq(ret, 0); +} + +Test(bitset, cmp) +{ + int ret; + struct bitset bs1, bs2; + + ret = bitset_init(&bs1, LEN); + cr_assert_eq(ret, 0); + + ret = bitset_init(&bs2, LEN); + cr_assert_eq(ret, 0); + + ret = bitset_set(&bs1, 525); + cr_assert_eq(ret, 0); + + ret = bitset_set(&bs2, 525); + cr_assert_eq(ret, 0); + + ret = bitset_cmp(&bs1, &bs2); + cr_assert_eq(ret, 0); + + ret = bitset_clear(&bs2, 525); + cr_assert_eq(ret, 0); + + ret = bitset_cmp(&bs1, &bs2); + cr_assert_neq(ret, 0); + + ret = bitset_destroy(&bs1); + cr_assert_eq(ret, 0); + + ret = bitset_destroy(&bs2); + cr_assert_eq(ret, 0); +} + +Test(bitset, all) +{ + int ret; + struct bitset bs; + + ret = bitset_init(&bs, LEN); + cr_assert_eq(ret, 0); + + for (int i = 0; i < LEN; i++) { + bitset_test(&bs, i); + cr_assert_eq(ret, 0); + } + + ret = bitset_destroy(&bs); + cr_assert_eq(ret, 0); +} diff --git a/common/tests/hash_table.c b/common/tests/hash_table.c new file mode 100644 index 000000000..6e511e4f9 --- /dev/null +++ b/common/tests/hash_table.c @@ -0,0 +1,71 @@ +/** Unit tests for hash table + * + * @author Steffen Vogel + * @copyright 2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +static char *keys[] = { "able", "achieve", "acoustics", "action", "activity", "aftermath", "afternoon", "afterthought", "apparel", "appliance", "beginner", "believe", "bomb", "border", "boundary", "breakfast", "cabbage", "cable", "calculator", "calendar", "caption", "carpenter", "cemetery", "channel", "circle", "creator", "creature", "education", "faucet", "feather", "friction", "fruit", "fuel", "galley", "guide", "guitar", "health", "heart", "idea", "kitten", "laborer", "language" }; +static char *values[] = { "lawyer", "linen", "locket", "lumber", "magic", "minister", "mitten", "money", "mountain", "music", "partner", "passenger", "pickle", "picture", "plantation", "plastic", "pleasure", "pocket", "police", "pollution", "railway", "recess", "reward", "route", "scene", "scent", "squirrel", "stranger", "suit", "sweater", "temper", "territory", "texture", "thread", "treatment", "veil", "vein", "volcano", "wealth", "weather", "wilderness", "wren" }; + +Test(hash_table, hash_table_lookup) +{ + int ret; + struct hash_table ht = { .state = STATE_DESTROYED }; + + ret = hash_table_init(&ht, 20); + cr_assert(!ret); + + /* Insert */ + for (int i = 0; i < ARRAY_LEN(keys); i++) { + ret = hash_table_insert(&ht, keys[i], values[i]); + cr_assert(!ret); + } + + /* Lookup */ + for (int i = 0; i < ARRAY_LEN(keys); i++) { + char *value = hash_table_lookup(&ht, keys[i]); + cr_assert_eq(values[i], value); + } + + /* Inserting the same key twice should fail */ + ret = hash_table_insert(&ht, keys[0], values[0]); + cr_assert(ret); + + hash_table_dump(&ht); + + /* Removing an entry */ + ret = hash_table_delete(&ht, keys[0]); + cr_assert(!ret); + + /* Removing the same entry twice should fail */ + ret = hash_table_delete(&ht, keys[0]); + cr_assert(ret); + + /* After removing, we should be able to insert it again */ + ret = hash_table_insert(&ht, keys[0], values[0]); + cr_assert(!ret); + + ret = hash_table_destroy(&ht, NULL, false); + cr_assert(!ret); +} diff --git a/common/tests/hist.c b/common/tests/hist.c new file mode 100644 index 000000000..a4ecac241 --- /dev/null +++ b/common/tests/hist.c @@ -0,0 +1,51 @@ +/** Unit tests for histogram + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +const double test_data[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + +/* Histogram of test_data with 200 buckets between -100 and 100 */ +const int hist_result[] = {}; + +Test(hist, simple) { + struct hist h; + int ret; + + ret = hist_init(&h, 0, 0); + cr_assert_eq(ret, 0); + + for (int i = 0; i < ARRAY_LEN(test_data); i++) + hist_put(&h, test_data[i]); + + cr_assert_float_eq(hist_mean(&h), 5.5, 1e-6); + cr_assert_float_eq(hist_var(&h), 9.1666, 1e-3,); + cr_assert_float_eq(hist_stddev(&h), 3.027650, 1e-6); + +// for (int i = 0; i < ARRAY_LEN(hist_result); i++) +// cr_assert_eq() + + hist_destroy(&h); +} diff --git a/common/tests/kernel.c b/common/tests/kernel.c new file mode 100644 index 000000000..fc9689c69 --- /dev/null +++ b/common/tests/kernel.c @@ -0,0 +1,114 @@ +/** Unit tests for kernel functions. + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +#ifdef __linux__ + +#if defined(__x86_64__) || defined(__i386__) + #define PAGESIZE (1 << 12) + #define CACHELINESIZE 64 + + #if defined(__x86_64__) + #define HUGEPAGESIZE (1 << 21) + #elif defined(__i386__) + #define HUGEPAGESIZE (1 << 22) + #endif +#else + #error "Unsupported architecture" +#endif + +/* This test is not portable, but we currently support x86 only */ +Test(kernel, sizes) +{ + int sz; + + sz = kernel_get_page_size(); + cr_assert_eq(sz, PAGESIZE); + + sz = kernel_get_hugepage_size(); + cr_assert(sz == HUGEPAGESIZE); + + sz = kernel_get_cacheline_size(); + cr_assert_eq(sz, CACHELINESIZE); +} + +Test(kernel, hugepages) +{ + int ret; + + ret = kernel_set_nr_hugepages(25); + cr_assert_eq(ret, 0); + + ret = kernel_get_nr_hugepages(); + cr_assert_eq(ret, 25); + + ret = kernel_set_nr_hugepages(10); + cr_assert_eq(ret, 0); + + ret = kernel_get_nr_hugepages(); + cr_assert_eq(ret, 10); +} + +Test(kernel, version) +{ + int ret; + + struct version ver; + + ret = kernel_get_version(&ver); + cr_assert_eq(ret, 0); + + ret = version_cmp(&ver, &(struct version) { 100, 5 }); + cr_assert_lt(ret, 0); + + ret = version_cmp(&ver, &(struct version) { 2, 6 }); + cr_assert_gt(ret, 0); +} + +Test(kernel, module, .disabled = true) +{ + int ret; + + ret = kernel_module_loaded("nf_nat"); + cr_assert_eq(ret, 0); + + ret = kernel_module_loaded("does_not_exist"); + cr_assert_neq(ret, 0); +} + +Test(kernel, frequency) +{ + int ret; + uint64_t freq; + + ret = kernel_get_cpu_frequency(&freq); + cr_assert_eq(ret, 0); + + /* Check for plausability only */ + cr_assert(freq > 1e9 && freq < 5e9); +} + +#endif /* __linux__ */ diff --git a/common/tests/list.c b/common/tests/list.c new file mode 100644 index 000000000..626a8b414 --- /dev/null +++ b/common/tests/list.c @@ -0,0 +1,163 @@ +/** Unit tests for array-based list + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include +#include + +static char *nouns[] = { "time", "person", "year", "way", "day", "thing", "man", "world", "life", "hand", "part", "child", "eye", "woman", "place", "work", "week", "case", "point", "government", "company", "number", "group", "problem", "fact" }; + +struct data { + char *tag; + int data; +}; + +Test(list, list_lookup) +{ + struct list l = { .state = STATE_DESTROYED }; + + list_init(&l); + + for (int i = 0; i < ARRAY_LEN(nouns); i++) { + struct data *d = malloc(sizeof(struct data)); + + d->tag = nouns[i]; + d->data = i; + + list_push(&l, d); + } + + struct data *found = list_lookup(&l, "woman"); + + cr_assert_eq(found->data, 13); + + list_destroy(&l, NULL, true); +} + +Test(list, list_search) +{ + struct list l = { .state = STATE_DESTROYED }; + + list_init(&l); + + /* Fill list */ + for (int i = 0; i < ARRAY_LEN(nouns); i++) + list_push(&l, nouns[i]); + + cr_assert_eq(list_length(&l), ARRAY_LEN(nouns)); + + /* Declare on stack! */ + char positive[] = "woman"; + char negative[] = "dinosaurrier"; + + char *found = list_search(&l, (cmp_cb_t) strcmp, positive); + cr_assert_not_null(found); + cr_assert_eq(found, nouns[13], "found = %p, nouns[13] = %p", found, nouns[13]); + cr_assert_str_eq(found, positive); + + char *not_found = (char *) list_search(&l, (cmp_cb_t) strcmp, negative); + cr_assert_null(not_found); + + list_destroy(&l, NULL, false); +} + +struct content { + int destroyed; +}; + +static int dtor(void *ptr) +{ + struct content *elm = (struct content *) ptr; + + elm->destroyed = 1; + + return 0; +} + +Test(list, destructor) +{ + struct list l = { .state = STATE_DESTROYED }; + struct content elm = { .destroyed = 0 }; + + list_init(&l); + list_push(&l, &elm); + + cr_assert_eq(list_length(&l), 1); + + list_destroy(&l, dtor, false); + + cr_assert_eq(elm.destroyed, 1); +} + +static int compare(const void *a, const void *b) { + return b - a; +} + +Test(list, basics) +{ + intptr_t i; + int ret; + struct list l = { .state = STATE_DESTROYED }; + + list_init(&l); + + for (i = 0; i < 100; i++) { + cr_assert_eq(list_length(&l), i); + + list_push(&l, (void *) i); + } + + cr_assert_eq(list_at_safe(&l, 555), NULL); + cr_assert_eq(list_last(&l), (void *) 99); + cr_assert_eq(list_first(&l), (void *) 0); + + for (size_t j = 0, i = 0; j < list_length(&l); j++) { + void *k = list_at(&l, j); + + cr_assert_eq(k, (void *) i++); + } + + list_sort(&l, compare); /* Reverse list */ + + for (size_t j = 0, i = 99; j < list_length(&l); j++) { + void *k = list_at(&l, j); + + cr_assert_eq(k, (void *) i, "Is %#zx, expected %p", i, k); + i--; + } + + ret = list_contains(&l, (void *) 55); + cr_assert(ret); + + list_remove(&l, (void *) 55); + + ret = list_contains(&l, (void *) 55); + cr_assert(!ret); + + list_destroy(&l, NULL, false); + + ret = list_length(&l); + cr_assert_eq(ret, -1, "List not properly destroyed: l.length = %zd", l.length); +} diff --git a/common/tests/log.c b/common/tests/log.c new file mode 100644 index 000000000..32df36ab8 --- /dev/null +++ b/common/tests/log.c @@ -0,0 +1,67 @@ +/** Unit tests for log functions + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include +#include + +struct param { + char *expression; + long expected; +}; + +static struct log l; + +static void init() +{ + log_init(&l, V, LOG_ALL); +} + +static void fini() +{ + log_destroy(&l); +} + +ParameterizedTestParameters(log, facility_expression) +{ + static struct param params[] = { + { "all,!pool", LOG_ALL & ~LOG_POOL }, + { "pool,!pool", LOG_POOL & ~LOG_POOL }, + { "pool,nodes,!socket", (LOG_POOL | LOG_NODES) & ~LOG_SOCKET }, + { "kernel", LOG_KERNEL }, + { "ngsi", LOG_NGSI }, + { "all", LOG_ALL }, + { "!all", 0 }, + { "", 0 } + }; + + return cr_make_param_array(struct param, params, ARRAY_LEN(params)); +} + +ParameterizedTest(struct param *p, log, facility_expression, .init = init, .fini = fini) +{ + log_set_facility_expression(&l, p->expression); + + cr_assert_eq(l.facilities, p->expected, "log.faciltities is %#lx not %#lx", l.facilities, p->expected); +} diff --git a/common/tests/task.c b/common/tests/task.c new file mode 100644 index 000000000..547303196 --- /dev/null +++ b/common/tests/task.c @@ -0,0 +1,92 @@ +/** Unit tests for periodic tasks + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include + +#include +#include + +Test(task, rate, .timeout = 10) +{ + int ret; + int runs = 10; + double rate = 5, waited; + struct timespec start, end; + struct task task; + + ret = task_init(&task, rate, CLOCK_MONOTONIC); + cr_assert_eq(ret, 0); + + int i; + for (i = 0; i < runs; i++) { + clock_gettime(CLOCK_MONOTONIC, &start); + + task_wait(&task); + + clock_gettime(CLOCK_MONOTONIC, &end); + + waited = time_delta(&start, &end); + + if (fabs(waited - 1.0 / rate) > 10e-3) + break; + } + + if (i < runs) + cr_assert_float_eq(waited, 1.0 / rate, 1e-2, "We slept for %f instead of %f secs in round %d", waited, 1.0 / rate, i); + + ret = task_destroy(&task); + cr_assert_eq(ret, 0); +} + +Test(task, wait_until, .timeout = 5) +{ + int ret; + struct task task; + struct timespec start, end, diff, future; + + ret = task_init(&task, 1, CLOCK_REALTIME); + cr_assert_eq(ret, 0); + + double waitfor = 3.423456789; + + start = time_now(); + diff = time_from_double(waitfor); + future = time_add(&start, &diff); + + ret = task_set_next(&task, &future); + cr_assert_eq(ret, 0); + + ret = task_wait(&task); + + end = time_now(); + + cr_assert_eq(ret, 1); + + double waited = time_delta(&start, &end); + + cr_assert_float_eq(waited, waitfor, 1e-2, "We slept for %f instead of %f secs", waited, waitfor); + + ret = task_destroy(&task); + cr_assert_eq(ret, 0); +} diff --git a/common/tests/timing.c b/common/tests/timing.c new file mode 100644 index 000000000..7ac3c8f4e --- /dev/null +++ b/common/tests/timing.c @@ -0,0 +1,92 @@ +/** Unit tests for time related utlities + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include + +#include + +Test(timing, time_now) +{ + struct timespec now1 = time_now(); + struct timespec now2 = time_now(); + + double delta = time_delta(&now1, &now2); + + cr_assert_float_eq(delta, 0, 1e-5, "time_now() shows large variance!"); + cr_assert_gt(delta, 0, "time_now() was reordered!"); +} + +Test(timing, time_diff) +{ + struct timespec ts1 = { .tv_sec = 0, .tv_nsec = 1}; /* Value doesnt matter */ + struct timespec ts2 = { .tv_sec = 1, .tv_nsec = 0}; /* Overflow in nano seconds! */ + + struct timespec ts3 = time_diff(&ts1, &ts2); + + /* ts4 == ts2? */ + cr_assert_eq(ts3.tv_sec, 0); + cr_assert_eq(ts3.tv_nsec, 999999999); +} + +Test(timing, time_add) +{ + struct timespec ts1 = { .tv_sec = 1, .tv_nsec = 999999999}; /* Value doesnt matter */ + struct timespec ts2 = { .tv_sec = 1, .tv_nsec = 1}; /* Overflow in nano seconds! */ + + struct timespec ts3 = time_add(&ts1, &ts2); + + /* ts4 == ts2? */ + cr_assert_eq(ts3.tv_sec, 3); + cr_assert_eq(ts3.tv_nsec, 0); +} + +Test(timing, time_delta) +{ + struct timespec ts1 = { .tv_sec = 1, .tv_nsec = 123}; /* Value doesnt matter */ + struct timespec ts2 = { .tv_sec = 5, .tv_nsec = 246}; /* Overflow in nano seconds! */ + + double delta = time_delta(&ts1, &ts2); + + cr_assert_float_eq(delta, 4 + 123e-9, 1e-9); +} + +Test(timing, time_from_double) +{ + double ref = 1234.56789; + + struct timespec ts = time_from_double(ref); + + cr_assert_eq(ts.tv_sec, 1234); + cr_assert_eq(ts.tv_nsec, 567890000); +} + +Test(timing, time_to_from_double) +{ + double ref = 1234.56789; + + struct timespec ts = time_from_double(ref); + double dbl = time_to_double(&ts); + + cr_assert_float_eq(dbl, ref, 1e-9); +} diff --git a/common/tests/tsc.c b/common/tests/tsc.c new file mode 100644 index 000000000..2e843b231 --- /dev/null +++ b/common/tests/tsc.c @@ -0,0 +1,74 @@ +/** Unit tests for rdtsc + * + * @author Steffen Vogel + * @copyright 2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include +#include + +#define CNT (1 << 18) + +Test(tsc, increasing) +{ + int ret; + struct tsc tsc; + uint64_t *cntrs; + + ret = tsc_init(&tsc); + cr_assert_eq(ret, 0); + + cntrs = alloc(sizeof(uint64_t) * CNT); + cr_assert_not_null(cntrs); + + for (int i = 0; i < CNT; i++) + cntrs[i] = tsc_now(&tsc); + + for (int i = 1; i < CNT; i++) + cr_assert_lt(cntrs[i-1], cntrs[i]); + + free(cntrs); +} + +Test(tsc, sleep) +{ + int ret; + double delta, duration = 1; + struct timespec start, stop; + struct tsc tsc; + uint64_t start_cycles, end_cycles; + + ret = tsc_init(&tsc); + cr_assert_eq(ret, 0); + + clock_gettime(CLOCK_MONOTONIC, &start); + + start_cycles = tsc_now(&tsc); + end_cycles = start_cycles + duration * tsc.frequency; + + while (tsc_now(&tsc) < end_cycles); + + clock_gettime(CLOCK_MONOTONIC, &stop); + delta = time_delta(&start, &stop); + + cr_assert_float_eq(delta, duration, 1e-4, "Error: %f, Delta: %lf, Freq: %llu", delta - duration, delta, tsc.frequency); +} diff --git a/common/tests/utils.c b/common/tests/utils.c new file mode 100644 index 000000000..ed06c86b3 --- /dev/null +++ b/common/tests/utils.c @@ -0,0 +1,196 @@ +/** Unit tests for utilities + * + * @author Steffen Vogel + * @copyright 2017-2018, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLAScommon + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +#include +#include + +/* Simple normality test for 1,2,3s intervals */ +Test(utils, box_muller) +{ + double n; + unsigned sigma[3] = { 0 }; + unsigned iter = 1000000; + + for (int i = 0; i < iter; i++) { + n = box_muller(0, 1); + + if (n > 2 || n < -2) sigma[2]++; + else if (n > 1 || n < -1) sigma[1]++; + else sigma[0]++; + } + +#if 0 + printf("%f %f %f\n", + (double) sigma[2] / iter, + (double) sigma[1] / iter, + (double) sigma[0] / iter); +#endif + + /* The random variable generated by the Box Muller transform is + * not an ideal normal distributed variable. + * The numbers from below are empirically measured. */ + cr_assert_float_eq((double) sigma[2] / iter, 0.045527, 1e-2); + cr_assert_float_eq((double) sigma[1] / iter, 0.271644, 1e-2); + cr_assert_float_eq((double) sigma[0] / iter, 0.682829, 1e-2); +} + +#ifdef __linux__ +Test(utils, cpuset) +{ + int ret; + char str[512]; + + cpu_set_t cset1; + cpu_set_t cset2; + uintmax_t int1 = 0x1234567890ABCDEFULL; + uintmax_t int2 = 0; + + cpuset_from_integer(int1, &cset1); + + cpulist_create(str, sizeof(str), &cset1); + + ret = cpulist_parse(str, &cset2, 1); + cr_assert_eq(ret, 0); + + cr_assert(CPU_EQUAL(&cset1, &cset2)); + + cpuset_to_integer(&cset2, &int2); + + cr_assert_eq(int1, int2); +} +#endif /* __linux__ */ + +Test(utils, memdup) +{ + char orig[1024], *copy; + size_t len; + + len = read_random(orig, sizeof(orig)); + cr_assert_eq(len, sizeof(orig)); + + copy = memdup(orig, sizeof(orig)); + cr_assert_not_null(copy); + cr_assert_arr_eq(copy, orig, sizeof(orig)); + + free(copy); +} + +Test(utils, is_aligned) +{ + /* Positive */ + cr_assert(IS_ALIGNED(1, 1)); + cr_assert(IS_ALIGNED(128, 64)); + + /* Negative */ + cr_assert(!IS_ALIGNED(55, 16)); + cr_assert(!IS_ALIGNED(55, 55)); + cr_assert(!IS_ALIGNED(1128, 256)); +} + +Test(utils, ceil) +{ + cr_assert_eq(CEIL(10, 3), 4); + cr_assert_eq(CEIL(10, 5), 2); + cr_assert_eq(CEIL(4, 3), 2); +} + +Test(utils, is_pow2) +{ + /* Positive */ + cr_assert(IS_POW2(1)); + cr_assert(IS_POW2(2)); + cr_assert(IS_POW2(64)); + + /* Negative */ + cr_assert(!IS_POW2(0)); + cr_assert(!IS_POW2(3)); + cr_assert(!IS_POW2(11111)); + cr_assert(!IS_POW2(-1)); +} + +Test(utils, strf) +{ + char *buf = NULL; + + buf = strf("Hallo %s", "Steffen."); + cr_assert_str_eq(buf, "Hallo Steffen."); + + strcatf(&buf, " Its Monday %uth %s %u.", 13, "August", 2018); + cr_assert_str_eq(buf, "Hallo Steffen. Its Monday 13th August 2018."); + + free(buf); +} + +struct version_param { + const char *v1, *v2; + int result; +}; + +Test(utils, version) +{ + struct version v1, v2, v3, v4; + + version_parse("1.2", &v1); + version_parse("1.3", &v2); + version_parse("55", &v3); + version_parse("66", &v4); + + cr_assert_lt(version_cmp(&v1, &v2), 0); + cr_assert_eq(version_cmp(&v1, &v1), 0); + cr_assert_gt(version_cmp(&v2, &v1), 0); + cr_assert_lt(version_cmp(&v3, &v4), 0); +} + +Test(utils, sha1sum) +{ + int ret; + FILE *f = tmpfile(); + + unsigned char hash[SHA_DIGEST_LENGTH]; + unsigned char expected[SHA_DIGEST_LENGTH] = { 0x69, 0xdf, 0x29, 0xdf, 0x1f, 0xf2, 0xd2, 0x5d, 0xb8, 0x68, 0x6c, 0x02, 0x8d, 0xdf, 0x40, 0xaf, 0xb3, 0xc1, 0xc9, 0x4d }; + + /* Write the first 512 fibonaccia numbers to the file */ + for (int i = 0, a = 0, b = 1, c; i < 512; i++, a = b, b = c) { + c = a + b; + + fwrite((void *) &c, sizeof(c), 1, f); + } + + ret = sha1sum(f, hash); + + cr_assert_eq(ret, 0); + cr_assert_arr_eq(hash, expected, SHA_DIGEST_LENGTH); + + fclose(f); +} + +Test(utils, decolor) +{ + char str[] = "This " CLR_RED("is") " a " CLR_BLU("colored") " " CLR_BLD("text!"); + char expect[] = "This is a colored text!"; + + decolor(str); + + cr_assert_str_eq(str, expect); +}