config: enable subsitution of environment variables and resolvement of @include directives

include/villas/config.hpp

@@ -23,11 +23,14 @@
 
 #pragma once
 
+#include <cstdio>
+
 #include <functional>
 #include <regex>
 #include <jansson.h>
 
 #include <villas/node/config.h>
+#include <villas/log.hpp>
 #include <villas/advio.hpp>
 
 namespace villas {
@@ -64,13 +67,13 @@ protected:
 	AFILE * loadFromRemoteFile(const std::string &u);
 
 	/** Resolve custom include directives. */
-	void resolveIncludes();
+	json_t * resolveIncludes(json_t *in);
 
 	/** To shell-like subsitution of environment variables in strings. */
-	void expandEnvVars();
+	json_t * expandEnvVars(json_t *in);
 
 	/** Run a callback function for each string in the config */
-	json_t * walkStrings(json_t *root, str_walk_fcn_t cb);
+	json_t * walkStrings(json_t *in, str_walk_fcn_t cb);
 
 public:
 	json_t *root;
@@ -80,7 +83,9 @@ public:
 
 	~Config();
 
-	json_t * load(const std::string &u);
+	json_t * load(std::FILE *f, bool resolveIncludes=true, bool resolveEnvVars=true);
+
+	json_t * load(const std::string &u, bool resolveIncludes=true, bool resolveEnvVars=true);
 };
 
 } /* namespace node */

lib/config.cpp

@@ -57,7 +57,20 @@ Config::~Config()
 		json_decref(root);
 }
 
-json_t * Config::load(const std::string &u)
+json_t * Config::load(std::FILE *f, bool resolveInc, bool resolveEnvVars)
+{
+	root = decode(f);
+
+	if (resolveInc)
+		root = resolveIncludes(root);
+
+	if (resolveEnvVars)
+		root = expandEnvVars(root);
+
+	return root;
+}
+
+json_t * Config::load(const std::string &u, bool resolveInc, bool resolveEnvVars)
 {
 	FILE *f;
 	AFILE *af = nullptr;
@@ -71,7 +84,7 @@ json_t * Config::load(const std::string &u)
 		f = af->file;
 	}
 
-	root = decode(f);
+	root = load(f, resolveInc, resolveEnvVars);
 
 	if (af)
 		afclose(af);
@@ -166,52 +179,48 @@ json_t * Config::libconfigDecode(FILE *f)
 }
 #endif /* WITH_CONFIG */
 
-json_t * Config::walkStrings(json_t *root, str_walk_fcn_t cb)
+json_t * Config::walkStrings(json_t *in, str_walk_fcn_t cb)
 {
 	const char *key;
 	size_t index;
 	json_t *val, *new_val, *new_root;
 
-	switch (json_typeof(root)) {
+	switch (json_typeof(in)) {
 		case JSON_STRING:
-			return cb(root);
+			return cb(in);
 
 		case JSON_OBJECT:
 			new_root = json_object();
 
-			json_object_foreach(root, key, val) {
+			json_object_foreach(in, key, val) {
 				new_val = walkStrings(val, cb);
 
-				json_object_set_new(new_root, key, new_val);
+				json_object_set(new_root, key, new_val);
 			}
 
-			json_decref(root);
-
 			return new_root;
 
 		case JSON_ARRAY:
 			new_root = json_array();
 
-			json_array_foreach(root, index, val) {
+			json_array_foreach(in, index, val) {
 				new_val = walkStrings(val, cb);
 
-				json_array_append_new(new_root, new_val);
+				json_array_append(new_root, new_val);
 			}
 
-			json_decref(root);
-
 			return new_root;
 
 		default:
-			return root;
+			return in;
 	};
 }
 
-void Config::expandEnvVars()
+json_t * Config::expandEnvVars(json_t *in)
 {
 	static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
 
-	root = walkStrings(root, [this](json_t *str) -> json_t * {
+	return walkStrings(in, [this](json_t *str) -> json_t * {
 		std::string text = json_string_value(str);
 
 		std::smatch match;
@@ -230,9 +239,9 @@ void Config::expandEnvVars()
 	});
 }
 
-void Config::resolveIncludes()
+json_t * Config::resolveIncludes(json_t *in)
 {
-	root = walkStrings(root, [this](json_t *str) -> json_t * {
+	return walkStrings(in, [this](json_t *str) -> json_t * {
 		std::string text = json_string_value(str);
 		static const std::string kw = "@include ";
 

lib/node_direction.cpp

@@ -27,7 +27,9 @@
 #include <villas/hook_list.hpp>
 #include <villas/node.h>
 #include <villas/node_direction.h>
+#include <villas/exceptions.hpp>
 
+using namespace villas;
 using namespace villas::node;
 using namespace villas::utils;
 
@@ -135,10 +137,12 @@ int node_direction_parse(struct node_direction *nd, struct node *n, json_t *cfg)
 		const char *type_str = "float";
 
 		if (json_is_object(json_signals)) {
-			json_unpack_ex(json_signals, &err, 0, "{ s: i, s: s }",
+			ret = json_unpack_ex(json_signals, &err, 0, "{ s: i, s: s }",
 				"count", &count,
 				"type", &type_str
 			);
+			if  (ret)
+				throw ConfigError(json_signals, "node-config-node-signals", "Invalid signal definition");
 		}
 
 		enum SignalType type = signal_type_from_str(type_str);

tests/unit/CMakeLists.txt

@@ -22,6 +22,7 @@
 
 set(TEST_SRC
 	config_json.cpp
+	config.cpp
 	main.cpp
 	mapping.cpp
 	memory.cpp

tests/unit/config.cpp

@@ -0,0 +1,92 @@
+/** Unit tests for config features.
+ *
+ * @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
+ * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
+ * @license GNU General Public License (version 3)
+ *
+ * VILLASnode
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *********************************************************************************/
+
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <criterion/criterion.h>
+
+#include <villas/utils.hpp>
+#include <villas/config.hpp>
+
+namespace fs = std::filesystem;
+using namespace villas::node;
+
+Test(config, env)
+{
+	const char *cfg_f = "test = \"${MY_ENV_VAR}\"\n";
+
+	std::FILE *f = std::tmpfile();
+	std::fputs(cfg_f, f);
+	std::rewind(f);
+
+	auto c = Config();
+
+	char env[] = "MY_ENV_VAR=mobydick";
+	putenv(env);
+
+	auto *r = c.load(f);
+	cr_assert_not_null(r);
+
+	auto *j = json_object_get(r, "test");
+	cr_assert_not_null(j);
+
+	cr_assert(json_is_string(j));
+	cr_assert_str_eq("mobydick", json_string_value(j));
+}
+
+
+Test(config, include)
+{
+	const char *cfg_f2 = "magic = 1234\n";
+
+	std::string f2_fn = std::tmpnam(nullptr);
+	std::FILE *f2 = std::fopen(f2_fn.c_str(), "w");
+	std::fputs(cfg_f2, f2);
+	std::rewind(f2);
+
+	auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn);
+
+	std::FILE *f1 = std::tmpfile();
+	std::fputs(cfg_f1.c_str(), f1);
+	std::rewind(f1);
+
+	auto env = fmt::format("INCLUDE_FILE={}", f2_fn).c_str();
+	putenv((char *) env);
+
+	auto c = Config();
+
+	auto *r = c.load(f1);
+	cr_assert_not_null(r);
+
+	auto *j = json_object_get(r, "subval");
+	cr_assert_not_null(j);
+
+	auto *j2 = json_object_get(j, "magic");
+	cr_assert_not_null(j2);
+
+	cr_assert(json_is_integer(j2));
+	cr_assert_eq(json_number_value(j2), 1234);
+
+	std::fclose(f2);
+	std::remove(f2_fn.c_str());
+}

tests/unit/config_json.cpp

@@ -59,7 +59,7 @@ const char *json_example = "{\n"
 			   "  ]\n"
 			   "}";
 
-Test(utils, config_to_json)
+Test(config, config_to_json)
 {
 	int ret;
 	config_t cfg;
@@ -87,7 +87,7 @@ Test(utils, config_to_json)
 	config_destroy(&cfg);
 }
 
-Test(utils, json_to_config)
+Test(config, json_to_config)
 {
 	config_t cfg;
 	config_setting_t *cfg_root;