diff --git a/doc/index.rst b/doc/index.rst index 56fbdf4..1b2183b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -10,6 +10,7 @@ Criterion assert hooks env + parameterized theories internal faq diff --git a/doc/parameterized.rst b/doc/parameterized.rst new file mode 100644 index 0000000..8a7fc04 --- /dev/null +++ b/doc/parameterized.rst @@ -0,0 +1,110 @@ +Using parameterized tests +========================= + +Parameterized tests are useful to repeat a specific test logic over a finite +set of parameters. + +Due to limitations on how generated parameters are passed, parameterized tests +can only accept one pointer parameter; however, this is not that much of a +problem since you can just pass a structure containing the context you need. + +Adding parameterized tests +-------------------------- + +Adding parameterized tests is done by defining the parameterized test function, +and the parameter generator function: + +.. code-block:: c + + #include + + ParameterizedTestParameter(suite_name, test_name) = { + void *params; + size_t nb_params; + + // generate parameter set + return cr_make_param_array(Type, params, nb_params); + } + + ParameterizedTest(Type *param, suite_name, test_name) { + // contents of the test + } + +``suite_name`` and ``test_name`` are the identifiers of the test suite and +the test, respectively. These identifiers must follow the language +identifier format. + +``Type`` is the compound type of the generated array. ``params`` and ``nb_params`` +are the pointer and the length of the generated array, respectively. + +Passing multiple parameters +--------------------------- + +As said earlier, parameterized tests only take one parameter, so passing +multiple parameters is, in the strict sense, not possible. However, one can +easily use a struct to hold the context as a workaround: + +.. code-block:: c + + #include + + struct my_params { + int param0; + double param1; + ... + }; + + ParameterizedTestParameter(suite_name, test_name) = { + size_t nb_params = 32; + struct my_params *params = malloc(sizeof (struct my_params) * nb_params); + + // generate parameter set + + params[0] = ... + params[1] = ... + + ... + + return cr_make_param_array(struct my_params, params, nb_params); + } + + ParameterizedTest(struct my_params *param, suite_name, test_name) { + // access param.param0, param.param1, ... + } + +There is, however, one absolute rule that must be respected, unless you don't +want your tests to run on windows, ever: parameters must not contain any +pointer to dynamically allocated data. + +Hence, this is not permitted: + +.. code-block:: c + + #include + + struct my_params { + int *some_int_ptr; + }; + + ParameterizedTestParameter(suite_name, test_name) = { + static my_params param = { + .some_int_ptr = malloc(sizeof (int)); // Don't do this. + }; + *param.some_int_ptr = 42; + + return cr_make_param_array(struct my_params, ¶m, 1); + } + +and **will crash the test** on Windows. + +Configuring parameterized tests +------------------------------- + +Parameterized tests can optionally recieve configuration parameters to alter +their own behaviour, and are applied to each iteration of the parameterized +test individually (this means that the initialization and finalization runs once +per iteration). +Those parameters are the same ones as the ones of the ``Test`` macro function +(c.f. :ref:`test-config-ref`). + + diff --git a/include/criterion/criterion.h b/include/criterion/criterion.h index f72eba0..0beb926 100644 --- a/include/criterion/criterion.h +++ b/include/criterion/criterion.h @@ -48,6 +48,8 @@ TEST_PROTOTYPE_(Category, Name); \ struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = \ CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data, \ + .kind_ = CR_TEST_NORMAL, \ + .param_ = (struct criterion_test_params(*)(void)) NULL, \ .identifier_ = #Category "/" #Name, \ .file_ = __FILE__, \ .line_ = __LINE__, \ diff --git a/include/criterion/parameterized.h b/include/criterion/parameterized.h new file mode 100644 index 0000000..6bfec4e --- /dev/null +++ b/include/criterion/parameterized.h @@ -0,0 +1,50 @@ +#ifndef CRITERION_PARAMETERIZED_H_ +# define CRITERION_PARAMETERIZED_H_ + +# include "criterion.h" + +# ifdef __cplusplus +# define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \ + extern "C" void IDENTIFIER_(Category, Name, impl)(Param) +# else +# define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \ + void IDENTIFIER_(Category, Name, impl)(Param) +# endif + +# define ParameterizedTest(...) \ + CR_EXPAND(ParameterizedTest_(__VA_ARGS__, .sentinel_ = 0)) + +# define ParameterizedTest_(Param, Category, Name, ...) \ + CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name); \ + struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = \ + CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data, \ + .kind_ = CR_TEST_PARAMETERIZED, \ + .param_ = IDENTIFIER_(Category, Name, param), \ + .identifier_ = #Category "/" #Name, \ + .file_ = __FILE__, \ + .line_ = __LINE__, \ + __VA_ARGS__ \ + )); \ + struct criterion_test IDENTIFIER_(Category, Name, meta) = { \ + #Name, \ + #Category, \ + (void(*)(void)) IDENTIFIER_(Category, Name, impl), \ + &IDENTIFIER_(Category, Name, extra) \ + }; \ + SECTION_("cr_tst") \ + struct criterion_test *IDENTIFIER_(Category, Name, ptr) \ + = &IDENTIFIER_(Category, Name, meta) SECTION_SUFFIX_; \ + CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) + +# define ParameterizedTestParameters(Category, Name) \ + static struct criterion_test_params IDENTIFIER_(Category, Name, param)(void) + +# ifdef __cplusplus +# define cr_make_param_array(Type, Array, ...) \ + criterion_test_params(sizeof (Type), (Array), __VA_ARGS__) +# else +# define cr_make_param_array(Type, Array, ...) \ + (struct criterion_test_params) { .size = sizeof (Type), (void*)(Array), __VA_ARGS__ } +# endif + +#endif /* !CRITERION_PARAMETERIZED_H_ */ diff --git a/include/criterion/types.h b/include/criterion/types.h index df2c3bc..7e98bc8 100644 --- a/include/criterion/types.h +++ b/include/criterion/types.h @@ -33,8 +33,39 @@ using std::size_t; # endif # include "common.h" +enum criterion_test_kind { + CR_TEST_NORMAL, + CR_TEST_PARAMETERIZED, +}; + +struct criterion_test_params { + size_t size; + void *params; + size_t length; + void (*cleanup)(struct criterion_test_params *); + +# ifdef __cplusplus + constexpr criterion_test_params(size_t size, void *params, size_t length) + : size(size) + , params(params) + , length(length) + , cleanup(nullptr) + {} + + constexpr criterion_test_params(size_t size, void *params, size_t length, + void (*cleanup)(struct criterion_test_params *)) + : size(size) + , params(params) + , length(length) + , cleanup(cleanup) + {} +# endif +}; + struct criterion_test_extra_data { int sentinel_; + enum criterion_test_kind kind_; + struct criterion_test_params (*param_)(void); const char *identifier_; const char *file_; unsigned line_; diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 6ae8e1a..a80f1af 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -17,6 +17,7 @@ set(SAMPLES theories.c timeout.c redirect.c + parameterized.c signal.cc report.cc diff --git a/samples/outputs/parameterized.c.bin.err.expected b/samples/outputs/parameterized.c.bin.err.expected new file mode 100644 index 0000000..f4b3134 --- /dev/null +++ b/samples/outputs/parameterized.c.bin.err.expected @@ -0,0 +1,19 @@ +[----] parameterized.c:60: Assertion failed: Parameters: (1, 2.000000) +[FAIL] params::cleanup: (0.00s) +[----] parameterized.c:60: Assertion failed: Parameters: (3, 4.000000) +[FAIL] params::cleanup: (0.00s) +[----] parameterized.c:60: Assertion failed: Parameters: (5, 6.000000) +[FAIL] params::cleanup: (0.00s) +[----] parameterized.c:36: Assertion failed: Parameters: (1, 2.000000) +[FAIL] params::multiple: (0.00s) +[----] parameterized.c:36: Assertion failed: Parameters: (3, 4.000000) +[FAIL] params::multiple: (0.00s) +[----] parameterized.c:36: Assertion failed: Parameters: (5, 6.000000) +[FAIL] params::multiple: (0.00s) +[----] parameterized.c:15: Assertion failed: Parameter: foo +[FAIL] params::str: (0.00s) +[----] parameterized.c:15: Assertion failed: Parameter: bar +[FAIL] params::str: (0.00s) +[----] parameterized.c:15: Assertion failed: Parameter: baz +[FAIL] params::str: (0.00s) +[====] Synthesis: Tested: 9 | Passing: 0 | Failing: 9 | Crashing: 0  diff --git a/samples/outputs/parameterized.c.bin.out.expected b/samples/outputs/parameterized.c.bin.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/parameterized.c b/samples/parameterized.c new file mode 100644 index 0000000..6c58b9a --- /dev/null +++ b/samples/parameterized.c @@ -0,0 +1,61 @@ +#include +#include + +// Basic usage + +ParameterizedTestParameters(params, str) { + static const char *strings[] = { + "foo", "bar", "baz" + }; + + return cr_make_param_array(const char *, strings, sizeof (strings) / sizeof (const char *)); +} + +ParameterizedTest(const char **str, params, str) { + cr_assert_fail("Parameter: %s", *str); +} + +// Multiple parameters must be coalesced in a single parameter + +struct parameter_tuple { + int i; + double d; +}; + +ParameterizedTestParameters(params, multiple) { + static struct parameter_tuple params[] = { + {1, 2}, + {3, 4}, + {5, 6}, + }; + + return cr_make_param_array(struct parameter_tuple, params, sizeof (params) / sizeof (struct parameter_tuple)); +} + +ParameterizedTest(struct parameter_tuple *tup, params, multiple) { + cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d); +} + +// Cleaning up dynamically generated parameters + +// Note: Do **NOT** embed dynamically allocated pointers inside of structures +// or this will fail on windows + +void free_params(struct criterion_test_params *crp) { + free(crp->params); +} + +ParameterizedTestParameters(params, cleanup) { + const size_t nb_tuples = 3; + + struct parameter_tuple *params = malloc(sizeof(struct parameter_tuple) * nb_tuples); + params[0] = (struct parameter_tuple) { 1, 2 }; + params[1] = (struct parameter_tuple) { 3, 4 }; + params[2] = (struct parameter_tuple) { 5, 6 }; + + return cr_make_param_array(struct parameter_tuple, params, nb_tuples, free_params); +} + +ParameterizedTest(struct parameter_tuple *tup, params, cleanup) { + cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d); +} diff --git a/src/compat/process.c b/src/compat/process.c index f5c03bc..8a67623 100644 --- a/src/compat/process.c +++ b/src/compat/process.c @@ -34,6 +34,7 @@ #include #ifdef VANILLA_WIN32 +# include # define CREATE_SUSPENDED_(Filename, CmdLine, StartupInfo, Info) \ CreateProcessW(Filename, \ CmdLine, \ @@ -87,8 +88,6 @@ static int get_win_status(HANDLE handle) { } #endif - - struct worker_context g_worker_context = {.test = NULL}; #ifdef VANILLA_WIN32 @@ -99,11 +98,12 @@ struct full_context { struct criterion_test_extra_data suite_data; f_worker_func func; struct pipe_handle pipe; - volatile int resumed; + struct test_single_param param; + HANDLE sync; + DWORD extra_size; }; -static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker"); -#define MAPPING_SIZE sizeof (struct full_context) +static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker_%lu"); static struct full_context local_ctx; #endif @@ -168,10 +168,13 @@ static void CALLBACK handle_child_terminated(PVOID lpParameter, int resume_child(void) { #ifdef VANILLA_WIN32 + TCHAR mapping_name[128]; + _sntprintf(mapping_name, 128, g_mapping_name, GetCurrentProcessId()); + HANDLE sharedMem = OpenFileMapping( FILE_MAP_ALL_ACCESS, FALSE, - g_mapping_name); + mapping_name); if (sharedMem == NULL) return 0; @@ -180,30 +183,57 @@ int resume_child(void) { FILE_MAP_ALL_ACCESS, 0, 0, - MAPPING_SIZE); + sizeof (struct full_context)); - if (ctx == NULL) + if (ctx == NULL) { + CloseHandle(sharedMem); exit(-1); + } local_ctx = *ctx; + UnmapViewOfFile(ctx); + + struct test_single_param *param = NULL; + if (local_ctx.param.size != 0) { + ctx = (struct full_context*) MapViewOfFile(sharedMem, + FILE_MAP_ALL_ACCESS, + 0, + 0, + sizeof (struct full_context) + local_ctx.extra_size); + + if (ctx == NULL) { + CloseHandle(sharedMem); + exit(-1); + } + + param = malloc(sizeof (struct test_single_param) + local_ctx.param.size); + *param = (struct test_single_param) { + .size = local_ctx.param.size, + .ptr = param + 1, + }; + memcpy(param + 1, ctx + 1, param->size); + UnmapViewOfFile(ctx); + } + + CloseHandle(sharedMem); + g_worker_context = (struct worker_context) { - &local_ctx.test, - &local_ctx.suite, - local_ctx.func, - &local_ctx.pipe + .test = &local_ctx.test, + .suite = &local_ctx.suite, + .func = local_ctx.func, + .pipe = &local_ctx.pipe, + .param = param, }; local_ctx.test.data = &local_ctx.test_data; local_ctx.suite.data = &local_ctx.suite_data; - ctx->resumed = 1; - - UnmapViewOfFile(ctx); - CloseHandle(sharedMem); + SetEvent(local_ctx.sync); SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); run_worker(&g_worker_context); + free(param); return 1; #else # if defined(__unix__) || defined(__APPLE__) @@ -227,6 +257,16 @@ s_proc_handle *fork_process() { ZeroMemory(&info, sizeof (info)); + SECURITY_ATTRIBUTES inherit_handle = { + .nLength = sizeof (SECURITY_ATTRIBUTES), + .bInheritHandle = TRUE + }; + + // Create the synchronization event + HANDLE sync = CreateEvent(&inherit_handle, FALSE, FALSE, NULL); + if (sync == NULL) + return (void *) -1; + // Create the suspended child process wchar_t filename[MAX_PATH]; GetModuleFileNameW(NULL, filename, MAX_PATH); @@ -235,22 +275,29 @@ s_proc_handle *fork_process() { return (void *) -1; // Copy context over + TCHAR mapping_name[128]; + _sntprintf(mapping_name, 128, g_mapping_name, info.dwProcessId); + + DWORD mapping_size = sizeof (struct full_context); + if (g_worker_context.param) + mapping_size += g_worker_context.param->size; + HANDLE sharedMem = CreateFileMapping( INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, - MAPPING_SIZE, - g_mapping_name); + mapping_size, + mapping_name); - if (sharedMem == NULL) + if (sharedMem == NULL || GetLastError() == ERROR_ALREADY_EXISTS) return (void *) -1; struct full_context *ctx = (struct full_context *) MapViewOfFile(sharedMem, FILE_MAP_ALL_ACCESS, 0, 0, - MAPPING_SIZE); + mapping_size); if (ctx == NULL) { CloseHandle(sharedMem); @@ -263,16 +310,28 @@ s_proc_handle *fork_process() { .suite = *g_worker_context.suite, .func = g_worker_context.func, .pipe = *g_worker_context.pipe, - .resumed = 0, + .sync = sync, }; + if (g_worker_context.param) { + ctx->extra_size = g_worker_context.param->size; + ctx->param = *g_worker_context.param; + memcpy(ctx + 1, g_worker_context.param->ptr, g_worker_context.param->size); + } + if (g_worker_context.suite->data) ctx->suite_data = *g_worker_context.suite->data; - ResumeThread(info.hThread); - CloseHandle(info.hThread); + if (ResumeThread(info.hThread) == (DWORD) -1) + goto failure; - while (!ctx->resumed); // wait until the child has initialized itself + // wait until the child has initialized itself + HANDLE handles[] = { info.hProcess, sync }; + DWORD wres = WaitForMultipleObjects(sizeof (handles) / sizeof (HANDLE), handles, FALSE, INFINITE); + if (wres == WAIT_OBJECT_0) + goto failure; + + CloseHandle(info.hThread); UnmapViewOfFile(ctx); CloseHandle(sharedMem); @@ -293,6 +352,14 @@ s_proc_handle *fork_process() { s_proc_handle *handle = smalloc(sizeof (s_proc_handle)); *handle = (s_proc_handle) { info.hProcess }; return handle; + +failure: + UnmapViewOfFile(ctx); + CloseHandle(sharedMem); + CloseHandle(info.hThread); + CloseHandle(info.hProcess); + return (void *) -1; + #else pid_t pid = fork(); if (pid == -1) diff --git a/src/compat/process.h b/src/compat/process.h index 2e26db4..0d70770 100644 --- a/src/compat/process.h +++ b/src/compat/process.h @@ -42,6 +42,7 @@ struct worker_context { struct criterion_suite *suite; f_worker_func func; struct pipe_handle *pipe; + struct test_single_param *param; }; extern struct worker_context g_worker_context; diff --git a/src/core/runner.c b/src/core/runner.c index 4333ebd..a998843 100644 --- a/src/core/runner.c +++ b/src/core/runner.c @@ -186,7 +186,14 @@ static void run_test_child(struct criterion_test *test, struct timespec_compat ts; if (!setjmp(g_pre_test)) { timer_start(&ts); - (test->test ? test->test : nothing)(); + if (test->test) { + if (!test->data->param_) { + test->test(); + } else { + void(*param_test_func)(void *) = (void(*)(void*)) test->test; + param_test_func(g_worker_context.param->ptr); + } + } } double elapsed_time; @@ -297,7 +304,8 @@ static void handle_worker_terminated(struct event *ev, static void run_test(struct criterion_global_stats *stats, struct criterion_suite_stats *suite_stats, struct criterion_test *test, - struct criterion_suite *suite) { + struct criterion_suite *suite, + struct test_single_param *param) { struct criterion_test_stats *test_stats = test_stats_init(test); struct process *proc = NULL; @@ -310,7 +318,7 @@ static void run_test(struct criterion_global_stats *stats, goto cleanup; } - proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe); + proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe, param); if (proc == NULL && !is_runner()) goto cleanup; @@ -370,6 +378,45 @@ cleanup: sfree(proc); } +static void run_test_param(struct criterion_global_stats *stats, + struct criterion_suite_stats *suite_stats, + struct criterion_test *test, + struct criterion_suite *suite) { + + if (!test->data->param_) + return; + + struct criterion_test_params params = test->data->param_(); + for (size_t i = 0; i < params.length; ++i) { + struct test_single_param param = { params.size, (char *) params.params + i * params.size }; + + run_test(stats, suite_stats, test, suite, ¶m); + if (criterion_options.fail_fast && stats->tests_failed > 0) + break; + if (!is_runner()) + break; + } + + if (params.cleanup) + params.cleanup(¶ms); +} + +static void run_test_switch(struct criterion_global_stats *stats, + struct criterion_suite_stats *suite_stats, + struct criterion_test *test, + struct criterion_suite *suite) { + + switch (test->data->kind_) { + case CR_TEST_NORMAL: + run_test(stats, suite_stats, test, suite, NULL); + break; + case CR_TEST_PARAMETERIZED: + run_test_param(stats, suite_stats, test, suite); + break; + default: break; + } +} + #ifdef HAVE_PCRE void disable_unmatching(struct criterion_test_set *set) { FOREACH_SET(struct criterion_suite_set *s, set->suites) { @@ -414,7 +461,7 @@ static int criterion_run_all_tests_impl(struct criterion_test_set *set) { abort(); struct criterion_global_stats *stats = stats_init(); - map_tests(set, stats, run_test); + map_tests(set, stats, run_test_switch); int result = is_runner() ? stats->tests_failed == 0 : -1; diff --git a/src/core/worker.c b/src/core/worker.c index 7f53279..85a945a 100644 --- a/src/core/worker.c +++ b/src/core/worker.c @@ -77,19 +77,21 @@ void run_worker(struct worker_context *ctx) { struct process *spawn_test_worker(struct criterion_test *test, struct criterion_suite *suite, f_worker_func func, - s_pipe_handle *pipe) { + s_pipe_handle *pipe, + struct test_single_param *param) { g_worker_context = (struct worker_context) { .test = test, .suite = suite, .func = func, - .pipe = pipe + .pipe = pipe, + .param = param, }; struct process *ptr = NULL; s_proc_handle *proc = fork_process(); if (proc == (void *) -1) { - return NULL; + abort(); } else if (proc == NULL) { run_worker(&g_worker_context); return NULL; diff --git a/src/core/worker.h b/src/core/worker.h index 5cc7af7..ab8bff9 100644 --- a/src/core/worker.h +++ b/src/core/worker.h @@ -47,6 +47,11 @@ struct worker_status { struct process_status status; }; +struct test_single_param { + size_t size; + void *ptr; +}; + void run_worker(struct worker_context *ctx); void set_runner_process(void); void unset_runner_process(void); @@ -56,7 +61,8 @@ struct process_status get_status(int status); struct process *spawn_test_worker(struct criterion_test *test, struct criterion_suite *suite, f_worker_func func, - s_pipe_handle *pipe); + s_pipe_handle *pipe, + struct test_single_param *param); struct event *worker_read_event(struct process *proc); #endif /* !PROCESS_H_ */