Merge branch 'features/parameterized-tests' into bleeding

This commit is contained in:
Snaipe 2015-09-20 19:43:17 +02:00
commit 5b42df6db9
14 changed files with 430 additions and 32 deletions

View file

@ -10,6 +10,7 @@ Criterion
assert
hooks
env
parameterized
theories
internal
faq

110
doc/parameterized.rst Normal file
View file

@ -0,0 +1,110 @@
Using parameterized tests
=========================
Parameterized tests are useful to repeat a specific test logic over a finite
set of parameters.
Due to limitations on how generated parameters are passed, parameterized tests
can only accept one pointer parameter; however, this is not that much of a
problem since you can just pass a structure containing the context you need.
Adding parameterized tests
--------------------------
Adding parameterized tests is done by defining the parameterized test function,
and the parameter generator function:
.. code-block:: c
#include <criterion/parameterized.h>
ParameterizedTestParameter(suite_name, test_name) = {
void *params;
size_t nb_params;
// generate parameter set
return cr_make_param_array(Type, params, nb_params);
}
ParameterizedTest(Type *param, suite_name, test_name) {
// contents of the test
}
``suite_name`` and ``test_name`` are the identifiers of the test suite and
the test, respectively. These identifiers must follow the language
identifier format.
``Type`` is the compound type of the generated array. ``params`` and ``nb_params``
are the pointer and the length of the generated array, respectively.
Passing multiple parameters
---------------------------
As said earlier, parameterized tests only take one parameter, so passing
multiple parameters is, in the strict sense, not possible. However, one can
easily use a struct to hold the context as a workaround:
.. code-block:: c
#include <criterion/parameterized.h>
struct my_params {
int param0;
double param1;
...
};
ParameterizedTestParameter(suite_name, test_name) = {
size_t nb_params = 32;
struct my_params *params = malloc(sizeof (struct my_params) * nb_params);
// generate parameter set
params[0] = ...
params[1] = ...
...
return cr_make_param_array(struct my_params, params, nb_params);
}
ParameterizedTest(struct my_params *param, suite_name, test_name) {
// access param.param0, param.param1, ...
}
There is, however, one absolute rule that must be respected, unless you don't
want your tests to run on windows, ever: parameters must not contain any
pointer to dynamically allocated data.
Hence, this is not permitted:
.. code-block:: c
#include <criterion/parameterized.h>
struct my_params {
int *some_int_ptr;
};
ParameterizedTestParameter(suite_name, test_name) = {
static my_params param = {
.some_int_ptr = malloc(sizeof (int)); // Don't do this.
};
*param.some_int_ptr = 42;
return cr_make_param_array(struct my_params, &param, 1);
}
and **will crash the test** on Windows.
Configuring parameterized tests
-------------------------------
Parameterized tests can optionally recieve configuration parameters to alter
their own behaviour, and are applied to each iteration of the parameterized
test individually (this means that the initialization and finalization runs once
per iteration).
Those parameters are the same ones as the ones of the ``Test`` macro function
(c.f. :ref:`test-config-ref`).

View file

@ -48,6 +48,8 @@
TEST_PROTOTYPE_(Category, Name); \
struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = \
CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data, \
.kind_ = CR_TEST_NORMAL, \
.param_ = (struct criterion_test_params(*)(void)) NULL, \
.identifier_ = #Category "/" #Name, \
.file_ = __FILE__, \
.line_ = __LINE__, \

View file

@ -0,0 +1,50 @@
#ifndef CRITERION_PARAMETERIZED_H_
# define CRITERION_PARAMETERIZED_H_
# include "criterion.h"
# ifdef __cplusplus
# define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \
extern "C" void IDENTIFIER_(Category, Name, impl)(Param)
# else
# define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \
void IDENTIFIER_(Category, Name, impl)(Param)
# endif
# define ParameterizedTest(...) \
CR_EXPAND(ParameterizedTest_(__VA_ARGS__, .sentinel_ = 0))
# define ParameterizedTest_(Param, Category, Name, ...) \
CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name); \
struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = \
CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data, \
.kind_ = CR_TEST_PARAMETERIZED, \
.param_ = IDENTIFIER_(Category, Name, param), \
.identifier_ = #Category "/" #Name, \
.file_ = __FILE__, \
.line_ = __LINE__, \
__VA_ARGS__ \
)); \
struct criterion_test IDENTIFIER_(Category, Name, meta) = { \
#Name, \
#Category, \
(void(*)(void)) IDENTIFIER_(Category, Name, impl), \
&IDENTIFIER_(Category, Name, extra) \
}; \
SECTION_("cr_tst") \
struct criterion_test *IDENTIFIER_(Category, Name, ptr) \
= &IDENTIFIER_(Category, Name, meta) SECTION_SUFFIX_; \
CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name)
# define ParameterizedTestParameters(Category, Name) \
static struct criterion_test_params IDENTIFIER_(Category, Name, param)(void)
# ifdef __cplusplus
# define cr_make_param_array(Type, Array, ...) \
criterion_test_params(sizeof (Type), (Array), __VA_ARGS__)
# else
# define cr_make_param_array(Type, Array, ...) \
(struct criterion_test_params) { .size = sizeof (Type), (void*)(Array), __VA_ARGS__ }
# endif
#endif /* !CRITERION_PARAMETERIZED_H_ */

View file

@ -33,8 +33,39 @@ using std::size_t;
# endif
# include "common.h"
enum criterion_test_kind {
CR_TEST_NORMAL,
CR_TEST_PARAMETERIZED,
};
struct criterion_test_params {
size_t size;
void *params;
size_t length;
void (*cleanup)(struct criterion_test_params *);
# ifdef __cplusplus
constexpr criterion_test_params(size_t size, void *params, size_t length)
: size(size)
, params(params)
, length(length)
, cleanup(nullptr)
{}
constexpr criterion_test_params(size_t size, void *params, size_t length,
void (*cleanup)(struct criterion_test_params *))
: size(size)
, params(params)
, length(length)
, cleanup(cleanup)
{}
# endif
};
struct criterion_test_extra_data {
int sentinel_;
enum criterion_test_kind kind_;
struct criterion_test_params (*param_)(void);
const char *identifier_;
const char *file_;
unsigned line_;

View file

@ -17,6 +17,7 @@ set(SAMPLES
theories.c
timeout.c
redirect.c
parameterized.c
signal.cc
report.cc

View file

@ -0,0 +1,19 @@
[----] parameterized.c:60: Assertion failed: Parameters: (1, 2.000000)
[FAIL] params::cleanup: (0.00s)
[----] parameterized.c:60: Assertion failed: Parameters: (3, 4.000000)
[FAIL] params::cleanup: (0.00s)
[----] parameterized.c:60: Assertion failed: Parameters: (5, 6.000000)
[FAIL] params::cleanup: (0.00s)
[----] parameterized.c:36: Assertion failed: Parameters: (1, 2.000000)
[FAIL] params::multiple: (0.00s)
[----] parameterized.c:36: Assertion failed: Parameters: (3, 4.000000)
[FAIL] params::multiple: (0.00s)
[----] parameterized.c:36: Assertion failed: Parameters: (5, 6.000000)
[FAIL] params::multiple: (0.00s)
[----] parameterized.c:15: Assertion failed: Parameter: foo
[FAIL] params::str: (0.00s)
[----] parameterized.c:15: Assertion failed: Parameter: bar
[FAIL] params::str: (0.00s)
[----] parameterized.c:15: Assertion failed: Parameter: baz
[FAIL] params::str: (0.00s)
[====] Synthesis: Tested: 9 | Passing: 0 | Failing: 9 | Crashing: 0 

61
samples/parameterized.c Normal file
View file

@ -0,0 +1,61 @@
#include <criterion/parameterized.h>
#include <stdio.h>
// Basic usage
ParameterizedTestParameters(params, str) {
static const char *strings[] = {
"foo", "bar", "baz"
};
return cr_make_param_array(const char *, strings, sizeof (strings) / sizeof (const char *));
}
ParameterizedTest(const char **str, params, str) {
cr_assert_fail("Parameter: %s", *str);
}
// Multiple parameters must be coalesced in a single parameter
struct parameter_tuple {
int i;
double d;
};
ParameterizedTestParameters(params, multiple) {
static struct parameter_tuple params[] = {
{1, 2},
{3, 4},
{5, 6},
};
return cr_make_param_array(struct parameter_tuple, params, sizeof (params) / sizeof (struct parameter_tuple));
}
ParameterizedTest(struct parameter_tuple *tup, params, multiple) {
cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d);
}
// Cleaning up dynamically generated parameters
// Note: Do **NOT** embed dynamically allocated pointers inside of structures
// or this will fail on windows
void free_params(struct criterion_test_params *crp) {
free(crp->params);
}
ParameterizedTestParameters(params, cleanup) {
const size_t nb_tuples = 3;
struct parameter_tuple *params = malloc(sizeof(struct parameter_tuple) * nb_tuples);
params[0] = (struct parameter_tuple) { 1, 2 };
params[1] = (struct parameter_tuple) { 3, 4 };
params[2] = (struct parameter_tuple) { 5, 6 };
return cr_make_param_array(struct parameter_tuple, params, nb_tuples, free_params);
}
ParameterizedTest(struct parameter_tuple *tup, params, cleanup) {
cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d);
}

View file

@ -34,6 +34,7 @@
#include <signal.h>
#ifdef VANILLA_WIN32
# include <tchar.h>
# define CREATE_SUSPENDED_(Filename, CmdLine, StartupInfo, Info) \
CreateProcessW(Filename, \
CmdLine, \
@ -87,8 +88,6 @@ static int get_win_status(HANDLE handle) {
}
#endif
struct worker_context g_worker_context = {.test = NULL};
#ifdef VANILLA_WIN32
@ -99,11 +98,12 @@ struct full_context {
struct criterion_test_extra_data suite_data;
f_worker_func func;
struct pipe_handle pipe;
volatile int resumed;
struct test_single_param param;
HANDLE sync;
DWORD extra_size;
};
static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker");
#define MAPPING_SIZE sizeof (struct full_context)
static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker_%lu");
static struct full_context local_ctx;
#endif
@ -168,10 +168,13 @@ static void CALLBACK handle_child_terminated(PVOID lpParameter,
int resume_child(void) {
#ifdef VANILLA_WIN32
TCHAR mapping_name[128];
_sntprintf(mapping_name, 128, g_mapping_name, GetCurrentProcessId());
HANDLE sharedMem = OpenFileMapping(
FILE_MAP_ALL_ACCESS,
FALSE,
g_mapping_name);
mapping_name);
if (sharedMem == NULL)
return 0;
@ -180,30 +183,57 @@ int resume_child(void) {
FILE_MAP_ALL_ACCESS,
0,
0,
MAPPING_SIZE);
sizeof (struct full_context));
if (ctx == NULL)
if (ctx == NULL) {
CloseHandle(sharedMem);
exit(-1);
}
local_ctx = *ctx;
UnmapViewOfFile(ctx);
struct test_single_param *param = NULL;
if (local_ctx.param.size != 0) {
ctx = (struct full_context*) MapViewOfFile(sharedMem,
FILE_MAP_ALL_ACCESS,
0,
0,
sizeof (struct full_context) + local_ctx.extra_size);
if (ctx == NULL) {
CloseHandle(sharedMem);
exit(-1);
}
param = malloc(sizeof (struct test_single_param) + local_ctx.param.size);
*param = (struct test_single_param) {
.size = local_ctx.param.size,
.ptr = param + 1,
};
memcpy(param + 1, ctx + 1, param->size);
UnmapViewOfFile(ctx);
}
CloseHandle(sharedMem);
g_worker_context = (struct worker_context) {
&local_ctx.test,
&local_ctx.suite,
local_ctx.func,
&local_ctx.pipe
.test = &local_ctx.test,
.suite = &local_ctx.suite,
.func = local_ctx.func,
.pipe = &local_ctx.pipe,
.param = param,
};
local_ctx.test.data = &local_ctx.test_data;
local_ctx.suite.data = &local_ctx.suite_data;
ctx->resumed = 1;
UnmapViewOfFile(ctx);
CloseHandle(sharedMem);
SetEvent(local_ctx.sync);
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
run_worker(&g_worker_context);
free(param);
return 1;
#else
# if defined(__unix__) || defined(__APPLE__)
@ -227,6 +257,16 @@ s_proc_handle *fork_process() {
ZeroMemory(&info, sizeof (info));
SECURITY_ATTRIBUTES inherit_handle = {
.nLength = sizeof (SECURITY_ATTRIBUTES),
.bInheritHandle = TRUE
};
// Create the synchronization event
HANDLE sync = CreateEvent(&inherit_handle, FALSE, FALSE, NULL);
if (sync == NULL)
return (void *) -1;
// Create the suspended child process
wchar_t filename[MAX_PATH];
GetModuleFileNameW(NULL, filename, MAX_PATH);
@ -235,22 +275,29 @@ s_proc_handle *fork_process() {
return (void *) -1;
// Copy context over
TCHAR mapping_name[128];
_sntprintf(mapping_name, 128, g_mapping_name, info.dwProcessId);
DWORD mapping_size = sizeof (struct full_context);
if (g_worker_context.param)
mapping_size += g_worker_context.param->size;
HANDLE sharedMem = CreateFileMapping(
INVALID_HANDLE_VALUE,
NULL,
PAGE_READWRITE,
0,
MAPPING_SIZE,
g_mapping_name);
mapping_size,
mapping_name);
if (sharedMem == NULL)
if (sharedMem == NULL || GetLastError() == ERROR_ALREADY_EXISTS)
return (void *) -1;
struct full_context *ctx = (struct full_context *) MapViewOfFile(sharedMem,
FILE_MAP_ALL_ACCESS,
0,
0,
MAPPING_SIZE);
mapping_size);
if (ctx == NULL) {
CloseHandle(sharedMem);
@ -263,16 +310,28 @@ s_proc_handle *fork_process() {
.suite = *g_worker_context.suite,
.func = g_worker_context.func,
.pipe = *g_worker_context.pipe,
.resumed = 0,
.sync = sync,
};
if (g_worker_context.param) {
ctx->extra_size = g_worker_context.param->size;
ctx->param = *g_worker_context.param;
memcpy(ctx + 1, g_worker_context.param->ptr, g_worker_context.param->size);
}
if (g_worker_context.suite->data)
ctx->suite_data = *g_worker_context.suite->data;
ResumeThread(info.hThread);
CloseHandle(info.hThread);
if (ResumeThread(info.hThread) == (DWORD) -1)
goto failure;
while (!ctx->resumed); // wait until the child has initialized itself
// wait until the child has initialized itself
HANDLE handles[] = { info.hProcess, sync };
DWORD wres = WaitForMultipleObjects(sizeof (handles) / sizeof (HANDLE), handles, FALSE, INFINITE);
if (wres == WAIT_OBJECT_0)
goto failure;
CloseHandle(info.hThread);
UnmapViewOfFile(ctx);
CloseHandle(sharedMem);
@ -293,6 +352,14 @@ s_proc_handle *fork_process() {
s_proc_handle *handle = smalloc(sizeof (s_proc_handle));
*handle = (s_proc_handle) { info.hProcess };
return handle;
failure:
UnmapViewOfFile(ctx);
CloseHandle(sharedMem);
CloseHandle(info.hThread);
CloseHandle(info.hProcess);
return (void *) -1;
#else
pid_t pid = fork();
if (pid == -1)

View file

@ -42,6 +42,7 @@ struct worker_context {
struct criterion_suite *suite;
f_worker_func func;
struct pipe_handle *pipe;
struct test_single_param *param;
};
extern struct worker_context g_worker_context;

View file

@ -186,7 +186,14 @@ static void run_test_child(struct criterion_test *test,
struct timespec_compat ts;
if (!setjmp(g_pre_test)) {
timer_start(&ts);
(test->test ? test->test : nothing)();
if (test->test) {
if (!test->data->param_) {
test->test();
} else {
void(*param_test_func)(void *) = (void(*)(void*)) test->test;
param_test_func(g_worker_context.param->ptr);
}
}
}
double elapsed_time;
@ -297,7 +304,8 @@ static void handle_worker_terminated(struct event *ev,
static void run_test(struct criterion_global_stats *stats,
struct criterion_suite_stats *suite_stats,
struct criterion_test *test,
struct criterion_suite *suite) {
struct criterion_suite *suite,
struct test_single_param *param) {
struct criterion_test_stats *test_stats = test_stats_init(test);
struct process *proc = NULL;
@ -310,7 +318,7 @@ static void run_test(struct criterion_global_stats *stats,
goto cleanup;
}
proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe);
proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe, param);
if (proc == NULL && !is_runner())
goto cleanup;
@ -370,6 +378,45 @@ cleanup:
sfree(proc);
}
static void run_test_param(struct criterion_global_stats *stats,
struct criterion_suite_stats *suite_stats,
struct criterion_test *test,
struct criterion_suite *suite) {
if (!test->data->param_)
return;
struct criterion_test_params params = test->data->param_();
for (size_t i = 0; i < params.length; ++i) {
struct test_single_param param = { params.size, (char *) params.params + i * params.size };
run_test(stats, suite_stats, test, suite, &param);
if (criterion_options.fail_fast && stats->tests_failed > 0)
break;
if (!is_runner())
break;
}
if (params.cleanup)
params.cleanup(&params);
}
static void run_test_switch(struct criterion_global_stats *stats,
struct criterion_suite_stats *suite_stats,
struct criterion_test *test,
struct criterion_suite *suite) {
switch (test->data->kind_) {
case CR_TEST_NORMAL:
run_test(stats, suite_stats, test, suite, NULL);
break;
case CR_TEST_PARAMETERIZED:
run_test_param(stats, suite_stats, test, suite);
break;
default: break;
}
}
#ifdef HAVE_PCRE
void disable_unmatching(struct criterion_test_set *set) {
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
@ -414,7 +461,7 @@ static int criterion_run_all_tests_impl(struct criterion_test_set *set) {
abort();
struct criterion_global_stats *stats = stats_init();
map_tests(set, stats, run_test);
map_tests(set, stats, run_test_switch);
int result = is_runner() ? stats->tests_failed == 0 : -1;

View file

@ -77,19 +77,21 @@ void run_worker(struct worker_context *ctx) {
struct process *spawn_test_worker(struct criterion_test *test,
struct criterion_suite *suite,
f_worker_func func,
s_pipe_handle *pipe) {
s_pipe_handle *pipe,
struct test_single_param *param) {
g_worker_context = (struct worker_context) {
.test = test,
.suite = suite,
.func = func,
.pipe = pipe
.pipe = pipe,
.param = param,
};
struct process *ptr = NULL;
s_proc_handle *proc = fork_process();
if (proc == (void *) -1) {
return NULL;
abort();
} else if (proc == NULL) {
run_worker(&g_worker_context);
return NULL;

View file

@ -47,6 +47,11 @@ struct worker_status {
struct process_status status;
};
struct test_single_param {
size_t size;
void *ptr;
};
void run_worker(struct worker_context *ctx);
void set_runner_process(void);
void unset_runner_process(void);
@ -56,7 +61,8 @@ struct process_status get_status(int status);
struct process *spawn_test_worker(struct criterion_test *test,
struct criterion_suite *suite,
f_worker_func func,
s_pipe_handle *pipe);
s_pipe_handle *pipe,
struct test_single_param *param);
struct event *worker_read_event(struct process *proc);
#endif /* !PROCESS_H_ */