Merge branch 'features/parameterized-tests' into bleeding

doc/index.rst

@@ -10,6 +10,7 @@ Criterion
     assert
     hooks
     env
+    parameterized
     theories
     internal
     faq

doc/parameterized.rst

@@ -0,0 +1,110 @@
+Using parameterized tests
+=========================
+
+Parameterized tests are useful to repeat a specific test logic over a finite
+set of parameters.
+
+Due to limitations on how generated parameters are passed, parameterized tests
+can only accept one pointer parameter; however, this is not that much of a
+problem since you can just pass a structure containing the context you need.
+
+Adding parameterized tests
+--------------------------
+
+Adding parameterized tests is done by defining the parameterized test function,
+and the parameter generator function:
+
+.. code-block:: c
+
+    #include <criterion/parameterized.h>
+
+    ParameterizedTestParameter(suite_name, test_name) = {
+        void *params;
+        size_t nb_params;
+
+        // generate parameter set
+        return cr_make_param_array(Type, params, nb_params);
+    }
+
+    ParameterizedTest(Type *param, suite_name, test_name) {
+        // contents of the test
+    }
+
+``suite_name`` and ``test_name`` are the identifiers of the test suite and
+the test, respectively. These identifiers must follow the language
+identifier format.
+
+``Type`` is the compound type of the generated array. ``params`` and ``nb_params``
+are the pointer and the length of the generated array, respectively.
+
+Passing multiple parameters
+---------------------------
+
+As said earlier, parameterized tests only take one parameter, so passing
+multiple parameters is, in the strict sense, not possible. However, one can
+easily use a struct to hold the context as a workaround:
+
+.. code-block:: c
+
+    #include <criterion/parameterized.h>
+
+    struct my_params {
+        int param0;
+        double param1;
+        ...
+    };
+
+    ParameterizedTestParameter(suite_name, test_name) = {
+        size_t nb_params = 32;
+        struct my_params *params = malloc(sizeof (struct my_params) * nb_params);
+
+        // generate parameter set
+
+        params[0] = ...
+        params[1] = ...
+
+        ...
+
+        return cr_make_param_array(struct my_params, params, nb_params);
+    }
+
+    ParameterizedTest(struct my_params *param, suite_name, test_name) {
+        // access param.param0, param.param1, ...
+    }
+
+There is, however, one absolute rule that must be respected, unless you don't
+want your tests to run on windows, ever: parameters must not contain any
+pointer to dynamically allocated data.
+
+Hence, this is not permitted:
+
+.. code-block:: c
+
+    #include <criterion/parameterized.h>
+
+    struct my_params {
+        int *some_int_ptr;
+    };
+
+    ParameterizedTestParameter(suite_name, test_name) = {
+        static my_params param = {
+            .some_int_ptr = malloc(sizeof (int)); // Don't do this.
+        };
+        *param.some_int_ptr = 42;
+
+        return cr_make_param_array(struct my_params, &param, 1);
+    }
+
+and **will crash the test** on Windows.
+
+Configuring parameterized tests
+-------------------------------
+
+Parameterized tests can optionally recieve configuration parameters to alter
+their own behaviour, and are applied to each iteration of the parameterized
+test individually (this means that the initialization and finalization runs once
+per iteration).
+Those parameters are the same ones as the ones of the ``Test`` macro function
+(c.f. :ref:`test-config-ref`).
+
+

include/criterion/criterion.h

@@ -48,6 +48,8 @@
     TEST_PROTOTYPE_(Category, Name);                                           \
     struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) =      \
         CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data,      \
+            .kind_ = CR_TEST_NORMAL,                                           \
+            .param_ = (struct criterion_test_params(*)(void)) NULL,            \
             .identifier_ = #Category "/" #Name,                                \
             .file_    = __FILE__,                                              \
             .line_    = __LINE__,                                              \

include/criterion/parameterized.h

@@ -0,0 +1,50 @@
+#ifndef CRITERION_PARAMETERIZED_H_
+# define CRITERION_PARAMETERIZED_H_
+
+# include "criterion.h"
+
+# ifdef __cplusplus
+#  define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \
+    extern "C" void IDENTIFIER_(Category, Name, impl)(Param)
+# else
+#  define CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name) \
+    void IDENTIFIER_(Category, Name, impl)(Param)
+# endif
+
+# define ParameterizedTest(...) \
+    CR_EXPAND(ParameterizedTest_(__VA_ARGS__, .sentinel_ = 0))
+
+# define ParameterizedTest_(Param, Category, Name, ...)                        \
+    CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name);                           \
+    struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) =      \
+        CR_EXPAND(CRITERION_MAKE_STRUCT(struct criterion_test_extra_data,      \
+            .kind_ = CR_TEST_PARAMETERIZED,                                    \
+            .param_ = IDENTIFIER_(Category, Name, param),                      \
+            .identifier_ = #Category "/" #Name,                                \
+            .file_    = __FILE__,                                              \
+            .line_    = __LINE__,                                              \
+            __VA_ARGS__                                                        \
+        ));                                                                    \
+    struct criterion_test IDENTIFIER_(Category, Name, meta) = {                \
+        #Name,                                                                 \
+        #Category,                                                             \
+        (void(*)(void)) IDENTIFIER_(Category, Name, impl),                     \
+        &IDENTIFIER_(Category, Name, extra)                                    \
+    };                                                                         \
+    SECTION_("cr_tst")                                                         \
+    struct criterion_test *IDENTIFIER_(Category, Name, ptr)                    \
+            = &IDENTIFIER_(Category, Name, meta) SECTION_SUFFIX_;              \
+    CR_PARAM_TEST_PROTOTYPE_(Param, Category, Name)
+
+# define ParameterizedTestParameters(Category, Name) \
+    static struct criterion_test_params IDENTIFIER_(Category, Name, param)(void)
+
+# ifdef __cplusplus
+#  define cr_make_param_array(Type, Array, ...) \
+    criterion_test_params(sizeof (Type), (Array), __VA_ARGS__)
+# else
+#  define cr_make_param_array(Type, Array, ...) \
+    (struct criterion_test_params) { .size = sizeof (Type), (void*)(Array), __VA_ARGS__ }
+# endif
+
+#endif /* !CRITERION_PARAMETERIZED_H_ */

include/criterion/types.h

@@ -33,8 +33,39 @@ using std::size_t;
 # endif
 # include "common.h"
 
+enum criterion_test_kind {
+    CR_TEST_NORMAL,
+    CR_TEST_PARAMETERIZED,
+};
+
+struct criterion_test_params {
+    size_t size;
+    void *params;
+    size_t length;
+    void (*cleanup)(struct criterion_test_params *);
+
+# ifdef __cplusplus
+    constexpr criterion_test_params(size_t size, void *params, size_t length)
+        : size(size)
+        , params(params)
+        , length(length)
+        , cleanup(nullptr)
+    {}
+
+    constexpr criterion_test_params(size_t size, void *params, size_t length,
+            void (*cleanup)(struct criterion_test_params *))
+        : size(size)
+        , params(params)
+        , length(length)
+        , cleanup(cleanup)
+    {}
+# endif
+};
+
 struct criterion_test_extra_data {
     int sentinel_;
+    enum criterion_test_kind kind_;
+    struct criterion_test_params (*param_)(void);
     const char *identifier_;
     const char *file_;
     unsigned line_;

samples/CMakeLists.txt

@@ -17,6 +17,7 @@ set(SAMPLES
   theories.c
   timeout.c
   redirect.c
+  parameterized.c
 
   signal.cc
   report.cc

samples/outputs/parameterized.c.bin.err.expected

@@ -0,0 +1,19 @@
+[----] parameterized.c:60: Assertion failed: Parameters: (1, 2.000000)
+[FAIL] params::cleanup: (0.00s)
+[----] parameterized.c:60: Assertion failed: Parameters: (3, 4.000000)
+[FAIL] params::cleanup: (0.00s)
+[----] parameterized.c:60: Assertion failed: Parameters: (5, 6.000000)
+[FAIL] params::cleanup: (0.00s)
+[----] parameterized.c:36: Assertion failed: Parameters: (1, 2.000000)
+[FAIL] params::multiple: (0.00s)
+[----] parameterized.c:36: Assertion failed: Parameters: (3, 4.000000)
+[FAIL] params::multiple: (0.00s)
+[----] parameterized.c:36: Assertion failed: Parameters: (5, 6.000000)
+[FAIL] params::multiple: (0.00s)
+[----] parameterized.c:15: Assertion failed: Parameter: foo
+[FAIL] params::str: (0.00s)
+[----] parameterized.c:15: Assertion failed: Parameter: bar
+[FAIL] params::str: (0.00s)
+[----] parameterized.c:15: Assertion failed: Parameter: baz
+[FAIL] params::str: (0.00s)
+[====] Synthesis: Tested: 9 | Passing: 0 | Failing: 9 | Crashing: 0 

samples/parameterized.c

@@ -0,0 +1,61 @@
+#include <criterion/parameterized.h>
+#include <stdio.h>
+
+// Basic usage
+
+ParameterizedTestParameters(params, str) {
+    static const char *strings[] = {
+        "foo", "bar", "baz"
+    };
+
+    return cr_make_param_array(const char *, strings, sizeof (strings) / sizeof (const char *));
+}
+
+ParameterizedTest(const char **str, params, str) {
+    cr_assert_fail("Parameter: %s", *str);
+}
+
+// Multiple parameters must be coalesced in a single parameter
+
+struct parameter_tuple {
+    int i;
+    double d;
+};
+
+ParameterizedTestParameters(params, multiple) {
+    static struct parameter_tuple params[] = {
+        {1, 2},
+        {3, 4},
+        {5, 6},
+    };
+
+    return cr_make_param_array(struct parameter_tuple, params, sizeof (params) / sizeof (struct parameter_tuple));
+}
+
+ParameterizedTest(struct parameter_tuple *tup, params, multiple) {
+    cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d);
+}
+
+// Cleaning up dynamically generated parameters
+
+// Note: Do **NOT** embed dynamically allocated pointers inside of structures
+// or this will fail on windows
+
+void free_params(struct criterion_test_params *crp) {
+    free(crp->params);
+}
+
+ParameterizedTestParameters(params, cleanup) {
+    const size_t nb_tuples = 3;
+
+    struct parameter_tuple *params = malloc(sizeof(struct parameter_tuple) * nb_tuples);
+    params[0] = (struct parameter_tuple) { 1, 2 };
+    params[1] = (struct parameter_tuple) { 3, 4 };
+    params[2] = (struct parameter_tuple) { 5, 6 };
+
+    return cr_make_param_array(struct parameter_tuple, params, nb_tuples, free_params);
+}
+
+ParameterizedTest(struct parameter_tuple *tup, params, cleanup) {
+    cr_assert_fail("Parameters: (%d, %f)", tup->i, tup->d);
+}

src/compat/process.c

@@ -34,6 +34,7 @@
 #include <signal.h>
 
 #ifdef VANILLA_WIN32
+# include <tchar.h>
 # define CREATE_SUSPENDED_(Filename, CmdLine, StartupInfo, Info)    \
     CreateProcessW(Filename,                                        \
             CmdLine,                                                \
@@ -87,8 +88,6 @@ static int get_win_status(HANDLE handle) {
 }
 #endif
 
-
-
 struct worker_context g_worker_context = {.test = NULL};
 
 #ifdef VANILLA_WIN32
@@ -99,11 +98,12 @@ struct full_context {
     struct criterion_test_extra_data suite_data;
     f_worker_func func;
     struct pipe_handle pipe;
-    volatile int resumed;
+    struct test_single_param param;
+    HANDLE sync;
+    DWORD extra_size;
 };
 
-static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker");
-#define MAPPING_SIZE sizeof (struct full_context)
+static TCHAR g_mapping_name[] = TEXT("WinCriterionWorker_%lu");
 
 static struct full_context local_ctx;
 #endif
@@ -168,10 +168,13 @@ static void CALLBACK handle_child_terminated(PVOID lpParameter,
 
 int resume_child(void) {
 #ifdef VANILLA_WIN32
+    TCHAR mapping_name[128];
+    _sntprintf(mapping_name, 128, g_mapping_name, GetCurrentProcessId());
+
     HANDLE sharedMem = OpenFileMapping(
            FILE_MAP_ALL_ACCESS,
            FALSE,
-           g_mapping_name);
+           mapping_name);
 
     if (sharedMem == NULL)
         return 0;
@@ -180,30 +183,57 @@ int resume_child(void) {
            FILE_MAP_ALL_ACCESS,
            0,
            0,
-           MAPPING_SIZE);
+           sizeof (struct full_context));
 
-    if (ctx == NULL)
+    if (ctx == NULL) {
+        CloseHandle(sharedMem);
         exit(-1);
+    }
 
     local_ctx = *ctx;
+    UnmapViewOfFile(ctx);
+
+    struct test_single_param *param = NULL;
+    if (local_ctx.param.size != 0) {
+        ctx = (struct full_context*) MapViewOfFile(sharedMem,
+               FILE_MAP_ALL_ACCESS,
+               0,
+               0,
+               sizeof (struct full_context) + local_ctx.extra_size);
+
+        if (ctx == NULL) {
+            CloseHandle(sharedMem);
+            exit(-1);
+        }
+
+        param = malloc(sizeof (struct test_single_param) + local_ctx.param.size);
+        *param = (struct test_single_param) {
+            .size = local_ctx.param.size,
+            .ptr = param + 1,
+        };
+        memcpy(param + 1, ctx + 1, param->size);
+        UnmapViewOfFile(ctx);
+    }
+
+    CloseHandle(sharedMem);
+
     g_worker_context = (struct worker_context) {
-        &local_ctx.test,
-        &local_ctx.suite,
-        local_ctx.func,
-        &local_ctx.pipe
+        .test = &local_ctx.test,
+        .suite = &local_ctx.suite,
+        .func = local_ctx.func,
+        .pipe = &local_ctx.pipe,
+        .param = param,
     };
 
     local_ctx.test.data  = &local_ctx.test_data;
     local_ctx.suite.data = &local_ctx.suite_data;
 
-    ctx->resumed = 1;
-
-    UnmapViewOfFile(ctx);
-    CloseHandle(sharedMem);
+    SetEvent(local_ctx.sync);
 
     SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
 
     run_worker(&g_worker_context);
+    free(param);
     return 1;
 #else
 # if defined(__unix__) || defined(__APPLE__)
@@ -227,6 +257,16 @@ s_proc_handle *fork_process() {
 
     ZeroMemory(&info, sizeof (info));
 
+    SECURITY_ATTRIBUTES inherit_handle = {
+        .nLength = sizeof (SECURITY_ATTRIBUTES),
+        .bInheritHandle = TRUE
+    };
+
+    // Create the synchronization event
+    HANDLE sync = CreateEvent(&inherit_handle, FALSE, FALSE, NULL);
+    if (sync == NULL)
+        return (void *) -1;
+
     // Create the suspended child process
     wchar_t filename[MAX_PATH];
     GetModuleFileNameW(NULL, filename, MAX_PATH);
@@ -235,22 +275,29 @@ s_proc_handle *fork_process() {
         return (void *) -1;
 
     // Copy context over
+    TCHAR mapping_name[128];
+    _sntprintf(mapping_name, 128, g_mapping_name, info.dwProcessId);
+
+    DWORD mapping_size = sizeof (struct full_context);
+    if (g_worker_context.param)
+        mapping_size += g_worker_context.param->size;
+
     HANDLE sharedMem = CreateFileMapping(
             INVALID_HANDLE_VALUE,
             NULL,
             PAGE_READWRITE,
             0,
-            MAPPING_SIZE,
-            g_mapping_name);
+            mapping_size,
+            mapping_name);
 
-    if (sharedMem == NULL)
+    if (sharedMem == NULL || GetLastError() == ERROR_ALREADY_EXISTS)
         return (void *) -1;
 
     struct full_context *ctx = (struct full_context *) MapViewOfFile(sharedMem,
             FILE_MAP_ALL_ACCESS,
             0,
             0,
-            MAPPING_SIZE);
+            mapping_size);
 
     if (ctx == NULL) {
         CloseHandle(sharedMem);
@@ -263,16 +310,28 @@ s_proc_handle *fork_process() {
         .suite     = *g_worker_context.suite,
         .func      = g_worker_context.func,
         .pipe      = *g_worker_context.pipe,
-        .resumed   = 0,
+        .sync      = sync,
     };
 
+    if (g_worker_context.param) {
+        ctx->extra_size = g_worker_context.param->size;
+        ctx->param = *g_worker_context.param;
+        memcpy(ctx + 1, g_worker_context.param->ptr, g_worker_context.param->size);
+    }
+
     if (g_worker_context.suite->data)
         ctx->suite_data = *g_worker_context.suite->data;
 
-    ResumeThread(info.hThread);
-    CloseHandle(info.hThread);
+    if (ResumeThread(info.hThread) == (DWORD) -1)
+        goto failure;
 
-    while (!ctx->resumed); // wait until the child has initialized itself
+    // wait until the child has initialized itself
+    HANDLE handles[] = { info.hProcess, sync };
+    DWORD wres = WaitForMultipleObjects(sizeof (handles) / sizeof (HANDLE), handles, FALSE, INFINITE);
+    if (wres == WAIT_OBJECT_0)
+        goto failure;
+
+    CloseHandle(info.hThread);
 
     UnmapViewOfFile(ctx);
     CloseHandle(sharedMem);
@@ -293,6 +352,14 @@ s_proc_handle *fork_process() {
     s_proc_handle *handle = smalloc(sizeof (s_proc_handle));
     *handle = (s_proc_handle) { info.hProcess };
     return handle;
+
+failure:
+    UnmapViewOfFile(ctx);
+    CloseHandle(sharedMem);
+    CloseHandle(info.hThread);
+    CloseHandle(info.hProcess);
+    return (void *) -1;
+
 #else
     pid_t pid = fork();
     if (pid == -1)

src/compat/process.h

@@ -42,6 +42,7 @@ struct worker_context {
     struct criterion_suite *suite;
     f_worker_func func;
     struct pipe_handle *pipe;
+    struct test_single_param *param;
 };
 
 extern struct worker_context g_worker_context;

src/core/runner.c

@@ -186,7 +186,14 @@ static void run_test_child(struct criterion_test *test,
     struct timespec_compat ts;
     if (!setjmp(g_pre_test)) {
         timer_start(&ts);
-        (test->test ? test->test : nothing)();
+        if (test->test) {
+            if (!test->data->param_) {
+                test->test();
+            } else {
+                void(*param_test_func)(void *) = (void(*)(void*)) test->test;
+                param_test_func(g_worker_context.param->ptr);
+            }
+        }
     }
 
     double elapsed_time;
@@ -297,7 +304,8 @@ static void handle_worker_terminated(struct event *ev,
 static void run_test(struct criterion_global_stats *stats,
         struct criterion_suite_stats *suite_stats,
         struct criterion_test *test,
-        struct criterion_suite *suite) {
+        struct criterion_suite *suite,
+        struct test_single_param *param) {
 
     struct criterion_test_stats *test_stats = test_stats_init(test);
     struct process *proc = NULL;
@@ -310,7 +318,7 @@ static void run_test(struct criterion_global_stats *stats,
         goto cleanup;
     }
 
-    proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe);
+    proc = spawn_test_worker(test, suite, run_test_child, g_worker_pipe, param);
     if (proc == NULL && !is_runner())
         goto cleanup;
 
@@ -370,6 +378,45 @@ cleanup:
     sfree(proc);
 }
 
+static void run_test_param(struct criterion_global_stats *stats,
+        struct criterion_suite_stats *suite_stats,
+        struct criterion_test *test,
+        struct criterion_suite *suite) {
+
+    if (!test->data->param_)
+        return;
+
+    struct criterion_test_params params = test->data->param_();
+    for (size_t i = 0; i < params.length; ++i) {
+        struct test_single_param param = { params.size, (char *) params.params + i * params.size };
+
+        run_test(stats, suite_stats, test, suite, &param);
+        if (criterion_options.fail_fast && stats->tests_failed > 0)
+            break;
+        if (!is_runner())
+            break;
+    }
+
+    if (params.cleanup)
+        params.cleanup(&params);
+}
+
+static void run_test_switch(struct criterion_global_stats *stats,
+        struct criterion_suite_stats *suite_stats,
+        struct criterion_test *test,
+        struct criterion_suite *suite) {
+
+    switch (test->data->kind_) {
+        case CR_TEST_NORMAL:
+            run_test(stats, suite_stats, test, suite, NULL);
+            break;
+        case CR_TEST_PARAMETERIZED:
+            run_test_param(stats, suite_stats, test, suite);
+            break;
+        default: break;
+    }
+}
+
 #ifdef HAVE_PCRE
 void disable_unmatching(struct criterion_test_set *set) {
     FOREACH_SET(struct criterion_suite_set *s, set->suites) {
@@ -414,7 +461,7 @@ static int criterion_run_all_tests_impl(struct criterion_test_set *set) {
         abort();
 
     struct criterion_global_stats *stats = stats_init();
-    map_tests(set, stats, run_test);
+    map_tests(set, stats, run_test_switch);
 
     int result = is_runner() ? stats->tests_failed == 0 : -1;
 

src/core/worker.c

@@ -77,19 +77,21 @@ void run_worker(struct worker_context *ctx) {
 struct process *spawn_test_worker(struct criterion_test *test,
                                   struct criterion_suite *suite,
                                   f_worker_func func,
-                                  s_pipe_handle *pipe) {
+                                  s_pipe_handle *pipe,
+                                  struct test_single_param *param) {
     g_worker_context = (struct worker_context) {
         .test = test,
         .suite = suite,
         .func = func,
-        .pipe = pipe
+        .pipe = pipe,
+        .param = param,
     };
 
     struct process *ptr = NULL;
 
     s_proc_handle *proc = fork_process();
     if (proc == (void *) -1) {
-        return NULL;
+        abort();
     } else if (proc == NULL) {
         run_worker(&g_worker_context);
         return NULL;

src/core/worker.h

@@ -47,6 +47,11 @@ struct worker_status {
     struct process_status status;
 };
 
+struct test_single_param {
+    size_t size;
+    void *ptr;
+};
+
 void run_worker(struct worker_context *ctx);
 void set_runner_process(void);
 void unset_runner_process(void);
@@ -56,7 +61,8 @@ struct process_status get_status(int status);
 struct process *spawn_test_worker(struct criterion_test *test,
                                   struct criterion_suite *suite,
                                   f_worker_func func,
-                                  s_pipe_handle *pipe);
+                                  s_pipe_handle *pipe,
+                                  struct test_single_param *param);
 struct event *worker_read_event(struct process *proc);
 
 #endif /* !PROCESS_H_ */