Added timeout code for tests

include/criterion/logging.h

@@ -104,6 +104,7 @@ struct criterion_output_provider {
     void (*log_pre_test     )(struct criterion_test *test);
     void (*log_assert       )(struct criterion_assert_stats *stats);
     void (*log_theory_fail  )(struct criterion_theory_stats *stats);
+    void (*log_test_timeout )(struct criterion_test_stats *stats);
     void (*log_test_crash   )(struct criterion_test_stats *stats);
     void (*log_other_crash  )(struct criterion_test_stats *stats);
     void (*log_abnormal_exit)(struct criterion_test_stats *stats);

include/criterion/stats.h

@@ -44,6 +44,7 @@ struct criterion_test_stats {
     int signal;
     int exit_code;
     float elapsed_time;
+    bool timed_out;
     unsigned progress;
     const char *file;
 

include/criterion/types.h

@@ -44,6 +44,7 @@ struct criterion_test_extra_data {
     int exit_code;
     bool disabled;
     const char *description;
+    float timeout;
     void *data;
 };
 

po/fr.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: criterion 1.0.0\n"
 "Report-Msgid-Bugs-To: franklinmathieu+criterion@gmail.com\n"
-"POT-Creation-Date: 2015-09-10 03:35+0200\n"
+"POT-Creation-Date: 2015-09-11 16:47+0200\n"
 "PO-Revision-Date: 2015-04-03 17:58+0200\n"
 "Last-Translator:  <franklinmathieu@gmail.com>\n"
 "Language-Team: French\n"
@@ -60,18 +60,23 @@ msgstr ""
 "  La théorie %1$s::%2$s a échoué avec les paramètres suivants: (%3$s)\n"
 
 #: src/log/normal.c:61
+#, fuzzy, c-format
+msgid "%1$s::%2$s: Timed out.\n"
+msgstr "%1$s::%2$s: Délai expiré.\n"
+
+#: src/log/normal.c:62
 #, c-format
 msgid "%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n"
 msgstr ""
 "%1$s%2$s%3$s:%4$s%5$u%6$s: Un signal inattendu a été reçu après cette "
 "ligne!\n"
 
-#: src/log/normal.c:62
+#: src/log/normal.c:63
 #, c-format
 msgid "%1$s::%2$s: CRASH!\n"
 msgstr "%1$s::%2$s: PLANTAGE!\n"
 
-#: src/log/normal.c:63
+#: src/log/normal.c:64
 #, fuzzy, c-format
 msgid ""
 "%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown."
@@ -80,7 +85,7 @@ msgstr ""
 "%1$sAttention! Le test `%2$s::%3$s` a planté pendant son initialisation ou "
 "sa finalisation.%4$s\n"
 
-#: src/log/normal.c:64
+#: src/log/normal.c:65
 #, fuzzy, c-format
 msgid ""
 "%1$sWarning! The test `%2$s::%3$s` exited during its setup or teardown.%4$s\n"
@@ -88,14 +93,14 @@ msgstr ""
 "%1$sAttention! Le test `%2$s::%3$s` a quitté pendant son initialisation ou "
 "sa finalisation.%4$s\n"
 
-#: src/log/normal.c:65
+#: src/log/normal.c:66
 #, c-format
 msgid "Running %1$s%2$lu%3$s test from %4$s%5$s%6$s:\n"
 msgid_plural "Running %1$s%2$lu%3$s tests from %4$s%5$s%6$s:\n"
 msgstr[0] "Lancement de %1$s%2$lu%3$s test dans %4$s%5$s%6$s:\n"
 msgstr[1] "Lancement de %1$s%2$lu%3$s tests dans %4$s%5$s%6$s:\n"
 
-#: src/log/normal.c:67
+#: src/log/normal.c:68
 #, c-format
 msgid ""
 "%1$sSynthesis: Tested: %2$s%3$lu%4$s | Passing: %5$s%6$lu%7$s | Failing: %8$s"

samples/CMakeLists.txt

@@ -15,6 +15,7 @@ set(SAMPLES
   description.c
   simple.c
   theories.c
+  timeout.c
 
   signal.cc
   report.cc

samples/timeout.c

@@ -0,0 +1,12 @@
+#include <criterion/criterion.h>
+
+#ifdef _WIN32
+# include <Windows.h>
+# define sleep(x) Sleep(x * 1000)
+#else
+# include <unistd.h>
+#endif
+
+Test(timeout, simple, .timeout = 1.) {
+    sleep(10);
+}

src/log/normal.c

@@ -58,6 +58,7 @@ static msg_t msg_post_suite_test = N_("%1$s::%2$s: Test is disabled\n");
 static msg_t msg_post_suite_suite = N_("%1$s::%2$s: Suite is disabled\n");
 static msg_t msg_assert_fail = N_("%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n");
 static msg_t msg_theory_fail = N_("  Theory %1$s::%2$s failed with the following parameters: (%3$s)\n");
+static msg_t msg_test_timeout = N_("%1$s::%2$s: Timed out.\n");
 static msg_t msg_test_crash_line = N_("%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n");
 static msg_t msg_test_crash = N_("%1$s::%2$s: CRASH!\n");
 static msg_t msg_test_other_crash = N_("%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown.%4$s\n");
@@ -77,6 +78,7 @@ static msg_t msg_post_suite_test = "%s::%s: Test is disabled\n";
 static msg_t msg_post_suite_suite = "%s::%s: Suite is disabled\n";
 static msg_t msg_assert_fail = "%s%s%s:%s%d%s: Assertion failed: %s\n";
 static msg_t msg_theory_fail = "  Theory %s::%s failed with the following parameters: %s\n";
+static msg_t msg_test_timeout = "%s::%s: timed out.\n";
 static msg_t msg_test_crash_line = "%s%s%s:%s%u%s: Unexpected signal caught below this line!\n";
 static msg_t msg_test_crash = "%s::%s: CRASH!\n";
 static msg_t msg_test_other_crash = "%sWarning! The test `%s::%s` crashed during its setup or teardown.%s\n";
@@ -220,12 +222,20 @@ void normal_log_theory_fail(struct criterion_theory_stats *stats) {
             stats->formatted_args);
 }
 
+void normal_log_test_timeout(UNUSED struct criterion_test_stats *stats) {
+    criterion_pimportant(CRITERION_PREFIX_FAIL,
+            _(msg_test_timeout),
+            stats->test->category,
+            stats->test->name);
+}
+
 struct criterion_output_provider normal_logging = {
     .log_pre_all        = normal_log_pre_all,
     .log_pre_init       = normal_log_pre_init,
     .log_pre_suite      = normal_log_pre_suite,
     .log_assert         = normal_log_assert,
     .log_theory_fail    = normal_log_theory_fail,
+    .log_test_timeout   = normal_log_test_timeout,
     .log_test_crash     = normal_log_test_crash,
     .log_other_crash    = normal_log_other_crash,
     .log_abnormal_exit  = normal_log_abnormal_exit,

src/log/tap.c

@@ -118,10 +118,17 @@ void tap_log_test_crash(struct criterion_test_stats *stats) {
             stats->progress);
 }
 
+void tap_log_test_timeout(struct criterion_test_stats *stats) {
+    criterion_important("not ok - %s::%s timed out\n",
+            stats->test->category,
+            stats->test->name);
+}
+
 struct criterion_output_provider tap_logging = {
-    .log_pre_all    = tap_log_pre_all,
-    .log_pre_suite  = tap_log_pre_suite,
-    .log_test_crash = tap_log_test_crash,
-    .log_post_test  = tap_log_post_test,
-    .log_post_suite = tap_log_post_suite,
+    .log_pre_all        = tap_log_pre_all,
+    .log_pre_suite      = tap_log_pre_suite,
+    .log_test_crash     = tap_log_test_crash,
+    .log_test_timeout   = tap_log_test_timeout,
+    .log_post_test      = tap_log_post_test,
+    .log_post_suite     = tap_log_post_suite,
 };

src/posix-compat.c

@@ -52,9 +52,6 @@
         WriteProcessMemory(Proc, &What, &What, Size, NULL);
 
 # include <signal.h>
-# ifndef SIGALRM
-#  define SIGALRM 14
-# endif
 
 #else
 # include <unistd.h>
@@ -240,7 +237,7 @@ void wait_process(s_proc_handle *handle, int *status) {
         case STATUS_PRIVILEGED_INSTRUCTION:
         case STATUS_NONCONTINUABLE_EXCEPTION:   sig = SIGILL; break;
 
-        case STATUS_TIMEOUT:                    sig = SIGALRM; break;
+        case CR_EXCEPTION_TIMEOUT:              sig = SIGPROF; break;
 
         case STATUS_ACCESS_VIOLATION:
         case STATUS_DATATYPE_MISALIGNMENT:

src/posix-compat.h

@@ -43,6 +43,9 @@
 #  define WTERMSIG(Status)    ((Status) & 0x7F)
 #  define WIFEXITED(Status)   (WTERMSIG(Status) == 0)
 #  define WIFSIGNALED(Status) (((signed char) (WTERMSIG(Status) + 1) >> 1) > 0)
+
+#  define SIGPROF 27
+#  define CR_EXCEPTION_TIMEOUT 0xC0001042
 # else
 #  include <sys/wait.h>
 # endif

src/runner.c

@@ -172,6 +172,11 @@ static void map_tests(struct criterion_test_set *set,
 static void run_test_child(struct criterion_test *test,
                            struct criterion_suite *suite) {
 
+    if (suite->data && suite->data->timeout != 0 && test->data->timeout == 0)
+        setup_timeout((uint64_t) (suite->data->timeout * 1e9));
+    else if (test->data->timeout != 0)
+        setup_timeout((uint64_t) (test->data->timeout * 1e9));
+
     send_event(PRE_INIT, NULL, 0);
     if (suite->data)
         (suite->data->init ? suite->data->init : nothing)();
@@ -274,6 +279,17 @@ static void run_test(struct criterion_global_stats *stats,
 
     struct process_status status = wait_proc(proc);
     if (status.kind == SIGNAL) {
+        if (status.status == SIGPROF) {
+            test_stats->timed_out = true;
+            double elapsed_time = test->data->timeout;
+            if (elapsed_time == 0 && suite->data)
+                elapsed_time = suite->data->timeout;
+            push_event(POST_TEST, .data = &elapsed_time);
+            push_event(POST_FINI);
+            log(test_timeout, test_stats);
+            goto cleanup;
+        }
+
         if (normal_finish || !test_started) {
             log(other_crash, test_stats);
             if (!test_started) {

src/stats.c

@@ -209,15 +209,20 @@ static void push_post_test(s_glob_stats *stats,
 
     test->elapsed_time = (float) *data;
     if (test->failed_asserts > 0
+            || test->timed_out
             || test->signal != test->test->data->signal
             || test->exit_code != test->test->data->exit_code) {
         test->failed = 1;
+    }
+
+    if (test->failed) {
         ++stats->tests_failed;
         ++suite->tests_failed;
     } else {
         ++stats->tests_passed;
         ++suite->tests_passed;
     }
+
 }
 
 static void push_test_crash(s_glob_stats *stats,

src/timer.c

@@ -1,6 +1,9 @@
 #include <errno.h>
 #include <inttypes.h>
+#include <stdlib.h>
 #include "timer.h"
+#include "criterion/common.h"
+#include "posix-compat.h"
 
 #define GIGA 1000000000
 
@@ -17,6 +20,7 @@ extern __attribute__ ((weak)) int clock_gettime(clockid_t, struct timespec *);
 #elif defined(__APPLE__)
 # include <mach/clock.h>
 # include <mach/mach.h>
+# include <pthread.h>
 #elif defined(_WIN32) || defined(__CYGWIN__)
 # define VC_EXTRALEAN
 # define WIN32_LEAN_AND_MEAN
@@ -81,3 +85,63 @@ int timer_end(double *time, struct timespec_compat *state) {
     *time = (last.tv_sec - state->tv_sec) + (last.tv_nsec - state->tv_nsec) / (double) GIGA;
     return 1;
 }
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+void win_raise_timeout(UNUSED HWND hwnd, 
+        UNUSED UINT uMsg, 
+        UNUSED UINT_PTR idEvent, 
+        UNUSED DWORD dwTime) {
+    RaiseException(CR_EXCEPTION_TIMEOUT, EXCEPTION_NONCONTINUABLE, 0, NULL);
+}
+#endif
+
+#if defined(__APPLE__)
+void *wait_and_raise(void *ptr) {
+    uint64_t *nanos = ptr;
+    struct timespec elapsed = {
+        .tv_sec = *nanos / GIGA,
+        .tv_nsec = *nanos % GIGA,
+    };
+    free(nanos);
+    if (!nanosleep(&elapsed, NULL))
+        raise(SIGPROF);
+    return NULL;
+}
+#endif
+
+int setup_timeout(uint64_t nanos) {
+#if   defined(__APPLE__)
+    uint64_t *nanos_copy = malloc(sizeof (uint64_t));
+    *nanos_copy = nanos;
+
+    pthread_t thread;
+    int res = pthread_create(&thread, NULL, wait_and_raise, nanos_copy);
+
+    return res ? -1 : 0;
+#elif defined(_WIN32) || defined(__CYGWIN__)
+    return SetTimer(NULL, 0, nanos / 1000000, (TIMERPROC) win_raise_timeout) ? 0 : -1;
+#elif defined(__unix__)
+    if (!can_measure_time()) {
+        errno = ENOTSUP;
+        return -1;
+    }
+
+    timer_t timer;
+    int res = timer_create(CLOCK_MONOTONIC, &(struct sigevent) {
+            .sigev_notify = SIGEV_SIGNAL,
+            .sigev_signo  = SIGPROF,
+        }, &timer);
+
+    if (res == -1)
+        return res;
+
+    struct itimerspec schedule = {
+        .it_value = { .tv_sec = nanos / GIGA, .tv_nsec = (nanos % GIGA) }
+    };
+
+    return timer_settime(timer, 0, &schedule, NULL);
+#else
+    errno = ENOTSUP;
+    return -1;
+#endif
+}

src/timer.h

@@ -1,6 +1,7 @@
 #ifndef TIMER_H_
 # define TIMER_H_
 
+# include <signal.h>
 # include <time.h>
 # include <inttypes.h>
 # include <stdbool.h>
@@ -13,5 +14,6 @@ struct timespec_compat {
 bool can_measure_time(void);
 int timer_start(struct timespec_compat *state);
 int timer_end(double *time, struct timespec_compat *state);
+int setup_timeout(uint64_t nanos);
 
 #endif /* !TIMER_H_ */