From 3733c858cb195878f11fb84962346430d339a173 Mon Sep 17 00:00:00 2001 From: Snaipe Date: Tue, 6 Oct 2015 15:44:09 +0200 Subject: [PATCH] Fixed regression tests sometimes failing because of them being time-dependent --- include/criterion/options.h | 1 + samples/CMakeLists.txt | 1 + samples/outputs/asserts.c.bin.err.expected | 4 ++-- samples/outputs/asserts.cc.bin.err.expected | 8 ++++---- samples/outputs/description.c.bin.err.expected | 2 +- .../outputs/description.cc.bin.err.expected | 2 +- .../outputs/parameterized.c.bin.err.expected | 18 +++++++++--------- samples/outputs/report.c.bin.err.expected | 2 +- samples/outputs/report.cc.bin.err.expected | 2 +- samples/outputs/signal.c.bin.err.expected | 2 +- samples/outputs/signal.cc.bin.err.expected | 2 +- samples/outputs/simple.c.bin.err.expected | 2 +- samples/outputs/simple.cc.bin.err.expected | 2 +- .../outputs/failmessages.c.bin.err.expected | 4 ++-- .../outputs/failmessages.cc.bin.err.expected | 4 ++-- .../outputs/long-messages.c.bin.err.expected | 2 +- .../outputs/long-messages.cc.bin.err.expected | 2 +- .../theories_regression.c.bin.err.expected | 4 ++-- .../theories_regression.cc.bin.err.expected | 4 ++-- src/compat/time.c | 9 ++++----- src/entry/main.c | 2 ++ src/entry/options.c | 1 + 22 files changed, 42 insertions(+), 38 deletions(-) diff --git a/include/criterion/options.h b/include/criterion/options.h index 49c0548..ee3a9c3 100644 --- a/include/criterion/options.h +++ b/include/criterion/options.h @@ -37,6 +37,7 @@ struct criterion_options { const char *pattern; bool short_filename; size_t jobs; + bool measure_time; }; CR_BEGIN_C_API diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 7bde575..ada809f 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -67,6 +67,7 @@ macro(add_samples DIR_ SAMPLES_) ENVIRONMENT "CRITERION_SHORT_FILENAME=1" ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage ENVIRONMENT "CRITERION_JOBS=1" # for output ordering + ENVIRONMENT "CRITERION_DISABLE_TIME_MEASUREMENTS=1" # do not compare timings ) endif () endforeach() diff --git a/samples/outputs/asserts.c.bin.err.expected b/samples/outputs/asserts.c.bin.err.expected index 95d9a94..10e2e90 100644 --- a/samples/outputs/asserts.c.bin.err.expected +++ b/samples/outputs/asserts.c.bin.err.expected @@ -1,7 +1,7 @@ [----] asserts.c:11: Assertion failed: assert is fatal, expect isn't [----] asserts.c:12: Assertion failed: This assert runs -[FAIL] asserts::base: (0.00s) +[FAIL] asserts::base [----] asserts.c:17: Assertion failed: You can fail an assertion with a message from anywhere [----] asserts.c:18: Assertion failed: The conditions for this assertion were not met. -[FAIL] asserts::old_school: (0.00s) +[FAIL] asserts::old_school [====] Synthesis: Tested: 6 | Passing: 4 | Failing: 2 | Crashing: 0  diff --git a/samples/outputs/asserts.cc.bin.err.expected b/samples/outputs/asserts.cc.bin.err.expected index ebeb355..ddc7623 100644 --- a/samples/outputs/asserts.cc.bin.err.expected +++ b/samples/outputs/asserts.cc.bin.err.expected @@ -1,11 +1,11 @@ [----] asserts.cc:83: Assertion failed: The expression (&s1)[0 .. 2] == (&s2)[0 .. 2] is false. -[FAIL] asserts::array: (0.00s) +[FAIL] asserts::array [----] asserts.cc:13: Assertion failed: assert is fatal, expect isn't [----] asserts.cc:14: Assertion failed: This assert runs -[FAIL] asserts::base: (0.00s) +[FAIL] asserts::base [----] asserts.cc:89: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception. -[FAIL] asserts::exception: (0.00s) +[FAIL] asserts::exception [----] asserts.cc:19: Assertion failed: You can fail an assertion with a message from anywhere [----] asserts.cc:20: Assertion failed: The conditions for this assertion were not met. -[FAIL] asserts::old_school: (0.00s) +[FAIL] asserts::old_school [====] Synthesis: Tested: 7 | Passing: 3 | Failing: 4 | Crashing: 0  diff --git a/samples/outputs/description.c.bin.err.expected b/samples/outputs/description.c.bin.err.expected index 2cc0d10..ffcd02d 100644 --- a/samples/outputs/description.c.bin.err.expected +++ b/samples/outputs/description.c.bin.err.expected @@ -1,3 +1,3 @@ [----] description.c:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing: (0.00s) +[FAIL] misc::failing [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/description.cc.bin.err.expected b/samples/outputs/description.cc.bin.err.expected index b9cebe4..ff63cc4 100644 --- a/samples/outputs/description.cc.bin.err.expected +++ b/samples/outputs/description.cc.bin.err.expected @@ -1,3 +1,3 @@ [----] description.cc:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing: (0.00s) +[FAIL] misc::failing [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/parameterized.c.bin.err.expected b/samples/outputs/parameterized.c.bin.err.expected index a080c6d..6299298 100644 --- a/samples/outputs/parameterized.c.bin.err.expected +++ b/samples/outputs/parameterized.c.bin.err.expected @@ -1,19 +1,19 @@ [----] parameterized.c:76: Assertion failed: Parameters: (1, 2.000000) -[FAIL] params::cleanup: (0.00s) +[FAIL] params::cleanup [----] parameterized.c:76: Assertion failed: Parameters: (3, 4.000000) -[FAIL] params::cleanup: (0.00s) +[FAIL] params::cleanup [----] parameterized.c:76: Assertion failed: Parameters: (5, 6.000000) -[FAIL] params::cleanup: (0.00s) +[FAIL] params::cleanup [----] parameterized.c:36: Assertion failed: Parameters: (1, 2.000000) -[FAIL] params::multiple: (0.00s) +[FAIL] params::multiple [----] parameterized.c:36: Assertion failed: Parameters: (3, 4.000000) -[FAIL] params::multiple: (0.00s) +[FAIL] params::multiple [----] parameterized.c:36: Assertion failed: Parameters: (5, 6.000000) -[FAIL] params::multiple: (0.00s) +[FAIL] params::multiple [----] parameterized.c:15: Assertion failed: Parameter: foo -[FAIL] params::str: (0.00s) +[FAIL] params::str [----] parameterized.c:15: Assertion failed: Parameter: bar -[FAIL] params::str: (0.00s) +[FAIL] params::str [----] parameterized.c:15: Assertion failed: Parameter: baz -[FAIL] params::str: (0.00s) +[FAIL] params::str [====] Synthesis: Tested: 9 | Passing: 0 | Failing: 9 | Crashing: 0  diff --git a/samples/outputs/report.c.bin.err.expected b/samples/outputs/report.c.bin.err.expected index f08bf9a..5de6825 100644 --- a/samples/outputs/report.c.bin.err.expected +++ b/samples/outputs/report.c.bin.err.expected @@ -1,3 +1,3 @@ [----] report.c:5: Assertion failed: The expression 0 is false. -[FAIL] sample::test: (0.00s) +[FAIL] sample::test [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/report.cc.bin.err.expected b/samples/outputs/report.cc.bin.err.expected index 72c59aa..0436a29 100644 --- a/samples/outputs/report.cc.bin.err.expected +++ b/samples/outputs/report.cc.bin.err.expected @@ -1,3 +1,3 @@ [----] report.cc:5: Assertion failed: The expression 0 is false. -[FAIL] sample::test: (0.00s) +[FAIL] sample::test [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/signal.c.bin.err.expected b/samples/outputs/signal.c.bin.err.expected index 8bc2b30..02d96bb 100644 --- a/samples/outputs/signal.c.bin.err.expected +++ b/samples/outputs/signal.c.bin.err.expected @@ -1,4 +1,4 @@ [----] signal.c:16: Unexpected signal caught below this line! [FAIL] simple::uncaught: CRASH! -[FAIL] simple::wrong_signal: (0.00s) +[FAIL] simple::wrong_signal [====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1  diff --git a/samples/outputs/signal.cc.bin.err.expected b/samples/outputs/signal.cc.bin.err.expected index 8ac44c1..8ec5103 100644 --- a/samples/outputs/signal.cc.bin.err.expected +++ b/samples/outputs/signal.cc.bin.err.expected @@ -1,4 +1,4 @@ [----] signal.cc:16: Unexpected signal caught below this line! [FAIL] simple::uncaught: CRASH! -[FAIL] simple::wrong_signal: (0.00s) +[FAIL] simple::wrong_signal [====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1  diff --git a/samples/outputs/simple.c.bin.err.expected b/samples/outputs/simple.c.bin.err.expected index a687898..4898f35 100644 --- a/samples/outputs/simple.c.bin.err.expected +++ b/samples/outputs/simple.c.bin.err.expected @@ -1,3 +1,3 @@ [----] simple.c:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing: (0.00s) +[FAIL] misc::failing [====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/simple.cc.bin.err.expected b/samples/outputs/simple.cc.bin.err.expected index 816d6cd..e2e6e05 100644 --- a/samples/outputs/simple.cc.bin.err.expected +++ b/samples/outputs/simple.cc.bin.err.expected @@ -1,3 +1,3 @@ [----] simple.cc:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing: (0.00s) +[FAIL] misc::failing [====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/failmessages.c.bin.err.expected b/samples/tests/outputs/failmessages.c.bin.err.expected index 3a6b0e7..fc2dc0c 100644 --- a/samples/tests/outputs/failmessages.c.bin.err.expected +++ b/samples/tests/outputs/failmessages.c.bin.err.expected @@ -17,7 +17,7 @@ [----] failmessages.c:22: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false. [----] failmessages.c:23: Assertion failed: The expression (as strings) ("abc") > ("abd") is false. [----] failmessages.c:24: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false. -[FAIL] messages::default: (0.00s) +[FAIL] messages::default [----] failmessages.c:28: Assertion failed: foo bar [----] failmessages.c:29: Assertion failed: foo bar [----] failmessages.c:30: Assertion failed: foo bar @@ -37,5 +37,5 @@ [----] failmessages.c:46: Assertion failed: foo bar [----] failmessages.c:47: Assertion failed: foo bar [----] failmessages.c:48: Assertion failed: foo bar -[FAIL] messages::user: (0.00s) +[FAIL] messages::user [====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/samples/tests/outputs/failmessages.cc.bin.err.expected b/samples/tests/outputs/failmessages.cc.bin.err.expected index f060bac..5a57355 100644 --- a/samples/tests/outputs/failmessages.cc.bin.err.expected +++ b/samples/tests/outputs/failmessages.cc.bin.err.expected @@ -21,7 +21,7 @@ [----] failmessages.cc:28: Assertion failed: The statement `throw std::exception()` did throw an instance of the `std::exception` exception. [----] failmessages.cc:29: Assertion failed: The statement `{}` did not throw any exception. [----] failmessages.cc:30: Assertion failed: The statement `throw std::exception()` threw some exception. -[FAIL] messages::default: (0.00s) +[FAIL] messages::default [----] failmessages.cc:34: Assertion failed: foo bar [----] failmessages.cc:35: Assertion failed: foo bar [----] failmessages.cc:36: Assertion failed: foo bar @@ -45,5 +45,5 @@ [----] failmessages.cc:57: Assertion failed: foo bar [----] failmessages.cc:58: Assertion failed: foo bar [----] failmessages.cc:59: Assertion failed: foo bar -[FAIL] messages::user: (0.00s) +[FAIL] messages::user [====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/samples/tests/outputs/long-messages.c.bin.err.expected b/samples/tests/outputs/long-messages.c.bin.err.expected index 9658ae5..f97a7f4 100644 --- a/samples/tests/outputs/long-messages.c.bin.err.expected +++ b/samples/tests/outputs/long-messages.c.bin.err.expected @@ -2,5 +2,5 @@ [----] A long message [----] Spawning multiple lines. [----] Formatting is respected. -[FAIL] sample::long_msg: (0.00s) +[FAIL] sample::long_msg [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/long-messages.cc.bin.err.expected b/samples/tests/outputs/long-messages.cc.bin.err.expected index fa56cf2..0237997 100644 --- a/samples/tests/outputs/long-messages.cc.bin.err.expected +++ b/samples/tests/outputs/long-messages.cc.bin.err.expected @@ -2,5 +2,5 @@ [----] A long message [----] Spawning multiple lines. [----] Formatting is respected. -[FAIL] sample::long_msg: (0.00s) +[FAIL] sample::long_msg [====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/theories_regression.c.bin.err.expected b/samples/tests/outputs/theories_regression.c.bin.err.expected index 0db8e12..fa88f2b 100644 --- a/samples/tests/outputs/theories_regression.c.bin.err.expected +++ b/samples/tests/outputs/theories_regression.c.bin.err.expected @@ -8,8 +8,8 @@ [----] Theory theory::gen failed with the following parameters: (4) [----] theories_regression.c:51: Assertion failed: The conditions for this assertion were not met. [----] Theory theory::gen failed with the following parameters: (5) -[FAIL] theory::gen: (0.00s) +[FAIL] theory::gen [----] theories_regression.c:34: Assertion failed: The conditions for this assertion were not met. [----] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") -[FAIL] theory::misc: (0.00s) +[FAIL] theory::misc [====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/samples/tests/outputs/theories_regression.cc.bin.err.expected b/samples/tests/outputs/theories_regression.cc.bin.err.expected index a4e378e..0303937 100644 --- a/samples/tests/outputs/theories_regression.cc.bin.err.expected +++ b/samples/tests/outputs/theories_regression.cc.bin.err.expected @@ -8,8 +8,8 @@ [----] Theory theory::gen failed with the following parameters: (4) [----] theories_regression.cc:53: Assertion failed: The conditions for this assertion were not met. [----] Theory theory::gen failed with the following parameters: (5) -[FAIL] theory::gen: (0.00s) +[FAIL] theory::gen [----] theories_regression.cc:36: Assertion failed: The conditions for this assertion were not met. [----] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") -[FAIL] theory::misc: (0.00s) +[FAIL] theory::misc [====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/src/compat/time.c b/src/compat/time.c index 14de484..f8c8fee 100644 --- a/src/compat/time.c +++ b/src/compat/time.c @@ -2,6 +2,7 @@ #include #include #include "criterion/common.h" +#include "criterion/options.h" #include "compat/time.h" #include "compat/posix.h" @@ -28,6 +29,9 @@ extern __attribute__ ((weak)) int clock_gettime(clockid_t, struct timespec *); #endif bool can_measure_time(void) { + if (!criterion_options.measure_time) + return false; + #if defined(__unix__) && !defined(__CYGWIN__) return clock_gettime != NULL; #else @@ -128,11 +132,6 @@ int setup_timeout(uint64_t nanos) { CloseHandle(thread); return 0; #elif defined(__unix__) - if (!can_measure_time()) { - errno = ENOTSUP; - return -1; - } - timer_t timer; int res = timer_create(CLOCK_MONOTONIC, &(struct sigevent) { .sigev_notify = SIGEV_SIGNAL, diff --git a/src/entry/main.c b/src/entry/main.c index d67989f..6de122a 100644 --- a/src/entry/main.c +++ b/src/entry/main.c @@ -183,6 +183,8 @@ int criterion_handle_args(int argc, char *argv[], bool handle_unknown_arg) { bool use_tap = !strcmp("1", DEF(getenv("CRITERION_ENABLE_TAP"), "0")); + opt->measure_time = !!strcmp("1", DEF(getenv("CRITERION_DISABLE_TIME_MEASUREMENTS"), "0")); + bool do_list_tests = false; bool do_print_version = false; bool do_print_usage = false; diff --git a/src/entry/options.c b/src/entry/options.c index c83b294..20295f9 100644 --- a/src/entry/options.c +++ b/src/entry/options.c @@ -26,4 +26,5 @@ struct criterion_options criterion_options = { .logging_threshold = CRITERION_IMPORTANT, .output_provider = &normal_logging, + .measure_time = true, };