Fixed regression tests sometimes failing because of them being time-dependent
This commit is contained in:
Snaipe
parent
ec6ea15491
commit
3733c858cb
22 changed files with 42 additions and 38 deletions
|
|
@ -37,6 +37,7 @@ struct criterion_options {
|
|||
const char *pattern;
|
||||
bool short_filename;
|
||||
size_t jobs;
|
||||
bool measure_time;
|
||||
};
|
||||
|
||||
CR_BEGIN_C_API
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ macro(add_samples DIR_ SAMPLES_)
|
|||
ENVIRONMENT "CRITERION_SHORT_FILENAME=1"
|
||||
ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage
|
||||
ENVIRONMENT "CRITERION_JOBS=1" # for output ordering
|
||||
ENVIRONMENT "CRITERION_DISABLE_TIME_MEASUREMENTS=1" # do not compare timings
|
||||
)
|
||||
endif ()
|
||||
endforeach()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[[0;34m----[0m] [0;1masserts.c[0m:[0;31m11[0m: Assertion failed: assert is fatal, expect isn't
|
||||
[[0;34m----[0m] [0;1masserts.c[0m:[0;31m12[0m: Assertion failed: This assert runs
|
||||
[[0;31mFAIL[0m] asserts::base: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::base
|
||||
[[0;34m----[0m] [0;1masserts.c[0m:[0;31m17[0m: Assertion failed: You can fail an assertion with a message from anywhere
|
||||
[[0;34m----[0m] [0;1masserts.c[0m:[0;31m18[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;31mFAIL[0m] asserts::old_school: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::old_school
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m6[0;1m | Passing: [0;32m4[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m83[0m: Assertion failed: The expression (&s1)[0 .. 2] == (&s2)[0 .. 2] is false.
|
||||
[[0;31mFAIL[0m] asserts::array: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::array
|
||||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m13[0m: Assertion failed: assert is fatal, expect isn't
|
||||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m14[0m: Assertion failed: This assert runs
|
||||
[[0;31mFAIL[0m] asserts::base: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::base
|
||||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m89[0m: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception.
|
||||
[[0;31mFAIL[0m] asserts::exception: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::exception
|
||||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m19[0m: Assertion failed: You can fail an assertion with a message from anywhere
|
||||
[[0;34m----[0m] [0;1masserts.cc[0m:[0;31m20[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;31mFAIL[0m] asserts::old_school: (0.00s)
|
||||
[[0;31mFAIL[0m] asserts::old_school
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m7[0;1m | Passing: [0;32m3[0;1m | Failing: [0;31m4[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1mdescription.c[0m:[0;31m4[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] misc::failing: (0.00s)
|
||||
[[0;31mFAIL[0m] misc::failing
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1mdescription.cc[0m:[0;31m4[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] misc::failing: (0.00s)
|
||||
[[0;31mFAIL[0m] misc::failing
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m76[0m: Assertion failed: Parameters: (1, 2.000000)
|
||||
[[0;31mFAIL[0m] params::cleanup: (0.00s)
|
||||
[[0;31mFAIL[0m] params::cleanup
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m76[0m: Assertion failed: Parameters: (3, 4.000000)
|
||||
[[0;31mFAIL[0m] params::cleanup: (0.00s)
|
||||
[[0;31mFAIL[0m] params::cleanup
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m76[0m: Assertion failed: Parameters: (5, 6.000000)
|
||||
[[0;31mFAIL[0m] params::cleanup: (0.00s)
|
||||
[[0;31mFAIL[0m] params::cleanup
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m36[0m: Assertion failed: Parameters: (1, 2.000000)
|
||||
[[0;31mFAIL[0m] params::multiple: (0.00s)
|
||||
[[0;31mFAIL[0m] params::multiple
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m36[0m: Assertion failed: Parameters: (3, 4.000000)
|
||||
[[0;31mFAIL[0m] params::multiple: (0.00s)
|
||||
[[0;31mFAIL[0m] params::multiple
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m36[0m: Assertion failed: Parameters: (5, 6.000000)
|
||||
[[0;31mFAIL[0m] params::multiple: (0.00s)
|
||||
[[0;31mFAIL[0m] params::multiple
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m15[0m: Assertion failed: Parameter: foo
|
||||
[[0;31mFAIL[0m] params::str: (0.00s)
|
||||
[[0;31mFAIL[0m] params::str
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m15[0m: Assertion failed: Parameter: bar
|
||||
[[0;31mFAIL[0m] params::str: (0.00s)
|
||||
[[0;31mFAIL[0m] params::str
|
||||
[[0;34m----[0m] [0;1mparameterized.c[0m:[0;31m15[0m: Assertion failed: Parameter: baz
|
||||
[[0;31mFAIL[0m] params::str: (0.00s)
|
||||
[[0;31mFAIL[0m] params::str
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m9[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m9[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1mreport.c[0m:[0;31m5[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] sample::test: (0.00s)
|
||||
[[0;31mFAIL[0m] sample::test
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1mreport.cc[0m:[0;31m5[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] sample::test: (0.00s)
|
||||
[[0;31mFAIL[0m] sample::test
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
[[0;34m----[0m] [0;1msignal.c[0m:[0;31m16[0m: Unexpected signal caught below this line!
|
||||
[[0;31mFAIL[0m] simple::uncaught: CRASH!
|
||||
[[0;31mFAIL[0m] simple::wrong_signal: (0.00s)
|
||||
[[0;31mFAIL[0m] simple::wrong_signal
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m3[0;1m | Passing: [0;32m1[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m1[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
[[0;34m----[0m] [0;1msignal.cc[0m:[0;31m16[0m: Unexpected signal caught below this line!
|
||||
[[0;31mFAIL[0m] simple::uncaught: CRASH!
|
||||
[[0;31mFAIL[0m] simple::wrong_signal: (0.00s)
|
||||
[[0;31mFAIL[0m] simple::wrong_signal
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m3[0;1m | Passing: [0;32m1[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m1[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1msimple.c[0m:[0;31m4[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] misc::failing: (0.00s)
|
||||
[[0;31mFAIL[0m] misc::failing
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m1[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[[0;34m----[0m] [0;1msimple.cc[0m:[0;31m4[0m: Assertion failed: The expression 0 is false.
|
||||
[[0;31mFAIL[0m] misc::failing: (0.00s)
|
||||
[[0;31mFAIL[0m] misc::failing
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m1[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m22[0m: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false.
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m23[0m: Assertion failed: The expression (as strings) ("abc") > ("abd") is false.
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m24[0m: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false.
|
||||
[[0;31mFAIL[0m] messages::default: (0.00s)
|
||||
[[0;31mFAIL[0m] messages::default
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m28[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m29[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m30[0m: Assertion failed: foo bar
|
||||
|
|
@ -37,5 +37,5 @@
|
|||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m46[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m47[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.c[0m:[0;31m48[0m: Assertion failed: foo bar
|
||||
[[0;31mFAIL[0m] messages::user: (0.00s)
|
||||
[[0;31mFAIL[0m] messages::user
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m28[0m: Assertion failed: The statement `throw std::exception()` did throw an instance of the `std::exception` exception.
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m29[0m: Assertion failed: The statement `{}` did not throw any exception.
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m30[0m: Assertion failed: The statement `throw std::exception()` threw some exception.
|
||||
[[0;31mFAIL[0m] messages::default: (0.00s)
|
||||
[[0;31mFAIL[0m] messages::default
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m34[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m35[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m36[0m: Assertion failed: foo bar
|
||||
|
|
@ -45,5 +45,5 @@
|
|||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m57[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m58[0m: Assertion failed: foo bar
|
||||
[[0;34m----[0m] [0;1mfailmessages.cc[0m:[0;31m59[0m: Assertion failed: foo bar
|
||||
[[0;31mFAIL[0m] messages::user: (0.00s)
|
||||
[[0;31mFAIL[0m] messages::user
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -2,5 +2,5 @@
|
|||
[[0;34m----[0m] A long message
|
||||
[[0;34m----[0m] Spawning multiple lines.
|
||||
[[0;34m----[0m] Formatting is respected.
|
||||
[[0;31mFAIL[0m] sample::long_msg: (0.00s)
|
||||
[[0;31mFAIL[0m] sample::long_msg
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -2,5 +2,5 @@
|
|||
[[0;34m----[0m] A long message
|
||||
[[0;34m----[0m] Spawning multiple lines.
|
||||
[[0;34m----[0m] Formatting is respected.
|
||||
[[0;31mFAIL[0m] sample::long_msg: (0.00s)
|
||||
[[0;31mFAIL[0m] sample::long_msg
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m1[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m1[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
[[0;34m----[0m] Theory theory::gen failed with the following parameters: (4)
|
||||
[[0;34m----[0m] [0;1mtheories_regression.c[0m:[0;31m51[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;34m----[0m] Theory theory::gen failed with the following parameters: (5)
|
||||
[[0;31mFAIL[0m] theory::gen: (0.00s)
|
||||
[[0;31mFAIL[0m] theory::gen
|
||||
[[0;34m----[0m] [0;1mtheories_regression.c[0m:[0;31m34[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;34m----[0m] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test")
|
||||
[[0;31mFAIL[0m] theory::misc: (0.00s)
|
||||
[[0;31mFAIL[0m] theory::misc
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
[[0;34m----[0m] Theory theory::gen failed with the following parameters: (4)
|
||||
[[0;34m----[0m] [0;1mtheories_regression.cc[0m:[0;31m53[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;34m----[0m] Theory theory::gen failed with the following parameters: (5)
|
||||
[[0;31mFAIL[0m] theory::gen: (0.00s)
|
||||
[[0;31mFAIL[0m] theory::gen
|
||||
[[0;34m----[0m] [0;1mtheories_regression.cc[0m:[0;31m36[0m: Assertion failed: The conditions for this assertion were not met.
|
||||
[[0;34m----[0m] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test")
|
||||
[[0;31mFAIL[0m] theory::misc: (0.00s)
|
||||
[[0;31mFAIL[0m] theory::misc
|
||||
[[0;34m====[0m] [0;1mSynthesis: Tested: [0;34m2[0;1m | Passing: [0;32m0[0;1m | Failing: [0;31m2[0;1m | Crashing: [0;31m0[0;1m [0m
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include <inttypes.h>
|
||||
#include <stdlib.h>
|
||||
#include "criterion/common.h"
|
||||
#include "criterion/options.h"
|
||||
#include "compat/time.h"
|
||||
#include "compat/posix.h"
|
||||
|
||||
|
|
@ -28,6 +29,9 @@ extern __attribute__ ((weak)) int clock_gettime(clockid_t, struct timespec *);
|
|||
#endif
|
||||
|
||||
bool can_measure_time(void) {
|
||||
if (!criterion_options.measure_time)
|
||||
return false;
|
||||
|
||||
#if defined(__unix__) && !defined(__CYGWIN__)
|
||||
return clock_gettime != NULL;
|
||||
#else
|
||||
|
|
@ -128,11 +132,6 @@ int setup_timeout(uint64_t nanos) {
|
|||
CloseHandle(thread);
|
||||
return 0;
|
||||
#elif defined(__unix__)
|
||||
if (!can_measure_time()) {
|
||||
errno = ENOTSUP;
|
||||
return -1;
|
||||
}
|
||||
|
||||
timer_t timer;
|
||||
int res = timer_create(CLOCK_MONOTONIC, &(struct sigevent) {
|
||||
.sigev_notify = SIGEV_SIGNAL,
|
||||
|
|
|
|||
|
|
@ -183,6 +183,8 @@ int criterion_handle_args(int argc, char *argv[], bool handle_unknown_arg) {
|
|||
|
||||
bool use_tap = !strcmp("1", DEF(getenv("CRITERION_ENABLE_TAP"), "0"));
|
||||
|
||||
opt->measure_time = !!strcmp("1", DEF(getenv("CRITERION_DISABLE_TIME_MEASUREMENTS"), "0"));
|
||||
|
||||
bool do_list_tests = false;
|
||||
bool do_print_version = false;
|
||||
bool do_print_usage = false;
|
||||
|
|
|
|||
|
|
@ -26,4 +26,5 @@
|
|||
struct criterion_options criterion_options = {
|
||||
.logging_threshold = CRITERION_IMPORTANT,
|
||||
.output_provider = &normal_logging,
|
||||
.measure_time = true,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue