diff --git a/.gitignore b/.gitignore index dc27c60..a051573 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ !*.rst !*.po !*.in +!*.t !.cmake/Modules/*.cmake !samples/tests/*.sh !samples/**/*.expected diff --git a/.travis.yml b/.travis.yml index 57fa759..3d08ffd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,10 @@ before_install: brew unlink cmake brew install llvm cmake fi + - | + if [ "${TESTS:-ON}" = "ON" ]; then + pip install --user cram + fi - export CXX=${CC/gcc/g++}; export CXX=${CXX/clang/clang++} - $CC --version - $CXX --version @@ -122,7 +126,7 @@ after_success: after_failure: - | - if [ "$TESTS" = "ON" ]; then + if [ "${TESTS:-ON}" = "ON" ]; then cat Testing/Temporary/LastTest.log samples/*.{out,err} ../samples/tests/*.{out,err} fi diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 799a758..eaba4da 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -70,42 +70,9 @@ macro(add_samples DIR_ SAMPLES_) ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage ) - - if (NOT MSVC) # we disable the scripted tests when building with MSVC - add_test(${sample}_compare sh ${DIR_}/run_test.sh "${CMAKE_CURRENT_LIST_DIR}/outputs" . . ${sample}.bin) - set_property(TEST ${sample}_compare PROPERTY - ENVIRONMENT "LC_ALL=en_US.utf8" - ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" - ENVIRONMENT "CRITERION_SHORT_FILENAME=1" - ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage - ENVIRONMENT "CRITERION_JOBS=1" # for output ordering - ENVIRONMENT "CRITERION_DISABLE_TIME_MEASUREMENTS=1" # do not compare timings - ) - endif () endforeach() endmacro() add_samples("${CMAKE_CURRENT_LIST_DIR}/tests" "${SAMPLES}") -if (NOT MSVC) # we disable the scripted tests when building with MSVC - -foreach(script ${SCRIPTS}) - add_test(${script} sh "${CMAKE_CURRENT_LIST_DIR}/tests/${script}.sh") - set_property(TEST ${script} PROPERTY - ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" - ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage - ) - - add_test(${script}_compare sh ${CMAKE_CURRENT_LIST_DIR}/tests/run_test.sh "${CMAKE_CURRENT_LIST_DIR}" . "${CMAKE_CURRENT_LIST_DIR}" "tests/${script}.sh") - set_property(TEST ${script}_compare PROPERTY - ENVIRONMENT "LC_ALL=en_US.utf8" - ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" - ENVIRONMENT "CRITERION_SHORT_FILENAME=1" - ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage - ENVIRONMENT "CRITERION_JOBS=1" # for output ordering - ) -endforeach() - -endif() - add_subdirectory(tests) diff --git a/samples/outputs/asserts.c.bin.err.expected b/samples/outputs/asserts.c.bin.err.expected deleted file mode 100644 index 10e2e90..0000000 --- a/samples/outputs/asserts.c.bin.err.expected +++ /dev/null @@ -1,7 +0,0 @@ -[----] asserts.c:11: Assertion failed: assert is fatal, expect isn't -[----] asserts.c:12: Assertion failed: This assert runs -[FAIL] asserts::base -[----] asserts.c:17: Assertion failed: You can fail an assertion with a message from anywhere -[----] asserts.c:18: Assertion failed: The conditions for this assertion were not met. -[FAIL] asserts::old_school -[====] Synthesis: Tested: 6 | Passing: 4 | Failing: 2 | Crashing: 0  diff --git a/samples/outputs/asserts.c.bin.out.expected b/samples/outputs/asserts.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/asserts.cc.bin.err.expected b/samples/outputs/asserts.cc.bin.err.expected deleted file mode 100644 index ddc7623..0000000 --- a/samples/outputs/asserts.cc.bin.err.expected +++ /dev/null @@ -1,11 +0,0 @@ -[----] asserts.cc:83: Assertion failed: The expression (&s1)[0 .. 2] == (&s2)[0 .. 2] is false. -[FAIL] asserts::array -[----] asserts.cc:13: Assertion failed: assert is fatal, expect isn't -[----] asserts.cc:14: Assertion failed: This assert runs -[FAIL] asserts::base -[----] asserts.cc:89: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception. -[FAIL] asserts::exception -[----] asserts.cc:19: Assertion failed: You can fail an assertion with a message from anywhere -[----] asserts.cc:20: Assertion failed: The conditions for this assertion were not met. -[FAIL] asserts::old_school -[====] Synthesis: Tested: 7 | Passing: 3 | Failing: 4 | Crashing: 0  diff --git a/samples/outputs/asserts.cc.bin.out.expected b/samples/outputs/asserts.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/description.c.bin.err.expected b/samples/outputs/description.c.bin.err.expected deleted file mode 100644 index ffcd02d..0000000 --- a/samples/outputs/description.c.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] description.c:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/description.c.bin.out.expected b/samples/outputs/description.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/description.cc.bin.err.expected b/samples/outputs/description.cc.bin.err.expected deleted file mode 100644 index ff63cc4..0000000 --- a/samples/outputs/description.cc.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] description.cc:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/description.cc.bin.out.expected b/samples/outputs/description.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/fixtures.c.bin.err.expected b/samples/outputs/fixtures.c.bin.err.expected deleted file mode 100644 index a34f221..0000000 --- a/samples/outputs/fixtures.c.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/fixtures.c.bin.out.expected b/samples/outputs/fixtures.c.bin.out.expected deleted file mode 100644 index a74afed..0000000 --- a/samples/outputs/fixtures.c.bin.out.expected +++ /dev/null @@ -1,2 +0,0 @@ -Runs before the test -Runs after the test diff --git a/samples/outputs/fixtures.cc.bin.err.expected b/samples/outputs/fixtures.cc.bin.err.expected deleted file mode 100644 index a34f221..0000000 --- a/samples/outputs/fixtures.cc.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/fixtures.cc.bin.out.expected b/samples/outputs/fixtures.cc.bin.out.expected deleted file mode 100644 index a74afed..0000000 --- a/samples/outputs/fixtures.cc.bin.out.expected +++ /dev/null @@ -1,2 +0,0 @@ -Runs before the test -Runs after the test diff --git a/samples/outputs/more-suites.c.bin.err.expected b/samples/outputs/more-suites.c.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/more-suites.c.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/more-suites.c.bin.out.expected b/samples/outputs/more-suites.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/more-suites.cc.bin.err.expected b/samples/outputs/more-suites.cc.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/more-suites.cc.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/more-suites.cc.bin.out.expected b/samples/outputs/more-suites.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/other-crashes.c.bin.err.expected b/samples/outputs/other-crashes.c.bin.err.expected deleted file mode 100644 index 41d3cb8..0000000 --- a/samples/outputs/other-crashes.c.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown. -[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown. -[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1  diff --git a/samples/outputs/other-crashes.c.bin.out.expected b/samples/outputs/other-crashes.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/other-crashes.cc.bin.err.expected b/samples/outputs/other-crashes.cc.bin.err.expected deleted file mode 100644 index 41d3cb8..0000000 --- a/samples/outputs/other-crashes.cc.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown. -[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown. -[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1  diff --git a/samples/outputs/other-crashes.cc.bin.out.expected b/samples/outputs/other-crashes.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/parameterized.c.bin.err.expected b/samples/outputs/parameterized.c.bin.err.expected deleted file mode 100644 index 6299298..0000000 --- a/samples/outputs/parameterized.c.bin.err.expected +++ /dev/null @@ -1,19 +0,0 @@ -[----] parameterized.c:76: Assertion failed: Parameters: (1, 2.000000) -[FAIL] params::cleanup -[----] parameterized.c:76: Assertion failed: Parameters: (3, 4.000000) -[FAIL] params::cleanup -[----] parameterized.c:76: Assertion failed: Parameters: (5, 6.000000) -[FAIL] params::cleanup -[----] parameterized.c:36: Assertion failed: Parameters: (1, 2.000000) -[FAIL] params::multiple -[----] parameterized.c:36: Assertion failed: Parameters: (3, 4.000000) -[FAIL] params::multiple -[----] parameterized.c:36: Assertion failed: Parameters: (5, 6.000000) -[FAIL] params::multiple -[----] parameterized.c:15: Assertion failed: Parameter: foo -[FAIL] params::str -[----] parameterized.c:15: Assertion failed: Parameter: bar -[FAIL] params::str -[----] parameterized.c:15: Assertion failed: Parameter: baz -[FAIL] params::str -[====] Synthesis: Tested: 9 | Passing: 0 | Failing: 9 | Crashing: 0  diff --git a/samples/outputs/parameterized.c.bin.out.expected b/samples/outputs/parameterized.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/redirect.c.bin.err.expected b/samples/outputs/redirect.c.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/redirect.c.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/redirect.c.bin.out.expected b/samples/outputs/redirect.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/redirect.cc.bin.err.expected b/samples/outputs/redirect.cc.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/redirect.cc.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/redirect.cc.bin.out.expected b/samples/outputs/redirect.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/report.c.bin.err.expected b/samples/outputs/report.c.bin.err.expected deleted file mode 100644 index 5de6825..0000000 --- a/samples/outputs/report.c.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] report.c:5: Assertion failed: The expression 0 is false. -[FAIL] sample::test -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/report.c.bin.out.expected b/samples/outputs/report.c.bin.out.expected deleted file mode 100644 index d560277..0000000 --- a/samples/outputs/report.c.bin.out.expected +++ /dev/null @@ -1,4 +0,0 @@ -criterion_init -testing test in category sample -Asserts: [1 passed, 1 failed, 2 total] -criterion_fini diff --git a/samples/outputs/report.cc.bin.err.expected b/samples/outputs/report.cc.bin.err.expected deleted file mode 100644 index 0436a29..0000000 --- a/samples/outputs/report.cc.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] report.cc:5: Assertion failed: The expression 0 is false. -[FAIL] sample::test -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/report.cc.bin.out.expected b/samples/outputs/report.cc.bin.out.expected deleted file mode 100644 index d560277..0000000 --- a/samples/outputs/report.cc.bin.out.expected +++ /dev/null @@ -1,4 +0,0 @@ -criterion_init -testing test in category sample -Asserts: [1 passed, 1 failed, 2 total] -criterion_fini diff --git a/samples/outputs/signal.c.bin.err.expected b/samples/outputs/signal.c.bin.err.expected deleted file mode 100644 index 02d96bb..0000000 --- a/samples/outputs/signal.c.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] signal.c:16: Unexpected signal caught below this line! -[FAIL] simple::uncaught: CRASH! -[FAIL] simple::wrong_signal -[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1  diff --git a/samples/outputs/signal.c.bin.out.expected b/samples/outputs/signal.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/signal.cc.bin.err.expected b/samples/outputs/signal.cc.bin.err.expected deleted file mode 100644 index 8ec5103..0000000 --- a/samples/outputs/signal.cc.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] signal.cc:16: Unexpected signal caught below this line! -[FAIL] simple::uncaught: CRASH! -[FAIL] simple::wrong_signal -[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1  diff --git a/samples/outputs/signal.cc.bin.out.expected b/samples/outputs/signal.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/simple.c.bin.err.expected b/samples/outputs/simple.c.bin.err.expected deleted file mode 100644 index 4898f35..0000000 --- a/samples/outputs/simple.c.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] simple.c:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing -[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/simple.c.bin.out.expected b/samples/outputs/simple.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/simple.cc.bin.err.expected b/samples/outputs/simple.cc.bin.err.expected deleted file mode 100644 index e2e6e05..0000000 --- a/samples/outputs/simple.cc.bin.err.expected +++ /dev/null @@ -1,3 +0,0 @@ -[----] simple.cc:4: Assertion failed: The expression 0 is false. -[FAIL] misc::failing -[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0  diff --git a/samples/outputs/simple.cc.bin.out.expected b/samples/outputs/simple.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/suites.c.bin.err.expected b/samples/outputs/suites.c.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/suites.c.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/suites.c.bin.out.expected b/samples/outputs/suites.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/outputs/suites.cc.bin.err.expected b/samples/outputs/suites.cc.bin.err.expected deleted file mode 100644 index 09eb50f..0000000 --- a/samples/outputs/suites.cc.bin.err.expected +++ /dev/null @@ -1 +0,0 @@ -[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/outputs/suites.cc.bin.out.expected b/samples/outputs/suites.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/early_exit.sh b/samples/tests/early_exit.sh deleted file mode 100755 index e0566dd..0000000 --- a/samples/tests/early_exit.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -./simple.c.bin --always-succeed -CRITERION_NO_EARLY_EXIT=0 ./simple.c.bin --always-succeed diff --git a/samples/tests/fail_fast.sh b/samples/tests/fail_fast.sh deleted file mode 100755 index fcc02e1..0000000 --- a/samples/tests/fail_fast.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -./simple.c.bin --fail-fast --always-succeed diff --git a/samples/tests/help.sh b/samples/tests/help.sh deleted file mode 100755 index 6752cc6..0000000 --- a/samples/tests/help.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -./simple.c.bin --help -./simple.c.bin --version diff --git a/samples/tests/json_test.sh b/samples/tests/json_test.sh deleted file mode 100755 index 2546c00..0000000 --- a/samples/tests/json_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -./simple.c.bin --json --always-succeed -./signal.c.bin --json --always-succeed -./asserts.c.bin --json --always-succeed -./more-suites.c.bin --json --always-succeed -./tests/long-messages.c.bin --json --always-succeed -./description.c.bin --json --always-succeed diff --git a/samples/tests/list.sh b/samples/tests/list.sh deleted file mode 100755 index 6ab72bf..0000000 --- a/samples/tests/list.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -./simple.c.bin --list -./simple.c.bin --list --ascii diff --git a/samples/tests/outputs/exit.c.bin.err.expected b/samples/tests/outputs/exit.c.bin.err.expected deleted file mode 100644 index 3ffe51b..0000000 --- a/samples/tests/outputs/exit.c.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] Warning! The test `exit::unexpected_exit` exited during its setup or teardown. -[----] Warning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown. -[----] Warning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown. -[====] Synthesis: Tested: 5 | Passing: 3 | Failing: 2 | Crashing: 2  diff --git a/samples/tests/outputs/exit.c.bin.out.expected b/samples/tests/outputs/exit.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/exit.cc.bin.err.expected b/samples/tests/outputs/exit.cc.bin.err.expected deleted file mode 100644 index 3ffe51b..0000000 --- a/samples/tests/outputs/exit.cc.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] Warning! The test `exit::unexpected_exit` exited during its setup or teardown. -[----] Warning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown. -[----] Warning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown. -[====] Synthesis: Tested: 5 | Passing: 3 | Failing: 2 | Crashing: 2  diff --git a/samples/tests/outputs/exit.cc.bin.out.expected b/samples/tests/outputs/exit.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/failmessages.c.bin.err.expected b/samples/tests/outputs/failmessages.c.bin.err.expected deleted file mode 100644 index fc2dc0c..0000000 --- a/samples/tests/outputs/failmessages.c.bin.err.expected +++ /dev/null @@ -1,41 +0,0 @@ -[----] failmessages.c:4: Assertion failed: The expression 0 is false. -[----] failmessages.c:5: Assertion failed: The expression (0) == (1) is false. -[----] failmessages.c:6: Assertion failed: The expression (1) != (1) is false. -[----] failmessages.c:7: Assertion failed: The expression (2) < (1) is false. -[----] failmessages.c:8: Assertion failed: The expression (2) <= (1) is false. -[----] failmessages.c:9: Assertion failed: The expression (1) > (2) is false. -[----] failmessages.c:10: Assertion failed: The expression (1) >= (2) is false. -[----] failmessages.c:11: Assertion failed: "" is not null. -[----] failmessages.c:12: Assertion failed: ((void *)0) is null. -[----] failmessages.c:14: Assertion failed: The expression (2) - (1) <= (0.1) && (1) - (2) <= (0.1) is false. -[----] failmessages.c:15: Assertion failed: The expression (2) - (2) > (0.1) || (2) - (2) > (0.1) is false. -[----] failmessages.c:17: Assertion failed: "foo" is not empty. -[----] failmessages.c:18: Assertion failed: "" is empty. -[----] failmessages.c:19: Assertion failed: The expression (as strings) ("abc") == ("abd") is false. -[----] failmessages.c:20: Assertion failed: The expression (as strings) ("abc") != ("abc") is false. -[----] failmessages.c:21: Assertion failed: The expression (as strings) ("abc") < ("aba") is false. -[----] failmessages.c:22: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false. -[----] failmessages.c:23: Assertion failed: The expression (as strings) ("abc") > ("abd") is false. -[----] failmessages.c:24: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false. -[FAIL] messages::default -[----] failmessages.c:28: Assertion failed: foo bar -[----] failmessages.c:29: Assertion failed: foo bar -[----] failmessages.c:30: Assertion failed: foo bar -[----] failmessages.c:31: Assertion failed: foo bar -[----] failmessages.c:32: Assertion failed: foo bar -[----] failmessages.c:33: Assertion failed: foo bar -[----] failmessages.c:34: Assertion failed: foo bar -[----] failmessages.c:35: Assertion failed: foo bar -[----] failmessages.c:36: Assertion failed: foo bar -[----] failmessages.c:38: Assertion failed: foo bar -[----] failmessages.c:39: Assertion failed: foo bar -[----] failmessages.c:41: Assertion failed: foo bar -[----] failmessages.c:42: Assertion failed: foo bar -[----] failmessages.c:43: Assertion failed: foo bar -[----] failmessages.c:44: Assertion failed: foo bar -[----] failmessages.c:45: Assertion failed: foo bar -[----] failmessages.c:46: Assertion failed: foo bar -[----] failmessages.c:47: Assertion failed: foo bar -[----] failmessages.c:48: Assertion failed: foo bar -[FAIL] messages::user -[====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/samples/tests/outputs/failmessages.c.bin.out.expected b/samples/tests/outputs/failmessages.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/failmessages.cc.bin.err.expected b/samples/tests/outputs/failmessages.cc.bin.err.expected deleted file mode 100644 index 5a57355..0000000 --- a/samples/tests/outputs/failmessages.cc.bin.err.expected +++ /dev/null @@ -1,49 +0,0 @@ -[----] failmessages.cc:5: Assertion failed: The expression 0 is false. -[----] failmessages.cc:6: Assertion failed: The expression (0) == (1) is false. -[----] failmessages.cc:7: Assertion failed: The expression (1) != (1) is false. -[----] failmessages.cc:8: Assertion failed: The expression (2) < (1) is false. -[----] failmessages.cc:9: Assertion failed: The expression (2) <= (1) is false. -[----] failmessages.cc:10: Assertion failed: The expression (1) > (2) is false. -[----] failmessages.cc:11: Assertion failed: The expression (1) >= (2) is false. -[----] failmessages.cc:12: Assertion failed: "" is not null. -[----] failmessages.cc:13: Assertion failed: __null is null. -[----] failmessages.cc:15: Assertion failed: The expression (2) - (1) <= (0.1) && (1) - (2) <= (0.1) is false. -[----] failmessages.cc:16: Assertion failed: The expression (2) - (2) > (0.1) || (2) - (2) > (0.1) is false. -[----] failmessages.cc:18: Assertion failed: "foo" is not empty. -[----] failmessages.cc:19: Assertion failed: "" is empty. -[----] failmessages.cc:20: Assertion failed: The expression (as strings) ("abc") == ("abd") is false. -[----] failmessages.cc:21: Assertion failed: The expression (as strings) ("abc") != ("abc") is false. -[----] failmessages.cc:22: Assertion failed: The expression (as strings) ("abc") < ("aba") is false. -[----] failmessages.cc:23: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false. -[----] failmessages.cc:24: Assertion failed: The expression (as strings) ("abc") > ("abd") is false. -[----] failmessages.cc:25: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false. -[----] failmessages.cc:27: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception. -[----] failmessages.cc:28: Assertion failed: The statement `throw std::exception()` did throw an instance of the `std::exception` exception. -[----] failmessages.cc:29: Assertion failed: The statement `{}` did not throw any exception. -[----] failmessages.cc:30: Assertion failed: The statement `throw std::exception()` threw some exception. -[FAIL] messages::default -[----] failmessages.cc:34: Assertion failed: foo bar -[----] failmessages.cc:35: Assertion failed: foo bar -[----] failmessages.cc:36: Assertion failed: foo bar -[----] failmessages.cc:37: Assertion failed: foo bar -[----] failmessages.cc:38: Assertion failed: foo bar -[----] failmessages.cc:39: Assertion failed: foo bar -[----] failmessages.cc:40: Assertion failed: foo bar -[----] failmessages.cc:41: Assertion failed: foo bar -[----] failmessages.cc:42: Assertion failed: foo bar -[----] failmessages.cc:44: Assertion failed: foo bar -[----] failmessages.cc:45: Assertion failed: foo bar -[----] failmessages.cc:47: Assertion failed: foo bar -[----] failmessages.cc:48: Assertion failed: foo bar -[----] failmessages.cc:49: Assertion failed: foo bar -[----] failmessages.cc:50: Assertion failed: foo bar -[----] failmessages.cc:51: Assertion failed: foo bar -[----] failmessages.cc:52: Assertion failed: foo bar -[----] failmessages.cc:53: Assertion failed: foo bar -[----] failmessages.cc:54: Assertion failed: foo bar -[----] failmessages.cc:56: Assertion failed: foo bar -[----] failmessages.cc:57: Assertion failed: foo bar -[----] failmessages.cc:58: Assertion failed: foo bar -[----] failmessages.cc:59: Assertion failed: foo bar -[FAIL] messages::user -[====] Synthesis: Tested: 2 | Passing: 0 | Failing: 2 | Crashing: 0  diff --git a/samples/tests/outputs/failmessages.cc.bin.out.expected b/samples/tests/outputs/failmessages.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/json_test.sh.err.expected b/samples/tests/outputs/json_test.sh.err.expected deleted file mode 100644 index 001f0dc..0000000 --- a/samples/tests/outputs/json_test.sh.err.expected +++ /dev/null @@ -1,237 +0,0 @@ -{ - "id": "Criterion v2.1.0", - "passed": 1, - "failed": 1, - "errored": 0, - "skipped": 0, - "test_suites": [ - { - "name": "misc", - "passed": 1, - "failed": 1, - "errored": 0, - "skipped": 0, - "tests": [ - { - "name": "passing", - "assertions": 1, - "status": "PASSED" - }, - { - "name": "failing", - "assertions": 1, - "status": "FAILED", - "messages": [ - "simple.c:4: The expression 0 is false." - ] - } - ] - } - ] -} -{ - "id": "Criterion v2.1.0", - "passed": 1, - "failed": 2, - "errored": 1, - "skipped": 0, - "test_suites": [ - { - "name": "simple", - "passed": 1, - "failed": 2, - "errored": 1, - "skipped": 0, - "tests": [ - { - "name": "wrong_signal", - "assertions": 0, - "status": "FAILED", - "messages": [ - - ] - }, - { - "name": "uncaught", - "assertions": 0, - "status": "ERRORED", - "messages": ["The test crashed."] - }, - { - "name": "caught", - "assertions": 0, - "status": "PASSED" - } - ] - } - ] -} -{ - "id": "Criterion v2.1.0", - "passed": 4, - "failed": 2, - "errored": 0, - "skipped": 0, - "test_suites": [ - { - "name": "asserts", - "passed": 4, - "failed": 2, - "errored": 0, - "skipped": 0, - "tests": [ - { - "name": "string", - "assertions": 10, - "status": "PASSED" - }, - { - "name": "old_school", - "assertions": 2, - "status": "FAILED", - "messages": [ - "asserts.c:18: The conditions for this assertion were not met.", - "asserts.c:17: You can fail an assertion with a message from anywhere" - ] - }, - { - "name": "native", - "assertions": 8, - "status": "PASSED" - }, - { - "name": "float", - "assertions": 2, - "status": "PASSED" - }, - { - "name": "base", - "assertions": 6, - "status": "FAILED", - "messages": [ - "asserts.c:12: This assert runs", - "asserts.c:11: assert is fatal, expect isn't" - ] - }, - { - "name": "array", - "assertions": 3, - "status": "PASSED" - } - ] - } - ] -} -{ - "id": "Criterion v2.1.0", - "passed": 2, - "failed": 0, - "errored": 0, - "skipped": 1, - "test_suites": [ - { - "name": "suite2", - "passed": 1, - "failed": 0, - "errored": 0, - "skipped": 0, - "tests": [ - { - "name": "test", - "assertions": 1, - "status": "PASSED" - } - ] - }, - { - "name": "suite1", - "passed": 1, - "failed": 0, - "errored": 0, - "skipped": 0, - "tests": [ - { - "name": "test", - "assertions": 1, - "status": "PASSED" - } - ] - }, - { - "name": "disabled", - "passed": 0, - "failed": 0, - "errored": 0, - "skipped": 1, - "tests": [ - { - "name": "test", - "assertions": 0, - "status": "SKIPPED", - "messages": ["The test was skipped."] - } - ] - } - ] -} -{ - "id": "Criterion v2.1.0", - "passed": 0, - "failed": 1, - "errored": 0, - "skipped": 0, - "test_suites": [ - { - "name": "sample", - "passed": 0, - "failed": 1, - "errored": 0, - "skipped": 0, - "tests": [ - { - "name": "long_msg", - "assertions": 1, - "status": "FAILED", - "messages": [ - "long-messages.c:4: This is", - " A long message", - " Spawning multiple lines.", - " Formatting is respected." - ] - } - ] - } - ] -} -{ - "id": "Criterion v2.1.0", - "passed": 0, - "failed": 1, - "errored": 0, - "skipped": 1, - "test_suites": [ - { - "name": "misc", - "passed": 0, - "failed": 1, - "errored": 0, - "skipped": 1, - "tests": [ - { - "name": "skipped", - "assertions": 0, - "status": "SKIPPED", - "messages": ["The test was skipped."] - }, - { - "name": "failing", - "assertions": 1, - "status": "FAILED", - "messages": [ - "description.c:4: The expression 0 is false." - ] - } - ] - } - ] -} diff --git a/samples/tests/outputs/json_test.sh.out.expected b/samples/tests/outputs/json_test.sh.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/long-messages.c.bin.err.expected b/samples/tests/outputs/long-messages.c.bin.err.expected deleted file mode 100644 index f97a7f4..0000000 --- a/samples/tests/outputs/long-messages.c.bin.err.expected +++ /dev/null @@ -1,6 +0,0 @@ -[----] long-messages.c:4: Assertion failed: This is -[----] A long message -[----] Spawning multiple lines. -[----] Formatting is respected. -[FAIL] sample::long_msg -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/long-messages.c.bin.out.expected b/samples/tests/outputs/long-messages.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/long-messages.cc.bin.err.expected b/samples/tests/outputs/long-messages.cc.bin.err.expected deleted file mode 100644 index 0237997..0000000 --- a/samples/tests/outputs/long-messages.cc.bin.err.expected +++ /dev/null @@ -1,6 +0,0 @@ -[----] long-messages.cc:4: Assertion failed: This is -[----] A long message -[----] Spawning multiple lines. -[----] Formatting is respected. -[FAIL] sample::long_msg -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/long-messages.cc.bin.out.expected b/samples/tests/outputs/long-messages.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/tap_test.sh.err.expected b/samples/tests/outputs/tap_test.sh.err.expected deleted file mode 100644 index 56f3268..0000000 --- a/samples/tests/outputs/tap_test.sh.err.expected +++ /dev/null @@ -1,61 +0,0 @@ -TAP version 13 -1..2 -# Criterion v2.1.0 - -# Running 2 tests from misc -ok - misc::passing (0.00s) -not ok - misc::failing (0.00s) - simple.c:4: Assertion failed: The expression 0 is false. -TAP version 13 -1..3 -# Criterion v2.1.0 - -# Running 3 tests from simple -not ok - simple::wrong_signal (0.00s) -not ok - simple::uncaught unexpected signal after signal.c:16 -ok - simple::caught (0.00s) -TAP version 13 -1..6 -# Criterion v2.1.0 - -# Running 6 tests from asserts -ok - asserts::string (0.00s) -not ok - asserts::old_school (0.00s) - asserts.c:18: Assertion failed: The conditions for this assertion were not met. - asserts.c:17: Assertion failed: You can fail an assertion with a message from anywhere -ok - asserts::native (0.00s) -ok - asserts::float (0.00s) -not ok - asserts::base (0.00s) - asserts.c:12: Assertion failed: This assert runs - asserts.c:11: Assertion failed: assert is fatal, expect isn't -ok - asserts::array (0.00s) -TAP version 13 -1..3 -# Criterion v2.1.0 - -# Running 1 tests from suite2 -ok - suite2::test (0.00s) - -# Running 1 tests from suite1 -ok - suite1::test (0.00s) - -# Running 1 tests from disabled -ok - disabled::test # SKIP suite is disabled -TAP version 13 -1..1 -# Criterion v2.1.0 - -# Running 1 tests from sample -not ok - sample::long_msg (0.00s) - long-messages.c:4: Assertion failed: This is - A long message - Spawning multiple lines. - Formatting is respected. -TAP version 13 -1..2 -# Criterion v2.1.0 - -# Running 2 tests from misc -ok - misc::skipped This one is skipped # SKIP test is disabled -not ok - misc::failing Just a failing test (0.00s) - description.c:4: Assertion failed: The expression 0 is false. diff --git a/samples/tests/outputs/tap_test.sh.out.expected b/samples/tests/outputs/tap_test.sh.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/theories_regression.c.bin.err.expected b/samples/tests/outputs/theories_regression.c.bin.err.expected deleted file mode 100644 index 1262f80..0000000 --- a/samples/tests/outputs/theories_regression.c.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] theories_regression.c:34: Assertion failed: The conditions for this assertion were not met. -[----] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") -[FAIL] theory::misc -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/theories_regression.c.bin.out.expected b/samples/tests/outputs/theories_regression.c.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/theories_regression.cc.bin.err.expected b/samples/tests/outputs/theories_regression.cc.bin.err.expected deleted file mode 100644 index e149d58..0000000 --- a/samples/tests/outputs/theories_regression.cc.bin.err.expected +++ /dev/null @@ -1,4 +0,0 @@ -[----] theories_regression.cc:36: Assertion failed: The conditions for this assertion were not met. -[----] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") -[FAIL] theory::misc -[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/tests/outputs/theories_regression.cc.bin.out.expected b/samples/tests/outputs/theories_regression.cc.bin.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/outputs/xml_test.sh.err.expected b/samples/tests/outputs/xml_test.sh.err.expected deleted file mode 100644 index b3f8362..0000000 --- a/samples/tests/outputs/xml_test.sh.err.expected +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - simple.c:4: The expression 0 is false. - - - - - - - - - - - - - - - - - - - - - - - - asserts.c:18: The conditions for this assertion were not met. asserts.c:17: You can fail an assertion with a message from anywhere - - - - - - - asserts.c:12: This assert runs asserts.c:11: assert is fatal, expect isn't - - - - - - - - - - - - - - - - - - - - - - - - - - - - long-messages.c:4: This is A long message Spawning multiple lines. Formatting is respected. - - - - - - - - - - - - description.c:4: The expression 0 is false. - - - diff --git a/samples/tests/outputs/xml_test.sh.out.expected b/samples/tests/outputs/xml_test.sh.out.expected deleted file mode 100644 index e69de29..0000000 diff --git a/samples/tests/pattern.sh b/samples/tests/pattern.sh deleted file mode 100755 index c1b10a9..0000000 --- a/samples/tests/pattern.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -./simple.c.bin --pattern '*/passing' -./simple.c.bin --pattern '!(*/passing)' -./simple.c.bin --pattern '[pf]a@(ss|il)ing' -./simple.c.bin --pattern '@(+(nest)ed))' -./simple.c.bin --pattern '?(*(a|b))' -! ./simple.c.bin --pattern '?(malformed' -./simple.c.bin --pattern '[!azerty]assing' -./simple.c.bin --pattern '|pipe' -./simple.c.bin --pattern '\!(escaped' diff --git a/samples/tests/run_test.sh b/samples/tests/run_test.sh deleted file mode 100755 index 14d386d..0000000 --- a/samples/tests/run_test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -e - -cmp_dir=$1; shift -out_dir=$1; shift -bin_dir=$1; shift - -mkdir -p $out_dir - -if [ -f $cmp_dir/$1.in ]; then - sh -c "$bin_dir/$*" > $out_dir/$1.out 2> $out_dir/$1.err < $cmp_dir/$1.in -else - mkdir -p $(dirname $out_dir/$1) - sh -c "$bin_dir/$*" > $out_dir/$1.out 2> $out_dir/$1.err -fi - -if [ -f $cmp_dir/$1.out.expected ] && [ "$(md5sum $out_dir/$1.out | cut -d' ' -f1)" != "$(md5sum $cmp_dir/$1.out.expected | cut -d' ' -f1)" ]; then - if ! diff --strip-trailing-cr $out_dir/$1.out $cmp_dir/$1.out.expected ; then - exit 255 - fi -fi - -if [ -f $cmp_dir/$1.err.expected ] && [ "$(md5sum $out_dir/$1.err | cut -d' ' -f1)" != "$(md5sum $cmp_dir/$1.err.expected | cut -d' ' -f1)" ]; then - if ! diff --strip-trailing-cr $out_dir/$1.err $cmp_dir/$1.err.expected ; then - exit 255 - fi -fi diff --git a/samples/tests/tap_test.sh b/samples/tests/tap_test.sh deleted file mode 100755 index 9faca2d..0000000 --- a/samples/tests/tap_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -./simple.c.bin --tap --always-succeed -./signal.c.bin --tap --always-succeed -./asserts.c.bin --tap --always-succeed -./more-suites.c.bin --tap --always-succeed -./tests/long-messages.c.bin --tap --always-succeed -./description.c.bin --tap --always-succeed diff --git a/samples/tests/verbose.sh b/samples/tests/verbose.sh deleted file mode 100755 index 6bb1ef2..0000000 --- a/samples/tests/verbose.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -./simple.c.bin --verbose --always-succeed diff --git a/samples/tests/xml_test.sh b/samples/tests/xml_test.sh deleted file mode 100755 index d0740d9..0000000 --- a/samples/tests/xml_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -./simple.c.bin --xml --always-succeed -./signal.c.bin --xml --always-succeed -./asserts.c.bin --xml --always-succeed -./more-suites.c.bin --xml --always-succeed -./tests/long-messages.c.bin --xml --always-succeed -./description.c.bin --xml --always-succeed diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f23ac22..29c1d1a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,3 +32,20 @@ add_test(criterion_unit_tests criterion_unit_tests) set_property(TEST criterion_unit_tests PROPERTY ENVIRONMENT "CRITERION_NO_EARLY_EXIT=1" # for coverage ) + +if (NOT MSVC) # we disable the scripted tests when building with MSVC + +add_custom_target(cram_tests ${CMAKE_COMMAND} -E env + "PATH=$ENV{PATH}:${PROJECT_BINARY_DIR}/samples:${PROJECT_BINARY_DIR}/samples/tests" + "LC_ALL=en_US.utf8" + "CRITERION_ALWAYS_SUCCEED=1" + "CRITERION_SHORT_FILENAME=1" + "CRITERION_NO_EARLY_EXIT=1" # for coverage + "CRITERION_JOBS=1" # for output ordering + "CRITERION_DISABLE_TIME_MEASUREMENTS=1" # do not compare timings + cram -v ${CMAKE_CURRENT_SOURCE_DIR}/cram) +add_dependencies(cram_tests criterion_samples) + +add_test(cram_tests make cram_tests) + +endif () diff --git a/test/cram/asserts.t b/test/cram/asserts.t new file mode 100644 index 0000000..7802a11 --- /dev/null +++ b/test/cram/asserts.t @@ -0,0 +1,155 @@ +Test C assertions: + + $ asserts.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.c\x1b[0m:\x1b[0;31m11\x1b[0m: Assertion failed: assert is fatal, expect isn't (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.c\x1b[0m:\x1b[0;31m12\x1b[0m: Assertion failed: This assert runs (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::base (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.c\x1b[0m:\x1b[0;31m17\x1b[0m: Assertion failed: You can fail an assertion with a message from anywhere (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.c\x1b[0m:\x1b[0;31m18\x1b[0m: Assertion failed: The conditions for this assertion were not met. (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::old_school (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m6\x1b[0;1m | Passing: \x1b[0;32m4\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Test C++ assertions: + + $ asserts.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m83\x1b[0m: Assertion failed: The expression (&s1)[0 .. 2] == (&s2)[0 .. 2] is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::array (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m13\x1b[0m: Assertion failed: assert is fatal, expect isn't (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: This assert runs (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::base (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m89\x1b[0m: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception. (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::exception (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m19\x1b[0m: Assertion failed: You can fail an assertion with a message from anywhere (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1masserts.cc\x1b[0m:\x1b[0;31m20\x1b[0m: Assertion failed: The conditions for this assertion were not met. (esc) + [\x1b[0;31mFAIL\x1b[0m] asserts::old_school (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m7\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m4\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Using redirections + + $ redirect.c.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ redirect.cc.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing all assert messages + + $ failmessages.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m5\x1b[0m: Assertion failed: The expression (0) == (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m6\x1b[0m: Assertion failed: The expression (1) != (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m7\x1b[0m: Assertion failed: The expression (2) < (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m8\x1b[0m: Assertion failed: The expression (2) <= (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m9\x1b[0m: Assertion failed: The expression (1) > (2) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m10\x1b[0m: Assertion failed: The expression (1) >= (2) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m11\x1b[0m: Assertion failed: "" is not null. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m12\x1b[0m: Assertion failed: ((void *)0) is null. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: The expression (2) - (1) <= (0.1) && (1) - (2) <= (0.1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: The expression (2) - (2) > (0.1) || (2) - (2) > (0.1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m17\x1b[0m: Assertion failed: "foo" is not empty. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m18\x1b[0m: Assertion failed: "" is empty. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m19\x1b[0m: Assertion failed: The expression (as strings) ("abc") == ("abd") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m20\x1b[0m: Assertion failed: The expression (as strings) ("abc") != ("abc") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m21\x1b[0m: Assertion failed: The expression (as strings) ("abc") < ("aba") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m22\x1b[0m: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m23\x1b[0m: Assertion failed: The expression (as strings) ("abc") > ("abd") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m24\x1b[0m: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] messages::default (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m28\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m29\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m30\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m31\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m32\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m33\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m34\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m38\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m39\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m41\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m42\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m43\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m44\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m45\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m46\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m47\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.c\x1b[0m:\x1b[0;31m48\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;31mFAIL\x1b[0m] messages::user (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ failmessages.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m5\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m6\x1b[0m: Assertion failed: The expression (0) == (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m7\x1b[0m: Assertion failed: The expression (1) != (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m8\x1b[0m: Assertion failed: The expression (2) < (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m9\x1b[0m: Assertion failed: The expression (2) <= (1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m10\x1b[0m: Assertion failed: The expression (1) > (2) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m11\x1b[0m: Assertion failed: The expression (1) >= (2) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m12\x1b[0m: Assertion failed: "" is not null. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m13\x1b[0m: Assertion failed: __null is null. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: The expression (2) - (1) <= (0.1) && (1) - (2) <= (0.1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m16\x1b[0m: Assertion failed: The expression (2) - (2) > (0.1) || (2) - (2) > (0.1) is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m18\x1b[0m: Assertion failed: "foo" is not empty. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m19\x1b[0m: Assertion failed: "" is empty. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m20\x1b[0m: Assertion failed: The expression (as strings) ("abc") == ("abd") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m21\x1b[0m: Assertion failed: The expression (as strings) ("abc") != ("abc") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m22\x1b[0m: Assertion failed: The expression (as strings) ("abc") < ("aba") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m23\x1b[0m: Assertion failed: The expression (as strings) ("abc") <= ("aba") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m24\x1b[0m: Assertion failed: The expression (as strings) ("abc") > ("abd") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m25\x1b[0m: Assertion failed: The expression (as strings) ("abc") >= ("abd") is false. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m27\x1b[0m: Assertion failed: The statement `throw std::exception()` did not throw an instance of the `std::bad_alloc` exception. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m28\x1b[0m: Assertion failed: The statement `throw std::exception()` did throw an instance of the `std::exception` exception. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m29\x1b[0m: Assertion failed: The statement `{}` did not throw any exception. (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m30\x1b[0m: Assertion failed: The statement `throw std::exception()` threw some exception. (esc) + [\x1b[0;31mFAIL\x1b[0m] messages::default (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m34\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m37\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m38\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m39\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m40\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m41\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m42\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m44\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m45\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m47\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m48\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m49\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m50\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m51\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m52\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m53\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m54\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m56\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m57\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m58\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mfailmessages.cc\x1b[0m:\x1b[0;31m59\x1b[0m: Assertion failed: foo bar (esc) + [\x1b[0;31mFAIL\x1b[0m] messages::user (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Long assert messages should wrap + + $ long-messages.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mlong-messages.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: This is (esc) + [\x1b[0;34m----\x1b[0m] A long message (esc) + [\x1b[0;34m----\x1b[0m] Spawning multiple lines. (esc) + [\x1b[0;34m----\x1b[0m] Formatting is respected. (esc) + [\x1b[0;31mFAIL\x1b[0m] sample::long_msg (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ long-messages.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mlong-messages.cc\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: This is (esc) + [\x1b[0;34m----\x1b[0m] A long message (esc) + [\x1b[0;34m----\x1b[0m] Spawning multiple lines. (esc) + [\x1b[0;34m----\x1b[0m] Formatting is respected. (esc) + [\x1b[0;31mFAIL\x1b[0m] sample::long_msg (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Using redirections + + $ redirect.c.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ redirect.cc.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/core.t b/test/cram/core.t new file mode 100644 index 0000000..6910f7d --- /dev/null +++ b/test/cram/core.t @@ -0,0 +1,153 @@ +Checking the version + + $ simple.c.bin --version + Tests compiled with Criterion v2.2.0 + + $ simple.cc.bin --version + Tests compiled with Criterion v2.2.0 + +Running the simplest test + + $ simple.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ simple.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.cc\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Running with verbose output + + $ simple.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) + [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ simple.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.cc\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) + [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Display the help message + + $ simple.c.bin --help + Tests compiled with Criterion v2.2.0 + + usage: simple.c.bin OPTIONS + options: + -h or --help: prints this message + -q or --quiet: disables all logging + -v or --version: prints the version of criterion these tests have been linked against + -l or --list: prints all the tests in a list + -jN or --jobs N: use N concurrent jobs + -f or --fail-fast: exit after the first failure + --ascii: don't use fancy unicode symbols or colors in the output + -S or --short-filename: only display the base name of the source file on a failure + --pattern [PATTERN]: run tests matching the given pattern + --tap[=FILE]: writes TAP report in FILE (no file or "-" means stderr) + --xml[=FILE]: writes XML report in FILE (no file or "-" means stderr) + --always-succeed: always exit with 0 + --no-early-exit: do not exit the test worker prematurely after the test + --verbose[=level]: sets verbosity to level (1 by default) + -OP:F or --output=PROVIDER=FILE: write test report to FILE using the specified provider + + $ simple.cc.bin --help + Tests compiled with Criterion v2.2.0 + + usage: simple.cc.bin OPTIONS + options: + -h or --help: prints this message + -q or --quiet: disables all logging + -v or --version: prints the version of criterion these tests have been linked against + -l or --list: prints all the tests in a list + -jN or --jobs N: use N concurrent jobs + -f or --fail-fast: exit after the first failure + --ascii: don't use fancy unicode symbols or colors in the output + -S or --short-filename: only display the base name of the source file on a failure + --pattern [PATTERN]: run tests matching the given pattern + --tap[=FILE]: writes TAP report in FILE (no file or "-" means stderr) + --xml[=FILE]: writes XML report in FILE (no file or "-" means stderr) + --always-succeed: always exit with 0 + --no-early-exit: do not exit the test worker prematurely after the test + --verbose[=level]: sets verbosity to level (1 by default) + -OP:F or --output=PROVIDER=FILE: write test report to FILE using the specified provider + +Listing the tests + + $ simple.c.bin --list + misc: 2 tests + ├── failing + └── passing + + $ simple.cc.bin --list + misc: 2 tests + ├── failing + └── passing + +Testing the fixtures + + $ fixtures.c.bin + Runs before the test + Runs after the test + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ fixtures.cc.bin + Runs before the test + Runs after the test + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing descriptions + + $ description.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] Just a failing test (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mdescription.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test is disabled (esc) + [\x1b[0;34m----\x1b[0m] This one is skipped (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ description.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] Just a failing test (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mdescription.cc\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test is disabled (esc) + [\x1b[0;34m----\x1b[0m] This one is skipped (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing report hooks + + $ report.c.bin + criterion_init + [\x1b[0;34m----\x1b[0m] \x1b[0;1mreport.c\x1b[0m:\x1b[0;31m5\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] sample::test (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + testing test in category sample + Asserts: [1 passed, 1 failed, 2 total] + criterion_fini + + $ report.cc.bin + criterion_init + [\x1b[0;34m----\x1b[0m] \x1b[0;1mreport.cc\x1b[0m:\x1b[0;31m5\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] sample::test (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + testing test in category sample + Asserts: [1 passed, 1 failed, 2 total] + criterion_fini diff --git a/test/cram/crashes.t b/test/cram/crashes.t new file mode 100644 index 0000000..8db3f35 --- /dev/null +++ b/test/cram/crashes.t @@ -0,0 +1,113 @@ +Simple crashes & signal testing + + $ signal.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1msignal.c\x1b[0m:\x1b[0;31m16\x1b[0m: Unexpected signal caught below this line! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::uncaught: CRASH! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::wrong_signal (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m3\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + + $ signal.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1msignal.cc\x1b[0m:\x1b[0;31m16\x1b[0m: Unexpected signal caught below this line! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::uncaught: CRASH! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::wrong_signal (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m3\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + +Simple crashes & signal testing (verbose) + + $ signal.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33msimple\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] simple::caught (esc) + [\x1b[0;32mPASS\x1b[0m] simple::caught (esc) + [\x1b[0;34mRUN \x1b[0m] simple::uncaught (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msignal.c\x1b[0m:\x1b[0;31m16\x1b[0m: Unexpected signal caught below this line! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::uncaught: CRASH! (esc) + [\x1b[0;34mRUN \x1b[0m] simple::wrong_signal (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::wrong_signal (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m3\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + + $ signal.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33msimple\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] simple::caught (esc) + [\x1b[0;32mPASS\x1b[0m] simple::caught (esc) + [\x1b[0;34mRUN \x1b[0m] simple::uncaught (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msignal.cc\x1b[0m:\x1b[0;31m16\x1b[0m: Unexpected signal caught below this line! (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::uncaught: CRASH! (esc) + [\x1b[0;34mRUN \x1b[0m] simple::wrong_signal (esc) + [\x1b[0;31mFAIL\x1b[0m] simple::wrong_signal (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m3\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + +Crashes outside of the test + + $ other-crashes.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::setup_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::teardown_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + + $ other-crashes.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::setup_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::teardown_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + +Crashes outside of the test (verbose) + + $ other-crashes.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::setup_crash (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::setup_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34mRUN \x1b[0m] misc::teardown_crash (esc) + [\x1b[0;32mPASS\x1b[0m] misc::teardown_crash (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::teardown_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + + $ other-crashes.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::setup_crash (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::setup_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34mRUN \x1b[0m] misc::teardown_crash (esc) + [\x1b[0;32mPASS\x1b[0m] misc::teardown_crash (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `misc::teardown_crash` crashed during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc) + +C++ Exceptions should be handled + + $ exception.cc.bin + [\x1b[0;34m----\x1b[0m] exception::raise_any: Caught some unexpected exception during the test execution. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_any (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_any_fini: Caught some unexpected exception during the test finalization. (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_any_init: Caught some unexpected exception during the test initialization. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_any_init (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std: Caught an unexpected exception during the test execution: Some exception message. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_std (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std_fini: Caught an unexpected exception during the test finalization: Some exception message. (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std_init: Caught an unexpected exception during the test initialization: Some exception message. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_std_init (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m6\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m4\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +C++ Exceptions should be handled (verbose) + + $ exception.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m6\x1b[0m tests from \x1b[0;33mexception\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_any (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_any: Caught some unexpected exception during the test execution. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_any (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_any_fini (esc) + [\x1b[0;32mPASS\x1b[0m] exception::raise_any_fini (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_any_fini: Caught some unexpected exception during the test finalization. (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_any_init (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_any_init: Caught some unexpected exception during the test initialization. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_any_init (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_std (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std: Caught an unexpected exception during the test execution: Some exception message. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_std (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_std_fini (esc) + [\x1b[0;32mPASS\x1b[0m] exception::raise_std_fini (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std_fini: Caught an unexpected exception during the test finalization: Some exception message. (esc) + [\x1b[0;34mRUN \x1b[0m] exception::raise_std_init (esc) + [\x1b[0;34m----\x1b[0m] exception::raise_std_init: Caught an unexpected exception during the test initialization: Some exception message. (esc) + [\x1b[0;31mFAIL\x1b[0m] exception::raise_std_init (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m6\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m4\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/early-exit.t b/test/cram/early-exit.t new file mode 100644 index 0000000..32f9143 --- /dev/null +++ b/test/cram/early-exit.t @@ -0,0 +1,13 @@ +Testing --no-early-exit + + $ simple.c.bin --no-early-exit + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing CRITERION_NO_EARLY_EXIT + + $ CRITERION_NO_EARLY_EXIT=0 simple.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/exit.t b/test/cram/exit.t new file mode 100644 index 0000000..127851a --- /dev/null +++ b/test/cram/exit.t @@ -0,0 +1,35 @@ +Testing tests that call exit() + + $ exit.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33mexit\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] exit::expected_exit (esc) + [\x1b[0;32mPASS\x1b[0m] exit::expected_exit (esc) + [\x1b[0;34mRUN \x1b[0m] exit::normal (esc) + [\x1b[0;32mPASS\x1b[0m] exit::normal (esc) + [\x1b[0;34mRUN \x1b[0m] exit::unexpected_exit (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit::unexpected_exit` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mexit_with_fixtures\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::fini_exits (esc) + [\x1b[0;32mPASS\x1b[0m] exit_with_fixtures::fini_exits (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::init_exits (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m2\x1b[0;1m \x1b[0m (esc) + + $ exit.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33mexit\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] exit::expected_exit (esc) + [\x1b[0;32mPASS\x1b[0m] exit::expected_exit (esc) + [\x1b[0;34mRUN \x1b[0m] exit::normal (esc) + [\x1b[0;32mPASS\x1b[0m] exit::normal (esc) + [\x1b[0;34mRUN \x1b[0m] exit::unexpected_exit (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit::unexpected_exit` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mexit_with_fixtures\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::fini_exits (esc) + [\x1b[0;32mPASS\x1b[0m] exit_with_fixtures::fini_exits (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::init_exits (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown.\x1b[0m (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m2\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/fail-fast.t b/test/cram/fail-fast.t new file mode 100644 index 0000000..9e0ee0a --- /dev/null +++ b/test/cram/fail-fast.t @@ -0,0 +1,7 @@ +Testing --fail-fast + + $ simple.c.bin --fail-fast --verbose + +Testing CRITERION_FAIL_FAST + + $ CRITERION_FAIL_FAST=1 simple.c.bin --verbose diff --git a/test/cram/json.t b/test/cram/json.t new file mode 100644 index 0000000..2b18e7c --- /dev/null +++ b/test/cram/json.t @@ -0,0 +1,320 @@ +Testing multiple samples with --json + + $ simple.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "misc", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "passing", + "assertions": 1, + "status": "PASSED" + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + "simple.c:4: The expression 0 is false." + ] + } + ] + } + ] + } + + $ signal.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 1, + "failed": 2, + "errored": 1, + "skipped": 0, + "test_suites": [ + { + "name": "simple", + "passed": 1, + "failed": 2, + "errored": 1, + "skipped": 0, + "tests": [ + { + "name": "wrong_signal", + "assertions": 0, + "status": "FAILED", + "messages": [ + + ] + }, + { + "name": "uncaught", + "assertions": 0, + "status": "ERRORED", + "messages": ["The test crashed."] + }, + { + "name": "caught", + "assertions": 0, + "status": "PASSED" + } + ] + } + ] + } + + $ asserts.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 4, + "failed": 2, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "asserts", + "passed": 4, + "failed": 2, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "string", + "assertions": 10, + "status": "PASSED" + }, + { + "name": "old_school", + "assertions": 2, + "status": "FAILED", + "messages": [ + "asserts.c:18: The conditions for this assertion were not met.", + "asserts.c:17: You can fail an assertion with a message from anywhere" + ] + }, + { + "name": "native", + "assertions": 8, + "status": "PASSED" + }, + { + "name": "float", + "assertions": 2, + "status": "PASSED" + }, + { + "name": "base", + "assertions": 6, + "status": "FAILED", + "messages": [ + "asserts.c:12: This assert runs", + "asserts.c:11: assert is fatal, expect isn't" + ] + }, + { + "name": "array", + "assertions": 3, + "status": "PASSED" + } + ] + } + ] + } + + $ more-suites.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 2, + "failed": 0, + "errored": 0, + "skipped": 1, + "test_suites": [ + { + "name": "suite2", + "passed": 1, + "failed": 0, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "test", + "assertions": 1, + "status": "PASSED" + } + ] + }, + { + "name": "suite1", + "passed": 1, + "failed": 0, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "test", + "assertions": 1, + "status": "PASSED" + } + ] + }, + { + "name": "disabled", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 1, + "tests": [ + { + "name": "test", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + } + ] + } + ] + } + + $ long-messages.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "sample", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "long_msg", + "assertions": 1, + "status": "FAILED", + "messages": [ + "long-messages.c:4: This is", + " A long message", + " Spawning multiple lines.", + " Formatting is respected." + ] + } + ] + } + ] + } + + $ description.c.bin --json + { + "id": "Criterion v2.2.0", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 1, + "test_suites": [ + { + "name": "misc", + "passed": 0, + "failed": 1, + "errored": 0, + "skipped": 1, + "tests": [ + { + "name": "skipped", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + "description.c:4: The expression 0 is false." + ] + } + ] + } + ] + } + +Testing --output=json + + $ simple.c.bin --output=json:- + { + "id": "Criterion v2.2.0", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "misc", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "passing", + "assertions": 1, + "status": "PASSED" + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + "simple.c:4: The expression 0 is false." + ] + } + ] + } + ] + } + +Testing CRITERION_OUTPUTS + + $ CRITERION_OUTPUTS=json:- simple.c.bin + { + "id": "Criterion v2.2.0", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "test_suites": [ + { + "name": "misc", + "passed": 1, + "failed": 1, + "errored": 0, + "skipped": 0, + "tests": [ + { + "name": "passing", + "assertions": 1, + "status": "PASSED" + }, + { + "name": "failing", + "assertions": 1, + "status": "FAILED", + "messages": [ + "simple.c:4: The expression 0 is false." + ] + } + ] + } + ] + } diff --git a/test/cram/parameterized.t b/test/cram/parameterized.t new file mode 100644 index 0000000..ceb478f --- /dev/null +++ b/test/cram/parameterized.t @@ -0,0 +1,117 @@ +Using parameterized tests + + $ parameterized.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: foo (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: bar (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: baz (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m9\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m9\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ parameterized.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: foo (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: bar (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: baz (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m9\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m9\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Using parameterized tests (verbose) + + $ parameterized.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33mparams\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m76\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: foo (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: bar (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.c\x1b[0m:\x1b[0;31m15\x1b[0m: Assertion failed: Parameter: baz (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m9\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m9\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ parameterized.cc.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m3\x1b[0m tests from \x1b[0;33mparams\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::cleanup (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m74\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::cleanup (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (1, 2.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (3, 4.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::multiple (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m35\x1b[0m: Assertion failed: Parameters: (5, 6.000000) (esc) + [\x1b[0;31mFAIL\x1b[0m] params::multiple (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: foo (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: bar (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34mRUN \x1b[0m] params::str (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1mparameterized.cc\x1b[0m:\x1b[0;31m14\x1b[0m: Assertion failed: Parameter: baz (esc) + [\x1b[0;31mFAIL\x1b[0m] params::str (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m9\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m9\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Parameterized tests should be listed as one test + + $ parameterized.c.bin --list + params: 3 tests + ├── cleanup + ├── multiple + └── str diff --git a/test/cram/pattern.t b/test/cram/pattern.t new file mode 100644 index 0000000..fdfc55a --- /dev/null +++ b/test/cram/pattern.t @@ -0,0 +1,84 @@ +Selecting misc::passing + + $ simple.c.bin --pattern '*/passing' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) + [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Selecting misc::failing + + $ simple.c.bin --pattern '!(*/passing)' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Selecting both misc::passing and misc::failing + + $ simple.c.bin --pattern 'misc/[pf]a@(ss|il)ing' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) + [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) + [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) + [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) + [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing nested extglob patterns + + $ simple.c.bin --pattern '@(+(nest)ed))' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing one or more + + $ simple.c.bin --pattern '?(*(a|b))' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing malformed pattern + + $ simple.c.bin --pattern '?(malformed' --verbose + pattern error: mismatching parenthesis + [1] + +Testing range negation + + $ simple.c.bin --pattern 'misc/[!azerty]assing' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) + [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing unparenthesized pipe + + $ simple.c.bin --pattern '|pipe' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Testing special char escaping + + $ simple.c.bin --pattern '\!(escaped' --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/redirect.t b/test/cram/redirect.t new file mode 100644 index 0000000..8f92a0d --- /dev/null +++ b/test/cram/redirect.t @@ -0,0 +1,7 @@ +Using redirections + + $ redirect.c.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ redirect.cc.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/suites.t b/test/cram/suites.t new file mode 100644 index 0000000..1d37be6 --- /dev/null +++ b/test/cram/suites.t @@ -0,0 +1,45 @@ +Using test suites + + $ suites.c.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ suites.cc.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ more-suites.c.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ more-suites.cc.bin + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Listing tests in multiple suites + + $ suites.c.bin --list + first_suite: 1 test + └── test + second_suite: 1 test + └── test + +Running the tests with verbose output + + $ suites.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33mfirst_suite\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] first_suite::test (esc) + [\x1b[0;32mPASS\x1b[0m] first_suite::test (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33msecond_suite\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] second_suite::test (esc) + [\x1b[0;32mPASS\x1b[0m] second_suite::test (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ more-suites.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.0 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33mdisabled\x1b[0m: (esc) + [\x1b[0;33mSKIP\x1b[0m] disabled::test: Suite is disabled (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33msuite1\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] suite1::test (esc) + [\x1b[0;32mPASS\x1b[0m] suite1::test (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33msuite2\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] suite2::test (esc) + [\x1b[0;32mPASS\x1b[0m] suite2::test (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m2\x1b[0;1m | Passing: \x1b[0;32m2\x1b[0;1m | Failing: \x1b[0;31m0\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/tap.t b/test/cram/tap.t new file mode 100644 index 0000000..eac398f --- /dev/null +++ b/test/cram/tap.t @@ -0,0 +1,98 @@ +Testing multiple samples with --tap + + $ simple.c.bin --tap + TAP version 13 + 1..2 + # Criterion v2.2.0 + + # Running 2 tests from misc + ok - misc::passing + not ok - misc::failing + simple.c:4: Assertion failed: The expression 0 is false. + + $ signal.c.bin --tap + TAP version 13 + 1..3 + # Criterion v2.2.0 + + # Running 3 tests from simple + not ok - simple::wrong_signal + not ok - simple::uncaught unexpected signal after signal.c:16 + ok - simple::caught + + $ asserts.c.bin --tap + TAP version 13 + 1..6 + # Criterion v2.2.0 + + # Running 6 tests from asserts + ok - asserts::string + not ok - asserts::old_school + asserts.c:18: Assertion failed: The conditions for this assertion were not met. + asserts.c:17: Assertion failed: You can fail an assertion with a message from anywhere + ok - asserts::native + ok - asserts::float + not ok - asserts::base + asserts.c:12: Assertion failed: This assert runs + asserts.c:11: Assertion failed: assert is fatal, expect isn't + ok - asserts::array + + $ more-suites.c.bin --tap + TAP version 13 + 1..3 + # Criterion v2.2.0 + + # Running 1 tests from suite2 + ok - suite2::test + + # Running 1 tests from suite1 + ok - suite1::test + + # Running 1 tests from disabled + ok - disabled::test # SKIP suite is disabled + + $ long-messages.c.bin --tap + TAP version 13 + 1..1 + # Criterion v2.2.0 + + # Running 1 tests from sample + not ok - sample::long_msg + long-messages.c:4: Assertion failed: This is + A long message + Spawning multiple lines. + Formatting is respected. + + $ description.c.bin --tap + TAP version 13 + 1..2 + # Criterion v2.2.0 + + # Running 2 tests from misc + ok - misc::skipped This one is skipped # SKIP test is disabled + not ok - misc::failing Just a failing test + description.c:4: Assertion failed: The expression 0 is false. + +Testing --output=tap + + $ simple.c.bin --output=tap:- + TAP version 13 + 1..2 + # Criterion v2.2.0 + + # Running 2 tests from misc + ok - misc::passing + not ok - misc::failing + simple.c:4: Assertion failed: The expression 0 is false. + +Testing CRITERION_OUTPUTS + + $ CRITERION_OUTPUTS=tap:- simple.c.bin + TAP version 13 + 1..2 + # Criterion v2.2.0 + + # Running 2 tests from misc + ok - misc::passing + not ok - misc::failing + simple.c:4: Assertion failed: The expression 0 is false. diff --git a/test/cram/theories.t b/test/cram/theories.t new file mode 100644 index 0000000..53d3eac --- /dev/null +++ b/test/cram/theories.t @@ -0,0 +1,19 @@ +Using theories + + $ theories_regression.c.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mtheories_regression.c\x1b[0m:\x1b[0;31m34\x1b[0m: Assertion failed: The conditions for this assertion were not met. (esc) + [\x1b[0;34m----\x1b[0m] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") (esc) + [\x1b[0;31mFAIL\x1b[0m] theory::misc (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + + $ theories_regression.cc.bin + [\x1b[0;34m----\x1b[0m] \x1b[0;1mtheories_regression.cc\x1b[0m:\x1b[0;31m36\x1b[0m: Assertion failed: The conditions for this assertion were not met. (esc) + [\x1b[0;34m----\x1b[0m] Theory theory::misc failed with the following parameters: ('a', true, 1, 1, 3.14f, 3.14, "test", "other test") (esc) + [\x1b[0;31mFAIL\x1b[0m] theory::misc (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0;31m0\x1b[0;1m \x1b[0m (esc) + +Theories should be listed as only one test + + $ theories_regression.c.bin --list + theory: 1 test + └── misc diff --git a/test/cram/xml.t b/test/cram/xml.t new file mode 100644 index 0000000..cd6699a --- /dev/null +++ b/test/cram/xml.t @@ -0,0 +1,125 @@ +Testing multiple samples with --xml + + $ simple.c.bin --xml + + + + + + + + simple.c:4: The expression 0 is false. + + + + + $ signal.c.bin --xml + + + + + + + + + + + + + + + $ asserts.c.bin --xml + + + + + + + + asserts.c:18: The conditions for this assertion were not met. asserts.c:17: You can fail an assertion with a message from anywhere + + + + + + + asserts.c:12: This assert runs asserts.c:11: assert is fatal, expect isn't + + + + + + + $ more-suites.c.bin --xml + + + + + + + + + + + + + + + + + + + $ long-messages.c.bin --xml + + + + + + long-messages.c:4: This is A long message Spawning multiple lines. Formatting is respected. + + + + + $ description.c.bin --xml + + + + + + + + + description.c:4: The expression 0 is false. + + + + +Testing --output=xml + + $ simple.c.bin --output=xml:- + + + + + + + + simple.c:4: The expression 0 is false. + + + + +Testing CRITERION_OUTPUTS + + $ CRITERION_OUTPUTS=xml:- simple.c.bin + + + + + + + + simple.c:4: The expression 0 is false. + + +