From 9db9b5776a4f794a9381856158f26a14bdb4a43e Mon Sep 17 00:00:00 2001 From: Snaipe Date: Thu, 20 Aug 2015 07:20:29 +0200 Subject: [PATCH] Added output comparison tests to prevent regressions --- .gitignore | 1 + .travis.yml | 2 +- samples/CMakeLists.txt | 10 ++++++++++ samples/asserts.err.expected | 6 ++++++ samples/asserts.out.expected | 0 samples/description.err.expected | 3 +++ samples/description.out.expected | 0 samples/fixtures.err.expected | 1 + samples/fixtures.out.expected | 2 ++ samples/long-messages.err.expected | 6 ++++++ samples/long-messages.out.expected | 0 samples/more-suites.err.expected | 1 + samples/more-suites.out.expected | 0 samples/other-crashes.err.expected | 3 +++ samples/other-crashes.out.expected | 0 samples/report.err.expected | 3 +++ samples/report.out.expected | 5 +++++ samples/signal.err.expected | 4 ++++ samples/signal.out.expected | 0 samples/simple.err.expected | 3 +++ samples/simple.out.expected | 0 samples/suites.err.expected | 1 + samples/suites.out.expected | 0 samples/tests/run_test.sh | 17 +++++++++++++++++ 24 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 samples/asserts.err.expected create mode 100644 samples/asserts.out.expected create mode 100644 samples/description.err.expected create mode 100644 samples/description.out.expected create mode 100644 samples/fixtures.err.expected create mode 100644 samples/fixtures.out.expected create mode 100644 samples/long-messages.err.expected create mode 100644 samples/long-messages.out.expected create mode 100644 samples/more-suites.err.expected create mode 100644 samples/more-suites.out.expected create mode 100644 samples/other-crashes.err.expected create mode 100644 samples/other-crashes.out.expected create mode 100644 samples/report.err.expected create mode 100644 samples/report.out.expected create mode 100644 samples/signal.err.expected create mode 100644 samples/signal.out.expected create mode 100644 samples/simple.err.expected create mode 100644 samples/simple.out.expected create mode 100644 samples/suites.err.expected create mode 100644 samples/suites.out.expected create mode 100644 samples/tests/run_test.sh diff --git a/.gitignore b/.gitignore index 48d1edb..8dd4a19 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ !*.rst !samples/tests/*.sh !*.po +!samples/*.expected !LICENSE !HEADER diff --git a/.travis.yml b/.travis.yml index eabcb44..4df2cb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ script: after_success: - make coveralls after_failure: -- cat Testing/Temporary/LastTest.log +- cat Testing/Temporary/LastTest.log samples/*.{out,err} ../samples/tests/*.{out,err} env: global: secure: bzZcWjdqoTgceC40kEBucx7NuWYJPk+rxgF3UJJDXi+ijQAFYPv70p5eVsGR6rfc+XgqXCxcUFQtuL4ZVt7QEfVk1ZOJITNeHbKIeKaEYS4nX8mFf+CBeEm9bJGZ04KiQJdJu5mzzAHvXbW7roGXDGWe1Bjnk5wwA+dNUCa7H04= diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 206c049..49456a8 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -34,6 +34,11 @@ foreach(sample ${SAMPLES}) set_property(TEST ${sample} PROPERTY ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" ) + + add_test(${sample}_compare sh ${CMAKE_CURRENT_LIST_DIR}/tests/run_test.sh ./${sample} "${CMAKE_CURRENT_LIST_DIR}") + set_property(TEST ${sample}_compare PROPERTY + ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" + ) endforeach() foreach(script ${SCRIPTS}) @@ -41,4 +46,9 @@ foreach(script ${SCRIPTS}) set_property(TEST ${script} PROPERTY ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" ) + + add_test(${script}_compare sh ${CMAKE_CURRENT_LIST_DIR}/tests/run_test.sh ${CMAKE_CURRENT_LIST_DIR}/tests/${script}.sh "${CMAKE_CURRENT_LIST_DIR}") + set_property(TEST ${script}_compare PROPERTY + ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1" + ) endforeach() diff --git a/samples/asserts.err.expected b/samples/asserts.err.expected new file mode 100644 index 0000000..edb004b --- /dev/null +++ b/samples/asserts.err.expected @@ -0,0 +1,6 @@ +[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:11: Assertion failed: assert is fatal, expect isn't +[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:12: Assertion failed: This assert runs +[FAIL] asserts::base: (0,00s) +[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:20: Assertion failed: The conditions for this test were not met. +[FAIL] asserts::old_school: (0,00s) +[====] Synthesis: Tested: 6 | Passing: 4 | Failing: 2 | Crashing: 0  diff --git a/samples/asserts.out.expected b/samples/asserts.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/description.err.expected b/samples/description.err.expected new file mode 100644 index 0000000..b5209d0 --- /dev/null +++ b/samples/description.err.expected @@ -0,0 +1,3 @@ +[----] /home/snaipe/workspace/c/criterion/samples/description.c:4: Assertion failed: 0 +[FAIL] misc::failing: (0,00s) +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/description.out.expected b/samples/description.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/fixtures.err.expected b/samples/fixtures.err.expected new file mode 100644 index 0000000..a34f221 --- /dev/null +++ b/samples/fixtures.err.expected @@ -0,0 +1 @@ +[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0  diff --git a/samples/fixtures.out.expected b/samples/fixtures.out.expected new file mode 100644 index 0000000..a74afed --- /dev/null +++ b/samples/fixtures.out.expected @@ -0,0 +1,2 @@ +Runs before the test +Runs after the test diff --git a/samples/long-messages.err.expected b/samples/long-messages.err.expected new file mode 100644 index 0000000..2fb7513 --- /dev/null +++ b/samples/long-messages.err.expected @@ -0,0 +1,6 @@ +[----] /home/snaipe/workspace/c/criterion/samples/long-messages.c:4: Assertion failed: This is +[----] A long message +[----] Spawning multiple lines. +[----] Formatting is respected. +[FAIL] sample::long_msg: (0,00s) +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/long-messages.out.expected b/samples/long-messages.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/more-suites.err.expected b/samples/more-suites.err.expected new file mode 100644 index 0000000..09eb50f --- /dev/null +++ b/samples/more-suites.err.expected @@ -0,0 +1 @@ +[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/more-suites.out.expected b/samples/more-suites.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/other-crashes.err.expected b/samples/other-crashes.err.expected new file mode 100644 index 0000000..41d3cb8 --- /dev/null +++ b/samples/other-crashes.err.expected @@ -0,0 +1,3 @@ +[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown. +[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown. +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1  diff --git a/samples/other-crashes.out.expected b/samples/other-crashes.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/report.err.expected b/samples/report.err.expected new file mode 100644 index 0000000..e1ca348 --- /dev/null +++ b/samples/report.err.expected @@ -0,0 +1,3 @@ +[----] /home/snaipe/workspace/c/criterion/samples/report.c:5: Assertion failed: 0 +[FAIL] sample::test: (0,00s) +[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0  diff --git a/samples/report.out.expected b/samples/report.out.expected new file mode 100644 index 0000000..20217bd --- /dev/null +++ b/samples/report.out.expected @@ -0,0 +1,5 @@ +criterion_init +criterion_init +testing test in category sample +Asserts: [1 passed, 1 failed, 2 total] +criterion_fini diff --git a/samples/signal.err.expected b/samples/signal.err.expected new file mode 100644 index 0000000..5fe1a9b --- /dev/null +++ b/samples/signal.err.expected @@ -0,0 +1,4 @@ +[----] /home/snaipe/workspace/c/criterion/samples/signal.c:16: Unexpected signal caught below this line! +[FAIL] simple::uncaught: CRASH! +[FAIL] simple::wrong_signal: (0,00s) +[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1  diff --git a/samples/signal.out.expected b/samples/signal.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/simple.err.expected b/samples/simple.err.expected new file mode 100644 index 0000000..8227be5 --- /dev/null +++ b/samples/simple.err.expected @@ -0,0 +1,3 @@ +[----] /home/snaipe/workspace/c/criterion/samples/simple.c:4: Assertion failed: 0 +[FAIL] misc::failing: (0,00s) +[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0  diff --git a/samples/simple.out.expected b/samples/simple.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/suites.err.expected b/samples/suites.err.expected new file mode 100644 index 0000000..09eb50f --- /dev/null +++ b/samples/suites.err.expected @@ -0,0 +1 @@ +[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0  diff --git a/samples/suites.out.expected b/samples/suites.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/samples/tests/run_test.sh b/samples/tests/run_test.sh new file mode 100644 index 0000000..88e5d94 --- /dev/null +++ b/samples/tests/run_test.sh @@ -0,0 +1,17 @@ +#!/bin/sh -e + +if [ -f $1.in ]; then + sh -c "$1" > $1.out 2> $1.err < $1.in +else + sh -c "$1" > $1.out 2> $1.err +fi + +if [ -f $2/$1.out.expected ] && [ "$(md5sum $1.out | cut -d' ' -f1)" != "$(md5sum $2/$1.out.expected | cut -d' ' -f1)" ]; then + diff $1.out $2/$1.out.expected + exit 255 +fi + +if [ -f $2/$1.err.expected ] && [ "$(md5sum $1.err | cut -d' ' -f1)" != "$(md5sum $2/$1.err.expected | cut -d' ' -f1)" ]; then + diff $1.err $2/$1.err.expected + exit 255 +fi