Added output comparison tests to prevent regressions

.gitignore

@@ -9,6 +9,7 @@
 !*.rst
 !samples/tests/*.sh
 !*.po
+!samples/*.expected
 
 !LICENSE
 !HEADER

.travis.yml

@@ -19,7 +19,7 @@ script:
 after_success:
 - make coveralls
 after_failure:
-- cat Testing/Temporary/LastTest.log
+- cat Testing/Temporary/LastTest.log samples/*.{out,err} ../samples/tests/*.{out,err}
 env:
   global:
     secure: bzZcWjdqoTgceC40kEBucx7NuWYJPk+rxgF3UJJDXi+ijQAFYPv70p5eVsGR6rfc+XgqXCxcUFQtuL4ZVt7QEfVk1ZOJITNeHbKIeKaEYS4nX8mFf+CBeEm9bJGZ04KiQJdJu5mzzAHvXbW7roGXDGWe1Bjnk5wwA+dNUCa7H04=

samples/CMakeLists.txt

@@ -34,6 +34,11 @@ foreach(sample ${SAMPLES})
     set_property(TEST ${sample} PROPERTY
         ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1"
     )
+
+    add_test(${sample}_compare sh ${CMAKE_CURRENT_LIST_DIR}/tests/run_test.sh ./${sample} "${CMAKE_CURRENT_LIST_DIR}")
+    set_property(TEST ${sample}_compare PROPERTY
+        ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1"
+    )
 endforeach()
 
 foreach(script ${SCRIPTS})
@@ -41,4 +46,9 @@ foreach(script ${SCRIPTS})
     set_property(TEST ${script} PROPERTY
         ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1"
     )
+
+    add_test(${script}_compare sh ${CMAKE_CURRENT_LIST_DIR}/tests/run_test.sh ${CMAKE_CURRENT_LIST_DIR}/tests/${script}.sh "${CMAKE_CURRENT_LIST_DIR}")
+    set_property(TEST ${script}_compare PROPERTY
+        ENVIRONMENT "CRITERION_ALWAYS_SUCCEED=1"
+    )
 endforeach()

samples/asserts.err.expected

@@ -0,0 +1,6 @@
+[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:11: Assertion failed: assert is fatal, expect isn't
+[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:12: Assertion failed: This assert runs
+[FAIL] asserts::base: (0,00s)
+[----] /home/snaipe/workspace/c/criterion/samples/asserts.c:20: Assertion failed: The conditions for this test were not met.
+[FAIL] asserts::old_school: (0,00s)
+[====] Synthesis: Tested: 6 | Passing: 4 | Failing: 2 | Crashing: 0 

samples/description.err.expected

@@ -0,0 +1,3 @@
+[----] /home/snaipe/workspace/c/criterion/samples/description.c:4: Assertion failed: 0
+[FAIL] misc::failing: (0,00s)
+[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 

samples/fixtures.err.expected

@@ -0,0 +1 @@
+[====] Synthesis: Tested: 1 | Passing: 1 | Failing: 0 | Crashing: 0 

samples/fixtures.out.expected

@@ -0,0 +1,2 @@
+Runs before the test
+Runs after the test

samples/long-messages.err.expected

@@ -0,0 +1,6 @@
+[----] /home/snaipe/workspace/c/criterion/samples/long-messages.c:4: Assertion failed: This is
+[----]   A long message
+[----]   Spawning multiple lines.
+[----]   Formatting is respected.
+[FAIL] sample::long_msg: (0,00s)
+[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 

samples/more-suites.err.expected

@@ -0,0 +1 @@
+[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0 

samples/other-crashes.err.expected

@@ -0,0 +1,3 @@
+[----] Warning! The test `misc::setup_crash` crashed during its setup or teardown.
+[----] Warning! The test `misc::teardown_crash` crashed during its setup or teardown.
+[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 1 

samples/report.err.expected

@@ -0,0 +1,3 @@
+[----] /home/snaipe/workspace/c/criterion/samples/report.c:5: Assertion failed: 0
+[FAIL] sample::test: (0,00s)
+[====] Synthesis: Tested: 1 | Passing: 0 | Failing: 1 | Crashing: 0 

samples/report.out.expected

@@ -0,0 +1,5 @@
+criterion_init
+criterion_init
+testing test in category sample
+Asserts: [1 passed, 1 failed, 2 total]
+criterion_fini

samples/signal.err.expected

@@ -0,0 +1,4 @@
+[----] /home/snaipe/workspace/c/criterion/samples/signal.c:16: Unexpected signal caught below this line!
+[FAIL] simple::uncaught: CRASH!
+[FAIL] simple::wrong_signal: (0,00s)
+[====] Synthesis: Tested: 3 | Passing: 1 | Failing: 2 | Crashing: 1 

samples/simple.err.expected

@@ -0,0 +1,3 @@
+[----] /home/snaipe/workspace/c/criterion/samples/simple.c:4: Assertion failed: 0
+[FAIL] misc::failing: (0,00s)
+[====] Synthesis: Tested: 2 | Passing: 1 | Failing: 1 | Crashing: 0 

samples/suites.err.expected

@@ -0,0 +1 @@
+[====] Synthesis: Tested: 2 | Passing: 2 | Failing: 0 | Crashing: 0 

samples/tests/run_test.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -e
+
+if [ -f $1.in ]; then
+    sh -c "$1" > $1.out 2> $1.err < $1.in
+else
+    sh -c "$1" > $1.out 2> $1.err
+fi
+
+if [ -f $2/$1.out.expected ] && [ "$(md5sum $1.out | cut -d' ' -f1)" != "$(md5sum $2/$1.out.expected | cut -d' ' -f1)" ]; then
+    diff $1.out $2/$1.out.expected
+    exit 255
+fi
+
+if [ -f $2/$1.err.expected ] && [ "$(md5sum $1.err | cut -d' ' -f1)" != "$(md5sum $2/$1.err.expected | cut -d' ' -f1)" ]; then
+    diff $1.err $2/$1.err.expected
+    exit 255
+fi