stv0g/Criterion
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

core: .exit_code = 0 now enforces the test exit status to be 0.

Browse source 
Before this change, an exit_code of 0 would allow a test to pass for any
exit status, because checking for 0 meant that the value was optional
and people did not care.

However, the semantics of exit() tells that 0 shall be synonymous of
success, and allowing a nonzero status to pass the test is unexpected.

This fixes issue #137.
This commit is contained in:
Snaipe 2016-07-28 18:11:45 +02:00
parent 1f6d05adca
commit 9fbd28bf69
2 changed files with 9 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/core/client.c
View file

@ -445,16 +445,11 @@ bool handle_death(struct server_ctx *sctx, struct client_ctx *ctx, const criteri
}
ctx->tstats->exit_code = death->status;
if (ctx->state == CS_MAIN) {
if (ctx->test->data->exit_code == 0) {
push_event(TEST_CRASH);
log(abnormal_exit, ctx->tstats);
} else {
double elapsed_time = 0;
push_event(POST_TEST, .data = &elapsed_time);
log(post_test, ctx->tstats);
push_event(POST_FINI);
log(post_fini, ctx->tstats);
}
double elapsed_time = 0;
push_event(POST_TEST, .data = &elapsed_time);
log(post_test, ctx->tstats);
push_event(POST_FINI);
log(post_fini, ctx->tstats);
}
} break;
default: break;

8
test/cram/exit.t
View file

@ -8,14 +8,14 @@ Testing tests that call exit()
[\x1b[0;34mRUN \x1b[0m] exit::normal (esc)
[\x1b[0;32mPASS\x1b[0m] exit::normal (esc)
[\x1b[0;34mRUN \x1b[0m] exit::unexpected_exit (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit::unexpected_exit` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;31mFAIL\x1b[0m] exit::unexpected_exit (esc)
[\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mexit_with_fixtures\x1b[0m: (esc)
[\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::fini_exits (esc)
[\x1b[0;32mPASS\x1b[0m] exit_with_fixtures::fini_exits (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::init_exits (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m2\x1b[0;1m \x1b[0m (esc)
[\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc)
$ exit.cc.bin --verbose
[\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc)
@ -25,11 +25,11 @@ Testing tests that call exit()
[\x1b[0;34mRUN \x1b[0m] exit::normal (esc)
[\x1b[0;32mPASS\x1b[0m] exit::normal (esc)
[\x1b[0;34mRUN \x1b[0m] exit::unexpected_exit (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit::unexpected_exit` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;31mFAIL\x1b[0m] exit::unexpected_exit (esc)
[\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mexit_with_fixtures\x1b[0m: (esc)
[\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::fini_exits (esc)
[\x1b[0;32mPASS\x1b[0m] exit_with_fixtures::fini_exits (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::fini_exits` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;34mRUN \x1b[0m] exit_with_fixtures::init_exits (esc)
[\x1b[0;34m----\x1b[0m] \x1b[0;1mWarning! The test `exit_with_fixtures::init_exits` exited during its setup or teardown.\x1b[0m (esc)
[\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m2\x1b[0;1m \x1b[0m (esc)
[\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m5\x1b[0;1m | Passing: \x1b[0;32m3\x1b[0;1m | Failing: \x1b[0;31m2\x1b[0;1m | Crashing: \x1b[0;31m1\x1b[0;1m \x1b[0m (esc)