diff --git a/include/criterion/abort.h b/include/criterion/abort.h index 065ab44..ef5d155 100644 --- a/include/criterion/abort.h +++ b/include/criterion/abort.h @@ -39,6 +39,13 @@ CR_BEGIN_C_API */ CR_API CR_NORETURN void criterion_abort_test(void); +/** + * Aborts the current test, marking it as skipped. + * + * This function does not return. + */ +CR_API CR_NORETURN void criterion_skip_test(const char *format, ...); + /** * Continues the current test. * diff --git a/include/criterion/assert.h b/include/criterion/assert.h index a0d76e2..fe14a37 100644 --- a/include/criterion/assert.h +++ b/include/criterion/assert.h @@ -50,6 +50,13 @@ * *****************************************************************************/ # define cr_assert_fail(FormatString, ...) +/** + * Skips the test + * + * The test is marked as skipped and the execution of the function is aborted. + * + *****************************************************************************/ +# define cr_skip_test(FormatString, ...) /** * Fails always. diff --git a/include/criterion/internal/assert.h b/include/criterion/internal/assert.h index 237edc3..ef135ae 100644 --- a/include/criterion/internal/assert.h +++ b/include/criterion/internal/assert.h @@ -128,16 +128,21 @@ CR_END_C_API Fail(); \ } while (0) -# define cr_fail(Fail, ...) \ - CR_EXPAND(cr_assert_impl( \ - Fail, \ - 0, \ - dummy, \ - CRITERION_ASSERT_MSG_FAIL, \ - (), \ - __VA_ARGS__ \ + +# define cr_fail(Fail, ...) \ + CR_EXPAND(cr_assert_impl( \ + Fail, \ + 0, \ + dummy, \ + CRITERION_ASSERT_MSG_FAIL, \ + (), \ + __VA_ARGS__ \ )) +# define cr_skip(...) \ + criterion_skip_test("" __VA_ARGS__) + + # define cr_assert_(...) \ CR_EXPAND(cr_assert_impl( \ CR_FAIL_ABORT_, \ @@ -460,6 +465,8 @@ CR_END_C_API // Actual implementation +# undef cr_skip_test +# define cr_skip_test(...) CR_EXPAND(cr_skip(__VA_ARGS__)) # undef cr_assert_fail # define cr_assert_fail(...) CR_EXPAND(cr_fail(CR_FAIL_ABORT_, __VA_ARGS__)) # undef cr_expect_fail diff --git a/include/criterion/internal/deprecation.h b/include/criterion/internal/deprecation.h index 72b217c..0af6204 100644 --- a/include/criterion/internal/deprecation.h +++ b/include/criterion/internal/deprecation.h @@ -32,4 +32,10 @@ # define CR_DEPRECATED_(Msg) _Pragma(#Msg) # endif +# ifdef __GNUC__ +# define CR_DEPRECATED_MEMBER(Member) Member __attribute__((deprecated)) +# else +# define CR_DEPRECATED_MEMBER(Member) Member +# endif + #endif /* !CRITERION_INTERNAL_DEPRECATION_H_ */ diff --git a/include/criterion/stats.h b/include/criterion/stats.h index 1d98bcd..7f63d4d 100644 --- a/include/criterion/stats.h +++ b/include/criterion/stats.h @@ -29,6 +29,13 @@ # define CRITERION_STATS_H_ # include "types.h" +# include "internal/deprecation.h" + +enum criterion_test_status { + CR_STATUS_PASSED = 0, + CR_STATUS_FAILED = 1, + CR_STATUS_SKIPPED = 2, +}; struct criterion_assert_stats { const char *message; @@ -42,7 +49,8 @@ struct criterion_assert_stats { struct criterion_test_stats { struct criterion_test *test; struct criterion_assert_stats *asserts; - bool failed; + bool CR_DEPRECATED_MEMBER(failed); + enum criterion_test_status test_status; int passed_asserts; int failed_asserts; int signal; @@ -52,6 +60,7 @@ struct criterion_test_stats { bool crashed; unsigned progress; const char *file; + const char *message; struct criterion_test_stats *next; }; diff --git a/include/criterion/types.h b/include/criterion/types.h index 525d24c..69290cc 100644 --- a/include/criterion/types.h +++ b/include/criterion/types.h @@ -132,7 +132,7 @@ struct criterion_test_extra_data { int exit_code; /** - * If `true`, disables the test. + * If `true`, skips the test. * * The test will still appear in the test list, but will be marked as * skipped and will not be executed. diff --git a/po/de.po b/po/de.po index 195194c..3661cc0 100644 --- a/po/de.po +++ b/po/de.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Criterion \n" "Report-Msgid-Bugs-To: franklinmathieu+criterion@gmail.com\n" -"POT-Creation-Date: 2016-09-03 20:06+0200\n" +"POT-Creation-Date: 2016-09-03 23:05+0200\n" "PO-Revision-Date: 2016-02-12 11:12+0100\n" "Last-Translator: \n" "Language-Team: German\n" @@ -40,48 +40,43 @@ msgstr "%1$s::%2$s: (%3$3.2fs)\n" #: src/log/normal.c:49 #, c-format -msgid "%1$s::%2$s: Test is disabled\n" -msgstr "%1$s::%2$s: Test ist deaktiviert\n" +msgid "%1$s::%2$s: Test was skipped\n" +msgstr "%1$s::%2$s: Test wurde übersprungen\n" #: src/log/normal.c:50 #, c-format -msgid "%1$s::%2$s: Suite is disabled\n" -msgstr "%1$s::%2$s: Suite ist deaktiviert\n" - -#: src/log/normal.c:51 -#, c-format msgid "%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n" msgstr "%1$s%2$s%3$s:%4$s%5$d%6$s: Fehlerhafte Behauptung: %7$s\n" -#: src/log/normal.c:52 +#: src/log/normal.c:51 #, c-format msgid " Theory %1$s::%2$s failed with the following parameters: (%3$s)\n" msgstr "" " Hypothese %1$s::%2$s schlug fehl mit den folgenden Parametern: (%3$s)\n" -#: src/log/normal.c:53 +#: src/log/normal.c:52 #, c-format msgid "%1$s::%2$s: Timed out. (%3$3.2fs)\n" msgstr "%1$s::%2$s: Außerhalb der Vorgabezeit. (%3$3.2fs)\n" -#: src/log/normal.c:54 +#: src/log/normal.c:53 #, c-format msgid "%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n" msgstr "" "%1$s%2$s%3$s:%4$s%5$u%6$s: Unerwartetes Signal gefangen unterhalb dieser " "Zeile!\n" -#: src/log/normal.c:55 +#: src/log/normal.c:54 #, c-format msgid "%1$s::%2$s: CRASH!\n" msgstr "%1$s::%2$s: ABSTURZ!\n" -#: src/log/normal.c:56 +#: src/log/normal.c:55 #, c-format msgid "%1$s::%2$s: %3$s\n" msgstr "%1$s::%2$s: %3$s\n" -#: src/log/normal.c:57 +#: src/log/normal.c:56 #, c-format msgid "" "%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown." @@ -90,7 +85,7 @@ msgstr "" "%1$sWarnung! Der Test `%2$s::%3$s` ist abgestürzt während der " "Initialisierung oder Beendung %4$s\n" -#: src/log/normal.c:58 +#: src/log/normal.c:57 #, c-format msgid "" "%1$sWarning! The test `%2$s::%3$s` exited during its setup or teardown.%4$s\n" @@ -98,14 +93,14 @@ msgstr "" "%1$sWarnung! Der Test `%2$s::%3$s` hat sich beendet während der " "Initialisierung oder Beendung.%4$s\n" -#: src/log/normal.c:59 +#: src/log/normal.c:58 #, c-format msgid "Running %1$s%2$lu%3$s test from %4$s%5$s%6$s:\n" msgid_plural "Running %1$s%2$lu%3$s tests from %4$s%5$s%6$s:\n" msgstr[0] "Führe %1$s%2$lu%3$s Test durch von %4$s%5$s%6$s:\n" msgstr[1] "Führe %1$s%2$lu%3$s Tests durch von %4$s%5$s%6$s:\n" -#: src/log/normal.c:61 +#: src/log/normal.c:60 #, c-format msgid "" "%1$sSynthesis: Tested: %2$s%3$lu%4$s | Passing: %5$s%6$lu%7$s | Failing: %8$s" @@ -202,3 +197,6 @@ msgstr "" #, c-format msgid "Writing %1$s report in `%2$s`.\n" msgstr "Schreibe Bericht von %1$s nach `%2$s`.\n" + +#~ msgid "%1$s::%2$s: Test is disabled\n" +#~ msgstr "%1$s::%2$s: Test ist deaktiviert\n" diff --git a/po/fr.po b/po/fr.po index f501e09..71611a6 100644 --- a/po/fr.po +++ b/po/fr.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: criterion 2.0.0\n" "Report-Msgid-Bugs-To: franklinmathieu+criterion@gmail.com\n" -"POT-Creation-Date: 2016-09-03 20:06+0200\n" +"POT-Creation-Date: 2016-09-03 23:05+0200\n" "PO-Revision-Date: 2015-04-03 17:58+0200\n" "Last-Translator: \n" "Language-Team: French\n" @@ -34,54 +34,49 @@ msgid "%1$s::%2$s\n" msgstr "%1$s::%2$s\n" #: src/log/normal.c:47 -#, fuzzy, c-format +#, c-format msgid "%1$s::%2$s: (%3$3.2fs)\n" msgstr "%1$s::%2$s: (%3$3.2fs)\n" #: src/log/normal.c:49 #, c-format -msgid "%1$s::%2$s: Test is disabled\n" -msgstr "%1$s::%2$s: Le test est désactivé\n" +msgid "%1$s::%2$s: Test was skipped\n" +msgstr "%1$s::%2$s: Le test a été sauté\n" #: src/log/normal.c:50 #, c-format -msgid "%1$s::%2$s: Suite is disabled\n" -msgstr "%1$s::%2$s: La suite est désactivée\n" - -#: src/log/normal.c:51 -#, c-format msgid "%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n" msgstr "%1$s%2$s%3$s:%4$s%5$d%6$s: Échec d'assertion: %7$s\n" -#: src/log/normal.c:52 +#: src/log/normal.c:51 #, fuzzy, c-format msgid " Theory %1$s::%2$s failed with the following parameters: (%3$s)\n" msgstr "" " La théorie %1$s::%2$s a échoué avec les paramètres suivants: (%3$s)\n" -#: src/log/normal.c:53 +#: src/log/normal.c:52 #, fuzzy, c-format msgid "%1$s::%2$s: Timed out. (%3$3.2fs)\n" msgstr "%1$s::%2$s: Délai expiré. (%3$3.2fs)\n" -#: src/log/normal.c:54 +#: src/log/normal.c:53 #, c-format msgid "%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n" msgstr "" "%1$s%2$s%3$s:%4$s%5$u%6$s: Un signal inattendu a été reçu après cette " "ligne!\n" -#: src/log/normal.c:55 +#: src/log/normal.c:54 #, c-format msgid "%1$s::%2$s: CRASH!\n" msgstr "%1$s::%2$s: PLANTAGE!\n" -#: src/log/normal.c:56 +#: src/log/normal.c:55 #, fuzzy, c-format msgid "%1$s::%2$s: %3$s\n" msgstr "%1$s::%2$s: (%3$3.2fs)\n" -#: src/log/normal.c:57 +#: src/log/normal.c:56 #, fuzzy, c-format msgid "" "%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown." @@ -90,7 +85,7 @@ msgstr "" "%1$sAttention! Le test `%2$s::%3$s` a planté pendant son initialisation ou " "sa finalisation.%4$s\n" -#: src/log/normal.c:58 +#: src/log/normal.c:57 #, fuzzy, c-format msgid "" "%1$sWarning! The test `%2$s::%3$s` exited during its setup or teardown.%4$s\n" @@ -98,14 +93,14 @@ msgstr "" "%1$sAttention! Le test `%2$s::%3$s` a quitté pendant son initialisation ou " "sa finalisation.%4$s\n" -#: src/log/normal.c:59 +#: src/log/normal.c:58 #, c-format msgid "Running %1$s%2$lu%3$s test from %4$s%5$s%6$s:\n" msgid_plural "Running %1$s%2$lu%3$s tests from %4$s%5$s%6$s:\n" msgstr[0] "Lancement de %1$s%2$lu%3$s test dans %4$s%5$s%6$s:\n" msgstr[1] "Lancement de %1$s%2$lu%3$s tests dans %4$s%5$s%6$s:\n" -#: src/log/normal.c:61 +#: src/log/normal.c:60 #, c-format msgid "" "%1$sSynthesis: Tested: %2$s%3$lu%4$s | Passing: %5$s%6$lu%7$s | Failing: %8$s" @@ -204,3 +199,6 @@ msgstr "" #, c-format msgid "Writing %1$s report in `%2$s`.\n" msgstr "Écriture du rapport %1$s dans `%2$s`.\n" + +#~ msgid "%1$s::%2$s: Test is disabled\n" +#~ msgstr "%1$s::%2$s: Le test est désactivé\n" diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 4549e88..4070d1e 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -23,6 +23,7 @@ set(SAMPLES redirect.c parameterized.c log.c + skip.c ) if (CMAKE_CXX_COMPILER_WORKS) @@ -38,6 +39,7 @@ if (CMAKE_CXX_COMPILER_WORKS) redirect.cc parameterized.cc log.cc + skip.cc ) endif () @@ -48,7 +50,6 @@ if (THEORIES) endif () endif () - add_custom_target(criterion_samples) add_dependencies(criterion_tests criterion_samples) diff --git a/samples/skip.c b/samples/skip.c new file mode 100644 index 0000000..b6d5cbb --- /dev/null +++ b/samples/skip.c @@ -0,0 +1,10 @@ +#include + +Test(misc, skipping) { + cr_skip_test(); + cr_assert_fail("We want to skip before failing"); +} + +Test(misc, message) { + cr_skip_test("Skips may take %s", "printf-like messages"); +} diff --git a/samples/skip.cc b/samples/skip.cc new file mode 100644 index 0000000..b6d5cbb --- /dev/null +++ b/samples/skip.cc @@ -0,0 +1,10 @@ +#include + +Test(misc, skipping) { + cr_skip_test(); + cr_assert_fail("We want to skip before failing"); +} + +Test(misc, message) { + cr_skip_test("Skips may take %s", "printf-like messages"); +} diff --git a/src/common.h b/src/common.h index 56b57e3..e9d30a2 100644 --- a/src/common.h +++ b/src/common.h @@ -29,14 +29,6 @@ # include "criterion/internal/common.h" -#ifdef __GNUC__ -# define INLINE __attribute__((always_inline)) inline -#elif defined(_MSC_VER) -# define INLINE __forceinline -#else -# define INLINE -#endif - # define DEF(X, Y) ((X) ? (X) : (Y)) #endif /* !COMMON_H_ */ diff --git a/src/compat/pipe.h b/src/compat/pipe.h index 59c955f..5a7cc08 100644 --- a/src/compat/pipe.h +++ b/src/compat/pipe.h @@ -65,7 +65,7 @@ void pipe_std_redirect(s_pipe_handle *pipe, enum criterion_std_fd fd); int pipe_write(const void *buf, size_t size, s_pipe_file_handle *pipe); int pipe_read(void *buf, size_t size, s_pipe_file_handle *pipe); -INLINE FILE* get_std_file(enum criterion_std_fd fd_kind) { +CR_INLINE FILE* get_std_file(enum criterion_std_fd fd_kind) { switch (fd_kind) { case CR_STDIN: return stdin; case CR_STDOUT: return stdout; diff --git a/src/core/abort.c b/src/core/abort.c index 5498f8d..840e8c7 100644 --- a/src/core/abort.c +++ b/src/core/abort.c @@ -40,6 +40,28 @@ void criterion_abort_test(void) { longjmp(g_pre_test, 1); } +void criterion_skip_test(const char *format, ...) { + char *msg = NULL; + if(*format) + { + va_list args; + va_start(args, format); + cr_vasprintf(&msg, format, args); + va_end(args); + } + + criterion_protocol_msg skip_msg = criterion_message(phase, + .phase = criterion_protocol_phase_kind_SKIP, + .name = (char *) criterion_current_test->name, + .message = msg, + ); + criterion_message_set_id(skip_msg); + cr_send_to_runner(&skip_msg); + free(msg); + + exit(0); +} + void criterion_continue_test(void) { if (criterion_options.crash) debug_break(); diff --git a/src/core/client.c b/src/core/client.c index 682935c..34202e7 100644 --- a/src/core/client.c +++ b/src/core/client.c @@ -45,6 +45,7 @@ static enum client_state phase_to_state[] = { [criterion_protocol_phase_kind_END] = CS_END, [criterion_protocol_phase_kind_ABORT] = CS_ABORT, [criterion_protocol_phase_kind_TIMEOUT] = CS_TIMEOUT, + [criterion_protocol_phase_kind_SKIP] = CS_SKIP, }; static const char *state_to_string[] = { @@ -54,6 +55,7 @@ static const char *state_to_string[] = { [CS_END] = "end", [CS_ABORT] = "abort", [CS_TIMEOUT] = "timeout", + [CS_SKIP] = "skip", }; typedef bool message_handler(struct server_ctx *, struct client_ctx *, const criterion_protocol_msg *); @@ -308,7 +310,7 @@ bool handle_abort(struct server_ctx *sctx, struct client_ctx *ctx, const criteri enum client_state curstate = ctx->state & (CS_MAX_CLIENT_STATES - 1); if (ctx->state < CS_MAX_CLIENT_STATES) { - ctx->tstats->failed = 1; + ctx->tstats->test_status = CR_STATUS_FAILED; log(test_abort, ctx->tstats, msg->message ? msg->message : ""); if (curstate < CS_TEARDOWN) { @@ -347,6 +349,19 @@ bool handle_timeout(struct server_ctx *sctx, struct client_ctx *ctx, const crite return false; } +bool handle_skip(struct server_ctx *sctx, struct client_ctx *ctx, const criterion_protocol_phase *msg) { + (void) sctx; + (void) msg; + + if (ctx->state < CS_MAX_CLIENT_STATES) { + ctx->tstats->test_status = CR_STATUS_SKIPPED; + ctx->tstats->message = msg->message ? strdup(msg->message) : NULL; + double elapsed_time = 0; + push_event(POST_TEST, .data = &elapsed_time); + push_event(POST_FINI); + } + return false; +} # define MAX_TEST_DEPTH 16 bool handle_phase(struct server_ctx *sctx, struct client_ctx *ctx, const criterion_protocol_msg *msg) { @@ -393,6 +408,7 @@ bool handle_phase(struct server_ctx *sctx, struct client_ctx *ctx, const criteri [CS_END] = handle_post_fini, [CS_ABORT] = handle_abort, [CS_TIMEOUT] = handle_timeout, + [CS_SKIP] = handle_skip, }; bool ack = handlers[new_state](sctx, ctx, phase_msg); diff --git a/src/core/client.h b/src/core/client.h index 9c647af..d9bd3e8 100644 --- a/src/core/client.h +++ b/src/core/client.h @@ -38,6 +38,7 @@ enum client_state { // added in the state count CS_ABORT, CS_TIMEOUT, + CS_SKIP, }; // always make it a power of 2 diff --git a/src/core/ordered-set.c b/src/core/ordered-set.c index 1ecdeab..7738ca7 100644 --- a/src/core/ordered-set.c +++ b/src/core/ordered-set.c @@ -31,7 +31,7 @@ static void destroy_ordered_set(void *ptr, CR_UNUSED void *meta) { sfree(((struct criterion_ordered_set *) ptr)->first); } -static INLINE void nothing(CR_UNUSED void *ptr, CR_UNUSED void *meta) {} +static CR_INLINE void nothing(CR_UNUSED void *ptr, CR_UNUSED void *meta) {} static void destroy_ordered_set_node(void *ptr, void *meta) { struct criterion_ordered_set *set = *(void **) meta; diff --git a/src/core/runner.c b/src/core/runner.c index ec57afd..c1f6cea 100644 --- a/src/core/runner.c +++ b/src/core/runner.c @@ -275,7 +275,7 @@ static void run_tests_async(struct criterion_test_set *set, continue; if (!cctx->alive) { - if (cctx->tstats->failed && criterion_options.fail_fast) { + if ((cctx->tstats->test_status == CR_STATUS_FAILED) && criterion_options.fail_fast) { cr_terminate(cctx->gstats); } diff --git a/src/core/runner_coroutine.c b/src/core/runner_coroutine.c index ff5561b..59ca8e6 100644 --- a/src/core/runner_coroutine.c +++ b/src/core/runner_coroutine.c @@ -319,7 +319,7 @@ static bxf_instance *run_test(struct run_next_context *ctx, return instance; } -static INLINE bool is_disabled(struct criterion_test *t, +static CR_INLINE bool is_disabled(struct criterion_test *t, struct criterion_suite *s) { @@ -331,6 +331,7 @@ static int skip_disabled(struct run_next_context *ctx) { if (is_disabled(ctx->test, ctx->suite_stats->suite)) { ctx->test_stats = test_stats_init(ctx->test); + ctx->test_stats->test_status = CR_STATUS_SKIPPED; stat_push_event(ctx->stats, ctx->suite_stats, ctx->test_stats, diff --git a/src/core/stats.c b/src/core/stats.c index 65efc10..ee58f7f 100644 --- a/src/core/stats.c +++ b/src/core/stats.c @@ -35,25 +35,25 @@ typedef struct criterion_test_stats s_test_stats; typedef struct criterion_assert_stats s_assert_stats; static void push_pre_suite(s_glob_stats *stats, - s_suite_stats *sstats, - s_test_stats *tstats, - void *data); + s_suite_stats *sstats, + s_test_stats *tstats, + void *data); static void push_pre_init(s_glob_stats *stats, - s_suite_stats *sstats, - s_test_stats *tstats, - void *data); + s_suite_stats *sstats, + s_test_stats *tstats, + void *data); static void push_assert(s_glob_stats *stats, - s_suite_stats *sstats, - s_test_stats *tstats, - void *data); + s_suite_stats *sstats, + s_test_stats *tstats, + void *data); static void push_post_test(s_glob_stats *stats, - s_suite_stats *sstats, - s_test_stats *tstats, - void *data); + s_suite_stats *sstats, + s_test_stats *tstats, + void *data); static void push_test_crash(s_glob_stats *stats, - s_suite_stats *sstats, - s_test_stats *tstats, - void *data); + s_suite_stats *sstats, + s_test_stats *tstats, + void *data); static void nothing(CR_UNUSED s_glob_stats *stats, CR_UNUSED s_suite_stats *sstats, @@ -103,6 +103,7 @@ static void destroy_test_stats(void *ptr, CR_UNUSED void *meta) { next = a->next; sfree(a); } + free((void*)stats->message); } static void destroy_assert_stats(void *ptr, CR_UNUSED void *meta) { @@ -160,11 +161,6 @@ static void push_pre_suite(s_glob_stats *stats, ++stats->nb_suites; } -static INLINE bool is_disabled(struct criterion_test *t, - struct criterion_suite *s) { - - return t->data->disabled || (s->data && s->data->disabled); -} static void push_pre_init(s_glob_stats *stats, s_suite_stats *suite, @@ -175,7 +171,7 @@ static void push_pre_init(s_glob_stats *stats, ++stats->nb_tests; ++suite->nb_tests; - if (is_disabled(test->test, suite->suite)) { + if (test->test_status == CR_STATUS_SKIPPED) { ++stats->tests_skipped; ++suite->tests_skipped; } @@ -223,15 +219,22 @@ static void push_post_test(s_glob_stats *stats, || test->timed_out || test->signal != test->test->data->signal || test->exit_code != test->test->data->exit_code) { - test->failed = 1; + test->test_status = CR_STATUS_FAILED; } - if (test->failed) { - ++stats->tests_failed; - ++suite->tests_failed; - } else { - ++stats->tests_passed; - ++suite->tests_passed; + switch (test->test_status) { + case CR_STATUS_FAILED: + ++stats->tests_failed; + ++suite->tests_failed; + break; + case CR_STATUS_PASSED: + ++stats->tests_passed; + ++suite->tests_passed; + break; + case CR_STATUS_SKIPPED: + ++stats->tests_skipped; + ++suite->tests_skipped; + break; } } @@ -240,7 +243,7 @@ static void push_test_crash(s_glob_stats *stats, s_suite_stats *suite, s_test_stats *test, CR_UNUSED void *ptr) { - test->failed = 1; + test->test_status = CR_STATUS_FAILED; test->crashed = 1; ++suite->tests_failed; ++suite->tests_crashed; diff --git a/src/entry/params.c b/src/entry/params.c index a8b1393..f1fcd04 100644 --- a/src/entry/params.c +++ b/src/entry/params.c @@ -121,7 +121,7 @@ int list_tests(bool unicode) { --tests == 0 ? end : node, join, t->name, - is_disabled(&s->suite, t) ? " (disabled)" : ""); + is_disabled(&s->suite, t) ? " (skipped)" : ""); } } diff --git a/src/io/json.c b/src/io/json.c index c15902d..6671282 100644 --- a/src/io/json.c +++ b/src/io/json.c @@ -64,9 +64,15 @@ ",\n" \ " \"messages\": [\"The test timed out.\"]" -#define JSON_SKIPPED_MSG_ENTRY \ +#define JSON_TEST_SKIPPED_TEMPLATE_BEGIN \ ",\n" \ - " \"messages\": [\"The test was skipped.\"]" + " \"messages\": [\"" + +#define JSON_TEST_SKIPPED_TEMPLATE_END \ + "\"]" + +#define JSON_SKIPPED_MSG_ENTRY \ + "The test was skipped." #define JSON_TEST_LIST_TEMPLATE_BEGIN \ " \"tests\": [\n" @@ -102,41 +108,31 @@ #define JSON_BASE_TEMPLATE_END \ "}\n" -static INLINE bool is_disabled(struct criterion_test *t, struct criterion_suite *s) { - return t->data->disabled || (s->data && s->data->disabled); + +static CR_INLINE const char *get_status_string(struct criterion_test_stats *ts){ + return (ts->crashed || ts->timed_out) ? "ERRORED" : + ts->test_status == CR_STATUS_FAILED ? "FAILED" : + ts->test_status == CR_STATUS_SKIPPED ? "SKIPPED" : + "PASSED"; } -static CR_INLINE -const char *get_status_string(struct criterion_test_stats *ts, - struct criterion_suite_stats *ss) { - - const char *status = "PASSED"; - if (ts->crashed || ts->timed_out) - status = "ERRORED"; - else if (ts->failed) - status = "FAILED"; - else if (is_disabled(ts->test, ss->suite)) - status = "SKIPPED"; - return status; -} - -static void print_test(FILE *f, - struct criterion_test_stats *ts, - struct criterion_suite_stats *ss) { +static void print_test(FILE *f, struct criterion_test_stats *ts){ fprintf(f, JSON_TEST_TEMPLATE_BEGIN, ts->test->name, (size_t) (ts->passed_asserts + ts->failed_asserts), - get_status_string(ts, ss) + get_status_string(ts) ); - if (is_disabled(ts->test, ss->suite)) { - fprintf(f, JSON_SKIPPED_MSG_ENTRY); + if (ts->test_status == CR_STATUS_SKIPPED) { + fprintf(f,"%s%s%s", JSON_TEST_SKIPPED_TEMPLATE_BEGIN, + ts->message ? ts->message : JSON_SKIPPED_MSG_ENTRY, + JSON_TEST_SKIPPED_TEMPLATE_END); } else if (ts->crashed) { fprintf(f, JSON_CRASH_MSG_ENTRY); } else if (ts->timed_out) { fprintf(f, JSON_TIMEOUT_MSG_ENTRY); - } else if (ts->failed) { + } else if (ts->test_status == CR_STATUS_FAILED) { fprintf(f, JSON_TEST_FAILED_TEMPLATE_BEGIN); bool first = true; @@ -192,7 +188,7 @@ void json_report(FILE *f, struct criterion_global_stats *stats) { fprintf(f, JSON_TEST_LIST_TEMPLATE_BEGIN); for (struct criterion_test_stats *ts = ss->tests; ts; ts = ts->next) { - print_test(f, ts, ss); + print_test(f, ts); fprintf(f, ts->next ? ",\n" : "\n"); } fprintf(f, JSON_TEST_LIST_TEMPLATE_END); diff --git a/src/io/tap.c b/src/io/tap.c index 52bcd67..4c507bd 100644 --- a/src/io/tap.c +++ b/src/io/tap.c @@ -48,15 +48,12 @@ static void print_pre_suite(FILE *f, struct criterion_suite_stats *stats) { stats->suite->name); } -static INLINE bool is_disabled(struct criterion_test *t, struct criterion_suite *s) { - return t->data->disabled || (s->data && s->data->disabled); -} static void print_test_normal(FILE *f, struct criterion_test_stats *stats) { const char *format = can_measure_time() ? "%s - %s::%s %s (%3.2fs)\n" : "%s - %s::%s %s\n"; fprintf(f, format, - stats->failed ? "not ok" : "ok", + stats->test_status == CR_STATUS_FAILED ? "not ok" : "ok", stats->test->category, stats->test->name, DEF(stats->test->data->description, ""), @@ -95,16 +92,14 @@ static void print_test_timeout(FILE *f, struct criterion_test_stats *stats) { stats->elapsed_time); } -static void print_test(FILE *f, - struct criterion_test_stats *ts, - struct criterion_suite_stats *ss) { +static void print_test(FILE *f, struct criterion_test_stats *ts){ - if (is_disabled(ts->test, ss->suite)) { - fprintf(f, "ok - %s::%s %s # SKIP %s is disabled\n", + if (ts->test_status == CR_STATUS_SKIPPED){ + fprintf(f, "ok - %s::%s %s # SKIP %s\n", ts->test->category, ts->test->name, DEF(ts->test->data->description, ""), - ts->test->data->disabled ? "test" : "suite"); + ts->message ? ts->message : "test was skipped"); } else if (ts->crashed) { print_test_crashed(f, ts); } else if (ts->timed_out) { @@ -121,7 +116,7 @@ void tap_report(FILE *f, struct criterion_global_stats *stats) { print_pre_suite(f, ss); for (struct criterion_test_stats *ts = ss->tests; ts; ts = ts->next) { - print_test(f, ts, ss); + print_test(f, ts); } } } diff --git a/src/io/xml.c b/src/io/xml.c index 268192c..b113e9f 100644 --- a/src/io/xml.c +++ b/src/io/xml.c @@ -98,22 +98,12 @@ #define XML_TIMEOUT_MSG_ENTRY \ " " -static INLINE bool is_disabled(struct criterion_test *t, struct criterion_suite *s) { - return t->data->disabled || (s->data && s->data->disabled); -} -static INLINE -const char *get_status_string(struct criterion_test_stats *ts, - struct criterion_suite_stats *ss) { - - const char *status = "PASSED"; - if (ts->crashed || ts->timed_out) - status = "ERRORED"; - else if (ts->failed) - status = "FAILED"; - else if (is_disabled(ts->test, ss->suite)) - status = "SKIPPED"; - return status; +static CR_INLINE const char *get_status_string(struct criterion_test_stats *ts){ + return (ts->crashed || ts->timed_out) ? "ERRORED" : + ts->test_status == CR_STATUS_FAILED ? "FAILED" : + ts->test_status == CR_STATUS_SKIPPED ? "SKIPPED" : + "PASSED"; } /* @@ -134,25 +124,23 @@ static int fprintf_locale(FILE *stream, return result; } -static void print_test(FILE *f, - struct criterion_test_stats *ts, - struct criterion_suite_stats *ss) { +static void print_test(FILE *f, struct criterion_test_stats *ts){ fprintf_locale(f, XML_TEST_TEMPLATE_BEGIN, ts->test->name, (size_t) (ts->passed_asserts + ts->failed_asserts), - get_status_string(ts, ss), + get_status_string(ts), ts->elapsed_time ); - if (is_disabled(ts->test, ss->suite)) { + if (ts->test_status == CR_STATUS_SKIPPED){ fprintf(f, XML_TEST_SKIPPED); } else if (ts->crashed) { fprintf(f, XML_CRASH_MSG_ENTRY); } else if (ts->timed_out) { fprintf(f, XML_TIMEOUT_MSG_ENTRY); } else { - if (ts->failed) { + if (ts->test_status == CR_STATUS_FAILED) { fprintf(f, XML_TEST_FAILED_TEMPLATE_BEGIN, ts->failed_asserts); for (struct criterion_assert_stats *asrt = ts->asserts; asrt; asrt = asrt->next) { if (!asrt->passed) { @@ -179,7 +167,7 @@ static void print_test(FILE *f, fprintf(f, XML_TEST_TEMPLATE_END); } -static INLINE float get_time_elapsed_suite(struct criterion_suite_stats *ss) +static CR_INLINE float get_time_elapsed_suite(struct criterion_suite_stats *ss) { float result = 0; for (struct criterion_test_stats *ts = ss->tests; ts; ts = ts->next) { @@ -209,7 +197,7 @@ void xml_report(FILE *f, struct criterion_global_stats *stats) { ); for (struct criterion_test_stats *ts = ss->tests; ts; ts = ts->next) { - print_test(f, ts, ss); + print_test(f, ts); } fprintf(f, XML_TESTSUITE_TEMPLATE_END); diff --git a/src/log/normal.c b/src/log/normal.c index dabe7e1..513cf81 100644 --- a/src/log/normal.c +++ b/src/log/normal.c @@ -46,8 +46,7 @@ static msg_t msg_desc = N_(" %s\n"); static msg_t msg_pre_init = N_("%1$s::%2$s\n"); static msg_t msg_post_test_timed = N_("%1$s::%2$s: (%3$3.2fs)\n"); static msg_t msg_post_test = N_("%1$s::%2$s\n"); -static msg_t msg_post_suite_test = N_("%1$s::%2$s: Test is disabled\n"); -static msg_t msg_post_suite_suite = N_("%1$s::%2$s: Suite is disabled\n"); +static msg_t msg_post_test_skip = N_("%1$s::%2$s: Test was skipped\n"); static msg_t msg_assert_fail = N_("%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n"); static msg_t msg_theory_fail = N_(" Theory %1$s::%2$s failed with the following parameters: (%3$s)\n"); static msg_t msg_test_timeout = N_("%1$s::%2$s: Timed out. (%3$3.2fs)\n"); @@ -67,8 +66,7 @@ static msg_t msg_post_all = N_("%1$sSynthesis: Tested: %2$s%3$lu%4$s " static msg_t msg_pre_init = "%s::%s\n"; static msg_t msg_post_test_timed = "%s::%s: (%3.2fs)\n"; static msg_t msg_post_test = "%s::%s\n"; -static msg_t msg_post_suite_test = "%s::%s: Test is disabled\n"; -static msg_t msg_post_suite_suite = "%s::%s: Suite is disabled\n"; +static msg_t msg_post_test_skip = "%1$s::%2$s: Test was skipped\n"; static msg_t msg_assert_fail = "%s%s%s:%s%d%s: Assertion failed: %s\n"; static msg_t msg_theory_fail = " Theory %s::%s failed with the following parameters: (%s)\n"; static msg_t msg_test_timeout = "%s::%s: Timed out. (%3.2fs)\n"; @@ -104,9 +102,10 @@ void normal_log_post_test(struct criterion_test_stats *stats) { const char *format = can_measure_time() ? msg_post_test_timed : msg_post_test; const enum criterion_logging_level level - = stats->failed ? CRITERION_IMPORTANT : CRITERION_INFO; + = stats->test_status == CR_STATUS_FAILED ? CRITERION_IMPORTANT : CRITERION_INFO; const struct criterion_prefix_data *prefix - = stats->failed ? CRITERION_PREFIX_FAIL : CRITERION_PREFIX_PASS; + = stats->test_status == CR_STATUS_FAILED ? CRITERION_PREFIX_FAIL : + CRITERION_PREFIX_PASS; criterion_plog(level, prefix, _(format), stats->test->category, @@ -114,21 +113,19 @@ void normal_log_post_test(struct criterion_test_stats *stats) { stats->elapsed_time); } -static INLINE bool is_disabled(struct criterion_test *t, - struct criterion_suite *s) { - return t->data->disabled || (s->data && s->data->disabled); -} - void normal_log_post_suite(struct criterion_suite_stats *stats) { for (struct criterion_test_stats *ts = stats->tests; ts; ts = ts->next) { - if (is_disabled(ts->test, stats->suite)) { - const char *format = ts->test->data->disabled - ? _(msg_post_suite_test) - : _(msg_post_suite_suite); - - criterion_pinfo(CRITERION_PREFIX_SKIP, format, - ts->test->category, - ts->test->name); + if(ts->test_status == CR_STATUS_SKIPPED){ + if(!ts->message) { + criterion_pinfo(CRITERION_PREFIX_SKIP, _(msg_post_test_skip), + ts->test->category, + ts->test->name); + } else { + criterion_pinfo(CRITERION_PREFIX_SKIP, "%s::%s: %s\n", + ts->test->category, + ts->test->name, + ts->message); + } if (ts->test->data->description) criterion_pinfo(CRITERION_PREFIX_DASHES, msg_desc, @@ -203,7 +200,7 @@ void normal_log_pre_suite(struct criterion_suite_set *set) { void normal_log_theory_fail(struct criterion_theory_stats *stats) { criterion_pimportant(CRITERION_PREFIX_DASHES, - _(msg_theory_fail), + _(msg_theory_fail), stats->stats->test->category, stats->stats->test->name, stats->formatted_args); diff --git a/src/protocol/criterion.pb.c b/src/protocol/criterion.pb.c index 878c484..53c6973 100644 --- a/src/protocol/criterion.pb.c +++ b/src/protocol/criterion.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.3.5-dev at Sun Jan 17 02:06:50 2016. */ +/* Generated by nanopb-0.3.5-dev at Mon Aug 8 22:00:27 2016. */ #include "criterion.pb.h" diff --git a/src/protocol/criterion.pb.h b/src/protocol/criterion.pb.h index 4fa906e..658a0ac 100644 --- a/src/protocol/criterion.pb.h +++ b/src/protocol/criterion.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.5-dev at Sun Jan 17 02:06:50 2016. */ +/* Generated by nanopb-0.3.5-dev at Mon Aug 8 22:00:27 2016. */ #ifndef PB_CRITERION_PB_H_INCLUDED #define PB_CRITERION_PB_H_INCLUDED @@ -20,7 +20,8 @@ typedef enum _criterion_protocol_phase_kind { criterion_protocol_phase_kind_TEARDOWN = 3, criterion_protocol_phase_kind_END = 4, criterion_protocol_phase_kind_ABORT = 5, - criterion_protocol_phase_kind_TIMEOUT = 6 + criterion_protocol_phase_kind_TIMEOUT = 6, + criterion_protocol_phase_kind_SKIP = 7 } criterion_protocol_phase_kind; typedef enum _criterion_protocol_death_result_type { diff --git a/src/protocol/criterion.proto b/src/protocol/criterion.proto index e97b96e..81610ad 100644 --- a/src/protocol/criterion.proto +++ b/src/protocol/criterion.proto @@ -15,6 +15,7 @@ message phase { ABORT = 5; TIMEOUT = 6; + SKIP = 7; } required kind phase = 1; @@ -34,6 +35,7 @@ message death { optional int64 timestamp = 3; } + message assert { required string message = 1; required bool passed = 2; diff --git a/test/cram/core.t b/test/cram/core.t index d2c9e8d..7e000e7 100644 --- a/test/cram/core.t +++ b/test/cram/core.t @@ -73,7 +73,7 @@ Testing descriptions [\x1b[0;34m----\x1b[0m] Just a failing test (esc) [\x1b[0;34m----\x1b[0m] \x1b[0;1mdescription.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test was skipped (esc) [\x1b[0;34m----\x1b[0m] This one is skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -84,7 +84,7 @@ Testing descriptions [\x1b[0;34m----\x1b[0m] Just a failing test (esc) [\x1b[0;34m----\x1b[0m] \x1b[0;1mdescription.cc\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::skipped: Test was skipped (esc) [\x1b[0;34m----\x1b[0m] This one is skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/json.t b/test/cram/json.t index 16dd2b8..3236d8a 100644 --- a/test/cram/json.t +++ b/test/cram/json.t @@ -318,3 +318,35 @@ Testing CRITERION_OUTPUTS } ] } + + $ skip.c.bin --json + { + "id": "Criterion v2.2.1", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 2, + "test_suites": [ + { + "name": "misc", + "passed": 0, + "failed": 0, + "errored": 0, + "skipped": 2, + "tests": [ + { + "name": "skipping", + "assertions": 0, + "status": "SKIPPED", + "messages": ["The test was skipped."] + }, + { + "name": "message", + "assertions": 0, + "status": "SKIPPED", + "messages": ["Skips may take printf-like messages"] + } + ] + } + ] + } diff --git a/test/cram/pattern.t b/test/cram/pattern.t index 0a8de09..b781492 100644 --- a/test/cram/pattern.t +++ b/test/cram/pattern.t @@ -5,7 +5,7 @@ Testing normal globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='*/pa?sing' --verbose @@ -13,7 +13,7 @@ Testing normal globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/passing?' --verbose @@ -21,7 +21,7 @@ Testing normal globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/*' --verbose @@ -44,7 +44,7 @@ Testing character classes [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/[p-s]assing' --verbose @@ -52,14 +52,14 @@ Testing character classes [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/[!f]ailing' --verbose [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/[p-f]ailing' --verbose @@ -68,7 +68,7 @@ Testing character classes [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/[!azerty]assing' --verbose @@ -76,7 +76,7 @@ Testing character classes [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -91,7 +91,7 @@ Testing extended globs [\x1b[0;34mRUN \x1b[0m] misc::failing (esc) [\x1b[0;34m----\x1b[0m] \x1b[0;1msimple.c\x1b[0m:\x1b[0;31m4\x1b[0m: Assertion failed: The expression 0 is false. (esc) [\x1b[0;31mFAIL\x1b[0m] misc::failing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0;31m1\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='@(misc)/passing' --verbose @@ -99,7 +99,7 @@ Testing extended globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -108,7 +108,7 @@ Testing extended globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -128,7 +128,7 @@ Testing extended globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -137,7 +137,7 @@ Testing extended globs [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/?(passing|failing)' --verbose @@ -169,15 +169,15 @@ Testing nested extglob patterns $ simple.c.bin --filter='@(+(nest)ed)' --verbose [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='?(*(a|b))' --verbose [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) $ simple.c.bin --filter='misc/pa@(s|*(s))ing' --verbose @@ -185,7 +185,7 @@ Testing nested extglob patterns [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] misc::passing (esc) [\x1b[0;32mPASS\x1b[0m] misc::passing (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m1\x1b[0;1m | Passing: \x1b[0;32m1\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) @@ -227,8 +227,8 @@ Testing special char escaping $ simple.c.bin --filter='\!escaped' --verbose [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test is disabled (esc) - [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::passing: Test was skipped (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::failing: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/skip.t b/test/cram/skip.t new file mode 100644 index 0000000..7aceca1 --- /dev/null +++ b/test/cram/skip.t @@ -0,0 +1,10 @@ +Testing skipping + + $ skip.c.bin --verbose + [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) + [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m2\x1b[0m tests from \x1b[0;33mmisc\x1b[0m: (esc) + [\x1b[0;34mRUN \x1b[0m] misc::message (esc) + [\x1b[0;34mRUN \x1b[0m] misc::skipping (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::skipping: Test was skipped (esc) + [\x1b[0;33mSKIP\x1b[0m] misc::message: Skips may take printf-like messages (esc) + [\x1b[0;34m====\x1b[0m] \x1b[0;1mSynthesis: Tested: \x1b[0;34m0\x1b[0;1m | Passing: \x1b[0;32m0\x1b[0;1m | Failing: \x1b[0m0\x1b[0;1m | Crashing: \x1b[0m0\x1b[0;1m \x1b[0m (esc) diff --git a/test/cram/suites.t b/test/cram/suites.t index 23c12eb..4e7a436 100644 --- a/test/cram/suites.t +++ b/test/cram/suites.t @@ -35,7 +35,7 @@ Running the tests with verbose output $ more-suites.c.bin --verbose [\x1b[0;34m----\x1b[0m] Criterion v2.2.1 (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33mdisabled\x1b[0m: (esc) - [\x1b[0;33mSKIP\x1b[0m] disabled::test: Suite is disabled (esc) + [\x1b[0;33mSKIP\x1b[0m] disabled::test: Test was skipped (esc) [\x1b[0;34m====\x1b[0m] Running \x1b[0;34m1\x1b[0m test from \x1b[0;33msuite1\x1b[0m: (esc) [\x1b[0;34mRUN \x1b[0m] suite1::test (esc) [\x1b[0;32mPASS\x1b[0m] suite1::test (esc) diff --git a/test/cram/tap.t b/test/cram/tap.t index 026f9fd..0c0f30d 100644 --- a/test/cram/tap.t +++ b/test/cram/tap.t @@ -49,7 +49,7 @@ Testing multiple samples with --tap ok - suite1::test # Running 1 tests from disabled - ok - disabled::test # SKIP suite is disabled + ok - disabled::test # SKIP test was skipped $ long-messages.c.bin --tap TAP version 13 @@ -69,7 +69,7 @@ Testing multiple samples with --tap # Criterion v2.2.1 # Running 2 tests from misc - ok - misc::skipped This one is skipped # SKIP test is disabled + ok - misc::skipped This one is skipped # SKIP test was skipped not ok - misc::failing Just a failing test description.c:4: Assertion failed: The expression 0 is false. @@ -96,3 +96,14 @@ Testing CRITERION_OUTPUTS ok - misc::passing not ok - misc::failing simple.c:4: Assertion failed: The expression 0 is false. + + + + $ skip.c.bin --tap + TAP version 13 + 1..2 + # Criterion v2.2.1 + + # Running 2 tests from misc + ok - misc::skipping # SKIP test was skipped + ok - misc::message # SKIP Skips may take printf-like messages diff --git a/test/cram/xml.t b/test/cram/xml.t index d6fd059..a06c1d2 100644 --- a/test/cram/xml.t +++ b/test/cram/xml.t @@ -123,3 +123,19 @@ Testing CRITERION_OUTPUTS + + + + $ skip.c.bin --xml + + + + + + + + + + + +