From 1757752dd1bd78307e157eeda24141f3fd393610 Mon Sep 17 00:00:00 2001 From: Snaipe Date: Thu, 3 Sep 2015 17:27:50 +0200 Subject: [PATCH] Added reporting & logging on theory failure --- doc/hooks.rst | 2 + include/criterion/hooks.h | 2 + include/criterion/logging.h | 1 + include/criterion/stats.h | 5 ++ include/criterion/theories.h | 8 ++- po/fr.po | 16 ++++-- src/abort.c | 3 +- src/abort.h | 3 ++ src/event.c | 20 ++++++++ src/log/normal.c | 11 +++++ src/report.c | 2 + src/report.h | 1 + src/runner.c | 8 +++ src/stats.c | 1 + src/theories.c | 95 +++++++++++++++++++++++++++++++++++- 15 files changed, 168 insertions(+), 10 deletions(-) diff --git a/doc/hooks.rst b/doc/hooks.rst index a9b8bd6..ff334a6 100644 --- a/doc/hooks.rst +++ b/doc/hooks.rst @@ -31,6 +31,7 @@ The flow of the test process goes as follows: #. ``PRE_INIT``: occurs before a test is initialized. #. ``PRE_TEST``: occurs after the test initialization, but before the test is run. #. ``ASSERT``: occurs when an assertion is hit +#. ``THEORY_FAIL``: occurs when a theory iteration fails. #. ``TEST_CRASH``: occurs when a test crashes unexpectedly. #. ``POST_TEST``: occurs after a test ends, but before the test finalization. #. ``POST_FINI``: occurs after a test finalization. @@ -50,6 +51,7 @@ Valid types for each phases are: * ``struct criterion_suite_set *`` for ``PRE_SUITE``. * ``struct criterion_test *`` for ``PRE_INIT`` and ``PRE_TEST``. * ``struct criterion_assert_stats *`` for ``ASSERT``. +* ``struct criterion_theory_stats *`` for ``THEORY_FAIL``. * ``struct criterion_test_stats *`` for ``POST_TEST``, ``POST_FINI``, and ``TEST_CRASH``. * ``struct criterion_suite_stats *`` for ``POST_SUITE``. * ``struct criterion_global_stats *`` for ``POST_ALL``. diff --git a/include/criterion/hooks.h b/include/criterion/hooks.h index c597467..0c86841 100644 --- a/include/criterion/hooks.h +++ b/include/criterion/hooks.h @@ -32,6 +32,7 @@ typedef enum { PRE_INIT, PRE_TEST, ASSERT, + THEORY_FAIL, TEST_CRASH, POST_TEST, POST_FINI, @@ -54,6 +55,7 @@ typedef void (*f_report_hook)(); # define HOOK_SECTION_PRE_INIT cr_pri # define HOOK_SECTION_PRE_TEST cr_prt # define HOOK_SECTION_ASSERT cr_ast +# define HOOK_SECTION_THEORY_FAIL cr_thf # define HOOK_SECTION_TEST_CRASH cr_tsc # define HOOK_SECTION_POST_TEST cr_pot # define HOOK_SECTION_POST_FINI cr_pof diff --git a/include/criterion/logging.h b/include/criterion/logging.h index 616dac9..344b8a7 100644 --- a/include/criterion/logging.h +++ b/include/criterion/logging.h @@ -96,6 +96,7 @@ struct criterion_output_provider { void (*log_pre_init )(struct criterion_test *test); void (*log_pre_test )(struct criterion_test *test); void (*log_assert )(struct criterion_assert_stats *stats); + void (*log_theory_fail)(struct criterion_theory_stats *stats); void (*log_test_crash )(struct criterion_test_stats *stats); void (*log_other_crash)(struct criterion_test_stats *stats); void (*log_post_test )(struct criterion_test_stats *stats); diff --git a/include/criterion/stats.h b/include/criterion/stats.h index 8a1b4e7..3effc4d 100644 --- a/include/criterion/stats.h +++ b/include/criterion/stats.h @@ -51,6 +51,11 @@ struct criterion_test_stats { struct criterion_test_stats *next; }; +struct criterion_theory_stats { + const char *formatted_args; + struct criterion_test_stats *stats; +}; + struct criterion_suite_stats { struct criterion_suite *suite; struct criterion_test_stats *tests; diff --git a/include/criterion/theories.h b/include/criterion/theories.h index 4da3527..7d233c9 100644 --- a/include/criterion/theories.h +++ b/include/criterion/theories.h @@ -41,8 +41,12 @@ void cr_theory_call(struct criterion_theory_context *ctx, void (*fnptr)(void)); # define TheoryDataPoints(Category, Name) \ static struct criterion_datapoints IDENTIFIER_(Category, Name, dps)[] -# define DataPoints(Type, ...) \ - { sizeof (Type), sizeof ((Type[]) { __VA_ARGS__ }) / sizeof (Type), #Type, &(Type[]) { __VA_ARGS__ } } +# define DataPoints(Type, ...) { \ + sizeof (Type), \ + sizeof ((Type[]) { __VA_ARGS__ }) / sizeof (Type), \ + #Type, \ + &(Type[]) { __VA_ARGS__ }, \ + } struct criterion_datapoints { size_t size; diff --git a/po/fr.po b/po/fr.po index af25ab8..eb73de0 100644 --- a/po/fr.po +++ b/po/fr.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: criterion 1.0.0\n" "Report-Msgid-Bugs-To: franklinmathieu+criterion@gmail.com\n" -"POT-Creation-Date: 2015-08-05 11:37+0200\n" +"POT-Creation-Date: 2015-09-03 17:24+0200\n" "PO-Revision-Date: 2015-04-03 17:58+0200\n" "Last-Translator: \n" "Language-Team: French\n" @@ -54,18 +54,24 @@ msgid "%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n" msgstr "%1$s%2$s%3$s:%4$s%5$d%6$s: Échec d'assertion: %7$s\n" #: src/log/normal.c:61 +#, fuzzy, c-format +msgid " Theory %1$s::%2$s failed with the following parameters: (%3$s)\n" +msgstr "" +" La théorie %1$s::%2$s a échoué avec les paramètres suivants: (%3$s)\n" + +#: src/log/normal.c:62 #, c-format msgid "%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n" msgstr "" "%1$s%2$s%3$s:%4$s%5$u%6$s: Un signal inattendu a été reçu après cette " "ligne!\n" -#: src/log/normal.c:62 +#: src/log/normal.c:63 #, c-format msgid "%1$s::%2$s: CRASH!\n" msgstr "%1$s::%2$s: PLANTAGE!\n" -#: src/log/normal.c:63 +#: src/log/normal.c:64 #, fuzzy, c-format msgid "" "%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown." @@ -74,14 +80,14 @@ msgstr "" "%1$sAttention! Le test `%2$s::%3$s` a planté pendant son initialisation ou " "sa finalisation.%4$s\n" -#: src/log/normal.c:64 +#: src/log/normal.c:65 #, c-format msgid "Running %1$s%2$lu%3$s test from %4$s%5$s%6$s:\n" msgid_plural "Running %1$s%2$lu%3$s tests from %4$s%5$s%6$s:\n" msgstr[0] "Lancement de %1$s%2$lu%3$s test dans %4$s%5$s%6$s:\n" msgstr[1] "Lancement de %1$s%2$lu%3$s tests dans %4$s%5$s%6$s:\n" -#: src/log/normal.c:66 +#: src/log/normal.c:67 #, c-format msgid "" "%1$sSynthesis: Tested: %2$s%3$lu%4$s | Passing: %5$s%6$lu%7$s | Failing: %8$s" diff --git a/src/abort.c b/src/abort.c index ea61bb9..596d9d8 100644 --- a/src/abort.c +++ b/src/abort.c @@ -21,10 +21,9 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#include #include "abort.h" -static jmp_buf g_pre_test; +jmp_buf g_pre_test; int setup_abort_test(void) { return !setjmp(g_pre_test); diff --git a/src/abort.h b/src/abort.h index 4d85ba3..9eab3d9 100644 --- a/src/abort.h +++ b/src/abort.h @@ -25,6 +25,9 @@ # define ABORT_H_ # include +# include + +extern jmp_buf g_pre_test; int setup_abort_test(void); diff --git a/src/event.c b/src/event.c index 6146c95..6907c7c 100644 --- a/src/event.c +++ b/src/event.c @@ -54,6 +54,26 @@ struct event *read_event(FILE *f) { .value = { .kind = kind, .data = buf }, .dtor = destroy_event); } + case THEORY_FAIL: { + size_t *len = malloc(sizeof (size_t)); + if (fread(len, sizeof (size_t), 1, f) == 0) { + free(len); + return NULL; + } + + char *buf = malloc(*len); + if (fread(buf, *len, 1, f) == 0) { + free(len); + free(buf); + return NULL; + } + free(len); + + return unique_ptr(struct event, + .value = { .kind = kind, .data = buf }, + .dtor = destroy_event); + + } case POST_TEST: { double *elapsed_time = malloc(sizeof (double)); if (fread(elapsed_time, sizeof (double), 1, f) == 0) { diff --git a/src/log/normal.c b/src/log/normal.c index 565e763..560a0f4 100644 --- a/src/log/normal.c +++ b/src/log/normal.c @@ -58,6 +58,7 @@ static msg_t msg_post_test = N_("%1$s::%2$s\n"); static msg_t msg_post_suite_test = N_("%1$s::%2$s: Test is disabled\n"); static msg_t msg_post_suite_suite = N_("%1$s::%2$s: Suite is disabled\n"); static msg_t msg_assert_fail = N_("%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n"); +static msg_t msg_theory_fail = N_(" Theory %1$s::%2$s failed with the following parameters: (%3$s)\n"); static msg_t msg_test_crash_line = N_("%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n"); static msg_t msg_test_crash = N_("%1$s::%2$s: CRASH!\n"); static msg_t msg_test_other_crash = N_("%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown.%4$s\n"); @@ -75,6 +76,7 @@ static msg_t msg_post_test = "%s::%s\n"; static msg_t msg_post_suite_test = "%s::%s: Test is disabled\n"; static msg_t msg_post_suite_suite = "%s::%s: Suite is disabled\n"; static msg_t msg_assert_fail = "%s%s%s:%s%d%s: Assertion failed: %s\n"; +static msg_t msg_theory_fail = " Theory %s::%s failed with the following parameters: %s\n"; static msg_t msg_test_crash_line = "%s%s%s:%s%u%s: Unexpected signal caught below this line!\n"; static msg_t msg_test_crash = "%s::%s: CRASH!\n"; static msg_t msg_test_other_crash = "%sWarning! The test `%s::%s` crashed during its setup or teardown.%s\n"; @@ -205,11 +207,20 @@ void normal_log_pre_suite(struct criterion_suite_set *set) { FG_GOLD, set->suite.name, RESET); } +void normal_log_theory_fail(struct criterion_theory_stats *stats) { + criterion_pimportant(CRITERION_PREFIX_DASHES, + _(msg_theory_fail), + stats->stats->test->category, + stats->stats->test->name, + stats->formatted_args); +} + struct criterion_output_provider normal_logging = { .log_pre_all = normal_log_pre_all, .log_pre_init = normal_log_pre_init, .log_pre_suite = normal_log_pre_suite, .log_assert = normal_log_assert, + .log_theory_fail = normal_log_theory_fail, .log_test_crash = normal_log_test_crash, .log_other_crash = normal_log_other_crash, .log_post_test = normal_log_post_test, diff --git a/src/report.c b/src/report.c index 64a6861..b839542 100644 --- a/src/report.c +++ b/src/report.c @@ -50,6 +50,7 @@ IMPL_CALL_REPORT_HOOKS(PRE_SUITE); IMPL_CALL_REPORT_HOOKS(PRE_INIT); IMPL_CALL_REPORT_HOOKS(PRE_TEST); IMPL_CALL_REPORT_HOOKS(ASSERT); +IMPL_CALL_REPORT_HOOKS(THEORY_FAIL); IMPL_CALL_REPORT_HOOKS(TEST_CRASH); IMPL_CALL_REPORT_HOOKS(POST_TEST); IMPL_CALL_REPORT_HOOKS(POST_FINI); @@ -61,6 +62,7 @@ ReportHook(PRE_SUITE)() {} ReportHook(PRE_INIT)() {} ReportHook(PRE_TEST)() {} ReportHook(ASSERT)() {} +ReportHook(THEORY_FAIL)() {} ReportHook(TEST_CRASH)() {} ReportHook(POST_TEST)() {} ReportHook(POST_FINI)() {} diff --git a/src/report.h b/src/report.h index aab2a10..02aeed9 100644 --- a/src/report.h +++ b/src/report.h @@ -37,6 +37,7 @@ DECL_CALL_REPORT_HOOKS(PRE_SUITE); DECL_CALL_REPORT_HOOKS(PRE_INIT); DECL_CALL_REPORT_HOOKS(PRE_TEST); DECL_CALL_REPORT_HOOKS(ASSERT); +DECL_CALL_REPORT_HOOKS(THEORY_FAIL); DECL_CALL_REPORT_HOOKS(TEST_CRASH); DECL_CALL_REPORT_HOOKS(POST_TEST); DECL_CALL_REPORT_HOOKS(POST_FINI); diff --git a/src/runner.c b/src/runner.c index c28da67..1a22164 100644 --- a/src/runner.c +++ b/src/runner.c @@ -220,6 +220,14 @@ static void run_test(struct criterion_global_stats *stats, log(pre_test, test); test_started = true; break; + case THEORY_FAIL: { + struct criterion_theory_stats ths = { + .formatted_args = (char*) ev->data, + .stats = test_stats, + }; + report(THEORY_FAIL, &ths); + log(theory_fail, &ths); + } break; case ASSERT: report(ASSERT, ev->data); log(assert, ev->data); diff --git a/src/stats.c b/src/stats.c index e7d60a6..91b561a 100644 --- a/src/stats.c +++ b/src/stats.c @@ -91,6 +91,7 @@ void stat_push_event(s_glob_stats *stats, push_pre_init, // PRE_INIT nothing, // PRE_TEST push_assert, // ASSERT + nothing, // THEORY_FAIL push_test_crash, // TEST_CRASH push_post_test, // POST_TEST nothing, // POST_FINI diff --git a/src/theories.c b/src/theories.c index e846543..ff22ff5 100644 --- a/src/theories.c +++ b/src/theories.c @@ -27,6 +27,7 @@ #include #include #include "criterion/theories.h" +#include "abort.h" struct criterion_theory_context { DCCallVM* vm; @@ -99,6 +100,75 @@ static bool contains_word(const char *str, const char *pattern, size_t sz) { && (!res[sz - 1] || res[sz - 1] == ' '); } +static bool is_string(const char *name) { + return !strcmp(name, "char*") + || !strcmp(name, "char *") + || !strcmp(name, "const char*") + || !strcmp(name, "const char *") + || !strcmp(name, "char const *") + || !strcmp(name, "char const*") + || !strcmp(name, "char[]") + || !strcmp(name, "char []") + || !strcmp(name, "const char[]") + || !strcmp(name, "const char []") + || !strcmp(name, "char const[]") + || !strcmp(name, "char const []"); +} + +static bool is_float(const char *name) { + return contains_word(name, "float", sizeof ("float")) + || contains_word(name, "double", sizeof ("double")); +} + +static bool is_unsigned_int(const char *name) { + return contains_word(name, "unsigned", sizeof ("unsigned")) + || !strncmp(name, "uint", 4); +} + +static void format_arg(char (*arg)[1024], struct criterion_datapoints *dp, void *data) { + if (is_float(dp->name)) { + if (dp->size == sizeof (float)) { + snprintf(*arg, sizeof (*arg) - 1, "%g", *(float*) data); + } else if (dp->size == sizeof (double)) { + snprintf(*arg, sizeof (*arg) - 1, "%g", *(double*) data); + } else if (dp->size == sizeof (long double)) { + snprintf(*arg, sizeof (*arg) - 1, "%g", (double) *(long double*) data); + } + } else { + if (is_string(dp->name)) { + snprintf(*arg, sizeof (*arg) - 1, "%s", *(char**) data); + } else if (dp->size == sizeof (char)) { + snprintf(*arg, sizeof (*arg) - 1, "%c", *(char*) data); + } else if (dp->size == sizeof (short)) { + const char *fmt = is_unsigned_int(dp->name) ? "%hu" : "%hd"; + snprintf(*arg, sizeof (*arg) - 1, fmt, *(short*) data); + } else if (dp->size == sizeof (int)) { + const char *fmt = is_unsigned_int(dp->name) ? "%u" : "%d"; + snprintf(*arg, sizeof (*arg) - 1, fmt, *(int*) data); + } else if (dp->size == sizeof (bool)) { + snprintf(*arg, sizeof (*arg) - 1, "%d", *(bool*) data); + } else if (dp->size == sizeof (long)) { + const char *fmt = is_unsigned_int(dp->name) ? "%lu" : "%ld"; + snprintf(*arg, sizeof (*arg) - 1, fmt, *(long*) data); + } else if (dp->size == sizeof (long long)) { + const char *fmt = is_unsigned_int(dp->name) ? "%llu" : "%lld"; + snprintf(*arg, sizeof (*arg) - 1, fmt, *(long long*) data); + } else if (dp->size == sizeof (void*)) { + snprintf(*arg, sizeof (*arg) - 1, "%p", *(void**) data); + } else { + snprintf(*arg, sizeof (*arg) - 1, "%s", ""); + } + } +} + +static void concat_arg(char (*msg)[4096], struct criterion_datapoints *dps, size_t *indices, size_t i) { + void *data = ((char*) dps[i].arr) + dps[i].size * indices[i]; + + char arg[1024]; + format_arg(&arg, dps + i, data); + strncat(*msg, arg, sizeof (*msg) - 1); +} + void cr_theory_main(struct criterion_datapoints *dps, size_t datapoints, void (*fnptr)(void)) { struct criterion_theory_context *ctx = cr_theory_init(); @@ -119,7 +189,30 @@ void cr_theory_main(struct criterion_datapoints *dps, size_t datapoints, void (* dps[i].size, ((char*) dps[i].arr) + dps[i].size * indices[i]); } - cr_theory_call(ctx, fnptr); + + jmp_buf backup; + memcpy(backup, g_pre_test, sizeof (jmp_buf)); + int abort = 0; + if (!setjmp(g_pre_test)) { + cr_theory_call(ctx, fnptr); + } else { + abort = 1; + struct { + size_t len; + char msg[4096]; + } result; + for (size_t i = 0; i < datapoints - 1; ++i) { + concat_arg(&result.msg, dps, indices, i); + strncat(result.msg, ", ", sizeof (result.msg) - 1); + } + concat_arg(&result.msg, dps, indices, datapoints - 1); + result.len = strlen(result.msg) + 1; + + send_event(THEORY_FAIL, &result, result.len + sizeof (size_t)); + } + memcpy(g_pre_test, backup, sizeof (jmp_buf)); + if (abort) + criterion_abort_test(); } for (size_t i = 0; i < datapoints; ++i) {