stv0g/Criterion
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Added reporting & logging on theory failure

Browse source
This commit is contained in:
Snaipe 2015-09-03 17:27:50 +02:00
parent 029b5e43c3
commit 1757752dd1
15 changed files with 168 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/hooks.rst
View file

@ -31,6 +31,7 @@ The flow of the test process goes as follows:
#. ``PRE_INIT``: occurs before a test is initialized.
#. ``PRE_TEST``: occurs after the test initialization, but before the test is run.
#. ``ASSERT``: occurs when an assertion is hit
#. ``THEORY_FAIL``: occurs when a theory iteration fails.
#. ``TEST_CRASH``: occurs when a test crashes unexpectedly.
#. ``POST_TEST``: occurs after a test ends, but before the test finalization.
#. ``POST_FINI``: occurs after a test finalization.
@ -50,6 +51,7 @@ Valid types for each phases are:
* ``struct criterion_suite_set *`` for ``PRE_SUITE``.
* ``struct criterion_test *`` for ``PRE_INIT`` and ``PRE_TEST``.
* ``struct criterion_assert_stats *`` for ``ASSERT``.
* ``struct criterion_theory_stats *`` for ``THEORY_FAIL``.
* ``struct criterion_test_stats *`` for ``POST_TEST``, ``POST_FINI``, and ``TEST_CRASH``.
* ``struct criterion_suite_stats *`` for ``POST_SUITE``.
* ``struct criterion_global_stats *`` for ``POST_ALL``.

2
include/criterion/hooks.h
View file

@ -32,6 +32,7 @@ typedef enum {
PRE_INIT,
PRE_TEST,
ASSERT,
THEORY_FAIL,
TEST_CRASH,
POST_TEST,
POST_FINI,
@ -54,6 +55,7 @@ typedef void (*f_report_hook)();
# define HOOK_SECTION_PRE_INIT cr_pri
# define HOOK_SECTION_PRE_TEST cr_prt
# define HOOK_SECTION_ASSERT cr_ast
# define HOOK_SECTION_THEORY_FAIL cr_thf
# define HOOK_SECTION_TEST_CRASH cr_tsc
# define HOOK_SECTION_POST_TEST cr_pot
# define HOOK_SECTION_POST_FINI cr_pof

1
include/criterion/logging.h
View file

@ -96,6 +96,7 @@ struct criterion_output_provider {
void (*log_pre_init )(struct criterion_test *test);
void (*log_pre_test )(struct criterion_test *test);
void (*log_assert )(struct criterion_assert_stats *stats);
void (*log_theory_fail)(struct criterion_theory_stats *stats);
void (*log_test_crash )(struct criterion_test_stats *stats);
void (*log_other_crash)(struct criterion_test_stats *stats);
void (*log_post_test )(struct criterion_test_stats *stats);

5
include/criterion/stats.h
View file

@ -51,6 +51,11 @@ struct criterion_test_stats {
struct criterion_test_stats *next;
};
struct criterion_theory_stats {
const char *formatted_args;
struct criterion_test_stats *stats;
};
struct criterion_suite_stats {
struct criterion_suite *suite;
struct criterion_test_stats *tests;

8
include/criterion/theories.h
View file

@ -41,8 +41,12 @@ void cr_theory_call(struct criterion_theory_context *ctx, void (*fnptr)(void));
# define TheoryDataPoints(Category, Name) \
static struct criterion_datapoints IDENTIFIER_(Category, Name, dps)[]
# define DataPoints(Type, ...) \
{ sizeof (Type), sizeof ((Type[]) { __VA_ARGS__ }) / sizeof (Type), #Type, &(Type[]) { __VA_ARGS__ } }
# define DataPoints(Type, ...) { \
sizeof (Type), \
sizeof ((Type[]) { __VA_ARGS__ }) / sizeof (Type), \
#Type, \
&(Type[]) { __VA_ARGS__ }, \
}
struct criterion_datapoints {
size_t size;

16
po/fr.po
View file

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: criterion 1.0.0\n"
"Report-Msgid-Bugs-To: franklinmathieu+criterion@gmail.com\n"
"POT-Creation-Date: 2015-08-05 11:37+0200\n"
"POT-Creation-Date: 2015-09-03 17:24+0200\n"
"PO-Revision-Date: 2015-04-03 17:58+0200\n"
"Last-Translator: <franklinmathieu@gmail.com>\n"
"Language-Team: French\n"
@ -54,18 +54,24 @@ msgid "%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n"
msgstr "%1$s%2$s%3$s:%4$s%5$d%6$s: Échec d'assertion: %7$s\n"
#: src/log/normal.c:61
#, fuzzy, c-format
msgid " Theory %1$s::%2$s failed with the following parameters: (%3$s)\n"
msgstr ""
" La théorie %1$s::%2$s a échoué avec les paramètres suivants: (%3$s)\n"
#: src/log/normal.c:62
#, c-format
msgid "%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n"
msgstr ""
"%1$s%2$s%3$s:%4$s%5$u%6$s: Un signal inattendu a été reçu après cette "
"ligne!\n"
#: src/log/normal.c:62
#: src/log/normal.c:63
#, c-format
msgid "%1$s::%2$s: CRASH!\n"
msgstr "%1$s::%2$s: PLANTAGE!\n"
#: src/log/normal.c:63
#: src/log/normal.c:64
#, fuzzy, c-format
msgid ""
"%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown."
@ -74,14 +80,14 @@ msgstr ""
"%1$sAttention! Le test `%2$s::%3$s` a planté pendant son initialisation ou "
"sa finalisation.%4$s\n"
#: src/log/normal.c:64
#: src/log/normal.c:65
#, c-format
msgid "Running %1$s%2$lu%3$s test from %4$s%5$s%6$s:\n"
msgid_plural "Running %1$s%2$lu%3$s tests from %4$s%5$s%6$s:\n"
msgstr[0] "Lancement de %1$s%2$lu%3$s test dans %4$s%5$s%6$s:\n"
msgstr[1] "Lancement de %1$s%2$lu%3$s tests dans %4$s%5$s%6$s:\n"
#: src/log/normal.c:66
#: src/log/normal.c:67
#, c-format
msgid ""
"%1$sSynthesis: Tested: %2$s%3$lu%4$s | Passing: %5$s%6$lu%7$s | Failing: %8$s"

3
src/abort.c
View file

@ -21,10 +21,9 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <setjmp.h>
#include "abort.h"
static jmp_buf g_pre_test;
jmp_buf g_pre_test;
int setup_abort_test(void) {
return !setjmp(g_pre_test);

3
src/abort.h
View file

@ -25,6 +25,9 @@
# define ABORT_H_
# include <criterion/abort.h>
# include <setjmp.h>
extern jmp_buf g_pre_test;
int setup_abort_test(void);

20
src/event.c
View file

@ -54,6 +54,26 @@ struct event *read_event(FILE *f) {
.value = { .kind = kind, .data = buf },
.dtor = destroy_event);
}
case THEORY_FAIL: {
size_t *len = malloc(sizeof (size_t));
if (fread(len, sizeof (size_t), 1, f) == 0) {
free(len);
return NULL;
}
char *buf = malloc(*len);
if (fread(buf, *len, 1, f) == 0) {
free(len);
free(buf);
return NULL;
}
free(len);
return unique_ptr(struct event,
.value = { .kind = kind, .data = buf },
.dtor = destroy_event);
}
case POST_TEST: {
double *elapsed_time = malloc(sizeof (double));
if (fread(elapsed_time, sizeof (double), 1, f) == 0) {

11
src/log/normal.c
View file

@ -58,6 +58,7 @@ static msg_t msg_post_test = N_("%1$s::%2$s\n");
static msg_t msg_post_suite_test = N_("%1$s::%2$s: Test is disabled\n");
static msg_t msg_post_suite_suite = N_("%1$s::%2$s: Suite is disabled\n");
static msg_t msg_assert_fail = N_("%1$s%2$s%3$s:%4$s%5$d%6$s: Assertion failed: %7$s\n");
static msg_t msg_theory_fail = N_(" Theory %1$s::%2$s failed with the following parameters: (%3$s)\n");
static msg_t msg_test_crash_line = N_("%1$s%2$s%3$s:%4$s%5$u%6$s: Unexpected signal caught below this line!\n");
static msg_t msg_test_crash = N_("%1$s::%2$s: CRASH!\n");
static msg_t msg_test_other_crash = N_("%1$sWarning! The test `%2$s::%3$s` crashed during its setup or teardown.%4$s\n");
@ -75,6 +76,7 @@ static msg_t msg_post_test = "%s::%s\n";
static msg_t msg_post_suite_test = "%s::%s: Test is disabled\n";
static msg_t msg_post_suite_suite = "%s::%s: Suite is disabled\n";
static msg_t msg_assert_fail = "%s%s%s:%s%d%s: Assertion failed: %s\n";
static msg_t msg_theory_fail = " Theory %s::%s failed with the following parameters: %s\n";
static msg_t msg_test_crash_line = "%s%s%s:%s%u%s: Unexpected signal caught below this line!\n";
static msg_t msg_test_crash = "%s::%s: CRASH!\n";
static msg_t msg_test_other_crash = "%sWarning! The test `%s::%s` crashed during its setup or teardown.%s\n";
@ -205,11 +207,20 @@ void normal_log_pre_suite(struct criterion_suite_set *set) {
FG_GOLD, set->suite.name, RESET);
}
void normal_log_theory_fail(struct criterion_theory_stats *stats) {
criterion_pimportant(CRITERION_PREFIX_DASHES,
_(msg_theory_fail),
stats->stats->test->category,
stats->stats->test->name,
stats->formatted_args);
}
struct criterion_output_provider normal_logging = {
.log_pre_all = normal_log_pre_all,
.log_pre_init = normal_log_pre_init,
.log_pre_suite = normal_log_pre_suite,
.log_assert = normal_log_assert,
.log_theory_fail = normal_log_theory_fail,
.log_test_crash = normal_log_test_crash,
.log_other_crash = normal_log_other_crash,
.log_post_test = normal_log_post_test,

2
src/report.c
View file

@ -50,6 +50,7 @@ IMPL_CALL_REPORT_HOOKS(PRE_SUITE);
IMPL_CALL_REPORT_HOOKS(PRE_INIT);
IMPL_CALL_REPORT_HOOKS(PRE_TEST);
IMPL_CALL_REPORT_HOOKS(ASSERT);
IMPL_CALL_REPORT_HOOKS(THEORY_FAIL);
IMPL_CALL_REPORT_HOOKS(TEST_CRASH);
IMPL_CALL_REPORT_HOOKS(POST_TEST);
IMPL_CALL_REPORT_HOOKS(POST_FINI);
@ -61,6 +62,7 @@ ReportHook(PRE_SUITE)() {}
ReportHook(PRE_INIT)() {}
ReportHook(PRE_TEST)() {}
ReportHook(ASSERT)() {}
ReportHook(THEORY_FAIL)() {}
ReportHook(TEST_CRASH)() {}
ReportHook(POST_TEST)() {}
ReportHook(POST_FINI)() {}

1
src/report.h
View file

@ -37,6 +37,7 @@ DECL_CALL_REPORT_HOOKS(PRE_SUITE);
DECL_CALL_REPORT_HOOKS(PRE_INIT);
DECL_CALL_REPORT_HOOKS(PRE_TEST);
DECL_CALL_REPORT_HOOKS(ASSERT);
DECL_CALL_REPORT_HOOKS(THEORY_FAIL);
DECL_CALL_REPORT_HOOKS(TEST_CRASH);
DECL_CALL_REPORT_HOOKS(POST_TEST);
DECL_CALL_REPORT_HOOKS(POST_FINI);

8
src/runner.c
View file

@ -220,6 +220,14 @@ static void run_test(struct criterion_global_stats *stats,
log(pre_test, test);
test_started = true;
break;
case THEORY_FAIL: {
struct criterion_theory_stats ths = {
.formatted_args = (char*) ev->data,
.stats = test_stats,
};
report(THEORY_FAIL, &ths);
log(theory_fail, &ths);
} break;
case ASSERT:
report(ASSERT, ev->data);
log(assert, ev->data);

1
src/stats.c
View file

@ -91,6 +91,7 @@ void stat_push_event(s_glob_stats *stats,
push_pre_init, // PRE_INIT
nothing, // PRE_TEST
push_assert, // ASSERT
nothing, // THEORY_FAIL
push_test_crash, // TEST_CRASH
push_post_test, // POST_TEST
nothing, // POST_FINI

95
src/theories.c
View file

@ -27,6 +27,7 @@
#include <dyncall.h>
#include <assert.h>
#include "criterion/theories.h"
#include "abort.h"
struct criterion_theory_context {
DCCallVM* vm;
@ -99,6 +100,75 @@ static bool contains_word(const char *str, const char *pattern, size_t sz) {
&& (!res[sz - 1] || res[sz - 1] == ' ');
}
static bool is_string(const char *name) {
return !strcmp(name, "char*")
|| !strcmp(name, "char *")
|| !strcmp(name, "const char*")
|| !strcmp(name, "const char *")
|| !strcmp(name, "char const *")
|| !strcmp(name, "char const*")
|| !strcmp(name, "char[]")
|| !strcmp(name, "char []")
|| !strcmp(name, "const char[]")
|| !strcmp(name, "const char []")
|| !strcmp(name, "char const[]")
|| !strcmp(name, "char const []");
}
static bool is_float(const char *name) {
return contains_word(name, "float", sizeof ("float"))
|| contains_word(name, "double", sizeof ("double"));
}
static bool is_unsigned_int(const char *name) {
return contains_word(name, "unsigned", sizeof ("unsigned"))
|| !strncmp(name, "uint", 4);
}
static void format_arg(char (*arg)[1024], struct criterion_datapoints *dp, void *data) {
if (is_float(dp->name)) {
if (dp->size == sizeof (float)) {
snprintf(*arg, sizeof (*arg) - 1, "%g", *(float*) data);
} else if (dp->size == sizeof (double)) {
snprintf(*arg, sizeof (*arg) - 1, "%g", *(double*) data);
} else if (dp->size == sizeof (long double)) {
snprintf(*arg, sizeof (*arg) - 1, "%g", (double) *(long double*) data);
}
} else {
if (is_string(dp->name)) {
snprintf(*arg, sizeof (*arg) - 1, "%s", *(char**) data);
} else if (dp->size == sizeof (char)) {
snprintf(*arg, sizeof (*arg) - 1, "%c", *(char*) data);
} else if (dp->size == sizeof (short)) {
const char *fmt = is_unsigned_int(dp->name) ? "%hu" : "%hd";
snprintf(*arg, sizeof (*arg) - 1, fmt, *(short*) data);
} else if (dp->size == sizeof (int)) {
const char *fmt = is_unsigned_int(dp->name) ? "%u" : "%d";
snprintf(*arg, sizeof (*arg) - 1, fmt, *(int*) data);
} else if (dp->size == sizeof (bool)) {
snprintf(*arg, sizeof (*arg) - 1, "%d", *(bool*) data);
} else if (dp->size == sizeof (long)) {
const char *fmt = is_unsigned_int(dp->name) ? "%lu" : "%ld";
snprintf(*arg, sizeof (*arg) - 1, fmt, *(long*) data);
} else if (dp->size == sizeof (long long)) {
const char *fmt = is_unsigned_int(dp->name) ? "%llu" : "%lld";
snprintf(*arg, sizeof (*arg) - 1, fmt, *(long long*) data);
} else if (dp->size == sizeof (void*)) {
snprintf(*arg, sizeof (*arg) - 1, "%p", *(void**) data);
} else {
snprintf(*arg, sizeof (*arg) - 1, "%s", "<np>");
}
}
}
static void concat_arg(char (*msg)[4096], struct criterion_datapoints *dps, size_t *indices, size_t i) {
void *data = ((char*) dps[i].arr) + dps[i].size * indices[i];
char arg[1024];
format_arg(&arg, dps + i, data);
strncat(*msg, arg, sizeof (*msg) - 1);
}
void cr_theory_main(struct criterion_datapoints *dps, size_t datapoints, void (*fnptr)(void)) {
struct criterion_theory_context *ctx = cr_theory_init();
@ -119,7 +189,30 @@ void cr_theory_main(struct criterion_datapoints *dps, size_t datapoints, void (*
dps[i].size,
((char*) dps[i].arr) + dps[i].size * indices[i]);
}
cr_theory_call(ctx, fnptr);
jmp_buf backup;
memcpy(backup, g_pre_test, sizeof (jmp_buf));
int abort = 0;
if (!setjmp(g_pre_test)) {
cr_theory_call(ctx, fnptr);
} else {
abort = 1;
struct {
size_t len;
char msg[4096];
} result;
for (size_t i = 0; i < datapoints - 1; ++i) {
concat_arg(&result.msg, dps, indices, i);
strncat(result.msg, ", ", sizeof (result.msg) - 1);
}
concat_arg(&result.msg, dps, indices, datapoints - 1);
result.len = strlen(result.msg) + 1;
send_event(THEORY_FAIL, &result, result.len + sizeof (size_t));
}
memcpy(g_pre_test, backup, sizeof (jmp_buf));
if (abort)
criterion_abort_test();
}
for (size_t i = 0; i < datapoints; ++i) {