diff --git a/.travis.yml b/.travis.yml index 7d2629c..95ce90a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,4 +12,4 @@ script: after_success: - coveralls --gcov gcov-4.9 --exclude samples --exclude dependencies --gcov-options '\-lp' -b . after_failure: - - cat $(find check -iname '*.log') /dev/null + - cat $(find samples -iname '*.log') /dev/null diff --git a/ChangeLog b/ChangeLog index 4548437..c909ef7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2015-03-18 Franklin "Snaipe" Mathieu + * src/timer.*: Added test timings + * src/, include/: Changed assert prototypes + * include/: ANSI compliance over header files + * *: Windows + FreeBSD compatibility + 2015-03-11 Franklin "Snaipe" Mathieu * include/criterion/logging.h, src/logging.c: A logging interface diff --git a/Makefile.am b/Makefile.am index f373ec2..371c649 100644 --- a/Makefile.am +++ b/Makefile.am @@ -45,4 +45,6 @@ libcriterion_la_SOURCES = \ src/stats.h \ src/logging.c \ src/options.c \ + src/timer.c \ + src/timer.h \ src/main.c diff --git a/README.md b/README.md index 2cdb128..c665ff3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Criterion [![License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](https://github.com/Snaipe/Criterion/blob/master/LICENSE) [![Version](https://img.shields.io/github/tag/Snaipe/Criterion.svg?label=version&style=flat)](https://github.com/Snaipe/Criterion/releases) -A dead-simple, yet extensible, C test framework. +A dead-simple, yet extensible, C unit testing framework. ![Screencast](./doc/screencast.gif) @@ -30,13 +30,14 @@ the user would have with other frameworks: reported and tested. * [x] Progress and statistics can be followed in real time with report hooks. * [x] TAP output format can be enabled with an option. -* [x] Runs on Linux, Mac OS X, and Windows (compiles only with Cygwin for the moment). +* [x] Runs on Linux, FreeBSD, Mac OS X, and Windows (compiles only with Cygwin for the moment). ## Downloads -* [Linux (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v0.3/criterion-0.3-linux-x86_64.tar.bz2) -* [OS X (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v0.3/criterion-0.3-osx-x86_64.tar.bz2) -* [Windows (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v0.3/criterion-0.3-win-x86_64.tar.bz2) +* [Linux (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-linux-x86_64.tar.bz2) +* [OS X (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-osx-x86_64.tar.bz2) +* [Windows (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-win-x86_64.tar.bz2) +* [FreeBSD (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-freebsd-x86_64.tar.bz2) If you have a different platform, you can still [build the library from source](http://criterion.readthedocs.org/en/latest/setup.html#installation) @@ -51,6 +52,7 @@ Sample tests can be found in the [sample directory][samples]. * [A simple test][sample-simple] * [Using multiple suites][sample-suites] +* [Writing assertions][sample-asserts] * [Adding test fixtures][sample-fixtures] * [Tests with signals][sample-signal] * [Using report hooks][sample-report] @@ -68,9 +70,8 @@ A. I worked with CUnit and Check, and I must say that they do their job **Q. Where has this been tested?** A. Currently, on Linux 2.6.32 and Linux 3.15.7, although it should work on - most \*nix systems; Mac OS X Yosemite 10.10, and finally Windows 7 (with - the Cygwin port of GCC). - More tests will be added on the build matrix. + most \*nix systems; Mac OS X Yosemite 10.10, FreeBSD 10.0, and finally + Windows 7 (with the Cygwin port of GCC). **Q. Can I use it on Windows without Cygwin?** A. Yes, you can, Cygwin is only required to compile the static library if @@ -85,6 +86,7 @@ A. Yes, you can, Cygwin is only required to compile the static library if [samples]: ./samples/ [sample-simple]: ./samples/simple.c [sample-suites]: ./samples/suites.c +[sample-asserts]: ./samples/asserts.c [sample-fixtures]: ./samples/fixtures.c [sample-signal]: ./samples/signal.c [sample-report]: ./samples/report.c diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..5ef46fa --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,70 @@ +version: 1.0.0_b{build}-{branch} + +os: Windows Server 2012 + +init: + - ps: (New-Object System.Net.WebClient).DownloadFile("https://cygwin.com/setup-x86.exe", "c:\setup-x86.exe") + - git config --global core.autocrlf input + - c:\setup-x86.exe -qnNdO -R %CYG_ROOT% -s %CYG_MIRROR% -l %CYG_CACHE% \ + -P autoconf \ + -P automake \ + -P gcc-core \ + -P mingw-runtime \ + -P mingw-binutils \ + -P mingw-gcc-core \ + -P mingw-pthreads \ + -P mingw-w32api \ + -P libtool \ + -P make \ + -P python \ + -P gettext-devel \ + -P gettext \ + -P expat \ + -P intltool \ + -P libiconv \ + -P pkg-config \ + -P check \ + -P git \ + -P wget \ + -P curl + - "%CYG_BASH% -lc 'echo $PATH'" + +environment: + global: + CYG_ROOT: C:\cygwin + CYG_MIRROR: http://cygwin.mirror.constant.com + CYG_CACHE: C:\cygwin\var\cache\setup + CYG_BASH: C:\cygwin\bin\bash + +cache: + - '%CYG_CACHE%' + +clone_depth: 5 + +matrix: + fast_finish: true # set this flag to immediately finish build once one of the jobs fails. + +platform: + - x86 + - Any CPU + +configuration: Release + +install: + - "%CYG_BASH% -lc 'cd $APPVEYOR_BUILD_FOLDER; ./autogen.sh'" + - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0, "" __VA_ARGS__) -# define expect_gt(Actual, Expected, ...) \ - expect_op(Actual, Expected, >, "" __VA_ARGS__) +# define assert_gt(...) assert_op_(>, __VA_ARGS__, .sentinel_ = 0) +# define expect_gt(...) expect_op_(>, __VA_ARGS__, .sentinel_ = 0) -# define assert_leq(Actual, Expected, ...) \ - assert_op(Actual, Expected, <=, "" __VA_ARGS__) -# define expect_leq(Actual, Expected, ...) \ - expect_op(Actual, Expected, <=, "" __VA_ARGS__) +# define assert_leq(...) assert_op_(<=, __VA_ARGS__, .sentinel_ = 0) +# define expect_leq(...) expect_op_(<=, __VA_ARGS__, .sentinel_ = 0) -# define assert_geq(Actual, Expected, ...) \ - assert_op(Actual, Expected, >=, "" __VA_ARGS__) -# define expect_geq(Actual, Expected, ...) \ - expect_op(Actual, Expected, >=, "" __VA_ARGS__) +# define assert_geq(...) assert_op_(>=, __VA_ARGS__, .sentinel_ = 0) +# define expect_geq(...) expect_op_(>=, __VA_ARGS__, .sentinel_ = 0) // Floating-point asserts -# define assert_float_equal(Actual, Expected, Epsilon, ...) \ - assert((Expected) - (Actual) <= (Epsilon) && (Actual) - (Expected) <= (Epsilon), "" __VA_ARGS__) -# define expect_float_equal(Actual, Expected, Epsilon, ...) \ - expect((Expected) - (Actual) <= (Epsilon) && (Actual) - (Expected) <= (Epsilon), "" __VA_ARGS__) +# define assert_float_eq(...) \ + assert_float_eq_(__VA_ARGS__, .sentinel_ = 0) +# define expect_float_eq(...) \ + expect_float_eq_(__VA_ARGS__, .sentinel_ = 0) -# define assert_float_not_equal(Actual, Expected, Epsilon, ...) \ - assert((Expected) - (Actual) > (Epsilon) || (Actual) - (Expected) > (Epsilon), "" __VA_ARGS__) -# define expect_float_not_equal(Actual, Expected, Epsilon, ...) \ - expect((Expected) - (Actual) > (Epsilon) || (Actual) - (Expected) > (Epsilon), "" __VA_ARGS__) +# define assert_float_eq_(Actual, Expected, Epsilon, ...) \ + assert_impl(FATAL, (Expected) - (Actual) <= (Epsilon) \ + && (Actual) - (Expected) <= (Epsilon), \ + __VA_ARGS__) +# define expect_float_eq_(Actual, Expected, Epsilon, ...) \ + assert_impl(NORMAL, (Expected) - (Actual) <= (Epsilon) \ + && (Actual) - (Expected) <= (Epsilon), \ + __VA_ARGS__) + +# define assert_float_neq(...) \ + assert_float_neq_(__VA_ARGS__, .sentinel_ = 0) +# define expect_float_neq(...) \ + expect_float_neq_(__VA_ARGS__, .sentinel_ = 0) + +# define assert_float_neq_(Actual, Expected, Epsilon, ...) \ + assert_impl(FATAL, (Expected) - (Actual) > (Epsilon) \ + || (Actual) - (Expected) > (Epsilon), \ + __VA_ARGS__) +# define expect_float_neq_(Actual, Expected, Epsilon, ...) \ + assert_impl(NORMAL, (Expected) - (Actual) > (Epsilon) \ + || (Actual) - (Expected) > (Epsilon), \ + __VA_ARGS__) // String asserts -# define assert_strings(Actual, Expected, Op, ...) \ - assert(strcmp((Actual), (Expected)) Op 0, "" __VA_ARGS__) -# define expect_strings(Actual, Expected, Op, ...) \ - expect(strcmp((Actual), (Expected)) Op 0, "" __VA_ARGS__) +# define assert_strings_(Op, Actual, Expected, ...) \ + assert_impl(FATAL, strcmp((Actual), (Expected)) Op 0, __VA_ARGS__) +# define expect_strings_(Op, Actual, Expected, ...) \ + assert_impl(NORMAL, strcmp((Actual), (Expected)) Op 0, __VA_ARGS__) -# define assert_strings_equal(Actual, Expected, ...) \ - assert_strings(Actual, Expected, ==, "" __VA_ARGS__) -# define expect_strings_equal(Actual, Expected, ...) \ - expect_strings(Actual, Expected, ==, "" __VA_ARGS__) +# define assert_strings_eq(...) \ + assert_strings_(==, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_eq(...) \ + expect_strings_(==, __VA_ARGS__, .sentinel_ = 0) -# define assert_strings_gt(Actual, Expected, ...) \ - assert_strings(Actual, Expected, >, "" __VA_ARGS__) -# define expect_strings_gt(Actual, Expected, ...) \ - expect_strings(Actual, Expected, >, "" __VA_ARGS__) +# define assert_strings_neq(...) \ + assert_strings_(!=, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_neq(...) \ + expect_strings_(!=, __VA_ARGS__, .sentinel_ = 0) -# define assert_strings_lt(Actual, Expected, ...) \ - assert_strings(Actual, Expected, <, "" __VA_ARGS__) -# define expect_strings_lt(Actual, Expected, ...) \ - expect_strings(Actual, Expected, <, "" __VA_ARGS__) +# define assert_strings_gt(...) assert_strings_(>, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_gt(...) expect_strings_(>, __VA_ARGS__, .sentinel_ = 0) -# define assert_strings_geq(Actual, Expected, ...) \ - assert_strings(Actual, Expected, >=, "" __VA_ARGS__) -# define expect_strings_geq(Actual, Expected, ...) \ - expect_strings(Actual, Expected, >=, "" __VA_ARGS__) +# define assert_strings_lt(...) assert_strings_(<, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_lt(...) expect_strings_(<, __VA_ARGS__, .sentinel_ = 0) -# define assert_strings_leq(Actual, Expected, ...) \ - assert_strings(Actual, Expected, <=, "" __VA_ARGS__) -# define expect_strings_leq(Actual, Expected, ...) \ - expect_strings(Actual, Expected, <=, "" __VA_ARGS__) +# define assert_strings_leq(...) assert_strings_(<=, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_leq(...) expect_strings_(<=, __VA_ARGS__, .sentinel_ = 0) -# define assert_strings_not_equal(Actual, Expected, ...) \ - assert_strings(Actual, Expected, !=, "" __VA_ARGS__) -# define expect_strings_not_equal(Actual, Expected, ...) \ - expect_strings(Actual, Expected, !=, "" __VA_ARGS__) +# define assert_strings_geq(...) assert_strings_(>=, __VA_ARGS__, .sentinel_ = 0) +# define expect_strings_geq(...) expect_strings_(>=, __VA_ARGS__, .sentinel_ = 0) // Array asserts -# define assert_arrays_equal(A, B, Size, ...) \ - assert(!memcmp((A), (B), (Size)), "" __VA_ARGS__) -# define expect_arrays_equal(A, B, Size, ...) \ - expect(!memcmp((A), (B), (Size)), "" __VA_ARGS__) +# define assert_arrays_eq(...) \ + assert_arrays_eq_(__VA_ARGS__, .sentinel = 0) +# define expect_arrays_eq(...) \ + expect_arrays_eq_(__VA_ARGS__, .sentinel = 0) -# define assert_arrays_not_equal(A, B, Size, ...) \ - assert(memcmp((A), (B), (Size)), "" __VA_ARGS__) -# define expect_arrays_not_equal(A, B, Size, ...) \ - expect(memcmp((A), (B), (Size)), "" __VA_ARGS__) +# define assert_arrays_neq(...) \ + assert_arrays_neq_(__VA_ARGS__, .sentinel = 0) +# define expect_arrays_neq(...) \ + expect_arrays_neq_(__VA_ARGS__, .sentinel = 0) + +# define assert_arrays_eq_(A, B, Size, ...) \ + assert_impl(FATAL, !memcmp((A), (B), (Size)), __VA_ARGS__) +# define expect_arrays_eq_(A, B, Size, ...) \ + assert_impl(NORMAL, !memcmp((A), (B), (Size)), __VA_ARGS__) + +# define assert_arrays_neq_(A, B, Size, ...) \ + assert_impl(FATAL, memcmp((A), (B), (Size)), __VA_ARGS__) +# define expect_arrays_neq_(A, B, Size, ...) \ + assert_impl(NORMAL, memcmp((A), (B), (Size)), __VA_ARGS__) #endif /* !CRITERION_ASSERT_H_ */ diff --git a/include/criterion/common.h b/include/criterion/common.h index 32d6420..cdeffa9 100644 --- a/include/criterion/common.h +++ b/include/criterion/common.h @@ -57,6 +57,12 @@ # define UNUSED __attribute__((unused)) +# ifdef _WIN32 +# define SIZE_T_FORMAT "%Iu" +# else +# define SIZE_T_FORMAT "%zu" +# endif + # ifdef __GNUC__ # define FORMAT(Archetype, Index, Ftc) __attribute__((format(Archetype, Index, Ftc))) # else diff --git a/include/criterion/criterion.h b/include/criterion/criterion.h index 6c59180..2c0d963 100644 --- a/include/criterion/criterion.h +++ b/include/criterion/criterion.h @@ -30,6 +30,7 @@ # include "assert.h" struct criterion_test_extra_data { + int sentinel_; const char *const file_; const unsigned line_; void (*init)(void); @@ -56,7 +57,8 @@ struct criterion_test_set { # define TEST_PROTOTYPE_(Category, Name) \ void IDENTIFIER_(Category, Name, impl)(void) -# define Test(Category, Name, ...) \ +# define Test(...) Test_(__VA_ARGS__, .sentinel_ = 0) +# define Test_(Category, Name, ...) \ TEST_PROTOTYPE_(Category, Name); \ struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = { \ .file_ = __FILE__, \ diff --git a/include/criterion/event.h b/include/criterion/event.h index 37080ee..75f4a7c 100644 --- a/include/criterion/event.h +++ b/include/criterion/event.h @@ -24,6 +24,8 @@ #ifndef CRITERION_EVENT_H_ # define CRITERION_EVENT_H_ +# include + extern int EVENT_PIPE; void send_event(int kind, void *data, size_t size); diff --git a/include/criterion/options.h b/include/criterion/options.h index 2d24fb5..d04db21 100644 --- a/include/criterion/options.h +++ b/include/criterion/options.h @@ -24,6 +24,7 @@ #ifndef CRITERION_OPTIONS_H_ # define CRITERION_OPTIONS_H_ +# include # include "logging.h" struct criterion_options { diff --git a/include/criterion/stats.h b/include/criterion/stats.h index c836cbe..2225896 100644 --- a/include/criterion/stats.h +++ b/include/criterion/stats.h @@ -25,6 +25,7 @@ # define CRITERION_STATS_H_ # include +# include # include "criterion.h" struct criterion_assert_stats { @@ -45,6 +46,7 @@ struct criterion_test_stats { int passed_asserts; int failed_asserts; int signal; + float elapsed_time; unsigned progress; const char *file; diff --git a/samples/Makefile.am b/samples/Makefile.am index e92ca78..6660f1c 100644 --- a/samples/Makefile.am +++ b/samples/Makefile.am @@ -1,4 +1,4 @@ -TESTS = \ +BIN_TESTS = \ signal \ report \ suites \ @@ -6,18 +6,22 @@ TESTS = \ asserts \ simple -TESTS += tests/tap_test.sh \ - tests/early_exit.sh \ - tests/verbose.sh \ - tests/help.sh +TESTS_ENVIRONMENT = CRITERION_ALWAYS_SUCCEED=1 + +check_PROGRAMS := $(BIN_TESTS) +CFLAGS = -I$(top_srcdir)/include/ -std=c99 -Wall -Wextra -pedantic +LDADD = -L$(top_srcdir)/ -lcriterion + +SCRIPT_TESTS = tests/tap_test.sh \ + tests/early_exit.sh \ + tests/verbose.sh \ + tests/help.sh + +EXTRA_DIST = $(SCRIPT_TESTS) tests/tap_test.sh: simple signal asserts tests/early_exit.sh: simple tests/verbose.sh: simple tests/help.sh: simple -TESTS_ENVIRONMENT = CRITERION_ALWAYS_SUCCEED=1 - -check_PROGRAMS = $(TESTS) -CFLAGS = -I$(top_srcdir)/include/ -std=c99 -LDADD = -L$(top_srcdir)/ -lcriterion +TESTS = $(BIN_TESTS) $(SCRIPT_TESTS) diff --git a/samples/asserts.c b/samples/asserts.c index deec0b4..811824f 100644 --- a/samples/asserts.c +++ b/samples/asserts.c @@ -6,14 +6,23 @@ Test(asserts, base) { assert(true, "Assertions may take failure messages"); + assert(true, .msg = "You can use explicit named arguments"); + expect(false, "assert is fatal, expect isn't"); assert(false, "This assert runs"); assert(false, "This does not"); } +Test(asserts, old_school) { + if (false) + abort_test("You can abort the test with a message from anywhere"); + + abort_test(NULL); // or without a message +} + Test(asserts, string) { - assert_strings_equal("hello", "hello"); - assert_strings_not_equal("hello", "olleh"); + assert_strings_eq("hello", "hello"); + assert_strings_neq("hello", "olleh"); assert_strings_gt("hello", "hell"); assert_strings_geq("hello", "hell"); @@ -25,8 +34,8 @@ Test(asserts, string) { } Test(asserts, native) { - assert_equal(1, 1); - assert_not_equal(1, 2); + assert_eq(1, 1); + assert_neq(1, 2); assert_lt(1, 2); assert_leq(1, 2); @@ -38,6 +47,6 @@ Test(asserts, native) { } Test(asserts, float) { - assert_not_equal(0.1 * 0.1, 0.01); - assert_float_equal(0.1 * 0.1, 0.01, 0.001); + assert_neq(0.1 * 0.1, 0.01); + assert_float_eq(0.1 * 0.1, 0.01, 0.001); } diff --git a/src/event.c b/src/event.c index 0e569d7..e5df964 100644 --- a/src/event.c +++ b/src/event.c @@ -21,14 +21,9 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#include "criterion/assert.h" -#undef assert -#include #include -#include #include -#include "criterion/criterion.h" #include "criterion/stats.h" #include "criterion/hooks.h" #include "event.h" @@ -45,15 +40,25 @@ struct event *read_event(int fd) { if (read(fd, &kind, sizeof (unsigned)) < (ssize_t) sizeof (unsigned)) return NULL; - if (kind != ASSERT) - return unique_ptr(struct event, ({ .kind = kind, .data = NULL })); + switch (kind) { + case ASSERT: { + const size_t assert_size = sizeof (struct criterion_assert_stats); + unsigned char *buf = malloc(assert_size); + if (read(fd, buf, assert_size) < (ssize_t) assert_size) + return NULL; - const size_t assert_size = sizeof (struct criterion_assert_stats); - unsigned char *buf = malloc(assert_size); - if (read(fd, buf, assert_size) < (ssize_t) assert_size) - return NULL; + return unique_ptr(struct event, ({ .kind = kind, .data = buf }), destroy_event); + } + case POST_TEST: { + double *elapsed_time = malloc(sizeof (double)); + if (read(fd, elapsed_time, sizeof (double)) < (ssize_t) sizeof (double)) + return NULL; - return unique_ptr(struct event, ({ .kind = kind, .data = buf }), destroy_event); + return unique_ptr(struct event, ({ .kind = kind, .data = elapsed_time }), destroy_event); + } + default: + return unique_ptr(struct event, ({ .kind = kind, .data = NULL })); + } } void send_event(int kind, void *data, size_t size) { diff --git a/src/main.c b/src/main.c index 3c5f26e..ad350c0 100644 --- a/src/main.c +++ b/src/main.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE #include -#include #include #include #include @@ -44,5 +43,5 @@ int main(int argc, char *argv[]) { } } - return criterion_run_all_tests(); + return !criterion_run_all_tests(); } diff --git a/src/process.c b/src/process.c index ffc971e..17ef862 100644 --- a/src/process.c +++ b/src/process.c @@ -22,9 +22,7 @@ * THE SOFTWARE. */ #include -#include #include -#include #include #include @@ -71,7 +69,7 @@ struct process *spawn_test_worker(struct criterion_test *test, void (*func)(stru func(test); close(fds[1]); - if (!criterion_options.no_early_exit) + if (criterion_options.no_early_exit) return NULL; else _exit(0); diff --git a/src/process.h b/src/process.h index fcf9e62..712648c 100644 --- a/src/process.h +++ b/src/process.h @@ -24,6 +24,8 @@ #ifndef PROCESS_H_ # define PROCESS_H_ +# include + struct process; enum status_kind { diff --git a/src/report.c b/src/report.c index 0a79eb9..e135732 100644 --- a/src/report.c +++ b/src/report.c @@ -27,6 +27,7 @@ #include "criterion/logging.h" #include "criterion/options.h" #include "report.h" +#include "timer.h" #define IMPL_CALL_REPORT_HOOKS(Kind) \ IMPL_SECTION_LIMITS(f_report_hook, crit_ ## Kind); \ @@ -57,11 +58,14 @@ ReportHook(PRE_INIT)(struct criterion_test *test) { ReportHook(POST_TEST)(struct criterion_test_stats *stats) { if (criterion_options.enable_tap_format) { - criterion_important("%s %lu - %s::%s\n", + const char *format = can_measure_time() ? "%s " SIZE_T_FORMAT " - %s::%s (%3.2fs)\n" + : "%s " SIZE_T_FORMAT " - %s::%s\n"; + criterion_important(format, stats->failed ? "not ok" : "ok", tap_test_index++, stats->test->category, - stats->test->name); + stats->test->name, + stats->elapsed_time); for (struct criterion_assert_stats *asrt = stats->asserts; asrt; asrt = asrt->next) { if (!asrt->passed) { char *dup = strdup(*asrt->message ? asrt->message : asrt->condition), *saveptr = NULL; @@ -76,11 +80,13 @@ ReportHook(POST_TEST)(struct criterion_test_stats *stats) { } } } else { + const char *format = can_measure_time() ? "%s::%s: %s (%3.2fs)\n" : "%s::%s: %s\n"; criterion_log(stats->failed ? CRITERION_IMPORTANT : CRITERION_INFO, - "%s::%s: %s\n", + format, stats->test->category, stats->test->name, - stats->failed ? "FAILURE" : "SUCCESS"); + stats->failed ? "FAILURE" : "SUCCESS", + stats->elapsed_time); } } @@ -93,13 +99,13 @@ ReportHook(PRE_ALL)(struct criterion_test_set *set) { for (struct criterion_test **test = set->tests; i < set->nb_tests; ++i) if (!(test[i])->data->disabled) ++enabled_count; - criterion_important("1..%lu\n", enabled_count); + criterion_important("1.." SIZE_T_FORMAT "\n", enabled_count); } } ReportHook(POST_ALL)(struct criterion_global_stats *stats) { if (criterion_options.enable_tap_format) return; - criterion_important("Synthesis: %lu tests were run. %lu passed, %lu failed (with %lu crashes)\n", + criterion_important("Synthesis: " SIZE_T_FORMAT " tests were run. " SIZE_T_FORMAT " passed, " SIZE_T_FORMAT " failed (with " SIZE_T_FORMAT " crashes)\n", stats->nb_tests, stats->tests_passed, stats->tests_failed, @@ -119,7 +125,7 @@ ReportHook(ASSERT)(struct criterion_assert_stats *stats) { ReportHook(TEST_CRASH)(struct criterion_test_stats *stats) { if (criterion_options.enable_tap_format) { - criterion_important("not ok %lu - %s::%s unexpected signal after %s:%u\n", + criterion_important("not ok " SIZE_T_FORMAT " - %s::%s unexpected signal after %s:%u\n", tap_test_index++, stats->test->category, stats->test->name, diff --git a/src/runner.c b/src/runner.c index cc749ba..0572c6c 100644 --- a/src/runner.c +++ b/src/runner.c @@ -24,15 +24,14 @@ #include #include #include -#include #include -#include "criterion/assert.h" #include "criterion/options.h" #include "stats.h" #include "runner.h" #include "report.h" #include "event.h" #include "process.h" +#include "timer.h" IMPL_SECTION_LIMITS(struct criterion_test, criterion_tests); @@ -89,8 +88,15 @@ static void run_test_child(struct criterion_test *test) { send_event(PRE_INIT, NULL, 0); (test->data->init ?: nothing)(); send_event(PRE_TEST, NULL, 0); + + struct timespec_compat ts; + timer_start(&ts); (test->test ?: nothing)(); - send_event(POST_TEST, NULL, 0); + double elapsed_time; + if (!timer_end(&elapsed_time, &ts)) + elapsed_time = -1; + + send_event(POST_TEST, &elapsed_time, sizeof (double)); (test->data->fini ?: nothing)(); send_event(POST_FINI, NULL, 0); } @@ -126,11 +132,12 @@ static void run_test(struct criterion_global_stats *stats, struct criterion_test stat_push_event(stats, test_stats, &ev); report(TEST_CRASH, test_stats); } else { - struct event ev = { .kind = POST_TEST }; + double elapsed_time = 0; + struct event ev = { .kind = POST_TEST, .data = &elapsed_time }; stat_push_event(stats, test_stats, &ev); report(POST_TEST, test_stats); - ev.kind = POST_FINI; + ev = (struct event) { .kind = POST_FINI, .data = NULL }; stat_push_event(stats, test_stats, &ev); report(POST_FINI, test_stats); } @@ -152,7 +159,7 @@ static int criterion_run_all_tests_impl(void) { return -1; report(POST_ALL, stats); - return stats->tests_failed > 0; + return stats->tests_failed == 0; } int criterion_run_all_tests(void) { @@ -160,5 +167,5 @@ int criterion_run_all_tests(void) { if (res == -1) // if this is the test worker terminating _exit(0); - return !criterion_options.always_succeed && res; + return criterion_options.always_succeed || res; } diff --git a/src/stats.c b/src/stats.c index e12db58..4c33793 100644 --- a/src/stats.c +++ b/src/stats.c @@ -114,7 +114,8 @@ static void push_assert(s_glob_stats *stats, static void push_post_test(s_glob_stats *stats, s_test_stats *test, - UNUSED void *ptr) { + double *ptr) { + test->elapsed_time = *ptr; if (test->failed_asserts > 0 || test->signal != test->test->data->signal) { test->failed = 1; ++stats->tests_failed; diff --git a/src/timer.c b/src/timer.c new file mode 100644 index 0000000..edb181e --- /dev/null +++ b/src/timer.c @@ -0,0 +1,83 @@ +#include +#include +#include "timer.h" + +#define GIGA 1000000000 + +#if defined(__unix__) && !defined(__CYGWIN__) + +# ifdef CLOCK_MONOTONIC_RAW +# define CLOCK CLOCK_MONOTONIC_RAW +# else +# define CLOCK CLOCK_MONOTONIC +# endif + +extern __attribute__ ((weak)) int clock_gettime(clockid_t, struct timespec *); + +#elif defined(__APPLE__) +# include +# include +#elif defined(_WIN32) || defined(__CYGWIN__) +# define VC_EXTRALEAN +# define WIN32_LEAN_AND_MEAN +# include +#endif + +bool can_measure_time(void) { +#if defined(__unix__) && !defined(__CYGWIN__) + return clock_gettime != NULL; +#else + return true; +#endif +} + +int gettime_compat(struct timespec_compat *ts) { +#if defined(__APPLE__) + clock_serv_t cclock; + mach_timespec_t mts; + + host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); + int res = clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + + *ts = (struct timespec_compat) { mts.tv_sec, mts.tv_nsec }; + return res > 0 ? -1 : 0; +#elif defined(_WIN32) || defined(__CYGWIN__) + LARGE_INTEGER freq, count; + if (!QueryPerformanceFrequency(&freq) + || !QueryPerformanceCounter(&count)) + return -1; + + int64_t sec = count.QuadPart / freq.QuadPart; + int64_t nano = (int64_t) ((double) count.QuadPart * GIGA / (double) freq.QuadPart) % GIGA; + + *ts = (struct timespec_compat) { sec, nano }; + return 0; +#elif defined(__unix__) + if (!can_measure_time()) { + errno = ENOTSUP; + return -1; + } + + struct timespec ts_; + int res = clock_gettime(CLOCK, &ts_); + + *ts = (struct timespec_compat) { ts_.tv_sec, ts_.tv_nsec }; + return res; +#else + return -1; +#endif +} + +int timer_start(struct timespec_compat *state) { + return gettime_compat(state) == -1 ? 0 : 1; +} + +int timer_end(double *time, struct timespec_compat *state) { + struct timespec_compat last; + if (gettime_compat(&last) == -1) + return 0; + + *time = (last.tv_sec - state->tv_sec) + (last.tv_nsec - state->tv_nsec) / (double) GIGA; + return 1; +} diff --git a/src/timer.h b/src/timer.h new file mode 100644 index 0000000..2d334cb --- /dev/null +++ b/src/timer.h @@ -0,0 +1,17 @@ +#ifndef TIMER_H_ +# define TIMER_H_ + +# include +# include +# include + +struct timespec_compat { + int64_t tv_sec; + int64_t tv_nsec; +}; + +bool can_measure_time(void); +int timer_start(struct timespec_compat *state); +int timer_end(double *time, struct timespec_compat *state); + +#endif /* !TIMER_H_ */