diff --git a/ChangeLog b/ChangeLog index 42b12c2..4548437 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,12 @@ +2015-03-11 Franklin "Snaipe" Mathieu + + * include/criterion/logging.h, src/logging.c: A logging interface + * README.md, doc/*: Various readme and documentation changes + * src/main.c, include/criterion/options.h, src/options.c: Default CLI options & environment variables + * *: Mac OS X compatibility + * include/criterion/assert.h: Comparison assertions, floating-point equality assertions + 2015-02-06 Franklin "Snaipe" Mathieu * src/: Added criterion internals. - * include/: Added Test, ReportHook, assert and expect macros. + * include/: Added Test, ReportHook, assert and expect macros. diff --git a/Makefile.am b/Makefile.am index 00a090d..f373ec2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,6 +28,8 @@ subdirinclude_HEADERS = \ include/criterion/criterion.h \ include/criterion/event.h \ include/criterion/hooks.h \ + include/criterion/logging.h \ + include/criterion/options.h \ include/criterion/stats.h libcriterion_la_SOURCES = \ @@ -41,4 +43,6 @@ libcriterion_la_SOURCES = \ src/process.h \ src/stats.c \ src/stats.h \ + src/logging.c \ + src/options.c \ src/main.c diff --git a/README.md b/README.md index d04edfb..95a1845 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,12 @@ Criterion [![Build Status](https://travis-ci.org/Snaipe/Criterion.svg?branch=master)](https://travis-ci.org/Snaipe/Criterion) [![Coverage Status](https://coveralls.io/repos/Snaipe/Criterion/badge.svg?branch=master)](https://coveralls.io/r/Snaipe/Criterion?branch=master) [![License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](https://github.com/Snaipe/Criterion/blob/master/LICENSE) -[![Version](https://img.shields.io/github/tag/Snaipe/Criterion.svg?label=version&style=flat)](https://github.com/Snaipe/Criterion/releases) +[![Version](https://img.shields.io/github/tag/Snaipe/Criterion.svg?label=version&style=flat)](https://github.com/Snaipe/Criterion/releases) A dead-simple, yet extensible, C test framework. +![Screencast](./doc/screencast.gif) + ## Philosophy Most test frameworks for C require a lot of boilerplate code to @@ -21,16 +23,18 @@ This gives the user great control, at the unfortunate cost of simplicity. Criterion follows the KISS principle, while keeping the control the user would have with other frameworks: -* Tests are automatically registered when declared. -* A default entry point is provided, no need to declare a main +* [x] Tests are automatically registered when declared. +* [x] A default entry point is provided, no need to declare a main unless you want to do special handling. -* Test are isolated in their own process, crashes and signals can be +* [x] Test are isolated in their own process, crashes and signals can be reported and tested. -* Progress and statistics can be followed in real time with report hooks. +* [x] Progress and statistics can be followed in real time with report hooks. +* [x] TAP output format can be enabled with an option. +* [x] Runs on Linux and OS X. ## Documentation -An online documentation is available on [ReadTheDocs][online-docs] +An online documentation is available on [ReadTheDocs][online-docs] ([PDF][pdf-docs] | [Zip][zip-docs] | [Epub][epub-docs]) ## Samples @@ -39,6 +43,7 @@ Sample tests can be found in the [sample directory][samples]. * [A simple test][sample-simple] * [Using multiple suites][sample-suites] +* [Adding test fixtures][sample-fixtures] * [Tests with signals][sample-signal] * [Using report hooks][sample-report] @@ -67,8 +72,9 @@ A. Windows support with MinGW/MSVC is coming, but MSVC is a bit of a lost cause [zip-docs]: http://readthedocs.org/projects/criterion/downloads/htmlzip/latest/ [epub-docs]: http://readthedocs.org/projects/criterion/downloads/epub/latest/ -[samples]: https://github.com/Snaipe/Criterion/tree/master/samples -[sample-simple]: https://github.com/Snaipe/Criterion/blob/master/samples/simple.c -[sample-suites]: https://github.com/Snaipe/Criterion/blob/master/samples/suites.c -[sample-signal]: https://github.com/Snaipe/Criterion/blob/master/samples/signal.c -[sample-report]: https://github.com/Snaipe/Criterion/blob/master/samples/report.c +[samples]: ./samples/ +[sample-simple]: ./samples/simple.c +[sample-suites]: ./samples/suites.c +[sample-fixtures]: ./samples/fixtures.c +[sample-signal]: ./samples/signal.c +[sample-report]: ./samples/report.c diff --git a/dependencies/csptr b/dependencies/csptr index 333c650..30b3380 160000 --- a/dependencies/csptr +++ b/dependencies/csptr @@ -1 +1 @@ -Subproject commit 333c650c515204d02b40aed4c3c531b6c56159bf +Subproject commit 30b3380a9aee3c49a2206d2cced12a6adf26469b diff --git a/doc/hooks.rst b/doc/hooks.rst index 1450305..dc476b8 100644 --- a/doc/hooks.rst +++ b/doc/hooks.rst @@ -22,14 +22,14 @@ Testing Phases The flow of the test process goes as follows: -1. ``PRE_EVERYTHING``: occurs before running the tests. +1. ``PRE_ALL``: occurs before running the tests. #. ``PRE_INIT``: occurs before a test is initialized. #. ``PRE_TEST``: occurs after the test initialization, but before the test is run. #. ``ASSERT``: occurs when an assertion is hit #. ``TEST_CRASH``: occurs when a test crashes unexpectedly. #. ``POST_TEST``: occurs after a test ends, but before the test finalization. #. ``POST_FINI``: occurs after a test finalization. -#. ``POST_EVERYTHING``: occurs after all the tests are done. +#. ``POST_ALL``: occurs after all the tests are done. Hook Parameters --------------- @@ -43,6 +43,6 @@ Valid types for each phases are: * ``struct criterion_test *`` for ``PRE_INIT`` and ``PRE_TEST``. * ``struct criterion_test_stats *`` for ``POST_TEST``, ``POST_FINI``, and ``TEST_CRASH``. * ``struct criterion_assert_stats *`` for ``ASSERT``. -* ``struct criterion_global_stats *`` for ``POST_EVERYTHING``. +* ``struct criterion_global_stats *`` for ``POST_ALL``. -``PRE_EVERYTHING`` does not take any parameter. +``PRE_ALL`` does not take any parameter. diff --git a/doc/screencast.gif b/doc/screencast.gif new file mode 100644 index 0000000..4b3e7eb Binary files /dev/null and b/doc/screencast.gif differ diff --git a/doc/starter.rst b/doc/starter.rst index 7b17da6..54e3819 100644 --- a/doc/starter.rst +++ b/doc/starter.rst @@ -50,15 +50,26 @@ parameter, and an optional failure message: On top of those, more assertions are available for common operations: -* ``{assert,expect}Equal(Actual, Expected, [Message])`` -* ``{assert,expect}NotEqual(Actual, Unexpected, [Message])`` -* ``{assert,expect}StringsEqual(Actual, Expected, [Message])`` -* ``{assert,expect}StringsNotEqual(Actual, Unexpected, [Message])`` -* ``{assert,expect}ArraysEqual(Actual, Expected, Size, [Message])`` -* ``{assert,expect}ArraysNotEqual(Actual, Unexpected, Size, [Message])`` +* ``{assert,expect}_not(Actual, Expected, [Message])`` +* ``{assert,expect}_equal(Actual, Expected, [Message])`` +* ``{assert,expect}_not_equal(Actual, Unexpected, [Message])`` +* ``{assert,expect}_lt(Actual, Expected, [Message])`` +* ``{assert,expect}_leq(Actual, Expected, [Message])`` +* ``{assert,expect}_gt(Actual, Expected, [Message])`` +* ``{assert,expect}_geq(Actual, Expected, [Message])`` +* ``{assert,expect}_float_equal(Actual, Expected, Epsilon, [Message])`` +* ``{assert,expect}_float_not_equal(Actual, Unexpected, Epsilon, [Message])`` +* ``{assert,expect}_strings_equal(Actual, Expected, [Message])`` +* ``{assert,expect}_strings_not_equal(Actual, Unexpected, [Message])`` +* ``{assert,expect}_strings_lt(Actual, Expected, [Message])`` +* ``{assert,expect}_strings_leq(Actual, Expected, [Message])`` +* ``{assert,expect}_strings_gt(Actual, Expected, [Message])`` +* ``{assert,expect}_strings_geq(Actual, Expected, [Message])`` +* ``{assert,expect}_arrays_equal(Actual, Expected, Size, [Message])`` +* ``{assert,expect}_arrays_not_equal(Actual, Unexpected, Size, [Message])`` -Initialization and finalization -------------------------------- +Fixtures +-------- Tests that need some setup and teardown can register functions that will run before and after the test function: diff --git a/include/criterion/assert.h b/include/criterion/assert.h index e832ca3..a5a6bed 100644 --- a/include/criterion/assert.h +++ b/include/criterion/assert.h @@ -37,7 +37,7 @@ enum criterion_assert_kind { FATAL }; -# define assertImpl(Kind, Condition, ...) \ +# define assert_impl(Kind, Condition, ...) \ do { \ int passed = !!(Condition); \ struct criterion_assert_stats stat = { \ @@ -53,40 +53,110 @@ enum criterion_assert_kind { return; \ } while (0) -# define assert(Condition, ...) assertImpl(FATAL, (Condition), ## __VA_ARGS__) -# define expect(Condition, ...) assertImpl(NORMAL, (Condition), ## __VA_ARGS__) +// Common asserts -# define assertArraysEqual(A, B, Size, ...) \ - assert(!memcmp((A), (B), (Size)), ## __VA_ARGS__) -# define expectArraysEqual(A, B, Size, ...) \ - expect(!memcmp((A), (B), (Size)), ## __VA_ARGS__) +# define assert(Condition, ...) assert_impl(FATAL, Condition, "" __VA_ARGS__) +# define expect(Condition, ...) assert_impl(NORMAL, Condition, "" __VA_ARGS__) -# define assertEqual(Actual, Expected, ...) \ - assert((Actual) == (Expected), ## __VA_ARGS__) -# define expectEqual(Actual, Expected, ...) \ - expect((Actual) == (Expected), ## __VA_ARGS__) +# define assert_not(Condition, ...) assert(!(Condition), "" __VA_ARGS__) +# define expect_not(Condition, ...) expect(!(Condition), "" __VA_ARGS__) -# define assertStringsEqual(Actual, Expected, ...) \ - assert(!strcmp((Actual), (Expected)), ## __VA_ARGS__) -# define expectStringsEqual(Actual, Expected, ...) \ - expect(!strcmp((Actual), (Expected)), ## __VA_ARGS__) +// Native asserts -# define assertNot(Condition, ...) assert(!(Condition), ## __VA_ARGS__) -# define expectNot(Condition, ...) expect(!(Condition), ## __VA_ARGS__) +# define assert_op(Actual, Expected, Op, ...) \ + assert((Actual) Op (Expected), "" __VA_ARGS__) +# define expect_op(Actual, Expected, Op, ...) \ + expect((Actual) Op (Expected), "" __VA_ARGS__) -# define assertNotEqual(Actual, Expected, ...) \ - assert((Actual) != (Expected), ## __VA_ARGS__) +# define assert_equal(Actual, Expected, ...) \ + assert_op(Actual, Expected, ==, "" __VA_ARGS__) +# define expect_equal(Actual, Expected, ...) \ + expect_op(Actual, Expected, ==, "" __VA_ARGS__) + +# define assert_not_equal(Actual, Expected, ...) \ + assert_op(Actual, Expected, !=, "" __VA_ARGS__) # define expectNotEqual(Actual, Expected, ...) \ - expect((Actual) != (Expected), ## __VA_ARGS__) + expect_op(Actual, Expected, !=, "" __VA_ARGS__) -# define assertArraysNotEqual(A, B, Size, ...) \ - assert(memcmp((A), (B), (Size)), ## __VA_ARGS__) -# define expectArraysNotEqual(A, B, Size, ...) \ - expect(memcmp((A), (B), (Size)), ## __VA_ARGS__) +# define assert_lt(Actual, Expected, ...) \ + assert_op(Actual, Expected, <, "" __VA_ARGS__) +# define expect_lt(Actual, Expected, ...) \ + expect_op(Actual, Expected, <, "" __VA_ARGS__) -# define assertStringsNotEqual(Actual, Expected, ...) \ - assert(strcmp((Actual), (Expected)), ## __VA_ARGS__) -# define expectStringsNotEqual(Actual, Expected, ...) \ - expect(strcmp((Actual), (Expected)), ## __VA_ARGS__) +# define assert_gt(Actual, Expected, ...) \ + assert_op(Actual, Expected, >, "" __VA_ARGS__) +# define expect_gt(Actual, Expected, ...) \ + expect_op(Actual, Expected, >, "" __VA_ARGS__) + +# define assert_leq(Actual, Expected, ...) \ + assert_op(Actual, Expected, <=, "" __VA_ARGS__) +# define expect_leq(Actual, Expected, ...) \ + expect_op(Actual, Expected, <=, "" __VA_ARGS__) + +# define assert_geq(Actual, Expected, ...) \ + assert_op(Actual, Expected, >=, "" __VA_ARGS__) +# define expect_geq(Actual, Expected, ...) \ + expect_op(Actual, Expected, >=, "" __VA_ARGS__) + +// Floating-point asserts + +# define assert_float_equal(Actual, Expected, Epsilon, ...) \ + assert((Expected) - (Actual) <= (Epsilon) && (Actual) - (Expected) <= (Epsilon), "" __VA_ARGS__) +# define expect_float_equal(Actual, Expected, Epsilon, ...) \ + expect((Expected) - (Actual) <= (Epsilon) && (Actual) - (Expected) <= (Epsilon), "" __VA_ARGS__) + +# define assert_float_not_equal(Actual, Expected, Epsilon, ...) \ + assert((Expected) - (Actual) > (Epsilon) || (Actual) - (Expected) > (Epsilon), "" __VA_ARGS__) +# define expect_float_not_equal(Actual, Expected, Epsilon, ...) \ + expect((Expected) - (Actual) > (Epsilon) || (Actual) - (Expected) > (Epsilon), "" __VA_ARGS__) + +// String asserts + +# define assert_strings(Actual, Expected, Op, ...) \ + assert(strcmp((Actual), (Expected)) Op 0, "" __VA_ARGS__) +# define expect_strings(Actual, Expected, Op, ...) \ + expect(strcmp((Actual), (Expected)) Op 0, "" __VA_ARGS__) + +# define assert_strings_equal(Actual, Expected, ...) \ + assert_strings(Actual, Expected, ==, "" __VA_ARGS__) +# define expect_strings_equal(Actual, Expected, ...) \ + expect_strings(Actual, Expected, ==, "" __VA_ARGS__) + +# define assert_strings_gt(Actual, Expected, ...) \ + assert_strings(Actual, Expected, >, "" __VA_ARGS__) +# define expect_strings_gt(Actual, Expected, ...) \ + expect_strings(Actual, Expected, >, "" __VA_ARGS__) + +# define assert_strings_lt(Actual, Expected, ...) \ + assert_strings(Actual, Expected, <, "" __VA_ARGS__) +# define expect_strings_lt(Actual, Expected, ...) \ + expect_strings(Actual, Expected, <, "" __VA_ARGS__) + +# define assert_strings_geq(Actual, Expected, ...) \ + assert_strings(Actual, Expected, >=, "" __VA_ARGS__) +# define expect_strings_geq(Actual, Expected, ...) \ + expect_strings(Actual, Expected, >=, "" __VA_ARGS__) + +# define assert_strings_leq(Actual, Expected, ...) \ + assert_strings(Actual, Expected, <=, "" __VA_ARGS__) +# define expect_strings_leq(Actual, Expected, ...) \ + expect_strings(Actual, Expected, <=, "" __VA_ARGS__) + +# define assert_strings_not_equal(Actual, Expected, ...) \ + assert_strings(Actual, Expected, !=, "" __VA_ARGS__) +# define expect_strings_not_equal(Actual, Expected, ...) \ + expect_strings(Actual, Expected, !=, "" __VA_ARGS__) + +// Array asserts + +# define assert_arrays_equal(A, B, Size, ...) \ + assert(!memcmp((A), (B), (Size)), "" __VA_ARGS__) +# define expect_arrays_equal(A, B, Size, ...) \ + expect(!memcmp((A), (B), (Size)), "" __VA_ARGS__) + +# define assert_arrays_not_equal(A, B, Size, ...) \ + assert(memcmp((A), (B), (Size)), "" __VA_ARGS__) +# define expect_arrays_not_equal(A, B, Size, ...) \ + expect(memcmp((A), (B), (Size)), "" __VA_ARGS__) #endif /* !CRITERION_ASSERT_H_ */ diff --git a/include/criterion/common.h b/include/criterion/common.h index a2ae32a..32d6420 100644 --- a/include/criterion/common.h +++ b/include/criterion/common.h @@ -24,7 +24,43 @@ #ifndef CRITERION_COMMON_H_ # define CRITERION_COMMON_H_ -# define SECTION_(Name) __attribute__((section(Name))) +# ifdef __APPLE__ +# define SECTION_START_PREFIX __first +# define SECTION_END_PREFIX __last +# define SECTION_START_SUFFIX(Name) __asm("section$start$__DATA$" Name) +# define SECTION_END_SUFFIX(Name) __asm("section$end$__DATA$" Name) +# define SECTION_(Name) __attribute__((section("__DATA," Name))) +# else +# define SECTION_START_PREFIX __start +# define SECTION_END_PREFIX __stop +# define SECTION_START_SUFFIX(Name) +# define SECTION_END_SUFFIX(Name) +# define SECTION_(Name) __attribute__((section(Name))) +# endif + +# define MAKE_IDENTIFIER_(Prefix, Id) MAKE_IDENTIFIER__(Prefix, Id) +# define MAKE_IDENTIFIER__(Prefix, Id) Prefix ## _ ## Id + +# define SECTION_START_(Name) MAKE_IDENTIFIER_(SECTION_START_PREFIX, Name) +# define SECTION_END_(Name) MAKE_IDENTIFIER_(SECTION_END_PREFIX, Name) + +# define SECTION_START(Name) g_ ## Name ## _section_start +# define SECTION_END(Name) g_ ## Name ## _section_end + +# define DECL_SECTION_LIMITS(Type, Name) \ + extern Type SECTION_START_(Name) SECTION_START_SUFFIX(#Name); \ + extern Type SECTION_END_(Name) SECTION_END_SUFFIX(#Name) + +# define IMPL_SECTION_LIMITS(Type, Name) \ + Type *const SECTION_START(Name) = &SECTION_START_(Name); \ + Type *const SECTION_END(Name) = &SECTION_END_(Name) + # define UNUSED __attribute__((unused)) +# ifdef __GNUC__ +# define FORMAT(Archetype, Index, Ftc) __attribute__((format(Archetype, Index, Ftc))) +# else +# define FORMAT(Archetype, Index, Ftc) +# endif + #endif /* !CRITERION_COMMON_H_ */ diff --git a/include/criterion/criterion.h b/include/criterion/criterion.h index 0565e69..6c59180 100644 --- a/include/criterion/criterion.h +++ b/include/criterion/criterion.h @@ -30,18 +30,25 @@ # include "assert.h" struct criterion_test_extra_data { - const char *file_; - unsigned line_; + const char *const file_; + const unsigned line_; void (*init)(void); void (*fini)(void); int signal; + bool disabled; + void *data; }; struct criterion_test { const char *name; const char *category; void (*test)(void); - const struct criterion_test_extra_data *data; + struct criterion_test_extra_data *const data; +}; + +struct criterion_test_set { + struct criterion_test **tests; + size_t nb_tests; }; # define IDENTIFIER_(Category, Name, Suffix) \ @@ -49,12 +56,12 @@ struct criterion_test { # define TEST_PROTOTYPE_(Category, Name) \ void IDENTIFIER_(Category, Name, impl)(void) -# define Test(Category, Name, Args...) \ +# define Test(Category, Name, ...) \ TEST_PROTOTYPE_(Category, Name); \ struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = { \ .file_ = __FILE__, \ .line_ = __LINE__, \ - Args \ + __VA_ARGS__ \ }; \ SECTION_("criterion_tests") \ const struct criterion_test IDENTIFIER_(Category, Name, meta) = { \ diff --git a/include/criterion/hooks.h b/include/criterion/hooks.h index 129e7c1..b47b330 100644 --- a/include/criterion/hooks.h +++ b/include/criterion/hooks.h @@ -24,17 +24,17 @@ #ifndef CRITERION_HOOKS_H_ # define CRITERION_HOOKS_H_ -#include "common.h" +# include "common.h" typedef enum { - PRE_EVERYTHING, + PRE_ALL, PRE_INIT, PRE_TEST, ASSERT, TEST_CRASH, POST_TEST, POST_FINI, - POST_EVERYTHING, + POST_ALL, } e_report_status; typedef void (*f_report_hook)(); @@ -48,7 +48,7 @@ typedef void (*f_report_hook)(); # define ReportHook(Kind) \ HOOK_PROTOTYPE_(); \ - SECTION_("criterion_hooks_" #Kind) \ + SECTION_("crit_" #Kind) \ const f_report_hook HOOK_IDENTIFIER_(func) = HOOK_IDENTIFIER_(impl); \ HOOK_PROTOTYPE_ diff --git a/include/criterion/logging.h b/include/criterion/logging.h new file mode 100644 index 0000000..c810933 --- /dev/null +++ b/include/criterion/logging.h @@ -0,0 +1,41 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef CRITERION_LOGGING_H_ +# define CRITERION_LOGGING_H_ + +# include +# include "common.h" + +enum criterion_logging_level { + CRITERION_INFO = 1, + CRITERION_IMPORTANT, +}; + +FORMAT(printf, 2, 3) +void criterion_log(enum criterion_logging_level level, const char *msg, ...); + +# define criterion_info(...) criterion_log(CRITERION_INFO, __VA_ARGS__) +# define criterion_important(...) criterion_log(CRITERION_IMPORTANT, __VA_ARGS__) + +#endif /* !CRITERION_LOGGING_H_ */ diff --git a/include/criterion/options.h b/include/criterion/options.h new file mode 100644 index 0000000..2d24fb5 --- /dev/null +++ b/include/criterion/options.h @@ -0,0 +1,38 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef CRITERION_OPTIONS_H_ +# define CRITERION_OPTIONS_H_ + +# include "logging.h" + +struct criterion_options { + enum criterion_logging_level logging_threshold; + bool enable_tap_format; + bool no_early_exit; + bool always_succeed; +}; + +extern struct criterion_options criterion_options; + +#endif /*!CRITERION_OPTIONS_H_ */ diff --git a/samples/Makefile.am b/samples/Makefile.am index 84ffec3..3fb962a 100644 --- a/samples/Makefile.am +++ b/samples/Makefile.am @@ -2,13 +2,12 @@ TESTS = \ signal \ report \ suites \ + fixtures \ + asserts \ simple TESTS_ENVIRONMENT = CRITERION_ALWAYS_SUCCEED=1 check_PROGRAMS = $(TESTS) -CFLAGS = -I$(top_srcdir)/include/ +CFLAGS = -I$(top_srcdir)/include/ -std=c99 LDADD = -L$(top_srcdir)/ -lcriterion - -signal_SOURCES = signal.c -simple_SOURCES = simple.c diff --git a/samples/asserts.c b/samples/asserts.c new file mode 100644 index 0000000..deec0b4 --- /dev/null +++ b/samples/asserts.c @@ -0,0 +1,43 @@ +#include + +Test(asserts, base) { + assert(true); + expect(true); + + assert(true, "Assertions may take failure messages"); + + expect(false, "assert is fatal, expect isn't"); + assert(false, "This assert runs"); + assert(false, "This does not"); +} + +Test(asserts, string) { + assert_strings_equal("hello", "hello"); + assert_strings_not_equal("hello", "olleh"); + + assert_strings_gt("hello", "hell"); + assert_strings_geq("hello", "hell"); + assert_strings_geq("hello", "hello"); + + assert_strings_lt("hell", "hello"); + assert_strings_leq("hell", "hello"); + assert_strings_leq("hello", "hello"); +} + +Test(asserts, native) { + assert_equal(1, 1); + assert_not_equal(1, 2); + + assert_lt(1, 2); + assert_leq(1, 2); + assert_leq(2, 2); + + assert_gt(2, 1); + assert_geq(2, 1); + assert_geq(2, 2); +} + +Test(asserts, float) { + assert_not_equal(0.1 * 0.1, 0.01); + assert_float_equal(0.1 * 0.1, 0.01, 0.001); +} diff --git a/samples/fixtures.c b/samples/fixtures.c new file mode 100644 index 0000000..e077d8e --- /dev/null +++ b/samples/fixtures.c @@ -0,0 +1,14 @@ +#include +#include + +void setup(void) { + puts("Runs before the test"); +} + +void teardown(void) { + puts("Runs after the test"); +} + +Test(simple, fixtures, .init = setup, .fini = teardown) { + assert(1); +} diff --git a/samples/report.c b/samples/report.c index e806322..5c92b90 100644 --- a/samples/report.c +++ b/samples/report.c @@ -15,10 +15,10 @@ ReportHook(POST_TEST)(struct criterion_test_stats *stats) { stats->passed_asserts, stats->failed_asserts, stats->passed_asserts + stats->failed_asserts); } -ReportHook(PRE_EVERYTHING)() { +ReportHook(PRE_ALL)() { puts("criterion_init"); } -ReportHook(POST_EVERYTHING)() { +ReportHook(POST_ALL)() { puts("criterion_fini"); } diff --git a/src/logging.c b/src/logging.c new file mode 100644 index 0000000..f26d1e2 --- /dev/null +++ b/src/logging.c @@ -0,0 +1,38 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include "criterion/logging.h" +#include "criterion/options.h" + +void criterion_log(enum criterion_logging_level level, const char *msg, ...) { + va_list args; + + if (level < criterion_options.logging_threshold) + return; + + va_start(args, msg); + vfprintf(stderr, msg, args); + va_end(args); +} diff --git a/src/main.c b/src/main.c index bcaac01..3c5f26e 100644 --- a/src/main.c +++ b/src/main.c @@ -1,5 +1,48 @@ +#define _GNU_SOURCE #include +#include +#include +#include +#include + +# define USAGE \ + "usage: %s OPTIONS\n" \ + "options: \n" \ + " -h or --help: prints this message\n" \ + " --verbose [level]: sets verbosity to level\n" + +int print_usage(char *progname) { + fprintf(stderr, USAGE, progname); + return 0; +} + +int main(int argc, char *argv[]) { + static struct option opts[] = { + {"verbose", optional_argument, 0, 'v'}, + {"tap", no_argument, 0, 't'}, + {"help", no_argument, 0, 'h'}, + {"always-succeed", no_argument, 0, 'y'}, + {"no-early-exit", no_argument, 0, 'z'}, + {0, 0, 0, 0 } + }; + + criterion_options = (struct criterion_options) { + .always_succeed = !strcmp("1", getenv("CRITERION_ALWAYS_SUCCEED") ?: "0"), + .no_early_exit = !strcmp("1", getenv("CRITERION_NO_EARLY_EXIT") ?: "0"), + .enable_tap_format = !strcmp("1", getenv("CRITERION_ENABLE_TAP") ?: "0"), + .logging_threshold = atoi(getenv("CRITERION_VERBOSITY_LEVEL") ?: "2"), + }; + + for (int c; (c = getopt_long(argc, argv, "h", opts, NULL)) != -1;) { + switch (c) { + case 'v': criterion_options.logging_threshold = atoi(optarg ?: "1"); break; + case 't': criterion_options.enable_tap_format = true; break; + case 'y': criterion_options.always_succeed = true; break; + case 'z': criterion_options.no_early_exit = true; break; + case 'h': + default : return print_usage(argv[0]); + } + } -int main(void) { return criterion_run_all_tests(); } diff --git a/src/options.c b/src/options.c new file mode 100644 index 0000000..9b47c44 --- /dev/null +++ b/src/options.c @@ -0,0 +1,26 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +# include "criterion/options.h" + +struct criterion_options criterion_options = { .logging_threshold = CRITERION_IMPORTANT }; diff --git a/src/process.c b/src/process.c index 3cbab45..ffc971e 100644 --- a/src/process.c +++ b/src/process.c @@ -1,3 +1,26 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #include #include #include @@ -6,6 +29,7 @@ #include #include "criterion/criterion.h" +#include "criterion/options.h" #include "process.h" #include "event.h" @@ -47,7 +71,7 @@ struct process *spawn_test_worker(struct criterion_test *test, void (*func)(stru func(test); close(fds[1]); - if (!strcmp("1", getenv("CRITERION_NO_EARLY_EXIT") ?: "0")) + if (!criterion_options.no_early_exit) return NULL; else _exit(0); diff --git a/src/process.h b/src/process.h index 9eb3358..fcf9e62 100644 --- a/src/process.h +++ b/src/process.h @@ -1,3 +1,26 @@ +/* + * The MIT License (MIT) + * + * Copyright © 2015 Franklin "Snaipe" Mathieu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ #ifndef PROCESS_H_ # define PROCESS_H_ diff --git a/src/report.c b/src/report.c index 06e1854..0a79eb9 100644 --- a/src/report.c +++ b/src/report.c @@ -24,49 +24,93 @@ #include #include "criterion/criterion.h" #include "criterion/stats.h" +#include "criterion/logging.h" +#include "criterion/options.h" #include "report.h" #define IMPL_CALL_REPORT_HOOKS(Kind) \ - static f_report_hook * const g_##Kind##_section_start = \ - &__start_criterion_hooks_##Kind; \ - static f_report_hook * const g_##Kind##_section_end = \ - &__stop_criterion_hooks_##Kind; \ + IMPL_SECTION_LIMITS(f_report_hook, crit_ ## Kind); \ void call_report_hooks_##Kind(void *data) { \ - for (f_report_hook *hook = g_##Kind##_section_start;\ - hook < g_##Kind##_section_end; \ + for (f_report_hook *hook = SECTION_START(crit_ ## Kind); \ + hook < SECTION_END(crit_ ## Kind); \ ++hook) { \ (*hook)(data); \ } \ } -IMPL_CALL_REPORT_HOOKS(PRE_EVERYTHING); +static size_t tap_test_index = 1; + +IMPL_CALL_REPORT_HOOKS(PRE_ALL); IMPL_CALL_REPORT_HOOKS(PRE_INIT); IMPL_CALL_REPORT_HOOKS(PRE_TEST); IMPL_CALL_REPORT_HOOKS(ASSERT); IMPL_CALL_REPORT_HOOKS(TEST_CRASH); IMPL_CALL_REPORT_HOOKS(POST_TEST); IMPL_CALL_REPORT_HOOKS(POST_FINI); -IMPL_CALL_REPORT_HOOKS(POST_EVERYTHING); +IMPL_CALL_REPORT_HOOKS(POST_ALL); ReportHook(PRE_INIT)(struct criterion_test *test) { - fprintf(stderr, "%s::%s: RUNNING\n", test->category, test->name); + if (criterion_options.enable_tap_format) return; + + criterion_info("%s::%s: RUNNING\n", test->category, test->name); } ReportHook(POST_TEST)(struct criterion_test_stats *stats) { - fprintf(stderr, "%s::%s: %s\n", stats->test->category, stats->test->name, stats->failed ? "FAILURE" : "SUCCESS"); + if (criterion_options.enable_tap_format) { + criterion_important("%s %lu - %s::%s\n", + stats->failed ? "not ok" : "ok", + tap_test_index++, + stats->test->category, + stats->test->name); + for (struct criterion_assert_stats *asrt = stats->asserts; asrt; asrt = asrt->next) { + if (!asrt->passed) { + char *dup = strdup(*asrt->message ? asrt->message : asrt->condition), *saveptr = NULL; + char *line = strtok_r(dup, "\n", &saveptr); + criterion_important("\t%s:%u: Assertion failed: %s\n", + asrt->file, + asrt->line, + line); + while ((line = strtok_r(NULL, "\n", &saveptr))) + criterion_important("\t%s\n", line); + free(dup); + } + } + } else { + criterion_log(stats->failed ? CRITERION_IMPORTANT : CRITERION_INFO, + "%s::%s: %s\n", + stats->test->category, + stats->test->name, + stats->failed ? "FAILURE" : "SUCCESS"); + } } ReportHook(PRE_TEST)() {} ReportHook(POST_FINI)() {} -ReportHook(PRE_EVERYTHING)() {} -ReportHook(POST_EVERYTHING)(struct criterion_global_stats *stats) { - fprintf(stderr, "Synthesis: %lu tests were run. %lu passed, %lu failed (with %lu crashes)\n", stats->nb_tests, stats->tests_passed, stats->tests_failed, stats->tests_crashed); +ReportHook(PRE_ALL)(struct criterion_test_set *set) { + if (criterion_options.enable_tap_format) { + size_t enabled_count = 0, i = 0; + for (struct criterion_test **test = set->tests; i < set->nb_tests; ++i) + if (!(test[i])->data->disabled) + ++enabled_count; + criterion_important("1..%lu\n", enabled_count); + } +} +ReportHook(POST_ALL)(struct criterion_global_stats *stats) { + if (criterion_options.enable_tap_format) return; + + criterion_important("Synthesis: %lu tests were run. %lu passed, %lu failed (with %lu crashes)\n", + stats->nb_tests, + stats->tests_passed, + stats->tests_failed, + stats->tests_crashed); } ReportHook(ASSERT)(struct criterion_assert_stats *stats) { + if (criterion_options.enable_tap_format) return; + if (!stats->passed) { - fprintf(stderr, "\t%s:%d: Assertion failed: %s\n", + criterion_important("%s:%d: Assertion failed: %s\n", stats->file, stats->line, *stats->message ? stats->message : stats->condition); @@ -74,9 +118,18 @@ ReportHook(ASSERT)(struct criterion_assert_stats *stats) { } ReportHook(TEST_CRASH)(struct criterion_test_stats *stats) { - fprintf(stderr, "\tUnexpected signal after %s:%u!\n%s::%s: FAILURE (CRASH!)\n", - stats->file, - stats->progress, - stats->test->category, - stats->test->name); + if (criterion_options.enable_tap_format) { + criterion_important("not ok %lu - %s::%s unexpected signal after %s:%u\n", + tap_test_index++, + stats->test->category, + stats->test->name, + stats->file, + stats->progress); + } else { + criterion_important("Unexpected signal after %s:%u!\n%s::%s: FAILURE (CRASH!)\n", + stats->file, + stats->progress, + stats->test->category, + stats->test->name); + } } diff --git a/src/report.h b/src/report.h index 4b02caa..c9506b8 100644 --- a/src/report.h +++ b/src/report.h @@ -28,18 +28,17 @@ # define report(Kind, Data) call_report_hooks_##Kind(Data) -# define DECL_CALL_REPORT_HOOKS(Kind) \ - extern f_report_hook __start_criterion_hooks_##Kind; \ - extern f_report_hook __stop_criterion_hooks_##Kind; \ +# define DECL_CALL_REPORT_HOOKS(Kind) \ + DECL_SECTION_LIMITS(f_report_hook, crit_ ## Kind); \ void call_report_hooks_##Kind(void *data) -DECL_CALL_REPORT_HOOKS(PRE_EVERYTHING); +DECL_CALL_REPORT_HOOKS(PRE_ALL); DECL_CALL_REPORT_HOOKS(PRE_INIT); DECL_CALL_REPORT_HOOKS(PRE_TEST); DECL_CALL_REPORT_HOOKS(ASSERT); DECL_CALL_REPORT_HOOKS(TEST_CRASH); DECL_CALL_REPORT_HOOKS(POST_TEST); DECL_CALL_REPORT_HOOKS(POST_FINI); -DECL_CALL_REPORT_HOOKS(POST_EVERYTHING); +DECL_CALL_REPORT_HOOKS(POST_ALL); #endif /* !REPORT_H_ */ diff --git a/src/runner.c b/src/runner.c index e00d4fe..cc749ba 100644 --- a/src/runner.c +++ b/src/runner.c @@ -27,19 +27,14 @@ #include #include #include "criterion/assert.h" +#include "criterion/options.h" #include "stats.h" #include "runner.h" #include "report.h" #include "event.h" #include "process.h" -static struct criterion_test * const g_section_start = &__start_criterion_tests; -static struct criterion_test * const g_section_end = &__stop_criterion_tests; - -struct test_set { - struct criterion_test **tests; - size_t nb_tests; -}; +IMPL_SECTION_LIMITS(struct criterion_test, criterion_tests); static int compare_test(const void *a, const void *b) { struct criterion_test *first = *(struct criterion_test **) a; @@ -55,30 +50,30 @@ static int compare_test(const void *a, const void *b) { } static void destroy_test_set(void *ptr, UNUSED void *meta) { - struct test_set *set = ptr; + struct criterion_test_set *set = ptr; free(set->tests); } -static struct test_set *read_all_tests(void) { - size_t nb_tests = g_section_end - g_section_start; +static struct criterion_test_set *read_all_tests(void) { + size_t nb_tests = SECTION_END(criterion_tests) - SECTION_START(criterion_tests); struct criterion_test **tests = malloc(nb_tests * sizeof (void *)); if (tests == NULL) return NULL; size_t i = 0; - for (struct criterion_test *test = g_section_start; test < g_section_end; ++test) + for (struct criterion_test *test = SECTION_START(criterion_tests); test < SECTION_END(criterion_tests); ++test) tests[i++] = test; qsort(tests, nb_tests, sizeof (void *), compare_test); - return unique_ptr(struct test_set, ({ + return unique_ptr(struct criterion_test_set, ({ .tests = tests, .nb_tests = nb_tests }), destroy_test_set); } -static void map_tests(struct test_set *set, struct criterion_global_stats *stats, void (*fun)(struct criterion_global_stats *, struct criterion_test *)) { +static void map_tests(struct criterion_test_set *set, struct criterion_global_stats *stats, void (*fun)(struct criterion_global_stats *, struct criterion_test *)) { size_t i = 0; for (struct criterion_test **t = set->tests; i < set->nb_tests; ++i, ++t) { fun(stats, *t); @@ -101,6 +96,9 @@ static void run_test_child(struct criterion_test *test) { } static void run_test(struct criterion_global_stats *stats, struct criterion_test *test) { + if (test->data->disabled) + return; + smart struct criterion_test_stats *test_stats = test_stats_init(test); smart struct process *proc = spawn_test_worker(test, run_test_child); @@ -139,12 +137,12 @@ static void run_test(struct criterion_global_stats *stats, struct criterion_test } } -// TODO: disable & change tests at runtime static int criterion_run_all_tests_impl(void) { - report(PRE_EVERYTHING, NULL); + smart struct criterion_test_set *set = read_all_tests(); + + report(PRE_ALL, set); set_runner_pid(); - smart struct test_set *set = read_all_tests(); smart struct criterion_global_stats *stats = stats_init(); if (!set) abort(); @@ -153,14 +151,14 @@ static int criterion_run_all_tests_impl(void) { if (!is_runner()) return -1; - report(POST_EVERYTHING, stats); + report(POST_ALL, stats); return stats->tests_failed > 0; } int criterion_run_all_tests(void) { int res = criterion_run_all_tests_impl(); if (res == -1) // if this is the test worker terminating - exit(0); + _exit(0); - return strcmp("1", getenv("CRITERION_ALWAYS_SUCCEED") ?: "0") && res; + return !criterion_options.always_succeed && res; } diff --git a/src/runner.h b/src/runner.h index 6723104..10ccfbb 100644 --- a/src/runner.h +++ b/src/runner.h @@ -26,7 +26,6 @@ # include "criterion/criterion.h" -extern struct criterion_test __start_criterion_tests; -extern struct criterion_test __stop_criterion_tests; +DECL_SECTION_LIMITS(struct criterion_test, criterion_tests); #endif /* !CRITERION_RUNNER_H_ */ diff --git a/src/stats.c b/src/stats.c index 24199ad..ab67456 100644 --- a/src/stats.c +++ b/src/stats.c @@ -64,14 +64,14 @@ void stat_push_event(s_glob_stats *stats, s_test_stats *test, struct event *data) { static void (*const handles[])(s_glob_stats *, s_test_stats *, void *) = { - nothing, // PRE_EVERYTHING + nothing, // PRE_ALL nothing, // PRE_INIT push_pre_test, // PRE_TEST push_assert, // ASSERT push_test_crash, // TEST_CRASH push_post_test, // POST_TEST nothing, // POST_FINI - nothing, // POST_EVERYTHING + nothing, // POST_ALL }; assert(data->kind > 0);