stv0g/Criterion
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

[v1.1.0] Merge branch 'bleeding'

Browse source
This commit is contained in:
Snaipe 2015-03-24 16:02:40 +01:00
commit 227ee452a2
45 changed files with 1067 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
ChangeLog
View file

@ -1,3 +1,15 @@
2015-03-24 Franklin "Snaipe" Mathieu <franklinmathieu@gmail.com>
* src/log/: Refactored logging system
* src/log/: Changed output format to a better-looking one
* src/log/: Added Syntactic coloration
* include/, src/: Added test suite separation
* include/, src/: Added test suite statistics
* src/main.c: Added --list option
* src/main.c: Added --fail-fast option
* src/main.c, src/report.c: Added --pattern option
2015-03-18 Franklin "Snaipe" Mathieu <franklinmathieu@gmail.com>
* src/timer.*: Added test timings
* src/, include/: Changed assert prototypes

27
Makefile.am
View file

@ -4,12 +4,12 @@ SUBDIRS = dependencies/csptr samples
lib_LTLIBRARIES = libcriterion.la
WARNINGS = -Wall -Wextra \
-Wno-unused-result -Wno-missing-field-initializers
-Wno-unused-result
libcriterion_la_CFLAGS = \
$(WARNINGS) \
-std=gnu99 \
-fplan9-extensions \
-std=gnu11 \
-I$(top_srcdir)/src/ \
-I$(top_srcdir)/include/ \
-I$(top_srcdir)/dependencies/csptr/include/ \
$(COVERAGE_CFLAGS)
@ -29,7 +29,9 @@ subdirinclude_HEADERS = \
include/criterion/event.h \
include/criterion/hooks.h \
include/criterion/logging.h \
include/criterion/types.h \
include/criterion/options.h \
include/criterion/ordered-set.h \
include/criterion/stats.h
libcriterion_la_SOURCES = \
@ -43,8 +45,25 @@ libcriterion_la_SOURCES = \
src/process.h \
src/stats.c \
src/stats.h \
src/logging.c \
src/log/logging.c \
src/log/tap.c \
src/log/normal.c \
src/options.c \
src/timer.c \
src/timer.h \
src/ordered-set.c \
src/main.c
TARGET = $(PACKAGE)-$(VERSION)
package: all
rm -Rf $(TARGET)
mkdir -p $(TARGET)
cp -Rf .libs $(TARGET)/lib/
rm -f $(TARGET)/lib/libcriterion.la
cp -f libcriterion.la $(TARGET)/lib
cp -Rf include $(TARGET)
tar -cvjf $(TARGET).tar.bz2 $(TARGET)
clean-local:
rm -Rf $(TARGET) $(TARGET).tar.bz2

9
README.md
View file

@ -24,6 +24,7 @@ Criterion follows the KISS principle, while keeping the control
the user would have with other frameworks:
* [x] Tests are automatically registered when declared.
* [x] Implements a xUnit framework structure.
* [x] A default entry point is provided, no need to declare a main
unless you want to do special handling.
* [x] Test are isolated in their own process, crashes and signals can be
@ -34,10 +35,10 @@ the user would have with other frameworks:
## Downloads
* [Linux (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-linux-x86_64.tar.bz2)
* [OS X (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-osx-x86_64.tar.bz2)
* [Windows (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-win-x86_64.tar.bz2)
* [FreeBSD (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.0.0/criterion-1.0.0-freebsd-x86_64.tar.bz2)
* [Linux (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.1.0/criterion-1.1.0-linux-x86_64.tar.bz2)
* [OS X (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.1.0/criterion-1.1.0-osx-x86_64.tar.bz2)
* [Windows (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.1.0/criterion-1.1.0-win-x86_64.tar.bz2)
* [FreeBSD (x86_64)](https://github.com/Snaipe/Criterion/releases/download/v1.1.0/criterion-1.1.0-freebsd-x86_64.tar.bz2)
If you have a different platform, you can still [build the library from source](http://criterion.readthedocs.org/en/latest/setup.html#installation)

2
configure.ac
View file

@ -1,6 +1,6 @@
AC_PREREQ([2.60])
AC_INIT([criterion], [1.0.0], [], [criterion], [franklinmathieu@gmail.com])
AC_INIT([criterion], [1.1.0], [], [criterion], [franklinmathieu@gmail.com])
AC_CONFIG_SRCDIR([src/runner.c])
LT_PREREQ([2.2.4])

2
dependencies/csptr vendored

@ -1 +1 @@
Subproject commit 3b6b26f8b3464a70cc76628e4b65b776a7760ba4
Subproject commit 15b825ffb8ffe7309bf524659eb900d6bb1d7c04

2
doc/conf.py
View file

@ -39,7 +39,7 @@ copyright = u'2015, Franklin "Snaipe" Mathieu'
# built documents.
#
# The short X.Y version.
version = '0.1.0'
version = '1.1.0'
# The full version, including alpha/beta/rc tags.
release = version

65
doc/env.rst
View file

@ -1,15 +1,64 @@
Environment and CLI
===================
Tests built with Criterion support environment variables to alter
their runtime behaviour.
Tests built with Criterion expose by default various command line switchs
and environment variables to alter their runtime behaviour.
Command line arguments
----------------------
* ``-h or --help``: Show a help message with the available switches.
* ``-v or --version``: Prints the version of criterion that has been
linked against.
* ``-l or --list``: Print all the tests in a list.
* ``-f or --fail-fast``: Exit after the first test failure.
* ``--ascii``: Don't use fancy unicode symbols or colors in the output.
* ``--pattern [PATTERN]``: Run tests whose string identifier matches
the given shell wildcard pattern (see dedicated section below).
* ``--no-early-exit``: The test workers shall not prematurely exit when done and
will properly return from the main, cleaning up their process space.
This is useful when tracking memory leaks with ``valgrind --tool=memcheck``.
* ``--always-succeed``: The process shall exit with a status of ``0``.
* ``--tap``: Enables the TAP (Test Anything Protocol) output format.
* ``--verbose[=level]``: Makes the output verbose. When provided with an integer,
sets the verbosity level to that integer.
Shell Wildcard Pattern
----------------------
Patterns in criterion are matched against a test's string identifier with
``fnmatch``.
Special characters used in shell-style wildcard patterns are:
=========== ===================================
Pattern Meaning
=========== ===================================
``*`` matches everything
----------- -----------------------------------
``?`` matches any character
----------- -----------------------------------
``[seq]`` matches any character in *seq*
----------- -----------------------------------
``[!seq]`` matches any character not in *seq*
=========== ===================================
A test string identifier is of the form ``suite-name/test-name``, so a pattern
of ``simple/*`` matches every tests in the ``simple`` suite, ``*/passing``
matches all tests named ``passing`` regardless of the suite, and ``*`` matches
every possible test.
Environment Variables
---------------------
* `CRITERION_ALWAYS_SUCCEED`: when set to `1`, the exit status of the test
process will be 0, regardless if the tests failed or not.
* `CRITERION_NO_EARLY_EXIT`: when set to `1`, the test workers shall not
call `_exit` when done and will properly return from the main and
clean up their process space. This is useful when tracking memory leaks with
`valgrind --tool=memcheck`.
Environment variables are alternatives to command line switches when set to 1.
* ``CRITERION_ALWAYS_SUCCEED``: Same as ``--always-succeed``.
* ``CRITERION_NO_EARLY_EXIT``: Same as ``--no-early-exit``.
* ``CRITERION_ENABLE_TAP``: Same as ``--tap``.
* ``CRITERION_FAIL_FAST``: Same as ``--fail-fast``.
* ``CRITERION_USE_ASCII``: Same as ``--ascii``.
* ``CRITERION_VERBOSITY_LEVEL``: Same as ``--verbose``. Sets the verbosity level
to its value.
* ``CRITERION_TEST_PATTERN``: Same as ``--pattern``. Sets the test pattern
to its value.

17
doc/faq.rst Normal file
View file

@ -0,0 +1,17 @@
F.A.Q
=====
**Q. When running the test suite in Windows' cmd.exe, the test executable
prints weird characters, how do I fix that?**
A. Windows' ``cmd.exe`` is not an unicode ANSI-compatible terminal emulator.
There are plenty of ways to fix that behaviour:
* Pass ``--ascii`` to the test suite when executing.
* Define the ``CRITERION_USE_ASCII`` environment variable to ``1``.
* Get a better terminal emulator, such as the one shipped with Git or Cygwin.
**Q. I'm having an issue with the library, what can I do ?**
A. Open a new issue on the `github issue tracker <https://github.com/Snaipe/Criterion/issues>`_,
and describe the problem you are experiencing.

9
doc/hooks.rst
View file

@ -23,12 +23,14 @@ Testing Phases
The flow of the test process goes as follows:
1. ``PRE_ALL``: occurs before running the tests.
#. ``PRE_SUITE``: occurs before a suite is initialized.
#. ``PRE_INIT``: occurs before a test is initialized.
#. ``PRE_TEST``: occurs after the test initialization, but before the test is run.
#. ``ASSERT``: occurs when an assertion is hit
#. ``TEST_CRASH``: occurs when a test crashes unexpectedly.
#. ``POST_TEST``: occurs after a test ends, but before the test finalization.
#. ``POST_FINI``: occurs after a test finalization.
#. ``POST_SUITE``: occurs before a suite is finalized.
#. ``POST_ALL``: occurs after all the tests are done.
Hook Parameters
@ -40,9 +42,10 @@ type for that phase.
Valid types for each phases are:
* ``struct criterion_test_set *`` for ``PRE_ALL``.
* ``struct criterion_suite_set *`` for ``PRE_SUITE``.
* ``struct criterion_test *`` for ``PRE_INIT`` and ``PRE_TEST``.
* ``struct criterion_test_stats *`` for ``POST_TEST``, ``POST_FINI``, and ``TEST_CRASH``.
* ``struct criterion_assert_stats *`` for ``ASSERT``.
* ``struct criterion_test_stats *`` for ``POST_TEST``, ``POST_FINI``, and ``TEST_CRASH``.
* ``struct criterion_suite_stats *`` for ``POST_SUITE``.
* ``struct criterion_global_stats *`` for ``POST_ALL``.
``PRE_ALL`` does not take any parameter.

1
doc/index.rst
View file

@ -9,3 +9,4 @@ Criterion
starter
hooks
env
faq

4
doc/intro.rst
View file

@ -26,3 +26,7 @@ Features
* Test are isolated in their own process, crashes and signals can be
reported and tested.
* Progress and statistics can be followed in real time with report hooks.
* TAP output format can be enabled with an option.
* Runs on Linux, FreeBSD, Mac OS X, and Windows (compiles only with Cygwin
for the moment).
* xUnit framework structure

BIN
doc/screencast.gif
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.3 MiB

After

Width:  |  Height:  |  Size: 1.4 MiB

3
doc/setup.rst
View file

@ -8,7 +8,8 @@ Currently, this library only works under \*nix systems.
To compile the static library and its dependencies, GCC 4.9+ is needed.
To use the static library, GCC or Clang are needed.
To use the static library, any GNU-C compatible compiler will suffice
(GCC, Clang/LLVM, ICC, MinGW-GCC, ...).
Installation
------------

2
include/criterion/assert.h
View file

@ -27,7 +27,7 @@
# include <string.h>
# include <stdlib.h>
# include <stdbool.h>
# include "criterion.h"
# include "types.h"
# include "stats.h"
# include "hooks.h"
# include "event.h"

4
include/criterion/common.h
View file

@ -51,8 +51,8 @@
extern Type SECTION_START_(Name) SECTION_START_SUFFIX(#Name); \
extern Type SECTION_END_(Name) SECTION_END_SUFFIX(#Name)
# define IMPL_SECTION_LIMITS(Type, Name) \
Type *const SECTION_START(Name) = &SECTION_START_(Name); \
# define IMPL_SECTION_LIMITS(Type, Name) \
Type *const SECTION_START(Name) = &SECTION_START_(Name); \
Type *const SECTION_END(Name) = &SECTION_END_(Name)
# define UNUSED __attribute__((unused))

70
include/criterion/criterion.h
View file

@ -1,68 +1,25 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef CRITERION_H_
# define CRITERION_H_
# include <stdbool.h>
# include <stddef.h>
# include "common.h"
# include "assert.h"
struct criterion_test_extra_data {
int sentinel_;
const char *const file_;
const unsigned line_;
void (*init)(void);
void (*fini)(void);
int signal;
bool disabled;
void *data;
};
struct criterion_test {
const char *name;
const char *category;
void (*test)(void);
struct criterion_test_extra_data *const data;
};
struct criterion_test_set {
struct criterion_test **tests;
size_t nb_tests;
};
# include "types.h"
# define IDENTIFIER_(Category, Name, Suffix) \
Category ## _ ## Name ## _ ## Suffix
# define TEST_PROTOTYPE_(Category, Name) \
void IDENTIFIER_(Category, Name, impl)(void)
# define SUITE_IDENTIFIER_(Name, Suffix) \
suite_ ## Name ## _ ## Suffix
# define Test(...) Test_(__VA_ARGS__, .sentinel_ = 0)
# define Test_(Category, Name, ...) \
TEST_PROTOTYPE_(Category, Name); \
struct criterion_test_extra_data IDENTIFIER_(Category, Name, extra) = { \
.file_ = __FILE__, \
.line_ = __LINE__, \
.identifier_ = #Category "/" #Name, \
.file_ = __FILE__, \
.line_ = __LINE__, \
__VA_ARGS__ \
}; \
SECTION_("criterion_tests") \
@ -74,6 +31,19 @@ struct criterion_test_set {
}; \
TEST_PROTOTYPE_(Category, Name)
# define TestSuite(...) TestSuite_(__VA_ARGS__, .sentinel_ = 0)
# define TestSuite_(Name, ...) \
struct criterion_test_extra_data SUITE_IDENTIFIER_(Name, extra) = { \
.file_ = __FILE__, \
.line_ = 0, \
__VA_ARGS__ \
}; \
SECTION_("crit_suites") \
const struct criterion_suite SUITE_IDENTIFIER_(Name, meta) = { \
.name = #Name, \
.data = &SUITE_IDENTIFIER_(Name, extra), \
}
int criterion_run_all_tests(void);
#endif /* !CRITERION_H_ */

2
include/criterion/hooks.h
View file

@ -28,12 +28,14 @@
typedef enum {
PRE_ALL,
PRE_SUITE,
PRE_INIT,
PRE_TEST,
ASSERT,
TEST_CRASH,
POST_TEST,
POST_FINI,
POST_SUITE,
POST_ALL,
} e_report_status;

21
include/criterion/logging.h
View file

@ -26,6 +26,8 @@
# include <stdbool.h>
# include "common.h"
# include "ordered-set.h"
# include "stats.h"
enum criterion_logging_level {
CRITERION_INFO = 1,
@ -38,4 +40,23 @@ void criterion_log(enum criterion_logging_level level, const char *msg, ...);
# define criterion_info(...) criterion_log(CRITERION_INFO, __VA_ARGS__)
# define criterion_important(...) criterion_log(CRITERION_IMPORTANT, __VA_ARGS__)
struct criterion_output_provider {
void (*log_pre_all )(struct criterion_test_set *set);
void (*log_pre_suite )(struct criterion_suite_set *set);
void (*log_pre_init )(struct criterion_test *test);
void (*log_pre_test )(struct criterion_test *test);
void (*log_assert )(struct criterion_assert_stats *stats);
void (*log_test_crash)(struct criterion_test_stats *stats);
void (*log_post_test )(struct criterion_test_stats *stats);
void (*log_post_fini )(struct criterion_test_stats *stats);
void (*log_post_suite)(struct criterion_suite_stats *stats);
void (*log_post_all )(struct criterion_global_stats *stats);
};
extern struct criterion_output_provider normal_logging;
extern struct criterion_output_provider tap_logging;
#define NORMAL_LOGGING (&normal_logging)
#define TAP_LOGGING (&tap_logging)
#endif /* !CRITERION_LOGGING_H_ */

5
include/criterion/options.h
View file

@ -29,9 +29,12 @@
struct criterion_options {
enum criterion_logging_level logging_threshold;
bool enable_tap_format;
struct criterion_output_provider *output_provider;
bool no_early_exit;
bool always_succeed;
bool use_ascii;
bool fail_fast;
const char *pattern;
};
extern struct criterion_options criterion_options;

59
include/criterion/ordered-set.h Normal file
View file

@ -0,0 +1,59 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef CRITERION_ORDERED_SET_H_
# define CRITERION_ORDERED_SET_H_
# include "types.h"
struct criterion_ordered_set {
struct criterion_ordered_set_node *first;
size_t size;
int (*const cmp)(void *, void *);
void (*const dtor)(void *, void *);
};
struct criterion_ordered_set_node {
struct criterion_ordered_set_node *next;
char data[0];
};
struct criterion_suite_set {
struct criterion_suite suite;
struct criterion_ordered_set *tests;
};
struct criterion_test_set {
struct criterion_ordered_set *suites;
size_t tests;
};
struct criterion_ordered_set *new_ordered_set(int (*cmp)(void *, void *), void (*dtor)(void *, void *));
void *insert_ordered_set(struct criterion_ordered_set *l, void *ptr, size_t size);
# define FOREACH_SET(Elt, Set) \
for (struct criterion_ordered_set_node *n = Set->first; n; n = n->next) \
for (int cond = 1; cond;) \
for (Elt = (void*) n->data; cond && (cond = 0, 1);)
#endif /* !CRITERION_ORDERED_SET_H_ */

21
include/criterion/stats.h
View file

@ -24,9 +24,7 @@
#ifndef CRITERION_STATS_H_
# define CRITERION_STATS_H_
# include <stdbool.h>
# include <stddef.h>
# include "criterion.h"
# include "types.h"
struct criterion_assert_stats {
int kind;
@ -53,7 +51,8 @@ struct criterion_test_stats {
struct criterion_test_stats *next;
};
struct criterion_global_stats {
struct criterion_suite_stats {
struct criterion_suite *suite;
struct criterion_test_stats *tests;
size_t nb_tests;
size_t nb_asserts;
@ -62,6 +61,20 @@ struct criterion_global_stats {
size_t tests_passed;
size_t asserts_failed;
size_t asserts_passed;
struct criterion_suite_stats *next;
};
struct criterion_global_stats {
struct criterion_suite_stats *suites;
size_t nb_suites;
size_t nb_tests;
size_t nb_asserts;
size_t tests_failed;
size_t tests_crashed;
size_t tests_passed;
size_t asserts_failed;
size_t asserts_passed;
};
#endif /* !CRITERION_STATS_H_ */

55
include/criterion/types.h Normal file
View file

@ -0,0 +1,55 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef CRITERION_TYPES_H_
# define CRITERION_TYPES_H_
# include <stdbool.h>
# include <stddef.h>
struct criterion_test_extra_data {
int sentinel_;
const char *const identifier_;
const char *const file_;
const unsigned line_;
void (*init)(void);
void (*fini)(void);
int signal;
bool disabled;
const char *description;
void *data;
};
struct criterion_test {
const char *name;
const char *category;
void (*test)(void);
struct criterion_test_extra_data *const data;
};
struct criterion_suite {
const char *name;
struct criterion_test_extra_data *const data;
};
#endif /* !CRITERION_TYPES_H_ */

6
samples/Makefile.am
View file

@ -4,6 +4,9 @@ BIN_TESTS = \
suites \
fixtures \
asserts \
more-suites \
long-messages \
description \
simple
TESTS_ENVIRONMENT = CRITERION_ALWAYS_SUCCEED=1
@ -15,6 +18,9 @@ LDADD = -L$(top_srcdir)/ -lcriterion
SCRIPT_TESTS = tests/tap_test.sh \
tests/early_exit.sh \
tests/verbose.sh \
tests/list.sh \
tests/pattern.sh \
tests/fail_fast.sh \
tests/help.sh
EXTRA_DIST = $(SCRIPT_TESTS)

5
samples/description.c Normal file
View file

@ -0,0 +1,5 @@
#include <criterion/criterion.h>
Test(misc, failing, .description = "Just a failing test") {
assert(0);
}

5
samples/long-messages.c Normal file
View file

@ -0,0 +1,5 @@
#include <criterion/criterion.h>
Test(sample, long_msg) {
assert(0, "This is\nA long message\nSpawning multiple lines.\n\nFormatting is respected.");
}

19
samples/more-suites.c Normal file
View file

@ -0,0 +1,19 @@
#include <criterion/criterion.h>
void setup_suite(void) {
// setup suite
}
TestSuite(suite1, .init = setup_suite);
Test(suite1, test) {
assert(1);
}
Test(suite2, test) {
assert(1);
}
TestSuite(disabled, .disabled = true);
Test(disabled, test) {}

2
samples/tests/fail_fast.sh Executable file
View file

@ -0,0 +1,2 @@
#!/bin/sh
./simple --fail-fast --always-succeed

1
samples/tests/help.sh
View file

@ -1,2 +1,3 @@
#!/bin/sh
./simple --help
./simple --version

3
samples/tests/list.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/sh
./simple --list
./simple --list --ascii

2
samples/tests/pattern.sh Executable file
View file

@ -0,0 +1,2 @@
#!/bin/sh
./simple --pattern '*/passing'

3
samples/tests/tap_test.sh
View file

@ -2,3 +2,6 @@
./simple --tap --always-succeed
./signal --tap --always-succeed
./asserts --tap --always-succeed
./more-suites --tap --always-succeed
./long-messages --tap --always-succeed
./description --tap --always-succeed

6
src/event.c
View file

@ -47,17 +47,17 @@ struct event *read_event(int fd) {
if (read(fd, buf, assert_size) < (ssize_t) assert_size)
return NULL;
return unique_ptr(struct event, ({ .kind = kind, .data = buf }), destroy_event);
return unique_ptr(struct event, { .kind = kind, .data = buf }, destroy_event);
}
case POST_TEST: {
double *elapsed_time = malloc(sizeof (double));
if (read(fd, elapsed_time, sizeof (double)) < (ssize_t) sizeof (double))
return NULL;
return unique_ptr(struct event, ({ .kind = kind, .data = elapsed_time }), destroy_event);
return unique_ptr(struct event, { .kind = kind, .data = elapsed_time }, destroy_event);
}
default:
return unique_ptr(struct event, ({ .kind = kind, .data = NULL }));
return unique_ptr(struct event, { .kind = kind, .data = NULL });
}
}

0
src/logging.c → src/log/logging.c
View file

158
src/log/normal.c Normal file
View file

@ -0,0 +1,158 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "criterion/stats.h"
#include "criterion/logging.h"
#include "criterion/options.h"
#include "criterion/ordered-set.h"
#include "timer.h"
#include "config.h"
#define NORMALIZE(Str) (criterion_options.use_ascii ? "" : Str)
#define FG_BOLD NORMALIZE("\e[0;1m")
#define FG_RED NORMALIZE("\e[0;31m")
#define FG_GREEN NORMALIZE("\e[0;32m")
#define FG_GOLD NORMALIZE("\e[0;33m")
#define FG_BLUE NORMALIZE("\e[0;34m")
#define RESET NORMALIZE("\e[0m")
void normal_log_pre_all(UNUSED struct criterion_test_set *set) {
criterion_info("[%s====%s] Criterion v%s\n", FG_BLUE, RESET, VERSION);
}
void normal_log_pre_init(struct criterion_test *test) {
criterion_info("[%sRUN%s ] %s::%s\n", FG_BLUE, RESET, test->category, test->name);
if (test->data->description)
criterion_info("[%s----%s] %s\n", FG_BLUE, RESET, test->data->description);
}
void normal_log_post_test(struct criterion_test_stats *stats) {
const char *format = can_measure_time() ? "%s::%s: (%3.2fs)\n" : "%s::%s\n";
const enum criterion_logging_level level = stats->failed ? CRITERION_IMPORTANT
: CRITERION_INFO;
const char *color = stats->failed ? FG_RED : FG_GREEN;
criterion_log(level, "[%s%s%s] ", color, stats->failed ? "FAIL" : "PASS", RESET);
criterion_log(level, format,
stats->test->category,
stats->test->name,
stats->elapsed_time);
}
__attribute__((always_inline))
static inline bool is_disabled(struct criterion_test *t, struct criterion_suite *s) {
return t->data->disabled || (s->data && s->data->disabled);
}
void normal_log_post_suite(struct criterion_suite_stats *stats) {
for (struct criterion_test_stats *ts = stats->tests; ts; ts = ts->next) {
if (is_disabled(ts->test, stats->suite)) {
criterion_info("[%sSKIP%s] %s::%s: %s is disabled\n",
FG_GOLD,
RESET,
ts->test->category,
ts->test->name,
ts->test->data->disabled ? "test" : "suite");
if (ts->test->data->description)
criterion_info("[%s----%s] %s\n", FG_BLUE, RESET, ts->test->data->description);
}
}
}
void normal_log_post_all(struct criterion_global_stats *stats) {
criterion_important("[%s====%s] ", FG_BLUE, RESET);
criterion_important("%sSynthesis: " SIZE_T_FORMAT " test%s run. " SIZE_T_FORMAT " passed, " SIZE_T_FORMAT " failed (with " SIZE_T_FORMAT " crash%s)%s\n",
FG_BOLD,
stats->nb_tests,
stats->nb_tests == 1 ? " was" : "s were",
stats->tests_passed,
stats->tests_failed,
stats->tests_crashed,
stats->tests_crashed == 1 ? "" : "es",
RESET);
}
void normal_log_assert(struct criterion_assert_stats *stats) {
if (!stats->passed) {
char *dup = strdup(*stats->message ? stats->message : stats->condition), *saveptr = NULL;
char *line = strtok_r(dup, "\n", &saveptr);
criterion_important("[%s----%s] ", FG_BLUE, RESET);
criterion_important("%s%s%s:%s%d%s: Assertion failed: %s\n",
FG_BOLD,
stats->file,
RESET,
FG_RED,
stats->line,
RESET,
line);
while ((line = strtok_r(NULL, "\n", &saveptr)))
criterion_important("[%s----%s] %s\n", FG_BLUE, RESET, line);
free(dup);
}
}
void normal_log_test_crash(struct criterion_test_stats *stats) {
criterion_important("[%s----%s] ", FG_BLUE, RESET);
criterion_important("%s%s%s:%s%u%s: Unexpected signal caught below this line!\n",
FG_BOLD,
stats->file,
RESET,
FG_RED,
stats->progress,
RESET);
criterion_important("[%sFAIL%s] %s::%s: CRASH!\n",
FG_RED,
RESET,
stats->test->category,
stats->test->name);
}
void normal_log_pre_suite(struct criterion_suite_set *set) {
criterion_info("[%s====%s] ", FG_BLUE, RESET);
criterion_info("Running %s" SIZE_T_FORMAT "%s test%s from %s%s%s:\n",
FG_BLUE,
set->tests->size,
RESET,
set->tests->size == 1 ? "" : "s",
FG_GOLD,
set->suite.name,
RESET);
}
struct criterion_output_provider normal_logging = {
.log_pre_all = normal_log_pre_all,
.log_pre_init = normal_log_pre_init,
.log_pre_suite = normal_log_pre_suite,
.log_assert = normal_log_assert,
.log_test_crash = normal_log_test_crash,
.log_post_test = normal_log_post_test,
.log_post_suite = normal_log_post_suite,
.log_post_all = normal_log_post_all,
};

111
src/log/tap.c Normal file
View file

@ -0,0 +1,111 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "criterion/stats.h"
#include "criterion/logging.h"
#include "criterion/options.h"
#include "criterion/ordered-set.h"
#include "timer.h"
#include "config.h"
void tap_log_pre_all(struct criterion_test_set *set) {
size_t enabled_count = 0;
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
if ((s->suite.data && s->suite.data->disabled) || !s->tests)
continue;
FOREACH_SET(struct criterion_test *test, s->tests) {
if (!test->data->disabled)
++enabled_count;
}
}
criterion_important("TAP version 13\n1.." SIZE_T_FORMAT "\n", set->tests);
criterion_important("# Criterion v%s\n", VERSION);
}
void tap_log_pre_suite(struct criterion_suite_set *set) {
criterion_important("\n# Running " SIZE_T_FORMAT " tests from %s\n",
set->tests->size,
set->suite.name);
}
__attribute__((always_inline))
static inline bool is_disabled(struct criterion_test *t, struct criterion_suite *s) {
return t->data->disabled || (s->data && s->data->disabled);
}
void tap_log_post_suite(struct criterion_suite_stats *stats) {
for (struct criterion_test_stats *ts = stats->tests; ts; ts = ts->next) {
if (is_disabled(ts->test, stats->suite)) {
criterion_important("ok - %s::%s %s # SKIP %s is disabled\n",
ts->test->category,
ts->test->name,
ts->test->data->description ?: "",
ts->test->data->disabled ? "test" : "suite");
}
}
}
void tap_log_post_test(struct criterion_test_stats *stats) {
const char *format = can_measure_time() ? "%s - %s::%s %s (%3.2fs)\n"
: "%s - %s::%s %s\n";
criterion_important(format,
stats->failed ? "not ok" : "ok",
stats->test->category,
stats->test->name,
stats->test->data->description ?: "",
stats->elapsed_time);
for (struct criterion_assert_stats *asrt = stats->asserts; asrt; asrt = asrt->next) {
if (!asrt->passed) {
char *dup = strdup(*asrt->message ? asrt->message : asrt->condition), *saveptr = NULL;
char *line = strtok_r(dup, "\n", &saveptr);
criterion_important(" %s:%u: Assertion failed: %s\n",
asrt->file,
asrt->line,
line);
while ((line = strtok_r(NULL, "\n", &saveptr)))
criterion_important(" %s\n", line);
free(dup);
}
}
}
void tap_log_test_crash(struct criterion_test_stats *stats) {
criterion_important("not ok - %s::%s unexpected signal after %s:%u\n",
stats->test->category,
stats->test->name,
stats->file,
stats->progress);
}
struct criterion_output_provider tap_logging = {
.log_pre_all = tap_log_pre_all,
.log_pre_suite = tap_log_pre_suite,
.log_test_crash = tap_log_test_crash,
.log_post_test = tap_log_post_test,
.log_post_suite = tap_log_post_suite,
};

136
src/main.c
View file

@ -1,25 +1,118 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#define _GNU_SOURCE
#include <criterion/criterion.h>
#include <criterion/options.h>
#include <criterion/ordered-set.h>
#include <stdio.h>
#include <getopt.h>
#include <csptr/smart_ptr.h>
#include "runner.h"
#include "config.h"
# define USAGE \
"usage: %s OPTIONS\n" \
"options: \n" \
" -h or --help: prints this message\n" \
" --verbose [level]: sets verbosity to level\n"
# define VERSION_MSG "Tests compiled with Criterion v" VERSION "\n"
# define USAGE \
VERSION_MSG "\n" \
"usage: %s OPTIONS\n" \
"options: \n" \
" -h or --help: prints this message\n" \
" -v or --version: prints the version of criterion " \
"these tests have been linked against\n" \
" -l or --list: prints all the tests in a list\n" \
" -f or --fail-fast: exit after the first failure\n" \
" --ascii: don't use fancy unicode symbols " \
"or colors in the output\n" \
" --pattern [PATTERN]: run tests matching the " \
"given pattern\n" \
" --tap: enables TAP formatting\n" \
" --always-succeed: always exit with 0\n" \
" --no-early-exit: do not exit the test worker " \
"prematurely after the test\n" \
" --verbose[=level]: sets verbosity to level " \
"(1 by default)\n"
int print_usage(char *progname) {
fprintf(stderr, USAGE, progname);
return 0;
}
int print_version(void) {
fputs(VERSION_MSG, stderr);
return 0;
}
# define UTF8_TREE_NODE "├"
# define UTF8_TREE_END "└"
# define UTF8_TREE_JOIN "──"
# define ASCII_TREE_NODE "|"
# define ASCII_TREE_END "`"
# define ASCII_TREE_JOIN "--"
bool is_disabled(struct criterion_suite *s, struct criterion_test *t) {
return (s->data && s->data->disabled) || t->data->disabled;
}
int list_tests(bool unicode) {
smart struct criterion_test_set *set = criterion_init();
const char *node = unicode ? UTF8_TREE_NODE : ASCII_TREE_NODE;
const char *join = unicode ? UTF8_TREE_JOIN : ASCII_TREE_JOIN;
const char *end = unicode ? UTF8_TREE_END : ASCII_TREE_END;
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
size_t tests = s->tests ? s->tests->size : 0;
if (!tests)
continue;
printf("%s: " SIZE_T_FORMAT " test%s\n",
s->suite.name,
tests,
tests == 1 ? "" : "s");
FOREACH_SET(struct criterion_test *t, s->tests) {
printf("%s%s %s%s\n",
--tests == 0 ? end : node,
join,
t->name,
is_disabled(&s->suite, t) ? " (disabled)" : "");
}
}
return 0;
}
int main(int argc, char *argv[]) {
static struct option opts[] = {
{"verbose", optional_argument, 0, 'v'},
{"verbose", optional_argument, 0, 'b'},
{"version", no_argument, 0, 'v'},
{"tap", no_argument, 0, 't'},
{"help", no_argument, 0, 'h'},
{"list", no_argument, 0, 'l'},
{"ascii", no_argument, 0, 'k'},
{"fail-fast", no_argument, 0, 'f'},
{"pattern", required_argument, 0, 'p'},
{"always-succeed", no_argument, 0, 'y'},
{"no-early-exit", no_argument, 0, 'z'},
{0, 0, 0, 0 }
@ -28,20 +121,41 @@ int main(int argc, char *argv[]) {
criterion_options = (struct criterion_options) {
.always_succeed = !strcmp("1", getenv("CRITERION_ALWAYS_SUCCEED") ?: "0"),
.no_early_exit = !strcmp("1", getenv("CRITERION_NO_EARLY_EXIT") ?: "0"),
.enable_tap_format = !strcmp("1", getenv("CRITERION_ENABLE_TAP") ?: "0"),
.fail_fast = !strcmp("1", getenv("CRITERION_FAIL_FAST") ?: "0"),
.use_ascii = !strcmp("1", getenv("CRITERION_USE_ASCII") ?: "0"),
.logging_threshold = atoi(getenv("CRITERION_VERBOSITY_LEVEL") ?: "2"),
.pattern = getenv("CRITERION_TEST_PATTERN"),
.output_provider = NORMAL_LOGGING,
};
for (int c; (c = getopt_long(argc, argv, "h", opts, NULL)) != -1;) {
bool use_tap = !strcmp("1", getenv("CRITERION_ENABLE_TAP") ?: "0");
bool do_list_tests = false;
bool do_print_version = false;
bool do_print_usage = false;
for (int c; (c = getopt_long(argc, argv, "hvlf", opts, NULL)) != -1;) {
switch (c) {
case 'v': criterion_options.logging_threshold = atoi(optarg ?: "1"); break;
case 't': criterion_options.enable_tap_format = true; break;
case 'b': criterion_options.logging_threshold = atoi(optarg ?: "1"); break;
case 'y': criterion_options.always_succeed = true; break;
case 'z': criterion_options.no_early_exit = true; break;
case 'k': criterion_options.use_ascii = true; break;
case 'f': criterion_options.fail_fast = true; break;
case 'p': criterion_options.pattern = optarg; break;
case 't': use_tap = true; break;
case 'l': do_list_tests = true; break;
case 'v': do_print_version = true; break;
case 'h':
default : return print_usage(argv[0]);
default : do_print_usage = true; break;
}
}
if (use_tap)
criterion_options.output_provider = TAP_LOGGING;
if (do_print_usage)
return print_usage(argv[0]);
if (do_print_version)
return print_version();
if (do_list_tests)
return list_tests(!criterion_options.use_ascii);
return !criterion_run_all_tests();
}

74
src/ordered-set.c Normal file
View file

@ -0,0 +1,74 @@
/*
* The MIT License (MIT)
*
* Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <criterion/common.h>
#include <criterion/ordered-set.h>
#include <csptr/smart_ptr.h>
static void destroy_ordered_set(void *ptr, UNUSED void *meta) {
sfree(((struct criterion_ordered_set *) ptr)->first);
}
__attribute__ ((always_inline))
static inline void nothing() {}
static void destroy_ordered_set_node(void *ptr, void *meta) {
struct criterion_ordered_set *set = *(void **) meta;
struct criterion_ordered_set_node *n = ptr;
(set->dtor ?: nothing)(n->data, NULL);
sfree(((struct criterion_ordered_set_node *) ptr)->next);
}
struct criterion_ordered_set *new_ordered_set(int (*cmp)(void *, void *), f_destructor dtor) {
return unique_ptr(struct criterion_ordered_set,
{ .cmp = cmp, .dtor = dtor }, destroy_ordered_set);
}
void *insert_ordered_set(struct criterion_ordered_set *l, void *ptr, size_t size) {
int cmp;
struct criterion_ordered_set_node *n, *prev = NULL;
for (n = l->first; n && (cmp = l->cmp(ptr, n->data)) > 0; n = n->next)
prev = n;
if (n && !cmp) // element already exists
return n->data;
struct criterion_ordered_set_node *new = smalloc(
.size = sizeof(struct criterion_ordered_set_node) + size,
.dtor = destroy_ordered_set_node,
.meta = { &l, sizeof (void *) },
);
if (!new)
return NULL;
memcpy(new->data, ptr, size);
new->next = n;
if (prev) {
prev->next = new;
} else {
l->first = new;
}
++l->size;
return new->data;
}

13
src/process.c
View file

@ -22,11 +22,12 @@
* THE SOFTWARE.
*/
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/wait.h>
#include <csptr/smart_ptr.h>
#include "criterion/criterion.h"
#include "criterion/types.h"
#include "criterion/options.h"
#include "process.h"
#include "event.h"
@ -54,7 +55,9 @@ struct event *worker_read_event(struct process *proc) {
return read_event(proc->in);
}
struct process *spawn_test_worker(struct criterion_test *test, void (*func)(struct criterion_test *)) {
struct process *spawn_test_worker(struct criterion_test *test,
struct criterion_suite *suite,
void (*func)(struct criterion_test *, struct criterion_suite *)) {
int fds[2];
if (pipe(fds) == -1)
abort();
@ -67,8 +70,10 @@ struct process *spawn_test_worker(struct criterion_test *test, void (*func)(stru
close(fds[0]);
EVENT_PIPE = fds[1];
func(test);
func(test, suite);
close(fds[1]);
fflush(NULL); // flush all opened streams
if (criterion_options.no_early_exit)
return NULL;
else
@ -76,7 +81,7 @@ struct process *spawn_test_worker(struct criterion_test *test, void (*func)(stru
}
close(fds[1]);
return unique_ptr(struct process, ({ .pid = pid, .in = fds[0] }), close_process);
return unique_ptr(struct process, { .pid = pid, .in = fds[0] }, close_process);
}
struct process_status wait_proc(struct process *proc) {

4
src/process.h
View file

@ -42,7 +42,9 @@ struct process_status {
void set_runner_pid(void);
bool is_runner(void);
struct process_status wait_proc(struct process *proc);
struct process *spawn_test_worker(struct criterion_test *test, void (*func)(struct criterion_test *));
struct process *spawn_test_worker(struct criterion_test *test,
struct criterion_suite *suite,
void (*func)(struct criterion_test *, struct criterion_suite *));
struct event *worker_read_event(struct process *proc);
#endif /* !PROCESS_H_ */

155
src/report.c
View file

@ -21,121 +21,84 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include "criterion/criterion.h"
#include <stdlib.h>
#include <fnmatch.h>
#include "criterion/types.h"
#include "criterion/stats.h"
#include "criterion/logging.h"
#include "criterion/options.h"
#include "criterion/ordered-set.h"
#include "report.h"
#include "timer.h"
#define IMPL_CALL_REPORT_HOOKS(Kind) \
IMPL_SECTION_LIMITS(f_report_hook, crit_ ## Kind); \
void call_report_hooks_##Kind(void *data) { \
for (f_report_hook *hook = SECTION_START(crit_ ## Kind); \
hook < SECTION_END(crit_ ## Kind); \
++hook) { \
(*hook)(data); \
} \
#define IMPL_CALL_REPORT_HOOKS(Kind) \
IMPL_SECTION_LIMITS(f_report_hook, crit_ ## Kind); \
void call_report_hooks_##Kind(void *data) { \
for (f_report_hook *hook = SECTION_START(crit_ ## Kind); \
hook < SECTION_END(crit_ ## Kind); \
++hook) { \
(*hook)(data); \
} \
}
static size_t tap_test_index = 1;
#define IMPL_REPORT_HOOK(Type) \
IMPL_CALL_REPORT_HOOKS(Type); \
ReportHook(Type)
IMPL_CALL_REPORT_HOOKS(PRE_ALL);
IMPL_CALL_REPORT_HOOKS(PRE_INIT);
IMPL_CALL_REPORT_HOOKS(PRE_TEST);
IMPL_CALL_REPORT_HOOKS(ASSERT);
IMPL_CALL_REPORT_HOOKS(TEST_CRASH);
IMPL_CALL_REPORT_HOOKS(POST_TEST);
IMPL_CALL_REPORT_HOOKS(POST_FINI);
IMPL_CALL_REPORT_HOOKS(POST_ALL);
#define log(Type, Arg) \
(criterion_options.output_provider->log_ ## Type ?: nothing)(Arg);
ReportHook(PRE_INIT)(struct criterion_test *test) {
if (criterion_options.enable_tap_format) return;
__attribute__((always_inline))
static inline void nothing() {}
criterion_info("%s::%s: RUNNING\n", test->category, test->name);
}
IMPL_REPORT_HOOK(PRE_ALL)(struct criterion_test_set *set) {
if (criterion_options.pattern) {
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
if ((s->suite.data && s->suite.data->disabled) || !s->tests)
continue;
ReportHook(POST_TEST)(struct criterion_test_stats *stats) {
if (criterion_options.enable_tap_format) {
const char *format = can_measure_time() ? "%s " SIZE_T_FORMAT " - %s::%s (%3.2fs)\n"
: "%s " SIZE_T_FORMAT " - %s::%s\n";
criterion_important(format,
stats->failed ? "not ok" : "ok",
tap_test_index++,
stats->test->category,
stats->test->name,
stats->elapsed_time);
for (struct criterion_assert_stats *asrt = stats->asserts; asrt; asrt = asrt->next) {
if (!asrt->passed) {
char *dup = strdup(*asrt->message ? asrt->message : asrt->condition), *saveptr = NULL;
char *line = strtok_r(dup, "\n", &saveptr);
criterion_important("\t%s:%u: Assertion failed: %s\n",
asrt->file,
asrt->line,
line);
while ((line = strtok_r(NULL, "\n", &saveptr)))
criterion_important("\t%s\n", line);
free(dup);
FOREACH_SET(struct criterion_test *test, s->tests) {
if (fnmatch(criterion_options.pattern, test->data->identifier_, 0))
test->data->disabled = true;
}
}
} else {
const char *format = can_measure_time() ? "%s::%s: %s (%3.2fs)\n" : "%s::%s: %s\n";
criterion_log(stats->failed ? CRITERION_IMPORTANT : CRITERION_INFO,
format,
stats->test->category,
stats->test->name,
stats->failed ? "FAILURE" : "SUCCESS",
stats->elapsed_time);
}
log(pre_all, set);
}
ReportHook(PRE_TEST)() {}
ReportHook(POST_FINI)() {}
ReportHook(PRE_ALL)(struct criterion_test_set *set) {
if (criterion_options.enable_tap_format) {
size_t enabled_count = 0, i = 0;
for (struct criterion_test **test = set->tests; i < set->nb_tests; ++i)
if (!(test[i])->data->disabled)
++enabled_count;
criterion_important("1.." SIZE_T_FORMAT "\n", enabled_count);
}
}
ReportHook(POST_ALL)(struct criterion_global_stats *stats) {
if (criterion_options.enable_tap_format) return;
criterion_important("Synthesis: " SIZE_T_FORMAT " tests were run. " SIZE_T_FORMAT " passed, " SIZE_T_FORMAT " failed (with " SIZE_T_FORMAT " crashes)\n",
stats->nb_tests,
stats->tests_passed,
stats->tests_failed,
stats->tests_crashed);
IMPL_REPORT_HOOK(PRE_SUITE)(struct criterion_suite_set *set) {
log(pre_suite, set);
}
ReportHook(ASSERT)(struct criterion_assert_stats *stats) {
if (criterion_options.enable_tap_format) return;
if (!stats->passed) {
criterion_important("%s:%d: Assertion failed: %s\n",
stats->file,
stats->line,
*stats->message ? stats->message : stats->condition);
}
IMPL_REPORT_HOOK(PRE_INIT)(struct criterion_test *test) {
log(pre_init, test);
}
ReportHook(TEST_CRASH)(struct criterion_test_stats *stats) {
if (criterion_options.enable_tap_format) {
criterion_important("not ok " SIZE_T_FORMAT " - %s::%s unexpected signal after %s:%u\n",
tap_test_index++,
stats->test->category,
stats->test->name,
stats->file,
stats->progress);
} else {
criterion_important("Unexpected signal after %s:%u!\n%s::%s: FAILURE (CRASH!)\n",
stats->file,
stats->progress,
stats->test->category,
stats->test->name);
}
IMPL_REPORT_HOOK(PRE_TEST)(struct criterion_test *test) {
log(pre_test, test);
}
IMPL_REPORT_HOOK(ASSERT)(struct criterion_assert_stats *stats) {
log(assert, stats);
}
IMPL_REPORT_HOOK(TEST_CRASH)(struct criterion_test_stats *stats) {
log(test_crash, stats);
}
IMPL_REPORT_HOOK(POST_TEST)(struct criterion_test_stats *stats) {
log(post_test, stats);
}
IMPL_REPORT_HOOK(POST_FINI)(struct criterion_test_stats *stats) {
log(post_fini, stats);
}
IMPL_REPORT_HOOK(POST_SUITE)(struct criterion_suite_stats *stats) {
log(post_suite, stats);
}
IMPL_REPORT_HOOK(POST_ALL)(struct criterion_global_stats *stats) {
log(post_all, stats);
}

2
src/report.h
View file

@ -33,12 +33,14 @@
void call_report_hooks_##Kind(void *data)
DECL_CALL_REPORT_HOOKS(PRE_ALL);
DECL_CALL_REPORT_HOOKS(PRE_SUITE);
DECL_CALL_REPORT_HOOKS(PRE_INIT);
DECL_CALL_REPORT_HOOKS(PRE_TEST);
DECL_CALL_REPORT_HOOKS(ASSERT);
DECL_CALL_REPORT_HOOKS(TEST_CRASH);
DECL_CALL_REPORT_HOOKS(POST_TEST);
DECL_CALL_REPORT_HOOKS(POST_FINI);
DECL_CALL_REPORT_HOOKS(POST_SUITE);
DECL_CALL_REPORT_HOOKS(POST_ALL);
#endif /* !REPORT_H_ */

157
src/runner.c
View file

@ -25,7 +25,9 @@
#include <stdio.h>
#include <unistd.h>
#include <csptr/smart_ptr.h>
#include "criterion/criterion.h"
#include "criterion/options.h"
#include "criterion/ordered-set.h"
#include "stats.h"
#include "runner.h"
#include "report.h"
@ -34,58 +36,96 @@
#include "timer.h"
IMPL_SECTION_LIMITS(struct criterion_test, criterion_tests);
IMPL_SECTION_LIMITS(struct criterion_suite, crit_suites);
static int compare_test(const void *a, const void *b) {
struct criterion_test *first = *(struct criterion_test **) a;
struct criterion_test *second = *(struct criterion_test **) b;
TestSuite(default);
// likely to happen
if (first->category == second->category) {
return strcmp(first->name, second->name);
} else {
return strcmp(first->category, second->category)
?: strcmp(first->name, second->name);
int cmp_suite(void *a, void *b) {
struct criterion_suite *s1 = a, *s2 = b;
return strcmp(s1->name, s2->name);
}
int cmp_test(void *a, void *b) {
struct criterion_test *s1 = a, *s2 = b;
return strcmp(s1->name, s2->name);
}
static void dtor_suite_set(void *ptr, UNUSED void *meta) {
struct criterion_suite_set *s = ptr;
sfree(s->tests);
}
static void dtor_test_set(void *ptr, UNUSED void *meta) {
struct criterion_test_set *t = ptr;
sfree(t->suites);
}
struct criterion_test_set *criterion_init(void) {
struct criterion_ordered_set *suites = new_ordered_set(cmp_suite, dtor_suite_set);
FOREACH_SUITE_SEC(s) {
struct criterion_suite_set css = {
.suite = *s,
};
insert_ordered_set(suites, &css, sizeof (css));
}
FOREACH_TEST_SEC(test) {
struct criterion_suite_set css = {
.suite = { .name = test->category },
};
struct criterion_suite_set *s = insert_ordered_set(suites, &css, sizeof (css));
if (!s->tests)
s->tests = new_ordered_set(cmp_test, NULL);
insert_ordered_set(s->tests, test, sizeof(*test));
}
const size_t nb_tests = SECTION_END(criterion_tests)
- SECTION_START(criterion_tests);
return unique_ptr(struct criterion_test_set, {
suites,
nb_tests,
}, dtor_test_set);
}
static void destroy_test_set(void *ptr, UNUSED void *meta) {
struct criterion_test_set *set = ptr;
free(set->tests);
}
typedef void (*f_test_run)(struct criterion_global_stats *,
struct criterion_suite_stats *,
struct criterion_test *,
struct criterion_suite *);
static struct criterion_test_set *read_all_tests(void) {
size_t nb_tests = SECTION_END(criterion_tests) - SECTION_START(criterion_tests);
static void map_tests(struct criterion_test_set *set, struct criterion_global_stats *stats, f_test_run fun) {
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
if (!s->tests)
continue;
struct criterion_test **tests = malloc(nb_tests * sizeof (void *));
if (tests == NULL)
return NULL;
report(PRE_SUITE, s);
size_t i = 0;
for (struct criterion_test *test = SECTION_START(criterion_tests); test < SECTION_END(criterion_tests); ++test)
tests[i++] = test;
smart struct criterion_suite_stats *suite_stats = suite_stats_init(&s->suite);
qsort(tests, nb_tests, sizeof (void *), compare_test);
struct event ev = { .kind = PRE_SUITE };
stat_push_event(stats, suite_stats, NULL, &ev);
return unique_ptr(struct criterion_test_set, ({
.tests = tests,
.nb_tests = nb_tests
}), destroy_test_set);
}
FOREACH_SET(struct criterion_test *t, s->tests) {
fun(stats, suite_stats, t, &s->suite);
if (criterion_options.fail_fast && stats->tests_failed > 0)
break;
if (!is_runner())
return;
}
static void map_tests(struct criterion_test_set *set, struct criterion_global_stats *stats, void (*fun)(struct criterion_global_stats *, struct criterion_test *)) {
size_t i = 0;
for (struct criterion_test **t = set->tests; i < set->nb_tests; ++i, ++t) {
fun(stats, *t);
if (!is_runner())
return;
report(POST_SUITE, suite_stats);
}
}
__attribute__ ((always_inline))
static inline void nothing() {}
static void run_test_child(struct criterion_test *test) {
static void run_test_child(struct criterion_test *test, struct criterion_suite *suite) {
send_event(PRE_INIT, NULL, 0);
if (suite->data)
(suite->data->init ?: nothing)();
(test->data->init ?: nothing)();
send_event(PRE_TEST, NULL, 0);
@ -98,22 +138,44 @@ static void run_test_child(struct criterion_test *test) {
send_event(POST_TEST, &elapsed_time, sizeof (double));
(test->data->fini ?: nothing)();
if (suite->data)
(suite->data->fini ?: nothing)();
send_event(POST_FINI, NULL, 0);
}
static void run_test(struct criterion_global_stats *stats, struct criterion_test *test) {
if (test->data->disabled)
return;
__attribute__((always_inline))
static inline bool is_disabled(struct criterion_test *t, struct criterion_suite *s) {
return t->data->disabled || (s->data && s->data->disabled);
}
#define push_event(Kind, ...) \
do { \
stat_push_event(stats, \
suite_stats, \
test_stats, \
&(struct event) { .kind = Kind, __VA_ARGS__ }); \
report(Kind, test_stats); \
} while (0)
static void run_test(struct criterion_global_stats *stats,
struct criterion_suite_stats *suite_stats,
struct criterion_test *test,
struct criterion_suite *suite) {
smart struct criterion_test_stats *test_stats = test_stats_init(test);
smart struct process *proc = spawn_test_worker(test, run_test_child);
if (is_disabled(test, suite)) {
stat_push_event(stats, suite_stats, test_stats, &(struct event) { .kind = PRE_TEST });
return;
}
smart struct process *proc = spawn_test_worker(test, suite, run_test_child);
if (proc == NULL && !is_runner())
return;
struct event *ev;
while ((ev = worker_read_event(proc)) != NULL) {
stat_push_event(stats, test_stats, ev);
stat_push_event(stats, suite_stats, test_stats, ev);
switch (ev->kind) {
case PRE_INIT: report(PRE_INIT, test); break;
case PRE_TEST: report(PRE_TEST, test); break;
@ -128,31 +190,22 @@ static void run_test(struct criterion_global_stats *stats, struct criterion_test
if (status.kind == SIGNAL) {
test_stats->signal = status.status;
if (test->data->signal == 0) {
struct event ev = { .kind = TEST_CRASH };
stat_push_event(stats, test_stats, &ev);
report(TEST_CRASH, test_stats);
push_event(TEST_CRASH);
} else {
double elapsed_time = 0;
struct event ev = { .kind = POST_TEST, .data = &elapsed_time };
stat_push_event(stats, test_stats, &ev);
report(POST_TEST, test_stats);
ev = (struct event) { .kind = POST_FINI, .data = NULL };
stat_push_event(stats, test_stats, &ev);
report(POST_FINI, test_stats);
push_event(POST_TEST, .data = &elapsed_time);
push_event(POST_FINI);
}
}
}
static int criterion_run_all_tests_impl(void) {
smart struct criterion_test_set *set = read_all_tests();
smart struct criterion_test_set *set = criterion_init();
report(PRE_ALL, set);
set_runner_pid();
smart struct criterion_global_stats *stats = stats_init();
if (!set)
abort();
map_tests(set, stats, run_test);
if (!is_runner())

15
src/runner.h
View file

@ -24,8 +24,21 @@
#ifndef CRITERION_RUNNER_H_
# define CRITERION_RUNNER_H_
# include "criterion/criterion.h"
# include "criterion/types.h"
DECL_SECTION_LIMITS(struct criterion_test, criterion_tests);
DECL_SECTION_LIMITS(struct criterion_suite, crit_suites);
struct criterion_test_set *criterion_init(void);
# define FOREACH_TEST_SEC(Test) \
for (struct criterion_test *Test = SECTION_START(criterion_tests); \
Test < SECTION_END(criterion_tests); \
++Test)
# define FOREACH_SUITE_SEC(Suite) \
for (struct criterion_suite *Suite = SECTION_START(crit_suites); \
Suite < SECTION_END(crit_suites); \
++Suite)
#endif /* !CRITERION_RUNNER_H_ */

53
src/stats.c
View file

@ -28,22 +28,36 @@
#include <assert.h>
static void nothing() {};
static void push_pre_suite();
static void push_pre_test();
static void push_assert();
static void push_post_test();
static void push_test_crash();
typedef struct criterion_global_stats s_glob_stats;
typedef struct criterion_suite_stats s_suite_stats;
typedef struct criterion_test_stats s_test_stats;
typedef struct criterion_assert_stats s_assert_stats;
static void destroy_stats(void *ptr, UNUSED void *meta) {
s_glob_stats *stats = ptr;
sfree(stats->tests);
sfree(stats->suites);
}
s_glob_stats *stats_init(void) {
return unique_ptr(s_glob_stats, ({0}), destroy_stats);
return unique_ptr(s_glob_stats, .dtor = destroy_stats);
}
static void destroy_suite_stats(void *ptr, UNUSED void *meta) {
s_suite_stats *stats = ptr;
sfree(stats->tests);
sfree(stats->next);
}
s_suite_stats *suite_stats_init(struct criterion_suite *s) {
return shared_ptr(s_suite_stats, {
.suite = s,
}, destroy_suite_stats);
}
static void destroy_test_stats(void *ptr, UNUSED void *meta) {
@ -53,39 +67,53 @@ static void destroy_test_stats(void *ptr, UNUSED void *meta) {
}
s_test_stats *test_stats_init(struct criterion_test *t) {
return shared_ptr(s_test_stats, ({
return shared_ptr(s_test_stats, {
.test = t,
.progress = t->data->line_,
.file = t->data->file_
}), destroy_test_stats);
}, destroy_test_stats);
}
void stat_push_event(s_glob_stats *stats,
s_suite_stats *suite,
s_test_stats *test,
struct event *data) {
static void (*const handles[])(s_glob_stats *, s_test_stats *, void *) = {
static void (*const handles[])(s_glob_stats *, s_suite_stats *, s_test_stats *, void *) = {
nothing, // PRE_ALL
push_pre_suite, // PRE_SUITE
nothing, // PRE_INIT
push_pre_test, // PRE_TEST
push_assert, // ASSERT
push_test_crash, // TEST_CRASH
push_post_test, // POST_TEST
nothing, // POST_FINI
nothing, // PRE_SUITE
nothing, // POST_ALL
};
assert(data->kind > 0);
assert(data->kind <= (signed long long) (sizeof (handles) / sizeof (void (*)(void))));
handles[data->kind](stats, test, data->data);
handles[data->kind](stats, suite, test, data->data);
}
static void push_pre_suite(s_glob_stats *stats,
s_suite_stats *suite,
UNUSED s_test_stats *test,
UNUSED void *ptr) {
suite->next = stats->suites;
stats->suites = sref(suite);
++stats->nb_suites;
}
static void push_pre_test(s_glob_stats *stats,
s_suite_stats *suite,
s_test_stats *test,
UNUSED void *ptr) {
test->next = stats->tests;
stats->tests = sref(test);
test->next = suite->tests;
suite->tests = sref(test);
++stats->nb_tests;
++suite->nb_tests;
}
static void destroy_assert(void *ptr, UNUSED void *meta) {
@ -94,6 +122,7 @@ static void destroy_assert(void *ptr, UNUSED void *meta) {
}
static void push_assert(s_glob_stats *stats,
s_suite_stats *suite,
s_test_stats *test,
s_assert_stats *data) {
s_assert_stats *dup = unique_ptr(s_assert_stats, (*data), destroy_assert);
@ -102,9 +131,11 @@ static void push_assert(s_glob_stats *stats,
if (data->passed) {
++stats->asserts_passed;
++suite->asserts_passed;
++test->passed_asserts;
} else {
++stats->asserts_failed;
++suite->asserts_failed;
++test->failed_asserts;
}
@ -113,21 +144,27 @@ static void push_assert(s_glob_stats *stats,
}
static void push_post_test(s_glob_stats *stats,
s_suite_stats *suite,
s_test_stats *test,
double *ptr) {
test->elapsed_time = *ptr;
if (test->failed_asserts > 0 || test->signal != test->test->data->signal) {
test->failed = 1;
++stats->tests_failed;
++suite->tests_failed;
} else {
++stats->tests_passed;
++suite->tests_passed;
}
}
static void push_test_crash(s_glob_stats *stats,
s_suite_stats *suite,
s_test_stats *test,
UNUSED void *ptr) {
test->failed = 1;
++suite->tests_failed;
++suite->tests_crashed;
++stats->tests_failed;
++stats->tests_crashed;
}

2
src/stats.h
View file

@ -29,7 +29,9 @@
struct criterion_global_stats *stats_init(void);
struct criterion_test_stats *test_stats_init(struct criterion_test *t);
struct criterion_suite_stats *suite_stats_init(struct criterion_suite *s);
void stat_push_event(struct criterion_global_stats *stats,
struct criterion_suite_stats *suite,
struct criterion_test_stats *test,
struct event *data);