[Issue #73] Merge branch 'features/json-reporting' into bleeding

CMakeLists.txt

@@ -135,6 +135,7 @@ set(SOURCE_FILES
   src/io/output.h
   src/io/tap.c
   src/io/xml.c
+  src/io/json.c
   src/log/logging.c
   src/log/normal.c
   src/string/i18n.c

doc/env.rst

@@ -29,6 +29,9 @@ Command line arguments
 * ``--xml[=FILE]``: Writes JUnit4 XML report to FILE.
   No file or ``"-"`` means ``stderr``. This option is equivalent to
   ``--output=xml:FILE``.
+* ``--json[=FILE]``: Writes a JSON report to FILE.
+  No file or ``"-"`` means ``stderr``. This option is equivalent to
+  ``--output=json:FILE``.
 * ``--verbose[=level]``: Makes the output verbose. When provided with an integer,
   sets the verbosity level to that integer.
 * ``-OPROVIDER:FILE or --output=PROVIDER:FILE``: Write a test report to FILE

samples/CMakeLists.txt

@@ -35,6 +35,7 @@ set(SAMPLES
 set(SCRIPTS
   tap_test
   xml_test
+  json_test
   early_exit
   verbose
   list

samples/tests/json_test.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+./simple.c.bin --json --always-succeed
+./signal.c.bin --json --always-succeed
+./asserts.c.bin --json --always-succeed
+./more-suites.c.bin --json --always-succeed
+./tests/long-messages.c.bin --json --always-succeed
+./description.c.bin --json --always-succeed

samples/tests/outputs/json_test.sh.err.expected

@@ -0,0 +1,237 @@
+{
+  "id": "Criterion v2.1.0",
+  "passed": 1,
+  "failed": 1,
+  "errored": 0,
+  "skipped": 0,
+  "test_suites": [
+    {
+      "name": "misc",
+      "passed": 1,
+      "failed": 1,
+      "errored": 0,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "passing",
+          "assertions": 1,
+          "status": "PASSED"
+        },
+        {
+          "name": "failing",
+          "assertions": 1,
+          "status": "FAILED",
+          "messages": [
+            "simple.c:4: The expression 0 is false."
+          ]
+        }
+      ]
+    }
+  ]
+}
+{
+  "id": "Criterion v2.1.0",
+  "passed": 1,
+  "failed": 2,
+  "errored": 1,
+  "skipped": 0,
+  "test_suites": [
+    {
+      "name": "simple",
+      "passed": 1,
+      "failed": 2,
+      "errored": 1,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "wrong_signal",
+          "assertions": 0,
+          "status": "FAILED",
+          "messages": [
+
+          ]
+        },
+        {
+          "name": "uncaught",
+          "assertions": 0,
+          "status": "ERRORED",
+          "messages": ["The test crashed."]
+        },
+        {
+          "name": "caught",
+          "assertions": 0,
+          "status": "PASSED"
+        }
+      ]
+    }
+  ]
+}
+{
+  "id": "Criterion v2.1.0",
+  "passed": 4,
+  "failed": 2,
+  "errored": 0,
+  "skipped": 0,
+  "test_suites": [
+    {
+      "name": "asserts",
+      "passed": 4,
+      "failed": 2,
+      "errored": 0,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "string",
+          "assertions": 10,
+          "status": "PASSED"
+        },
+        {
+          "name": "old_school",
+          "assertions": 2,
+          "status": "FAILED",
+          "messages": [
+            "asserts.c:18: The conditions for this assertion were not met.",
+            "asserts.c:17: You can fail an assertion with a message from anywhere"
+          ]
+        },
+        {
+          "name": "native",
+          "assertions": 8,
+          "status": "PASSED"
+        },
+        {
+          "name": "float",
+          "assertions": 2,
+          "status": "PASSED"
+        },
+        {
+          "name": "base",
+          "assertions": 6,
+          "status": "FAILED",
+          "messages": [
+            "asserts.c:12: This assert runs",
+            "asserts.c:11: assert is fatal, expect isn't"
+          ]
+        },
+        {
+          "name": "array",
+          "assertions": 3,
+          "status": "PASSED"
+        }
+      ]
+    }
+  ]
+}
+{
+  "id": "Criterion v2.1.0",
+  "passed": 2,
+  "failed": 0,
+  "errored": 0,
+  "skipped": 1,
+  "test_suites": [
+    {
+      "name": "suite2",
+      "passed": 1,
+      "failed": 0,
+      "errored": 0,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "test",
+          "assertions": 1,
+          "status": "PASSED"
+        }
+      ]
+    },
+    {
+      "name": "suite1",
+      "passed": 1,
+      "failed": 0,
+      "errored": 0,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "test",
+          "assertions": 1,
+          "status": "PASSED"
+        }
+      ]
+    },
+    {
+      "name": "disabled",
+      "passed": 0,
+      "failed": 0,
+      "errored": 0,
+      "skipped": 1,
+      "tests": [
+        {
+          "name": "test",
+          "assertions": 0,
+          "status": "SKIPPED",
+          "messages": ["The test was skipped."]
+        }
+      ]
+    }
+  ]
+}
+{
+  "id": "Criterion v2.1.0",
+  "passed": 0,
+  "failed": 1,
+  "errored": 0,
+  "skipped": 0,
+  "test_suites": [
+    {
+      "name": "sample",
+      "passed": 0,
+      "failed": 1,
+      "errored": 0,
+      "skipped": 0,
+      "tests": [
+        {
+          "name": "long_msg",
+          "assertions": 1,
+          "status": "FAILED",
+          "messages": [
+            "long-messages.c:4: This is",
+            "  A long message",
+            "  Spawning multiple lines.",
+            "  Formatting is respected."
+          ]
+        }
+      ]
+    }
+  ]
+}
+{
+  "id": "Criterion v2.1.0",
+  "passed": 0,
+  "failed": 1,
+  "errored": 0,
+  "skipped": 1,
+  "test_suites": [
+    {
+      "name": "misc",
+      "passed": 0,
+      "failed": 1,
+      "errored": 0,
+      "skipped": 1,
+      "tests": [
+        {
+          "name": "skipped",
+          "assertions": 0,
+          "status": "SKIPPED",
+          "messages": ["The test was skipped."]
+        },
+        {
+          "name": "failing",
+          "assertions": 1,
+          "status": "FAILED",
+          "messages": [
+            "description.c:4: The expression 0 is false."
+          ]
+        }
+      ]
+    }
+  ]
+}

src/core/runner.c

@@ -350,6 +350,7 @@ struct criterion_test_set *criterion_initialize(void) {
 
     criterion_register_output_provider("tap", tap_report);
     criterion_register_output_provider("xml", xml_report);
+    criterion_register_output_provider("json", json_report);
 
     return criterion_init();
 }

src/entry/main.c

@@ -142,6 +142,7 @@ int criterion_handle_args(int argc, char *argv[], bool handle_unknown_arg) {
         {"version",         no_argument,        0, 'v'},
         {"tap",             optional_argument,  0, 't'},
         {"xml",             optional_argument,  0, 'x'},
+        {"json",            optional_argument,  0, 'n'},
         {"help",            no_argument,        0, 'h'},
         {"list",            no_argument,        0, 'l'},
         {"ascii",           no_argument,        0, 'k'},
@@ -218,6 +219,7 @@ int criterion_handle_args(int argc, char *argv[], bool handle_unknown_arg) {
             case 'q': quiet = true; break;
             case 't': quiet = true; criterion_add_output("tap", DEF(optarg, "-")); break;
             case 'x': quiet = true; criterion_add_output("xml", DEF(optarg, "-")); break;
+            case 'n': quiet = true; criterion_add_output("json", DEF(optarg, "-")); break;
             case 'l': do_list_tests = true; break;
             case 'v': do_print_version = true; break;
             case 'h': do_print_usage = true; break;

src/io/json.c

@@ -0,0 +1,206 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2015 Franklin "Snaipe" Mathieu <http://snai.pe/>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "criterion/stats.h"
+#include "criterion/logging.h"
+#include "criterion/options.h"
+#include "criterion/ordered-set.h"
+#include "compat/posix.h"
+#include "compat/strtok.h"
+#include "compat/time.h"
+#include "config.h"
+#include "common.h"
+
+#define JSON_TEST_TEMPLATE_BEGIN \
+    "        {\n" \
+    "          \"name\": \"%s\",\n" \
+    "          \"assertions\": " CR_SIZE_T_FORMAT ",\n"  \
+    "          \"status\": \"%s\""
+
+#define JSON_TEST_TEMPLATE_END \
+    "\n" \
+    "        }"
+
+#define JSON_TEST_FAILED_TEMPLATE_BEGIN \
+    ",\n" \
+    "          \"messages\": [\n"
+
+#define JSON_TEST_FAILED_TEMPLATE_END \
+    "\n" \
+    "          ]"
+
+#define JSON_FAILURE_MSG_ENTRY \
+    "            \"%s:%u: %s\""
+
+#define JSON_CRASH_MSG_ENTRY \
+    ",\n" \
+    "          \"messages\": [\"The test crashed.\"]"
+
+#define JSON_TIMEOUT_MSG_ENTRY \
+    ",\n" \
+    "          \"messages\": [\"The test timed out.\"]"
+
+#define JSON_SKIPPED_MSG_ENTRY \
+    ",\n" \
+    "          \"messages\": [\"The test was skipped.\"]"
+
+#define JSON_TEST_LIST_TEMPLATE_BEGIN \
+    "      \"tests\": [\n"
+
+#define JSON_TEST_LIST_TEMPLATE_END \
+    "      ]\n"
+
+#define JSON_TESTSUITE_TEMPLATE_BEGIN \
+    "    {\n" \
+    "      \"name\": \"%s\",\n" \
+    "      \"passed\": " CR_SIZE_T_FORMAT ",\n" \
+    "      \"failed\": " CR_SIZE_T_FORMAT ",\n" \
+    "      \"errored\": " CR_SIZE_T_FORMAT ",\n" \
+    "      \"skipped\": " CR_SIZE_T_FORMAT ",\n"
+
+#define JSON_TESTSUITE_TEMPLATE_END \
+    "    }"
+
+#define JSON_TESTSUITE_LIST_TEMPLATE_BEGIN \
+    "  \"test_suites\": [\n"
+
+#define JSON_TESTSUITE_LIST_TEMPLATE_END \
+    "  ]\n"
+
+#define JSON_BASE_TEMPLATE_BEGIN \
+    "{\n" \
+    "  \"id\": \"Criterion v" VERSION "\",\n" \
+    "  \"passed\": " CR_SIZE_T_FORMAT ",\n" \
+    "  \"failed\": " CR_SIZE_T_FORMAT ",\n" \
+    "  \"errored\": " CR_SIZE_T_FORMAT ",\n" \
+    "  \"skipped\": " CR_SIZE_T_FORMAT ",\n" \
+
+#define JSON_BASE_TEMPLATE_END \
+    "}\n"
+
+static INLINE bool is_disabled(struct criterion_test *t, struct criterion_suite *s) {
+    return t->data->disabled || (s->data && s->data->disabled);
+}
+
+static CR_INLINE
+const char *get_status_string(struct criterion_test_stats *ts,
+                              struct criterion_suite_stats *ss) {
+
+    const char *status = "PASSED";
+    if (ts->crashed || ts->timed_out)
+        status = "ERRORED";
+    else if (ts->failed)
+        status = "FAILED";
+    else if (is_disabled(ts->test, ss->suite))
+        status = "SKIPPED";
+    return status;
+}
+
+static void print_test(FILE *f,
+                       struct criterion_test_stats *ts,
+                       struct criterion_suite_stats *ss) {
+
+    fprintf(f, JSON_TEST_TEMPLATE_BEGIN,
+            ts->test->name,
+            (size_t) (ts->passed_asserts + ts->failed_asserts),
+            get_status_string(ts, ss)
+        );
+
+    if (is_disabled(ts->test, ss->suite)) {
+        fprintf(f, JSON_SKIPPED_MSG_ENTRY);
+    } else if (ts->crashed) {
+        fprintf(f, JSON_CRASH_MSG_ENTRY);
+    } else if (ts->timed_out) {
+        fprintf(f, JSON_TIMEOUT_MSG_ENTRY);
+    } else if (ts->failed) {
+        fprintf(f, JSON_TEST_FAILED_TEMPLATE_BEGIN);
+
+        bool first = true;
+        for (struct criterion_assert_stats *asrt = ts->asserts; asrt; asrt = asrt->next) {
+            if (!asrt->passed) {
+                if (!first) {
+                    fprintf(f, ",\n");
+                } else {
+                    first = false;
+                }
+
+                bool sf = criterion_options.short_filename;
+                char *dup = strdup(*asrt->message ? asrt->message : "");
+                char *saveptr = NULL;
+                char *line = strtok_r(dup, "\n", &saveptr);
+
+                fprintf(f, JSON_FAILURE_MSG_ENTRY,
+                        sf ? basename_compat(asrt->file) : asrt->file,
+                        asrt->line,
+                        line
+                    );
+
+                while ((line = strtok_r(NULL, "\n", &saveptr))) {
+                    fprintf(f, ",\n            \"  %s\"", line);
+                }
+                free(dup);
+            }
+        }
+        fprintf(f, JSON_TEST_FAILED_TEMPLATE_END);
+    }
+
+    fprintf(f, JSON_TEST_TEMPLATE_END);
+}
+
+void json_report(FILE *f, struct criterion_global_stats *stats) {
+    fprintf(f, JSON_BASE_TEMPLATE_BEGIN,
+            stats->tests_passed,
+            stats->tests_failed,
+            stats->tests_crashed,
+            stats->tests_skipped
+        );
+
+    fprintf(f, JSON_TESTSUITE_LIST_TEMPLATE_BEGIN);
+    for (struct criterion_suite_stats *ss = stats->suites; ss; ss = ss->next) {
+
+        fprintf(f, JSON_TESTSUITE_TEMPLATE_BEGIN,
+                ss->suite->name,
+                ss->tests_passed,
+                ss->tests_failed,
+                ss->tests_crashed,
+                ss->tests_skipped
+            );
+
+        fprintf(f, JSON_TEST_LIST_TEMPLATE_BEGIN);
+        for (struct criterion_test_stats *ts = ss->tests; ts; ts = ts->next) {
+            print_test(f, ts, ss);
+            fprintf(f, ts->next ? ",\n" : "\n");
+        }
+        fprintf(f, JSON_TEST_LIST_TEMPLATE_END);
+
+        fprintf(f, JSON_TESTSUITE_TEMPLATE_END);
+        fprintf(f, ss->next ? ",\n" : "\n");
+    }
+    fprintf(f, JSON_TESTSUITE_LIST_TEMPLATE_END);
+
+    fprintf(f, JSON_BASE_TEMPLATE_END);
+}

src/io/output.h

@@ -31,5 +31,6 @@ void criterion_free_output(void);
 
 void tap_report(FILE *f, struct criterion_global_stats *stats);
 void xml_report(FILE *f, struct criterion_global_stats *stats);
+void json_report(FILE *f, struct criterion_global_stats *stats);
 
 #endif /* !OUTPUT_H_ */