diff --git a/CMakeLists.txt b/CMakeLists.txt
index a180082..3890768 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -192,23 +192,23 @@ endif ()
 set(INTERFACE_FILES
   include/criterion/assert.h
   include/criterion/abort.h
-  include/criterion/common.h
   include/criterion/criterion.h
   include/criterion/event.h
   include/criterion/hooks.h
   include/criterion/logging.h
   include/criterion/types.h
   include/criterion/options.h
-  include/criterion/ordered-set.h
   include/criterion/stats.h
   include/criterion/theories.h
-  include/criterion/asprintf-compat.h
-  include/criterion/designated-initializer-compat.h
-  include/criterion/preprocess.h
   include/criterion/alloc.h
   include/criterion/parameterized.h
   include/criterion/redirect.h
   include/criterion/output.h
+  include/criterion/internal/common.h
+  include/criterion/internal/ordered-set.h
+  include/criterion/internal/asprintf-compat.h
+  include/criterion/internal/designated-initializer-compat.h
+  include/criterion/internal/preprocess.h
 )
 
 # Generate the configure file
diff --git a/dependencies/valgrind/include/valgrind/valgrind.h b/dependencies/valgrind/include/valgrind/valgrind.h
index 9cfb700..04c73ec 100644
--- a/dependencies/valgrind/include/valgrind/valgrind.h
+++ b/dependencies/valgrind/include/valgrind/valgrind.h
@@ -93,7 +93,7 @@
 
 
 #include <stdarg.h>
-#include <criterion/common.h>
+#include <criterion/internal/common.h>
 
 /* Nb: this file might be included in a file compiled with -ansi.  So
    we can't use C++ style "//" comments nor the "asm" keyword (instead
diff --git a/include/criterion/abort.h b/include/criterion/abort.h
index 449c0de..d6a649e 100644
--- a/include/criterion/abort.h
+++ b/include/criterion/abort.h
@@ -24,7 +24,7 @@
 #ifndef CRITERION_ABORT_H_
 # define CRITERION_ABORT_H_
 
-# include "common.h"
+# include "internal/common.h"
 
 CR_BEGIN_C_API
 
diff --git a/include/criterion/alloc.h b/include/criterion/alloc.h
index eb68794..71d4ca7 100644
--- a/include/criterion/alloc.h
+++ b/include/criterion/alloc.h
@@ -31,7 +31,7 @@ using std::size_t;
 # else
 #  include <stddef.h>
 # endif
-# include "common.h"
+# include "internal/common.h"
 
 CR_BEGIN_C_API
 
diff --git a/include/criterion/assert_base.h b/include/criterion/assert_base.h
index e390518..f539598 100644
--- a/include/criterion/assert_base.h
+++ b/include/criterion/assert_base.h
@@ -24,8 +24,8 @@
 #ifndef CRITERION_ASSERT_BASE_H_
 # define CRITERION_ASSERT_BASE_H_
 
-# include "preprocess.h"
-# include "asprintf-compat.h"
+# include "internal/preprocess.h"
+# include "internal/asprintf-compat.h"
 
 # ifdef __cplusplus
 #  include <cstring>
@@ -35,7 +35,7 @@
 #  include <stdlib.h>
 #  include <stdbool.h>
 # endif
-# include "designated-initializer-compat.h"
+# include "internal/designated-initializer-compat.h"
 # include "types.h"
 # include "stats.h"
 # include "hooks.h"
diff --git a/include/criterion/criterion.h b/include/criterion/criterion.h
index a2f4ae2..9549223 100644
--- a/include/criterion/criterion.h
+++ b/include/criterion/criterion.h
@@ -24,8 +24,8 @@
 #ifndef CRITERION_H_
 # define CRITERION_H_
 
-# include "designated-initializer-compat.h"
-# include "common.h"
+# include "internal/designated-initializer-compat.h"
+# include "internal/common.h"
 # include "types.h"
 # include "assert.h"
 # include "alloc.h"
diff --git a/include/criterion/event.h b/include/criterion/event.h
index 6ccc649..6d2d3a0 100644
--- a/include/criterion/event.h
+++ b/include/criterion/event.h
@@ -29,7 +29,7 @@
 # else
 #  include <stddef.h>
 # endif
-# include "common.h"
+# include "internal/common.h"
 
 CR_BEGIN_C_API
 
diff --git a/include/criterion/hooks.h b/include/criterion/hooks.h
index 497b047..5f622b2 100644
--- a/include/criterion/hooks.h
+++ b/include/criterion/hooks.h
@@ -24,7 +24,7 @@
 #ifndef CRITERION_HOOKS_H_
 # define CRITERION_HOOKS_H_
 
-# include "common.h"
+# include "internal/common.h"
 # include "types.h"
 
 typedef enum {
diff --git a/include/criterion/asprintf-compat.h b/include/criterion/internal/asprintf-compat.h
similarity index 100%
rename from include/criterion/asprintf-compat.h
rename to include/criterion/internal/asprintf-compat.h
diff --git a/include/criterion/common.h b/include/criterion/internal/common.h
similarity index 100%
rename from include/criterion/common.h
rename to include/criterion/internal/common.h
diff --git a/include/criterion/designated-initializer-compat.h b/include/criterion/internal/designated-initializer-compat.h
similarity index 100%
rename from include/criterion/designated-initializer-compat.h
rename to include/criterion/internal/designated-initializer-compat.h
diff --git a/include/criterion/ordered-set.h b/include/criterion/internal/ordered-set.h
similarity index 98%
rename from include/criterion/ordered-set.h
rename to include/criterion/internal/ordered-set.h
index 07c5148..aa93cc1 100644
--- a/include/criterion/ordered-set.h
+++ b/include/criterion/internal/ordered-set.h
@@ -24,7 +24,7 @@
 #ifndef CRITERION_ORDERED_SET_H_
 # define CRITERION_ORDERED_SET_H_
 
-# include "types.h"
+# include "../types.h"
 
 typedef int (*f_criterion_cmp)(void *, void *);
 
diff --git a/include/criterion/preprocess.h b/include/criterion/internal/preprocess.h
similarity index 100%
rename from include/criterion/preprocess.h
rename to include/criterion/internal/preprocess.h
diff --git a/include/criterion/logging.h b/include/criterion/logging.h
index 3fbc27c..ab78034 100644
--- a/include/criterion/logging.h
+++ b/include/criterion/logging.h
@@ -31,8 +31,8 @@ using std::va_list;
 #  include <stdbool.h>
 #  include <stdarg.h>
 # endif
-# include "common.h"
-# include "ordered-set.h"
+# include "internal/common.h"
+# include "internal/ordered-set.h"
 # include "stats.h"
 
 enum criterion_logging_level {
diff --git a/include/criterion/redirect.h b/include/criterion/redirect.h
index effb909..c00f42f 100644
--- a/include/criterion/redirect.h
+++ b/include/criterion/redirect.h
@@ -24,7 +24,7 @@
 #ifndef CRITERION_REDIRECT_H_
 # define CRITERION_REDIRECT_H_
 
-# include "common.h"
+# include "internal/common.h"
 # include "assert.h"
 
 # ifdef __cplusplus
diff --git a/include/criterion/types.h b/include/criterion/types.h
index c2c40a2..905b6ac 100644
--- a/include/criterion/types.h
+++ b/include/criterion/types.h
@@ -33,7 +33,7 @@ using std::size_t;
 #  include <stdbool.h>
 #  include <stddef.h>
 # endif
-# include "common.h"
+# include "internal/common.h"
 
 enum criterion_language {
     CR_LANG_C,
diff --git a/src/compat/section.h b/src/compat/section.h
index 7ff0755..c461ebf 100644
--- a/src/compat/section.h
+++ b/src/compat/section.h
@@ -24,7 +24,7 @@
 #ifndef SECTION_H_
 # define SECTION_H_
 
-# include "criterion/common.h"
+# include "criterion/internal/common.h"
 
 # ifdef _WIN32
 void *get_win_section_start(const char *section);
diff --git a/src/compat/time.c b/src/compat/time.c
index f8c8fee..2254d50 100644
--- a/src/compat/time.c
+++ b/src/compat/time.c
@@ -1,7 +1,7 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <stdlib.h>
-#include "criterion/common.h"
+#include "criterion/internal/common.h"
 #include "criterion/options.h"
 #include "compat/time.h"
 #include "compat/posix.h"
diff --git a/src/core/abort.c b/src/core/abort.c
index 0d32188..3fc3518 100644
--- a/src/core/abort.c
+++ b/src/core/abort.c
@@ -23,7 +23,7 @@
  */
 #include <string.h>
 #include "abort.h"
-#include "criterion/asprintf-compat.h"
+#include "criterion/internal/asprintf-compat.h"
 #include "io/event.h"
 
 jmp_buf g_pre_test;
diff --git a/src/core/ordered-set.c b/src/core/ordered-set.c
index c7c6f38..dde5d3c 100644
--- a/src/core/ordered-set.c
+++ b/src/core/ordered-set.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include <string.h>
-#include <criterion/common.h>
-#include <criterion/ordered-set.h>
+#include <criterion/internal/common.h>
+#include <criterion/internal/ordered-set.h>
 #include <csptr/smalloc.h>
 #include "common.h"
 
diff --git a/src/core/report.c b/src/core/report.c
index c91e780..f6c2398 100644
--- a/src/core/report.c
+++ b/src/core/report.c
@@ -28,7 +28,7 @@
 #include "criterion/stats.h"
 #include "criterion/logging.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "report.h"
 #include "config.h"
 #include "compat/posix.h"
diff --git a/src/core/runner.c b/src/core/runner.c
index fe20ef5..38a85fc 100644
--- a/src/core/runner.c
+++ b/src/core/runner.c
@@ -29,9 +29,9 @@
 #include <valgrind/valgrind.h>
 #include "criterion/criterion.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "criterion/logging.h"
-#include "criterion/preprocess.h"
+#include "criterion/internal/preprocess.h"
 #include "compat/time.h"
 #include "compat/posix.h"
 #include "compat/processor.h"
diff --git a/src/core/stats.c b/src/core/stats.c
index bb10bac..f8d46cd 100644
--- a/src/core/stats.c
+++ b/src/core/stats.c
@@ -23,7 +23,7 @@
  */
 #include <string.h>
 #include <csptr/smalloc.h>
-#include "criterion/common.h"
+#include "criterion/internal/common.h"
 #include "stats.h"
 #include "common.h"
 
diff --git a/src/entry/main.c b/src/entry/main.c
index 6e561de..792ff79 100644
--- a/src/entry/main.c
+++ b/src/entry/main.c
@@ -28,7 +28,7 @@
 #include <csptr/smalloc.h>
 #include "criterion/criterion.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "compat/posix.h"
 #include "compat/strtok.h"
 #include "core/runner.h"
diff --git a/src/io/asprintf.c b/src/io/asprintf.c
index c2a4a12..f340b1d 100644
--- a/src/io/asprintf.c
+++ b/src/io/asprintf.c
@@ -25,7 +25,7 @@
 #include <limits.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include "criterion/asprintf-compat.h"
+#include "criterion/internal/asprintf-compat.h"
 
 int cr_asprintf(char **strp, const char *fmt, ...) {
     va_list ap;
diff --git a/src/io/event.c b/src/io/event.c
index 9720426..8d45c4b 100644
--- a/src/io/event.c
+++ b/src/io/event.c
@@ -26,7 +26,7 @@
 #include <string.h>
 #include <csptr/smalloc.h>
 #include "criterion/stats.h"
-#include "criterion/common.h"
+#include "criterion/internal/common.h"
 #include "criterion/hooks.h"
 #include "criterion/logging.h"
 #include "core/worker.h"
diff --git a/src/io/json.c b/src/io/json.c
index a893e57..ea60073 100644
--- a/src/io/json.c
+++ b/src/io/json.c
@@ -28,7 +28,7 @@
 #include "criterion/stats.h"
 #include "criterion/logging.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "compat/posix.h"
 #include "compat/strtok.h"
 #include "compat/time.h"
diff --git a/src/io/xml.c b/src/io/xml.c
index 3bce465..a2cf30f 100644
--- a/src/io/xml.c
+++ b/src/io/xml.c
@@ -28,7 +28,7 @@
 #include "criterion/stats.h"
 #include "criterion/logging.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "compat/posix.h"
 #include "compat/strtok.h"
 #include "compat/time.h"
diff --git a/src/log/normal.c b/src/log/normal.c
index 9789d27..f6221e0 100644
--- a/src/log/normal.c
+++ b/src/log/normal.c
@@ -29,7 +29,7 @@
 #include "criterion/stats.h"
 #include "criterion/logging.h"
 #include "criterion/options.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 #include "compat/posix.h"
 #include "compat/strtok.h"
 #include "compat/time.h"
diff --git a/src/string/extmatch.c b/src/string/extmatch.c
index 3af6da9..de63088 100644
--- a/src/string/extmatch.c
+++ b/src/string/extmatch.c
@@ -27,7 +27,7 @@
 #include <setjmp.h>
 
 #include <stdio.h>
-#include "criterion/common.h"
+#include "criterion/internal/common.h"
 #include "common.h"
 
 struct context {
diff --git a/src/string/i18n.c b/src/string/i18n.c
index ecc2e4d..da55950 100644
--- a/src/string/i18n.c
+++ b/src/string/i18n.c
@@ -1,6 +1,6 @@
 #include "i18n.h"
 #include "criterion/assert.h"
-#include "criterion/asprintf-compat.h"
+#include "criterion/internal/asprintf-compat.h"
 
 void init_i18n(void) {
 #if ENABLE_NLS
diff --git a/test/asprintf.c b/test/asprintf.c
index 0fa34d9..f709bd0 100644
--- a/test/asprintf.c
+++ b/test/asprintf.c
@@ -1,6 +1,6 @@
 #include "criterion/criterion.h"
 #include "criterion/theories.h"
-#include "criterion/asprintf-compat.h"
+#include "criterion/internal/asprintf-compat.h"
 
 #include <stdio.h>
 
diff --git a/test/ordered-set.c b/test/ordered-set.c
index 3ecdbf1..92a4d28 100644
--- a/test/ordered-set.c
+++ b/test/ordered-set.c
@@ -1,7 +1,7 @@
 #include <csptr/smalloc.h>
 
 #include "criterion/criterion.h"
-#include "criterion/ordered-set.h"
+#include "criterion/internal/ordered-set.h"
 
 int compare_gt(void *a, void *b) {
     int *ia = a, *ib = b;