1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00

cmake: initial support for CMake build system

This commit is contained in:
Daniel Krebs 2017-04-03 18:08:12 +02:00
parent eca8be0236
commit 30021d5291
34 changed files with 1501 additions and 197 deletions

View file

@ -19,13 +19,17 @@
},
"files":
[{
"includePattern": "../(libhermit[^/]*deb$)", "uploadPattern": "$1",
[
{
"includePattern": "build/(libhermit[^/]*deb$)", "uploadPattern": "$1",
"matrixParams": {
"deb_distribution": "vivid",
"deb_component": "main",
"deb_architecture": "amd64",
"override": 1}
}],
"override": 1}
},
{"includePattern": "build/(libhermit[^/]*rpm$)", "uploadPattern": "$1", "override": 1}
{"includePattern": "build/(libhermit[^/]*tar.bz2$)", "uploadPattern": "$1", "override": 1}
],
"publish": true
}

2
.gitignore vendored
View file

@ -1,7 +1,7 @@
*.pcap
*.config
*.creator
*.creator.user
*.user
*.files
*.includes
*.pyc

View file

@ -1,26 +1,22 @@
sudo: required
dist: trusty
git:
submodules: false
submodules: true
language: c
compiler: gcc
before_install:
- echo "deb https://dl.bintray.com/rwth-os/hermitcore vivid main" | sudo tee -a /etc/apt/sources.list
- travis_retry sudo apt-get -qq update
- travis_retry sudo apt-get install -y curl qemu-system-x86 nasm texinfo libmpfr-dev libmpc-dev libgmp-dev libisl-dev flex bison packaging-dev
#- sudo apt-get install -y --force-yes binutils-hermit gcc-hermit-bootstrap
- travis_retry sudo apt-get install -y curl wget qemu-system-x86 nasm texinfo libmpfr-dev libmpc-dev libgmp-dev libisl-dev flex bison packaging-dev rpm
- travis_retry sudo apt-get install -y --force-yes binutils-hermit libhermit newlib-hermit pthread-embedded-hermit gcc-hermit
- git submodule update --init lwip usr/libomp
script:
- cd ..
- mv HermitCore libhermit-0.1
- tar -czf libhermit_0.1.orig.tar.gz libhermit-0.1 --exclude=.git
- cd libhermit-0.1
- debuild -e PATH -e CFLAGS_FOR_TARGET -e GOFLAGS_FOR_TARGET -e FCFLAGS_FOR_TARGET -e FFLAGS_FOR_TARGET -e CXXFLAGS_FOR_TARGET -us -uc -j2
- sudo dpkg -i ../libhermit*_amd64.deb
- make -j2 examples
- make test
- source cmake/local-cmake.sh
- mkdir build
- cd build
- cmake ..
- make -j1 package
deploy:
on: master
@ -30,16 +26,3 @@ deploy:
secure: wo0yLY7xhGZYDqdB05UE+IOXXgYhAwj+zvtZh3ET2253hy35y74cDKMBNGfynH0aauPk8EFmN/LJoYaV/T9QF6pc1bilbqSg1/LJuL7hjAVVFaBSjwlE/Wbbb+EHNX5CR3qmC42SUvbrU+0WNvRu7WJBpZIoH9EtPv2Cj7uRV9+7Mtp/O3Ykl3E5LgkF5iA6Lo03TN+bZ1Vog5MbuJagDTpNa18vJ3jVYaed5gqOkghYYDpc5U9yxN3debS7/8MCl3u9V9uanMAyjc6wtbvJLotVkUwdXK8u9FohuWQ4pTEQ/QspLDDBuxaS3kLNWtzBMNQn6rcYIOWWKyD3uqvIhqjcErX7WChf3oGS1XPFfTHebNrHyV4KGvzkMAJUR/1qC5f4nZCsvgPv/35d702nky3Di2/WZsjD5zUR9g3+vPO8mnM3z8eA6ShVJcxezZ73Co3p0ZnBaF3J438bAKiT4nlrbIVpRzZzo/5nYTx//SefXSAmYCILr/y7xTYBqjfjMQHQZEMZSvJYRPbtP7eVCfWGEa2tG4rQndegUtwS+n1r8atcESL9eeUUM6sa1QzZYDYVseOc9GWcB7SIJzPOm8QTuTlkISHMGJ8FY+KHorP15VkVlr0EZ5c7KY0cBIbWoZGNQTg9+nSgChyf7ElNbAxPiCZsR0zafjMDhfcnzco=
key:
secure: JfsqEUO3Z60yGfuK5RSzwMoWZtaYflZtW7QE6R1DVMPEQ+CytzEdV2JaTpY14xz4yz1YpBBuQ0P3Q3e2rf/ORp8N8j7/5m3gfiDi8bRH3gX10r6vCQaUBilj0pz3amWUacxwBUEYR/f1029OnJ1qug30f4ARk7DWyuAePt0OboDXZ3j4JOi8xfXKTzofyKGugU4EuzhmKAbpHaBoX97g8z+gETC+wsBEYio8iD2h0ZOe/qZ0S+JGYkphIKcxpQazKdi3YrmWm0BUZsQRtkgoH7KUZm8vqfOUyVOrK+UGOTz4vXqCWHZ+wG1QRrGPUv8ehLrB26y2o02mmaDWQhM+I3RtllL06JDvDw40xjRImYtzg6xr7Mvl0OTQHprXrkN8gw2IbIivV8v31O46Ov+KIaN3CCx1IncnllBWjEXAIs4zPtvNj2Ad338JTkI/opHmPG0DI0DKE36r8wPZYTs/pHVpc3xEzwKYDklJkICjMLUakUGKppS7eKzKGRMjWvbT0vM/U7hHUcz0lA+BUoXedNmJQ1wBT85Ud8uobuKS4C8QmlgIuF1PI2+6LJr5LsCGZRvg7Pl1SPc3ZQLPHX4ggoLAnZZJiV/0ZPCn7XCLiUS1qws37l0uZT1zJQMFLsw9MGuP58tpT7WDuYYAwma/pL+OKC/JKoDhuJwM1I7wB4s=
env:
global:
- PATH=$PATH:/opt/hermit/bin/
- HERMIT_ISLE=qemu
- HERMIT_CPUS=1
- HERMIT_MEM="512M"
- HERMIT_KVM="0"
- HERMIT_VERBOSE="1"
- CFLAGS_FOR_TARGET="-m64 -O3 -ftree-vectorize"
- GOFLAGS_FOR_TARGET"=-m64 -O3 -ftree-vectorize"
- FCFLAGS_FOR_TARGET"=-m64 -O3 -ftree-vectorize"
- FFLAGS_FOR_TARGET="-m64 -O3 -ftree-vectorize"
- CXXFLAGS_FOR_TARGET="-m64 -O3 -ftree-vectorize"

196
CMakeLists.txt Normal file
View file

@ -0,0 +1,196 @@
cmake_minimum_required(VERSION 3.7)
include(ExternalProject)
include(cmake/HermitCore.cmake)
project (HermitCore)
### Kernel
# generate config files
add_subdirectory(include/hermit)
add_kernel_module_sources("kernel" "kernel/*.c")
add_kernel_module_sources("libkern" "libkern/*.c")
add_kernel_module_sources("mm" "mm/*.c")
add_kernel_module_sources("drivers" "drivers/net/*.c")
set(LWIP_SRC lwip/src)
add_kernel_module_sources("lwip" "${LWIP_SRC}/api/*.c")
add_kernel_module_sources("lwip" "${LWIP_SRC}/arch/*.c")
add_kernel_module_sources("lwip" "${LWIP_SRC}/core/*.c")
add_kernel_module_sources("lwip" "${LWIP_SRC}/core/ipv4/*.c")
add_kernel_module_sources("lwip" "${LWIP_SRC}/core/ipv6/*.c")
add_kernel_module_sources("lwip" "${LWIP_SRC}/netif/*.c")
get_kernel_modules(KERNEL_MODULES)
foreach(MODULE ${KERNEL_MODULES})
get_kernel_module_sources(SOURCES ${MODULE})
# maintain list of all objects that will end up in libhermit.a
list(APPEND KERNEL_OBJECTS $<TARGET_OBJECTS:${MODULE}>)
add_library(${MODULE} OBJECT ${SOURCES})
# this is kernel code
target_compile_definitions(${MODULE}
PRIVATE -D__KERNEL__)
target_compile_options(${MODULE}
PRIVATE ${HERMIT_KERNEL_FLAGS})
target_include_directories(${MODULE}
PUBLIC ${HERMIT_KERNEL_INCLUDES})
# suppress all LwIP compiler warnings. Not our code, so we cannot fix
if("${MODULE}" STREQUAL "lwip")
target_compile_options(${MODULE}
PRIVATE -w)
endif()
endforeach()
# add arch/x86 and its objects
# TODO: make this conditional when new architectures are implemented
add_subdirectory(arch/x86)
list(APPEND KERNEL_OBJECTS
$<TARGET_OBJECTS:${X86_KERNEL_ASM_TARGET}>
$<TARGET_OBJECTS:${X86_KERNEL_C_TARGET}>)
# finally build libhermit.a
add_library(hermit STATIC ${KERNEL_OBJECTS})
# after compiling ASM sources, we need to post-process them. Adding this
# dependency makes sure that this is done before hermit is linked
add_dependencies(hermit ${X86_KERNEL_TARGET})
add_custom_command(
TARGET
hermit POST_BUILD
# rename sections in final library
COMMAND
${CMAKE_OBJCOPY} --rename-section .bss=.kbss
--rename-section .text=.ktext
--rename-section .data=.kdata
$<TARGET_FILE:hermit>
# copy libhermit.a into local prefix directory so that all subsequent
# targets can link against the freshly built version (as opposed to
# linking against the one supplied by the toolchain)
COMMAND
${CMAKE_COMMAND} -E make_directory ${LOCAL_PREFIX_ARCH_LIB_DIR}
COMMAND
${CMAKE_COMMAND} -E copy_if_different
$<TARGET_FILE:hermit>
${LOCAL_PREFIX_ARCH_LIB_DIR}/
# and also copy headers into local prefix
COMMAND
${CMAKE_COMMAND} -E make_directory ${LOCAL_PREFIX_ARCH_INCLUDE_DIR}/hermit
COMMAND
${CMAKE_COMMAND} -E copy_if_different
${CMAKE_BINARY_DIR}/include/hermit/*.h
${LOCAL_PREFIX_ARCH_INCLUDE_DIR}/hermit/
COMMAND
${CMAKE_COMMAND} -E copy_if_different
${CMAKE_BINARY_DIR}/include/hermit/*.asm
${LOCAL_PREFIX_ARCH_INCLUDE_DIR}/hermit/)
# deploy libhermit.a and headers for package creation
install(TARGETS hermit
DESTINATION ${TARGET_ARCH}/lib)
install(DIRECTORY include/hermit
DESTINATION ${TARGET_ARCH}/include/)
### External projects
#
# Build projects externally and deploy into temporary common prefix, will later
# be relocated for installation
## HermitCore's own tools such as Qemu/KVM proxy
build_external(tools ${HERMIT_ROOT}/tools "")
build_external(arch_x86_loader ${HERMIT_ROOT}/arch/x86/loader "")
## Intel's OpenMP runtime for x86 (libomp)
build_external(libiomp ${HERMIT_ROOT}/usr/libomp ""
-DHERMIT=1
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/${TARGET_ARCH})
# libomp is part of HermitCore's runtime and should be available before any
# application will link
add_dependencies(hermit libiomp)
## iRCCE
build_external(ircce ${HERMIT_ROOT}/usr/ircce "")
add_dependencies(hermit ircce)
## XRay profiler
build_external(xray ${HERMIT_ROOT}/usr/xray "")
add_dependencies(hermit xray)
## Tests and benchmarks
build_external(tests ${HERMIT_ROOT}/usr/tests hermit)
build_external(benchmarks ${HERMIT_ROOT}/usr/benchmarks hermit)
build_external(openmpbench ${HERMIT_ROOT}/usr/openmpbench hermit)
## relocate the local prefix to our install destination
install(DIRECTORY ${LOCAL_PREFIX_DIR}/
DESTINATION ${CMAKE_INSTALL_PREFIX}/
USE_SOURCE_PERMISSIONS)
### QEmu
# Start HermitCore as multi-kernel in a QEmu VM
add_custom_target(qemu
COMMAND
qemu-system-x86_64
-machine accel=kvm -cpu host
-smp 10 -m 8G -numa node,nodeid=0,cpus=0-4 -numa node,nodeid=1,cpus=5-9
-kernel ${HERMIT_ROOT}/config/bzImage
-append "root=/dev/ram0 rootfstype=ramfs init=init console=ttyS0"
-net nic,model=rtl8139 -net user -net dump
-nographic -monitor telnet:127.0.0.1:1235,server,nowait
-fsdev local,security_model=none,id=fsdev0,path=${LOCAL_PREFIX_DIR}
-device virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=hermit
-s
USES_TERMINAL VERBATIM)
# create a QEmu target that depends on everything
get_property(_TARGETS
DIRECTORY .
PROPERTY BUILDSYSTEM_TARGETS)
add_custom_target(qemu-dep
DEPENDS
${_TARGETS} qemu)
### Packaging
set(CPACK_PACKAGE_NAME libhermit)
set(CPACK_SYSTEM_NAME all)
set(CPACK_PACKAGE_VERSION_MAJOR 0)
set(CPACK_PACKAGE_VERSION_MINOR 1)
set(CPACK_PACKAGE_VERSION_PATCH 0)
set(CPACK_PACKAGE_CONTACT "Daniel Krebs <github@daniel-krebs.net>")
# build .deb, .rpm and .tar.bz2 packages
set(CPACK_GENERATOR DEB;RPM;TBZ2)
# needed in order for tests and bechmark to use correct install prefix
set(CPACK_SET_DESTDIR on)
## Debian specific
# not dependent on Debian system architecture
set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE all)
## RPM specific
# libhermit is currently not relocatable
set(CPACK_PACKAGE_RELOCATABLE FALSE)
include(CPack)

425
README.md
View file

@ -1,91 +1,268 @@
# HermitCore - A lightweight unikernel for a scalable and predictable runtime behavior
The project [HermitCore](http://www.hermitcore.org) is new [unikernel](http://unikernel.org) targeting at a scalable and predictable runtime for high-performance and cloud computing.
HermitCore extends the multi-kernel approach (like [McKernel](http://www-sys-aics.riken.jp/ResearchTopics/os/mckernel.html)) with unikernel features for a better programmability and scalability for hierarchical systems.
On the startup of HermitCore applications, cores are isolated from the Linux system enabling the bare-metal of the applications on these cores.
This approach achieves lower OS jitter and a better scalability compared to full-weight kernels.
Inter-kernel communication between HermitCore applications and the Linux system is realized by means of an IP interface.
The project [HermitCore]( http://www.hermitcore.org ) is a new
[unikernel](http://unikernel.org) targeting a scalable and predictable runtime
for high-performance and cloud computing. HermitCore extends the multi-kernel
approach (like
[McKernel](http://www-sys-aics.riken.jp/ResearchTopics/os/mckernel.html)) with
unikernel features for a better programmability and scalability for hierarchical
systems.
On the startup of HermitCore applications, cores are isolated from the Linux
system enabling bare-metal execution of on these cores. This approach achieves
lower OS jitter and a better scalability compared to full-weight kernels.
Inter-kernel communication between HermitCore applications and the Linux system
is realized by means of an IP interface.
In addition to the multi-kernel approach described above, HermitCore can be used
as a classical standalone unikernel as well. In this case, HermitCore runs a
single-kernel exclusively on the hardware or within a virtual machine. This
reduces the resource demand and loweres the boot time which is critical for
cloud computing applications. It is the result of a research project at RWTH
Aachen University and is currently an experimental approach, i.e., not
production ready. Please use it with caution.
In addition to the multi-kernel approach described above, HermitCore can be used as classical standalone unikernel as well.
In this case HermitCore runs a single-kernel exclusively on the hardware or within a virtual machine.
This reduces the resource demand and improves the boot time which is critical for cloud computing applications.
It is the result of a research project at RWTH Aachen University and is currently an experimental approach, i.e., not production ready.
Please use it with caution.
## Requirements
The build process works currently only on **x86-based Linux** systems. The following software packets are required to build HermitCore on a Linux system:
The build process works currently only on **x86-based Linux** systems. To build
the HermitCore kernel and applications you need:
* Netwide Assembler (NASM)
* GNU Make, GNU Binutils
* Tools and libraries to build *linux*, *binutils* and *gcc* (e.g. flex, bison, MPFR library, GMP library, MPC library, ISL library)
* texinfo
* Qemu
* CMake
* Netwide Assember (NASM)
* recent host compiler such as GCC
* HermitCore cross-toolchain, i.e. Binutils, GCC, newlib, pthreads-embedded
On Debian-based systems the packets can be installed by executing:
```
sudo apt-get install qemu-system-x86 nasm texinfo libmpfr-dev libmpc-dev libgmp-dev libisl-dev flex bison
### HermitCore cross-toolchain
We provide prebuilt packages (currently Debian-based only) of the HermitCore
toolchain, which can be installed as follows:
```bash
$ echo "deb [trusted=yes] https://dl.bintray.com/rwth-os/hermitcore vivid main" | sudo tee -a /etc/apt/sources.list
$ sudo apt-get -qq update
$ sudo apt-get install binutils-hermit newlib-hermit pthread-embedded-hermit gcc-hermit libhermit
```
## Installing HermitCore with by using debian packets
If you want to build the toolchain yourself, have a look at the following
repositories, especially at `debian/rules` in each repository:
We provide binary packets for debian-based systems containing the complete HermitCore toolchain including a cross-compiler.
To install the packets you have to execute the following commands:
* [GCC](https://github.com/RWTH-OS/gcc)
* [Binutils](https://github.com/RWTH-OS/binutils)
* [Newlib](https://github.com/RWTH-OS/newlib)
* [Pthread-embedded](https://github.com/RWTH-OS/pthread-embedded)
Depending on how you want to use HermitCore, you might need additional packages
such as:
* QEMU (`apt-get install qemu-system-x86`)
## CMake requirements
We require a fairly recent version of CMake (`3.7`) which is not yet present in
most Linux distributions. We therefore provide a helper script that fetches the
required CMake binaries from the upstream project and stores them locally, so
you only need to download it once.
```bash
$ . cmake/local-cmake.sh
-- Downloading CMake
--2017-03-28 16:13:37-- https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.tar.gz
Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'
Resolving cmake.org... 66.194.253.19
Connecting to cmake.org|66.194.253.19|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 30681434 (29M) [application/x-gzip]
Saving to: cmake-3.7.2-Linux-x86_64.tar.gz
cmake-3.7.2-Linux-x86_64.tar.gz 100%[===================>] 29,26M 3,74MB/s in 12s
2017-03-28 16:13:50 (2,48 MB/s) - cmake-3.7.2-Linux-x86_64.tar.gz saved [30681434/30681434]
-- Unpacking CMake
-- Local CMake v3.7.2 installed to cmake/cmake-3.7.2-Linux-x86_64
-- Next time you source this script, no download will be neccessary
```
echo "deb [trusted=yes] https://dl.bintray.com/rwth-os/hermitcore vivid main" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qq update
sudo apt-get install binutils-hermit newlib-hermit pthread-embedded-hermit gcc-hermit libhermit
So before you build HermitCore you have to source the `local-cmake.sh` script
everytime you open a new terminal.
## Building HermitCore
```bash
$ mkdir build
$ cd build
$ cmake ..
$ make
```
This toolchain is able to build applications for [classical unikernel](#building-and-testing-hermitcore-as-classical-standalone-unikernel) environments within virtual machines or bare-metal in a multi-kernel environment.
For the latter, you have to install the modified Linux kernel.
An introduction to this execution mode is provided in section [Building and testing HermitCore as multi-kernel on a real machine](#building-and-testing-hermitcore-as-multi-kernel-on a-real-machine).
## Building and testing HermitCore as multi-kernel within a virtual machine
If your toolchain is not located in `/opt/hermit/bin` then you have to supply
its location to the `cmake` command above like so:
1. Please make sure that you cloned this repository and all its submodules.
2. To configure the system, run the *configure* script in the directory, which contains this *README*.
With the flag `--with-toolchain`, the HermitCore's complete cross toolchain (cross compiler, binutils, etc.) will be downloaded and built.
**NOTE**: This requires write access to the installation directory, which is specified by the flag `--prefix`.
At the end of this *README* in section *Tips* you find hints to enable optimization for the target.
3. The command `make all` build the the HermitCore kernel and depending on the configuration flags the cross toolchain.
4. Install the kernel with `make install`.
5. Build all example applications with `make examples`.
6. To start a virtual machine and to boot a small Linux version use the command `make qemu`.
Per default, the virtual machine has 10 cores, 2 NUMA nodes, and 8 GiB RAM.
To increase or to decrease the machine size, the label `qemu` in the Makefile has to be modified accordingly.
7. Inside the VM runs a small Linux system, which already includes the patches for HermitCore.
Per NUMA node (= HermitCore isle) there is a directory called `isleX` under `/sys/hermit` , where `X` represents the NUMA node ID.
The demo applications are located in the directories `/hermit/usr/{tests,benchmarks}`.
A HermitCore loader is already registered.
By starting a HermitCore application, a proxy will be executed on the Linux system, while the HermitCore binary will be started on isle 0 with cpu 1.
To change the default behavior, the environment variable `HERMIT_ISLE` is used to specify the (memory) location of the isle, while the environment variable `HERMIT_CPUS` is used to specify the cores.
For instance, `HERMIT_ISLE=1 HERMIT_CPUS="3-5" /hermit/usr/tests/hello` starts a HelloWorld demo on the HermitCore isle 1, which uses the cores 3 to 5.
The output messages are forwarded to the Linux proxy and printed on the Linux system.
8. HermitCore's kernel messages of `isleX` are available via `cat /sys/hermit/isleX/log`.
9. There is a virtual IP device for the communication between the HermitCore isles and the Linux system (see output of `ifconfig`).
Per default, the Linux system has the IP address `192.168.28.1`.
The HermitCore isles starts with the IP address `192.168.28.2` for isle 0 and is increased by one for every isle.
10. More HermitCore applications are available at `/hermit/usr/{tests,benchmarks}` which is a shared directory between the host and QEmu.
```bash
$ cmake -DTOOLCHAIN_BIN_DIR=/home/user/hermit/bin
```
## Building and testing HermitCore as multi-kernel on a real machine
assuming that binaries like `x86_64-hermit-gcc` and friends are located in that
directory.
## Testing
### As multi-kernel within a virtual machine
```bash
$ cd build
$ make qemu
$ # or 'make qemu-dep' to build HermitCore dependencies before
```
Within the QEMU session you can start HermitCore application just the same as
traditional Linux programs:
```bash
(QEMU) $ /hermit/extra/tests/hello
smpboot: CPU 1 is now offline
Hello World!!!
argv[0] = /hermit/extra/tests/hello
Receive signal with number 30
Hostname: hermit.localdomain
x86: Booting SMP configuration:
smpboot: Booting Node 0 Processor 1 APIC 0x1
```
Per default, the virtual machine has 10 cores, 2 NUMA nodes, and 8 GiB RAM.
Inside the VM runs a small Linux system, which already includes the patches for
HermitCore. Per NUMA node (= HermitCore isle) there is a directory called
`isleX` under `/sys/hermit` , where `X` represents the NUMA node ID.
The demo applications are located in the directories
`/hermit/extra/{tests,benchmarks}`. A HermitCore loader is already registered.
By starting a HermitCore application, a proxy will be executed on the Linux
system, while the HermitCore binary will be started on isle 0 with cpu 1. To
change the default behavior, the environment variable `HERMIT_ISLE` is used to
specify the (memory) location of the isle, while the environment variable
`HERMIT_CPUS` is used to specify the cores.
For instance, `HERMIT_ISLE=1 HERMIT_CPUS="3-5" /hermit/extra/tests/hello` starts
a HelloWorld demo on the HermitCore isle 1, which uses the cores 3 to 5. The
output messages are forwarded to the Linux proxy and printed on the Linux
system.
HermitCore's kernel messages of `isleX` are available via `cat
/sys/hermit/isleX/log`.
There is a virtual IP device for the communication between the HermitCore isles
and the Linux system (see output of `ifconfig`). Per default, the Linux system
has the IP address `192.168.28.1`. The HermitCore isles starts with the IP
address `192.168.28.2` for isle 0 and is increased by one for every isle.
More HermitCore applications are available at `/hermit/usr/{tests,benchmarks}`
which is a shared directory between the host and QEMU.
### As classical standalone unikernel within a virtual machine
HermitCore applications can be directly started as standalone kernel within a
virtual machine. In this case,
[iRCCE](http://www.lfbs.rwth-aachen.de/publications/files/iRCCE.pdf ) is not
supported.
```bash
$ cd build
$ make install DESTDIR=~/hermit-build
$ cd ~/hermit-build/opt/hermit
$ # using QEMU
$ HERMIT_ISLE=qemu bin/proxy extra/tests/hello
$ # using uHyve
$ HERMIT_ISLE=uhyve bin/proxy extra/tests/hello
```
With `HERMIT_ISLE=qemu`, the application will be started within a QEMU VM.
Please note that the loader requires QEMU and uses per default *KVM*.
Furthermore, it expects that the executable is called `qemu-system-x86_64`.
With `HERMIT_ISLE=hyve`, the application will be started within a thin
hypervisor powered by Linux's KVM API and therefore requires *KVM* support.
uHyve has a considerably smaller startup time than QEMU, but lacks some features
such as GDB debugging.
In this context, the environment variable `HERMIT_CPUS` specifies the number of
cpus (and no longer a range of core ids). Furthermore, the variable `HERMIT_MEM`
defines the memory size of the virtual machine. The suffix of *M* or *G* can be
used to specify a value in megabytes or gigabytes respectively. Per default, the
loader initializes a system with one core and 2 GiB RAM.
The virtual machine opens two TCP/IP ports. One is used for the communication
between HermitCore application and its proxy. The second port is used to create
a connection via telnet to QEMU's system monitor. With the environment variable
`HERMIT_PORT`, the default port (18766) can be changed for the communication
between the HermitCore application and its proxy. The connection to the system
monitor used automatically `HERMIT_PORT+1`, i.e., the default port is 18767.
The following command starts the stream benchmark in a virtual machine, which
has 4 cores and 6GB memory.
```bash
$ HERMIT_ISLE=qemu HERMIT_CPUS=4 HERMIT_MEM=6G bin/proxy extra/benchmarks/stream
```
### As multi-kernel on a real machine
*Note*: to launch HermitCore applications, root privileges are required.
1. In principle you have to follow the tutorial above.
After the configuration, building of the cross-compilers and all example application (Step 5 in the [above tutorial](#building-and-testing-hermitcore-within-a-virtual-machine)), a modified Linux kernel has to be installed.
Please clone the repository with the [modified Linux kernel](https://github.com/RWTH-OS/linux).
Afterwards switch to the branch `hermit` for a relative new vanilla kernel or to `centos`, which is compatible to the current CentOS 7 kernel.
Configure the kernel with `make menuconfig` for your system.
Be sure, that the option `CONFIG_HERMIT_CORE` in `Processor type and features` is enabled.
2. Install the Linux kernel and its initial ramdisk on your system (see descriptions of your Linux distribution).
We recommend to disable Linux NO_HZ feature by setting the kernel parameter `nohz=off`.
3. After a reboot of the system, register the HermitCore loader at your system with following command: `sudo -c sh 'echo ":hermit:M:7:\\x42::/path2proyxy/proxy:" > /proc/sys/fs/binfmt_misc/register'`, in which `path2proxy` defines the path to the loader.
You find the loader `proxy` after building the HermiCore sources in the subdirectory `tools` of the directory, which contains this *README*.
4. The IP device between HermitCore and Linux currently does not support IPv6.
Consequently, disable IPv6 by adding following line to `/etc/sysctl.conf`: `net.ipv6.conf.mmnif.disable_ipv6 = 1`.
5. Per default, the IP device uses a static IP address range.
Linux has to use `162.168.28.1`, where HermitCore isles start with `192.168.28.2` (isle 0).
The network manager must be configured accordingly and therefore the file `/etc/sysconfig/network-scripts/ifcfg-mmnif` must be created with the following content:
A [modified Linux kernel](https://github.com/RWTH-OS/linux) has to be installed.
Afterwards switch to the branch `hermit` for a relative new vanilla kernel or to
`centos`, which is compatible to the current CentOS 7 kernel. Configure the
kernel with `make menuconfig` for your system. Be sure, that the option
`CONFIG_HERMIT_CORE` in `Processor type and features` is enabled.
```bash
$ git clone https://github.com/RWTH-OS/linux
$ cd linux
$ # see comments above
$ git checkout hermit
$ make menuconfig
$ make
```
Install the Linux kernel and its initial ramdisk on your system (see
descriptions of your Linux distribution). We recommend to disable Linux NO_HZ
feature by setting the kernel parameter `nohz=off`.
Install HermitCore to your system (by default to `/opt/hermit`):
```bash
$ cd build
$ sudo make install
$ ls -l /opt/hermit
```
After a reboot of the system, register the HermitCore loader at your system with
following command:
```bash
$ sudo -c sh 'echo ":hermit:M:7:\\x42::/opt/hermit/bin/proxy:" > /proc/sys/fs/binfmt_misc/register'
```
The IP device between HermitCore and Linux currently does not support IPv6.
Consequently, disable it (might be slightly different on your distribution):
```bash
$ echo 'net.ipv6.conf.mmnif.disable_ipv6 = 1' | sudo tee /etc/sysctl.conf
```
Per default, the IP device uses a static IP address range. Linux has to use
`162.168.28.1`, where HermitCore isles start with `192.168.28.2` (isle 0). The
interface is `mmnif`.
Please configure your network accordingly. For CentOS, you have to create the
file `/etc/sysconfig/network-scripts/ifcfg-mmnif`:
```
DEVICE=mmnif
@ -96,66 +273,72 @@ NETMASK=255.255.255.0
IPADDR=192.168.28.1
NM_CONTROLLED=yes
```
Finally, follow the [above tutorial](#building-and-testing-hermitcore-within-a-virtual-machine) from Step 5.
The demo applications are located in their subdirectories `usr/{tests,benchmarks}`.
## Building and testing HermitCore as classical standalone unikernel
You can now start applications the same way as from within a virtual machine
(see description above).
HermitCore applications can be directly started as standalone kernel within a virtual machine.
In this case, [iRCCE](http://www.lfbs.rwth-aachen.de/publications/files/iRCCE.pdf) is not supported.
Please build HermitCore and register the loader in the same way as done for the multi-kernel version (see [Building and testing HermitCore on a real machine](#building-and-testing-hermitcore-on-a-real-machine)).
If the environment variable `HERMIT_ISLE` is set to `qemu`, the application will be started within a VM.
Please note that the loader requires QEMU and uses per default *KVM*.
Furthermore, it expects that the executable is called `qemu-system-x86_64`.
You can adapt the name by setting the environment variable `HERMIT_QEMU`.
In this context, the environment variable `HERMIT_CPUS` specifies the number of cpus (and no longer a range of core ids).
Furthermore, the variable `HERMIT_MEM` defines the memory size of the virtual machine.
The suffix of *M* or *G* can be used to specify a value in megabytes or gigabytes respectively.
Per default, the loader initializes a system with one core and 2 GiB RAM.
## Building your own HermitCore applications
The virtual machine opens two TCP/IP ports.
One is used for the communication between HermitCore application and its proxy.
The second port is used to create a connection via telnet to QEMU's system monitor.
With the environment variable `HERMIT_PORT`, the default port (18766) can be changed for the communication between the HermitCore application and its proxy.
The connection to the system monitor used automatically `HERMIT_PORT+1`, i.e., the default port is 18767.
You can take `usr/tests` as a starting point to build your own applications. All
that is required is that you include
`[...]/HermitCore/cmake/HermitCore-Application.cmake` in your application's
`CMakeLists.txt`. It doesn't have to reside inside the HermitCore repository.
Other than that, it should behave like normal CMake.
## Profiling
We provide profiling support via the XRay profiler. See `usr/xray/README.md` for
more information on how to use it.
## Debugging
If the application is started via `make qemu`, debugging via GDB is enabled by
default on port 1234. When run via proxy (`HERMIT_ISLE=qemu`), set
`HERMIT_DEBUG=1`.
The following example starts the stream benchmark in a virtual machine, which has 4 cores and 6GB memory.
```
HERMIT_ISLE=qemu HERMIT_CPUS=4 HERMIT_MEM=6G usr/benchmarks/stream
$ gdb extra/tests/hello
(gdb) target extended-remote :1234
Remote debugging using :1234
0xffffffff8100b542 in ?? ()
```
## Building HermitCore applications
After successful building of HermitCore and its demo applications (see above), HermitCores cross toolchain (*gcc*, *g++*, *gfortran*, *gccgo*, *objdump*, etc.) is located at the subdiretory `usr/x86` of the directory, which contains this *README*.
To use these tools, add `usr/x86/bin` to your environment variable `PATH`.
As with any other cross toolchain, the tool names begin with the target architecture (*x86_64*) and the name of the operating system (*hermit*).
For instance, `x86_64-hermit-gcc` stands for the GNU C compiler, which is able to build HermitCore applications.
All tools can be used as the well-known GNU tools. Only the Go compiler works different to the typical workflow.
Instead of building Go application like
```
go build main.go
```
you have to use the compiler as follows
```
x86_64-hermit-gccgo -pthread -Wall -o main main.go
```
For network support, you have to link the Go application with the flag `-lnetgo`.
## Tips
1. The configuration flag `--with-mtune=name` specifies the name of the target processor for which GCC should tune the performance of the code.
You can use any architecture name, which is supported by GCC.
For instance, `--with-mtune=native` optimzes the code for the host system.
Please note, if the applications is started within a VM, the hypervisor has to support the specified architecture name.
Per default the system will be accelerated by KVM and the host architecture will be used as target processor.
2. If Qemu is started by our proxy and the environment variable `HERMIT_KVM` is set to `0`, the virtual machine will be not accelerated by KVM.
In this case, the configuration flag `--with-mtune=name` should be avoided.
With the environment variable `HERMIT_APP_PORT`, an additional port can be open to establish an TCP/IP connection with your application.
3. By setting the environment variable `HERMIT_VERBOSE` to `1`, the proxy prints at termination the kernel log messages onto the screen.
4. If `HERMIT_DEBUG` is set to `1`, Qemu will establish an gdbserver, which will be listen port 1234.
Afterwards you are able debug HermitCore applications remotely.
5. By setting the environment variable `HERMIT_CAPTURE_NET` to `1` and `HERMIT_ISLE` to `qemu`, Qemu captures the network traffic and
creates the trace file *qemu-vlan0.pcap*. For instance with [Wireshark](https://www.wireshark.org) you are able to analyze the file.
6. If `HERMIT_MONITOR` is set to `1` and `HERMIT_ISLE` to `qemu`, Qemu establishes a monitor which is available via telnet at port 18767.
### Optimization
You can configure the `-mtune=name` compiler flag by adding `-DMTUNE=name` to
the `cmake` command when configuring the project.
Please note, if the applications is started within a VM, the hypervisor has to
support the specified architecture name.
If QEMU is started by our proxy and the environment variable `HERMIT_KVM` is set
to `0`, the virtual machine will be not accelerated by KVM. In this case, the
`-mtune` flag should be avoided.
### TCP connections
With the environment variable `HERMIT_APP_PORT`, an additional port can be open
to establish an TCP/IP connection with your application.
### Dumping the kernel log
By setting the environment variable `HERMIT_VERBOSE` to `1`, the proxy prints at
termination the kernel log messages onto the screen.
### Network tracing
By setting the environment variable `HERMIT_CAPTURE_NET` to `1` and
`HERMIT_ISLE` to `qemu`, QEMU captures the network traffic and creates the trace
file *qemu-vlan0.pcap*. For instance with [Wireshark](https://www.wireshark.org)
you are able to analyze the file.
### Monitor
If `HERMIT_MONITOR` is set to `1` and `HERMIT_ISLE` to `qemu`, QEMU establishes
a monitor which is available via telnet at port 18767.

88
arch/x86/CMakeLists.txt Normal file
View file

@ -0,0 +1,88 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore.cmake)
project(arch_x86_kernel C ASM_NASM)
set_parent(X86_KERNEL_TARGET ${PROJECT_NAME})
set_parent(X86_KERNEL_ASM_TARGET ${X86_KERNEL_TARGET}_asm)
set_parent(X86_KERNEL_C_TARGET ${X86_KERNEL_TARGET}_c)
add_custom_target(${X86_KERNEL_TARGET})
# compiling kernel code here
add_definitions(-D__KERNEL__)
### ASM sources ###
add_library(${X86_KERNEL_ASM_TARGET} OBJECT
kernel/entry.asm
libkern/string.asm)
# HACK: We need to post-process the objects by running elfedit on them, but
# there is currently no way to get the list of objects out of CMake
# except for $<TARGET_OBJECTS:tgt>, which only works with add_library()
# and add_executable().
# So predict path to objects and add custom commands that depend on
# the asm target.
#
# Upstream issue: https://gitlab.kitware.com/cmake/cmake/issues/15226
#
set(_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}")
set(_BUILD_DIR "${_BUILD_DIR}/${X86_KERNEL_ASM_TARGET}.dir")
get_target_property(ASM_SOURCES ${X86_KERNEL_ASM_TARGET} SOURCES)
foreach(SOURCE ${ASM_SOURCES})
set(OBJECT "${SOURCE}.obj")
set(OBJECT_PATH "${_BUILD_DIR}/${OBJECT}")
# slash (/) not allowed in target names
string(REPLACE "/" "-"
OBJECT_TARGET_NAME
"${OBJECT}")
add_custom_target("${OBJECT_TARGET_NAME}"
COMMAND
${CMAKE_ELFEDIT} --output-osabi HermitCore ${OBJECT_PATH}
DEPENDS
${X86_KERNEL_ASM_TARGET})
# make main target depend on this
add_dependencies(${PROJECT_NAME} ${OBJECT_TARGET_NAME})
endforeach()
### C sources ###
file(GLOB KERNEL_SOURCES "kernel/*.c")
file(GLOB MM_SOURCES "mm/*.c")
# add boot.h as source to mark dependency boot.asm -> boot.h -> apic.c
add_library(${X86_KERNEL_C_TARGET} OBJECT
${KERNEL_SOURCES} ${MM_SOURCES}
${GENERATED_CONFIG_DIR}/hermit/boot.h)
target_include_directories(${X86_KERNEL_C_TARGET} BEFORE
PUBLIC ${HERMIT_KERNEL_INCLUDES}
PRIVATE ${GENERATED_CONFIG_DIR})
target_compile_options(${X86_KERNEL_C_TARGET}
PRIVATE ${HERMIT_KERNEL_FLAGS})
# assemble boot.asm and dump to C-array in boot.h
add_custom_command(
OUTPUT
${GENERATED_CONFIG_DIR}/hermit/boot.h
DEPENDS
kernel/boot.asm
COMMAND
echo "static const uint8_t boot_code[] = {" > boot.h
COMMAND
nasm -f bin -o boot.bin ${CMAKE_CURRENT_LIST_DIR}/kernel/boot.asm
COMMAND
hexdump -v -e "7/1 \"0x%02X, \" 1/1 \" 0x%02X,\\n\"" boot.bin >> boot.h
COMMAND
echo "};" >> boot.h
WORKING_DIRECTORY
${GENERATED_CONFIG_DIR}/hermit/
VERBATIM USES_TERMINAL)

View file

@ -42,7 +42,7 @@
#include <asm/io.h>
#include <asm/page.h>
#include <asm/apic.h>
#include "boot.h"
#include <hermit/boot.h>
/*
* Note that linker symbols are not variables, they have no memory allocated for

View file

@ -28,7 +28,7 @@
; perhaps setting up the GDT and segments. Please note that interrupts
; are disabled at this point: More on interrupts later!
%include "config.inc"
%include "hermit/config.asm"
[BITS 64]
@ -600,7 +600,7 @@ common_switch:
call get_current_stack ; get new rsp
mov rsp, rax
%ifdef SAVE_FPU
%ifidn SAVE_FPU,ON
; set task switched flag
mov rax, cr0
or rax, 8

View file

@ -8,7 +8,7 @@
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;
%include "config.inc"
%include "hermit/config.asm"
%ifdef CONFIG_X86_32
[BITS 32]

View file

@ -0,0 +1,50 @@
cmake_minimum_required(VERSION 3.7)
project(arch_x86_loader C ASM_NASM)
## ASM sources
file(GLOB ASM_SOURCES *.asm)
add_library(arch_x86_loader_asm STATIC ${ASM_SOURCES})
## C sources
file(GLOB C_SOURCES *.c)
add_executable(arch_x86_loader ${C_SOURCES})
target_include_directories(arch_x86_loader
PRIVATE include/)
target_compile_options(arch_x86_loader
PRIVATE -O2 -Wall -m64 -std=gnu99 -ffreestanding -mno-red-zone
-fstrength-reduce -fomit-frame-pointer -finline-functions)
target_link_libraries(arch_x86_loader
arch_x86_loader_asm
"-T ${CMAKE_CURRENT_LIST_DIR}/link.ld"
"-z max-page-size=4096"
-Wl,--build-id=none # required because CMake links with gcc, not ld
-nostdlib)
# tools/proxy looks for `ldhermit.elf`
set_target_properties(arch_x86_loader PROPERTIES
OUTPUT_NAME ldhermit.elf)
add_custom_command(
TARGET arch_x86_loader POST_BUILD
# Split debug symbols into seperate file
COMMAND
${CMAKE_OBJCOPY} --only-keep-debug
$<TARGET_FILE:arch_x86_loader>
$<TARGET_FILE:arch_x86_loader>.sym
# Qemu requires 32-bit ELF
COMMAND
${CMAKE_OBJCOPY} -O elf32-i386 --strip-debug
$<TARGET_FILE:arch_x86_loader>)
install(TARGETS arch_x86_loader
DESTINATION bin)
# Show include files in IDE
file(GLOB_RECURSE ARCH_X86_LOADER_INCLUDES "include/*")
add_custom_target(arch_x86_loader_includes_ide SOURCES ${ARCH_X86_LOADER_INCLUDES})

View file

@ -0,0 +1,9 @@
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore.cmake)
include_guard()
add_compile_options(${HERMIT_APP_FLAGS})
# link against and include locally built libraries instead of the ones
# supplied with the toolchain, if built from top-level
link_directories(${LOCAL_PREFIX_ARCH_LIB_DIR})
include_directories(BEFORE ${LOCAL_PREFIX_ARCH_INCLUDE_DIR})

View file

@ -0,0 +1,29 @@
set(PACKAGE_VERSION "0.1" CACHE STRING
"HermitCore current version")
set(MAX_CORES "512" CACHE STRING
"Maximum number of cores that can be managed")
set(MAX_TASKS "((MAX_CORES * 2) + 2)" CACHE STRING
"Maximum number of tasks")
set(MAX_ISLE "8" CACHE STRING
"Maximum number of NUMA isles")
set(KERNEL_STACK_SIZE 8192 CACHE STRING
"Kernel stack size in bytes")
set(DEFAULT_STACK_SIZE 262144 CACHE STRING
"Task stack size in bytes")
option(DYNAMIC_TICKS
"Don't use a periodic timer event to keep track of time" ON)
option(SAVE_FPU
"Save FPU registers on context switch" ON)
option(HAVE_ARCH_MEMSET "Use machine specific version of memset" OFF)
option(HAVE_ARCH_MEMCPY "Use machine specific version of memcpy" OFF)
option(HAVE_ARCH_STRLEN "Use machine specific version of strlen" OFF)
option(HAVE_ARCH_STRCPY "Use machine specific version of strcpy" OFF)
option(HAVE_ARCH_STRNCPY "Use machine specific version of strncpy" OFF)

View file

@ -0,0 +1,35 @@
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Utils.cmake)
# no include guard here because we have to include this file twice to correctly
# set CMAKE_INSTALL_PREFIX
# root of HermitCore project
set(HERMIT_ROOT ${CMAKE_CURRENT_LIST_DIR}/..)
# set default install prefix if user doesn't specify one
if(${CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT})
# See CMake docs for reference:
# https://cmake.org/cmake/help/v3.7/variable/CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT.html
set(CMAKE_INSTALL_PREFIX /opt/hermit CACHE PATH "..." FORCE)
endif()
# we install 3rd party libraries to an intermediate directory and relocate
# them here later when installing the whole project
if(NOT LOCAL_PREFIX_BASE_DIR)
# will be injected into external project because CMAKE_BINARY_DIR will be
# different there
set(LOCAL_PREFIX_BASE_DIR ${CMAKE_BINARY_DIR}/local_prefix)
endif()
# during build process libraries and external projects will be deployed into
# this directory structure
set(LOCAL_PREFIX_DIR ${LOCAL_PREFIX_BASE_DIR}/${CMAKE_INSTALL_PREFIX})
set(LOCAL_PREFIX_ARCH_DIR ${LOCAL_PREFIX_DIR}/${TARGET_ARCH})
set(LOCAL_PREFIX_ARCH_INCLUDE_DIR ${LOCAL_PREFIX_ARCH_DIR}/include)
# when building applications within the HermitCore project (tests, ...) they
# will link prefarably against libraries in this directory in order to test
# changes in the kernel
set(LOCAL_PREFIX_ARCH_LIB_DIR ${LOCAL_PREFIX_ARCH_DIR}/lib)
# generated configs will be put here
set(GENERATED_CONFIG_DIR ${CMAKE_BINARY_DIR}/include)

View file

@ -0,0 +1,25 @@
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Utils.cmake)
include_guard()
# let user provide a different path to the toolchain
set_default(TOOLCHAIN_BIN_DIR /opt/hermit/bin)
set(TARGET_ARCH x86_64-hermit)
set(CMAKE_SYSTEM_NAME Generic)
# point CMake to our toolchain, it will auto detect C++ compiler
set(CMAKE_C_COMPILER ${TOOLCHAIN_BIN_DIR}/${TARGET_ARCH}-gcc)
set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_BIN_DIR}/${TARGET_ARCH}-gfortran)
set(CMAKE_Go_COMPILER ${TOOLCHAIN_BIN_DIR}/${TARGET_ARCH}-gccgo)
# hinting the prefix and location is needed in order to correctly detect
# binutils
set(_CMAKE_TOOLCHAIN_PREFIX "${TARGET_ARCH}-")
set(_CMAKE_TOOLCHAIN_LOCATION ${TOOLCHAIN_BIN_DIR})
option(HAVE_ARCH_MEMSET "Use machine specific version of memset" ON)
option(HAVE_ARCH_MEMCPY "Use machine specific version of memcpy" ON)
option(HAVE_ARCH_STRLEN "Use machine specific version of strlen" ON)
option(HAVE_ARCH_STRCPY "Use machine specific version of strcpy" ON)
option(HAVE_ARCH_STRNCPY "Use machine specific version of strncpy" ON)

View file

@ -0,0 +1,142 @@
macro(include_guard)
if(DEFINED "_INCLUDE_GUARD_${CMAKE_CURRENT_LIST_FILE}")
return()
endif()
set("_INCLUDE_GUARD_${CMAKE_CURRENT_LIST_FILE}" INCLUDED)
endmacro(include_guard)
macro(add_kernel_module_sources MODULE SOURCE_GLOB)
file(GLOB SOURCES "${SOURCE_GLOB}")
if("${SOURCES}" STREQUAL "")
message(FATAL_ERROR "Module '${MODULE}' has no sources")
endif()
# make sure modules are unique, this is needed of multiple sources
# are added to the same module
list(APPEND _KERNEL_MODULES "${MODULE}")
list(REMOVE_DUPLICATES _KERNEL_MODULES)
# append sources for module
list(APPEND "_KERNEL_SOURCES_${MODULE}" "${SOURCES}")
endmacro(add_kernel_module_sources)
macro(get_kernel_module_sources VAR MODULE)
set(${VAR} ${_KERNEL_SOURCES_${MODULE}})
endmacro(get_kernel_module_sources)
macro(get_kernel_modules VAR)
set(${VAR} ${_KERNEL_MODULES})
endmacro(get_kernel_modules)
# find program in /toolchain/dir/prefix-NAME, only supply NAME
function(find_toolchain_program NAME)
string(TOUPPER "${NAME}" NAME_UPPER)
string(TOLOWER "${NAME}" NAME_LOWER)
set(VARNAME "CMAKE_${NAME_UPPER}")
find_program(${VARNAME}
NAMES ${_CMAKE_TOOLCHAIN_PREFIX}${NAME_LOWER}
HINTS ${TOOLCHAIN_BIN_DIR})
if(NOT ${VARNAME})
message(FATAL_ERROR
"Cannot find ${_CMAKE_TOOLCHAIN_PREFIX}${NAME_LOWER}")
endif()
endfunction(find_toolchain_program)
macro(set_parent VAR VALUE)
set(${VAR} ${VALUE} PARENT_SCOPE)
set(${VAR} ${VALUE})
endmacro(set_parent)
function(get_cmd_variables VAR)
set(_OUTPUT "")
get_cmake_property(vs VARIABLES)
foreach(v ${vs})
get_property(_HELPSTRING
CACHE ${v}
PROPERTY HELPSTRING)
if("${_HELPSTRING}" STREQUAL "No help, variable specified on the command line.")
list(APPEND _OUTPUT "${v}")
endif()
endforeach()
set(${VAR} ${_OUTPUT} PARENT_SCOPE)
endfunction(get_cmd_variables)
# any additional parameters will be handed over to the cmake command that the
# external project is invoked with
function(build_external NAME PATH DEPENDS)
if("${NAME}" IN_LIST PROFILE_APPS)
set(DO_PROFILING "-DPROFILING=true")
endif()
# pass through all command line variables
get_cmd_variables(CMD_VAR_NAMES)
foreach(var ${CMD_VAR_NAMES})
set(CMD_VARS ${CMD_VARS} -D${var}=${${var}})
endforeach()
ExternalProject_Add(${NAME}
SOURCE_DIR ${PATH}
BUILD_ALWAYS 1
DEPENDS ${DEPENDS}
INSTALL_COMMAND
${CMAKE_COMMAND} --build <BINARY_DIR>
--target install --
DESTDIR=${LOCAL_PREFIX_BASE_DIR}
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
-DLOCAL_PREFIX_BASE_DIR=${LOCAL_PREFIX_BASE_DIR}
-DCMAKE_INSTALL_MESSAGE=NEVER
--no-warn-unused-cli
${DO_PROFILING}
${CMD_VARS}
${ARGN})
ExternalProject_Add_Step(${NAME} relink
COMMAND find . -maxdepth 1 -type f -executable -exec rm {} "\\\;"
DEPENDEES configure
DEPENDERS build
WORKING_DIRECTORY <BINARY_DIR>)
ExternalProject_Add_StepDependencies(${NAME} relink ${DEPENDS})
endfunction(build_external)
# additional arguments are be treated as targets that will be excluded
function(install_local_targets PATH)
get_property(_TARGETS
DIRECTORY .
PROPERTY BUILDSYSTEM_TARGETS)
if(NOT "${ARGN}" STREQUAL "")
list(REMOVE_ITEM _TARGETS ${ARGN})
endif()
install(TARGETS ${_TARGETS}
DESTINATION ${PATH})
# if there are any .map files for profiling, install them too
foreach(TARGET ${_TARGETS})
install(FILES $<TARGET_FILE:${TARGET}>.map
DESTINATION ${PATH}
OPTIONAL)
endforeach()
endfunction(install_local_targets)
# set variable if not yet set
macro(set_default VARNAME)
if(NOT ${VARNAME})
set(${VARNAME} ${ARGN})
endif()
endmacro(set_default)

84
cmake/HermitCore.cmake Normal file
View file

@ -0,0 +1,84 @@
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Utils.cmake)
include_guard()
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Paths.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Configuration.cmake)
# scripts to detect HermitCore Go compiler
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/golang/)
if(NOT HERMIT_ARCH)
set(HERMIT_ARCH x86)
endif()
if(PROFILING)
# link everything against XRay
link_libraries(-lxray)
# generate symbol map file for XRay to resolve function names
link_libraries(-Wl,-Map=$<TARGET_PROPERTY:NAME>.map)
# enable profiling with XRay
add_compile_options(-falign-functions=32 -finstrument-functions
-finstrument-functions-exclude-function-list=_mm_pause,_mm_setcsr,_mm_getcsr)
add_definitions(-DXRAY -DXRAY_DISABLE_BROWSER_INTEGRATION
-DXRAY_NO_DEMANGLE -DXRAY_ANNOTATE)
endif()
# use default toolchain if not specified by user
if(NOT CMAKE_TOOLCHAIN_FILE)
set(CMAKE_TOOLCHAIN_FILE ${CMAKE_CURRENT_LIST_DIR}/HermitCore-Toolchain-${HERMIT_ARCH}.cmake)
endif()
# NASM detection will change binary format depending on host system, but
# we only want to generate elf64 for HermitCore
# Note: Has to be set *before* ASM_NASM is enabled
set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64)
enable_language(ASM_NASM)
# NASM hack, because it requires include paths to have a trailing /, whereas
# CMake explicitly will remove it when adding includes the usual way
# Note: Has to be set *after* ASM_NASM is enabled
set(CMAKE_ASM_NASM_FLAGS
"${CMAKE_ASM_NASM_FLAGS} -I ${CMAKE_BINARY_DIR}/include/")
set(HERMIT_KERNEL_FLAGS
-m64 -Wall -O2 -mno-red-zone
-fno-var-tracking-assignments -fstrength-reduce
-fomit-frame-pointer -finline-functions -ffreestanding
-nostdinc -fno-stack-protector -mno-sse -mno-mmx
-mno-sse2 -mno-3dnow -mno-avx
-fno-delete-null-pointer-checks
-falign-jumps=1 -falign-loops=1
-mno-80387 -mno-fp-ret-in-387 -mskip-rax-setup
-fno-common -Wframe-larger-than=1024
-fno-strict-aliasing -fno-asynchronous-unwind-tables
-fno-strict-overflow -maccumulate-outgoing-args)
set(HERMIT_APP_FLAGS
-m64 -mtls-direct-seg-refs -O3 -ftree-vectorize)
if(MTUNE)
set(HERMIT_KERNEL_FLAGS ${HERMIT_KERNEL_FLAGS} -mtune=${MTUNE})
set(HERMIT_APP_FLAGS ${HERMIT_APP_FLAGS} -mtune=${MTUNE})
endif()
set(HERMIT_KERNEL_INCLUDES
${CMAKE_BINARY_DIR}/include
${HERMIT_ROOT}/include
${HERMIT_ROOT}/arch/${HERMIT_ARCH}/include
${HERMIT_ROOT}/lwip/src/include
${HERMIT_ROOT}/drivers)
# HACK: when CMake detects compilers it taints CMAKE_INSTALL_PREFIX, so in
# order to rely on that variable (we redefine it), enable all languages
# here and source pathes again.
#
# Furthermore this will produce a sensible error message if the toolchain cannot
# be found.
enable_language(C CXX Fortran Go)
include(${CMAKE_CURRENT_LIST_DIR}/HermitCore-Paths.cmake)
# find elfedit, CMake doesn't use this program, so we have to find it ourself
find_toolchain_program(elfedit)

67
cmake/README.md Normal file
View file

@ -0,0 +1,67 @@
HermitCore CMake build system
=============================
HermitCore requires at least CMake version `3.7`, which (at the time of
introduction) is not yet available on most Linux distributions. We therefore
provide a script `cmake/local-cmake.sh` that fetches precompiled binaries from
the CMake project and installs them locally in `cmake/cmake-3.7*`. Only when
sourced for the first time it will download CMake, on further runs it detects
the existing download and adds it to `PATH` automatically.
```bash
$ . cmake/local-cmake.sh
-- Downloading CMake
cmake-3.7.2-Linux-x 100%[=================>] 29,26M 837KB/s in 19s
-- Unpacking CMake
-- Local CMake v3.7.2 installed to cmake/cmake-3.7.2-Linux-x86_64
-- Next time you source this script, no download will be neccessary
$ which cmake
/home/[...]/HermitCore/cmake/cmake-3.7.2-Linux-x86_64/bin/cmake
```
## Directory structure
```
cmake/
├── golang
│   ├── CMakeDetermineGoCompiler.cmake
│   ├── CMakeGoCompiler.cmake.in
│   ├── CMakeGoInformation.cmake
│   └── CMakeTestGoCompiler.cmake
├── HermitCore-Application.cmake
├── HermitCore.cmake
├── HermitCore-Paths.cmake
├── HermitCore-Toolchain-x86.cmake
├── HermitCore-Utils.cmake
├── local-cmake.sh
└── README.md
```
## Additional language support
Currently, HermitCore supports `C, C++, Fortran, Go` through Cmake. While the
former are supported and detected by CMake out-of-the-box, Go support has been
added manually for HermitCore.
Adding a new language requires you to provide CMake hints where to find the
toolchain and then how to compile and link binaries. The Go support in
`cmake/golang` may serve as an example on how to add a new language.
To finally enable the language it has to be added to CMake's module path in
`cmake/HermitCore.cmake`:
```
# scripts to detect HermitCore Go compiler
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/golang/)
# scripts to detect new language
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/new-language/)
```
## User applications
You just have to include `HermitCore-Application.cmake` before the `project()`
command in your `CMakeLists.txt` in order to build applications for HermitCore.
For configuration parameters of the project, please consult the `README.md` in
the root of this repository.

View file

@ -0,0 +1,43 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
# determine the compiler to use for Go programs
# NOTE, a generator may set CMAKE_Go_COMPILER before
# loading this file to force a compiler.
if(NOT CMAKE_Go_COMPILER)
# prefer the environment variable CC
if(NOT $ENV{GO_COMPILER} STREQUAL "")
get_filename_component(CMAKE_Go_COMPILER_INIT $ENV{GO_COMPILER} PROGRAM PROGRAM_ARGS CMAKE_Go_FLAGS_ENV_INIT)
if(CMAKE_Go_FLAGS_ENV_INIT)
set(CMAKE_Go_COMPILER_ARG1 "${CMAKE_Go_FLAGS_ENV_INIT}" CACHE STRING "First argument to Go compiler")
endif()
if(NOT EXISTS ${CMAKE_Go_COMPILER_INIT})
message(SEND_ERROR "Could not find compiler set in environment variable GO_COMPILER:\n$ENV{GO_COMPILER}.")
endif()
endif()
set(Go_BIN_PATH
$ENV{GOPATH}
$ENV{GOROOT}
$ENV{GOROOT}/../bin
$ENV{GO_COMPILER}
/usr/bin
/usr/local/bin
)
# if no compiler has been specified yet, then look for one
if(CMAKE_Go_COMPILER_INIT)
set(CMAKE_Go_COMPILER ${CMAKE_Go_COMPILER_INIT} CACHE PATH "Go Compiler")
else()
find_program(CMAKE_Go_COMPILER
NAMES go
PATHS ${Go_BIN_PATH}
)
endif()
endif()
mark_as_advanced(CMAKE_Go_COMPILER)
# configure variables set in this file for fast reload later on
configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeGoCompiler.cmake.in
${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY)
set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")

View file

@ -0,0 +1,11 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
set(CMAKE_Go_COMPILER "@CMAKE_Go_COMPILER@")
set(CMAKE_Go_COMPILER_LOADED 1)
set(CMAKE_Go_SOURCE_FILE_EXTENSIONS go)
set(CMAKE_Go_LINKER_PREFERENCE 40)
set(CMAKE_Go_OUTPUT_EXTENSION .6)
set(CMAKE_Go_OUTPUT_EXTENSION_REPLACE 1)
set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")

View file

@ -0,0 +1,30 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
# This should be included before the _INIT variables are
# used to initialize the cache. Since the rule variables
# have if blocks on them, users can still define them here.
# But, it should still be after the platform file so changes can
# be made to those values.
if(CMAKE_USER_MAKE_RULES_OVERRIDE)
# Save the full path of the file so try_compile can use it.
include(${CMAKE_USER_MAKE_RULES_OVERRIDE} RESULT_VARIABLE _override)
set(CMAKE_USER_MAKE_RULES_OVERRIDE "${_override}")
endif()
if(CMAKE_USER_MAKE_RULES_OVERRIDE_Go)
# Save the full path of the file so try_compile can use it.
include(${CMAKE_USER_MAKE_RULES_OVERRIDE_Go} RESULT_VARIABLE _override)
set(CMAKE_USER_MAKE_RULES_OVERRIDE_Go "${_override}")
endif()
# refer: /usr/share/cmake-3.7/Modules/CMakeCInformation.cmake
if(NOT CMAKE_Go_COMPILE_OBJECT)
set(CMAKE_Go_COMPILE_OBJECT "<CMAKE_Go_COMPILER> <FLAGS> -o <OBJECT> -c <SOURCE> ")
endif()
if(NOT CMAKE_Go_LINK_EXECUTABLE)
set(CMAKE_Go_LINK_EXECUTABLE "<CMAKE_Go_COMPILER> -pthread <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
endif()

View file

@ -0,0 +1,4 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
set(CMAKE_Go_COMPILER_WORKS 1 CACHE INTERNAL "")

69
cmake/local-cmake.sh Normal file
View file

@ -0,0 +1,69 @@
#!/bin/bash
# which version to fetch
MAJOR="3.7"
MINOR="2"
PLATFORM="Linux-x86_64"
# assemble url for desired version
URL="https://cmake.org/files/v${MAJOR}/cmake-${MAJOR}.${MINOR}-${PLATFORM}.tar.gz"
ARCHIVE="$(basename ${URL})"
DIR="$(basename ${ARCHIVE} .tar.gz)"
relpath() {
# workaround because Ubuntu seems to use an ancient realpath version
# https://stackoverflow.com/questions/2564634/convert-absolute-path-into-relative-path-given-a-current-directory-using-bash#comment12808306_7305217
python -c "import os.path; print(os.path.relpath('${2:-$PWD}','$1'))";
}
HERMIT_TOP="$(git rev-parse --show-toplevel)"
HERMIT_CMAKE="${HERMIT_TOP}/cmake"
CMAKE_DIR="${HERMIT_CMAKE}/${DIR}"
CMAKE_DIR_REL="$(relpath ${HERMIT_TOP} ${CMAKE_DIR})"
# make sure we're sourced, not executed
if [ "$0" = "$BASH_SOURCE" ]
then
echo "You have to source this script:"
echo "\$ . $0"
exit
fi
# quit if already in path
echo "$PATH" | grep "${CMAKE_DIR_REL}" &>/dev/null && return
# check if already installed
if which cmake &> /dev/null ; then
if cmake --version | grep "cmake version ${MAJOR}.${MINOR}" &> /dev/null; then
echo "You already have CMake ${MAJOR}.${MINOR}"
return
fi
fi
if [ ! -d "${CMAKE_DIR}" ]
then
echo "-- Downloading CMake"
wget "${URL}" -O "${ARCHIVE}" ||
(echo "Cannot download CMake"; return)
echo "-- Unpacking CMake"
tar -C "${HERMIT_CMAKE}" -xf "${ARCHIVE}" ||
(echo "Cannot unpack CMake archive"; return)
# delete temporary archive again
rm -f "${ARCHIVE}"
# add cmake dir to gitignore
GITIGNORE="${HERMIT_TOP}/.gitignore"
if ! grep "${CMAKE_DIR_REL}" "${GITIGNORE}" &>/dev/null
then
echo "${CMAKE_DIR_REL}/*" >> "${GITIGNORE}"
fi
echo "-- Local CMake v${MAJOR}.${MINOR} installed to ${CMAKE_DIR_REL}"
echo "-- Next time you source this script, no download will be neccessary"
fi
export PATH="${CMAKE_DIR}/bin:${PATH}"

View file

@ -0,0 +1,8 @@
cmake_minimum_required(VERSION 3.7)
configure_file(config.h.in config.h)
configure_file(config.asm.in config.asm)
# Show include files in IDE
file(GLOB_RECURSE HERMIT_INCLUDES "*")
add_custom_target(hermit_includes_ide SOURCES ${HERMIT_INCLUDES})

View file

@ -0,0 +1,3 @@
%define MAX_CORES @MAX_CORES@
%define KERNEL_STACK_SIZE @KERNEL_STACK_SIZE@
%define SAVE_FPU @SAVE_FPU@

53
include/hermit/config.h.in Executable file → Normal file
View file

@ -1,50 +1,25 @@
/* include/hermit/config.h.in. Generated from configure.ac by autoheader. */
#cmakedefine MAX_CORES (@MAX_CORES@)
#cmakedefine MAX_TASKS (@MAX_TASKS@)
#cmakedefine MAX_ISLE (@MAX_ISLE@)
#cmakedefine KERNEL_STACK_SIZE (@KERNEL_STACK_SIZE@)
#cmakedefine DEFAULT_STACK_SIZE (@DEFAULT_STACK_SIZE@)
#cmakedefine PACKAGE_VERSION "@PACKAGE_VERSION@"
#undef MAX_CORES
#undef MAX_TASKS
#undef MAX_ISLE
#undef MAX_FNAME
#undef SAVE_FPU
#undef VIDEO_MEM_ADDR
#cmakedefine SAVE_FPU
#cmakedefine DYNAMIC_TICKS
/* Define to use machine specific version of memcpy */
#undef HAVE_ARCH_MEMCPY
#cmakedefine HAVE_ARCH_MEMCPY
/* Define to use machine specific version of memset */
#undef HAVE_ARCH_MEMSET
#cmakedefine HAVE_ARCH_MEMSET
/* Define to use machine specific version of strcpy */
#undef HAVE_ARCH_STRCPY
#cmakedefine HAVE_ARCH_STRCPY
/* Define to use machine specific version of strlen */
#undef HAVE_ARCH_STRLEN
#cmakedefine HAVE_ARCH_STRLEN
/* Define to use machine specific version of strncpy */
#undef HAVE_ARCH_STRNCPY
/* Define the stack size of the idle task */
#undef KERNEL_STACK_SIZE
/* Define the default stack size */
#undef DEFAULT_STACK_SIZE
/* Define the maximum number of running tasks */
#undef MAX_TASKS
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the home page for this package. */
#undef PACKAGE_URL
/* Define to the version of this package. */
#undef PACKAGE_VERSION
#cmakedefine HAVE_ARCH_STRNCPY

19
tools/CMakeLists.txt Normal file
View file

@ -0,0 +1,19 @@
cmake_minimum_required(VERSION 3.7)
project(hermit_tools)
include(../cmake/HermitCore-Paths.cmake)
add_compile_options(-std=c99)
add_executable(proxy proxy.c uhyve.c)
target_link_libraries(proxy -pthread)
install(TARGETS proxy
DESTINATION bin)
install(FILES init.sh
DESTINATION tools)
# Show include files in IDE
file(GLOB_RECURSE TOOLS_INCLUDES "*.h")
add_custom_target(tools_includes_ide SOURCES ${TOOLS_INCLUDES})

View file

@ -1,6 +1,6 @@
#!/bin/sh
PROXY=/hermit/tools/proxy
PROXY=/hermit/bin/proxy
ELF_OSABI_OFFSET=7
ELF_OSABI="\\x42"

View file

@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore-Application.cmake)
project(hermit_benchmarks C)
add_executable(basic basic.c)
target_link_libraries(basic pthread)
add_executable(hg hg.c hist.c rdtsc.c run.c init.c opt.c report.c setup.c)
add_executable(netio netio.c)
add_executable(RCCE_pingpong RCCE_pingpong.c)
target_link_libraries(RCCE_pingpong ircce)
add_executable(stream stream.c)
target_compile_options(stream PRIVATE -fopenmp)
target_link_libraries(stream -fopenmp)
# deployment
install_local_targets(extra/benchmarks)

18
usr/ircce/CMakeLists.txt Normal file
View file

@ -0,0 +1,18 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore.cmake)
project(hermit_ircce C)
add_compile_options(${HERMIT_APP_FLAGS})
file(GLOB SOURCES *.c)
add_library(ircce STATIC ${SOURCES})
# deployment
install(TARGETS ircce
DESTINATION ${TARGET_ARCH}/lib)
install(FILES
iRCCE.h iRCCE_lib.h RCCE_debug.h RCCE.h RCCE_lib.h rte_memcpy.h
DESTINATION
${TARGET_ARCH}/include)

View file

@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore-Application.cmake)
project(hermit_openmpbench C)
add_executable(syncbench syncbench.c common.c)
target_link_libraries(syncbench PUBLIC -fopenmp)
add_executable(taskbench taskbench.c common.c)
target_link_libraries(taskbench PUBLIC -fopenmp)
add_library(common_sched STATIC common.c)
target_compile_definitions(common_sched PUBLIC -DSCHEDBENCH)
target_compile_options(common_sched PUBLIC -fopenmp)
target_link_libraries(common_sched PUBLIC -fopenmp)
add_executable(schedbench schedbench.c)
target_link_libraries(schedbench common_sched)
# deployment: exclude common_sched
install_local_targets(extra/benchmarks common_sched)

View file

@ -45,9 +45,9 @@ int main(int argc, char **argv) {
struct XRayTraceCapture* trace = XRayInit(
20, // max. call depth
32 * 1000 * 1000, // memory for report
16 * 1000 * 1000, // memory for report
13, // frame count
"/hermit/usr/openmpbench/syncbench.map");
"syncbench.map");
// Start Paraver tracing
#ifdef PARAVERTRACE
@ -128,7 +128,7 @@ int main(int argc, char **argv) {
#endif
XRaySaveReport(trace,
"/hermit/usr/openmpbench/syncbench.xray", // report file
"syncbench.xray", // report file
0.05f, // Only output funcs that have higher runtime [%]
1000); // Only output funcs that have higher runtime [cycles]
XRayShutdown(trace);

25
usr/tests/CMakeLists.txt Normal file
View file

@ -0,0 +1,25 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore-Application.cmake)
project(hermit_tests C CXX Fortran Go)
add_executable(hello hello.c)
add_executable(jacobi jacobi.c)
add_executable(hello++ hello++.cpp)
add_executable(hellof hellof.f90)
add_executable(pi pi.go)
add_executable(server server.go)
target_link_libraries(server netgo)
add_executable(RCCE_minimum RCCE_minimum.c)
target_link_libraries(RCCE_minimum ircce)
add_executable(thr_hello thr_hello.c)
target_link_libraries(thr_hello pthread)
add_executable(signals signals.c)
target_link_libraries(signals pthread)
# deployment
install_local_targets(extra/tests)

24
usr/xray/CMakeLists.txt Normal file
View file

@ -0,0 +1,24 @@
cmake_minimum_required(VERSION 3.7)
include(../../cmake/HermitCore.cmake)
project(hermit_xray C)
add_compile_options(${HERMIT_APP_FLAGS})
file(GLOB SOURCES *.c)
add_library(xray STATIC ${SOURCES})
target_compile_definitions(xray
PUBLIC
-DXRAY -DXRAY_ANNOTATE
-DXRAY_NO_DEMANGLE
-DXRAY_DISABLE_BROWSER_INTEGRATION)
# deployment
install(TARGETS xray
DESTINATION ${TARGET_ARCH}/lib)
install(FILES libxray.spec
DESTINATION ${TARGET_ARCH}/lib)
install(FILES xray.h
DESTINATION ${TARGET_ARCH}/include)

138
usr/xray/README.md Normal file
View file

@ -0,0 +1,138 @@
# Profiling with XRay
## Introduction
You can profile your application and parts of the system runtime using the XRay
profiler. It hooks into every function call (using GCC's
`-finstrument-functions` option) to record the execution time and create a call
graph.
It can generate a text-file report that lists the most expensive function calls
depending on the filtering that is configured.
## About XRay
XRay can divide the profiling into multiple "runs" called frames. In a graphical
application this could correspond to the rendering of a graphics frame, whereas
in a benchmark application a frame might correspond to each individual benchmark
run.
The profiling information is saved in a statically sized ring buffer so you must
decide on the size of the buffer and the max. number of frames. Those values
might need some fine tuning. If in doubt, increase the buffer size.
In order for XRay to resolve function names, a linker map file is needed. Using
this file, addresses can be resolved to function names.
## Limitations
If the compiler aggressively (or intendedly) inlines functions you won't see
them in the final report since no enter and exit hooks are inserted. Keep this
is mind if there's some function missing in the call hierarchy. Furthermore, the
name of static functions cannot be resolved because their names are not listed
in the linker file.
## Profile your application
To generate linker map files and inject enter and exit hooks, you have to tell
CMake that you want your application to be profiled:
```bash
$ cd build
$ cmake .. -DPROFILING=true
```
If you want to profile HermitCore internals or one of the example applications,
have a look at `CMakeLists.txt` in the root of the repository. Every target that
is built by `build_external(target_name ...)` can be profiled like this:
```bash
$ cd build
$ cmake .. -DPROFILE_APPS='openmpbench;tests'
```
### Code
You have to include the XRay header: `#include <xray.h>`.
Then you must initialize XRay and already do some configuration:
```c
struct XRayTraceCapture* XRayInit(int stack_size,
int buffer_size,
int frame_count,
const char* mapfilename);
struct XRayTraceCapture* trace = XRayInit(
5, // max. call depth in report
4 * 1000 * 1000, // ring buffer size for profiling information
10, // frame count
"/path/to/your/application.map");
```
To find the hotspots in your code you might want to start with a relatively
small call depth (maybe 5) and increase it to gain a better understanding of the
detailed call hierarchy. The maximum call depth / stack size is 255. Keep the
buffer size as small as possible and increase on demand.
Now you can wrap parts of your code into frames:
```c
XRayStartFrame(trace);
do_work();
XRayEndFrame(trace);
XRayStartFrame(trace);
do_even_more_work();
XRayEndFrame(trace);
```
And finally generate the report:
```c
void XRaySaveReport(struct XRayTraceCapture* capture,
const char* filename,
float percent_cutoff,
int cycle_cutoff);
XRaySaveReport(trace,
"/path/to/you/report/application.xray", // report file
10.0f, // Only output funcs that have a higher runtime [%]
2000); // Only output funcs that have a higher runtime [cycles]
XRayShutdown(trace);
```
Here you can do further filtering of the output. For a function call to be added
to the report, it's relative runtime (whole application) has be higher than
`percent_cutoff` and it's absolute runtime must be greater than `cycle_cutoff`
CPU cycles.
## Example
See [usr/openmpbench/syncbench.c](https://github.com/RWTH-OS/HermitCore/blob/master/usr/openmpbench/syncbench.c).
## Analysis
After tracing your code, you may want to analyse the report. While the XRay
report is already human-readable, it's hard to get an overview of the whole
trace. Therefore, it's possible to convert the XRay report to a format that
[kCacheGrind](https://kcachegrind.github.io) can read. You can find the tool
needed for conversion at `usr/xray/tools`.
```bash
$ ./conv2kcg.py libgomp_trace.xray
INFO:Parsing Header is done. Found 1 frames
INFO:Found frame 'PARALLEL' data
INFO:Frame 'PARALLEL' complete
INFO:Report file 'libgomp_trace.xray' parsed completely.
INFO:Create callgrind file for frame 'PARALLEL'
INFO:Writing to: libgomp_trace_PARALLEL.callgrind
```
This will create the file `libgomp_trace_PARALLEL.callgrind` which can be opened
using kCacheGrind (Open dialog: set Filter to 'All Files').