From 3942d62fe808a5b3d0d09ef254c2b488dd1d5e7a Mon Sep 17 00:00:00 2001 From: daniel-k Date: Wed, 1 Mar 2017 18:48:45 +0100 Subject: [PATCH 1/5] tools/uhyve: refactor syscalls definitions into new file --- tools/uhyve-syscalls.h | 64 ++++++++++++++++++++++++++++++++++++++++++ tools/uhyve.c | 39 +------------------------ 2 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 tools/uhyve-syscalls.h diff --git a/tools/uhyve-syscalls.h b/tools/uhyve-syscalls.h new file mode 100644 index 000000000..7b7998e03 --- /dev/null +++ b/tools/uhyve-syscalls.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2017, RWTH Aachen University + * Author(s): Daniel Krebs + * + * Permission to use, copy, modify, and/or distribute this software + * for any purpose with or without fee is hereby granted, provided + * that the above copyright notice and this permission notice appear + * in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL + * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE + * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef UHYVE_SYSCALLS_H +#define UHYVE_SYSCALLS_H + +#include +#include + +typedef enum { + UHYVE_PORT_WRITE = 0x499, + UHYVE_PORT_OPEN = 0x500, + UHYVE_PORT_CLOSE = 0x501, + UHYVE_PORT_READ = 0x502, + UHYVE_PORT_EXIT = 0x503, + UHYVE_PORT_LSEEK = 0x504 +} uhyve_syscall_t; + +typedef struct { + int fd; + const char* buf; + size_t len; +} __attribute__((packed)) uhyve_write_t; + +typedef struct { + const char* name; + int flags; + int mode; + int ret; +} __attribute__((packed)) uhyve_open_t; + +typedef struct { + int fd; + int ret; +} __attribute__((packed)) uhyve_close_t; + +typedef struct { + int fd; + char* buf; + size_t len; + ssize_t ret; +} __attribute__((packed)) uhyve_read_t; + +typedef struct { + int fd; + off_t offset; + int whence; +} __attribute__((packed)) uhyve_lseek_t; + +#endif // UHYVE_SYSCALLS_H diff --git a/tools/uhyve.c b/tools/uhyve.c index 791edfba2..fbb4ad041 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -58,6 +58,7 @@ #include #include "uhyve-cpu.h" +#include "uhyve-syscalls.h" #include "proxy.h" #define GUEST_OFFSET 0x0 @@ -79,13 +80,6 @@ #define KVM_32BIT_GAP_SIZE (768 << 20) #define KVM_32BIT_GAP_START (KVM_32BIT_MAX_MEM_SIZE - KVM_32BIT_GAP_SIZE) -#define UHYVE_PORT_WRITE 0x499 -#define UHYVE_PORT_OPEN 0x500 -#define UHYVE_PORT_CLOSE 0x501 -#define UHYVE_PORT_READ 0x502 -#define UHYVE_PORT_EXIT 0x503 -#define UHYVE_PORT_LSEEK 0x504 - #define kvm_ioctl(fd, cmd, arg) ({ \ int ret = ioctl(fd, cmd, arg); \ if(ret == -1) \ @@ -104,37 +98,6 @@ static int kvm = -1, vmfd = -1; static __thread struct kvm_run *run = NULL; static __thread int vcpufd = 1; -typedef struct { - int fd; - const char* buf; - size_t len; -} __attribute__((packed)) uhyve_write_t; - -typedef struct { - const char* name; - int flags; - int mode; - int ret; -} __attribute__((packed)) uhyve_open_t; - -typedef struct { - int fd; - int ret; -} __attribute__((packed)) uhyve_close_t; - -typedef struct { - int fd; - char* buf; - size_t len; - ssize_t ret; -} __attribute__((packed)) uhyve_read_t; - -typedef struct { - int fd; - off_t offset; - int whence; -} __attribute__((packed)) uhyve_lseek_t; - static size_t memparse(const char *ptr) { char *endptr; /* local pointer to end of parsed string */ From 728b4658268d4a79b30b415736d18611152b57bd Mon Sep 17 00:00:00 2001 From: daniel-k Date: Wed, 1 Mar 2017 18:55:23 +0100 Subject: [PATCH 2/5] tools/uhyve: general refactoring --- tools/uhyve.c | 227 +++++++++++++++++++++++++------------------------- 1 file changed, 114 insertions(+), 113 deletions(-) diff --git a/tools/uhyve.c b/tools/uhyve.c index fbb4ad041..d1b88dfbc 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -81,7 +81,7 @@ #define KVM_32BIT_GAP_START (KVM_32BIT_MAX_MEM_SIZE - KVM_32BIT_GAP_SIZE) #define kvm_ioctl(fd, cmd, arg) ({ \ - int ret = ioctl(fd, cmd, arg); \ + const int ret = ioctl(fd, cmd, arg); \ if(ret == -1) \ err(1, "KVM: ioctl " #cmd " failed"); \ ret; \ @@ -98,116 +98,119 @@ static int kvm = -1, vmfd = -1; static __thread struct kvm_run *run = NULL; static __thread int vcpufd = 1; -static size_t memparse(const char *ptr) +static uint64_t memparse(const char *ptr) { - char *endptr; /* local pointer to end of parsed string */ - size_t ret = strtoull(ptr, &endptr, 0); + // local pointer to end of parsed string + char *endptr; + // parse number + uint64_t size = strtoull(ptr, &endptr, 0); + + // parse size extension, intentional fall-through switch (*endptr) { - case 'E': - case 'e': - ret <<= 10; - case 'P': - case 'p': - ret <<= 10; - case 'T': - case 't': - ret <<= 10; - case 'G': - case 'g': - ret <<= 10; - case 'M': - case 'm': - ret <<= 10; - case 'K': - case 'k': - ret <<= 10; - endptr++; - default: - break; - } + case 'E': + case 'e': + size <<= 10; + case 'P': + case 'p': + size <<= 10; + case 'T': + case 't': + size <<= 10; + case 'G': + case 'g': + size <<= 10; + case 'M': + case 'm': + size <<= 10; + case 'K': + case 'k': + size <<= 10; + endptr++; + default: + break; + } - return ret; + return size; +} + +/// Just close file descriptor if not already done +static inline void close_fd(int* fd) +{ + if(*fd != -1) { + close(*fd); + *fd = -1; + } } static void sig_func(int sig) { - if (vcpufd != -1) - close(vcpufd); - vcpufd = -1; + (void) sig; - pthread_exit(0); + close_fd(&vcpufd); + pthread_exit(NULL); } static void uhyve_exit(void) { - char* str = getenv("HERMIT_VERBOSE"); - + // only the main thread will execute this if (vcpu_threads) { - for(uint32_t i=0; i '9')) && (*match != '\0') ) + match++; - while(fgets(line, 2048, fp)) { - if ((match = strstr(line, "cpu MHz")) == NULL) - continue; + freq = (uint32_t) atoi(match); + break; + } + } - // scan strinf for the next number - for(; (*match < 0x30) || (*match > 0x39); match++) - ; - - for(point = match; ((*point != '.') && (*point != '\0')); point++) - ; - *point = '\0'; - - freq = atoi(match); fclose(fp); - - return freq; } return freq; @@ -336,25 +339,27 @@ out: return 0; } +/// Filter CPUID functions that are not supported by the hypervisor and enable +/// features according to our needs. static void filter_cpuid(struct kvm_cpuid2 *kvm_cpuid) { - /* - * Filter CPUID functions that are not supported by the hypervisor. - */ for (uint32_t i = 0; i < kvm_cpuid->nent; i++) { struct kvm_cpuid_entry2 *entry = &kvm_cpuid->entries[i]; switch (entry->function) { - case 1: // CPUID to define basic cpu features - entry->ecx = entry->ecx | (1 << 31); // propagate that we are running on a hypervisor - //entry->ecx = entry->ecx & ~(1 << 21); // disable X2APIC support - entry->edx = entry->edx | (1 << 5); // enable msr support + case 1: + // CPUID to define basic cpu features + entry->ecx |= (1U << 31); // propagate that we are running on a hypervisor + entry->edx |= (1U << 5); // enable msr support break; + case CPUID_FUNC_PERFMON: - entry->eax = 0x00; /* disable it */ + // disable it + entry->eax = 0x00; break; + default: - /* Keep the CPUID function as -is */ + // Keep the CPUID function as-is break; }; } @@ -429,16 +434,13 @@ static void setup_system(int vcpufd, uint8_t *mem, uint32_t id) // all cores use the same startup code // => all cores use the same sregs // => only the boot processor has to initialize sregs - if (id == 0) - { + if (id == 0) { kvm_ioctl(vcpufd, KVM_GET_SREGS, &sregs); /* Set all cpu/mem system structures */ setup_system_gdt(&sregs, mem, BOOT_GDT); setup_system_page_tables(&sregs, mem); setup_system_64bit(&sregs); - - //printf("APIC is located at 0x%zx\n", (size_t)sregs.apic_base); } kvm_ioctl(vcpufd, KVM_SET_SREGS, &sregs); @@ -448,14 +450,17 @@ static void setup_system(int vcpufd, uint8_t *mem, uint32_t id) static void setup_cpuid(int kvm, int vcpufd) { struct kvm_cpuid2 *kvm_cpuid; - int max_entries = 100; + unsigned int max_entries = 100; - kvm_cpuid = calloc(1, sizeof(*kvm_cpuid) + max_entries * sizeof(*kvm_cpuid->entries)); + // allocate space for cpuid we get from KVM + kvm_cpuid = calloc(1, sizeof(*kvm_cpuid) + + (max_entries * sizeof(kvm_cpuid->entries[0])) ); kvm_cpuid->nent = max_entries; + kvm_ioctl(kvm, KVM_GET_SUPPORTED_CPUID, kvm_cpuid); + // set features filter_cpuid(kvm_cpuid); - kvm_ioctl(vcpufd, KVM_SET_CPUID2, kvm_cpuid); } @@ -587,18 +592,9 @@ static int vcpu_init(uint32_t id) /* Setup registers and memory. */ setup_system(vcpufd, guest_mem, id); - /* - * Initialize registers: instruction pointer for our code, addends, - * and initial flags required by x86 architecture. - * Arguments to the kernel main are passed using the x86_64 calling - * convention: RDI, RSI, RDX, RCX, R8, and R9 - */ struct kvm_regs regs = { - .rip = elf_entry, - .rax = 2, - .rbx = 2, - .rdx = 0, - .rflags = 0x2, + .rip = elf_entry, // entry point to HermitCore + .rflags = 0x2, // POR value required by x86 architecture }; kvm_ioctl(vcpufd, KVM_SET_REGS, ®s); @@ -625,11 +621,13 @@ static int vcpu_init(uint32_t id) static void* uhyve_thread(void* arg) { - size_t id = (size_t) arg; - size_t ret; + const size_t id = (size_t) arg; + // create new cpu vcpu_init(id); - ret = vcpu_loop(); + + // run cpu loop until thread gets killed + const size_t ret = vcpu_loop(); return (void*) ret; } @@ -642,9 +640,9 @@ int uhyve_init(char *path) // register routine to close the VM atexit(uhyve_exit); - char* str = getenv("HERMIT_MEM"); - if (str) - guest_size = memparse(str); + const char* hermit_memory = getenv("HERMIT_MEM"); + if (hermit_memory) + guest_size = memparse(hermit_memory); kvm = open("/dev/kvm", O_RDWR | O_CLOEXEC); if (kvm < 0) @@ -679,28 +677,31 @@ int uhyve_init(char *path) kvm_ioctl(vmfd, KVM_SET_USER_MEMORY_REGION, &kvm_region); kvm_ioctl(vmfd, KVM_CREATE_IRQCHIP, NULL); - //kvm_ioctl(vmfd, KVM_SET_BOOT_CPU_ID, 0); + // create first CPU, it will be the boot processor by default return vcpu_init(0); } int uhyve_loop(void) { - char* str = getenv("HERMIT_CPUS"); + const char* hermit_cpus = getenv("HERMIT_CPUS"); + if (hermit_cpus) + ncores = (uint32_t) atoi(hermit_cpus); - if (str) - ncores = atoi(str); *((uint32_t*) (mboot+0x24)) = ncores; vcpu_threads = (pthread_t*) calloc(ncores, sizeof(pthread_t)); if (!vcpu_threads) - err(1, "Not enough memoyr"); + err(1, "Not enough memory"); + // First CPU is special because it will boot the system. Other CPUs will + // be booted linearily after the first one. vcpu_threads[0] = pthread_self(); - // start threads to create VCPU - for(size_t i=1; i Date: Wed, 1 Mar 2017 18:49:23 +0100 Subject: [PATCH 3/5] tools/proxy: use switch-case for monitor enum --- tools/proxy.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/proxy.c b/tools/proxy.c index ef78368fa..a06e182f8 100644 --- a/tools/proxy.c +++ b/tools/proxy.c @@ -1043,7 +1043,18 @@ int main(int argc, char **argv) if (ret) return ret; - if (monitor != UHYVE) + + switch(monitor) { + case UHYVE: + return uhyve_loop(); + + case BAREMETAL: + case QEMU: return socket_loop(argc, argv); - return uhyve_loop(); + + default: + perror("Unknown monitor"); + } + + return 1; } From 76f9ad80f700f5094975283c39956791ede18829 Mon Sep 17 00:00:00 2001 From: daniel-k Date: Wed, 1 Mar 2017 18:49:55 +0100 Subject: [PATCH 4/5] tools/uhyve-cpu: remove unused definition of _kvm_segment --- tools/uhyve-cpu.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/uhyve-cpu.h b/tools/uhyve-cpu.h index 26744ec99..4a0f23084 100644 --- a/tools/uhyve-cpu.h +++ b/tools/uhyve-cpu.h @@ -70,16 +70,6 @@ (((base) & _AC(0x00ffffff, ULL)) << 16) | \ (((limit) & _AC(0x0000ffff, ULL)))) -struct _kvm_segment { - __u64 base; - __u32 limit; - __u16 selector; - __u8 type; - __u8 present, dpl, db, s, l, g, avl; - __u8 unusable; - __u8 padding; -}; - #define GDT_GET_G(x) (__u8)(((x) & 0x0080000000000000) >> 55) #define GDT_GET_DB(x) (__u8)(((x) & 0x0040000000000000) >> 54) #define GDT_GET_L(x) (__u8)(((x) & 0x0020000000000000) >> 53) From 5f949d4accb0a68604c676659f6a3815897a7cf8 Mon Sep 17 00:00:00 2001 From: daniel-k Date: Wed, 1 Mar 2017 18:51:32 +0100 Subject: [PATCH 5/5] tools/uhyve: fix memory leak --- tools/uhyve.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/uhyve.c b/tools/uhyve.c index d1b88dfbc..d7cae6196 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -462,6 +462,8 @@ static void setup_cpuid(int kvm, int vcpufd) // set features filter_cpuid(kvm_cpuid); kvm_ioctl(vcpufd, KVM_SET_CPUID2, kvm_cpuid); + + free(kvm_cpuid); } static int vcpu_loop(void)