diff --git a/hermit/Makefile b/hermit/Makefile index 6299f1a0a..77fbdd40e 100644 --- a/hermit/Makefile +++ b/hermit/Makefile @@ -54,10 +54,10 @@ ifdef PROFILING PROFILING_CFLAGS += -DXRAY_ANNOTATE endif -CFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -mno-red-zone -O3 -march=native -mtune=native -ftree-vectorize $(STACKPROT) -FCFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -mno-red-zone -O3 -march=native -mtune=native -ftree-vectorize -FFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -mno-red-zone -O3 -march=native -mtune=native -ftree-vectorize -CXXFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -mno-red-zone -O3 -march=native -mtune=native -ftree-vectorize +CFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -O3 -march=native -mtune=native -ftree-vectorize #$(STACKPROT) +FCFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -O3 -march=native -mtune=native -ftree-vectorize +FFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -O3 -march=native -mtune=native -ftree-vectorize +CXXFLAGS_FOR_NEWLIB = -m64 -mtls-direct-seg-refs -O3 -march=native -mtune=native -ftree-vectorize LDFLAGS_FOR_NEWLIB = NASMFLAGS_FOR_NEWLIB = -felf64 CFLAGS_FOR_TOOLS = -O2 -Wall diff --git a/hermit/arch/x86/include/asm/tss.h b/hermit/arch/x86/include/asm/tss.h index 43d4bdae1..e7a05e18d 100644 --- a/hermit/arch/x86/include/asm/tss.h +++ b/hermit/arch/x86/include/asm/tss.h @@ -59,9 +59,9 @@ typedef struct { uint16_t res6, bitmap; } __attribute__ ((packed)) tss_t; -/** @brief Set rsp0 in TSS of the current core +/** @brief Set rsp0 & ist1 in TSS of the current core */ -void tss_set_rsp0(size_t ptr); +void set_tss(size_t rsp0, size_t ist1); #ifdef __cplusplus } diff --git a/hermit/arch/x86/kernel/entry.asm b/hermit/arch/x86/kernel/entry.asm index a2bfc7897..d346fed8c 100644 --- a/hermit/arch/x86/kernel/entry.asm +++ b/hermit/arch/x86/kernel/entry.asm @@ -646,6 +646,9 @@ align 4096 global boot_stack boot_stack: TIMES (MAX_CORES*KERNEL_STACK_SIZE) DB 0xcd +global boot_ist +boot_ist: + TIMES KERNEL_STACK_SIZE DB 0xcd ; add some hints to the ELF file SECTION .note.GNU-stack noalloc noexec nowrite progbits diff --git a/hermit/arch/x86/kernel/gdt.c b/hermit/arch/x86/kernel/gdt.c index 047aa391b..b56f48381 100644 --- a/hermit/arch/x86/kernel/gdt.c +++ b/hermit/arch/x86/kernel/gdt.c @@ -35,11 +35,13 @@ #include #include +#define MAX_IST 3 + gdt_ptr_t gp; // currently, our kernel has full access to the ioports static gdt_entry_t gdt[GDT_ENTRIES] = {[0 ... GDT_ENTRIES-1] = {0, 0, 0, 0, 0, 0}}; static tss_t task_state_segments[MAX_CORES] __attribute__ ((aligned (PAGE_SIZE))); -static uint8_t stack_table[MAX_CORES*KERNEL_STACK_SIZE*3]; +static uint8_t stack_table[MAX_CORES*KERNEL_STACK_SIZE*MAX_IST] __attribute__ ((aligned (PAGE_SIZE))); extern const void boot_stack; @@ -51,9 +53,10 @@ extern void gdt_flush(void); extern const void boot_stack; -void tss_set_rsp0(size_t stptr) +void set_tss(size_t rps0, size_t ist1) { - task_state_segments[CORE_ID].rsp0 = stptr; + task_state_segments[CORE_ID].rsp0 = rps0; + task_state_segments[CORE_ID].ist1 = ist1; } /* Setup a descriptor in the Global Descriptor Table */ @@ -145,9 +148,10 @@ void gdt_install(void) */ for(i=0; irax, s->rbx, s->rcx, s->rdx, s->rbp, s->rsp, s->rdi, s->rsi, s->r8, s->r9, s->r10, s->r11, s->r12, s->r13, s->r14, s->r15); apic_eoi(s->int_no); - irq_enable(); //do_abort(); sys_exit(-EFAULT); } diff --git a/hermit/arch/x86/kernel/tasks.c b/hermit/arch/x86/kernel/tasks.c index 378a069c6..02447bab8 100644 --- a/hermit/arch/x86/kernel/tasks.c +++ b/hermit/arch/x86/kernel/tasks.c @@ -71,7 +71,7 @@ size_t* get_current_stack(void) stptr = (stptr + DEFAULT_STACK_SIZE - sizeof(size_t)) & ~0x1F; set_per_core(kernel_stack, stptr); - tss_set_rsp0(stptr); + set_tss(stptr, (size_t) curr_task->ist_addr + KERNEL_STACK_SIZE - 0x10); return curr_task->last_stack_pointer; } diff --git a/hermit/arch/x86/mm/page.c b/hermit/arch/x86/mm/page.c index cdd70edbb..b119ab029 100644 --- a/hermit/arch/x86/mm/page.c +++ b/hermit/arch/x86/mm/page.c @@ -271,7 +271,6 @@ default_handler: kprintf("Heap 0x%llx - 0x%llx\n", task->heap->start, task->heap->end); apic_eoi(s->int_no); - irq_enable(); //do_abort(); sys_exit(-EFAULT); } diff --git a/hermit/include/hermit/stdlib.h b/hermit/include/hermit/stdlib.h index 356669662..634d30376 100644 --- a/hermit/include/hermit/stdlib.h +++ b/hermit/include/hermit/stdlib.h @@ -73,11 +73,11 @@ void* kmalloc(size_t sz); /** @brief Create a stack with guard pages */ -void* create_stack(void); +void* create_stack(size_t sz); /** @brief Destroy stack with its guard pages */ -int destroy_stack(void* addr); +int destroy_stack(void* addr, size_t sz); /** @brief Release memory back to the buddy system * diff --git a/hermit/include/hermit/tasks_types.h b/hermit/include/hermit/tasks_types.h index af8687370..0c63a8ad6 100644 --- a/hermit/include/hermit/tasks_types.h +++ b/hermit/include/hermit/tasks_types.h @@ -80,6 +80,8 @@ typedef struct task { size_t* last_stack_pointer; /// start address of the stack void* stack; + /// interrupt stack for IST1 + void* ist_addr; /// Additional status flags. For instance, to signalize the using of the FPU uint8_t flags; /// Task priority diff --git a/hermit/kernel/tasks.c b/hermit/kernel/tasks.c index 73a76507d..9aae78185 100644 --- a/hermit/kernel/tasks.c +++ b/hermit/kernel/tasks.c @@ -36,6 +36,7 @@ #include #include #include +#include /* * Note that linker symbols are not variables, they have no memory allocated for @@ -51,8 +52,8 @@ extern const void tls_end; * A task's id will be its position in this array. */ static task_t task_table[MAX_TASKS] = { \ - [0] = {0, TASK_IDLE, 0, NULL, NULL, TASK_DEFAULT_FLAGS, 0, 0, 0, NULL, 0, NULL, NULL, 0, 0, 0}, \ - [1 ... MAX_TASKS-1] = {0, TASK_INVALID, 0, NULL, NULL, TASK_DEFAULT_FLAGS, 0, 0, 0, NULL, 0, NULL, NULL, 0, 0, 0}}; + [0] = {0, TASK_IDLE, 0, NULL, NULL, NULL, TASK_DEFAULT_FLAGS, 0, 0, 0, NULL, 0, NULL, NULL, 0, 0, 0}, \ + [1 ... MAX_TASKS-1] = {0, TASK_INVALID, 0, NULL, NULL, NULL, TASK_DEFAULT_FLAGS, 0, 0, 0, NULL, 0, NULL, NULL, 0, 0, 0}}; static spinlock_irqsave_t table_lock = SPINLOCK_IRQSAVE_INIT; @@ -69,6 +70,7 @@ DEFINE_PER_CORE(char*, kernel_stack, NULL); DEFINE_PER_CORE(uint32_t, __core_id, 0); #endif extern const void boot_stack; +extern const void boot_ist; /** @brief helper function for the assembly code to determine the current task * @return Pointer to the task_t structure of current task @@ -106,8 +108,10 @@ int multitasking_init(void) task_table[0].prio = IDLE_PRIO; task_table[0].stack = (char*) ((size_t)&boot_stack + core_id * KERNEL_STACK_SIZE); + task_table[0].ist_addr = (char*)&boot_ist; set_per_core(kernel_stack, task_table[0].stack + KERNEL_STACK_SIZE - 0x10); set_per_core(current_task, task_table+0); + set_tss((size_t) task_table[0].stack + KERNEL_STACK_SIZE - 0x10, (size_t) task_table[0].ist_addr + KERNEL_STACK_SIZE - 0x10); readyqueues[core_id].idle = task_table+0; @@ -158,6 +162,7 @@ int set_idle_task(void) task_table[i].last_core = core_id; task_table[i].last_stack_pointer = NULL; task_table[i].stack = (char*) ((size_t)&boot_stack + core_id * KERNEL_STACK_SIZE); + task_table[i].ist_addr = create_stack(KERNEL_STACK_SIZE); set_per_core(kernel_stack, task_table[i].stack + KERNEL_STACK_SIZE - 0x10); task_table[i].prio = IDLE_PRIO; task_table[i].heap = NULL; @@ -214,7 +219,7 @@ void finish_task_switch(void) /* cleanup task */ if (old->stack) { kprintf("Release stack at 0x%zx\n", old->stack); - destroy_stack(old->stack); + destroy_stack(old->stack, DEFAULT_STACK_SIZE); old->stack = NULL; } @@ -223,6 +228,11 @@ void finish_task_switch(void) old->heap = NULL; } + if (old->ist_addr) { + destroy_stack(old->ist_addr, KERNEL_STACK_SIZE); + old->ist_addr = NULL; + } + old->last_stack_pointer = NULL; readyqueues[core_id].old_task = NULL; @@ -322,6 +332,7 @@ int clone_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio) int ret = -EINVAL; uint32_t i; void* stack = NULL; + void* ist = NULL; task_t* curr_task; uint32_t core_id; @@ -334,10 +345,16 @@ int clone_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio) curr_task = per_core(current_task); - stack = create_stack(); + stack = create_stack(DEFAULT_STACK_SIZE); if (BUILTIN_EXPECT(!stack, 0)) return -ENOMEM; + ist = create_stack(KERNEL_STACK_SIZE); + if (BUILTIN_EXPECT(!ist, 0)) { + destroy_stack(stack, DEFAULT_STACK_SIZE); + return -ENOMEM; + } + spinlock_irqsave_lock(&table_lock); core_id = get_next_core_id(); @@ -357,6 +374,7 @@ int clone_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio) task_table[i].parent = curr_task->id; task_table[i].tls_addr = curr_task->tls_addr; task_table[i].tls_size = curr_task->tls_size; + task_table[i].ist_addr = ist; task_table[i].lwip_err = 0; if (id) @@ -392,7 +410,7 @@ int clone_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio) out: if (ret) - destroy_stack(stack); + destroy_stack(stack, DEFAULT_STACK_SIZE); #if 0 if (core_id != CORE_ID) @@ -407,6 +425,7 @@ int create_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio, uint32_t c int ret = -ENOMEM; uint32_t i; void* stack = NULL; + void* ist = NULL; void* counter = NULL; if (BUILTIN_EXPECT(!ep, 0)) @@ -420,13 +439,20 @@ int create_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio, uint32_t c if (BUILTIN_EXPECT(!readyqueues[core_id].idle, 0)) return -EINVAL; - stack = create_stack(); + stack = create_stack(DEFAULT_STACK_SIZE); if (BUILTIN_EXPECT(!stack, 0)) return -ENOMEM; + ist = create_stack(KERNEL_STACK_SIZE); + if (BUILTIN_EXPECT(!ist, 0)) { + destroy_stack(stack, DEFAULT_STACK_SIZE); + return -ENOMEM; + } + counter = kmalloc(sizeof(atomic_int64_t)); if (BUILTIN_EXPECT(!counter, 0)) { - destroy_stack(stack); + destroy_stack(stack, KERNEL_STACK_SIZE); + destroy_stack(stack, DEFAULT_STACK_SIZE); return -ENOMEM; } atomic_int64_set((atomic_int64_t*) counter, 0); @@ -444,6 +470,7 @@ int create_task(tid_t* id, entry_point_t ep, void* arg, uint8_t prio, uint32_t c task_table[i].heap = NULL; task_table[i].start_tick = get_clock_tick(); task_table[i].parent = 0; + task_table[i].ist_addr = ist; task_table[i].tls_addr = 0; task_table[i].tls_size = 0; task_table[i].lwip_err = 0; @@ -482,7 +509,7 @@ out: spinlock_irqsave_unlock(&table_lock); if (ret) { - destroy_stack(stack); + destroy_stack(stack, DEFAULT_STACK_SIZE); kfree(counter); } diff --git a/hermit/mm/malloc.c b/hermit/mm/malloc.c index 5c30fbe97..acd2ec4b3 100644 --- a/hermit/mm/malloc.c +++ b/hermit/mm/malloc.c @@ -167,14 +167,17 @@ void* palloc(size_t sz, uint32_t flags) return (void*) viraddr; } -void* create_stack(void) +void* create_stack(size_t sz) { size_t phyaddr, viraddr, bits; - uint32_t npages = PAGE_FLOOR(DEFAULT_STACK_SIZE) >> PAGE_BITS; + uint32_t npages = PAGE_FLOOR(sz) >> PAGE_BITS; int err; //kprintf("create_stack(0x%zx) (%lu pages)\n", DEFAULT_STACK_SIZE, npages); + if (BUILTIN_EXPECT(!sz, 0)) + return NULL; + // get free virtual address space viraddr = vma_alloc((npages+2)*PAGE_SIZE, VMA_READ|VMA_WRITE|VMA_CACHEABLE); if (BUILTIN_EXPECT(!viraddr, 0)) @@ -203,15 +206,17 @@ void* create_stack(void) return (void*) (viraddr+PAGE_SIZE); } -int destroy_stack(void* viraddr) +int destroy_stack(void* viraddr, size_t sz) { size_t phyaddr; - uint32_t npages = PAGE_FLOOR(DEFAULT_STACK_SIZE) >> PAGE_BITS; + uint32_t npages = PAGE_FLOOR(sz) >> PAGE_BITS; //kprintf("destroy_stack(0x%zx) (size 0x%zx)\n", viraddr, DEFAULT_STACK_SIZE); if (BUILTIN_EXPECT(!viraddr, 0)) - return -ENOMEM; + return -EINVAL; + if (BUILTIN_EXPECT(!sz, 0)) + return -EINVAL; phyaddr = virt_to_phys((size_t)viraddr); if (BUILTIN_EXPECT(!phyaddr, 0)) diff --git a/hermit/usr/benchmarks/Makefile b/hermit/usr/benchmarks/Makefile index bbbee7af5..0006125fe 100644 --- a/hermit/usr/benchmarks/Makefile +++ b/hermit/usr/benchmarks/Makefile @@ -62,7 +62,7 @@ stream: stream.o stream_icc.o: stream.c @echo [ICC] $@ - icc -m64 -c -o $@ -mno-red-zone -O3 -xHost -fno-stack-protector -I$(TOPDIR)/hermit/usr/x86/x86_64-hermit/include/ -openmp -ffreestanding $< + icc -m64 -c -o $@ -O3 -xHost -I$(TOPDIR)/hermit/usr/x86/x86_64-hermit/include/ -openmp -ffreestanding $< $Q$(ELFEDIT_FOR_TARGET) --output-osabi HermitCore $@ stream_icc: stream_icc.o