diff --git a/arch/aarch64/include/asm/page.h b/arch/aarch64/include/asm/page.h index b7562c6ec..53b4da747 100644 --- a/arch/aarch64/include/asm/page.h +++ b/arch/aarch64/include/asm/page.h @@ -240,4 +240,8 @@ static inline void tlb_flush_range(size_t start, size_t end) asm volatile ("isb" ::: "memory"); } +/** @brief Print the current page table tree + */ +void page_dump(void); + #endif diff --git a/arch/aarch64/include/asm/processor.h b/arch/aarch64/include/asm/processor.h index 3a1d2cb16..fd3686844 100644 --- a/arch/aarch64/include/asm/processor.h +++ b/arch/aarch64/include/asm/processor.h @@ -90,6 +90,13 @@ static inline uint32_t get_sctlr(void) return status; } +static inline uint64_t get_elr(void) +{ + uint64_t ret; + asm volatile("mrs %0, elr_el1" : "=r"(ret)); + return ret; +} + /** @brief get the current exception level * * Helper function to get the current exception level @@ -102,30 +109,16 @@ static inline uint32_t get_current_el(void) return (curr>>2) & 0x3; } -/** @brief Get thread local storage - * - * Helper function to get the TLS of the current task - */ -static inline size_t get_tls(void) { - uint64_t addr = 0; - asm volatile( - "mrs %0, tpidr_el0" - : "+r"(addr) - : - : ); +/** @brief Get thread id register */ +static inline size_t get_tpidr(void) { + uint64_t addr; + asm volatile("mrs %0, tpidr_el0" : "=r"(addr)); return addr; } -/** @brief Set thread local storage - * - * Helper function to set the TLS of the current task - */ -static inline void set_tls(size_t addr) { - asm volatile( - "msr tpidr_el0, %0" - : "=r"(addr) - : - : ); +/** @brief Set thread id register */ +static inline void set_tpidr(size_t addr) { + asm volatile("msr tpidr_el0, %0" :: "r"(addr)); } /** @brief Read id_aa64mmfr0_el1 register @@ -188,7 +181,7 @@ static inline void write_mair(size_t val) { /** @brief Read ttbr0_el1 register * @return ttbr0_el1's value */ -static inline size_t read_ttbr0(void) { +static inline size_t get_ttbr0(void) { size_t val; asm volatile("mrs %0, ttbr0_el1" : "=r"(val) :: "memory"); return val; @@ -197,14 +190,14 @@ static inline size_t read_ttbr0(void) { /** @brief Write a value into ttbr0_el1 register * @param val The value you want to write into ttbr0_el1 */ -static inline void write_ttbr0(size_t val) { +static inline void set_ttbr0(size_t val) { asm volatile("msr ttbr0_el1, %0" :: "r"(val) : "memory"); } /** @brief Read ttbr1_el1 register * @return ttbr1_el1's value */ -static inline size_t read_ttbr1(void) { +static inline size_t get_ttbr1(void) { size_t val; asm volatile("mrs %0, ttbr1_el1" : "=r"(val) :: "memory"); return val; @@ -213,7 +206,7 @@ static inline size_t read_ttbr1(void) { /** @brief Write a value into ttbr1_el1 register * @param val The value you want to write into ttbr1_el1 */ -static inline void write_ttbr1(size_t val) { +static inline void set_ttbr1(size_t val) { asm volatile("msr ttbr1_el1, %0" :: "r"(val) : "memory"); } diff --git a/arch/aarch64/include/asm/stddef.h b/arch/aarch64/include/asm/stddef.h index 01d5a36e2..23d083399 100644 --- a/arch/aarch64/include/asm/stddef.h +++ b/arch/aarch64/include/asm/stddef.h @@ -84,7 +84,7 @@ typedef wchar_t wint_t; struct state { uint64_t elr_el1; uint64_t spsr_el1; - uint64_t res; + uint64_t tpidr_el0; uint64_t x0; uint64_t x1; uint64_t x2; diff --git a/arch/aarch64/kernel/entry.S b/arch/aarch64/kernel/entry.S index 7eb570a9c..1fc77c60d 100644 --- a/arch/aarch64/kernel/entry.S +++ b/arch/aarch64/kernel/entry.S @@ -377,7 +377,9 @@ ldr \xreg, [sp], #8 stp x5, x6, [sp, #-16]! stp x3, x4, [sp, #-16]! stp x1, x2, [sp, #-16]! - str x0, [sp, #-16]! + + mrs x22, tpidr_el0 + stp x0, x22, [sp, #-16]! mrs x22, elr_el1 mrs x23, spsr_el1 @@ -389,7 +391,9 @@ ldr \xreg, [sp], #8 msr elr_el1, x22 msr spsr_el1, x23 - ldr x0, [sp], #16 + ldp x0, x22, [sp], #16 + msr tpidr_el0, x22 + ldp x1, x2, [sp], #16 ldp x3, x4, [sp], #16 ldp x5, x6, [sp], #16 diff --git a/arch/aarch64/kernel/irq.c b/arch/aarch64/kernel/irq.c index 554aeb243..b125d80ab 100644 --- a/arch/aarch64/kernel/irq.c +++ b/arch/aarch64/kernel/irq.c @@ -286,6 +286,10 @@ void do_sync(void *regs) return; LOG_ERROR("Unable to handle page fault at 0x%llx\n", far); + LOG_ERROR("Exception return address 0x%llx\n", get_elr()); + LOG_ERROR("Thread ID register 0x%llx\n", get_tpidr()); + LOG_ERROR("Table Base Register 0x%llx\n", get_ttbr0()); + LOG_ERROR("Exception Syndrome Register 0x%lx\n", esr); // send EOI gicc_write(GICC_EOIR, iar); diff --git a/arch/aarch64/kernel/tasks.c b/arch/aarch64/kernel/tasks.c index 6fe4d16d4..8279138f5 100644 --- a/arch/aarch64/kernel/tasks.c +++ b/arch/aarch64/kernel/tasks.c @@ -32,11 +32,6 @@ #include #include -#define TLS_ALIGNBITS 5 -#define TLS_ALIGNSIZE (1L << TLS_ALIGNBITS) -#define TSL_ALIGNMASK ((~0L) << TLS_ALIGNBITS) -#define TLS_FLOOR(addr) ((((size_t)addr) + TLS_ALIGNSIZE - 1) & TSL_ALIGNMASK) - extern int smp_main(void); /* @@ -45,12 +40,24 @@ extern int smp_main(void); */ extern const void tls_start; extern const void tls_end; +extern const void tdata_end; extern atomic_int32_t cpu_online; extern atomic_int32_t current_boot_id; -static char tls[16][DEFAULT_STACK_SIZE]; -static int id = 0; +typedef union dtv +{ + size_t counter; + struct { + void *val; + uint8_t is_static; + } pointer; +} dtv_t; + +typedef struct { + dtv_t *dtv; /* dtv */ + void *privat; /* private */ +} thread_block_t; static int init_tls(void) { @@ -58,33 +65,35 @@ static int init_tls(void) // do we have a thread local storage? if (((size_t) &tls_end - (size_t) &tls_start) > 0) { - char* tls_addr = NULL; - size_t tpidr_el0; + size_t tdata_size = (size_t) &tdata_end - (size_t) &tls_start; curr_task->tls_addr = (size_t) &tls_start; curr_task->tls_size = (size_t) &tls_end - (size_t) &tls_start; - //tls_addr = kmalloc(curr_task->tls_size + TLS_ALIGNSIZE + sizeof(size_t)); - tls_addr = tls[id]; - id++; - if (BUILTIN_EXPECT(!tls_addr, 0)) { + thread_block_t* tcb = (thread_block_t*) kmalloc(curr_task->tls_size+sizeof(thread_block_t)); + if (BUILTIN_EXPECT(!tcb, 0)) { LOG_ERROR("load_task: heap is missing!\n"); return -ENOMEM; } - memset(tls_addr, 0x00, TLS_ALIGNSIZE); - memcpy((void*) TLS_FLOOR(tls_addr), (void*) curr_task->tls_addr, curr_task->tls_size); - tpidr_el0 = (size_t) TLS_FLOOR(tls_addr) + curr_task->tls_size; - *((size_t*)tpidr_el0) = tpidr_el0; + memset((void*) tcb, 0x00, curr_task->tls_size+sizeof(thread_block_t)); + tcb->dtv = (dtv_t*) &tcb[1]; + memcpy((char*) tcb->dtv, (void*) curr_task->tls_addr, tdata_size); - // set tpidr_el0 register to the TLS segment - set_tls(tpidr_el0); - //LOG_INFO("TLS of task %d on core %d starts at 0x%zx (size 0x%zx)\n", curr_task->id, CORE_ID, TLS_FLOOR(tls_addr), curr_task->tls_size); - } else set_tls(0); // no TLS => clear tpidr_el0 register + set_tpidr((size_t) tcb); + LOG_INFO("TLS of task %d on core %d starts at 0x%zx (tls size 0x%zx)\n", curr_task->id, CORE_ID, get_tpidr(), curr_task->tls_size); + } else set_tpidr(0); // no TLS => clear tpidr_el0 register return 0; } +void destroy_tls(void) +{ + size_t tpidr = get_tpidr(); + if (tpidr) + kfree((void*)tpidr); +} + static int thread_entry(void* arg, size_t ep) { @@ -213,11 +222,3 @@ const int32_t is_uhyve(void) { return (uhyve != 0); } - -#if 0 -extern uint32_t single_kernel; -const int32_t is_single_kernel(void) -{ - return (single_kernel != 0); -} -#endif diff --git a/arch/aarch64/mm/page.c b/arch/aarch64/mm/page.c index f578a6600..68f706d27 100644 --- a/arch/aarch64/mm/page.c +++ b/arch/aarch64/mm/page.c @@ -91,8 +91,8 @@ int page_set_flags(size_t viraddr, uint32_t npages, int flags) int __page_map(size_t viraddr, size_t phyaddr, size_t npages, size_t bits) { int lvl, ret = -ENOMEM; - long vpn = viraddr >> PAGE_BITS; - long first[PAGE_LEVELS], last[PAGE_LEVELS]; + ssize_t vpn = viraddr >> PAGE_BITS; + ssize_t first[PAGE_LEVELS], last[PAGE_LEVELS]; size_t page_counter = 0; size_t cflags = 0; @@ -122,7 +122,7 @@ int __page_map(size_t viraddr, size_t phyaddr, size_t npages, size_t bits) self[lvl][vpn] = phyaddr | PT_PT; /* Fill new table with zeros */ - //LOG_INFO("Clear new page table at %p\n", &self[lvl-1][vpn<= 16)) cflags = PT_CONTIG; else if (cflags && !(viraddr & 0xFFFFULL) && (npages-page_counter < 16)) cflags = 0; +#endif if (bits & PG_DEVICE) self[lvl][vpn] = phyaddr | PT_DEVICE | cflags; @@ -195,8 +197,8 @@ int page_fault_handler(size_t viraddr) int check_pagetables(size_t vaddr) { int lvl; - long vpn = vaddr >> PAGE_BITS; - long index[PAGE_LEVELS]; + ssize_t vpn = vaddr >> PAGE_BITS; + ssize_t index[PAGE_LEVELS]; /* Calculate index boundaries for page map traversal */ for (lvl=0; lvlheap) && (viraddr >= task->heap->start) && (viraddr < task->heap->end)) { - size_t flags; - int ret; + //page_dump(); + if ((task->heap) && (viraddr >= task->heap->start) && (viraddr < task->heap->end)) { /* * do we have a valid page table entry? => flush TLB and return */ @@ -237,8 +238,8 @@ int page_fault_handler(size_t viraddr) goto default_handler; } - flags = PG_USER|PG_RW; - ret = __page_map(viraddr, phyaddr, 1, flags); + size_t flags = PG_USER|PG_RW; + int ret = __page_map(viraddr, phyaddr, 1, flags); if (BUILTIN_EXPECT(ret, 0)) { LOG_ERROR("map_region: could not map %#lx to %#lx, task = %u\n", phyaddr, viraddr, task->id); @@ -261,24 +262,98 @@ default_handler: // weak symbol is used to detect a Go application void __attribute__((weak)) runtime_osinit(); -/*static void dump_pgtable(void) +typedef size_t page_entry_t; + +/** @brief Get the corresponding virtual address to a page map entry */ +static inline size_t entry_to_virt(page_entry_t* entry, int level) { - size_t* l0 = &l0_pgtable; + size_t addr = (size_t) entry; - LOG_INFO("Dump page table tree: %p\n", l0); + addr <<= (level+1) * PAGE_MAP_BITS; + addr &= 0x0000FFFFFFFFFFFFULL; - for (int i=0; i<512; i++) { - if (l0[i] != 0) { - LOG_INFO("\tx[%d] = %zx\n", i, l0[i]); - size_t* l1 = (size_t*) l0[i]; - for(int j=0; j<512; j++) { - if (l1[j] != 0) { - LOG_INFO("\t\ty[%d] = %zx\n", j, l1[j]); + return addr; +} + +/** @brief Get the base address of the child table + * + * @param entry The parent entry + * @return The child entry + */ +static inline page_entry_t* get_child_entry(page_entry_t *entry) +{ + size_t child = (size_t) entry; + + child <<= PAGE_MAP_BITS; + child &= 0x0000FFFFFFFFFFFFULL; + + return (page_entry_t*) child; +} + +/** @brief Get the base address of the parent entry + * + * @param entry The child entry + * @return The parent entry + */ +static inline page_entry_t* get_parent_entry(page_entry_t *entry) +{ + ssize_t parent = (size_t) entry; + + parent >>= PAGE_MAP_BITS; + parent |= 0x0000FF8000000000ULL; //PAGE_MAP_PGT; + parent &= ~(sizeof(size_t) - 1); // align to page_entry_t + + return (page_entry_t*) parent; +} + +void page_dump(void) +{ + size_t flags = 0; + size_t start = 0; + size_t end; + + void print(size_t start, size_t end, size_t flags) { + size_t size = end - start; + + kprintf("%#018lx-%#018lx %#14x 0x%zx\n", start, end, size, flags & ~PAGE_MASK); + } + + void traverse(int level, page_entry_t* entry) { + page_entry_t* stop = entry + PAGE_MAP_ENTRIES; + for (; entry != stop; entry++) { + if (*entry) { + if (level) { // do "pre-order" traversal + // TODO: handle "inheritance" of page table flags (see get_page_flags()) + traverse(level-1, get_child_entry(entry)); + } else { + if (!flags) { + flags = *entry; //& ~PAGE_MASK; + start = entry_to_virt(entry, level); + } + else if ((flags & ~PAGE_MASK) != (*entry & ~PAGE_MASK)) { + end = entry_to_virt(entry, level); + print(start, end, flags); + + flags = *entry; // & ~PAGE_MASK; + start = end; + } } } + else if (flags) { + end = entry_to_virt(entry, level); + print(start, end, flags); + flags = 0; + } } } -}*/ + + kprintf("%-18s-%18s %14s %-6s\n", "start", "end", "size", "flags"); // header + + traverse(PAGE_LEVELS-1, (page_entry_t*) 0x0000FFFFFFFFF000ULL); + + if (flags) // workaround to print last mapping + print(start, 0L, flags); +} int page_init(void) { diff --git a/arch/x86_64/kernel/tasks.c b/arch/x86_64/kernel/tasks.c index c941376f0..eeb25c0b0 100644 --- a/arch/x86_64/kernel/tasks.c +++ b/arch/x86_64/kernel/tasks.c @@ -40,6 +40,7 @@ #include #include +#define TLS_OFFSET 8 #define TLS_ALIGNBITS 5 #define TLS_ALIGNSIZE (1L << TLS_ALIGNBITS) #define TSL_ALIGNMASK ((~0L) << TLS_ALIGNBITS) @@ -87,6 +88,18 @@ static int init_tls(void) return 0; } +void destroy_tls(void) +{ + task_t* curr_task = per_core(current_task); + + // do we need to release the TLS? + void* tls_addr = (void*)get_tls(); + if (tls_addr) { + //LOG_INFO("Release TLS at %p\n", (char*)tls_addr - curr_task->tls_size); + kfree((char*)tls_addr - curr_task->tls_size - TLS_OFFSET); + } +} + static int thread_entry(void* arg, size_t ep) { diff --git a/arch/x86_64/mm/page.c b/arch/x86_64/mm/page.c index 31af9547a..d91b8e85d 100644 --- a/arch/x86_64/mm/page.c +++ b/arch/x86_64/mm/page.c @@ -103,9 +103,9 @@ int page_set_flags(size_t viraddr, uint32_t npages, int flags) int __page_map(size_t viraddr, size_t phyaddr, size_t npages, size_t bits, uint8_t do_ipi) { int ret = -ENOMEM; - size_t vpn = viraddr >> PAGE_BITS; - size_t first[PAGE_LEVELS]; - size_t last[PAGE_LEVELS]; + ssize_t vpn = viraddr >> PAGE_BITS; + ssize_t first[PAGE_LEVELS]; + ssize_t last[PAGE_LEVELS]; int8_t send_ipi = 0; //kprintf("Map %d pages at 0x%zx\n", npages, viraddr); @@ -201,8 +201,8 @@ void page_fault_handler(struct state *s) int check_pagetables(size_t vaddr) { int lvl; - long vpn = vaddr >> PAGE_BITS; - long index[PAGE_LEVELS]; + ssize_t vpn = vaddr >> PAGE_BITS; + ssize_t index[PAGE_LEVELS]; /* Calculate index boundaries for page map traversal */ for (lvl=0; lvl copy first section to all other sections for(i=1; i 0) diff --git a/kernel/tasks.c b/kernel/tasks.c index fc558b0f8..9f1a8e8b8 100644 --- a/kernel/tasks.c +++ b/kernel/tasks.c @@ -47,8 +47,6 @@ extern atomic_int32_t cpu_online; volatile uint32_t go_down = 0; -#define TLS_OFFSET 8 - /** @brief Array of task structures (aka PCB) * * A task's id will be its position in this array. @@ -367,7 +365,6 @@ void finish_task_switch(void) void NORETURN do_exit(int arg) { task_t* curr_task = per_core(current_task); - void* tls_addr = NULL; const uint32_t core_id = CORE_ID; LOG_INFO("Terminate task: %u, return value %d\n", curr_task->id, arg); @@ -379,12 +376,8 @@ void NORETURN do_exit(int arg) readyqueues[core_id].nr_tasks--; spinlock_irqsave_unlock(&readyqueues[core_id].lock); - // do we need to release the TLS? - tls_addr = (void*)get_tls(); - if (tls_addr) { - //LOG_INFO("Release TLS at %p\n", (char*)tls_addr - curr_task->tls_size); - kfree((char*)tls_addr - curr_task->tls_size - TLS_OFFSET); - } + // release the thread local storage + destroy_tls(); curr_task->status = TASK_FINISHED; diff --git a/usr/tests/thr_hello.c b/usr/tests/thr_hello.c index e414778b4..cf40eb3ae 100644 --- a/usr/tests/thr_hello.c +++ b/usr/tests/thr_hello.c @@ -9,18 +9,20 @@ void* thread_func(void* arg) { id = *((int*) arg); - printf("Hello Thread!!! id = %d\n", id); + printf("Hello thread!!! id = %d\n", id); return 0; } int main(int argc, char** argv) { pthread_t threads[MAX_THREADS]; - int i, ret, param[MAX_THREADS]; + int param[MAX_THREADS]; - for(i=0; i