From a0bae35a884091d22c26bd7f7e47b057e59fec67 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 16 Aug 2011 03:29:54 -0700 Subject: [PATCH] add untested prototype of an SVM system --- arch/x86/include/asm/icc.h | 2 +- arch/x86/include/asm/processor.h | 9 +- arch/x86/include/asm/svm.h | 22 +++++ arch/x86/mm/page.c | 22 ++++- arch/x86/mm/svm.c | 149 +++++++++++++++++++++++++++++++ arch/x86/scc/icc.c | 6 +- include/metalsvm/vma.h | 9 +- kernel/main.c | 2 + kernel/tests.c | 12 ++- 9 files changed, 220 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index 9cca3390..33316d2f 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -40,7 +40,7 @@ typedef struct { extern bootinfo_t* bootinfo; #define ICC_TAG_IP 0 -#define ICC_TAG_SVM 1 +#define ICC_TAG_SVMREQUEST 1 #define ICC_TAG_PINGREQUEST 2 #define ICC_TAG_PINGRESPONSE 3 diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index da3b3556..5d79e4ac 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -106,7 +106,7 @@ inline static void flush_cache(void) { * The invd asm instruction which invalidates cache without writing back * is used here */ -inline static void invalid_cache(void) { +inline static void invalidate_cache(void) { asm volatile ("invd"); } @@ -123,6 +123,13 @@ inline static int get_return_value(void) { return ret; } +#ifdef CONFIG_ROCKCREEK +static inline void invalidate_cl1(void) +{ + asm volatile ( ".byte 0x0f; .byte 0x0a;\n" ); // CL1FLUSHMB +} +#endif + /* Force strict CPU ordering */ #ifdef CONFIG_ROCKCREEK inline static void mb(void) { asm volatile ("lock; addl $0,0(%%esp)" ::: "memory"); } diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 6c1e2d61..667d10a2 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -28,6 +28,14 @@ extern "C" { #ifdef CONFIG_ROCKCREEK +/** @brief Init routine of the SVM subsystem + * + * @return + * - 0 on success + * - -ENOMEM not enough memory + */ +int svm_init(void); + /** @brief Memory allocator of the SVM subsystem. * * Like RCCE function, belongs svmmalloc to the synchronous @@ -43,6 +51,20 @@ void* svmmalloc(size_t size); */ void svmfree(void* addr, size_t size); +/** @brief Request for exlusive access + * + * @return + * - 0 on success + */ +int svm_access_request(size_t addr); + +/** @brief emit page to core ue + * + * @return + * - 0 on success + */ +int svm_emit_page(size_t addr, int ue); + #endif #ifdef __cplusplus diff --git a/arch/x86/mm/page.c b/arch/x86/mm/page.c index 353f0626..5577e808 100644 --- a/arch/x86/mm/page.c +++ b/arch/x86/mm/page.c @@ -34,6 +34,7 @@ #ifdef CONFIG_ROCKCREEK #include #include +#include #include #endif @@ -357,7 +358,7 @@ size_t map_region(size_t viraddr, size_t phyaddr, uint32_t npages, uint32_t flag if (flags & MAP_SVM) pgt->entries[index] |= PG_SVM; if (flags & MAP_NO_ACCESS) - pgt->entries[index] &= ~(PG_PRESENT|PG_RW); + pgt->entries[index] &= ~PG_PRESENT; if (flags & MAP_USER_SPACE) atomic_int32_inc(&task->user_usage); @@ -399,8 +400,10 @@ int change_page_permissions(size_t start, size_t end, uint32_t flags) phyaddr = pgt->entries[index2] & 0xFFFFF000; newflags = pgt->entries[index2] & 0xFFF; // get old flags - if ((newflags & PG_SVM) && !(newflags & PG_PRESENT) && (flags & (VMA_WRITE|VMA_READ))) + if ((newflags & PG_SVM) && !(newflags & PG_PRESENT) && (flags & (VMA_READ|VMA_WRITE) && !(flags & VMA_NOACCESS))) newflags |= PG_PRESENT; + if ((newflags & PG_SVM) && (newflags & PG_PRESENT) && (flags & VMA_NOACCESS)) + newflags &= ~PG_PRESENT; // update flags if (!(flags & VMA_WRITE)) @@ -598,8 +601,11 @@ int print_paging_tree(size_t viraddr) static void pagefault_handler(struct state *s) { task_t* task = per_core(current_task); + page_dir_t* pgd = task->pgd; + page_table_t* pgt = NULL; size_t viraddr = read_cr2(); size_t phyaddr; + uint32_t index1, index2; if ((viraddr >= task->start_heap) && (viraddr <= task->end_heap) && (viraddr > KERNEL_SPACE)) { viraddr = viraddr & 0xFFFFF000; @@ -617,6 +623,18 @@ static void pagefault_handler(struct state *s) put_page(phyaddr); } + index1 = viraddr >> 22; + index2 = (viraddr >> 12) & 0x3FF; + + if (pgd) + pgt = (page_table_t*) (pgd->entries[index1] & 0xFFFFF000); + if (!pgt) + goto default_handler; + + if (pgt->entries[index2] & PG_SVM) + if (!svm_access_request(viraddr)) + return; + default_handler: kprintf("PAGE FAULT: Task %u got page fault at %p (irq %d, cs:eip 0x%x:0x%x)\n", task->id, viraddr, s->int_no, s->cs, s->eip); kprintf("Register state: eax = 0x%x, ebx = 0x%x, ecx = 0x%x, edx = 0x%x, edi = 0x%x, esi = 0x%x, ebp = 0x%x, esp = 0x%x\n", diff --git a/arch/x86/mm/svm.c b/arch/x86/mm/svm.c index 2fecbba7..0b5e0ae0 100644 --- a/arch/x86/mm/svm.c +++ b/arch/x86/mm/svm.c @@ -22,13 +22,113 @@ #include #include #include +#include #include #ifdef CONFIG_ROCKCREEK +#include #include +#include #include #include #include +#define SHARED_PAGES (RCCE_SHM_SIZE_MAX >> PAGE_SHIFT) +#define OWNER_SIZE ((SHARED_PAGES * sizeof(uint8_t) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) + +t_vcharp RC_SHM_BUFFER_START(); + +/* + * This array describes the owner of a specific page. + * Only the owner of a page is able to change the possession. + * => No lock is needded. + */ +static volatile uint8_t* page_owner = NULL; + +// helper array to convert a physical to a virtual address +static size_t phys2virt[SHARED_PAGES] = {[0 ... SHARED_PAGES-1] = 0}; +static size_t shmbegin = 0; +static int my_ue = 0; + +int svm_init(void) +{ + size_t phyaddr; + uint32_t flags; + + // iRCCE is not thread save => disable interrupts + flags = irq_nested_disable(); + my_ue = RCCE_ue(); + shmbegin = (size_t)RC_SHM_BUFFER_START(); + phyaddr = (size_t) RCCE_shmalloc(OWNER_SIZE); + irq_nested_enable(flags); + if (BUILTIN_EXPECT(!phyaddr, 0)) + return -ENOMEM; + + page_owner = (uint8_t*) map_region(0, phyaddr, OWNER_SIZE >> PAGE_SHIFT, MAP_KERNEL_SPACE|MAP_NO_CACHE/*MAP_MPE*/|MAP_SVM); + if (BUILTIN_EXPECT(!page_owner, 0)) { + flags = irq_nested_disable(); + RCCE_shfree((t_vcharp) phyaddr); + irq_nested_enable(flags); + return -ENOMEM; + } + + // per default is core 0 owner + if (!my_ue) + memset((void*)page_owner, 0x00, OWNER_SIZE); + + // iRCCE is not thread save => disable interrupts + flags = irq_nested_disable(); + RCCE_barrier(&RCCE_COMM_WORLD); + irq_nested_enable(flags); + + return 0; +} + +/* + * This function is called by the pagefault handler + * => the interrupt flags is already cleared + */ +int svm_access_request(size_t addr) +{ + size_t phyaddr = virt_to_phys(addr); + uint32_t pageid; + int remote_rank; + uint8_t payload[iRCCE_MAIL_HEADER_PAYLOAD]; + + if (phyaddr < shmbegin) + return -EINVAL; + if (phyaddr >= shmbegin + RCCE_SHM_SIZE_MAX) + return -EINVAL; + pageid = (phyaddr-shmbegin) >> PAGE_SHIFT; + + invalidate_cl1(); + if (page_owner[pageid] == my_ue) + return 0; + + kprintf("send access request to %d of 0x%x\n", remote_rank, phyaddr); + + remote_rank = page_owner[pageid]; + ((size_t*) payload)[0] = my_ue; + ((size_t*) payload)[1] = phyaddr; + + /* send ping request */ + iRCCE_mail_send(sizeof(size_t), ICC_TAG_SVMREQUEST, 0, payload, remote_rank); + + NOP8; + icc_send_irq(remote_rank); + + /* check for incoming messages */ + icc_mail_check(); + + invalidate_cl1(); + while (page_owner[pageid] != my_ue) + { + NOP4; + invalidate_cl1(); + }; + + return change_page_permissions(addr, addr+PAGE_SIZE, VMA_READ|VMA_WRITE|VMA_CACHEABLE); +} + void* svmmalloc(size_t size) { size_t phyaddr; @@ -46,8 +146,11 @@ void* svmmalloc(size_t size) if (RCCE_ue()) map_flags |= MAP_NO_ACCESS; irq_nested_enable(flags); + if (BUILTIN_EXPECT(!phyaddr, 0)) + return NULL; viraddr = map_region(0, phyaddr, size >> PAGE_SHIFT, map_flags); + phys2virt[(phyaddr - shmbegin) >> PAGE_SHIFT] = viraddr; //kprintf("shmmalloc: phyaddr 0x%x, viraddr 0x%x, size 0x%x\n", phyaddr, viraddr, size); @@ -70,6 +173,7 @@ void svmfree(void* addr, size_t size) //kprintf("shmmalloc: phyaddr 0x%x, viraddr 0x%x, size 0x%x\n", phyaddr, addr, size); unmap_region((size_t) addr, size >> PAGE_SHIFT); + phys2virt[(phyaddr - shmbegin) >> PAGE_SHIFT] = 0; // iRCCE is not thread save => disable interrupts flags = irq_nested_disable(); @@ -77,4 +181,49 @@ void svmfree(void* addr, size_t size) irq_nested_enable(flags); } +/* + * This function is called by icc_mail_check. + * => Interrupt flag is alread cleared. + */ +int svm_emit_page(size_t phyaddr, int ue) +{ + uint32_t pageid; + + kprintf("Try to emit page 0x%x to %d\n", phyaddr, ue); + + if (phyaddr < shmbegin) + return -EINVAL; + if (phyaddr >= shmbegin + RCCE_SHM_SIZE_MAX) + return -EINVAL; + pageid = (phyaddr-shmbegin) >> PAGE_SHIFT; + + invalidate_cl1(); + if (page_owner[pageid] != my_ue) { + // Core is nor owner => forward request to new owner + int remote_rank; + uint8_t payload[iRCCE_MAIL_HEADER_PAYLOAD]; + + kprintf("Ups, core %d is not owner of page 0x%x\n", my_ue, phyaddr); + + remote_rank = page_owner[pageid]; + ((size_t*) payload)[0] = ue; + ((size_t*) payload)[1] = phyaddr; + + /* send ping request */ + iRCCE_mail_send(sizeof(size_t), ICC_TAG_SVMREQUEST, 0, payload, remote_rank); + + NOP8; + icc_send_irq(remote_rank); + } else { + size_t viraddr = phys2virt[(phyaddr - shmbegin) >> PAGE_SHIFT]; + + change_page_permissions(viraddr, viraddr+PAGE_SIZE, VMA_NOACCESS|VMA_READ|VMA_CACHEABLE); + + page_owner[pageid] = ue; + invalidate_cl1(); + } + + return 0; +} + #endif diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 75bfdc78..2785f6d6 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -83,7 +84,7 @@ static void intr_handler(struct state *s) int tmp, z; #ifdef CONFIG_LWIP - mmnif_irqhandler(); +// mmnif_irqhandler(); #endif z = Z_PID(RC_COREID[my_ue]); @@ -279,6 +280,9 @@ void icc_mail_check(void) timer = rdtsc() - *((uint64_t*) header->payload); kprintf( "Response received in %d ticks!\n", timer ); break; + case ICC_TAG_SVMREQUEST: + svm_emit_page(((size_t*) header->payload)[1], ((size_t*) header->payload)[0]); + break; default: kprintf("Invalid mail: tag = %d\n", header->tag); break; diff --git a/include/metalsvm/vma.h b/include/metalsvm/vma.h index 449e81da..74c63233 100644 --- a/include/metalsvm/vma.h +++ b/include/metalsvm/vma.h @@ -32,10 +32,11 @@ extern "C" { #endif -#define VMA_READ 0x01 -#define VMA_WRITE 0x02 -#define VMA_EXECUTE 0x04 -#define VMA_CACHEABLE 0x08 +#define VMA_READ (1 << 0) +#define VMA_WRITE (1 << 1) +#define VMA_EXECUTE (1 << 2) +#define VMA_CACHEABLE (1 << 3) +#define VMA_NOACCESS (1 << 4) struct vma; diff --git a/kernel/main.c b/kernel/main.c index f89d2bc7..78594caa 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -32,6 +32,7 @@ #include #ifdef CONFIG_ROCKCREEK #include +#include #endif /* @@ -75,6 +76,7 @@ int main(void) mmu_init(); #ifdef CONFIG_ROCKCREEK icc_init(); + svm_init(); #endif initrd_init(); diff --git a/kernel/tests.c b/kernel/tests.c index a5113c09..bce1c56f 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef CONFIG_ROCKCREEK #include #include @@ -157,28 +158,31 @@ static int svm_test(void *arg) } // CL1FLUSH - cache_invalidate(); + invalidate_cl1(); // Now, we need only read access on A and B - change_page_permissions(A[0], A[0]+2*N*N, VMA_CACHEABLE|VMA_READ); + change_page_permissions((size_t) A[0], (size_t) (A[0]+2*N*N), VMA_CACHEABLE|VMA_READ); // iRCCE is not thread save => disable interrupts flags = irq_nested_disable(); RCCE_barrier(&RCCE_COMM_WORLD); irq_nested_enable(flags); + if (!my_ue) + kprintf("Start calculation...\n"); + // start calculation for(i=my_ue*(N/num_ues); i<(my_ue+1)*(N/num_ues); i++) for(j=0; j disable interrupts flags = irq_nested_disable(); RCCE_barrier(&RCCE_COMM_WORLD); irq_nested_enable(flags); - svmfree(A[0], 3*N*sizeof(int)); + svmfree((void*) A[0], 3*N*sizeof(int)); return 0; }