From 6892fa021a1a2773d0a6040e0bf37a78393592ee Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Sat, 13 Aug 2011 17:14:01 -0700 Subject: [PATCH] testing remote interrupts --- arch/x86/kernel/irq.c | 5 +- arch/x86/scc/iRCCE_mailbox.c | 21 +++++++- arch/x86/scc/icc.c | 96 ++++++++++++++++++------------------ include/metalsvm/tasks.h | 4 -- kernel/tests.c | 6 +-- 5 files changed, 72 insertions(+), 60 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 2bbc2855..f3fdfb8b 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -235,10 +235,9 @@ void irq_handler(struct state *s) void (*handler) (struct state * s); // at first, we check our work queues - if( s->int_no == 124 ) { - kprintf( "hello from rem_interrupt\n" ); +// if( s->int_no == 124 ) { check_workqueues(); - } +// } /* * Find out if we have a custom handler to run for this diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 2e16b4c0..64be600b 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -206,8 +206,20 @@ static int iRCCE_mailbox_check() { */ //------------------------------------------------------------------------------ int iRCCE_mail_check(int sender) { + uint32_t flags; + + /* disable interrupts */ + flags = irq_nested_disable(); + + + // verify sender's ID if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) { + if( sender == iRCCE_MAILBOX_ALL ) { + iRCCE_mailbox_check(); + return 0; + } + return iRCCE_ERROR_ID; } @@ -229,6 +241,10 @@ int iRCCE_mail_check(int sender) { else { return iRCCE_MAILBOX_EMPTY; } + + + /* enable interrupts */ + irq_nested_enable(flags); } //------------------------------------------------------------------------------ @@ -256,9 +272,10 @@ int iRCCE_mail_recv( iRCCE_MAIL_HEADER* help_header; // if there is no mail, check for incoming - if ( !iRCCE_mailbox_recv_queue[0].first ) { +/* if ( !iRCCE_mailbox_recv_queue[0].first ) { iRCCE_mailbox_check(); } +*/ // check priority queues for( i=0; isent ) { - iRCCE_mailbox_check(); +// iRCCE_mailbox_check(); RC_cache_invalidate(); } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index f11184e1..5a23d74c 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -221,7 +221,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 100000 +#define ROUNDS 1000 #define CORE_A 1 // sender #define CORE_B 4 // receiver @@ -232,8 +232,6 @@ int icc_send_gic_irq(int core_num) { // determine bit position and set according bit bit_pos = (1 << core_num); - kprintf( "bit_pos = 0x%x\n", bit_pos ); - kprintf( "irq_req_addr = %x\n", irq_request ); *irq_request = bit_pos; return 0; @@ -294,7 +292,7 @@ int icc_mail_ping( void ) if( my_ue == CORE_A ) { /* send ping request */ iRCCE_mail_send(0, PING_REQ, 0, NULL, CORE_B); - + /* wait for response */ do { res = iRCCE_mail_recv(&recv_header); @@ -364,6 +362,7 @@ int icc_mail_ping_irq( void ) kprintf( "my_rank = %d\n", my_ue ); kprintf( "rem_rank = %d\n", rem_rank ); kprintf( "rounds = %d\n", ROUNDS ); + // disable interrupts flags = irq_nested_disable(); @@ -372,28 +371,20 @@ int icc_mail_ping_irq( void ) iRCCE_mail_send(0, PING_REQ, 0, NULL, rem_rank); /* send interrupt */ - NOP8; - NOP8; - NOP8; - icc_send_irq(rem_rank); + icc_send_gic_irq(rem_rank); /* wait for response */ do { - res = iRCCE_mail_recv(&recv_header); - if( res == iRCCE_SUCCESS ) { - if( (recv_header->source == rem_rank) - && (recv_header->tag == PING_RESP) ) - break; - iRCCE_mail_release(&recv_header); - } - } while( 1 ); + res = iRCCE_mail_check(rem_rank); + } while( res != iRCCE_SUCCESS ); /* release mail */ + iRCCE_mail_recv(&recv_header); iRCCE_mail_release(&recv_header); /* start timer in first round */ if( i == 0 ) timer = rdtsc(); - + } /* stop timer */ @@ -404,19 +395,13 @@ int icc_mail_ping_irq( void ) timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); - /* recv noise mails */ - while(1) { - iRCCE_mail_recv(&recv_header); - if(recv_header) iRCCE_mail_release(&recv_header); - } - irq_nested_enable(flags); return 0; } int icc_mail_noise() { - int j, res; + int j, res, count = 0; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; @@ -428,6 +413,8 @@ int icc_mail_noise() { kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); for( ;; ) { + count ++; + kprintf( "%d ", count ); /* send a mail to each UE */ for( j=0; j>= 1, ++source ) { - if( (irq_status & 0x1) != 0 ) { - kprintf( "check mail for source %d\n", source ); - } + /* read status register */ + irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + irq_status = *irq_status_reg; + + // kprintf( "irq_status = %d\n", irq_status ); + + /* determine interrupt sources */ + irq_status >>= 6; // shift emac bits + + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( (irq_status & 0x1) != 0 ) { + // kprintf( "fetch mail from %d\n", source ); + // do { + res = iRCCE_mail_check(source); + // } while( res != iRCCE_SUCCESS ); } - - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = ~(0); - - kprintf( "irq_status: %x\n", *irq_status_reg ); + } - } - /* - int res; + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = ~(0); + + + /* empty mail queue */ while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); iRCCE_mail_release( &header ); } - */ + + /* enable interrupts */ + irq_nested_enable(flags); + } #endif diff --git a/include/metalsvm/tasks.h b/include/metalsvm/tasks.h index 289c7d8d..641c211f 100644 --- a/include/metalsvm/tasks.h +++ b/include/metalsvm/tasks.h @@ -140,13 +140,9 @@ int sys_execve(const char* fname, char** argv, char** env); static inline void check_workqueues(void) { -// uint32_t flags = irq_nested_disable(); - #ifdef CONFIG_ROCKCREEK icc_mail_check(); #endif - -// irq_nested_enable(flags); } #ifdef __cplusplus diff --git a/kernel/tests.c b/kernel/tests.c index 61e1c56d..f1b2e314 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -89,8 +89,8 @@ static int STDCALL foo(void* arg) #ifdef CONFIG_ROCKCREEK int STDCALL mail_ping(void* arg) { // icc_mail_ping(); - // icc_mail_ping_irq(); - icc_irq_ping(); + icc_mail_ping_irq(); + // icc_irq_ping(); icc_halt(); return 0; @@ -139,7 +139,7 @@ int test_init(void) //create_kernel_task(NULL, producer, NULL); //create_kernel_task(NULL, consumer, NULL); create_kernel_task(NULL, mail_ping, NULL); - //create_kernel_task(NULL, mail_noise, NULL); + create_kernel_task(NULL, mail_noise, NULL); //create_user_task(NULL, "/bin/hello", argv); //create_user_task(NULL, "/bin/tests", argv); //create_user_task(NULL, "/bin/jacobi", argv);