From 198c1dcebd1fe0be90baace04bb1a3d720371ac7 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 7 Nov 2011 13:01:57 -0800 Subject: [PATCH] merge Simon's current version of our mailbox system --- apps/tests.c | 2 + arch/x86/include/asm/icc.h | 1 + arch/x86/scc/iRCCE_mailbox.c | 157 +++++++++++++++++----------------- arch/x86/scc/icc.c | 160 +++++++++++++++++++++++++++++------ 4 files changed, 216 insertions(+), 104 deletions(-) diff --git a/apps/tests.c b/apps/tests.c index 65f4f465..b6926070 100644 --- a/apps/tests.c +++ b/apps/tests.c @@ -100,7 +100,9 @@ static int foo(void* arg) static int mail_ping(void* arg) { //icc_mail_ping(); icc_mail_ping_irq(); + icc_mail_ping_jitter(); //icc_irq_ping(); + //icc_mail_datarates(); //icc_halt(); return 0; diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index 3f41ccc1..47bbf1bc 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -53,6 +53,7 @@ void icc_mail_check(void); int icc_mail_ping(void); int icc_send_gic_irq(int core_num); int icc_mail_ping_irq(void); +int icc_mail_ping_jitter(void); int icc_mail_noise(void); void icc_wait(int tag); diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index dea78a5f..5c5c4724 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -144,73 +144,59 @@ static int iRCCE_mail_fetch( } -//------------------------------------------------------------------------------ -// FUNCTION: iRCCE_mailbox_check -//------------------------------------------------------------------------------ -/** - * @brief routine to check for new mail in mailboxes - * - * This function has to be called from time to time. It empties all mailboxes of - * the participating cores if the corresponding sent-flag is set and the mailbox - * is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset. - * Here we have to use a little trick because we can only write to the MPB in - * cacheline granularity. We set the appropriate flag to zero and afterwords - * touch the MPB on another cacheline. That causes the write combine buffer to - * write out the data. - */ -//------------------------------------------------------------------------------ -static iRCCE_MAIL_HEADER dummy_header = {0, 0, 0, NULL, 0, 0, 0, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} }; - - -static int iRCCE_mailbox_check(void) { - int i,j; - uint32_t flags; - - /* disable interrupts */ - flags = irq_nested_disable(); - - for( j=1; jsent ) { - iRCCE_mail_fetch(i); - - // reset senders flag - RC_cache_invalidate(); - *(iRCCE_mailbox_recv[i]) = dummy_header; - } - } - } - - /* enable interrupts */ - irq_nested_enable(flags); - return iRCCE_SUCCESS; -} - //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mail_check //------------------------------------------------------------------------------ /** - * @brief routine to check one specific mailbox - * @param sender is the core ID from which the mailbox is checked + * @brief routine to check one specific mailbox or all + * @param sender is the core ID from which the mailbox is checked use + * iRCCE_MAILBOX_ALL as wildcard to check all mailboxes * * This function may be called by the user application to check one specific - * mailbox. It is recommended to use it in combination with an inter core - * interrupt. + * mailbox. It is recommended to use it in combination with an inter-core + * interrupt. It empties one or all mailboxes of the participating cores if the + * corresponding sent-flag is set and the mailbox is not closed. After calling + * iRCCE_mail_fetch the sent-flag has to be reset. Here we have to use a little + * trick because we can only write to the MPB in cacheline granularity. We set + * the appropriate flag to zero and afterwords touch the MPB on another + * cacheline. That causes the write combine buffer to write out the data. * */ //------------------------------------------------------------------------------ +const iRCCE_MAIL_HEADER dummy_header = + {0, 0, 0, NULL, 0, 0, 0, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} }; + int iRCCE_mail_check(int sender) { uint32_t flags; + int j, i; + int found = 0; // check all mailboxes in case of wildcard if( sender == iRCCE_MAILBOX_ALL ) { - iRCCE_mailbox_check(); + /* disable interrupts */ + flags = irq_nested_disable(); - return iRCCE_SUCCESS; + for( j=1; jsent ) { + if( !found ) found = 1; + + iRCCE_mail_fetch(i); + // reset senders flag + RC_cache_invalidate(); + *(iRCCE_mailbox_recv[i]) = dummy_header; + } + } + } + + /* enable interrupts */ + irq_nested_enable(flags); + + return (found == 1)? iRCCE_SUCCESS : iRCCE_MAILBOX_EMPTY; } // verify sender's ID @@ -256,25 +242,28 @@ int iRCCE_mail_check(int sender) { * * The function checks if the receive queue with highest priority (priority 0) * contains any mail headers. In this case we pop the first element of that list - * in a FIFO maner. Otherwise iRCCE_mailbox_check() has to be called. Afterwards - * the first element of a non-empty receive queue with highest priority is - * returned. + * in a FIFO maner. Afterwards the first element of a non-empty receive queue + * with highest priority is returned. */ //------------------------------------------------------------------------------ int iRCCE_mail_recv( iRCCE_MAIL_HEADER** header // pointer to incoming header ) { // (memory allocated by iRCCE) - int i; + int i, found = 0; uint32_t flags; iRCCE_MAIL_HEADER* help_header; + + // if no mail queued pointer must be ZERO + *header = NULL; + + /* disable interrupts */ + flags = irq_nested_disable(); // check priority queues for( i=0; inext = NULL; - *header = help_header; - - /* enable interrupts */ - irq_nested_enable(flags); - - return iRCCE_SUCCESS; - + found = 1; + break; } + + } - // no mail queued - *header = NULL; - - return iRCCE_MAILBOX_EMPTY; + /* enable interrupts */ + irq_nested_enable(flags); + + return (found == 1)? iRCCE_SUCCESS : iRCCE_MAILBOX_EMPTY; } @@ -382,6 +369,8 @@ int iRCCE_mail_send( char* payload, // pointer to buffer for header payload int dest // UE that will receive the header ) { + uint32_t flags; + // verify sender's ID if( (dest < 0) || (dest > RCCE_NP) || (dest == RCCE_IAM) ) { return iRCCE_ERROR_TARGET; @@ -390,20 +379,28 @@ int iRCCE_mail_send( // if dest mailbox is full, check for incoming mail RC_cache_invalidate(); while( iRCCE_mailbox_send[dest]->sent ) { - iRCCE_mailbox_check(); + // iRCCE_mail_check(iRCCE_MAILBOX_ALL); RC_cache_invalidate(); + NOP8; + NOP8; + NOP8; } + + /* disable interrupts */ + flags = irq_nested_disable(); + // check if mailbox is closed RCCE_acquire_lock( dest ); RC_cache_invalidate(); if( iRCCE_mailbox_send[dest]->closed ) { RCCE_release_lock( dest ); + + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_MAILBOX_CLOSED; } - /* disable interrupts */ -// flags = irq_nested_disable(); - // prepare header iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio, RCCE_FLAG_UNSET, RCCE_FLAG_UNSET, @@ -425,11 +422,11 @@ int iRCCE_mail_send( *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); - /* enable interrupts */ -// irq_nested_enable(flags); - RCCE_release_lock( dest ); + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_SUCCESS; } @@ -474,7 +471,7 @@ int iRCCE_last_mail_recv(void) { //------------------------------------------------------------------------------ int iRCCE_mailbox_wait(void) { while( iRCCE_last_mail_recv() == iRCCE_LAST_MAILS_NOT_RECV ) { - iRCCE_mailbox_check(); + iRCCE_mail_check(iRCCE_MAILBOX_ALL); } return iRCCE_SUCCESS; @@ -531,7 +528,7 @@ int iRCCE_mailbox_flush(void) { * last-mail. * * This function closes a mailbox of the given rank. If the check flag is set - * an iRCCE_mailbox_check()-call is performed. The close procedure has to be + * an iRCCE_mail_check()-call is performed. The close procedure has to be * locked to be sure that no UE sends any mail while closing the mailbox. */ //------------------------------------------------------------------------------ diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index de6d4bea..d54f37c8 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -298,7 +298,7 @@ int icc_halt(void) #define ROUNDS 1000 #define CORE_A 0 // sender -#define CORE_B 1 // receiver +#define CORE_B 25 // receiver int icc_send_gic_irq(int core_num) { volatile uint32_t* irq_request = (volatile uint32_t*)(FPGA_BASE+IRQ_REQUEST+RC_MY_COREID*8); @@ -346,7 +346,7 @@ int icc_mail_ping(void) /* wait for response */ do { - res = iRCCE_mail_check(CORE_B); + res = iRCCE_mail_check(iRCCE_MAILBOX_ALL); } while( res != iRCCE_SUCCESS ); /* release mail */ @@ -357,7 +357,7 @@ int icc_mail_ping(void) else { /* wait for request */ do { - res = iRCCE_mail_check(CORE_A); + res = iRCCE_mail_check(iRCCE_MAILBOX_ALL); } while( res != iRCCE_SUCCESS ); /* check mail */ @@ -439,50 +439,162 @@ int icc_mail_ping_irq(void) return 0; } -#define _iRQ_NOISE_ 0 +int icc_mail_ping_jitter(void) +{ + kprintf( "Hello from gitter_test ... \n" ); + /* return if not core A */ + if( RCCE_IAM != CORE_A ) return 0; + + uint32_t flags; + uint64_t timer = 0; + uint64_t max = 0; + uint64_t min = ULONG_MAX; + uint64_t sum = 0; + + int i; + int res; + iRCCE_MAIL_HEADER* recv_header = NULL; + + kprintf( "my_rank = %d\n", RCCE_IAM ); + kprintf( "rem_rank = %d\n", CORE_B ); + kprintf( "rounds = %d\n", ROUNDS ); + + // disable interrupts + flags = irq_nested_disable(); + + for( i=0; i 0 ) { + max = ( max < timer )? timer : max; + min = ( min > timer )? timer : min; + sum += timer; + } + } + + kprintf( "Average was: %d nsec\n", sum*1000/(2*ROUNDS*533) ); + kprintf( "Maximum was: %d nsec\n", max*1000/(2*533) ); + kprintf( "Minimum was: %d nsec\n", min*1000/(2*533) ); + kprintf( "Jitter was: %d nsec\n", (max-min)*1000/(2*533) ); + + irq_nested_enable(flags); + + return 0; +} + +#undef _IRQ_NOISE_ +#define NOISE_PRIO 1 int icc_mail_noise(void) { int i, j, res; int num_ranks = RCCE_num_ues(); + int count = 0; iRCCE_MAIL_HEADER* recv_mail = NULL; + /* timer vars */ + uint64_t timer; + uint64_t tmr; + uint64_t tmr_send = 0; + uint64_t tmr_recv = 0; + uint64_t tmr_release = 0; + uint64_t tmr_chck = 0; + + kprintf( "my_ue = %d\n", RCCE_IAM ); + // leave function if not participating - if( !((RCCE_IAM == 4) || (RCCE_IAM == 2) || (RCCE_IAM == CORE_B)) ) { - kprintf( "mail_noise: leaving" ); - return -1; - } +// if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { +// kprintf( "mail_noise: leaving" ); +// return -1; +// } kprintf( "Hello from icc_mail_noise: my_ue = %d\n", RCCE_IAM ); kprintf( "num_ues = %d\n", num_ranks ); - - for( i=0; i<10000; ++i ) { + + timer = rdtsc(); + + for( i=0; i<20000; ++i ) { if( !(i%1000) ) kprintf( "%d ", i ); + tmr = rdtsc(); + iRCCE_mail_check(iRCCE_MAILBOX_ALL); + tmr = rdtsc() - tmr; + tmr_chck += tmr; + /* send a mail to each UE */ for( j=0; j