diff --git a/arch/x86/include/asm/iRCCE.h b/arch/x86/include/asm/iRCCE.h index 7300065c..9a3d7417 100644 --- a/arch/x86/include/asm/iRCCE.h +++ b/arch/x86/include/asm/iRCCE.h @@ -44,11 +44,14 @@ // status codes #define iRCCE_ERROR_ID RCCE_ERROR_ID -#define iRCCE_MAILBOX_OPEN -1 -#define iRCCE_MAILBOX_CLOSED -2 -#define iRCCE_MAILBOX_INVALID -3 -#define iRCCE_MAILBOX_EMPTY -4 -#define iRCCE_LAST_MAILS_NOT_RECV -5 +#define iRCCE_ERROR_TARGET RCCE_ERROR_TARGET +#define iRCCE_ERROR_SOURCE RCCE_ERROR_SOURCE +#define iRCCE_ERROR_GENERAL -1 +#define iRCCE_MAILBOX_OPEN -2 +#define iRCCE_MAILBOX_CLOSED -3 +#define iRCCE_MAILBOX_INVALID -4 +#define iRCCE_MAILBOX_EMPTY -5 +#define iRCCE_LAST_MAILS_NOT_RECV -6 // iRCCE-mailbox-system tags #define iRCCE_LAST_MAIL -1 diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 6a18464b..b1845e24 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -55,6 +55,7 @@ inline static uint32_t irq_nested_disable(void) { asm volatile("pushf; cli; popl %0": "=r"(flags) : : "memory"); if (flags & (1 << 9)) return 1; + return 0; } @@ -71,8 +72,9 @@ inline static void irq_enable(void) { * @param flags Flags to set. Could be the old ones you got from irq_nested_disable. */ inline static void irq_nested_enable(uint32_t flags) { - if (flags) + if (flags) { irq_enable(); + } } #ifdef __cplusplus diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index a0521317..b311970d 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -182,7 +182,6 @@ static int iRCCE_mailbox_check() { // only check open mailboxes if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) { - RC_cache_invalidate(); if( iRCCE_mailbox_recv[i]->sent ) { iRCCE_mail_fetch(i); @@ -193,10 +192,9 @@ static int iRCCE_mailbox_check() { } } } - + /* enable interrupts */ irq_nested_enable(flags); - return iRCCE_SUCCESS; } @@ -216,23 +214,18 @@ static int iRCCE_mailbox_check() { int iRCCE_mail_check(int sender) { uint32_t flags; - /* disable interrupts */ - flags = irq_nested_disable(); - + // check all mailboxes in case of wildcard + if( sender == iRCCE_MAILBOX_ALL ) { + iRCCE_mailbox_check(); + + return iRCCE_SUCCESS; + } // verify sender's ID if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) { - if( sender == iRCCE_MAILBOX_ALL ) { - iRCCE_mailbox_check(); - return 0; - } - - /* enable interrupts */ - irq_nested_enable(flags); - - return iRCCE_ERROR_ID; + return iRCCE_ERROR_SOURCE; } - + // check if mailbox is open if( iRCCE_mailbox_status[sender] == iRCCE_MAILBOX_CLOSED ) { return iRCCE_MAILBOX_CLOSED; @@ -240,20 +233,24 @@ int iRCCE_mail_check(int sender) { RC_cache_invalidate(); if( iRCCE_mailbox_recv[sender]->sent ) { + /* disable interrupts */ + flags = irq_nested_disable(); + iRCCE_mail_fetch(sender); // reset senders flag RC_cache_invalidate(); *(iRCCE_mailbox_recv[sender]) = dummy_header; + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_SUCCESS; } else { return iRCCE_MAILBOX_EMPTY; } - /* enable interrupts */ - irq_nested_enable(flags); } //------------------------------------------------------------------------------ @@ -280,12 +277,6 @@ int iRCCE_mail_recv( uint32_t flags; iRCCE_MAIL_HEADER* help_header; - // if there is no mail, check for incoming -/* if ( !iRCCE_mailbox_recv_queue[0].first ) { - iRCCE_mailbox_check(); - } -*/ - // check priority queues for( i=0; i RCCE_NP) || (dest == RCCE_IAM) ) { + return iRCCE_ERROR_TARGET; } // if dest mailbox is full, check for incoming mail @@ -412,7 +403,6 @@ int iRCCE_mail_send( iRCCE_mailbox_check(); RC_cache_invalidate(); } - // check if mailbox is closed RCCE_acquire_lock( dest ); RC_cache_invalidate(); @@ -421,6 +411,9 @@ int iRCCE_mail_send( return iRCCE_MAILBOX_CLOSED; } + /* disable interrupts */ +// flags = irq_nested_disable(); + // prepare header iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio, RCCE_FLAG_UNSET, RCCE_FLAG_UNSET, @@ -442,11 +435,11 @@ int iRCCE_mail_send( *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); - RCCE_release_lock( dest ); - /* enable interrupts */ // irq_nested_enable(flags); + RCCE_release_lock( dest ); + return iRCCE_SUCCESS; } @@ -512,7 +505,11 @@ int iRCCE_mailbox_wait(void) { int iRCCE_mailbox_flush(void) { int i; uint32_t flags; - + + /* disable interrupts */ + flags = irq_nested_disable(); + + // empty priority queues for( i=0; itag ) { case iRCCE_ANYLENGTH: recv_buffer = (char*)kmalloc( mail->size ); @@ -296,7 +295,6 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: -// count++; iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; @@ -339,15 +337,8 @@ int icc_mail_ping( void ) res = iRCCE_mail_check(CORE_B); } while( res != iRCCE_SUCCESS ); - recv_header = NULL; - /* release mail */ - do { - if( recv_header ) iRCCE_mail_release(&recv_header); - res = iRCCE_mail_recv(&recv_header); - } while( (recv_header->source != CORE_B) - && (recv_header->tag == PING_RESP) ); - /* release mail */ + iRCCE_mail_recv(&recv_header); iRCCE_mail_release(&recv_header); } /* receivers part */ @@ -356,12 +347,10 @@ int icc_mail_ping( void ) do { res = iRCCE_mail_check(CORE_A); } while( res != iRCCE_SUCCESS ); - - res = iRCCE_mail_recv(&recv_header); - if( (res == iRCCE_SUCCESS) && (recv_header->tag = PING_REQ) ); - /* send response */ - iRCCE_mail_send(0, PING_RESP, 0, NULL, recv_header->source); + /* check mail */ + res = iRCCE_mail_recv(&recv_header); + icc_mail_check_tag(recv_header); /* release mail */ iRCCE_mail_release(&recv_header); @@ -378,8 +367,8 @@ int icc_mail_ping( void ) if( my_ue == CORE_A ) { kprintf( "timer = %ld\n", timer ); - kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", - timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); + kprintf( "mail_pingpong needs in average %d ns (%d ticks)!\n", + timer*1000/(2*ROUNDS*533), timer/(2*ROUNDS) ); } irq_nested_enable(flags); @@ -449,7 +438,7 @@ int icc_mail_noise() { int flags; // leave function if not participating - if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { + if( !((my_ue == 4) || (my_ue == 2) || (my_ue == CORE_B)) ) { kprintf( "mail_noise: leaving" ); return -1; } @@ -457,37 +446,34 @@ int icc_mail_noise() { kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); kprintf( "num_ues = %d\n", num_ranks ); - int count = 0; - for( ;; ++count ) { - if( !(count%1000) ) kprintf( "STILL ALIVE! " ); + + for( i=0; i<10000; ++i ) { + if( !(i%1000) ) kprintf( "%d ", i ); /* send a mail to each UE */ for( j=0; j>= 6; // shift emac bits + /* determine interrupt sources */ + irq_status >>= 6; // shift emac bits - for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { - if( (irq_status & 0x1) != 0 ) { - res = iRCCE_mail_check(source); - } + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( (irq_status & 0x1) != 0 ) { + res = iRCCE_mail_check(source); } + } - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = irq_reset; -// } + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = irq_reset; } else { iRCCE_mail_check(iRCCE_MAILBOX_ALL); } + /* enable interrupts */ + irq_nested_enable(flags); + /* empty mail queue */ while( iRCCE_mail_recv(&header) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); iRCCE_mail_release( &header ); + NOP8; + NOP8; + NOP8; } - /* enable interrupts */ - irq_nested_enable(flags); - }