stv0g/metalsvm
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote branch 'origin/mailbox_integration' into demo

Browse source
This commit is contained in:
Stefan Lankes 2011-08-28 23:59:48 -07:00
commit 19cfc735e2
4 changed files with 87 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
arch/x86/include/asm/iRCCE.h
View file

@ -44,11 +44,14 @@
// status codes // status codes
#define iRCCE_ERROR_ID RCCE_ERROR_ID #define iRCCE_ERROR_ID RCCE_ERROR_ID
#define iRCCE_MAILBOX_OPEN -1 #define iRCCE_ERROR_TARGET RCCE_ERROR_TARGET
#define iRCCE_MAILBOX_CLOSED -2 #define iRCCE_ERROR_SOURCE RCCE_ERROR_SOURCE
#define iRCCE_MAILBOX_INVALID -3 #define iRCCE_ERROR_GENERAL -1
#define iRCCE_MAILBOX_EMPTY -4 #define iRCCE_MAILBOX_OPEN -2
#define iRCCE_LAST_MAILS_NOT_RECV -5 #define iRCCE_MAILBOX_CLOSED -3
#define iRCCE_MAILBOX_INVALID -4
#define iRCCE_MAILBOX_EMPTY -5
#define iRCCE_LAST_MAILS_NOT_RECV -6
// iRCCE-mailbox-system tags // iRCCE-mailbox-system tags
#define iRCCE_LAST_MAIL -1 #define iRCCE_LAST_MAIL -1

4
arch/x86/include/asm/irqflags.h
View file

@ -55,6 +55,7 @@ inline static uint32_t irq_nested_disable(void) {
asm volatile("pushf; cli; popl %0": "=r"(flags) : : "memory"); asm volatile("pushf; cli; popl %0": "=r"(flags) : : "memory");
if (flags & (1 << 9)) if (flags & (1 << 9))
return 1; return 1;
return 0; return 0;
} }
@ -71,8 +72,9 @@ inline static void irq_enable(void) {
* @param flags Flags to set. Could be the old ones you got from irq_nested_disable. * @param flags Flags to set. Could be the old ones you got from irq_nested_disable.
*/ */
inline static void irq_nested_enable(uint32_t flags) { inline static void irq_nested_enable(uint32_t flags) {
if (flags) if (flags) {
irq_enable(); irq_enable();
}
} }
#ifdef __cplusplus #ifdef __cplusplus

67
arch/x86/scc/iRCCE_mailbox.c
View file

@ -182,7 +182,6 @@ static int iRCCE_mailbox_check() {
// only check open mailboxes // only check open mailboxes
if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) { if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) {
RC_cache_invalidate(); RC_cache_invalidate();
if( iRCCE_mailbox_recv[i]->sent ) { if( iRCCE_mailbox_recv[i]->sent ) {
iRCCE_mail_fetch(i); iRCCE_mail_fetch(i);
@ -193,10 +192,9 @@ static int iRCCE_mailbox_check() {
} }
} }
} }
/* enable interrupts */ /* enable interrupts */
irq_nested_enable(flags); irq_nested_enable(flags);
return iRCCE_SUCCESS; return iRCCE_SUCCESS;
} }
@ -216,23 +214,18 @@ static int iRCCE_mailbox_check() {
int iRCCE_mail_check(int sender) { int iRCCE_mail_check(int sender) {
uint32_t flags; uint32_t flags;
/* disable interrupts */ // check all mailboxes in case of wildcard
flags = irq_nested_disable(); if( sender == iRCCE_MAILBOX_ALL ) {
iRCCE_mailbox_check();
return iRCCE_SUCCESS;
}
// verify sender's ID // verify sender's ID
if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) { if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) {
if( sender == iRCCE_MAILBOX_ALL ) { return iRCCE_ERROR_SOURCE;
iRCCE_mailbox_check();
return 0;
}
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_ERROR_ID;
} }
// check if mailbox is open // check if mailbox is open
if( iRCCE_mailbox_status[sender] == iRCCE_MAILBOX_CLOSED ) { if( iRCCE_mailbox_status[sender] == iRCCE_MAILBOX_CLOSED ) {
return iRCCE_MAILBOX_CLOSED; return iRCCE_MAILBOX_CLOSED;
@ -240,20 +233,24 @@ int iRCCE_mail_check(int sender) {
RC_cache_invalidate(); RC_cache_invalidate();
if( iRCCE_mailbox_recv[sender]->sent ) { if( iRCCE_mailbox_recv[sender]->sent ) {
/* disable interrupts */
flags = irq_nested_disable();
iRCCE_mail_fetch(sender); iRCCE_mail_fetch(sender);
// reset senders flag // reset senders flag
RC_cache_invalidate(); RC_cache_invalidate();
*(iRCCE_mailbox_recv[sender]) = dummy_header; *(iRCCE_mailbox_recv[sender]) = dummy_header;
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_SUCCESS; return iRCCE_SUCCESS;
} }
else { else {
return iRCCE_MAILBOX_EMPTY; return iRCCE_MAILBOX_EMPTY;
} }
/* enable interrupts */
irq_nested_enable(flags);
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -280,12 +277,6 @@ int iRCCE_mail_recv(
uint32_t flags; uint32_t flags;
iRCCE_MAIL_HEADER* help_header; iRCCE_MAIL_HEADER* help_header;
// if there is no mail, check for incoming
/* if ( !iRCCE_mailbox_recv_queue[0].first ) {
iRCCE_mailbox_check();
}
*/
// check priority queues // check priority queues
for( i=0; i<iRCCE_PRIOS; ++i ) { for( i=0; i<iRCCE_PRIOS; ++i ) {
if ( iRCCE_mailbox_recv_queue[i].first ) { if ( iRCCE_mailbox_recv_queue[i].first ) {
@ -399,11 +390,11 @@ int iRCCE_mail_send(
char* payload, // pointer to buffer for header payload char* payload, // pointer to buffer for header payload
int dest // UE that will receive the header int dest // UE that will receive the header
) { ) {
uint32_t flags; uint32_t flags;
// check for an attempt to send to own mailbox // verify sender's ID
if( dest == RCCE_IAM ) { if( (dest < 0) || (dest > RCCE_NP) || (dest == RCCE_IAM) ) {
return iRCCE_MAILBOX_INVALID; return iRCCE_ERROR_TARGET;
} }
// if dest mailbox is full, check for incoming mail // if dest mailbox is full, check for incoming mail
@ -412,7 +403,6 @@ int iRCCE_mail_send(
iRCCE_mailbox_check(); iRCCE_mailbox_check();
RC_cache_invalidate(); RC_cache_invalidate();
} }
// check if mailbox is closed // check if mailbox is closed
RCCE_acquire_lock( dest ); RCCE_acquire_lock( dest );
RC_cache_invalidate(); RC_cache_invalidate();
@ -421,6 +411,9 @@ int iRCCE_mail_send(
return iRCCE_MAILBOX_CLOSED; return iRCCE_MAILBOX_CLOSED;
} }
/* disable interrupts */
// flags = irq_nested_disable();
// prepare header // prepare header
iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio, iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio,
RCCE_FLAG_UNSET, RCCE_FLAG_UNSET, RCCE_FLAG_UNSET, RCCE_FLAG_UNSET,
@ -442,11 +435,11 @@ int iRCCE_mail_send(
*(int *)RCCE_fool_write_combine_buffer = 1; *(int *)RCCE_fool_write_combine_buffer = 1;
RC_cache_invalidate(); RC_cache_invalidate();
RCCE_release_lock( dest );
/* enable interrupts */ /* enable interrupts */
// irq_nested_enable(flags); // irq_nested_enable(flags);
RCCE_release_lock( dest );
return iRCCE_SUCCESS; return iRCCE_SUCCESS;
} }
@ -512,7 +505,11 @@ int iRCCE_mailbox_wait(void) {
int iRCCE_mailbox_flush(void) { int iRCCE_mailbox_flush(void) {
int i; int i;
uint32_t flags; uint32_t flags;
/* disable interrupts */
flags = irq_nested_disable();
// empty priority queues
for( i=0; i<iRCCE_PRIOS; ++i ) { for( i=0; i<iRCCE_PRIOS; ++i ) {
iRCCE_MAIL_HEADER* erase_header = iRCCE_MAIL_HEADER* erase_header =
iRCCE_mailbox_recv_queue[i].first; iRCCE_mailbox_recv_queue[i].first;
@ -524,9 +521,11 @@ int iRCCE_mailbox_flush(void) {
erase_header = iRCCE_mailbox_recv_queue[i].first; erase_header = iRCCE_mailbox_recv_queue[i].first;
} }
/* enable interrupts */
irq_nested_enable(flags);
} }
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_SUCCESS; return iRCCE_SUCCESS;
} }

99
arch/x86/scc/icc.c
View file

@ -259,7 +259,7 @@ int icc_halt(void)
#define ROUNDS 100000 #define ROUNDS 100000
#define CORE_A 0 // sender #define CORE_A 0 // sender
#define CORE_B 4 // receiver #define CORE_B 1 // receiver
int icc_send_gic_irq(int core_num) { int icc_send_gic_irq(int core_num) {
@ -284,10 +284,9 @@ int icc_irq_ping()
static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) {
char* recv_buffer; char* recv_buffer;
static int count = 0;
if( !mail ) return; if( !mail ) return;
switch( mail->tag ) { switch( mail->tag ) {
case iRCCE_ANYLENGTH: case iRCCE_ANYLENGTH:
recv_buffer = (char*)kmalloc( mail->size ); recv_buffer = (char*)kmalloc( mail->size );
@ -296,7 +295,6 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) {
break; break;
case PING_REQ: case PING_REQ:
// count++;
iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source);
break; break;
@ -339,15 +337,8 @@ int icc_mail_ping( void )
res = iRCCE_mail_check(CORE_B); res = iRCCE_mail_check(CORE_B);
} while( res != iRCCE_SUCCESS ); } while( res != iRCCE_SUCCESS );
recv_header = NULL;
/* release mail */
do {
if( recv_header ) iRCCE_mail_release(&recv_header);
res = iRCCE_mail_recv(&recv_header);
} while( (recv_header->source != CORE_B)
&& (recv_header->tag == PING_RESP) );
/* release mail */ /* release mail */
iRCCE_mail_recv(&recv_header);
iRCCE_mail_release(&recv_header); iRCCE_mail_release(&recv_header);
} }
/* receivers part */ /* receivers part */
@ -356,12 +347,10 @@ int icc_mail_ping( void )
do { do {
res = iRCCE_mail_check(CORE_A); res = iRCCE_mail_check(CORE_A);
} while( res != iRCCE_SUCCESS ); } while( res != iRCCE_SUCCESS );
res = iRCCE_mail_recv(&recv_header);
if( (res == iRCCE_SUCCESS) && (recv_header->tag = PING_REQ) );
/* send response */ /* check mail */
iRCCE_mail_send(0, PING_RESP, 0, NULL, recv_header->source); res = iRCCE_mail_recv(&recv_header);
icc_mail_check_tag(recv_header);
/* release mail */ /* release mail */
iRCCE_mail_release(&recv_header); iRCCE_mail_release(&recv_header);
@ -378,8 +367,8 @@ int icc_mail_ping( void )
if( my_ue == CORE_A ) { if( my_ue == CORE_A ) {
kprintf( "timer = %ld\n", timer ); kprintf( "timer = %ld\n", timer );
kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", kprintf( "mail_pingpong needs in average %d ns (%d ticks)!\n",
timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); timer*1000/(2*ROUNDS*533), timer/(2*ROUNDS) );
} }
irq_nested_enable(flags); irq_nested_enable(flags);
@ -449,7 +438,7 @@ int icc_mail_noise() {
int flags; int flags;
// leave function if not participating // leave function if not participating
if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { if( !((my_ue == 4) || (my_ue == 2) || (my_ue == CORE_B)) ) {
kprintf( "mail_noise: leaving" ); kprintf( "mail_noise: leaving" );
return -1; return -1;
} }
@ -457,37 +446,34 @@ int icc_mail_noise() {
kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue );
kprintf( "num_ues = %d\n", num_ranks ); kprintf( "num_ues = %d\n", num_ranks );
int count = 0;
for( ;; ++count ) { for( i=0; i<10000; ++i ) {
if( !(count%1000) ) kprintf( "STILL ALIVE! " ); if( !(i%1000) ) kprintf( "%d ", i );
/* send a mail to each UE */ /* send a mail to each UE */
for( j=0; j<num_ranks; ++j ) { for( j=0; j<num_ranks; ++j ) {
if( (j == CORE_A) /*|| (j == CORE_B)*/ ) if( !((j == 4) || (j == 2)/* || (j == CORE_B) */) )
continue; continue;
/* send noise mail */ /* send noise mail */
iRCCE_mail_send(0, NOISE, 1, NULL, j); iRCCE_mail_send(0, NOISE, 1, NULL, j);
#ifdef _IRQ_NOISE_ #ifdef _IRQ_NOISE_
kprintf( "sending irq ... " );
icc_send_gic_irq(j); icc_send_gic_irq(j);
#endif #endif
} iRCCE_mail_recv(&recv_mail);
#ifndef _IRQ_NOISE_
/* read some mails */
flags = irq_nested_disable;
iRCCE_mail_check(iRCCE_MAILBOX_ALL);
while( iRCCE_mail_recv(&recv_mail) == iRCCE_SUCCESS ) {
icc_mail_check_tag(recv_mail); icc_mail_check_tag(recv_mail);
iRCCE_mail_release(&recv_mail); if( recv_mail ) iRCCE_mail_release(&recv_mail);
} }
irq_nested_enable(flags);
#endif
} }
kprintf( "XXX XXX XXX" );
do {
iRCCE_mail_check(iRCCE_MAILBOX_ALL);
res = iRCCE_mail_recv(&recv_mail);
icc_mail_check_tag(recv_mail);
if( recv_mail ) iRCCE_mail_release(&recv_mail);
} while( res == iRCCE_SUCCESS );
return 0; return 0;
} }
@ -511,40 +497,41 @@ void icc_mail_check(int irq)
flags = irq_nested_disable(); flags = irq_nested_disable();
if( irq == 1 ) { if( irq == 1 ) {
// for( i=0; i<2; ++i ) { /* read status register */
/* read status register */ irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8);
irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); irq_status = irq_reset = *irq_status_reg;
irq_status = irq_reset = *irq_status_reg;
/* determine interrupt sources */ /* determine interrupt sources */
irq_status >>= 6; // shift emac bits irq_status >>= 6; // shift emac bits
for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) {
if( (irq_status & 0x1) != 0 ) { if( (irq_status & 0x1) != 0 ) {
res = iRCCE_mail_check(source); res = iRCCE_mail_check(source);
}
} }
}
/* reset status register */ /* reset status register */
irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8);
*irq_reset_reg = irq_reset; *irq_reset_reg = irq_reset;
// }
} }
else { else {
iRCCE_mail_check(iRCCE_MAILBOX_ALL); iRCCE_mail_check(iRCCE_MAILBOX_ALL);
} }
/* enable interrupts */
irq_nested_enable(flags);
/* empty mail queue */ /* empty mail queue */
while( iRCCE_mail_recv(&header) == iRCCE_SUCCESS ) { while( iRCCE_mail_recv(&header) == iRCCE_SUCCESS ) {
icc_mail_check_tag(header); icc_mail_check_tag(header);
iRCCE_mail_release( &header ); iRCCE_mail_release( &header );
NOP8;
NOP8;
NOP8;
} }
/* enable interrupts */
irq_nested_enable(flags);
} }