stv0g/metalsvm
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

merge Simon's current version of our mailbox system

Browse source
This commit is contained in:
Stefan Lankes 2011-11-07 13:01:57 -08:00
parent 2e1c062780
commit 198c1dcebd
4 changed files with 216 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/tests.c
View file

@ -100,7 +100,9 @@ static int foo(void* arg)
static int mail_ping(void* arg) {
//icc_mail_ping();
icc_mail_ping_irq();
icc_mail_ping_jitter();
//icc_irq_ping();
//icc_mail_datarates();
//icc_halt();
return 0;

1
arch/x86/include/asm/icc.h
View file

@ -53,6 +53,7 @@ void icc_mail_check(void);
int icc_mail_ping(void);
int icc_send_gic_irq(int core_num);
int icc_mail_ping_irq(void);
int icc_mail_ping_jitter(void);
int icc_mail_noise(void);
void icc_wait(int tag);

157
arch/x86/scc/iRCCE_mailbox.c
View file

@ -144,73 +144,59 @@ static int iRCCE_mail_fetch(
}
//------------------------------------------------------------------------------
// FUNCTION: iRCCE_mailbox_check
//------------------------------------------------------------------------------
/**
* @brief routine to check for new mail in mailboxes
*
* This function has to be called from time to time. It empties all mailboxes of
* the participating cores if the corresponding sent-flag is set and the mailbox
* is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset.
* Here we have to use a little trick because we can only write to the MPB in
* cacheline granularity. We set the appropriate flag to zero and afterwords
* touch the MPB on another cacheline. That causes the write combine buffer to
* write out the data.
*/
//------------------------------------------------------------------------------
static iRCCE_MAIL_HEADER dummy_header = {0, 0, 0, NULL, 0, 0, 0, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} };
static int iRCCE_mailbox_check(void) {
int i,j;
uint32_t flags;
/* disable interrupts */
flags = irq_nested_disable();
for( j=1; j<RCCE_NP; ++j ) {
i = (j+RCCE_IAM)%RCCE_NP;
// only check open mailboxes
if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) {
RC_cache_invalidate();
if( iRCCE_mailbox_recv[i]->sent ) {
iRCCE_mail_fetch(i);
// reset senders flag
RC_cache_invalidate();
*(iRCCE_mailbox_recv[i]) = dummy_header;
}
}
}
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_SUCCESS;
}
//------------------------------------------------------------------------------
// FUNCTION: iRCCE_mail_check
//------------------------------------------------------------------------------
/**
* @brief routine to check one specific mailbox
* @param sender is the core ID from which the mailbox is checked
* @brief routine to check one specific mailbox or all
* @param sender is the core ID from which the mailbox is checked use
* iRCCE_MAILBOX_ALL as wildcard to check all mailboxes
*
* This function may be called by the user application to check one specific
* mailbox. It is recommended to use it in combination with an inter core
* interrupt.
* mailbox. It is recommended to use it in combination with an inter-core
* interrupt. It empties one or all mailboxes of the participating cores if the
* corresponding sent-flag is set and the mailbox is not closed. After calling
* iRCCE_mail_fetch the sent-flag has to be reset. Here we have to use a little
* trick because we can only write to the MPB in cacheline granularity. We set
* the appropriate flag to zero and afterwords touch the MPB on another
* cacheline. That causes the write combine buffer to write out the data.
*
*/
//------------------------------------------------------------------------------
const iRCCE_MAIL_HEADER dummy_header =
{0, 0, 0, NULL, 0, 0, 0, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} };
int iRCCE_mail_check(int sender) {
uint32_t flags;
int j, i;
int found = 0;
// check all mailboxes in case of wildcard
if( sender == iRCCE_MAILBOX_ALL ) {
iRCCE_mailbox_check();
/* disable interrupts */
flags = irq_nested_disable();
return iRCCE_SUCCESS;
for( j=1; j<RCCE_NP; ++j ) {
i = (j+RCCE_IAM)%RCCE_NP;
// only check open mailboxes
if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) {
RC_cache_invalidate();
if( iRCCE_mailbox_recv[i]->sent ) {
if( !found ) found = 1;
iRCCE_mail_fetch(i);
// reset senders flag
RC_cache_invalidate();
*(iRCCE_mailbox_recv[i]) = dummy_header;
}
}
}
/* enable interrupts */
irq_nested_enable(flags);
return (found == 1)? iRCCE_SUCCESS : iRCCE_MAILBOX_EMPTY;
}
// verify sender's ID
@ -256,25 +242,28 @@ int iRCCE_mail_check(int sender) {
*
* The function checks if the receive queue with highest priority (priority 0)
* contains any mail headers. In this case we pop the first element of that list
* in a FIFO maner. Otherwise iRCCE_mailbox_check() has to be called. Afterwards
* the first element of a non-empty receive queue with highest priority is
* returned.
* in a FIFO maner. Afterwards the first element of a non-empty receive queue
* with highest priority is returned.
*/
//------------------------------------------------------------------------------
int iRCCE_mail_recv(
iRCCE_MAIL_HEADER** header // pointer to incoming header
) { // (memory allocated by iRCCE)
int i;
int i, found = 0;
uint32_t flags;
iRCCE_MAIL_HEADER* help_header;
// if no mail queued pointer must be ZERO
*header = NULL;
/* disable interrupts */
flags = irq_nested_disable();
// check priority queues
for( i=0; i<iRCCE_PRIOS; ++i ) {
if ( iRCCE_mailbox_recv_queue[i].first ) {
/* disable interrupts */
flags = irq_nested_disable();
if ( iRCCE_mailbox_recv_queue[i].first ) {
help_header = iRCCE_mailbox_recv_queue[i].first;
iRCCE_mailbox_recv_queue[i].first =
@ -283,23 +272,21 @@ int iRCCE_mail_recv(
if( iRCCE_mailbox_recv_queue[i].first == NULL ) {
iRCCE_mailbox_recv_queue[i].last = NULL;
}
/* prepare return value */
help_header->next = NULL;
*header = help_header;
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_SUCCESS;
found = 1;
break;
}
}
// no mail queued
*header = NULL;
return iRCCE_MAILBOX_EMPTY;
/* enable interrupts */
irq_nested_enable(flags);
return (found == 1)? iRCCE_SUCCESS : iRCCE_MAILBOX_EMPTY;
}
@ -382,6 +369,8 @@ int iRCCE_mail_send(
char* payload, // pointer to buffer for header payload
int dest // UE that will receive the header
) {
uint32_t flags;
// verify sender's ID
if( (dest < 0) || (dest > RCCE_NP) || (dest == RCCE_IAM) ) {
return iRCCE_ERROR_TARGET;
@ -390,20 +379,28 @@ int iRCCE_mail_send(
// if dest mailbox is full, check for incoming mail
RC_cache_invalidate();
while( iRCCE_mailbox_send[dest]->sent ) {
iRCCE_mailbox_check();
// iRCCE_mail_check(iRCCE_MAILBOX_ALL);
RC_cache_invalidate();
NOP8;
NOP8;
NOP8;
}
/* disable interrupts */
flags = irq_nested_disable();
// check if mailbox is closed
RCCE_acquire_lock( dest );
RC_cache_invalidate();
if( iRCCE_mailbox_send[dest]->closed ) {
RCCE_release_lock( dest );
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_MAILBOX_CLOSED;
}
/* disable interrupts */
// flags = irq_nested_disable();
// prepare header
iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio,
RCCE_FLAG_UNSET, RCCE_FLAG_UNSET,
@ -425,11 +422,11 @@ int iRCCE_mail_send(
*(int *)RCCE_fool_write_combine_buffer = 1;
RC_cache_invalidate();
/* enable interrupts */
// irq_nested_enable(flags);
RCCE_release_lock( dest );
/* enable interrupts */
irq_nested_enable(flags);
return iRCCE_SUCCESS;
}
@ -474,7 +471,7 @@ int iRCCE_last_mail_recv(void) {
//------------------------------------------------------------------------------
int iRCCE_mailbox_wait(void) {
while( iRCCE_last_mail_recv() == iRCCE_LAST_MAILS_NOT_RECV ) {
iRCCE_mailbox_check();
iRCCE_mail_check(iRCCE_MAILBOX_ALL);
}
return iRCCE_SUCCESS;
@ -531,7 +528,7 @@ int iRCCE_mailbox_flush(void) {
* last-mail.
*
* This function closes a mailbox of the given rank. If the check flag is set
* an iRCCE_mailbox_check()-call is performed. The close procedure has to be
* an iRCCE_mail_check()-call is performed. The close procedure has to be
* locked to be sure that no UE sends any mail while closing the mailbox.
*/
//------------------------------------------------------------------------------

160
arch/x86/scc/icc.c
View file

@ -298,7 +298,7 @@ int icc_halt(void)
#define ROUNDS 1000
#define CORE_A 0 // sender
#define CORE_B 1 // receiver
#define CORE_B 25 // receiver
int icc_send_gic_irq(int core_num) {
volatile uint32_t* irq_request = (volatile uint32_t*)(FPGA_BASE+IRQ_REQUEST+RC_MY_COREID*8);
@ -346,7 +346,7 @@ int icc_mail_ping(void)
/* wait for response */
do {
res = iRCCE_mail_check(CORE_B);
res = iRCCE_mail_check(iRCCE_MAILBOX_ALL);
} while( res != iRCCE_SUCCESS );
/* release mail */
@ -357,7 +357,7 @@ int icc_mail_ping(void)
else {
/* wait for request */
do {
res = iRCCE_mail_check(CORE_A);
res = iRCCE_mail_check(iRCCE_MAILBOX_ALL);
} while( res != iRCCE_SUCCESS );
/* check mail */
@ -439,50 +439,162 @@ int icc_mail_ping_irq(void)
return 0;
}
#define _iRQ_NOISE_ 0
int icc_mail_ping_jitter(void)
{
kprintf( "Hello from gitter_test ... \n" );
/* return if not core A */
if( RCCE_IAM != CORE_A ) return 0;
uint32_t flags;
uint64_t timer = 0;
uint64_t max = 0;
uint64_t min = ULONG_MAX;
uint64_t sum = 0;
int i;
int res;
iRCCE_MAIL_HEADER* recv_header = NULL;
kprintf( "my_rank = %d\n", RCCE_IAM );
kprintf( "rem_rank = %d\n", CORE_B );
kprintf( "rounds = %d\n", ROUNDS );
// disable interrupts
flags = irq_nested_disable();
for( i=0; i<ROUNDS+1; ++i ) {
/* start timer */
timer = rdtsc();
/* send ping request */
iRCCE_mail_send(0, PING_REQ, 0, NULL, CORE_B);
/* send interrupt */
icc_send_gic_irq(CORE_B);
/* wait for response */
do {
res = iRCCE_mail_check(CORE_B);
} while( res != iRCCE_SUCCESS );
iRCCE_mail_recv(&recv_header);
iRCCE_mail_release(&recv_header);
/* stop timer and update eval values */
timer = rdtsc() - timer;
if( i > 0 ) {
max = ( max < timer )? timer : max;
min = ( min > timer )? timer : min;
sum += timer;
}
}
kprintf( "Average was: %d nsec\n", sum*1000/(2*ROUNDS*533) );
kprintf( "Maximum was: %d nsec\n", max*1000/(2*533) );
kprintf( "Minimum was: %d nsec\n", min*1000/(2*533) );
kprintf( "Jitter was: %d nsec\n", (max-min)*1000/(2*533) );
irq_nested_enable(flags);
return 0;
}
#undef _IRQ_NOISE_
#define NOISE_PRIO 1
int icc_mail_noise(void) {
int i, j, res;
int num_ranks = RCCE_num_ues();
int count = 0;
iRCCE_MAIL_HEADER* recv_mail = NULL;
/* timer vars */
uint64_t timer;
uint64_t tmr;
uint64_t tmr_send = 0;
uint64_t tmr_recv = 0;
uint64_t tmr_release = 0;
uint64_t tmr_chck = 0;
kprintf( "my_ue = %d\n", RCCE_IAM );
// leave function if not participating
if( !((RCCE_IAM == 4) || (RCCE_IAM == 2) || (RCCE_IAM == CORE_B)) ) {
kprintf( "mail_noise: leaving" );
return -1;
}
// if( (my_ue == CORE_A) || (my_ue == CORE_B) ) {
// kprintf( "mail_noise: leaving" );
// return -1;
// }
kprintf( "Hello from icc_mail_noise: my_ue = %d\n", RCCE_IAM );
kprintf( "num_ues = %d\n", num_ranks );
for( i=0; i<10000; ++i ) {
timer = rdtsc();
for( i=0; i<20000; ++i ) {
if( !(i%1000) ) kprintf( "%d ", i );
tmr = rdtsc();
iRCCE_mail_check(iRCCE_MAILBOX_ALL);
tmr = rdtsc() - tmr;
tmr_chck += tmr;
/* send a mail to each UE */
for( j=0; j<num_ranks; ++j ) {
if( !((j == 4) || (j == 2)/* || (j == CORE_B) */) )
continue;
// if( (j == CORE_A) /* || (j == CORE_B)*/ ) continue;
/* send noise mail */
iRCCE_mail_send(0, NOISE, 1, NULL, j);
#ifdef _IRQ_NOISE_
kprintf( "sending irq ... " );
icc_send_gic_irq(j);
#endif
iRCCE_mail_recv(&recv_mail);
icc_mail_check_tag(recv_mail);
if( recv_mail ) iRCCE_mail_release(&recv_mail);
tmr = rdtsc();
iRCCE_mail_send(0, NOISE, NOISE_PRIO, NULL, j);
tmr = rdtsc() - tmr;
tmr_send += tmr;
tmr = rdtsc();
res = iRCCE_mail_recv(&recv_mail);
tmr = rdtsc() - tmr;
tmr_recv += tmr;
if( res == iRCCE_SUCCESS ) {
icc_mail_check_tag(recv_mail);
tmr = rdtsc();
iRCCE_mail_release(&recv_mail);
tmr = rdtsc() - tmr;
tmr_release += tmr;
count++;
}
}
}
kprintf( "XXX XXX XXX" );
do {
tmr = rdtsc();
iRCCE_mail_check(iRCCE_MAILBOX_ALL);
tmr = rdtsc() - tmr;
tmr_chck += tmr;
tmr = rdtsc();
res = iRCCE_mail_recv(&recv_mail);
icc_mail_check_tag(recv_mail);
if( recv_mail ) iRCCE_mail_release(&recv_mail);
tmr = rdtsc() - tmr;
tmr_recv += tmr;
if( res == iRCCE_SUCCESS ) {
icc_mail_check_tag(recv_mail);
tmr = rdtsc();
iRCCE_mail_release(&recv_mail);
tmr = rdtsc() - tmr;
tmr_release += tmr;
count++;
}
} while( res == iRCCE_SUCCESS );
timer = rdtsc() - timer;
kprintf( "Count = %d\n", count );
kprintf( "Time: %d ms\n", timer/(1000*get_cpu_frequency()) );
kprintf( "Time in send: %d ms\n", tmr_send/(1000*get_cpu_frequency()) );
kprintf( "Time in recv: %d ms\n", tmr_recv/(1000*get_cpu_frequency()) );
kprintf( "Time in chck: %d ms\n", tmr_chck/(1000*get_cpu_frequency()) );
kprintf( "Time in release: %d ms\n", tmr_release/(1000*get_cpu_frequency()) );
kprintf( "XXX XXX XXX" );
return 0;
}