From bc28b300e0152be7cf7b3b392036f9c216d47052 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Fri, 1 Jul 2011 12:18:15 +0200 Subject: [PATCH 01/36] doxygen comments in iRCCE_mailbox.c --- arch/x86/scc/iRCCE_mailbox.c | 132 ++++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 11 deletions(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index ae3132cb..1284acf5 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -41,6 +41,11 @@ static int iRCCE_mailbox_close_one(int rank, int check); //------------------------------------------------------------------------------ // routine for printing given header (debugging purpose) //------------------------------------------------------------------------------ +/** + * @brief routine for printing a given header + * @param header is a pointer to a given iRCCE_MAIL_HEADER structure +*/ + void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { kprintf( "\n" @@ -59,9 +64,20 @@ void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { } //------------------------------------------------------------------------------ -// FUNCTION: iRCCE_mailbox_enqueue_header +// FUNCTION: iRCCE_mail_fetch //------------------------------------------------------------------------------ -// routine to put push received header into iRCCE_mailbox_recv_queue +/** + * @brief routine to check for new mail in a given mailbox + * @param rank is the ID of the ranks mailbox to be emptied + * + * The function checks if the mailbox has new mail for a given rank. In case of + * new mail it needs memory for the received header. Either there is memory + * in the internal garbage collection or it has to allocated. The next step is + * to check wheter a last-mail was received or a normal one. A last-mail is + * indicated by the iRCCE_LAST_MAIL tag. A last-mail entails the appropriate + * flag in the iRCCE_last_mail array to be set. Otherwise the header has to be + * enqueued in the mailbox_recv_queue accordingly to the priority field. + */ //------------------------------------------------------------------------------ static int iRCCE_mail_fetch( int rank // rank from whom to empty mailbox @@ -119,7 +135,17 @@ static int iRCCE_mail_fetch( //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_check //------------------------------------------------------------------------------ -// routine to check for new mail in mailboxes +/** + * @brief routine to check for new mail in mailboxes + * + * This function has to be called from time to time. It empties all mailboxes of + * the participating cores if the corresponding sent-flag is set and the mailbox + * is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset. + * Here we have to use a little trick because we can only write to the MPB in + * cacheline granularity. We set the appropriate flag to zero and afterwords + * touch the MPB on another cacheline. That causes the write combine buffer to + * write out the data. + */ //------------------------------------------------------------------------------ static int iRCCE_mailbox_check() { int i; @@ -146,7 +172,18 @@ static int iRCCE_mailbox_check() { //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mail_recv //------------------------------------------------------------------------------ -// routine for fetching received headers out of iRCCE_mailbox_recv_queue +/** + * @brief routine for fetching received headers out of iRCCE_mailbox_recv_queue + * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure + * @return iRCCE_SUCCESS if there was new mail; iRCCE_MAILBOX_EMPTY else + * @todo implement fairer dequeue mechanism + * + * The function checks if the receive queue with highest priority (priority 0) + * contains any mail headers. In this case we pop the first element of that list + * in a FIFO maner. Otherwise iRCCE_mailbox_check() has to be called. Afterwards + * the first element of a non-empty receive queue with highest priority is + * returned. + */ //------------------------------------------------------------------------------ int iRCCE_mail_recv( iRCCE_MAIL_HEADER** header // pointer to incoming header @@ -172,6 +209,7 @@ int iRCCE_mail_recv( return iRCCE_SUCCESS; } } + // no mail queued *header = NULL; return iRCCE_MAILBOX_EMPTY; @@ -181,7 +219,15 @@ int iRCCE_mail_recv( //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mail_release //------------------------------------------------------------------------------ -// routine to free memory for given header +/** + * @brief routine to store released header by user in garbage collection + * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure + * @return iRCCE_SUCCESS in any case + * + * This function enqueus a pointer to memory for an iRCCE_MAIL_HEADER structure + * that is not used by the user program anymore. 'header' points to NULL by + * return of the function. + */ //------------------------------------------------------------------------------ int iRCCE_mail_release( iRCCE_MAIL_HEADER** header @@ -207,7 +253,31 @@ int iRCCE_mail_release( //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mail_send //------------------------------------------------------------------------------ -// routine for sending headers into receivers mailbox (blocking) +/** + * @brief routine to send a mail to a given destination (blocking) + * @param size is the size of the following message. This message may be + * contained in the payload or send by an isend()-call + * @param tag is an integer value to distinguish between different mail types + * @param prio indicates the priority of the mail. 0 is the highest priority + * whereas 4 is the lowest. + * @param payload is a pointer to byte array with a size of + * iRCCE_MAIL_HEADER_PAYLOAD. If NULL is passed nothing is done, otherwise array + * pointed by payload is copied into the header. + * @param dest indicates the destination of the mail in terms of the ID of + * one of the participating ranks + * @return iRCCE_SUCCESS if send was successful. If target mailbox is closed + * iRCCE_MAILBOX_CLOESD is returned. + * + * First it has to be checked if the target mailbox still contains an unread mail. + * If this is the case there is time to empty the own mailboxes. It blocks until + * the receiver has emptied its mailbox. The next step is to acquire the lock + * for the target mailbox to be sure that the mailbox is not closed by the + * receiver while the mail is delivered. After locking the mailbox an + * iRCCE_MAIL_HEADER is generated according with the parameters (but with a + * sent-flag set to zero)and is copied into the target mailbox. After all data + * beeing written the appropropriate sent-flag has to be set with the same trick + * already used in iRCCE_mail_check(). Now the lock can be released. + */ //------------------------------------------------------------------------------ int iRCCE_mail_send( size_t size, // size of following message expected to be send/received @@ -261,7 +331,14 @@ int iRCCE_mail_send( //------------------------------------------------------------------------------ // FUNCTION: iRCCE_last_mail_recv //------------------------------------------------------------------------------ -// check if all final headers are received from all UEs +/** + * @brief check if all final headers are received from all UEs + * @return iRCCE_SUCCES if all last-mails arrive iRCCE_LAST_MAILS_NOT_RECV + * otherwise + * + * This functions is used to determine if all last-mails arrived at the calling + * UE. Therefore it checks the iRCCE_last_mail array if all flags are set. + */ //------------------------------------------------------------------------------ int iRCCE_last_mail_recv(void) { int i; @@ -280,7 +357,15 @@ int iRCCE_last_mail_recv(void) { //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_wait //------------------------------------------------------------------------------ -// wait for all last-mails to be received +/** + * @brief wait for all last-mails to be received + * @return iRCCE_SUCCESS + * + * This functions blocks in a loop calling continously iRCCE_last_mail_recv() + * until its return value is iRCCE_SUCCESS what implicates that all last-mails + * of the participating UEs arrived at the calling UE. + * This function is used to shut down the mailbox environment. + */ //------------------------------------------------------------------------------ int iRCCE_mailbox_wait(void) { while( iRCCE_last_mail_recv() == iRCCE_LAST_MAILS_NOT_RECV ) { @@ -294,7 +379,13 @@ int iRCCE_mailbox_wait(void) { //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_flush //------------------------------------------------------------------------------ -// dequeue all iRCCE_mailbox_recv_queue elements and free memory +/** + * @brief dequeue all iRCCE_mailbox_recv_queue elements and free memory + * @return iRCCE_SUCCESS + * + * This function empties all iRCCE_mailbox_recv_queues whith no regard to their + * content. This function is used to shut down the mailbox environment. + */ //------------------------------------------------------------------------------ int iRCCE_mailbox_flush(void) { int i; @@ -314,7 +405,18 @@ int iRCCE_mailbox_flush(void) { //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_close_one //------------------------------------------------------------------------------ -// routine to close one mailbox +/** + * @brief routine to close one mailbox + * @return iRCCE_SUCCESS + * @param rank is the ID of the ranks mailbox to be closed + * @param check is a flag indicating wether the mailbox has to be emptied before + * closing or not. This is required for a close-call as a result of a received + * last-mail. + * + * This function closes a mailbox of the given rank. If the check flag is set + * an iRCCE_mail_check()-call is performed. The close procedure has to be locked + * to be sure that no UE sends any mail while closing the mailbox. + */ //------------------------------------------------------------------------------ static int iRCCE_mailbox_close_one(int rank, int check) { RCCE_acquire_lock( RCCE_IAM ); @@ -345,7 +447,15 @@ static int iRCCE_mailbox_close_one(int rank, int check) { //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_close() //------------------------------------------------------------------------------ -// routine to close one or all mailboxes +/** + * @brief routine to close one or all mailboxes + * @param rank is the ID of the UEs mailbox to be closed if iRCCE_MAILBOX_ALL + * is passed all mailboxes are closed by the calling UE + * @return iRCCE_SUCCESS + * + * This functions closed one or all mailboxes of the calling UE. This is done by + * calling iRCCE_mailbox_close_one for one or all mailboxes. + */ //------------------------------------------------------------------------------ int iRCCE_mailbox_close(int rank) { if( rank == iRCCE_MAILBOX_ALL ) { From f4443e6761d1297978582eb42e8eb7b3974a9e75 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Fri, 1 Jul 2011 15:41:41 +0200 Subject: [PATCH 02/36] new mail_pingpong test --- arch/x86/include/asm/icc.h | 5 +++ arch/x86/scc/icc.c | 67 +++++++++++++++++++++++--------------- kernel/tests.c | 11 ++----- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index cb806761..191a0566 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -39,6 +39,11 @@ typedef struct { extern bootinfo_t* bootinfo; +enum icc_mail_requests { + PING_REQ=1, + PING_RESP, +}; + /* #define ICC_TYPE_IP (1 << 0) #define ICC_TYPE_SVM (1 << 1) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index b2a517b9..3b7a9853 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -210,46 +210,59 @@ int icc_halt(void) return 0; } -#define MAIL_MSG_SIZE 20 +#define ROUNDS 1000 int icc_mail_ping( void ) { uint32_t flags; uint64_t timer; - int remote_rank = (my_ue+1)%2; + int rem_rank = (my_ue+1)%2; + int i; + iRCCE_MAIL_HEADER* recv_header = NULL; - char send_buffer[MAIL_MSG_SIZE]; - int length = MAIL_MSG_SIZE; - - if( my_ue ) { - return 0; - } - kprintf( "Hello from mail_ping ... \n" ); // disable interrupts flags = irq_nested_disable(); - - // start timer - timer = rdtsc(); - - // start anylength send - iRCCE_isend( send_buffer, -length, remote_rank, NULL ); - - // wait for response - iRCCE_MAIL_HEADER* mail; - do { - iRCCE_mail_recv( &mail ); - } while( !mail ); - // stop timer + for( i=0; itag == 2 ) { - kprintf( "Response received in %d ticks!\n", timer ); - } - else { - kprintf( "Response didn't arrive!\n" ); + if( my_rank == 0 ) { + printf( "mail_pingpong needs in average %.3f µsec (%d ticks)!\n", + timer/(2.0*ROUNDS*533), timer/(2*ROUNDS) ); } irq_nested_enable(flags); diff --git a/kernel/tests.c b/kernel/tests.c index 9ef0a393..e3528a7b 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -88,13 +88,8 @@ static int STDCALL foo(void* arg) #ifdef CONFIG_ROCKCREEK int STDCALL mail_ping(void* arg) { - int i; - - for(i=0; i<20; i++) { icc_mail_ping(); - //icc_halt(); - udelay(500000); - } + icc_halt(); return 0; } @@ -135,9 +130,9 @@ int test_init(void) //create_kernel_task(NULL, join_test, NULL); //create_kernel_task(NULL, producer, NULL); //create_kernel_task(NULL, consumer, NULL); - //create_kernel_task(NULL, mail_ping, NULL); + create_kernel_task(NULL, mail_ping, NULL); //create_user_task(NULL, "/bin/hello", argv); - create_user_task(NULL, "/bin/tests", argv); + //create_user_task(NULL, "/bin/tests", argv); //create_user_task(NULL, "/bin/jacobi", argv); //create_user_task(NULL, "/bin/jacobi", argv); From ae496a6d19f9f2f2815214f475cd63423117ab8b Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Fri, 1 Jul 2011 06:46:59 -0700 Subject: [PATCH 03/36] typos --- arch/x86/scc/iRCCE_isend.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/scc/iRCCE_isend.c b/arch/x86/scc/iRCCE_isend.c index 51cde3eb..ec1ee81e 100644 --- a/arch/x86/scc/iRCCE_isend.c +++ b/arch/x86/scc/iRCCE_isend.c @@ -262,18 +262,18 @@ int iRCCE_isend( iRCCE_mail_send( send_size, iRCCE_ANYLENGTH_PIGGYBACK, 0, privbuf, dest ); - NOP8; - NOP8; - send_irq( dest ); + //NOP8; + //NOP8; + //send_irq( dest ); return iRCCE_SUCCESS; } // we need an extra isend-call else { iRCCE_mail_send( send_size, iRCCE_ANYLENGTH, 0, NULL, dest ); - NOP8; - NOP8; - send_irq( dest ); + //NOP8; + //NOP8; + //send_irq( dest ); return iRCCE_isend_general( privbuf, send_size, dest, request ); } From c2341562c71b5412e8a4f05d5b61430689cb2dd2 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Fri, 1 Jul 2011 07:08:19 -0700 Subject: [PATCH 04/36] more rounds in mail_pingpong --- arch/x86/scc/icc.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 3b7a9853..ecb71e3a 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -210,7 +210,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 1000 +#define ROUNDS 100000 int icc_mail_ping( void ) { @@ -226,7 +226,7 @@ int icc_mail_ping( void ) flags = irq_nested_disable(); for( i=0; i Date: Sat, 2 Jul 2011 17:51:25 +0200 Subject: [PATCH 05/36] add mail_ping_irq example --- arch/x86/scc/icc.c | 81 ++++++++++++++++++++++++++++++++++++++++------ kernel/tests.c | 3 +- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index ecb71e3a..e0825836 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -210,7 +210,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 100000 +#define ROUNDS 10000 int icc_mail_ping( void ) { @@ -271,6 +271,61 @@ int icc_mail_ping( void ) return 0; } + +int icc_mail_ping_irq( void ) +{ + /* return if not core 0 */ + if( my_ue ) return; + + uint32_t flags; + uint64_t timer; + int rem_rank = (my_ue+1)%2; + int i; + iRCCE_MAIL_HEADER* recv_header = NULL; + + kprintf( "Hello from mail_ping ... \n" ); + + // disable interrupts + flags = irq_nested_disable(); + + for( i=0; itag == 1 ) { - iRCCE_mail_send( 0, 2, 0, NULL, header->source ); - } - else if( header->tag == iRCCE_ANYLENGTH ) { + switch( header->tag ) { + case iRCCE_ANYLENGTH: recv_buffer = (char*)kmalloc( header->size ); - iRCCE_irecv( recv_buffer, header->size, header->source, NULL ); - iRCCE_mail_send( 0, 2, 0, NULL, header->source ); - } - else if( header->tag == iRCCE_ANYLENGTH_PIGGYBACK ) { - iRCCE_mail_send( 0, 2, 0, NULL, header->source ); + iRCCE_irecv(recv_buffer, header->size, + header->source, NULL ); + iRCCE_mail_send(0, 2, 0, NULL, header->source); + break; + + case iRCCE_ANYLENGTH_PIGGYBACK: + iRCCE_mail_send(0, 2, 0, NULL, header->source); + break; + + case PING_REQ: + iRCCE_mail_send(0, PING_RESP, 0, NULL, header->source); + break; } + iRCCE_mail_release( &header ); } } diff --git a/kernel/tests.c b/kernel/tests.c index e3528a7b..6d5a0e72 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -88,7 +88,8 @@ static int STDCALL foo(void* arg) #ifdef CONFIG_ROCKCREEK int STDCALL mail_ping(void* arg) { - icc_mail_ping(); + // icc_mail_ping(); + icc_mail_ping_irq(); icc_halt(); return 0; From c0fe176945b3bf784d6be746f5fbbd53e4b8f8c6 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Sat, 2 Jul 2011 09:09:53 -0700 Subject: [PATCH 06/36] remove typos --- arch/x86/include/asm/icc.h | 1 + arch/x86/scc/icc.c | 10 +++++----- kernel/tests.c | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index 191a0566..cd059a42 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -56,6 +56,7 @@ int icc_halt(void); int icc_send_irq(int ue); void icc_mail_check(void); int icc_mail_ping(void); +int icc_mail_ping_irq(void); #endif diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index e0825836..2ef4ccfd 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -210,12 +210,12 @@ int icc_halt(void) return 0; } -#define ROUNDS 10000 +#define ROUNDS 100000 int icc_mail_ping( void ) { uint32_t flags; - uint64_t timer; + uint64_t timer = 0; int rem_rank = (my_ue+1)%2; int i; iRCCE_MAIL_HEADER* recv_header = NULL; @@ -275,15 +275,15 @@ int icc_mail_ping( void ) int icc_mail_ping_irq( void ) { /* return if not core 0 */ - if( my_ue ) return; + if( my_ue ) return 0; uint32_t flags; - uint64_t timer; + uint64_t timer = 0; int rem_rank = (my_ue+1)%2; int i; iRCCE_MAIL_HEADER* recv_header = NULL; - kprintf( "Hello from mail_ping ... \n" ); + kprintf( "Hello from mail_ping_irq ... \n" ); // disable interrupts flags = irq_nested_disable(); diff --git a/kernel/tests.c b/kernel/tests.c index 6d5a0e72..12e79e71 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -88,7 +88,7 @@ static int STDCALL foo(void* arg) #ifdef CONFIG_ROCKCREEK int STDCALL mail_ping(void* arg) { - // icc_mail_ping(); + icc_mail_ping(); icc_mail_ping_irq(); icc_halt(); From 439c56292ed457ece3347d0dbb76c5acfb715e5e Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Mon, 4 Jul 2011 12:27:28 +0200 Subject: [PATCH 07/36] add icc_mail_noise to generate noise on the other cores --- arch/x86/include/asm/icc.h | 1 + arch/x86/scc/icc.c | 31 ++++++++++++++++++++++++++++++- kernel/tests.c | 3 ++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index cd059a42..56e29ca2 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -57,6 +57,7 @@ int icc_send_irq(int ue); void icc_mail_check(void); int icc_mail_ping(void); int icc_mail_ping_irq(void); +int icc_mail_noise(void); #endif diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 2ef4ccfd..503250e1 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -271,11 +271,15 @@ int icc_mail_ping( void ) return 0; } +#define CORE_A 0 +#define CORE_B 1 +#define NOISE_ROUDNS 1000000 + int icc_mail_ping_irq( void ) { /* return if not core 0 */ - if( my_ue ) return 0; + if( my_ue != CORE_A ) return 0; uint32_t flags; uint64_t timer = 0; @@ -326,6 +330,31 @@ int icc_mail_ping_irq( void ) return 0; } + +int icc_mail_noise() { + int i, j; + int num_ranks = RCCE_num_ues(); + iRCCE_MAIL_HEADER* recv_mail = NULL; + + // leave function if not participating + if( (my_ue == 0) || (my_ue == 1) ) { + return -1; + } + + for( i=0; i Date: Mon, 4 Jul 2011 03:34:32 -0700 Subject: [PATCH 08/36] fix typos --- arch/x86/scc/icc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 2ef4ccfd..858e2abe 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -295,6 +295,7 @@ int icc_mail_ping_irq( void ) /* send interrupt */ + NOP8; icc_send_irq(rem_rank); /* wait for response */ From f7ca4a2ef1e690894906e56b13f90acb3cbf821d Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Mon, 4 Jul 2011 05:06:46 -0700 Subject: [PATCH 09/36] new test scenario with all cores --- arch/x86/scc/icc.c | 20 ++++++++++-------- kernel/tests.c | 2 +- tools/metalsvm.mt | 52 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 8074519c..d3cd427d 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -271,9 +271,9 @@ int icc_mail_ping( void ) return 0; } -#define CORE_A 0 -#define CORE_B 1 -#define NOISE_ROUDNS 1000000 +#define CORE_A 0 // sender +#define CORE_B RCCE_num_ues()-1 // receiver +#define NOISE_ROUNDS 1000000 int icc_mail_ping_irq( void ) @@ -283,12 +283,12 @@ int icc_mail_ping_irq( void ) uint32_t flags; uint64_t timer = 0; - int rem_rank = (my_ue+1)%2; + int rem_rank = CORE_B; int i; iRCCE_MAIL_HEADER* recv_header = NULL; kprintf( "Hello from mail_ping_irq ... \n" ); - + kprintf( "rem_rank = %d\n", rem_rank ); // disable interrupts flags = irq_nested_disable(); @@ -305,7 +305,9 @@ int icc_mail_ping_irq( void ) /* wait for response */ do { iRCCE_mail_recv(&recv_header); - } while( !recv_header ); + if( recv_header->tag == PING_RESP ) + break; + } while( 1 ); /* release mail */ iRCCE_mail_release(&recv_header); @@ -338,14 +340,14 @@ int icc_mail_noise() { iRCCE_MAIL_HEADER* recv_mail = NULL; // leave function if not participating - if( (my_ue == 0) || (my_ue == 1) ) { + if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { return -1; } - for( i=0; i Date: Wed, 13 Jul 2011 15:08:33 +0200 Subject: [PATCH 10/36] update to current mailbox-system --- arch/x86/include/asm/iRCCE_lib.h | 4 +- arch/x86/include/asm/iRCCE_lib.h.old | 56 ++++ arch/x86/scc/iRCCE_admin.c | 18 +- arch/x86/scc/iRCCE_admin.c.old | 131 ++++++++ arch/x86/scc/iRCCE_mailbox.c | 113 ++++--- arch/x86/scc/iRCCE_mailbox.c.old | 474 +++++++++++++++++++++++++++ 6 files changed, 747 insertions(+), 49 deletions(-) create mode 100644 arch/x86/include/asm/iRCCE_lib.h.old create mode 100644 arch/x86/scc/iRCCE_admin.c.old create mode 100644 arch/x86/scc/iRCCE_mailbox.c.old diff --git a/arch/x86/include/asm/iRCCE_lib.h b/arch/x86/include/asm/iRCCE_lib.h index 372b9cdd..70df2596 100644 --- a/arch/x86/include/asm/iRCCE_lib.h +++ b/arch/x86/include/asm/iRCCE_lib.h @@ -36,7 +36,7 @@ extern volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_send[RCCE_MAXNP]; extern volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_recv[RCCE_MAXNP]; // queue for received headers -extern iRCCE_MAIL_HEADER* iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; +extern iRCCE_MAIL_HEADER_LIST iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; // flags for last mail extern iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; @@ -45,7 +45,7 @@ extern iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; extern iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; // garbage collection for mailbox -extern iRCCE_MAIL_TRASH_BIN iRCCE_mail_garbage; +extern iRCCE_MAIL_HEADER_LIST iRCCE_mail_garbage; #ifdef _OPENMP #pragma omp threadprivate (iRCCE_isend_queue, iRCCE_irecv_queue) #endif diff --git a/arch/x86/include/asm/iRCCE_lib.h.old b/arch/x86/include/asm/iRCCE_lib.h.old new file mode 100644 index 00000000..372b9cdd --- /dev/null +++ b/arch/x86/include/asm/iRCCE_lib.h.old @@ -0,0 +1,56 @@ +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +#ifndef IRCCE_LIB_H +#define IRCCE_LIB_H +#include +#include + +extern iRCCE_SEND_REQUEST* iRCCE_isend_queue; +extern iRCCE_RECV_REQUEST* iRCCE_irecv_queue[RCCE_MAXNP]; +extern iRCCE_RECV_REQUEST* iRCCE_irecv_any_source_queue; + +// pointer to MPB-mailbox-space +extern volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_send[RCCE_MAXNP]; +extern volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_recv[RCCE_MAXNP]; + +// queue for received headers +extern iRCCE_MAIL_HEADER* iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; + +// flags for last mail +extern iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; + +// field to store open/closed status of mailboxes +extern iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; + +// garbage collection for mailbox +extern iRCCE_MAIL_TRASH_BIN iRCCE_mail_garbage; +#ifdef _OPENMP +#pragma omp threadprivate (iRCCE_isend_queue, iRCCE_irecv_queue) +#endif + +int iRCCE_test_flag(RCCE_FLAG, RCCE_FLAG_STATUS, int *); + +#endif + diff --git a/arch/x86/scc/iRCCE_admin.c b/arch/x86/scc/iRCCE_admin.c index 2a28f09e..cd119167 100644 --- a/arch/x86/scc/iRCCE_admin.c +++ b/arch/x86/scc/iRCCE_admin.c @@ -55,10 +55,10 @@ volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_recv[RCCE_MAXNP]; // store addresses f volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_send[RCCE_MAXNP]; // store addresses for sending headeres // mailbox recv queue -iRCCE_MAIL_HEADER* iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; +iRCCE_MAIL_HEADER_LIST iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; // mail garbage queue -iRCCE_MAIL_TRASH_BIN iRCCE_mail_garbage; +iRCCE_MAIL_HEADER_LIST iRCCE_mail_garbage; // flag indicating if last header was received iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; @@ -71,7 +71,16 @@ iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; //-------------------------------------------------------------------------------------- // FUNCTION: iRCCE_init //-------------------------------------------------------------------------------------- -// initialize the library +/** + * @brief initialize the library + * + * To initialize the mailbox system the function calles RCCE_malloc() as often + * as there are UEs participating. As a result the respective mailboxes are + * located at the begin of the local MPB directly behind the space reserved for + * flags by RCCE_init(). In iRCCE_mailbox_recv[i] a pointer to the mailbox is + * saved respectively. To access the send mailboxes at the receiving UEs the + * coresponding pointers are saved in iRCCE_mailbox_send[i] resprectively. + */ //-------------------------------------------------------------------------------------- int iRCCE_init(void) { int i; @@ -93,7 +102,8 @@ int iRCCE_init(void) { // init mail-priority lists for( i=0; i + +#ifdef CONFIG_ROCKCREEK + +#include + +// send request queue +iRCCE_SEND_REQUEST* iRCCE_isend_queue; +// recv request queue +iRCCE_RECV_REQUEST* iRCCE_irecv_queue[RCCE_MAXNP]; + +// recv request queue for those with source = iRCCE_ANY_SOURCE: +iRCCE_RECV_REQUEST* iRCCE_irecv_any_source_queue; + +// mailbox in MPB +volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_recv[RCCE_MAXNP]; // store addresses for receiving headers +volatile iRCCE_MAIL_HEADER* iRCCE_mailbox_send[RCCE_MAXNP]; // store addresses for sending headeres + +// mailbox recv queue +iRCCE_MAIL_HEADER* iRCCE_mailbox_recv_queue[iRCCE_PRIOS]; + +// mail garbage queue +iRCCE_MAIL_TRASH_BIN iRCCE_mail_garbage; + +// flag indicating if last header was received +iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; + +// field to store open/closed status of mailboxes +iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; + + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_init +//-------------------------------------------------------------------------------------- +// initialize the library +//-------------------------------------------------------------------------------------- +int iRCCE_init(void) { + int i; + + for(i=0; inext; + kfree( erase_header, sizeof(iRCCE_MAIL_HEADER) ); + } + + iRCCE_mail_garbage.first = iRCCE_mail_garbage.last = NULL; + return iRCCE_SUCCESS; +} +#endif diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 1284acf5..045a731d 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -1,28 +1,40 @@ -/* - * Copyright 2011 Simon Pickartz, Chair for Operating Systems, - * RWTH Aachen University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This file is part of MetalSVM. - */ +//****************************************************************************** +// Mailbox system. +//****************************************************************************** +// +// Author: Simon Pickartz, +// Chair for Operating Systems, +// RWTH Aachen University +// Date: 005/08/2011 +// +//****************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +#include "iRCCE_lib.h" +#include +#include +#include -/* - * Mailbox system +/** + * + * @file contains implementation of the mailbox system + * @author Simon Pickartz + * * - * [2011-05-08] implemented mailbox send/recv routines - * by Simon Pickartz, Chair for Operating Systems, - * RWTH Aachen University */ #include @@ -48,7 +60,7 @@ static int iRCCE_mailbox_close_one(int rank, int check); void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { - kprintf( "\n" + printf( "\n" "-------------------------\n" "| RCK%d\n" "-------------------------\n" @@ -117,14 +129,17 @@ static int iRCCE_mail_fetch( int prio = header->prio; // enqueue accordingly - if( iRCCE_mailbox_recv_queue[prio] == NULL ) { - iRCCE_mailbox_recv_queue[prio] = header; + header->next = NULL; + + if( iRCCE_mailbox_recv_queue[prio].first == NULL ) { + iRCCE_mailbox_recv_queue[prio].first = header; + iRCCE_mailbox_recv_queue[prio].last = header; } else { - iRCCE_MAIL_HEADER* run = iRCCE_mailbox_recv_queue[prio]; - while( run->next != NULL ) run = run->next; - run->next = header; + iRCCE_mailbox_recv_queue[prio].last->next = header; + iRCCE_mailbox_recv_queue[prio].last = header; } + } @@ -138,7 +153,7 @@ static int iRCCE_mail_fetch( /** * @brief routine to check for new mail in mailboxes * - * This function has to be called from time to time. It empties all mailboxes of + * This function has to be called from time to time. It empeies all mailboxes of * the participating cores if the corresponding sent-flag is set and the mailbox * is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset. * Here we have to use a little trick because we can only write to the MPB in @@ -148,9 +163,13 @@ static int iRCCE_mail_fetch( */ //------------------------------------------------------------------------------ static int iRCCE_mailbox_check() { - int i; + int i,j; + + for( j=1; jnext; + if ( iRCCE_mailbox_recv_queue[i].first ) { + help_header = iRCCE_mailbox_recv_queue[i].first; + + iRCCE_mailbox_recv_queue[i].first = + iRCCE_mailbox_recv_queue[i].first->next; + + if( iRCCE_mailbox_recv_queue[i].first == NULL ) { + iRCCE_mailbox_recv_queue[i].last = NULL; + } + help_header->next = NULL; *header = help_header; @@ -321,10 +346,10 @@ int iRCCE_mail_send( RC_cache_invalidate(); iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; *(int *)RCCE_fool_write_combine_buffer = 1; - RC_cache_invalidate(); + RC_cache_invalidate(); RCCE_release_lock( dest ); - + return iRCCE_SUCCESS; } @@ -391,12 +416,14 @@ int iRCCE_mailbox_flush(void) { int i; for( i=0; inext; + iRCCE_mailbox_recv_queue[i].first = + iRCCE_mailbox_recv_queue[i].first->next; kfree( erase_header, sizeof(iRCCE_MAIL_HEADER) ); - erase_header = iRCCE_mailbox_recv_queue[i]; + erase_header = iRCCE_mailbox_recv_queue[i].first; } } return iRCCE_SUCCESS; diff --git a/arch/x86/scc/iRCCE_mailbox.c.old b/arch/x86/scc/iRCCE_mailbox.c.old new file mode 100644 index 00000000..47935533 --- /dev/null +++ b/arch/x86/scc/iRCCE_mailbox.c.old @@ -0,0 +1,474 @@ +/* + * Copyright 2011 Simon Pickartz, Chair for Operating Systems, + * RWTH Aachen University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of MetalSVM. + */ + +/* + * Mailbox system + * + * [2011-05-08] implemented mailbox send/recv routines + * by Simon Pickartz, Chair for Operating Systems, + * RWTH Aachen University + */ + +#include +#include + + +#ifdef CONFIG_ROCKCREEK +#include + +// forward declaration +static int iRCCE_mailbox_close_one(int rank, int check); + + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mailbox_print_header +//------------------------------------------------------------------------------ +// routine for printing given header (debugging purpose) +//------------------------------------------------------------------------------ +/** + * @brief routine for printing a given header + * @param header is a pointer to a given iRCCE_MAIL_HEADER structure +*/ + +void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { + + kprintf( "\n" + "-------------------------\n" + "| RCK%d\n" + "-------------------------\n" + "| Sender\t: %d\t\n" + "| Size\t\t: %d\t\n" + "| Tag\t\t: %d\t\n" + "| Prio\t\t: %d\t\n" + "| Payload\t: %s\n" + "-------------------------\n\n", + RCCE_IAM, header->source, + header->size, header->tag, + header->prio, header->payload); +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mail_fetch +//------------------------------------------------------------------------------ +/** + * @brief routine to check for new mail in a given mailbox + * @param rank is the ID of the ranks mailbox to be emptied + * + * The function checks if the mailbox has new mail for a given rank. In case of + * new mail it needs memory for the received header. Either there is memory + * in the internal garbage collection or it has to allocated. The next step is + * to check wheter a last-mail was received or a normal one. A last-mail is + * indicated by the iRCCE_LAST_MAIL tag. A last-mail entails the appropriate + * flag in the iRCCE_last_mail array to be set. Otherwise the header has to be + * enqueued in the mailbox_recv_queue accordingly to the priority field. + */ +//------------------------------------------------------------------------------ +static int iRCCE_mail_fetch( + int rank // rank from whom to empty mailbox + ) { + + iRCCE_MAIL_HEADER* header; + + // check for memory in garbage collection or allocate new + if( iRCCE_mail_garbage.first ) { + header = iRCCE_mail_garbage.first; + iRCCE_mail_garbage.first = + iRCCE_mail_garbage.first->next; + + header->next = NULL; + if( iRCCE_mail_garbage.first == NULL ) { + iRCCE_mail_garbage.last = NULL; + } + } + else { + header = (iRCCE_MAIL_HEADER*)kmalloc(sizeof(iRCCE_MAIL_HEADER)); + } + + + // copy header to allocated memory + RC_cache_invalidate(); + iRCCE_memcpy_get( (void*)header, (void*)iRCCE_mailbox_recv[rank], + RCCE_LINE_SIZE ); + + // check if received a last-mail + if( header->tag == iRCCE_LAST_MAIL ) { + iRCCE_last_mail[rank] = 1; + iRCCE_mailbox_close_one( rank, 0 ); // we can close respective mailbox + iRCCE_mail_release( &header ); + } + else { + // check mail priority + int prio = header->prio; + + // enqueue accordingly + if( iRCCE_mailbox_recv_queue[prio] == NULL ) { + iRCCE_mailbox_recv_queue[prio] = header; + } + else { + iRCCE_MAIL_HEADER* run = iRCCE_mailbox_recv_queue[prio]; + while( run->next != NULL ) run = run->next; + run->next = header; + } + } + + + return iRCCE_SUCCESS; +} + + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mailbox_check +//------------------------------------------------------------------------------ +/** + * @brief routine to check for new mail in mailboxes + * + * This function has to be called from time to time. It empties all mailboxes of + * the participating cores if the corresponding sent-flag is set and the mailbox + * is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset. + * Here we have to use a little trick because we can only write to the MPB in + * cacheline granularity. We set the appropriate flag to zero and afterwords + * touch the MPB on another cacheline. That causes the write combine buffer to + * write out the data. + */ +//------------------------------------------------------------------------------ +static int iRCCE_mailbox_check() { + int i; + + for( i=0; isent ) { + iRCCE_mail_fetch(i); + + // reset senders flag + RC_cache_invalidate(); + iRCCE_mailbox_recv[i]->sent = RCCE_FLAG_UNSET; + *(int *)RCCE_fool_write_combine_buffer = 1; + } + } + } + + return iRCCE_SUCCESS; +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mail_recv +//------------------------------------------------------------------------------ +/** + * @brief routine for fetching received headers out of iRCCE_mailbox_recv_queue + * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure + * @return iRCCE_SUCCESS if there was new mail; iRCCE_MAILBOX_EMPTY else + * @todo implement fairer dequeue mechanism + * + * The function checks if the receive queue with highest priority (priority 0) + * contains any mail headers. In this case we pop the first element of that list + * in a FIFO maner. Otherwise iRCCE_mailbox_check() has to be called. Afterwards + * the first element of a non-empty receive queue with highest priority is + * returned. + */ +//------------------------------------------------------------------------------ +int iRCCE_mail_recv( + iRCCE_MAIL_HEADER** header // pointer to incoming header + ) { // (memory allocated by iRCCE) + + int i; + + // if there is no mail, check for incoming + if ( !iRCCE_mailbox_recv_queue[0] ) { + iRCCE_mailbox_check(); + } + + // check priority queues + for( i=0; inext; + help_header->next = NULL; + + *header = help_header; + return iRCCE_SUCCESS; + } + } + + // no mail queued + *header = NULL; + return iRCCE_MAILBOX_EMPTY; + +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mail_release +//------------------------------------------------------------------------------ +/** + * @brief routine to store released header by user in garbage collection + * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure + * @return iRCCE_SUCCESS in any case + * + * This function enqueus a pointer to memory for an iRCCE_MAIL_HEADER structure + * that is not used by the user program anymore. 'header' points to NULL by + * return of the function. + */ +//------------------------------------------------------------------------------ +int iRCCE_mail_release( + iRCCE_MAIL_HEADER** header + ) { + // put header in garbage collection + if( (iRCCE_mail_garbage.first == NULL) + && (iRCCE_mail_garbage.last == NULL ) ) { + + iRCCE_mail_garbage.first = *header; + iRCCE_mail_garbage.last = *header; + } + else { + iRCCE_mail_garbage.last->next = *header; + iRCCE_mail_garbage.last = *header; + } + iRCCE_mail_garbage.last->next = NULL; + + // reset header + *header = NULL; + return iRCCE_SUCCESS; +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mail_send +//------------------------------------------------------------------------------ +/** + * @brief routine to send a mail to a given destination (blocking) + * @param size is the size of the following message. This message may be + * contained in the payload or send by an isend()-call + * @param tag is an integer value to distinguish between different mail types + * @param prio indicates the priority of the mail. 0 is the highest priority + * whereas 4 is the lowest. + * @param payload is a pointer to byte array with a size of + * iRCCE_MAIL_HEADER_PAYLOAD. If NULL is passed nothing is done, otherwise array + * pointed by payload is copied into the header. + * @param dest indicates the destination of the mail in terms of the ID of + * one of the participating ranks + * @return iRCCE_SUCCESS if send was successful. If target mailbox is closed + * iRCCE_MAILBOX_CLOESD is returned. + * + * First it has to be checked if the target mailbox still contains an unread mail. + * If this is the case there is time to empty the own mailboxes. It blocks until + * the receiver has emptied its mailbox. The next step is to acquire the lock + * for the target mailbox to be sure that the mailbox is not closed by the + * receiver while the mail is delivered. After locking the mailbox an + * iRCCE_MAIL_HEADER is generated according with the parameters (but with a + * sent-flag set to zero)and is copied into the target mailbox. After all data + * beeing written the appropropriate sent-flag has to be set with the same trick + * already used in iRCCE_mail_check(). Now the lock can be released. + */ +//------------------------------------------------------------------------------ +int iRCCE_mail_send( + size_t size, // size of following message expected to be send/received + int tag, // tag to indicate message type + char prio, // mail priority + char* payload, // pointer to buffer for header payload + int dest // UE that will receive the header + ) { + + // if dest mailbox is full, check for incoming mail + RC_cache_invalidate(); + while( iRCCE_mailbox_send[dest]->sent ) { + iRCCE_mailbox_check(); + RC_cache_invalidate(); + } + + // check if mailbox is closed + RCCE_acquire_lock( dest ); + RC_cache_invalidate(); + if( iRCCE_mailbox_send[dest]->closed ) { + RCCE_release_lock( dest ); + return iRCCE_MAILBOX_CLOSED; + } + + // prepare header + iRCCE_MAIL_HEADER header = { RCCE_IAM, size, tag, NULL, prio, + RCCE_FLAG_UNSET, RCCE_FLAG_UNSET, + {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} }; + + // payload within the header? + if( payload ) { + memcpy( header.payload, payload, iRCCE_MAIL_HEADER_PAYLOAD ); + } + + // do the actual copy to MPB + RC_cache_invalidate(); + iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], + (void*)&header, RCCE_LINE_SIZE ); + + // set senders flag + RC_cache_invalidate(); + iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; + *(int *)RCCE_fool_write_combine_buffer = 1; + RC_cache_invalidate(); + + RCCE_release_lock( dest ); + + return iRCCE_SUCCESS; +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_last_mail_recv +//------------------------------------------------------------------------------ +/** + * @brief check if all final headers are received from all UEs + * @return iRCCE_SUCCES if all last-mails arrive iRCCE_LAST_MAILS_NOT_RECV + * otherwise + * + * This functions is used to determine if all last-mails arrived at the calling + * UE. Therefore it checks the iRCCE_last_mail array if all flags are set. + */ +//------------------------------------------------------------------------------ +int iRCCE_last_mail_recv(void) { + int i; + int res = iRCCE_SUCCESS; + + for( i=0; inext; + kfree( erase_header, sizeof(iRCCE_MAIL_HEADER) ); + erase_header = iRCCE_mailbox_recv_queue[i]; + } + } + return iRCCE_SUCCESS; +} + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mailbox_close_one +//------------------------------------------------------------------------------ +/** + * @brief routine to close one mailbox + * @return iRCCE_SUCCESS + * @param rank is the ID of the ranks mailbox to be closed + * @param check is a flag indicating wether the mailbox has to be emptied before + * closing or not. This is required for a close-call as a result of a received + * last-mail. + * + * This function closes a mailbox of the given rank. If the check flag is set + * an iRCCE_mail_check()-call is performed. The close procedure has to be locked + * to be sure that no UE sends any mail while closing the mailbox. + */ +//------------------------------------------------------------------------------ +static int iRCCE_mailbox_close_one(int rank, int check) { + RCCE_acquire_lock( RCCE_IAM ); + + // check if it contains new mail + RC_cache_invalidate(); + if( check && iRCCE_mailbox_recv[rank]->sent ) { + iRCCE_mail_fetch(rank); + + } + + // close mailbox + iRCCE_MAIL_HEADER help_header = { 0, 0, 0, NULL, 0, RCCE_FLAG_UNSET, + RCCE_FLAG_SET, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} }; + + RC_cache_invalidate(); + iRCCE_memcpy_put( (void*)iRCCE_mailbox_recv[rank], + &help_header, RCCE_LINE_SIZE ); + + RCCE_release_lock( RCCE_IAM ); + + iRCCE_mailbox_status[rank] = iRCCE_MAILBOX_CLOSED; + + return iRCCE_SUCCESS; +} + + +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mailbox_close() +//------------------------------------------------------------------------------ +/** + * @brief routine to close one or all mailboxes + * @param rank is the ID of the UEs mailbox to be closed if iRCCE_MAILBOX_ALL + * is passed all mailboxes are closed by the calling UE + * @return iRCCE_SUCCESS + * + * This functions closed one or all mailboxes of the calling UE. This is done by + * calling iRCCE_mailbox_close_one for one or all mailboxes. + */ +//------------------------------------------------------------------------------ +int iRCCE_mailbox_close(int rank) { + if( rank == iRCCE_MAILBOX_ALL ) { + int i; + for( i=0; i Date: Wed, 13 Jul 2011 15:16:52 +0200 Subject: [PATCH 11/36] add new iRCC.h --- arch/x86/include/asm/iRCCE.h | 29 +++-- arch/x86/include/asm/iRCCE.h.old | 201 +++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 arch/x86/include/asm/iRCCE.h.old diff --git a/arch/x86/include/asm/iRCCE.h b/arch/x86/include/asm/iRCCE.h index 7c071d5b..60250eab 100644 --- a/arch/x86/include/asm/iRCCE.h +++ b/arch/x86/include/asm/iRCCE.h @@ -112,21 +112,30 @@ typedef struct _iRCCE_WAIT_LIST { #define iRCCE_MAIL_HEADER_PAYLOAD 13 +/** + * @struct _iRCCE_MAIL_HEADER + * @brief mail that can be send/received by iRCCE_mail_send/iRCCE_mail_recv + * + * _iRCCE_MAIL_HEADER has exactly the size of one cacheline (32 byte). Create an + * object by using the iRCCE_MAIL_HEADER identifier. + */ typedef struct _iRCCE_MAIL_HEADER { - int source; // UE that will send the header - size_t size; // size of the message which will be send/received - int tag; // tag indicating which kind of message we have - struct _iRCCE_MAIL_HEADER* next;// pointer for queue - could be replaced by list-object - char prio; // priority of the mail - iRCCE_SHORT_FLAG sent; // flag indicating that header is new - iRCCE_SHORT_FLAG closed; // flag indication that mailbox is closed - char payload[iRCCE_MAIL_HEADER_PAYLOAD]; // payload for small messages + /** @{ */ + int source; /**< UE that will send the header */ + size_t size; /**< size of the message which will be send/received */ + int tag; /**< tag indicating which kind of message we have */ + struct _iRCCE_MAIL_HEADER* next;/**< pointer for queue - could be replaced by list-object */ + char prio; /**< priority of the mail */ + iRCCE_SHORT_FLAG sent; /**< flag indicating that header is new */ + iRCCE_SHORT_FLAG closed; /**< flag indication that mailbox is closed */ + char payload[iRCCE_MAIL_HEADER_PAYLOAD]; /**< payload for small messages */ + /** @} */ } iRCCE_MAIL_HEADER; -typedef struct _iRCCE_MAIL_TRASH_BIN { +typedef struct _iRCCE_MAIL_HEADER_LIST { iRCCE_MAIL_HEADER* first; iRCCE_MAIL_HEADER* last; -} iRCCE_MAIL_TRASH_BIN; +} iRCCE_MAIL_HEADER_LIST; /////////////////////////////////////////////////////////////// // diff --git a/arch/x86/include/asm/iRCCE.h.old b/arch/x86/include/asm/iRCCE.h.old new file mode 100644 index 00000000..7c071d5b --- /dev/null +++ b/arch/x86/include/asm/iRCCE.h.old @@ -0,0 +1,201 @@ +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes, University of Bayreuth +// +// [2010-12-09] added functions for a convenient handling of multiple +// pending non-blocking requests +// by Jacek Galowicz, Chair for Operating Systems +// RWTH Aachen University +// +#ifndef IRCCE_H +#define IRCCE_H + +#include "RCCE.h" + +#define iRCCE_SUCCESS RCCE_SUCCESS +#define iRCCE_PENDING -1 +#define iRCCE_RESERVED -2 +#define iRCCE_NOT_ENQUEUED -3 + +#define iRCCE_ANY_SOURCE -1 + +#define iRCCE_PRIOS 5 +#define iRCCE_MAILBOX_EMPTY -2 +#define iRCCE_LAST_MAILS_NOT_RECV -3 +#define iRCCE_MAILBOX_ALL -4 +#define iRCCE_MAILBOX_OPEN 0 +#define iRCCE_MAILBOX_CLOSED 1 + +// iRCCE-mailbox-system tags +#define iRCCE_LAST_MAIL -1 +#define iRCCE_ANYLENGTH -2 +#define iRCCE_ANYLENGTH_PIGGYBACK -3 + + +typedef volatile char iRCCE_SHORT_FLAG; + +typedef struct _iRCCE_SEND_REQUEST { + char *privbuf; // source buffer in local private memory (send buffer) + t_vcharp combuf; // intermediate buffer in MPB + size_t chunk; // size of MPB available for this message (bytes) + RCCE_FLAG *ready; // flag indicating whether receiver is ready + RCCE_FLAG *sent; // flag indicating whether message has been sent by source + size_t size; // size of message (bytes) + int dest; // UE that will receive the message + + size_t wsize; // offset within send buffer when putting in "chunk" bytes + size_t remainder; // bytes remaining to be sent + size_t nbytes; // number of bytes to be sent in single RCCE_put call + char *bufptr; // running pointer inside privbuf for current location + + int label; // jump/goto label for the reentrance of the respective poll function + int finished; // flag that indicates whether the request has already been finished + + struct _iRCCE_SEND_REQUEST *next; +} iRCCE_SEND_REQUEST; + + +typedef struct _iRCCE_RECV_REQUEST { + char *privbuf; // source buffer in local private memory (send buffer) + t_vcharp combuf; // intermediate buffer in MPB + size_t chunk; // size of MPB available for this message (bytes) + RCCE_FLAG *ready; // flag indicating whether receiver is ready + RCCE_FLAG *sent; // flag indicating whether message has been sent by source + size_t size; // size of message (bytes) + int source; // UE that will send the message + + size_t wsize; // offset within send buffer when putting in "chunk" bytes + size_t remainder; // bytes remaining to be sent + size_t nbytes; // number of bytes to be sent in single RCCE_put call + char *bufptr; // running pointer inside privbuf for current location + + int label; // jump/goto label for the reentrance of the respective poll function + int finished; // flag that indicates whether the request has already been finished + int started; // flag that indicates whether message parts have already been received + + struct _iRCCE_RECV_REQUEST *next; +} iRCCE_RECV_REQUEST; + +#define iRCCE_WAIT_LIST_RECV_TYPE 0 +#define iRCCE_WAIT_LIST_SEND_TYPE 1 + +typedef struct _iRCCE_WAIT_LISTELEM { + int type; + struct _iRCCE_WAIT_LISTELEM * next; + void * req; +} iRCCE_WAIT_LISTELEM; + +typedef struct _iRCCE_WAIT_LIST { + iRCCE_WAIT_LISTELEM * first; + iRCCE_WAIT_LISTELEM * last; +} iRCCE_WAIT_LIST; + + +#define iRCCE_MAIL_HEADER_PAYLOAD 13 +typedef struct _iRCCE_MAIL_HEADER { + int source; // UE that will send the header + size_t size; // size of the message which will be send/received + int tag; // tag indicating which kind of message we have + struct _iRCCE_MAIL_HEADER* next;// pointer for queue - could be replaced by list-object + char prio; // priority of the mail + iRCCE_SHORT_FLAG sent; // flag indicating that header is new + iRCCE_SHORT_FLAG closed; // flag indication that mailbox is closed + char payload[iRCCE_MAIL_HEADER_PAYLOAD]; // payload for small messages +} iRCCE_MAIL_HEADER; + +typedef struct _iRCCE_MAIL_TRASH_BIN { + iRCCE_MAIL_HEADER* first; + iRCCE_MAIL_HEADER* last; +} iRCCE_MAIL_TRASH_BIN; + +/////////////////////////////////////////////////////////////// +// +// THE iRCCE API: +// +// Initialize/Finalize functions: +int iRCCE_init(void); +int iRCCE_finalize(void); +// +// Non-blocking send/recv functions: +int iRCCE_isend(char *, ssize_t, int, iRCCE_SEND_REQUEST *); +int iRCCE_isend_test(iRCCE_SEND_REQUEST *, int *); +int iRCCE_isend_wait(iRCCE_SEND_REQUEST *); +int iRCCE_isend_push(void); +int iRCCE_irecv(char *, size_t, int, iRCCE_RECV_REQUEST *); +int iRCCE_irecv_test(iRCCE_RECV_REQUEST *, int *); +int iRCCE_irecv_wait(iRCCE_RECV_REQUEST *); +int iRCCE_irecv_push(void); +// +// Blocking but pipelined send/recv functions: +int iRCCE_send(char *, size_t, int); +int iRCCE_recv(char *, size_t, int); +// +// SCC-customized put/get and memcpy functions: +int iRCCE_put(t_vcharp, t_vcharp, int, int); +int iRCCE_get(t_vcharp, t_vcharp, int, int); +void* iRCCE_memcpy_put(void*, const void*, size_t); +void* iRCCE_memcpy_get(void*, const void*, size_t); +// +// Wait/test-all/any functions: +void iRCCE_init_wait_list(iRCCE_WAIT_LIST*); +void iRCCE_add_to_wait_list(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST *, iRCCE_RECV_REQUEST *); +int iRCCE_test_all(iRCCE_WAIT_LIST*, int *); +int iRCCE_wait_all(iRCCE_WAIT_LIST*); +int iRCCE_test_any(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST **, iRCCE_RECV_REQUEST **); +int iRCCE_wait_any(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST **, iRCCE_RECV_REQUEST **); +// +// Cancel functions for yet not started non-blocking requests: +int iRCCE_isend_cancel(iRCCE_SEND_REQUEST *, int *); +int iRCCE_irecv_cancel(iRCCE_RECV_REQUEST *, int *); +// +// Blocking send/recv functions for mailbox system +int iRCCE_mail_send(size_t, int, char, char*, int); +int iRCCE_mail_recv(iRCCE_MAIL_HEADER**); +// +// functions to empty mailbox-queue and to check for last mails: +int iRCCE_mail_release(iRCCE_MAIL_HEADER**); +int iRCCE_last_mail_recv(void); +int iRCCE_mailbox_wait(void); +int iRCCE_mailbox_flush(void); +int iRCCE_mailbox_close(int); +void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER*); +// +/////////////////////////////////////////////////////////////// +// +// Just for for convenience: +#if 1 +#define RCCE_isend iRCCE_isend +#define RCCE_isend_test iRCCE_isend_test +#define RCCE_isend_wait iRCCE_isend_wait +#define RCCE_isend_push iRCCE_isend_push +#define RCCE_irecv iRCCE_irecv +#define RCCE_irecv_test iRCCE_irecv_test +#define RCCE_irecv_wait iRCCE_irecv_wait +#define RCCE_irecv_push iRCCE_irecv_push +#define RCCE_SEND_REQUEST iRCCE_SEND_REQUEST +#define RCCE_RECV_REQUEST iRCCE_RECV_REQUEST +#endif +/////////////////////////////////////////////////////////////// + +#endif + From 6e14d56de8ea30b9fcf9b23c8a9fcd74dd79c129 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Mon, 18 Jul 2011 01:14:18 -0700 Subject: [PATCH 12/36] mailbox does not support sendings to own mailbox --- arch/x86/include/asm/iRCCE.h | 1 + arch/x86/kernel/irq.c | 6 +- arch/x86/scc/iRCCE_mailbox.c | 15 ++-- arch/x86/scc/icc.c | 147 ++++++++++++++++++----------------- kernel/tests.c | 8 +- 5 files changed, 96 insertions(+), 81 deletions(-) diff --git a/arch/x86/include/asm/iRCCE.h b/arch/x86/include/asm/iRCCE.h index 60250eab..ed762ba2 100644 --- a/arch/x86/include/asm/iRCCE.h +++ b/arch/x86/include/asm/iRCCE.h @@ -45,6 +45,7 @@ #define iRCCE_MAILBOX_ALL -4 #define iRCCE_MAILBOX_OPEN 0 #define iRCCE_MAILBOX_CLOSED 1 +#define iRCCE_MAILBOX_INVALID -1 // iRCCE-mailbox-system tags #define iRCCE_LAST_MAIL -1 diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 413c2932..7132026b 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -233,11 +233,9 @@ void irq_handler(struct state *s) { /* This is a blank function pointer */ void (*handler) (struct state * s); - + // at first, we check our work queues -// if( s->int_no == 124 ) { - check_workqueues(); -// } + check_workqueues(); /* * Find out if we have a custom handler to run for this diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 045a731d..b01907d0 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -24,10 +24,6 @@ // limitations under the License. // // -#include "iRCCE_lib.h" -#include -#include -#include /** * @@ -60,7 +56,7 @@ static int iRCCE_mailbox_close_one(int rank, int check); void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { - printf( "\n" + kprintf( "\n" "-------------------------\n" "| RCK%d\n" "-------------------------\n" @@ -185,6 +181,7 @@ static int iRCCE_mailbox_check() { } } + return iRCCE_SUCCESS; } @@ -210,7 +207,7 @@ int iRCCE_mail_recv( int i; iRCCE_MAIL_HEADER* help_header; - + // if there is no mail, check for incoming if ( !iRCCE_mailbox_recv_queue[0].first ) { iRCCE_mailbox_check(); @@ -237,6 +234,7 @@ int iRCCE_mail_recv( // no mail queued *header = NULL; + return iRCCE_MAILBOX_EMPTY; } @@ -312,6 +310,11 @@ int iRCCE_mail_send( int dest // UE that will receive the header ) { + // check for an attempt to send in own mailbox + if( dest == RCCE_IAM ) { + return iRCCE_MAILBOX_INVALID; + } + // if dest mailbox is full, check for incoming mail RC_cache_invalidate(); while( iRCCE_mailbox_send[dest]->sent ) { diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index d3cd427d..f0c8b957 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -192,25 +192,40 @@ int icc_halt(void) uint32_t flags; uint32_t do_send = 1; - do { - // iRCCE is not thread save => disable interrupts - flags = irq_nested_disable(); + // iRCCE is not thread save => disable interrupts + flags = irq_nested_disable(); - if (do_send) { - do_send = (iRCCE_isend_push() == iRCCE_PENDING); - iRCCE_irecv_push(); - } - icc_mail_check(); + icc_mail_check(); - irq_nested_enable(flags); - } while(do_send); + irq_nested_enable(flags); HALT; return 0; } -#define ROUNDS 100000 +#define ROUNDS 10000 + +static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { + char* recv_buffer; + + switch( mail->tag ) { + case iRCCE_ANYLENGTH: + recv_buffer = (char*)kmalloc( mail->size ); + iRCCE_irecv(recv_buffer, mail->size, + mail->source, NULL ); + break; + + case PING_REQ: + iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); + break; + + default: + break; + } + +} + int icc_mail_ping( void ) { @@ -271,11 +286,9 @@ int icc_mail_ping( void ) return 0; } + #define CORE_A 0 // sender -#define CORE_B RCCE_num_ues()-1 // receiver -#define NOISE_ROUNDS 1000000 - - +#define CORE_B 1 // receiver int icc_mail_ping_irq( void ) { /* return if not core 0 */ @@ -285,34 +298,36 @@ int icc_mail_ping_irq( void ) uint64_t timer = 0; int rem_rank = CORE_B; int i; + int res; iRCCE_MAIL_HEADER* recv_header = NULL; kprintf( "Hello from mail_ping_irq ... \n" ); + kprintf( "my_rank = %d\n", my_ue ); kprintf( "rem_rank = %d\n", rem_rank ); // disable interrupts flags = irq_nested_disable(); for( i=0; itag == PING_RESP ) + /* send interrupt */ + icc_send_irq(rem_rank); + + /* wait for response */ + do { + res = iRCCE_mail_recv(&recv_header); + if( res == iRCCE_SUCCESS ) { + if( (recv_header->source == rem_rank) + && (recv_header->tag == PING_RESP) ) break; - } while( 1 ); + iRCCE_mail_release(&recv_header); + } + } while( 1 ); + + /* release mail */ + iRCCE_mail_release(&recv_header); - /* release mail */ - iRCCE_mail_release(&recv_header); - } - /* start timer in first round */ if( i == 0 ) timer = rdtsc(); @@ -321,40 +336,52 @@ int icc_mail_ping_irq( void ) /* stop timer */ timer = rdtsc() - timer; + kprintf( "timer = %ld\n", timer ); + kprintf( "mail_pingpong needs in average %d µsec (%d ticks)!\n", + timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); - kprintf( "timer = %d\n", timer ); - if( my_ue == 0 ) { - kprintf( "mail_pingpong needs in average %f µsec (%d ticks)!\n", - timer/(2.0*ROUNDS*533), timer/(2*ROUNDS) ); + + /* recv noise mails */ + while(1) { + iRCCE_mail_recv(&recv_header); + if(recv_header) iRCCE_mail_release(&recv_header); } irq_nested_enable(flags); - return 0; } int icc_mail_noise() { - int i, j; + int j, res; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; - + + kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); + // leave function if not participating - if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { + if( (my_ue == CORE_A) /*|| (my_ue == CORE_B)*/) { return -1; - } + } - for( i=0; itag ) { - case iRCCE_ANYLENGTH: - recv_buffer = (char*)kmalloc( header->size ); - iRCCE_irecv(recv_buffer, header->size, - header->source, NULL ); - iRCCE_mail_send(0, 2, 0, NULL, header->source); - break; - - case iRCCE_ANYLENGTH_PIGGYBACK: - iRCCE_mail_send(0, 2, 0, NULL, header->source); - break; - - case PING_REQ: - iRCCE_mail_send(0, PING_RESP, 0, NULL, header->source); - break; - } - - + while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { + icc_mail_check_tag(header); iRCCE_mail_release( &header ); } } diff --git a/kernel/tests.c b/kernel/tests.c index fc4f4d39..67b3904d 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -90,11 +90,16 @@ static int STDCALL foo(void* arg) int STDCALL mail_ping(void* arg) { //icc_mail_ping(); icc_mail_ping_irq(); - //icc_mail_noise(); // generate noise in the mesh icc_halt(); return 0; } + +int STDCALL mail_noise(void*arg) { + icc_mail_noise(); // generate noise in the mesh + + return 0; +} #endif static int STDCALL join_test(void* arg) @@ -133,6 +138,7 @@ int test_init(void) //create_kernel_task(NULL, producer, NULL); //create_kernel_task(NULL, consumer, NULL); create_kernel_task(NULL, mail_ping, NULL); + create_kernel_task(NULL, mail_noise, NULL); //create_user_task(NULL, "/bin/hello", argv); //create_user_task(NULL, "/bin/tests", argv); //create_user_task(NULL, "/bin/jacobi", argv); From f344f65769ba96ca4ccf27e4b4076177d1d48aed Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Sat, 23 Jul 2011 12:27:08 -0700 Subject: [PATCH 13/36] mailbox-system thread-safe now --- arch/x86/kernel/irq.c | 5 ++- arch/x86/kernel/timer.c | 2 +- arch/x86/scc/iRCCE_mailbox.c | 56 ++++++++++++++++++++++-- arch/x86/scc/icc.c | 83 ++++++++++++++++++++++++------------ kernel/main.c | 2 +- kernel/syscall.c | 2 +- kernel/tests.c | 8 ++-- 7 files changed, 118 insertions(+), 40 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 7132026b..49602774 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -235,8 +235,9 @@ void irq_handler(struct state *s) void (*handler) (struct state * s); // at first, we check our work queues - check_workqueues(); - + if( s->int_no == 124 ) { + check_workqueues(); + } /* * Find out if we have a custom handler to run for this * IRQ and then finally, run it diff --git a/arch/x86/kernel/timer.c b/arch/x86/kernel/timer.c index 73c83a51..995fb747 100644 --- a/arch/x86/kernel/timer.c +++ b/arch/x86/kernel/timer.c @@ -81,7 +81,7 @@ void timer_wait(unsigned int ticks) uint64_t eticks = timer_ticks + ticks; while (timer_ticks < eticks) { - check_workqueues(); + //check_workqueues(); // recheck break condition if (timer_ticks >= eticks) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index b01907d0..d2d946e1 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -39,6 +39,7 @@ #ifdef CONFIG_ROCKCREEK #include +#include // forward declaration static int iRCCE_mailbox_close_one(int rank, int check); @@ -90,7 +91,8 @@ void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) { static int iRCCE_mail_fetch( int rank // rank from whom to empty mailbox ) { - + /* interrupts should already be disabled! */ + iRCCE_MAIL_HEADER* header; // check for memory in garbage collection or allocate new @@ -160,6 +162,10 @@ static int iRCCE_mail_fetch( //------------------------------------------------------------------------------ static int iRCCE_mailbox_check() { int i,j; + uint32_t flags; + + /* disable interrupts */ + flags = irq_nested_disable(); for( j=1; jnext = NULL; *header = help_header; + + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_SUCCESS; + } } @@ -255,6 +273,11 @@ int iRCCE_mail_recv( int iRCCE_mail_release( iRCCE_MAIL_HEADER** header ) { + uint32_t flags; + + /* disable interrupts */ + flags = irq_nested_disable(); + // put header in garbage collection if( (iRCCE_mail_garbage.first == NULL) && (iRCCE_mail_garbage.last == NULL ) ) { @@ -270,6 +293,10 @@ int iRCCE_mail_release( // reset header *header = NULL; + + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_SUCCESS; } @@ -309,6 +336,7 @@ int iRCCE_mail_send( char* payload, // pointer to buffer for header payload int dest // UE that will receive the header ) { + uint32_t flags; // check for an attempt to send in own mailbox if( dest == RCCE_IAM ) { @@ -344,6 +372,9 @@ int iRCCE_mail_send( RC_cache_invalidate(); iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], (void*)&header, RCCE_LINE_SIZE ); + + /* disable interrupts */ + flags = irq_nested_disable(); // set senders flag RC_cache_invalidate(); @@ -351,6 +382,9 @@ int iRCCE_mail_send( *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); + /* enable interrupts */ + irq_nested_enable(flags); + RCCE_release_lock( dest ); return iRCCE_SUCCESS; @@ -417,17 +451,24 @@ int iRCCE_mailbox_wait(void) { //------------------------------------------------------------------------------ int iRCCE_mailbox_flush(void) { int i; + uint32_t flags; for( i=0; inext; kfree( erase_header, sizeof(iRCCE_MAIL_HEADER) ); erase_header = iRCCE_mailbox_recv_queue[i].first; } + + /* enable interrupts */ + irq_nested_enable(flags); } return iRCCE_SUCCESS; } @@ -449,11 +490,16 @@ int iRCCE_mailbox_flush(void) { */ //------------------------------------------------------------------------------ static int iRCCE_mailbox_close_one(int rank, int check) { + uint32_t flags; RCCE_acquire_lock( RCCE_IAM ); + /* disable interrupts */ + flags = irq_nested_disable(); + // check if it contains new mail RC_cache_invalidate(); if( check && iRCCE_mailbox_recv[rank]->sent ) { + iRCCE_mail_fetch(rank); } @@ -465,10 +511,14 @@ static int iRCCE_mailbox_close_one(int rank, int check) { RC_cache_invalidate(); iRCCE_memcpy_put( (void*)iRCCE_mailbox_recv[rank], &help_header, RCCE_LINE_SIZE ); + + iRCCE_mailbox_status[rank] = iRCCE_MAILBOX_CLOSED; + + /* enable interrupts */ + irq_nested_enable(flags); RCCE_release_lock( RCCE_IAM ); - iRCCE_mailbox_status[rank] = iRCCE_MAILBOX_CLOSED; return iRCCE_SUCCESS; } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index f0c8b957..58db46ab 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -189,22 +189,15 @@ int icc_send_irq(int ue) int icc_halt(void) { - uint32_t flags; - uint32_t do_send = 1; - - // iRCCE is not thread save => disable interrupts - flags = irq_nested_disable(); - icc_mail_check(); - - irq_nested_enable(flags); - HALT; return 0; } -#define ROUNDS 10000 +#define ROUNDS 300000 +#define CORE_A 1 // sender +#define CORE_B 9 // receiver static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { char* recv_buffer; @@ -231,36 +224,62 @@ int icc_mail_ping( void ) { uint32_t flags; uint64_t timer = 0; - int rem_rank = (my_ue+1)%2; int i; + int res; iRCCE_MAIL_HEADER* recv_header = NULL; + /* leave function if not participating in pingpong */ + if( (my_ue != CORE_A) && (my_ue != CORE_B) ) return -1; + + kprintf( "my_ue = %d\n", my_ue ); kprintf( "Hello from mail_ping ... \n" ); - + kprintf( "rounds = %d\n", ROUNDS ); + + + /* wait 10 sec */ + timer = rdtsc(); + while( (rdtsc() - timer) < 533e8 ) { + kprintf( "waiting ...\n" ); + } + + kprintf( "waited ..\n" ); + // disable interrupts flags = irq_nested_disable(); for( i=0; isource == CORE_B ) + break; + iRCCE_mail_release(&recv_header); + } + } while( 1 ); /* release mail */ iRCCE_mail_release(&recv_header); } + /* receivers part */ else { /* wait for request */ do { - iRCCE_mail_recv(&recv_header); + res = iRCCE_mail_recv(&recv_header); + if( res == iRCCE_SUCCESS ) { + if( recv_header->tag == PING_REQ ) + break; + iRCCE_mail_release(&recv_header); + } } while( (!recv_header)); /* send response */ - iRCCE_mail_send(0, PING_RESP, 0, NULL, rem_rank); + iRCCE_mail_send(0, PING_RESP, 0, NULL, recv_header->source); /* release mail */ iRCCE_mail_release(&recv_header); @@ -275,10 +294,10 @@ int icc_mail_ping( void ) timer = rdtsc() - timer; - kprintf( "timer = %d\n", timer ); - if( my_ue == 0 ) { - kprintf( "mail_pingpong needs in average %f µsec (%d ticks)!\n", - timer/(2.0*ROUNDS*533), timer/(2*ROUNDS) ); + if( my_ue == CORE_A ) { + kprintf( "timer = %ld\n", timer ); + kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", + timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); } irq_nested_enable(flags); @@ -287,8 +306,6 @@ int icc_mail_ping( void ) } -#define CORE_A 0 // sender -#define CORE_B 1 // receiver int icc_mail_ping_irq( void ) { /* return if not core 0 */ @@ -304,6 +321,7 @@ int icc_mail_ping_irq( void ) kprintf( "Hello from mail_ping_irq ... \n" ); kprintf( "my_rank = %d\n", my_ue ); kprintf( "rem_rank = %d\n", rem_rank ); + kprintf( "rounds = %d\n", ROUNDS ); // disable interrupts flags = irq_nested_disable(); @@ -337,7 +355,7 @@ int icc_mail_ping_irq( void ) timer = rdtsc() - timer; kprintf( "timer = %ld\n", timer ); - kprintf( "mail_pingpong needs in average %d µsec (%d ticks)!\n", + kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); @@ -357,22 +375,26 @@ int icc_mail_noise() { int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; - kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); - // leave function if not participating - if( (my_ue == CORE_A) /*|| (my_ue == CORE_B)*/) { + if( (my_ue == CORE_A)/* || (my_ue == CORE_B)*/ ) { return -1; } + kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); + for( ;; ) { /* send a mail to each UE */ for( j=0; j Date: Mon, 1 Aug 2011 13:55:29 +0200 Subject: [PATCH 14/36] update mailbox_system to newest version --- arch/x86/include/asm/iRCCE.h | 18 ++++++---- arch/x86/include/asm/iRCCE_lib.h | 1 + arch/x86/scc/iRCCE_admin.c | 6 ++-- arch/x86/scc/iRCCE_mailbox.c | 60 +++++++++++++++++++++++++------- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/arch/x86/include/asm/iRCCE.h b/arch/x86/include/asm/iRCCE.h index ed762ba2..7300065c 100644 --- a/arch/x86/include/asm/iRCCE.h +++ b/arch/x86/include/asm/iRCCE.h @@ -40,19 +40,22 @@ #define iRCCE_ANY_SOURCE -1 #define iRCCE_PRIOS 5 -#define iRCCE_MAILBOX_EMPTY -2 -#define iRCCE_LAST_MAILS_NOT_RECV -3 #define iRCCE_MAILBOX_ALL -4 -#define iRCCE_MAILBOX_OPEN 0 -#define iRCCE_MAILBOX_CLOSED 1 -#define iRCCE_MAILBOX_INVALID -1 + +// status codes +#define iRCCE_ERROR_ID RCCE_ERROR_ID +#define iRCCE_MAILBOX_OPEN -1 +#define iRCCE_MAILBOX_CLOSED -2 +#define iRCCE_MAILBOX_INVALID -3 +#define iRCCE_MAILBOX_EMPTY -4 +#define iRCCE_LAST_MAILS_NOT_RECV -5 // iRCCE-mailbox-system tags #define iRCCE_LAST_MAIL -1 +#define iRCCE_LAST_MAIL -1 #define iRCCE_ANYLENGTH -2 #define iRCCE_ANYLENGTH_PIGGYBACK -3 - typedef volatile char iRCCE_SHORT_FLAG; typedef struct _iRCCE_SEND_REQUEST { @@ -182,7 +185,8 @@ int iRCCE_irecv_cancel(iRCCE_RECV_REQUEST *, int *); int iRCCE_mail_send(size_t, int, char, char*, int); int iRCCE_mail_recv(iRCCE_MAIL_HEADER**); // -// functions to empty mailbox-queue and to check for last mails: +// functions to empty mailbox-queue and to check for mails: +int iRCCE_mail_check(int); int iRCCE_mail_release(iRCCE_MAIL_HEADER**); int iRCCE_last_mail_recv(void); int iRCCE_mailbox_wait(void); diff --git a/arch/x86/include/asm/iRCCE_lib.h b/arch/x86/include/asm/iRCCE_lib.h index 70df2596..ff020884 100644 --- a/arch/x86/include/asm/iRCCE_lib.h +++ b/arch/x86/include/asm/iRCCE_lib.h @@ -46,6 +46,7 @@ extern iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; // garbage collection for mailbox extern iRCCE_MAIL_HEADER_LIST iRCCE_mail_garbage; + #ifdef _OPENMP #pragma omp threadprivate (iRCCE_isend_queue, iRCCE_irecv_queue) #endif diff --git a/arch/x86/scc/iRCCE_admin.c b/arch/x86/scc/iRCCE_admin.c index cd119167..9e59d2fb 100644 --- a/arch/x86/scc/iRCCE_admin.c +++ b/arch/x86/scc/iRCCE_admin.c @@ -66,8 +66,6 @@ iRCCE_SHORT_FLAG iRCCE_last_mail[RCCE_MAXNP]; // field to store open/closed status of mailboxes iRCCE_SHORT_FLAG iRCCE_mailbox_status[RCCE_MAXNP]; - - //-------------------------------------------------------------------------------------- // FUNCTION: iRCCE_init //-------------------------------------------------------------------------------------- @@ -105,6 +103,7 @@ int iRCCE_init(void) { iRCCE_mailbox_recv_queue[i].first = NULL; iRCCE_mailbox_recv_queue[i].last = NULL; } + // allocate space in MPB for mailbox and set senders mailbox-pointer for( i=0; inext; @@ -136,6 +137,7 @@ int iRCCE_finalize(void) { } iRCCE_mail_garbage.first = iRCCE_mail_garbage.last = NULL; + return iRCCE_SUCCESS; } #endif diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index b01907d0..3803238b 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -41,8 +41,7 @@ #include // forward declaration -static int iRCCE_mailbox_close_one(int rank, int check); - +int iRCCE_mailbox_close_one(int rank, int check); //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_print_header @@ -110,7 +109,7 @@ static int iRCCE_mail_fetch( // copy header to allocated memory - RC_cache_invalidate(); + //RC_cache_invalidate(); iRCCE_memcpy_get( (void*)header, (void*)iRCCE_mailbox_recv[rank], RCCE_LINE_SIZE ); @@ -158,14 +157,15 @@ static int iRCCE_mail_fetch( * write out the data. */ //------------------------------------------------------------------------------ +iRCCE_MAIL_HEADER dummy_header = {0, 0, 0, NULL, 0, 0, 0, {[0 ... iRCCE_MAIL_HEADER_PAYLOAD-1] = 0} }; + + static int iRCCE_mailbox_check() { int i,j; for( j=1; jsent = RCCE_FLAG_UNSET; - *(int *)RCCE_fool_write_combine_buffer = 1; + *(iRCCE_mailbox_recv[i]) = dummy_header; } } } - return iRCCE_SUCCESS; } +//------------------------------------------------------------------------------ +// FUNCTION: iRCCE_mail_check +//------------------------------------------------------------------------------ +/** + * @brief routine to check one specific mailbox + * @param sender is the core ID from which the mailbox is checked + * + * This function may be called by the user application to check one specific + * mailbox. It is recommended to use it in combination with an inter core + * interrupt. + * + */ +//------------------------------------------------------------------------------ +int iRCCE_mail_check(int sender) { + // verify sender's ID + if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) { + return iRCCE_ERROR_ID; + } + + // check if mailbox is open + if( iRCCE_mailbox_status[sender] == iRCCE_MAILBOX_CLOSED ) { + return iRCCE_MAILBOX_CLOSED; + } + + RC_cache_invalidate(); + if( iRCCE_mailbox_recv[sender]->sent ) { + iRCCE_mail_fetch(sender); + + // reset senders flag + RC_cache_invalidate(); + *(iRCCE_mailbox_recv[sender]) = dummy_header; + + return iRCCE_SUCCESS; + } + else { + return iRCCE_MAILBOX_EMPTY; + } +} + //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mail_recv //------------------------------------------------------------------------------ @@ -309,8 +346,7 @@ int iRCCE_mail_send( char* payload, // pointer to buffer for header payload int dest // UE that will receive the header ) { - - // check for an attempt to send in own mailbox + // check for an attempt to send to own mailbox if( dest == RCCE_IAM ) { return iRCCE_MAILBOX_INVALID; } @@ -444,8 +480,8 @@ int iRCCE_mailbox_flush(void) { * last-mail. * * This function closes a mailbox of the given rank. If the check flag is set - * an iRCCE_mail_check()-call is performed. The close procedure has to be locked - * to be sure that no UE sends any mail while closing the mailbox. + * an iRCCE_mailbox_check()-call is performed. The close procedure has to be + * locked to be sure that no UE sends any mail while closing the mailbox. */ //------------------------------------------------------------------------------ static int iRCCE_mailbox_close_one(int rank, int check) { From 3275d8cc1781ba3256fe62ef9e2c5e51f0bab570 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Mon, 1 Aug 2011 14:24:46 -0700 Subject: [PATCH 15/36] still testing remote interrupt --- arch/x86/include/asm/icc.h | 1 + arch/x86/kernel/irq.c | 13 ++++++++- arch/x86/kernel/timer.c | 2 +- arch/x86/scc/iRCCE_mailbox.c | 9 +++--- arch/x86/scc/icc.c | 56 +++++++++++++++++++++++------------- include/metalsvm/tasks.h | 4 +-- kernel/tests.c | 7 +++-- 7 files changed, 61 insertions(+), 31 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index 56e29ca2..b778e2f0 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -56,6 +56,7 @@ int icc_halt(void); int icc_send_irq(int ue); void icc_mail_check(void); int icc_mail_ping(void); +int icc_irq_ping(void); int icc_mail_ping_irq(void); int icc_mail_noise(void); diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 49602774..47d4d028 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -34,6 +34,7 @@ #include #include +#define FPGA_BASE 0xf9000000 /* * These are our own ISRs that point to our special IRQ handler * instead of the regular 'fault_handler' function @@ -236,8 +237,18 @@ void irq_handler(struct state *s) // at first, we check our work queues if( s->int_no == 124 ) { - check_workqueues(); + kprintf( "hello from rem_interrupt\n" ); } + int status_reg1 = FPGA_BASE + 0x0D008; + int cont1 = *((int*)status_reg1); + int comp = 1; + int core_num = -1; + + if( cont1 & comp ) { + kprintf( "Interrupt from core 1\n" ); + } + check_workqueues(); +// } /* * Find out if we have a custom handler to run for this * IRQ and then finally, run it diff --git a/arch/x86/kernel/timer.c b/arch/x86/kernel/timer.c index 995fb747..71af6a5e 100644 --- a/arch/x86/kernel/timer.c +++ b/arch/x86/kernel/timer.c @@ -81,7 +81,7 @@ void timer_wait(unsigned int ticks) uint64_t eticks = timer_ticks + ticks; while (timer_ticks < eticks) { - //check_workqueues(); +// check_workqueues(); // recheck break condition if (timer_ticks >= eticks) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index d2d946e1..dbf58f78 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -350,6 +350,9 @@ int iRCCE_mail_send( RC_cache_invalidate(); } + /* disable interrupts */ + flags = irq_nested_disable(); + // check if mailbox is closed RCCE_acquire_lock( dest ); RC_cache_invalidate(); @@ -373,19 +376,17 @@ int iRCCE_mail_send( iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], (void*)&header, RCCE_LINE_SIZE ); - /* disable interrupts */ - flags = irq_nested_disable(); - // set senders flag RC_cache_invalidate(); iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); + RCCE_release_lock( dest ); + /* enable interrupts */ irq_nested_enable(flags); - RCCE_release_lock( dest ); return iRCCE_SUCCESS; } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 58db46ab..71239ec9 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -195,9 +195,33 @@ int icc_halt(void) return 0; } -#define ROUNDS 300000 +#define ROUNDS 100000 #define CORE_A 1 // sender -#define CORE_B 9 // receiver +#define CORE_B 4 // receiver + + +int icc_send_spec_irq(int core_num) { + int addr = FPGA_BASE+0x0D608; + int tmp; + tmp=ReadConfigReg(addr); + + kprintf( "tmp = %x\n", tmp ); + tmp |= 1; + kprintf( "tmp = %x\n", tmp ); + SetConfigReg(addr, tmp); + return 0; +} + + +int icc_irq_ping() +{ + if( my_ue != 0 ) return -1; + + icc_send_spec_irq(1); + icc_send_irq(1); + kprintf( "sending irq to 1!\n"); + return 0; +} static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { char* recv_buffer; @@ -236,14 +260,6 @@ int icc_mail_ping( void ) kprintf( "rounds = %d\n", ROUNDS ); - /* wait 10 sec */ - timer = rdtsc(); - while( (rdtsc() - timer) < 533e8 ) { - kprintf( "waiting ...\n" ); - } - - kprintf( "waited ..\n" ); - // disable interrupts flags = irq_nested_disable(); @@ -276,7 +292,7 @@ int icc_mail_ping( void ) break; iRCCE_mail_release(&recv_header); } - } while( (!recv_header)); + } while( 1 ); /* send response */ iRCCE_mail_send(0, PING_RESP, 0, NULL, recv_header->source); @@ -330,6 +346,9 @@ int icc_mail_ping_irq( void ) iRCCE_mail_send(0, PING_REQ, 0, NULL, rem_rank); /* send interrupt */ + NOP8; + NOP8; + NOP8; icc_send_irq(rem_rank); /* wait for response */ @@ -376,7 +395,7 @@ int icc_mail_noise() { iRCCE_MAIL_HEADER* recv_mail = NULL; // leave function if not participating - if( (my_ue == CORE_A)/* || (my_ue == CORE_B)*/ ) { + if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { return -1; } @@ -387,14 +406,14 @@ int icc_mail_noise() { for( j=0; j Date: Tue, 9 Aug 2011 14:54:30 -0700 Subject: [PATCH 16/36] testing remote interrupts --- arch/x86/kernel/irq.c | 12 ++++---- arch/x86/scc/iRCCE_mailbox.c | 2 +- arch/x86/scc/icc.c | 54 ++++++++++++++++++++++++++++++------ 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 47d4d028..81804ca6 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -238,17 +238,17 @@ void irq_handler(struct state *s) // at first, we check our work queues if( s->int_no == 124 ) { kprintf( "hello from rem_interrupt\n" ); + check_workqueues(); } - int status_reg1 = FPGA_BASE + 0x0D008; - int cont1 = *((int*)status_reg1); - int comp = 1; + + int status_reg1 = FPGA_BASE + 0xD008; + int comp = (1<<5); int core_num = -1; - if( cont1 & comp ) { + if( *((volatile int*)status_reg1) & comp ) { kprintf( "Interrupt from core 1\n" ); } - check_workqueues(); -// } + /* * Find out if we have a custom handler to run for this * IRQ and then finally, run it diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 24a274a2..2e16b4c0 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -42,7 +42,7 @@ #include // forward declaration -int iRCCE_mailbox_close_one(int rank, int check); +static int iRCCE_mailbox_close_one(int rank, int check); //------------------------------------------------------------------------------ // FUNCTION: iRCCE_mailbox_print_header diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 71239ec9..57194ce4 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -26,6 +26,14 @@ #include #include +#define IRQ_STATUS 0xD000 +#define IRQ_MASK 0xD200 +#define IRQ_RESET 0xD400 +#define IRQ_REQUEST 0xD600 +#define IRQ_CONFIG 0xD800 + + + bootinfo_t* bootinfo = (bootinfo_t*) SCC_BOOTINFO; static int num_ues, my_ue; @@ -162,6 +170,29 @@ int icc_init(void) // set interrupt handler (INTR/LINT0) irq_install_handler(124, intr_handler); + + // unmask interrupts + int irq_mask = FPGA_BASE + IRQ_MASK + my_ue*8; + *(int*)irq_mask &= 0; + irq_mask += 4; + *(int*)irq_mask &= 0; + + // reset interrupt reg + int irq_reset = FPGA_BASE + IRQ_RESET + my_ue*8; + *(int*)irq_reset &= 0; + irq_reset += 4; + *(int*)irq_reset &= 0; + + kprintf( "irq_request = %x\n", *(int*)(FPGA_BASE + IRQ_REQUEST + my_ue*8)); + //int irq_request = FPGA_BASE + IRQ_REQUEST + my_ue*8; + //*(int*) irq_request = 1; + + kprintf( "irq_request = %x\n", *(int*)(FPGA_BASE + IRQ_REQUEST + my_ue*8)); + + kprintf( "irq_mask = %x\n", *(int*)(FPGA_BASE + IRQ_MASK + my_ue*8) ); + kprintf( "LUT = %x\n", *(int*)(FPGA_BASE + IRQ_CONFIG + my_ue*4) ); + kprintf( "reset_reg = %x\n", *(int*)(FPGA_BASE + IRQ_RESET + my_ue*8) ); + kprintf( "status_reg = %x\n", *(int*)(FPGA_BASE + IRQ_STATUS + my_ue*8) ); kputs("Now, the SCC is initialized!\n"); return 0; @@ -201,14 +232,22 @@ int icc_halt(void) int icc_send_spec_irq(int core_num) { - int addr = FPGA_BASE+0x0D608; - int tmp; - tmp=ReadConfigReg(addr); + unsigned int addr = FPGA_BASE+IRQ_REQUEST+8*my_ue; + + // determine bit position and set according bit + if( my_ue < 32 ) { + unsigned int tmp; + kprintf( "*addr = %x\n", addr ) ; + tmp = ReadConfigReg(addr); + tmp |= 2; + kprintf( "tmp = %x\n", tmp ); + SetConfigReg(addr, tmp); + kprintf( "request_reg_addr = %x\n", addr ); + kprintf( "request_reg = %x\n", *(unsigned int*)addr ); + } + else { + } - kprintf( "tmp = %x\n", tmp ); - tmp |= 1; - kprintf( "tmp = %x\n", tmp ); - SetConfigReg(addr, tmp); return 0; } @@ -218,7 +257,6 @@ int icc_irq_ping() if( my_ue != 0 ) return -1; icc_send_spec_irq(1); - icc_send_irq(1); kprintf( "sending irq to 1!\n"); return 0; } From 4b03e27a37f268e4dfc3a2237121a3068aeb3c18 Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Wed, 10 Aug 2011 00:21:21 -0700 Subject: [PATCH 17/36] testing global interrupt controller --- arch/x86/kernel/irq.c | 29 +++++++++++++++++-------- arch/x86/scc/icc.c | 49 +++++++++++++++++++++---------------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 81804ca6..e75b2a75 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -33,8 +33,17 @@ #include #include #include +#include #define FPGA_BASE 0xf9000000 +#define IRQ_STATUS 0xD000 +#define IRQ_MASK 0xD200 +#define IRQ_RESET 0xD400 +#define IRQ_REQUEST 0xD600 +#define IRQ_CONFIG 0xD800 + +static int my_ue; + /* * These are our own ISRs that point to our special IRQ handler * instead of the regular 'fault_handler' function @@ -211,6 +220,8 @@ int irq_init(void) isrs_install(); irq_install(); + my_ue = RCCE_ue(); + return 0; } @@ -234,21 +245,21 @@ void irq_handler(struct state *s) { /* This is a blank function pointer */ void (*handler) (struct state * s); - + + if( s->int_no != 123 ) kprintf( "irq_num = %d\n", s->int_no ); + // at first, we check our work queues if( s->int_no == 124 ) { kprintf( "hello from rem_interrupt\n" ); check_workqueues(); + + /* determine interrupt source */ + volatile int* irq_status = (volatile int*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + kprintf( "irq_status_1: %x\n", *irq_status ); + irq_status += 4; + kprintf( "irq_status_2: %x\n", *irq_status ); } - int status_reg1 = FPGA_BASE + 0xD008; - int comp = (1<<5); - int core_num = -1; - - if( *((volatile int*)status_reg1) & comp ) { - kprintf( "Interrupt from core 1\n" ); - } - /* * Find out if we have a custom handler to run for this * IRQ and then finally, run it diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 57194ce4..578ca657 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -172,27 +172,26 @@ int icc_init(void) // unmask interrupts - int irq_mask = FPGA_BASE + IRQ_MASK + my_ue*8; - *(int*)irq_mask &= 0; + volatile int* irq_mask = (volatile int*)(FPGA_BASE + IRQ_MASK + my_ue*8); + *irq_mask &= 0; irq_mask += 4; - *(int*)irq_mask &= 0; + *irq_mask &= 0; // reset interrupt reg - int irq_reset = FPGA_BASE + IRQ_RESET + my_ue*8; - *(int*)irq_reset &= 0; + volatile int* irq_reset = (volatile int*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset = 0xffff; irq_reset += 4; - *(int*)irq_reset &= 0; + *irq_reset = 0xffff; - kprintf( "irq_request = %x\n", *(int*)(FPGA_BASE + IRQ_REQUEST + my_ue*8)); - //int irq_request = FPGA_BASE + IRQ_REQUEST + my_ue*8; - //*(int*) irq_request = 1; - - kprintf( "irq_request = %x\n", *(int*)(FPGA_BASE + IRQ_REQUEST + my_ue*8)); + // set remote interrupts to LINT 0 + volatile int* irq_config = (volatile int*)(FPGA_BASE + IRQ_CONFIG + my_ue*4); + *irq_config = 0; - kprintf( "irq_mask = %x\n", *(int*)(FPGA_BASE + IRQ_MASK + my_ue*8) ); - kprintf( "LUT = %x\n", *(int*)(FPGA_BASE + IRQ_CONFIG + my_ue*4) ); - kprintf( "reset_reg = %x\n", *(int*)(FPGA_BASE + IRQ_RESET + my_ue*8) ); - kprintf( "status_reg = %x\n", *(int*)(FPGA_BASE + IRQ_STATUS + my_ue*8) ); + volatile int* irq_status = (volatile int*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + + kprintf( "irq_mask = %x\n", *irq_mask ); + kprintf( "irq_config = %x\n", *irq_config ); + kprintf( "status_reg = %x\n", *irq_status ); kputs("Now, the SCC is initialized!\n"); return 0; @@ -232,21 +231,20 @@ int icc_halt(void) int icc_send_spec_irq(int core_num) { - unsigned int addr = FPGA_BASE+IRQ_REQUEST+8*my_ue; - + volatile int* addr = (volatile int*)(FPGA_BASE+IRQ_REQUEST+8*core_num); + unsigned int bit_pos; // determine bit position and set according bit if( my_ue < 32 ) { - unsigned int tmp; - kprintf( "*addr = %x\n", addr ) ; - tmp = ReadConfigReg(addr); - tmp |= 2; - kprintf( "tmp = %x\n", tmp ); - SetConfigReg(addr, tmp); - kprintf( "request_reg_addr = %x\n", addr ); - kprintf( "request_reg = %x\n", *(unsigned int*)addr ); + bit_pos = 1 << my_ue; } else { + bit_pos = 1 << (my_ue - 32); } + + bit_pos = 0x8001; + kprintf( "bit_pos = 0x%x\n", bit_pos ); + + *addr = bit_pos; return 0; } @@ -257,6 +255,7 @@ int icc_irq_ping() if( my_ue != 0 ) return -1; icc_send_spec_irq(1); + kprintf( "my_ue = %d\n", my_ue ); kprintf( "sending irq to 1!\n"); return 0; } From 6995bb29e31480ccf9c7497c93f1d5fb01d0a147 Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Wed, 10 Aug 2011 22:44:16 -0700 Subject: [PATCH 18/36] gic irqs seem to work now --- arch/x86/kernel/irq.c | 25 +++++++++++++++---------- arch/x86/scc/icc.c | 41 ++++++++++++++++------------------------- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index e75b2a75..2cb343e3 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -42,8 +42,6 @@ #define IRQ_REQUEST 0xD600 #define IRQ_CONFIG 0xD800 -static int my_ue; - /* * These are our own ISRs that point to our special IRQ handler * instead of the regular 'fault_handler' function @@ -220,8 +218,6 @@ int irq_init(void) isrs_install(); irq_install(); - my_ue = RCCE_ue(); - return 0; } @@ -246,18 +242,27 @@ void irq_handler(struct state *s) /* This is a blank function pointer */ void (*handler) (struct state * s); - if( s->int_no != 123 ) kprintf( "irq_num = %d\n", s->int_no ); - // at first, we check our work queues if( s->int_no == 124 ) { + int my_ue = RCCE_ue(); kprintf( "hello from rem_interrupt\n" ); check_workqueues(); /* determine interrupt source */ - volatile int* irq_status = (volatile int*)(FPGA_BASE + IRQ_STATUS + my_ue*8); - kprintf( "irq_status_1: %x\n", *irq_status ); - irq_status += 4; - kprintf( "irq_status_2: %x\n", *irq_status ); + int i ; + volatile uint64_t* irq_status = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + for( i=0; i<100; ++i) kprintf( "test" ); + irq_status = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + kprintf( "my_ue = %d\n", my_ue ); + kprintf( "irq_status_addr: %x\n", irq_status ); + kprintf( "irq_status: %x\n", *irq_status ); + + /* reset status register */ + volatile uint64_t* irq_reset = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset = ~(0); + + kprintf( "irq_status: %x\n", *irq_status ); + } /* diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 578ca657..a55c4cd8 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -97,7 +97,7 @@ int icc_init(void) { int i, z, tmp; uint64_t start, end, ticks, freq = 533; - uint32_t cr4, msg = 0; + uint32_t cr4; kputs("Initialize Rock Creek!\n"); @@ -172,16 +172,12 @@ int icc_init(void) // unmask interrupts - volatile int* irq_mask = (volatile int*)(FPGA_BASE + IRQ_MASK + my_ue*8); - *irq_mask &= 0; - irq_mask += 4; + volatile uint64_t* irq_mask = (volatile uint64_t*)(FPGA_BASE + IRQ_MASK + my_ue*8); *irq_mask &= 0; // reset interrupt reg - volatile int* irq_reset = (volatile int*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset = 0xffff; - irq_reset += 4; - *irq_reset = 0xffff; + volatile uint64_t* irq_reset = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset = ~(0); // set remote interrupts to LINT 0 volatile int* irq_config = (volatile int*)(FPGA_BASE + IRQ_CONFIG + my_ue*4); @@ -230,21 +226,17 @@ int icc_halt(void) #define CORE_B 4 // receiver -int icc_send_spec_irq(int core_num) { - volatile int* addr = (volatile int*)(FPGA_BASE+IRQ_REQUEST+8*core_num); - unsigned int bit_pos; - // determine bit position and set according bit - if( my_ue < 32 ) { - bit_pos = 1 << my_ue; - } - else { - bit_pos = 1 << (my_ue - 32); - } - - bit_pos = 0x8001; - kprintf( "bit_pos = 0x%x\n", bit_pos ); +int icc_send_gic_irq(int core_num) { + volatile uint64_t* irq_request = (volatile uint64_t*)(FPGA_BASE+IRQ_REQUEST+my_ue*8); + uint64_t bit_pos; - *addr = bit_pos; + kprintf( "sizeof(unint64_t) = %d\n", sizeof(uint64_t) ); + + // determine bit position and set according bit + bit_pos = (1 << core_num); + kprintf( "bit_pos = 0x%x\n", bit_pos ); + kprintf( "irq_req_addr = %x\n", irq_request ); + *irq_request = bit_pos; return 0; } @@ -252,9 +244,8 @@ int icc_send_spec_irq(int core_num) { int icc_irq_ping() { - if( my_ue != 0 ) return -1; - - icc_send_spec_irq(1); + if( my_ue == 2 ) return -1 ; + icc_send_gic_irq(2); kprintf( "my_ue = %d\n", my_ue ); kprintf( "sending irq to 1!\n"); return 0; From 67a85668fabd74ac929367de40ed3c819a6387e5 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Thu, 11 Aug 2011 09:02:49 -0700 Subject: [PATCH 19/36] testing remote interrupts --- arch/x86/kernel/irq.c | 25 ------------------------- arch/x86/scc/icc.c | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 2cb343e3..2bbc2855 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -33,14 +33,6 @@ #include #include #include -#include - -#define FPGA_BASE 0xf9000000 -#define IRQ_STATUS 0xD000 -#define IRQ_MASK 0xD200 -#define IRQ_RESET 0xD400 -#define IRQ_REQUEST 0xD600 -#define IRQ_CONFIG 0xD800 /* * These are our own ISRs that point to our special IRQ handler @@ -244,25 +236,8 @@ void irq_handler(struct state *s) // at first, we check our work queues if( s->int_no == 124 ) { - int my_ue = RCCE_ue(); kprintf( "hello from rem_interrupt\n" ); check_workqueues(); - - /* determine interrupt source */ - int i ; - volatile uint64_t* irq_status = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); - for( i=0; i<100; ++i) kprintf( "test" ); - irq_status = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); - kprintf( "my_ue = %d\n", my_ue ); - kprintf( "irq_status_addr: %x\n", irq_status ); - kprintf( "irq_status: %x\n", *irq_status ); - - /* reset status register */ - volatile uint64_t* irq_reset = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset = ~(0); - - kprintf( "irq_status: %x\n", *irq_status ); - } /* diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index a55c4cd8..50d60dcd 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -230,8 +230,6 @@ int icc_send_gic_irq(int core_num) { volatile uint64_t* irq_request = (volatile uint64_t*)(FPGA_BASE+IRQ_REQUEST+my_ue*8); uint64_t bit_pos; - kprintf( "sizeof(unint64_t) = %d\n", sizeof(uint64_t) ); - // determine bit position and set according bit bit_pos = (1 << core_num); kprintf( "bit_pos = 0x%x\n", bit_pos ); @@ -458,12 +456,44 @@ int icc_mail_noise() { void icc_mail_check(void) { - iRCCE_MAIL_HEADER* header = NULL; + //iRCCE_MAIL_HEADER* header = NULL; + int source, i; + volatile uint64_t* irq_status_reg = NULL; + volatile uint64_t* irq_reset_reg = NULL; + uint64_t irq_status = 0; + + /* print status information */ + kprintf( "my_ue = %d\n", my_ue ); + kprintf( "irq_status_addr: %x\n", irq_status ); + + for( i=0; i<2; ++i ) { + /* read status register */ + irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + irq_status = *irq_status_reg; + kprintf( "irq_status: %x\n", irq_status ); + + /* determine interrupt sources */ + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( irq_status & 0x1 ) { + kprintf( "check mail for source %d\n", source ); + } + } + } + + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = ~(0); + + kprintf( "irq_status: %x\n", *irq_status_reg ); + + +/* int res; while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); iRCCE_mail_release( &header ); } + */ } #endif From 1e636dee97cce51552f1977a0402c27846b452b3 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Thu, 11 Aug 2011 09:13:31 -0700 Subject: [PATCH 20/36] fix in mail_check --- arch/x86/scc/icc.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 50d60dcd..f11184e1 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -465,29 +465,29 @@ void icc_mail_check(void) /* print status information */ kprintf( "my_ue = %d\n", my_ue ); kprintf( "irq_status_addr: %x\n", irq_status ); - + for( i=0; i<2; ++i ) { /* read status register */ irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); irq_status = *irq_status_reg; kprintf( "irq_status: %x\n", irq_status ); - + /* determine interrupt sources */ for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { - if( irq_status & 0x1 ) { + if( (irq_status & 0x1) != 0 ) { kprintf( "check mail for source %d\n", source ); - } + } } + + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = ~(0); + + kprintf( "irq_status: %x\n", *irq_status_reg ); + + } - - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = ~(0); - - kprintf( "irq_status: %x\n", *irq_status_reg ); - - -/* + /* int res; while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); From 6892fa021a1a2773d0a6040e0bf37a78393592ee Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Sat, 13 Aug 2011 17:14:01 -0700 Subject: [PATCH 21/36] testing remote interrupts --- arch/x86/kernel/irq.c | 5 +- arch/x86/scc/iRCCE_mailbox.c | 21 +++++++- arch/x86/scc/icc.c | 96 ++++++++++++++++++------------------ include/metalsvm/tasks.h | 4 -- kernel/tests.c | 6 +-- 5 files changed, 72 insertions(+), 60 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 2bbc2855..f3fdfb8b 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -235,10 +235,9 @@ void irq_handler(struct state *s) void (*handler) (struct state * s); // at first, we check our work queues - if( s->int_no == 124 ) { - kprintf( "hello from rem_interrupt\n" ); +// if( s->int_no == 124 ) { check_workqueues(); - } +// } /* * Find out if we have a custom handler to run for this diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 2e16b4c0..64be600b 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -206,8 +206,20 @@ static int iRCCE_mailbox_check() { */ //------------------------------------------------------------------------------ int iRCCE_mail_check(int sender) { + uint32_t flags; + + /* disable interrupts */ + flags = irq_nested_disable(); + + + // verify sender's ID if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) { + if( sender == iRCCE_MAILBOX_ALL ) { + iRCCE_mailbox_check(); + return 0; + } + return iRCCE_ERROR_ID; } @@ -229,6 +241,10 @@ int iRCCE_mail_check(int sender) { else { return iRCCE_MAILBOX_EMPTY; } + + + /* enable interrupts */ + irq_nested_enable(flags); } //------------------------------------------------------------------------------ @@ -256,9 +272,10 @@ int iRCCE_mail_recv( iRCCE_MAIL_HEADER* help_header; // if there is no mail, check for incoming - if ( !iRCCE_mailbox_recv_queue[0].first ) { +/* if ( !iRCCE_mailbox_recv_queue[0].first ) { iRCCE_mailbox_check(); } +*/ // check priority queues for( i=0; isent ) { - iRCCE_mailbox_check(); +// iRCCE_mailbox_check(); RC_cache_invalidate(); } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index f11184e1..5a23d74c 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -221,7 +221,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 100000 +#define ROUNDS 1000 #define CORE_A 1 // sender #define CORE_B 4 // receiver @@ -232,8 +232,6 @@ int icc_send_gic_irq(int core_num) { // determine bit position and set according bit bit_pos = (1 << core_num); - kprintf( "bit_pos = 0x%x\n", bit_pos ); - kprintf( "irq_req_addr = %x\n", irq_request ); *irq_request = bit_pos; return 0; @@ -294,7 +292,7 @@ int icc_mail_ping( void ) if( my_ue == CORE_A ) { /* send ping request */ iRCCE_mail_send(0, PING_REQ, 0, NULL, CORE_B); - + /* wait for response */ do { res = iRCCE_mail_recv(&recv_header); @@ -364,6 +362,7 @@ int icc_mail_ping_irq( void ) kprintf( "my_rank = %d\n", my_ue ); kprintf( "rem_rank = %d\n", rem_rank ); kprintf( "rounds = %d\n", ROUNDS ); + // disable interrupts flags = irq_nested_disable(); @@ -372,28 +371,20 @@ int icc_mail_ping_irq( void ) iRCCE_mail_send(0, PING_REQ, 0, NULL, rem_rank); /* send interrupt */ - NOP8; - NOP8; - NOP8; - icc_send_irq(rem_rank); + icc_send_gic_irq(rem_rank); /* wait for response */ do { - res = iRCCE_mail_recv(&recv_header); - if( res == iRCCE_SUCCESS ) { - if( (recv_header->source == rem_rank) - && (recv_header->tag == PING_RESP) ) - break; - iRCCE_mail_release(&recv_header); - } - } while( 1 ); + res = iRCCE_mail_check(rem_rank); + } while( res != iRCCE_SUCCESS ); /* release mail */ + iRCCE_mail_recv(&recv_header); iRCCE_mail_release(&recv_header); /* start timer in first round */ if( i == 0 ) timer = rdtsc(); - + } /* stop timer */ @@ -404,19 +395,13 @@ int icc_mail_ping_irq( void ) timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); - /* recv noise mails */ - while(1) { - iRCCE_mail_recv(&recv_header); - if(recv_header) iRCCE_mail_release(&recv_header); - } - irq_nested_enable(flags); return 0; } int icc_mail_noise() { - int j, res; + int j, res, count = 0; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; @@ -428,6 +413,8 @@ int icc_mail_noise() { kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); for( ;; ) { + count ++; + kprintf( "%d ", count ); /* send a mail to each UE */ for( j=0; j>= 1, ++source ) { - if( (irq_status & 0x1) != 0 ) { - kprintf( "check mail for source %d\n", source ); - } + /* read status register */ + irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + irq_status = *irq_status_reg; + + // kprintf( "irq_status = %d\n", irq_status ); + + /* determine interrupt sources */ + irq_status >>= 6; // shift emac bits + + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( (irq_status & 0x1) != 0 ) { + // kprintf( "fetch mail from %d\n", source ); + // do { + res = iRCCE_mail_check(source); + // } while( res != iRCCE_SUCCESS ); } - - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = ~(0); - - kprintf( "irq_status: %x\n", *irq_status_reg ); + } - } - /* - int res; + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = ~(0); + + + /* empty mail queue */ while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); iRCCE_mail_release( &header ); } - */ + + /* enable interrupts */ + irq_nested_enable(flags); + } #endif diff --git a/include/metalsvm/tasks.h b/include/metalsvm/tasks.h index 289c7d8d..641c211f 100644 --- a/include/metalsvm/tasks.h +++ b/include/metalsvm/tasks.h @@ -140,13 +140,9 @@ int sys_execve(const char* fname, char** argv, char** env); static inline void check_workqueues(void) { -// uint32_t flags = irq_nested_disable(); - #ifdef CONFIG_ROCKCREEK icc_mail_check(); #endif - -// irq_nested_enable(flags); } #ifdef __cplusplus diff --git a/kernel/tests.c b/kernel/tests.c index 61e1c56d..f1b2e314 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -89,8 +89,8 @@ static int STDCALL foo(void* arg) #ifdef CONFIG_ROCKCREEK int STDCALL mail_ping(void* arg) { // icc_mail_ping(); - // icc_mail_ping_irq(); - icc_irq_ping(); + icc_mail_ping_irq(); + // icc_irq_ping(); icc_halt(); return 0; @@ -139,7 +139,7 @@ int test_init(void) //create_kernel_task(NULL, producer, NULL); //create_kernel_task(NULL, consumer, NULL); create_kernel_task(NULL, mail_ping, NULL); - //create_kernel_task(NULL, mail_noise, NULL); + create_kernel_task(NULL, mail_noise, NULL); //create_user_task(NULL, "/bin/hello", argv); //create_user_task(NULL, "/bin/tests", argv); //create_user_task(NULL, "/bin/jacobi", argv); From f7be10461b57f84b9d4bf7db6e4dc010376ea3d6 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Mon, 15 Aug 2011 17:26:32 -0700 Subject: [PATCH 22/36] bug fix --- arch/x86/scc/icc.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 5a23d74c..a6509262 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -401,25 +401,26 @@ int icc_mail_ping_irq( void ) int icc_mail_noise() { - int j, res, count = 0; + int j, res; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; // leave function if not participating - if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { + if( (my_ue == CORE_A) /*|| (my_ue == CORE_B)*/ ) { return -1; } kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); for( ;; ) { - count ++; - kprintf( "%d ", count ); /* send a mail to each UE */ for( j=0; j>= 6; // shift emac bits for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { if( (irq_status & 0x1) != 0 ) { - // kprintf( "fetch mail from %d\n", source ); - // do { res = iRCCE_mail_check(source); - // } while( res != iRCCE_SUCCESS ); } } From 7bf0cf184c31cbb5a11ad64235d82174d0ab4af9 Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Mon, 15 Aug 2011 09:56:46 -0700 Subject: [PATCH 23/36] only check irq status reg if irq_no = 124 --- arch/x86/include/asm/icc.h | 2 +- arch/x86/kernel/irq.c | 9 ++++-- arch/x86/scc/icc.c | 63 ++++++++++++++++++++++---------------- include/metalsvm/tasks.h | 8 ++++- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index b778e2f0..56991381 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -54,7 +54,7 @@ enum icc_mail_requests { int icc_init(void); int icc_halt(void); int icc_send_irq(int ue); -void icc_mail_check(void); +void icc_mail_check(int irq); int icc_mail_ping(void); int icc_irq_ping(void); int icc_mail_ping_irq(void); diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index f3fdfb8b..9918cfd8 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -234,10 +234,13 @@ void irq_handler(struct state *s) /* This is a blank function pointer */ void (*handler) (struct state * s); - // at first, we check our work queues -// if( s->int_no == 124 ) { + // evaluate only irq status register if int_no = 124 + if( s->int_no == 124 ) { + check_workqueues_rem_irq(); + } + else { check_workqueues(); -// } + } /* * Find out if we have a custom handler to run for this diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index a6509262..3fa97c7e 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -215,15 +215,15 @@ int icc_send_irq(int ue) int icc_halt(void) { - icc_mail_check(); + icc_mail_check(0); HALT; return 0; } -#define ROUNDS 1000 -#define CORE_A 1 // sender -#define CORE_B 4 // receiver +#define ROUNDS 100000 +#define CORE_A 0 // sender +#define CORE_B 10 // receiver int icc_send_gic_irq(int core_num) { @@ -348,7 +348,7 @@ int icc_mail_ping( void ) int icc_mail_ping_irq( void ) { - /* return if not core 0 */ + /* return if not core A */ if( my_ue != CORE_A ) return 0; uint32_t flags; @@ -379,7 +379,11 @@ int icc_mail_ping_irq( void ) } while( res != iRCCE_SUCCESS ); /* release mail */ - iRCCE_mail_recv(&recv_header); + do { + if( recv_header ) iRCCE_mail_release(&recv_header); + res = iRCCE_mail_recv(&recv_header); + } while( (recv_header->source != rem_rank) + && (recv_header->tag == PING_RESP) ); iRCCE_mail_release(&recv_header); /* start timer in first round */ @@ -415,8 +419,8 @@ int icc_mail_noise() { for( ;; ) { /* send a mail to each UE */ for( j=0; j>= 6; // shift emac bits + /* determine interrupt sources */ + irq_status >>= 6; // shift emac bits - for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { - if( (irq_status & 0x1) != 0 ) { - res = iRCCE_mail_check(source); + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( (irq_status & 0x1) != 0 ) { + res = iRCCE_mail_check(source); + } } - } - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = ~(0); - + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = ~(0); + } + else { + iRCCE_mail_check(iRCCE_MAILBOX_ALL); + } /* empty mail queue */ while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { diff --git a/include/metalsvm/tasks.h b/include/metalsvm/tasks.h index 641c211f..d03a57ec 100644 --- a/include/metalsvm/tasks.h +++ b/include/metalsvm/tasks.h @@ -141,10 +141,16 @@ int sys_execve(const char* fname, char** argv, char** env); static inline void check_workqueues(void) { #ifdef CONFIG_ROCKCREEK - icc_mail_check(); + icc_mail_check(0); #endif } +static inline void check_workqueues_rem_irq(void) +{ +#ifdef CONFIG_ROCKCREEK + icc_mail_check(1); +#endif +} #ifdef __cplusplus } #endif From 9b19fe3114d359e180413cd748ea1c61efa2514e Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Tue, 16 Aug 2011 14:11:26 -0700 Subject: [PATCH 24/36] change in icc_mail_ping_irq-test --- arch/x86/scc/icc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 3fa97c7e..2325896e 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -229,11 +229,11 @@ int icc_halt(void) int icc_send_gic_irq(int core_num) { volatile uint64_t* irq_request = (volatile uint64_t*)(FPGA_BASE+IRQ_REQUEST+my_ue*8); uint64_t bit_pos; - + // determine bit position and set according bit bit_pos = (1 << core_num); *irq_request = bit_pos; - + return 0; } @@ -365,7 +365,6 @@ int icc_mail_ping_irq( void ) // disable interrupts flags = irq_nested_disable(); - for( i=0; isource != rem_rank) && (recv_header->tag == PING_RESP) ); + iRCCE_mail_release(&recv_header); /* start timer in first round */ @@ -419,8 +419,8 @@ int icc_mail_noise() { for( ;; ) { /* send a mail to each UE */ for( j=0; j Date: Wed, 17 Aug 2011 00:42:47 -0700 Subject: [PATCH 25/36] low mail prio for noise --- arch/x86/scc/icc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 2325896e..8e3102f8 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -425,7 +425,7 @@ int icc_mail_noise() { /* recv mails */ iRCCE_mail_check(iRCCE_MAILBOX_ALL); - iRCCE_mail_send(0, 100, 0, NULL, j); + iRCCE_mail_send(0, 100, 1, NULL, j); res = iRCCE_mail_recv(&recv_mail); if( res == iRCCE_SUCCESS ) { icc_mail_check_tag(recv_mail); From c53ee583f839f007a6f2327bb0adb666f9c72f42 Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Wed, 17 Aug 2011 02:14:27 -0700 Subject: [PATCH 26/36] bug fix --- arch/x86/scc/icc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 8e3102f8..93f980a9 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -221,7 +221,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 100000 +#define ROUNDS 1000 #define CORE_A 0 // sender #define CORE_B 10 // receiver @@ -425,7 +425,7 @@ int icc_mail_noise() { /* recv mails */ iRCCE_mail_check(iRCCE_MAILBOX_ALL); - iRCCE_mail_send(0, 100, 1, NULL, j); + iRCCE_mail_send(0, 100, 0, NULL, j); res = iRCCE_mail_recv(&recv_mail); if( res == iRCCE_SUCCESS ) { icc_mail_check_tag(recv_mail); From c0f9747c67ddcbe0d69aab241da3d70bf5847b90 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Thu, 18 Aug 2011 01:29:55 -0700 Subject: [PATCH 27/36] kernel task mail_ping no STDCALL any more --- arch/x86/scc/icc.c | 10 ++-------- kernel/init.c | 4 ++++ kernel/main.c | 3 +++ kernel/tests.c | 10 ++++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index bd0e3f9b..69473ecc 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -220,12 +220,7 @@ int icc_send_irq(int ue) int icc_halt(void) { - uint32_t flags; - - // iRCCE is not thread save => disable interrupts - flags = irq_nested_disable(); icc_mail_check(0); - irq_nested_enable(flags); NOP1; //HALT; @@ -235,7 +230,7 @@ int icc_halt(void) #define ROUNDS 1000 #define CORE_A 0 // sender -#define CORE_B 10 // receiver +#define CORE_B 11 // receiver int icc_send_gic_irq(int core_num) { @@ -360,7 +355,6 @@ int icc_mail_ping_irq( void ) { /* return if not core A */ if( my_ue != CORE_A ) return 0; - uint32_t flags; uint64_t timer = 0; int rem_rank = CORE_B; @@ -503,7 +497,7 @@ void icc_mail_check(int irq) icc_mail_check_tag(header); iRCCE_mail_release( &header ); } - + /* enable interrupts */ irq_nested_enable(flags); diff --git a/kernel/init.c b/kernel/init.c index a29807ac..5bf0fbac 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -217,9 +217,13 @@ static void list_root(void) { int initd(void* arg) { + kprintf("i1"); network_init(); + kprintf("i2"); list_root(); + kprintf("i3"); test_init(); + kprintf("i4"); return 0; } diff --git a/kernel/main.c b/kernel/main.c index 3e270d83..9df81676 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -83,13 +83,16 @@ int main(void) kprintf("Kernel starts at %p and ends at %p\n", &kernel_start, &kernel_end); system_calibration(); + kprintf("main3"); kprintf("Processor frequency: %u MHz\n", get_cpu_frequency()); kprintf("Total memory: %u MBytes\n", atomic_int32_read(&total_pages)/((1024*1024)/PAGE_SIZE)); kprintf("Current allocated memory: %u KBytes\n", atomic_int32_read(&total_allocated_pages)*(PAGE_SIZE/1024)); kprintf("Current available memory: %u MBytes\n", atomic_int32_read(&total_available_pages)/((1024*1024)/PAGE_SIZE)); sleep(5); + create_kernel_task(NULL, initd, NULL); + per_core(current_task)->time_slices = 0; // reset the number of time slices reschedule(); diff --git a/kernel/tests.c b/kernel/tests.c index 0ac4275d..8e593602 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -100,16 +100,17 @@ static int foo(void* arg) } #ifdef CONFIG_ROCKCREEK -int STDCALL mail_ping(void* arg) { +static int mail_ping(void* arg) { // icc_mail_ping(); icc_mail_ping_irq(); // icc_irq_ping(); - icc_halt(); - +// icc_halt(); + kprintf( "goodbye from mail_ping_task" ); return 0; } -int STDCALL mail_noise(void*arg) { + +static int STDCALL mail_noise(void*arg) { icc_mail_noise(); // generate noise in the mesh return 0; } @@ -289,5 +290,6 @@ int test_init(void) //sleep(5); //create_user_task(NULL, "/bin/client", client_argv); + kprintf( "####" ); return 0; } From ad2d1e4a3d0b0228b19896f1bf737eac6b13e98b Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Thu, 18 Aug 2011 02:07:42 -0700 Subject: [PATCH 28/36] problems with iRCCE_memcpy_put --- arch/x86/scc/iRCCE_mailbox.c | 3 ++- arch/x86/scc/icc.c | 6 +++++- kernel/tests.c | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 5269a638..f269edd2 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -438,13 +438,14 @@ int iRCCE_mail_send( RC_cache_invalidate(); iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], (void*)&header, RCCE_LINE_SIZE ); + // set senders flag RC_cache_invalidate(); iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); - + RCCE_release_lock( dest ); /* enable interrupts */ diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 69473ecc..eba075d4 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -230,7 +230,7 @@ int icc_halt(void) #define ROUNDS 1000 #define CORE_A 0 // sender -#define CORE_B 11 // receiver +#define CORE_B 1 // receiver int icc_send_gic_irq(int core_num) { @@ -264,10 +264,12 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: + kprintf( "icc_mail_check_tag: ping_req\n" ); iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; default: + kprintf( "icc_mail_check_tag: uknown tag id %d\n", mail->tag ); break; } @@ -469,6 +471,8 @@ void icc_mail_check(int irq) flags = irq_nested_disable(); if( irq == 1 ) { + + kprintf( "hello from gic irq" ); /* read status register */ irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); irq_status = *irq_status_reg; diff --git a/kernel/tests.c b/kernel/tests.c index 8e593602..502e2b87 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -104,8 +104,7 @@ static int mail_ping(void* arg) { // icc_mail_ping(); icc_mail_ping_irq(); // icc_irq_ping(); -// icc_halt(); - kprintf( "goodbye from mail_ping_task" ); + icc_halt(); return 0; } From b3c786b280912e2878dfb8cbf70b844e5e618f79 Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Thu, 18 Aug 2011 02:23:10 -0700 Subject: [PATCH 29/36] degub print in iRCCE_mail_send --- arch/x86/scc/iRCCE_mailbox.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index f269edd2..ff4e0e3e 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -439,13 +439,21 @@ int iRCCE_mail_send( iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], (void*)&header, RCCE_LINE_SIZE ); + RC_cache_invalidate(); + iRCCE_MAIL_HEADER test_header; + iRCCE_memcpy_get( (void*)&test_header, (void*)iRCCE_mailbox_send[dest], RCCE_LINE_SIZE ); + iRCCE_mailbox_print_header(&test_header); + kprintf( "'######" ); // set senders flag RC_cache_invalidate(); iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); - + + iRCCE_memcpy_get( (void*)&test_header, (void*)iRCCE_mailbox_send[dest], RCCE_LINE_SIZE ); + iRCCE_mailbox_print_header(&test_header); + RCCE_release_lock( dest ); /* enable interrupts */ From 09931d918d04b86fd96870fff57b2d32ca6c6bc2 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Sat, 20 Aug 2011 07:09:37 -0700 Subject: [PATCH 30/36] remove debug printf --- arch/x86/scc/iRCCE_mailbox.c | 7 ------- arch/x86/scc/icc.c | 15 ++++++++------- kernel/tests.c | 5 ++--- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index ff4e0e3e..05656136 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -440,10 +440,6 @@ int iRCCE_mail_send( (void*)&header, RCCE_LINE_SIZE ); RC_cache_invalidate(); - iRCCE_MAIL_HEADER test_header; - iRCCE_memcpy_get( (void*)&test_header, (void*)iRCCE_mailbox_send[dest], RCCE_LINE_SIZE ); - iRCCE_mailbox_print_header(&test_header); - kprintf( "'######" ); // set senders flag RC_cache_invalidate(); @@ -451,9 +447,6 @@ int iRCCE_mail_send( *(int *)RCCE_fool_write_combine_buffer = 1; RC_cache_invalidate(); - iRCCE_memcpy_get( (void*)&test_header, (void*)iRCCE_mailbox_send[dest], RCCE_LINE_SIZE ); - iRCCE_mailbox_print_header(&test_header); - RCCE_release_lock( dest ); /* enable interrupts */ diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index eba075d4..85910a9a 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -228,7 +228,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 1000 +#define ROUNDS 10000 #define CORE_A 0 // sender #define CORE_B 1 // receiver @@ -264,7 +264,6 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: - kprintf( "icc_mail_check_tag: ping_req\n" ); iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; @@ -383,13 +382,16 @@ int icc_mail_ping_irq( void ) res = iRCCE_mail_check(rem_rank); } while( res != iRCCE_SUCCESS ); + recv_header = NULL; /* release mail */ do { if( recv_header ) iRCCE_mail_release(&recv_header); res = iRCCE_mail_recv(&recv_header); } while( (recv_header->source != rem_rank) && (recv_header->tag == PING_RESP) ); + + if( !(i%1000) ) kprintf( "%d ", i ); iRCCE_mail_release(&recv_header); /* start timer in first round */ @@ -400,7 +402,7 @@ int icc_mail_ping_irq( void ) /* stop timer */ timer = rdtsc() - timer; - kprintf( "timer = %ld\n", timer ); + kprintf( "timer = %d\n", timer ); kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); @@ -416,7 +418,7 @@ int icc_mail_noise() { iRCCE_MAIL_HEADER* recv_mail = NULL; // leave function if not participating - if( (my_ue == CORE_A) /*|| (my_ue == CORE_B)*/ ) { + if( (my_ue == CORE_A) /*|| (my_ue == CORE_B) */) { return -1; } @@ -425,7 +427,7 @@ int icc_mail_noise() { for( ;; ) { /* send a mail to each UE */ for( j=0; j Date: Sun, 21 Aug 2011 11:38:09 -0700 Subject: [PATCH 31/36] remove debug printfs --- arch/x86/scc/icc.c | 64 ++++++++++++++++++++++------------------------ kernel/init.c | 6 +---- kernel/main.c | 1 - 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 85910a9a..55372f2a 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -228,7 +228,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 10000 +#define ROUNDS 1000 #define CORE_A 0 // sender #define CORE_B 1 // receiver @@ -255,7 +255,7 @@ int icc_irq_ping() static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { char* recv_buffer; - + static int count = 0; switch( mail->tag ) { case iRCCE_ANYLENGTH: recv_buffer = (char*)kmalloc( mail->size ); @@ -264,6 +264,8 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: +// kprintf( "%d ", count ); + count++; iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; @@ -298,16 +300,19 @@ int icc_mail_ping( void ) if( my_ue == CORE_A ) { /* send ping request */ iRCCE_mail_send(0, PING_REQ, 0, NULL, CORE_B); - + /* wait for response */ do { + res = iRCCE_mail_check(CORE_B); + } while( res != iRCCE_SUCCESS ); + + recv_header = NULL; + /* release mail */ + do { + if( recv_header ) iRCCE_mail_release(&recv_header); res = iRCCE_mail_recv(&recv_header); - if( res == iRCCE_SUCCESS ) { - if( recv_header->source == CORE_B ) - break; - iRCCE_mail_release(&recv_header); - } - } while( 1 ); + } while( (recv_header->source != CORE_B) + && (recv_header->tag == PING_RESP) ); /* release mail */ iRCCE_mail_release(&recv_header); @@ -316,13 +321,11 @@ int icc_mail_ping( void ) else { /* wait for request */ do { - res = iRCCE_mail_recv(&recv_header); - if( res == iRCCE_SUCCESS ) { - if( recv_header->tag == PING_REQ ) - break; - iRCCE_mail_release(&recv_header); - } - } while( 1 ); + res = iRCCE_mail_check(CORE_A); + } while( res != iRCCE_SUCCESS ); + + res = iRCCE_mail_recv(&recv_header); + if( (res == iRCCE_SUCCESS) && (recv_header->tag = PING_REQ) ); /* send response */ iRCCE_mail_send(0, PING_RESP, 0, NULL, recv_header->source); @@ -358,28 +361,27 @@ int icc_mail_ping_irq( void ) if( my_ue != CORE_A ) return 0; uint32_t flags; uint64_t timer = 0; - int rem_rank = CORE_B; int i; int res; iRCCE_MAIL_HEADER* recv_header = NULL; kprintf( "Hello from mail_ping_irq ... \n" ); kprintf( "my_rank = %d\n", my_ue ); - kprintf( "rem_rank = %d\n", rem_rank ); + kprintf( "rem_rank = %d\n", CORE_B ); kprintf( "rounds = %d\n", ROUNDS ); // disable interrupts flags = irq_nested_disable(); for( i=0; isource != rem_rank) + } while( (recv_header->source != CORE_B) && (recv_header->tag == PING_RESP) ); @@ -418,7 +420,7 @@ int icc_mail_noise() { iRCCE_MAIL_HEADER* recv_mail = NULL; // leave function if not participating - if( (my_ue == CORE_A) /*|| (my_ue == CORE_B) */) { + if( (my_ue == CORE_A) /*(my_ue == CORE_B)*/ ) { return -1; } @@ -432,22 +434,19 @@ int icc_mail_noise() { /* recv mails */ iRCCE_mail_check(iRCCE_MAILBOX_ALL); - + + /* send noise mail */ iRCCE_mail_send(0, 100, 0, NULL, j); - res = iRCCE_mail_recv(&recv_mail); - if( res == iRCCE_SUCCESS ) { - icc_mail_check_tag(recv_mail); - iRCCE_mail_release(&recv_mail); - } - - } + /* read some mails */ for( res = iRCCE_mail_recv(&recv_mail), j=0; (res == iRCCE_SUCCESS) && (j<48); res = iRCCE_mail_recv(&recv_mail), ++j ) { icc_mail_check_tag(recv_mail); iRCCE_mail_release(&recv_mail); } + + kprintf( "x " ); } @@ -463,7 +462,7 @@ int icc_mail_noise() { void icc_mail_check(int irq) { iRCCE_MAIL_HEADER* header = NULL; - int source, i, res; + int source, res; volatile uint64_t* irq_status_reg = NULL; volatile uint64_t* irq_reset_reg = NULL; uint64_t irq_status = 0; @@ -473,7 +472,6 @@ void icc_mail_check(int irq) flags = irq_nested_disable(); if( irq == 1 ) { - kprintf( "got rem irq" ); /* read status register */ irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); irq_status = *irq_status_reg; diff --git a/kernel/init.c b/kernel/init.c index 5bf0fbac..2d706944 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -217,14 +217,10 @@ static void list_root(void) { int initd(void* arg) { - kprintf("i1"); network_init(); - kprintf("i2"); list_root(); - kprintf("i3"); test_init(); - kprintf("i4"); - + return 0; } diff --git a/kernel/main.c b/kernel/main.c index 9df81676..57d7cbf8 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -83,7 +83,6 @@ int main(void) kprintf("Kernel starts at %p and ends at %p\n", &kernel_start, &kernel_end); system_calibration(); - kprintf("main3"); kprintf("Processor frequency: %u MHz\n", get_cpu_frequency()); kprintf("Total memory: %u MBytes\n", atomic_int32_read(&total_pages)/((1024*1024)/PAGE_SIZE)); kprintf("Current allocated memory: %u KBytes\n", atomic_int32_read(&total_allocated_pages)*(PAGE_SIZE/1024)); From c5befddfdfb9f85cfea54dbbaa09539470ad30fc Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Tue, 23 Aug 2011 14:45:08 -0700 Subject: [PATCH 32/36] testing rem irq with mail_noise --- arch/x86/include/asm/icc.h | 1 + arch/x86/scc/iRCCE_mailbox.c | 4 ++ arch/x86/scc/icc.c | 109 +++++++++++++++++++---------------- kernel/tests.c | 2 +- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/arch/x86/include/asm/icc.h b/arch/x86/include/asm/icc.h index 6d01b622..70e0a0ec 100644 --- a/arch/x86/include/asm/icc.h +++ b/arch/x86/include/asm/icc.h @@ -42,6 +42,7 @@ extern bootinfo_t* bootinfo; enum icc_mail_requests { PING_REQ=1, PING_RESP, + NOISE, }; diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 05656136..3f7ec2a6 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -292,6 +292,7 @@ int iRCCE_mail_recv( flags = irq_nested_disable(); + help_header = iRCCE_mailbox_recv_queue[i].first; iRCCE_mailbox_recv_queue[i].first = @@ -316,6 +317,9 @@ int iRCCE_mail_recv( // no mail queued *header = NULL; + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_MAILBOX_EMPTY; } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 55372f2a..ee7493c2 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -228,9 +228,9 @@ int icc_halt(void) return 0; } -#define ROUNDS 1000 -#define CORE_A 0 // sender -#define CORE_B 1 // receiver +#define ROUNDS 10000 +#define CORE_A 0 // sender +#define CORE_B 4 // receiver int icc_send_gic_irq(int core_num) { @@ -256,6 +256,9 @@ int icc_irq_ping() static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { char* recv_buffer; static int count = 0; + + if( !mail ) return; + switch( mail->tag ) { case iRCCE_ANYLENGTH: recv_buffer = (char*)kmalloc( mail->size ); @@ -264,13 +267,15 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: -// kprintf( "%d ", count ); - count++; +// kprintf( "PING_REQ" ); +// count++; iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; - + + case NOISE: +// kprintf( "XXX " ); default: - kprintf( "icc_mail_check_tag: uknown tag id %d\n", mail->tag ); +// kprintf( "icc_mail_check_tag: uknown tag id %d\n", mail->tag ); break; } @@ -372,6 +377,7 @@ int icc_mail_ping_irq( void ) // disable interrupts flags = irq_nested_disable(); + for( i=0; isource != CORE_B) - && (recv_header->tag == PING_RESP) ); - - - if( !(i%1000) ) kprintf( "%d ", i ); + iRCCE_mail_recv(&recv_header); iRCCE_mail_release(&recv_header); - + /* start timer in first round */ if( i == 0 ) timer = rdtsc(); @@ -415,41 +412,50 @@ int icc_mail_ping_irq( void ) int icc_mail_noise() { - int j, res; + int i, j, res; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; + int flags; // leave function if not participating - if( (my_ue == CORE_A) /*(my_ue == CORE_B)*/ ) { + if( (my_ue == CORE_A) /* || (my_ue == CORE_B) */ || (my_ue > 25) ) { + kprintf( "mail_noise: leaving" ); return -1; } kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); - - for( ;; ) { + kprintf( "num_ues = %d\n", num_ranks ); + int count = 0; + for( i=0; i<100000 ;++i ) { /* send a mail to each UE */ for( j=0; j 25) ) continue; - - /* recv mails */ - iRCCE_mail_check(iRCCE_MAILBOX_ALL); - + /* send noise mail */ - iRCCE_mail_send(0, 100, 0, NULL, j); + iRCCE_mail_send(0, NOISE, 1, NULL, j); + + icc_send_gic_irq(j); + + NOP8; + NOP8; + NOP8; + + } + + /* read some mails + if( !(i%1000) ) { + iRCCE_mail_send(0, NOISE, 0, NULL, j); } - - /* read some mails */ - for( res = iRCCE_mail_recv(&recv_mail), j=0; (res == iRCCE_SUCCESS) && (j<48); res = iRCCE_mail_recv(&recv_mail), ++j ) { + while( iRCCE_mail_recv(&recv_mail) == iRCCE_SUCCESS ) { icc_mail_check_tag(recv_mail); iRCCE_mail_release(&recv_mail); } +*/ - kprintf( "x " ); - } - + return 0; } @@ -462,44 +468,49 @@ int icc_mail_noise() { void icc_mail_check(int irq) { iRCCE_MAIL_HEADER* header = NULL; - int source, res; + int i, source, res; volatile uint64_t* irq_status_reg = NULL; volatile uint64_t* irq_reset_reg = NULL; uint64_t irq_status = 0; + uint64_t irq_reset = 0; uint32_t flags; /* disable interrupts */ flags = irq_nested_disable(); if( irq == 1 ) { - /* read status register */ - irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); - irq_status = *irq_status_reg; + for( i=0; i<2; ++i ) { + /* read status register */ + irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8); + irq_status = irq_reset = *irq_status_reg; - /* determine interrupt sources */ - irq_status >>= 6; // shift emac bits + /* determine interrupt sources */ + irq_status >>= 6; // shift emac bits - for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { - if( (irq_status & 0x1) != 0 ) { - res = iRCCE_mail_check(source); + for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) { + if( (irq_status & 0x1) != 0 ) { + res = iRCCE_mail_check(source); + } } - } - /* reset status register */ - irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); - *irq_reset_reg = ~(0); + /* reset status register */ + irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8); + *irq_reset_reg = irq_reset; + } } else { iRCCE_mail_check(iRCCE_MAILBOX_ALL); } /* empty mail queue */ - while( (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS ) { + i=0; +// while( (i<10) && (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS) { + iRCCE_mail_recv(&header); icc_mail_check_tag(header); iRCCE_mail_release( &header ); - } +// } /* enable interrupts */ irq_nested_enable(flags); diff --git a/kernel/tests.c b/kernel/tests.c index 3f51718e..92297e67 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -104,7 +104,7 @@ static int mail_ping(void* arg) { // icc_mail_ping(); icc_mail_ping_irq(); // icc_irq_ping(); - icc_halt(); +// icc_halt(); return 0; } From 0d4ff65bcbdab4b20cb355bed7e85129b82e328d Mon Sep 17 00:00:00 2001 From: Generic account for RWTHAachen Students Date: Wed, 24 Aug 2011 04:25:10 -0700 Subject: [PATCH 33/36] fix irq reset --- arch/x86/kernel/irq.c | 2 +- arch/x86/scc/iRCCE_mailbox.c | 3 --- arch/x86/scc/icc.c | 46 ++++++++++++++++++------------------ kernel/tests.c | 2 +- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 6d98a769..8246a2e5 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -235,7 +235,7 @@ void irq_handler(struct state *s) void (*handler) (struct state * s); // evaluate only irq status register if int_no = 124 - if( s->int_no == 124 ) { + if( s->int_no == 124 ) { check_workqueues_rem_irq(); } else { diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 3f7ec2a6..67fd45d7 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -317,9 +317,6 @@ int iRCCE_mail_recv( // no mail queued *header = NULL; - /* enable interrupts */ - irq_nested_enable(flags); - return iRCCE_MAILBOX_EMPTY; } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index ee7493c2..e0d814cd 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -230,7 +230,7 @@ int icc_halt(void) #define ROUNDS 10000 #define CORE_A 0 // sender -#define CORE_B 4 // receiver +#define CORE_B 1 // receiver int icc_send_gic_irq(int core_num) { @@ -362,15 +362,16 @@ int icc_mail_ping( void ) int icc_mail_ping_irq( void ) { + kprintf( "Hello from mail_ping_irq ... \n" ); /* return if not core A */ if( my_ue != CORE_A ) return 0; + uint32_t flags; uint64_t timer = 0; int i; int res; iRCCE_MAIL_HEADER* recv_header = NULL; - kprintf( "Hello from mail_ping_irq ... \n" ); kprintf( "my_rank = %d\n", my_ue ); kprintf( "rem_rank = %d\n", CORE_B ); kprintf( "rounds = %d\n", ROUNDS ); @@ -411,48 +412,49 @@ int icc_mail_ping_irq( void ) } +#define _iRQ_NOISE_ 0 + int icc_mail_noise() { int i, j, res; int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; int flags; - +/* // leave function if not participating - if( (my_ue == CORE_A) /* || (my_ue == CORE_B) */ || (my_ue > 25) ) { + if( (my_ue == CORE_A) || (my_ue == CORE_B) || (my_ue > 25) ) { kprintf( "mail_noise: leaving" ); return -1; } - +*/ + if( !((my_ue == CORE_A) || (my_ue == CORE_B) ) ) { + kprintf( "mail_noise: leaving" ); + return -1; + } + kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); kprintf( "num_ues = %d\n", num_ranks ); int count = 0; for( i=0; i<100000 ;++i ) { /* send a mail to each UE */ for( j=0; j 25) ) + if( !((j == CORE_A) || (j == CORE_B)) ) continue; /* send noise mail */ iRCCE_mail_send(0, NOISE, 1, NULL, j); - +#ifdef _IRQ_NOISE_ icc_send_gic_irq(j); - - NOP8; - NOP8; - NOP8; - +#endif } - - /* read some mails - if( !(i%1000) ) { - iRCCE_mail_send(0, NOISE, 0, NULL, j); - } - + +#ifndef _IRQ_NOISE_ + iRCCE_mail_check(iRCCE_MAILBOX_ALL); +#endif + /* read some mails */ while( iRCCE_mail_recv(&recv_mail) == iRCCE_SUCCESS ) { icc_mail_check_tag(recv_mail); iRCCE_mail_release(&recv_mail); } -*/ } @@ -505,12 +507,10 @@ void icc_mail_check(int irq) } /* empty mail queue */ - i=0; -// while( (i<10) && (res = iRCCE_mail_recv(&header)) == iRCCE_SUCCESS) { - iRCCE_mail_recv(&header); + while( iRCCE_mail_recv(&header) == iRCCE_SUCCESS ) { icc_mail_check_tag(header); iRCCE_mail_release( &header ); -// } + } /* enable interrupts */ irq_nested_enable(flags); diff --git a/kernel/tests.c b/kernel/tests.c index 92297e67..370af074 100644 --- a/kernel/tests.c +++ b/kernel/tests.c @@ -280,7 +280,7 @@ int test_init(void) //create_kernel_task(NULL, producer, NULL); //create_kernel_task(NULL, consumer, NULL); create_kernel_task(NULL, mail_ping, NULL); - create_kernel_task(NULL, mail_noise, NULL); +// create_kernel_task(NULL, mail_noise, NULL); //create_user_task(NULL, "/bin/hello", argv); //create_user_task(NULL, "/bin/tests", argv); //create_user_task(NULL, "/bin/jacobi", argv); From b3ee8f1a07c4ac0f61a1306b80cf9495a61d13c8 Mon Sep 17 00:00:00 2001 From: Simon Pickartz Date: Thu, 25 Aug 2011 12:59:32 -0700 Subject: [PATCH 34/36] fix noise production --- arch/x86/mm/svm.c | 2 +- arch/x86/scc/iRCCE_mailbox.c | 6 +++--- arch/x86/scc/icc.c | 39 +++++++++++++++++++----------------- kernel/tests.c | 2 +- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/arch/x86/mm/svm.c b/arch/x86/mm/svm.c index 1dd2075f..eb31c08c 100644 --- a/arch/x86/mm/svm.c +++ b/arch/x86/mm/svm.c @@ -128,7 +128,7 @@ int svm_access_request(size_t addr) icc_send_irq(remote_rank); /* check for incoming messages */ - icc_mail_check(); + icc_mail_check(0); while (page_owner[pageid] != my_ue) { NOP4; diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 67fd45d7..76424287 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -410,12 +410,12 @@ int iRCCE_mail_send( // if dest mailbox is full, check for incoming mail RC_cache_invalidate(); while( iRCCE_mailbox_send[dest]->sent ) { -// iRCCE_mailbox_check(); + iRCCE_mailbox_check(); RC_cache_invalidate(); } /* disable interrupts */ - flags = irq_nested_disable(); +// flags = irq_nested_disable(); // check if mailbox is closed RCCE_acquire_lock( dest ); @@ -451,7 +451,7 @@ int iRCCE_mail_send( RCCE_release_lock( dest ); /* enable interrupts */ - irq_nested_enable(flags); +// irq_nested_enable(flags); return iRCCE_SUCCESS; } diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index 2c2ca46d..a3571905 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -240,7 +240,7 @@ int icc_halt(void) #define ROUNDS 10000 #define CORE_A 0 // sender -#define CORE_B 1 // receiver +#define CORE_B 4 // receiver int icc_send_gic_irq(int core_num) { @@ -268,7 +268,7 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { static int count = 0; if( !mail ) return; - + switch( mail->tag ) { case iRCCE_ANYLENGTH: recv_buffer = (char*)kmalloc( mail->size ); @@ -277,13 +277,12 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case PING_REQ: -// kprintf( "PING_REQ" ); // count++; iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source); break; case NOISE: -// kprintf( "XXX " ); + kprintf( "XXX " ); default: // kprintf( "icc_mail_check_tag: uknown tag id %d\n", mail->tag ); break; @@ -429,43 +428,45 @@ int icc_mail_noise() { int num_ranks = RCCE_num_ues(); iRCCE_MAIL_HEADER* recv_mail = NULL; int flags; -/* + // leave function if not participating - if( (my_ue == CORE_A) || (my_ue == CORE_B) || (my_ue > 25) ) { + if( (my_ue == CORE_A) || (my_ue == CORE_B) ) { kprintf( "mail_noise: leaving" ); return -1; } -*/ - if( !((my_ue == CORE_A) || (my_ue == CORE_B) ) ) { - kprintf( "mail_noise: leaving" ); - return -1; - } + kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); kprintf( "num_ues = %d\n", num_ranks ); int count = 0; - for( i=0; i<100000 ;++i ) { + for( ;; ) { + kprintf( "SEND NOISE - " ); /* send a mail to each UE */ for( j=0; j Date: Sat, 27 Aug 2011 02:31:30 -0700 Subject: [PATCH 35/36] little changes --- arch/x86/scc/iRCCE_mailbox.c | 9 ++++----- arch/x86/scc/icc.c | 21 ++++++++++----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 76424287..4fb439fa 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -218,7 +218,6 @@ int iRCCE_mail_check(int sender) { /* disable interrupts */ flags = irq_nested_disable(); - // verify sender's ID @@ -227,7 +226,10 @@ int iRCCE_mail_check(int sender) { iRCCE_mailbox_check(); return 0; } - + + /* enable interrupts */ + irq_nested_enable(flags); + return iRCCE_ERROR_ID; } @@ -250,7 +252,6 @@ int iRCCE_mail_check(int sender) { return iRCCE_MAILBOX_EMPTY; } - /* enable interrupts */ irq_nested_enable(flags); } @@ -291,8 +292,6 @@ int iRCCE_mail_recv( /* disable interrupts */ flags = irq_nested_disable(); - - help_header = iRCCE_mailbox_recv_queue[i].first; iRCCE_mailbox_recv_queue[i].first = diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c index a3571905..c42c4fc4 100644 --- a/arch/x86/scc/icc.c +++ b/arch/x86/scc/icc.c @@ -238,7 +238,7 @@ int icc_halt(void) return 0; } -#define ROUNDS 10000 +#define ROUNDS 100000 #define CORE_A 0 // sender #define CORE_B 4 // receiver @@ -282,7 +282,7 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) { break; case NOISE: - kprintf( "XXX " ); +// kprintf( "XXX " ); default: // kprintf( "icc_mail_check_tag: uknown tag id %d\n", mail->tag ); break; @@ -412,8 +412,8 @@ int icc_mail_ping_irq( void ) timer = rdtsc() - timer; kprintf( "timer = %d\n", timer ); - kprintf( "mail_pingpong needs in average %d msec (%d ticks)!\n", - timer/(2*ROUNDS*533), timer/(2*ROUNDS) ); + kprintf( "mail_pingpong needs in average %d nsec (%d ticks)!\n", + timer*1000/(2*ROUNDS*533), timer/(2*ROUNDS) ); irq_nested_enable(flags); @@ -439,15 +439,15 @@ int icc_mail_noise() { kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue ); kprintf( "num_ues = %d\n", num_ranks ); int count = 0; - for( ;; ) { - kprintf( "SEND NOISE - " ); + for( ;; ++count ) { + if( !(count%1000) ) kprintf( "STILL ALIVE! " ); /* send a mail to each UE */ for( j=0; j Date: Sat, 27 Aug 2011 02:43:06 -0700 Subject: [PATCH 36/36] change in iRCCE_mailbox_close --- arch/x86/scc/iRCCE_mailbox.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c index 4fb439fa..a0521317 100644 --- a/arch/x86/scc/iRCCE_mailbox.c +++ b/arch/x86/scc/iRCCE_mailbox.c @@ -413,9 +413,6 @@ int iRCCE_mail_send( RC_cache_invalidate(); } - /* disable interrupts */ -// flags = irq_nested_disable(); - // check if mailbox is closed RCCE_acquire_lock( dest ); RC_cache_invalidate(); @@ -439,8 +436,6 @@ int iRCCE_mail_send( iRCCE_memcpy_put( (void*)iRCCE_mailbox_send[dest], (void*)&header, RCCE_LINE_SIZE ); - RC_cache_invalidate(); - // set senders flag RC_cache_invalidate(); iRCCE_mailbox_send[dest]->sent = RCCE_FLAG_SET; @@ -521,10 +516,7 @@ int iRCCE_mailbox_flush(void) { for( i=0; inext; @@ -577,13 +569,12 @@ static int iRCCE_mailbox_close_one(int rank, int check) { iRCCE_memcpy_put( (void*)iRCCE_mailbox_recv[rank], &help_header, RCCE_LINE_SIZE ); - iRCCE_mailbox_status[rank] = iRCCE_MAILBOX_CLOSED; - /* enable interrupts */ irq_nested_enable(flags); RCCE_release_lock( RCCE_IAM ); + iRCCE_mailbox_status[rank] = iRCCE_MAILBOX_CLOSED; return iRCCE_SUCCESS; }