From 762d39e69ac5981535dbd426bd53acef935e5ab7 Mon Sep 17 00:00:00 2001
From: Simon Pickartz <pickartz@lfbs.rwth-aachen.de>
Date: Sat, 27 Aug 2011 10:13:13 -0700
Subject: [PATCH] major fix in use of irqs in iRCCE_mailbox.c

---
 arch/x86/scc/iRCCE_mailbox.c | 55 +++++++++++++++---------------
 arch/x86/scc/icc.c           | 65 +++++++++++++++++++++---------------
 2 files changed, 65 insertions(+), 55 deletions(-)

diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c
index a0521317..0fc20331 100644
--- a/arch/x86/scc/iRCCE_mailbox.c
+++ b/arch/x86/scc/iRCCE_mailbox.c
@@ -216,23 +216,18 @@ static int iRCCE_mailbox_check() {
 int iRCCE_mail_check(int sender) {
 	uint32_t flags;
 
-	/* disable interrupts */
-	flags = irq_nested_disable();
-	
+	// check all mailboxes in case of wildcard
+	if( sender == iRCCE_MAILBOX_ALL ) {
+		iRCCE_mailbox_check();
+
+		return iRCCE_SUCCESS;
+	}
 
 	// verify sender's ID
 	if( (sender < 0) || (sender > RCCE_NP) || (sender == RCCE_IAM) ) {
-		if( sender == iRCCE_MAILBOX_ALL ) {
-			iRCCE_mailbox_check();
-			return 0;
-		}
-
-		/* enable interrupts */
-		irq_nested_enable(flags);
-
 		return iRCCE_ERROR_ID;		
 	}
-	
+
 	// check if mailbox is open
 	if( iRCCE_mailbox_status[sender] == iRCCE_MAILBOX_CLOSED ) {
 		return iRCCE_MAILBOX_CLOSED;	
@@ -240,20 +235,24 @@ int iRCCE_mail_check(int sender) {
 
 	RC_cache_invalidate();
 	if( iRCCE_mailbox_recv[sender]->sent ) {
+		/* disable interrupts */
+		flags = irq_nested_disable();
+
 		iRCCE_mail_fetch(sender);
 
 		// reset senders flag
 		RC_cache_invalidate();
 		*(iRCCE_mailbox_recv[sender]) = dummy_header;
 		
+		/* enable interrupts */
+		irq_nested_enable(flags);
+
 		return iRCCE_SUCCESS;
 	}
 	else {
 		return iRCCE_MAILBOX_EMPTY;
 	}
 
-	/* enable interrupts */
-	irq_nested_enable(flags);
 }
 
 //------------------------------------------------------------------------------
@@ -280,12 +279,6 @@ int iRCCE_mail_recv(
 	uint32_t flags;
 	iRCCE_MAIL_HEADER* help_header;
 
-	// if there is no mail, check for incoming
-/*	if ( !iRCCE_mailbox_recv_queue[0].first ) {
-		iRCCE_mailbox_check();
-	}
-*/
-
 	// check priority queues 
 	for( i=0; i<iRCCE_PRIOS; ++i ) {
 		if ( iRCCE_mailbox_recv_queue[i].first ) {
@@ -399,8 +392,6 @@ int iRCCE_mail_send(
 			char* payload,	// pointer to buffer for header payload
 			int dest	// UE that will receive the header
 			) {
-	uint32_t flags;	
-
 	// check for an attempt to send to own mailbox
 	if( dest == RCCE_IAM ) {
 		return iRCCE_MAILBOX_INVALID;
@@ -409,7 +400,10 @@ int iRCCE_mail_send(
 	// if dest mailbox is full, check for incoming mail
 	RC_cache_invalidate();
 	while( iRCCE_mailbox_send[dest]->sent ) {
-		iRCCE_mailbox_check();
+//		iRCCE_mailbox_check();
+		NOP8;
+		NOP8;
+		NOP8;
 		RC_cache_invalidate();
 	}
 
@@ -444,9 +438,6 @@ int iRCCE_mail_send(
 
 	RCCE_release_lock( dest );
 
-	/* enable interrupts */
-//	irq_nested_enable(flags);
-
 	return iRCCE_SUCCESS;
 }
 
@@ -512,7 +503,11 @@ int iRCCE_mailbox_wait(void) {
 int iRCCE_mailbox_flush(void) {
 	int i;
 	uint32_t flags;
-
+	
+	/* disable interrupts */
+	flags = irq_nested_disable();
+	
+	// empty priority queues
 	for( i=0; i<iRCCE_PRIOS; ++i ) {
 		iRCCE_MAIL_HEADER* erase_header = 
 			iRCCE_mailbox_recv_queue[i].first;
@@ -524,9 +519,11 @@ int iRCCE_mailbox_flush(void) {
 			erase_header = iRCCE_mailbox_recv_queue[i].first;
 		}
 
-		/* enable interrupts */
-		irq_nested_enable(flags);
 	}
+
+	/* enable interrupts */
+	irq_nested_enable(flags);
+
 	return iRCCE_SUCCESS;
 }
 
diff --git a/arch/x86/scc/icc.c b/arch/x86/scc/icc.c
index c42c4fc4..d4d60531 100644
--- a/arch/x86/scc/icc.c
+++ b/arch/x86/scc/icc.c
@@ -238,7 +238,7 @@ int icc_halt(void)
 	return 0;
 }
 
-#define ROUNDS 100000
+#define ROUNDS 10000
 #define CORE_A 0 		// sender
 #define CORE_B 4		// receiver 
 
@@ -268,7 +268,7 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) {
 	static int count = 0;
 	
 	if( !mail ) return;
-	
+
 	switch( mail->tag ) {
 	case iRCCE_ANYLENGTH:
 		recv_buffer = (char*)kmalloc( mail->size );
@@ -278,6 +278,7 @@ static inline void icc_mail_check_tag(iRCCE_MAIL_HEADER* mail) {
 
 	case PING_REQ:
 //		count++;
+
 		iRCCE_mail_send(0, PING_RESP, 0, NULL, mail->source);
 		break;
 	
@@ -430,7 +431,7 @@ int icc_mail_noise() {
 	int flags;
 
 	// leave function if not participating
-	if( (my_ue == CORE_A)  || (my_ue == CORE_B) ) {
+	if( (my_ue == CORE_A) /* || (my_ue == CORE_B)*/ ) {
 		kprintf( "mail_noise: leaving" );
 		return -1;
 	}  
@@ -438,9 +439,9 @@ int icc_mail_noise() {
 
 	kprintf( "Hello from icc_mail_noise: my_ue = %d\n", my_ue );
 	kprintf( "num_ues = %d\n", num_ranks );
-	int count = 0;	
-	for( ;; ++count ) {
-		if( !(count%1000) ) kprintf( "STILL ALIVE! " );
+	
+	for( i=0; i<10000; ++i ) {
+		if( !(i%1000) ) kprintf( "%d ", i );
 		/* send a mail to each UE */
 		for( j=0; j<num_ranks; ++j ) {
 			if( (j == CORE_A) /*|| (j == CORE_B)*/ )
@@ -449,14 +450,16 @@ int icc_mail_noise() {
 			/* send noise mail */
 			iRCCE_mail_send(0, NOISE, 1, NULL, j);
 #ifdef _IRQ_NOISE_
+			kprintf( "sending irq ... " );
 			icc_send_gic_irq(j);
 #endif
 		}
 
 #ifndef _IRQ_NOISE_
 		/* read some mails  */
-		flags = irq_nested_disable;
-
+		if( my_ue == CORE_B )
+			continue;
+		
 		iRCCE_mail_check(iRCCE_MAILBOX_ALL);
 		
 		while( iRCCE_mail_recv(&recv_mail) == iRCCE_SUCCESS ) {
@@ -464,11 +467,20 @@ int icc_mail_noise() {
 			iRCCE_mail_release(&recv_mail);	
 		}
 
-		irq_nested_enable(flags);
 #endif
 
 	}
 	
+	/* recv noise mails */
+	while( iRCCE_mail_recv(&recv_mail) == iRCCE_SUCCESS ) {
+		icc_mail_check_tag(recv_mail);
+		iRCCE_mail_release(&recv_mail);	
+		NOP8;
+		NOP8;
+		NOP8;
+	}
+
+	
 	return 0;
 }
 
@@ -492,40 +504,41 @@ void icc_mail_check(int irq)
 	flags = irq_nested_disable();
 
 	if( irq == 1 ) {
-//		for( i=0; i<2; ++i ) {
-			/* read status register */
-			irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8);
-			irq_status = irq_reset = *irq_status_reg;
+		/* read status register */
+		irq_status_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_STATUS + my_ue*8);
+		irq_status = irq_reset = *irq_status_reg;
 
 
-			/* determine interrupt sources */
-			irq_status >>= 6;	// shift emac bits
+		/* determine interrupt sources */
+		irq_status >>= 6;	// shift emac bits
 
-			for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) {
-				if( (irq_status & 0x1) != 0 ) {
-					res = iRCCE_mail_check(source);
-				} 
+		for( source = 0; irq_status != 0; irq_status >>= 1, ++source ) {
+			if( (irq_status & 0x1) != 0 ) {
+				res = iRCCE_mail_check(source);
 			} 
+		} 
 
 
-			/* reset status register */
-			irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8);
-			*irq_reset_reg = irq_reset;
-//		}
+		/* reset status register */
+		irq_reset_reg = (volatile uint64_t*)(FPGA_BASE + IRQ_RESET + my_ue*8);
+		*irq_reset_reg = irq_reset;
 	}
 	else {
 		iRCCE_mail_check(iRCCE_MAILBOX_ALL);
 	}
 
+	/* enable interrupts */
+	irq_nested_enable(flags);
+
 	/* empty mail queue */
 	while( iRCCE_mail_recv(&header) == iRCCE_SUCCESS ) {
 		icc_mail_check_tag(header);
 		iRCCE_mail_release( &header );
+		NOP8;
+		NOP8;
+		NOP8;
 	}
 
-	/* enable interrupts */
-	irq_nested_enable(flags);
-
 
 }