From 4b3efa14a324e6dfa2c482dd29cb2ba544f9b9f2 Mon Sep 17 00:00:00 2001
From: Stefan Lankes <lankes@lfbs.rwth-aachen.de>
Date: Sat, 2 Jul 2011 03:10:07 -0700
Subject: [PATCH] add Simon's patch

---
 arch/x86/scc/iRCCE_mailbox.c | 146 +++++++++++++++++++++++++++++++----
 1 file changed, 132 insertions(+), 14 deletions(-)

diff --git a/arch/x86/scc/iRCCE_mailbox.c b/arch/x86/scc/iRCCE_mailbox.c
index ae3132cb..257a12c9 100644
--- a/arch/x86/scc/iRCCE_mailbox.c
+++ b/arch/x86/scc/iRCCE_mailbox.c
@@ -28,19 +28,31 @@
 #include <metalsvm/stdlib.h>
 #include <metalsvm/string.h>
 
-
 #ifdef CONFIG_ROCKCREEK
 #include <asm/iRCCE_lib.h>
 
+/**
+ * 
+ * @file contains implementation of the mailbox system
+ * @author Simon Pickartz
+ *
+ *
+ */
+
+
 // forward declaration
 static int iRCCE_mailbox_close_one(int rank, int check); 
 
-
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_print_header
 //------------------------------------------------------------------------------
 // routine for printing given header (debugging purpose)
 //------------------------------------------------------------------------------
+/**
+ * @brief routine for printing a given header
+ * @param header is a pointer to a given iRCCE_MAIL_HEADER structure
+*/
+
 void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) {
 
                 kprintf( "\n"
@@ -59,9 +71,20 @@ void iRCCE_mailbox_print_header(iRCCE_MAIL_HEADER* header) {
 }
 
 //------------------------------------------------------------------------------
-// FUNCTION: iRCCE_mailbox_enqueue_header
+// FUNCTION: iRCCE_mail_fetch
 //------------------------------------------------------------------------------
-// routine to put push received header into iRCCE_mailbox_recv_queue 
+/**
+ * @brief routine to check for new mail in a given mailbox
+ * @param rank is the ID of the ranks mailbox to be emptied 
+ *
+ * The function checks if the mailbox has new mail for a given rank. In case of
+ * new mail it needs memory for the received header. Either there is memory
+ * in the internal garbage collection or it has to allocated. The next step is 
+ * to check wheter a last-mail was received or a normal one. A last-mail is 
+ * indicated by the iRCCE_LAST_MAIL tag. A last-mail entails the appropriate
+ * flag in the iRCCE_last_mail array to be set. Otherwise the header has to be
+ * enqueued in the mailbox_recv_queue accordingly to the priority field.
+ */
 //------------------------------------------------------------------------------
 static int iRCCE_mail_fetch(
 		int rank // rank from whom to empty mailbox
@@ -119,12 +142,23 @@ static int iRCCE_mail_fetch(
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_check
 //------------------------------------------------------------------------------
-// routine to check for new mail in mailboxes
+/** 
+ * @brief routine to check for new mail in mailboxes
+ *
+ * This function has to be called from time to time. It empties all mailboxes of
+ * the participating cores if the corresponding sent-flag is set and the mailbox
+ * is not closed. After calling iRCCE_mail_fetch the sent-flag has to be reset. 
+ * Here we have to use a little trick because we can only write to the MPB in 
+ * cacheline granularity. We set the appropriate flag to zero and afterwords 
+ * touch the MPB on another cacheline. That causes the write combine buffer to 
+ * write out the data.
+ */
 //------------------------------------------------------------------------------
 static int iRCCE_mailbox_check() {
 	int i;
 
 	for( i=0; i<RCCE_NP; ++i ) {
+		if( i == RCCE_IAM ) continue;
 		// only check open mailboxes
 		if( iRCCE_mailbox_status[i] == iRCCE_MAILBOX_OPEN ) {
 
@@ -146,7 +180,18 @@ static int iRCCE_mailbox_check() {
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mail_recv
 //------------------------------------------------------------------------------
-// routine for fetching received headers out of iRCCE_mailbox_recv_queue
+/** 
+ * @brief routine for fetching received headers out of iRCCE_mailbox_recv_queue
+ * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure
+ * @return iRCCE_SUCCESS if there was new mail; iRCCE_MAILBOX_EMPTY else
+ * @todo implement fairer dequeue mechanism
+ *
+ * The function checks if the receive queue with highest priority (priority 0) 
+ * contains any mail headers. In this case we pop the first element of that list
+ * in a FIFO maner. Otherwise iRCCE_mailbox_check() has to be called. Afterwards
+ * the first element of a non-empty receive queue with highest priority is 
+ * returned.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mail_recv(
 		iRCCE_MAIL_HEADER** header 	// pointer to incoming header 
@@ -172,6 +217,7 @@ int iRCCE_mail_recv(
 			return iRCCE_SUCCESS;
 		}
 	}
+
 	// no mail queued
 	*header = NULL;
 	return iRCCE_MAILBOX_EMPTY;
@@ -181,7 +227,15 @@ int iRCCE_mail_recv(
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mail_release
 //------------------------------------------------------------------------------
-// routine to free memory for given header 
+/**
+ * @brief routine to store released header by user in garbage collection
+ * @param header is the address of a pointer to an iRCCE_MAIL_HEADER structure
+ * @return iRCCE_SUCCESS in any case
+ *
+ * This function enqueus a pointer to memory for an iRCCE_MAIL_HEADER structure
+ * that is not used by the user program anymore. 'header' points to NULL by 
+ * return of the function.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mail_release(
 		iRCCE_MAIL_HEADER** header 
@@ -207,7 +261,31 @@ int iRCCE_mail_release(
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mail_send
 //------------------------------------------------------------------------------
-// routine for sending headers into receivers mailbox (blocking)
+/**
+ * @brief routine to send a mail to a given destination (blocking)
+ * @param size is the size of the following message. This message may be
+ * contained in the payload or send by an isend()-call
+ * @param tag is an integer value to distinguish between different mail types
+ * @param prio indicates the priority of the mail. 0 is the highest priority
+ * whereas 4 is the lowest.
+ * @param payload is a pointer to byte array with a size of 
+ * iRCCE_MAIL_HEADER_PAYLOAD. If NULL is passed nothing is done, otherwise array
+ * pointed by payload is copied into the header.
+ * @param dest indicates the destination of the mail in terms of the ID of 
+ * one of the participating ranks
+ * @return iRCCE_SUCCESS if send was successful. If target mailbox is closed
+ * iRCCE_MAILBOX_CLOESD is returned.
+ *
+ * First it has to be checked if the target mailbox still contains an unread mail.
+ * If this is the case there is time to empty the own mailboxes. It blocks until
+ * the receiver has emptied its mailbox. The next step is to acquire the lock 
+ * for the target mailbox to be sure that the mailbox is not closed by the 
+ * receiver while the mail is delivered. After locking the mailbox an 
+ * iRCCE_MAIL_HEADER is generated according with the parameters (but with a 
+ * sent-flag set to zero)and is copied into the target mailbox. After all data 
+ * beeing written the appropropriate sent-flag has to be set with the same trick
+ * already used in iRCCE_mail_check(). Now the lock can be released.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mail_send(
 			size_t size, 	// size of following message expected to be send/received
@@ -254,14 +332,21 @@ int iRCCE_mail_send(
 	RC_cache_invalidate();
 
 	RCCE_release_lock( dest );
-
+	
 	return iRCCE_SUCCESS;
 }
 
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_last_mail_recv
 //------------------------------------------------------------------------------
-// check if all final headers are received from all UEs 
+/**
+ * @brief check if all final headers are received from all UEs
+ * @return iRCCE_SUCCES if all last-mails arrive iRCCE_LAST_MAILS_NOT_RECV
+ * otherwise
+ *
+ * This functions is used to determine if all last-mails arrived at the calling
+ * UE. Therefore it checks the iRCCE_last_mail array if all flags are set.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_last_mail_recv(void) {
 	int i;
@@ -280,7 +365,15 @@ int iRCCE_last_mail_recv(void) {
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_wait
 //------------------------------------------------------------------------------
-// wait for all last-mails to be received
+/** 
+ * @brief wait for all last-mails to be received
+ * @return iRCCE_SUCCESS
+ *
+ * This functions blocks in a loop calling continously iRCCE_last_mail_recv() 
+ * until its return value is iRCCE_SUCCESS what implicates that all last-mails
+ * of the participating UEs arrived at the calling UE.
+ * This function is used to shut down the mailbox environment.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mailbox_wait(void) {
 	while( iRCCE_last_mail_recv() == iRCCE_LAST_MAILS_NOT_RECV ) {
@@ -294,7 +387,13 @@ int iRCCE_mailbox_wait(void) {
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_flush
 //------------------------------------------------------------------------------
-// dequeue all iRCCE_mailbox_recv_queue elements and free memory
+/**
+ * @brief dequeue all iRCCE_mailbox_recv_queue elements and free memory
+ * @return iRCCE_SUCCESS
+ *
+ * This function empties all iRCCE_mailbox_recv_queues whith no regard to their
+ * content. This function is used to shut down the mailbox environment.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mailbox_flush(void) {
 	int i;
@@ -314,7 +413,18 @@ int iRCCE_mailbox_flush(void) {
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_close_one
 //------------------------------------------------------------------------------
-// routine to close one mailbox
+/**
+ * @brief routine to close one mailbox
+ * @return iRCCE_SUCCESS
+ * @param rank is the ID of the ranks mailbox to be closed
+ * @param check is a flag indicating wether the mailbox has to be emptied before
+ * closing or not. This is required for a close-call as a result of a received
+ * last-mail.
+ * 
+ * This function closes a mailbox of the given rank. If the check flag is set
+ * an iRCCE_mail_check()-call is performed. The close procedure has to be locked
+ * to be sure that no UE sends any mail while closing the mailbox.
+ */
 //------------------------------------------------------------------------------
 static int iRCCE_mailbox_close_one(int rank, int check) {
 	RCCE_acquire_lock( RCCE_IAM );
@@ -345,7 +455,15 @@ static int iRCCE_mailbox_close_one(int rank, int check) {
 //------------------------------------------------------------------------------
 // FUNCTION: iRCCE_mailbox_close()
 //------------------------------------------------------------------------------
-// routine to close one or all mailboxes 
+/** 
+ * @brief routine to close one or all mailboxes
+ * @param rank is the ID of the UEs mailbox to be closed if iRCCE_MAILBOX_ALL
+ * is passed all mailboxes are closed by the calling UE
+ * @return iRCCE_SUCCESS
+ *
+ * This functions closed one or all mailboxes of the calling UE. This is done by
+ * calling iRCCE_mailbox_close_one for one or all mailboxes.
+ */
 //------------------------------------------------------------------------------
 int iRCCE_mailbox_close(int rank) {
 	if( rank == iRCCE_MAILBOX_ALL ) {