metalsvm/drivers/net/mmnif.c

/* 
 * Copyright 2011 Carl-Benedikt Krueger, Chair for Operating Systems,
 *                                       RWTH Aachen University
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * This file is part of MetalSVM. 
 */

/*
 * mmnif.c	---	memmory mapped interface
 *
 * Virutal IP Interface for the concept processor SCC
 *
 * Carl-Benedikt Krueger 2011
 *
 */

#include "mmnif.h"			/* definitions */

#ifdef WIN32
#include "mailbox.h"		/* mailbox_ptr_t */
#else
#include <metalsvm/mailbox.h>	/* mailbox_ptr_t */
#endif

#if defined(CONFIG_LWIP) && defined(CONFIG_ROCKCREEK)

#include <lwip/netif.h>		/* lwip netif */
#include <lwip/stats.h>		/* inteface stats */
#include <netif/etharp.h>	/* ethernet arp packets */
#include <lwip/ip.h>		/* struct iphdr*/
#include <lwip/tcpip.h>		/* tcpip_input()*/

//#define DEBUG_MMNIF

#ifdef DEBUG_MMNIF
#include "util.h"			/* hex dump */
#endif

#ifdef WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <system/threads/bthread.h>

typedef bthread_sem_t sem_t;
typedef bthread_t tid_t;

/*  "interrupt" of the other virutal network card*/
extern HANDLE remote_process_event;
/*  HANDLE to the other Process (for WPM and RPM)*/
extern HANDLE hProc;


#define DEBUGPRINTF(x,...)  printf(x,__VA_ARGS__)
#else
#define DEBUGPRINTF(x,...)  kprintf(x,##__VA_ARGS__)

#include <metalsvm/semaphore.h>

#include <asm/RCCE.h>
#include <asm/RCCE_lib.h>
#include <asm/iRCCE.h>
#include <asm/iRCCE_lib.h>

#include <asm/SCC_API.h>

#include <metalsvm/time.h>


#endif

#define MMNIF_TX_QUEUELEN		4
#define MMNIF_RX_QUEUELEN		8

#define MMNIF_RX_BUFFERLEN		1792
#define MMNIF_TX_BUFFERLEN		1792

#define MMNIF_CORES				2

#define MMNIF_WORKER_BUDGET		5

/* decide whether it's polling mode or not 
 */
static int no_irq = 0;

/* this will be set by open() and close() and shows wether the driver is running or not
*/
static int active = 0;

/* decide wheter it's uses locking or not
 */
static int disable_locking = 0;

/* IP address of the local core and the router core to get packets forwarded
 */
static unsigned int own_ip_address = 0xC0A80000;	/* 192.168.0.0 */

static unsigned int router_ip_address = 0xC0A80001;	/* 192.168.0.1 */

/* "message passing buffer" specific constants:
 * - start address
 * - size
 */
char*			mpb_start_address = NULL;
unsigned int	mpb_size = NULL;


/* Register offset for the CONFIG and LOCK registers */
#define RCK_GLCFG0          0x10
#define RCK_GLCFG1          0x18
#define RCK_TILEID          0x100

#define RCK_TILE_SIZE       0x01000000

/* Start address of the local register bank */
static int local_crb = 0xF8000000;

/* just set and reset the irq */
static int pulse_irq = 0;

/* Mask of the interrupt bits */
#define RCK_INTR_MASK       0x00000002
#define RCK_NMI_MASK        0x00000001



/*
 * the memory mapped network device
 */
static struct netif* mmnif_dev = NULL;

/* thread variables */
static tid_t	worker_thread;
static tid_t	polling_thread;

typedef struct mmnif_device_stats 
{
	/* device stats (granularity in packets):
	 * - recieve errors
	 * - recieve successes
	 * - transmit errors
	 * - transmit successes
	 */
	unsigned int rx_err;
	unsigned int rx;

	unsigned int rx_bytes;

	unsigned int tx_err;
	unsigned int tx;

	unsigned int tx_bytes;

	/* Heuristics :
	 *
	 */
	unsigned int bdg_overflow;
	unsigned int pll_empty;
	unsigned int rx_intr;
	unsigned int rx_poll;

} mmnif_device_stats_t;

typedef struct mm_rx_buffer
{
	/* memory rx buffer build
	 * - queued : how many packets are in the queue
	 * - pos : which is the next packet to be worked on
	 * - pending : how many packets are pending
	 * - iv_intr : inform via interrupt or not
	 * - lock: semaphore to lock the local variables to be multi access save
	 * 
	 * Note: this section will soon be complexer.
	 * I won't use a single buffer the whole time. I think i will use an descripor table
	 * and a table which descriptor is in use and use the buffer space dynamically with 
	 * descriptors
	 * 
	 * And this buffer needs a lock as soon as more as cores are availible =/
	 */
	uint8_t			queued;
	uint8_t			pos;
	uint8_t			pending;

	uint8_t			iv_intr;

	sem_t			lock;

//	void*			rx_desc[MMNIF_CORES * MMNIF_RX_QUEUELEN];
//	uint8_t			rx_inuse[MMNIF_CORES * MMNIF_RX_QUEUELEN]; /* bits 1: pending 2: finished 3: free ...*/
//	uint8_t			fin;
//	uint8_t*		data[MMNIF_RX_QUEUELEN];
} mm_rx_buffer_t;

typedef struct mmnif
{
	struct mmnif_device_stats	stats;

	/* Interface constants:
	 * - ehternet address
	 * - local ip address
	 */
	struct eth_addr*			ethaddr;
	uint32_t					ipaddr;
	
	/* memory interaction variables:
	 * - transmit queue
	 * - pointer to transmit buffer
	 * - pointer to recive buffer
	 */
	uint8_t						tx_queue;
	uint8_t*					tx_buff[MMNIF_TX_QUEUELEN];
	mm_rx_buffer_t*				rx_buff;

	sem_t						com_poll;

	/* comunication mailbox
	 */
	mailbox_ptr_t				mbox;
}mmnif_t;


#ifdef WIN32

__inline int RCCE_ue(void)
{
#ifndef RECV
	return 1;
#else
	return 0;
#endif
}

#endif

/*
 *	memory maped interface helper functions
 */

/* read the queue value from the remote buffer
 * and return it.
 */
__inline uint8_t mmnif_read_rx_queue(uint32_t dest_ip)
{
#ifdef WIN32
	/* Read the value of the forgein process
	 * form rx_buff->queued
	 */
	mm_rx_buffer_t	hdr;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	return hdr.queued;
#else
	mm_rx_buffer_t	hdr;
	memcpy(&hdr,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,sizeof(hdr));
	return hdr.queued;
#endif
};

/* read the pos value from the remote buffer
 * and return it.
 */
__inline uint8_t mmnif_read_rx_pos(uint32_t dest_ip)
{
#ifdef WIN32
	/* Read the value of the forgein process
	 * form rx_buff->pos
	 */
	mm_rx_buffer_t	hdr;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	return hdr.pos;
#else
	mm_rx_buffer_t	hdr;
	memcpy(&hdr,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,sizeof(hdr));
	return hdr.pos;
#endif
};

/* read the inv_intr from the remote buffer
 * and return it.
 */
__inline uint8_t mmnif_read_rx_inv_intr(uint32_t dest_ip)
{
#ifdef WIN32
	/* Read the value of the forgein process
	 * form rx_buff->inv_intr
	 */
	mm_rx_buffer_t	hdr;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	return hdr.iv_intr;
#else
	mm_rx_buffer_t	hdr;
	memcpy(&hdr,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,sizeof(hdr));
	return hdr.iv_intr;
#endif
};

/* read the pending value from the remote buffer
 * and return it.
 */
__inline uint8_t mmnif_read_rx_pending(uint32_t dest_ip)
{
#ifdef WIN32
	/* Read the value of the forgein process
	 * form rx_buff->pending
	 */
	mm_rx_buffer_t	hdr;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	return hdr.pending;
#else
	mm_rx_buffer_t	hdr;
	memcpy(&hdr,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,sizeof(hdr));
	return hdr.pending;
#endif
};

/* write data to the remote buffer
 * 
 */
__inline int mmnif_write_rx_buff(uint32_t dest_ip, uint32_t pos,void* data)
{
#ifdef WIN32
	/* we assume this is a correct buffer 
	 * therefore here is no further error checking
	 */
	uint32_t nr_of_bytes_written = 0;
	uint16_t length = *((uint16_t*)data);
	while(!WriteProcessMemory(hProc,(char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN,data,length+2,&nr_of_bytes_written));
	return nr_of_bytes_written;
#else
	uint16_t length = *((uint16_t*)data);
	memcpy((char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN,data,length+2);
	return 1;
#endif
};

__inline int mmnif_write_rx_buffl(uint32_t dest_ip, uint32_t pos,void* data,uint16_t length)
{
#ifdef WIN32
	/* we assume this is a correct buffer 
	 * therefore here is no further error checking
	 */
	uint32_t nr_of_bytes_written = 0;
	while(!WriteProcessMemory(hProc,(char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN,&length,2,&nr_of_bytes_written));
	while(!WriteProcessMemory(hProc,(char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN + 2,data,length,&nr_of_bytes_written));
	return nr_of_bytes_written+2;
#else
//	uint16_t length = *((uint16_t*)data);
	memcpy((char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN,&length,2);
	memcpy((char*)mpb_start_address +(dest_ip -1 ) * mpb_size + sizeof(mm_rx_buffer_t)+ pos * MMNIF_RX_BUFFERLEN + 2,data,length);
	return 1;
#endif
};

/* write the new queue value to the remote buffer
 * 
 */
__inline int mmnif_write_rx_queue(uint32_t dest_ip,uint8_t queue)
{
	/* tell the remote buffer/process
	 * that there is another packet in the queue
	 */
#ifdef WIN32
	uint32_t nr_of_bytes_written = 0;
	while(!WriteProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size ,&queue,1,&nr_of_bytes_written));
	return nr_of_bytes_written;
#else

	memcpy((char*)mpb_start_address + ( dest_ip -1 ) * mpb_size ,&queue,1);

	return 1;
#endif
};

/* write the new queue value to the remote buffer
 * 
 */
__inline int mmnif_write_rx_pending(uint32_t dest_ip,uint8_t pending)
{
	/* tell the remote buffer/process
	 * that there is another packet in the queue
	 */
#ifdef WIN32
	uint32_t nr_of_bytes_written = 0;
	while(!WriteProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size + 2 ,&pending,1,&nr_of_bytes_written));
	return nr_of_bytes_written;
#else

	memcpy((char*)mpb_start_address + ( dest_ip -1 ) * mpb_size + 2 , &pending,1);

	return 1;
#endif
};

/* trigger an interrupt on the remote processor
 * 
 */
__inline int mmnif_trigger_irq(dest_ip)
{
#ifdef WIN32
	return PulseEvent(remote_process_event);
#else
#if 0
	/* NOTE: have to check how the remote interrupt managment works
	 * on the SCC
	 */
	mmnif_t*		mmnif = mmnif_dev->state;
	int				core  = (dest_ip - 1) % 2;
	int				irq_address = mmnif->crb[dest_ip-1];
	unsigned int	value;

	if (core == 0) irq_address += RCK_GLCFG0;
	else		   irq_address += RCK_GLCFG1;

	/**/
	value = ReadConfigReg((void*)irq_address);

	if ((!pulse_irq) && (value & RCK_INTR_MASK))
	{
		value &= (~(RCK_INTR_MASK|RCK_NMI_MASK));
		SetConfigReg((void*)irq_address,value);
	}

	value |= RCK_INTR_MASK;
	SetConfigReg((void*) irq_address,value);

	/**/
	if (pulse_irq)
	{
		value &= (~(RCK_INTR_MASK|RCK_NMI_MASK));
		SetConfigReg((void*)irq_address,value);
	}
#endif

	int tmp, x, y, z, addr;

	int ue = dest_ip -1;

	z = Z_PID(RC_COREID[ue]);
	x = X_PID(RC_COREID[ue]);
	y = Y_PID(RC_COREID[ue]);
	addr = CRB_ADDR(x,y) + (z==0 ? GLCFG0 : GLCFG1);

	// send interrupt to ue
	do {
		NOP1;
		tmp=ReadConfigReg(addr);
	} while(tmp & 2);
	tmp |= 2;
	SetConfigReg(addr, tmp);

	return 0;
#endif
};

/* mmnif_device_schedule() : 
 * if there is no interupt used to indicate new packets
 * this creates a polling thread which looks for data
 * itself
 */
__inline int mmnif_device_schedule()
{
#ifdef WIN32
	bthread_create(&polling_thread,NULL,mmnif_poll,NULL);
	return NULL;
#else
	create_kernel_task(&polling_thread,mmnif_poll,NULL);
	return NULL;
#endif
}
/* mmnif_worker_schedule() : 
 * if there is no interupt used to indicate new packets
 * this creates a polling thread which looks for data
 * itself
 */
__inline int mmnif_worker_schedule()
{
#ifdef WIN32
	bthread_create(&worker_thread,NULL,mmnif_worker,NULL);
	return NULL;
#else
	create_kernel_task(&worker_thread,mmnif_worker,NULL);
	return NULL;
#endif
}
/* Allocate Shared Memory for communication this could be:
 * - in Message Passing Buffer
 * - Shared Memory Address Space (0x8000000 + )
 * 
 * Note: under windows this is kernel space so we take arbitrary 0x41000000 here
 */
__inline void* mmnif_shmalloc()
{
	/* Right now every core has the same buffer for every incoming packet
	 * this will be removed and a buffer for each Core will be implemented
	 * 
	 * but i'm first testing with two cores/processes.
	 */
#ifdef WIN32
	mpb_size = sizeof(mm_rx_buffer_t) + MMNIF_RX_QUEUELEN* MMNIF_RX_BUFFERLEN;
	mpb_start_address = VirtualAlloc((char*)0x41000000 /*+ 
						(mpb_size) * (own_ip_address - router_ip_address)*/, 
						mpb_size *MMNIF_CORES,MEM_COMMIT|MEM_RESERVE,PAGE_READWRITE);

	return (char*)0x41000000 + (mpb_size) * (own_ip_address - router_ip_address);
#else
	mpb_size = (sizeof(mm_rx_buffer_t) + MMNIF_RX_QUEUELEN* MMNIF_RX_BUFFERLEN);
	/* We choose a arbitrary address first 
	 * until i know how to properly allocate shared memory
	 */
//	RCCE_shmalloc_init(0x80000000,48*8192);
	mpb_start_address = RCCE_shmalloc(mpb_size*MMNIF_CORES);
	//mpb_start_address = 0x8000000+ mpb_size * (own_ip_address - router_ip_address);
	//SHMalloc(&mpb_start_address);
	//mpb_start_address = kmalloc(mpb_size*MMNIF_CORES);
//	mpb_start_address = 0xC0000000;
	return mpb_start_address + (mpb_size) * (own_ip_address - router_ip_address);
#endif
}

/* mmnif_lock_rx_hdr(): lock the header of mm_rx_buffer
 * so there is no race condition on the variables
 */
__inline void mmnif_lock_rx_hdr(int dest_ip)
{
#ifdef WIN32
	mm_rx_buffer_t	hdr;
	if(disable_locking) return;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	sem_wait(&hdr.lock);
#else
	if(disable_locking) return;
	mm_rx_buffer_t* hdr = (char*)mpb_start_address + ( dest_ip -1 ) * mpb_size;
	sem_wait(&hdr->lock);
#endif
}
/* mmnif_unlock_rx_hdr(): unlock the header
 * again
 */
__inline void mmnif_unlock_rx_hdr(int dest_ip)
{
#ifdef WIN32
	mm_rx_buffer_t	hdr;
	if(disable_locking) return;
	while(!ReadProcessMemory(hProc,(char*)mpb_start_address + ( dest_ip -1 ) * mpb_size,&hdr,sizeof(hdr),NULL));
	sem_post(&hdr.lock);
#else
	if(disable_locking) return;
	mm_rx_buffer_t* hdr = (char*)mpb_start_address + ( dest_ip -1 ) * mpb_size;
	sem_post(&hdr->lock);
#endif
}

/* mmnif_timestamp(): genereate a timestamp for the 
 * packets
 */
__inline int mmnif_timestamp()
{
#ifdef WIN32
	return GetTickCount();
#else
	return get_clock_tick();
#endif
}


/* mmnif_get_device_stats(): Returns a copy of the
 * current device
 */
mmnif_device_stats_t mmnif_get_device_stats()
{
	mmnif_device_stats_t stats = {0};

	if(!mmnif_dev)
		DEBUGPRINTF("mmnif_get_device_stats(): the device is not initialized yet.\n");
	else
		stats = ((mmnif_t*)mmnif_dev->state)->stats;

	return stats;
}


/* mmnif_print_stats(): Print the devices stats of the
 * current device
 */
void mmnif_print_stats()
{
	mmnif_t* mmnif;

	if (!mmnif_dev)
	{
		DEBUGPRINTF("mmnif_print_stats(): the device is not initialized yet.\n");
		return;
	}
	
	mmnif = (mmnif_t*)mmnif_dev->state;

	DEBUGPRINTF("/dev/mmnif - stats:\n");
	DEBUGPRINTF("Received: %d packets successfull\n",mmnif->stats.rx);
	DEBUGPRINTF("Received: %d bytes\n",mmnif->stats.rx_bytes);
	DEBUGPRINTF("interrupts: %d\n",mmnif->stats.rx_intr);
	DEBUGPRINTF("polling: %d\n",mmnif->stats.rx_poll);
	DEBUGPRINTF("Received: %d packets containuing errors\n",mmnif->stats.rx_err);
	DEBUGPRINTF("Transmitted: %d packests successfull\n",mmnif->stats.tx);
	DEBUGPRINTF("Transmitted: %d bytes\n",mmnif->stats.tx_bytes);
	DEBUGPRINTF("Transmitted: %d packests were dropped due to errors\n",mmnif->stats.tx_err);

}


/*
 *	memory maped interface main functions
 */

/* mmnif_get_destination(): low level transmid helper function
 * this function deals with some HW details, it checks to wich core this packet
 * should be routed and returns the destination
 */

uint8_t mmnif_get_destination(struct netif* netif, struct pbuf* p)
{
	struct ip_hdr*	iphdr;
	uint8_t			core;
	uint8_t*		ip4addr;
	uint8_t			addr[4];
	uint32_t		netmask = 0xFFFFFF00;


	/* grab the destination ip address out of the ip header
	 * for internal routing the last ocet is interpreted as core ID.
	 */

	iphdr = (struct ip_hdr*)(p->payload);
	ip4addr = &iphdr->dest.addr;
	
	/* revert the address to host format */
	addr[3] = ip4addr[0]; addr[2] = ip4addr[1]; 
	addr[1] = ip4addr[2]; addr[0] = ip4addr[3];

	/* check if the ip address is in the Local Area Network of the 48 cores */

	/* if it's not the same network the router core is addressed 
	 * Note: the router core is core 1
	 */

	if (!((netmask & *(uint32_t*)addr) == (netmask & own_ip_address) ))
		return 1;



	core = addr[0];

	/* check if the address is legitimata else return router core again */
	if ((core) < 1 || (core > MMNIF_CORES)) 
		core  = 1;

	return core;

}

/*
 * Transmid a packet (called by the lwip)
 */
err_t mmnif_tx(struct netif* netif, struct pbuf* p)
{
	mmnif_t*		mmnif = netif->state;
	uint8_t			slot = mmnif->tx_queue;
	uint8_t			queued;
	uint8_t			pos;
	uint8_t			pending;
	uint32_t		i;
	struct pbuf*	q; /* interator */
	uint8_t			build_buff = TRUE;
	uint8_t			dest_intr = FALSE;
	uint32_t		dest_ip = mmnif_get_destination(netif,p);

	/* take a place in the tx_queue */
	mmnif->tx_queue++;

	/* Perform serveral sanity checks on the packet and the buffers:
	 * - is the queue full?
	 * - is the output packet to big?
	 *
	 *
	 * HINT: MMNIF_TX_QUEUELEN should be 1 as long there is no mutex availible to ensure
	 * just one thread is writing to pos and queue of the mm_rx_buff
	 */

	if (mmnif->tx_queue >= MMNIF_TX_QUEUELEN) 
	{
		DEBUGPRINTF("mmnif_tx(): too many packets at once for tx_queue\n");
		goto drop_packet;
	}

	if (p->tot_len > MMNIF_TX_BUFFERLEN) 
	{
		DEBUGPRINTF("mmnif_tx(): packet is longer than 1792 bytes\n");
		goto drop_packet;
	}

	/* check if the pbuf consists only of one element
	 * if that is the case it would be much overhead
	 * copying that packet again
	 */
	if (!p->next)
		build_buff = FALSE;


	if (build_buff)
	{
		/* write packet length to start transmit buffer */
		*((unsigned short*)mmnif->tx_buff[slot]) = p->tot_len;

		/* build the payload out of the p's 
		 * ensure that the packet is in one memory chunk stored in the transmid buffer
		 */
		for (q = p, i = 2; q != 0; q = q->next) 
		{
			memcpy(mmnif->tx_buff[slot] + i, q->payload, q->len);
			i += q->len;
		}
	}
	else
	{
		/* because there is no copy operation to the tx_slots
		 * we don't need a place in the queue anymore
		 */
		mmnif->tx_queue--;
	}
	/* get the palce the router core is looking for the packet */

	/* lock the dest_ip mm_rx_buffer_hdr */
	mmnif_lock_rx_hdr(dest_ip);

	/* read and edit needed values */
	queued = mmnif_read_rx_queue(dest_ip);
	pos	   = mmnif_read_rx_pos(dest_ip);

	pending  = mmnif_read_rx_pending(dest_ip);

	pending++;
	mmnif_write_rx_pending(dest_ip,pending);

	/* and unlock the dest_ip mm_rx_buffer_hdr */
	mmnif_unlock_rx_hdr(dest_ip);

	/* check if there is a space in the queue without overwriting another packet */
	if ((queued + pending) > MMNIF_RX_QUEUELEN)
	{
		DEBUGPRINTF("mmnif_tx(): too many packet's at once for the remote queue : q:%d p:%d\n",queued , pending);
		goto drop_packet;
	}

	pos = (pos + queued + pending -1) % MMNIF_RX_QUEUELEN;

	/* write buffer to buffer & increment the queued packet count 
	 * this can be safely done without locking because this place is
	 * reserved for us because it has the status "pending"
	 */
	if (build_buff)
		mmnif_write_rx_buff(dest_ip, pos, mmnif->tx_buff[slot]);
	else
		mmnif_write_rx_buffl(dest_ip, pos, p->payload,p->tot_len);

//	udelay(50000);

	/* like above ensure we are the only ones editing the hdr */
	mmnif_lock_rx_hdr(dest_ip);

	queued = mmnif_read_rx_queue(dest_ip);
	pending  = mmnif_read_rx_pending(dest_ip);
	queued++;
	pending--;

	dest_intr = mmnif_read_rx_inv_intr(dest_ip);

#ifdef DEBUG_MMNIF
	DEBUGPRINTF("\n SEND 0x%.8X with length: %d\n",(char*)mpb_start_address + (dest_ip -1)*mpb_size + pos * 1792,p->tot_len +2);
	hex_dump(p->tot_len, p->payload);
#endif

	mmnif_write_rx_queue(dest_ip, queued);
	mmnif_write_rx_pending(dest_ip, pending);

	mmnif_unlock_rx_hdr(dest_ip);

	/* if driver is not in polling mode inform core that a message has arrived */
	if (dest_intr)
		mmnif_trigger_irq(dest_ip);

	/* free the transmid queue*/
	if(build_buff)
		mmnif->tx_queue--;

	/* gather stats:
	 * - firstly for lwip
	 * - secondly for us
	 */
	
	LINK_STATS_INC(link.xmit);

	mmnif->stats.tx++;
	mmnif->stats.tx_bytes += p->tot_len;

	return ERR_OK;

drop_packet: /* packet is lost. clean up and gather stats */

	mmnif->tx_queue--;

	LINK_STATS_INC(link.drop);

	mmnif->stats.tx_err++;

	return ERR_IF;
}

/* mmnif_link_layer(): wrapper function called by ip_output()
 * adding all needed headers for the link layer
 * because we have no link layer and everything is reliable we don't need
 * to add anything so we just pass it to our tx function
 */
static void mmnif_link_layer(struct netif *netif, struct pbuf *q, ip_addr_t *ipaddr)
{
	netif->linkoutput(netif,q);	
}


/*
 * Init the device (called from lwip)
 * It's invoked in netif_add
 */
err_t mmnif_init(struct netif* netif)
{
	mmnif_t* mmnif;
	uint32_t i;
	static int num = 0;
	uint16_t speed = 4000;


	/* Alloc and clear memory for the device struct
	 */
#ifdef WIN32
	mmnif =  malloc(sizeof(mmnif_t));
#else
	mmnif =  kmalloc(sizeof(mmnif_t));
#endif
	if (!mmnif) 
	{
		DEBUGPRINTF("mmnif init():out of memory\n");
		return ERR_MEM;
	}
	memset(mmnif, 0, sizeof(mmnif_t));

	/* Alloc and clear shared memory for rx_buff
	 */
	mmnif->rx_buff = mmnif_shmalloc();
	if (!(mmnif->rx_buff)) 
	{
		DEBUGPRINTF("mmnif init(): allocating shared memory failed\n");
		return ERR_MEM;
	}
	memset(mmnif->rx_buff, 0, mpb_size);

	/* init the lock for the hdr
	 */
	sem_init(&mmnif->rx_buff->lock,1);

	/* init the sems for communication art
	 */

	sem_init(&mmnif->com_poll,1);
	sem_wait(&mmnif->com_poll);

	/* inform via interrupt should be the dafault
	 */
	if (!no_irq)
		mmnif->rx_buff->iv_intr = TRUE;

	/* Alloc and clear internal memory for tx_buff
	 */
#ifdef WIN32
	mmnif->tx_buff[0] = malloc(MMNIF_TX_QUEUELEN * MMNIF_TX_BUFFERLEN);
#else
	mmnif->tx_buff[0] = kmalloc(MMNIF_TX_QUEUELEN * MMNIF_TX_BUFFERLEN);
#endif
	if (!(mmnif->tx_buff[0])) 
	{
		DEBUGPRINTF("mmnif init: out of memory tx\n");
#ifdef WIN32
		free(mmnif->rx_buff);
		free(mmnif);
#else
//		kfree(mmnif->rx_buff);
//		kfree(mmnif);
#endif
		return ERR_MEM;
	}
	mmnif->tx_queue = 0;	
	memset(mmnif->tx_buff[0], 0, MMNIF_TX_QUEUELEN * MMNIF_TX_BUFFERLEN);

	for (i = 0; i < MMNIF_TX_QUEUELEN -1 ; i++)
		mmnif->tx_buff[i+1] = mmnif->tx_buff[i] + MMNIF_TX_BUFFERLEN;

	/* initialize the mailbox system */
	mailbox_ptr_init(&mmnif->mbox);

	/* pass the device state to lwip */
	netif->state = mmnif;
	mmnif_dev = netif;

	/* Generate MAC address */
	mmnif_dev->hwaddr[0] = 0x11;mmnif_dev->hwaddr[1] = 0x22;mmnif_dev->hwaddr[2] = 0x33;
	mmnif_dev->hwaddr[3] = 0x44;mmnif_dev->hwaddr[4] = 0x55;mmnif_dev->hwaddr[5] = RCCE_ue()*0x11 +0x66;

	/*
	 * Initialize the snmp variables and counters inside the struct netif.
	 * The last argument should be replaced with your link speed, in units
	 * of bits per second.
	 */
	NETIF_INIT_SNMP(netif, snmp_ifType_ethernet_csmacd, speed);

	/* administrative details */
	netif->name[0] = 'm';
	netif->name[1] = 'n';
	netif->num = num;
	num++;
	/* downward functions */
	netif->output = mmnif_link_layer;
	/* there is no special link layer just the ip layer */
	netif->linkoutput = mmnif_tx;
	/* maximum transfer unit */
	netif->mtu = 1500;
	/* broadcast capability, keep all default flags*/
	netif->flags |= NETIF_FLAG_BROADCAST /*| NETIF_FLAG_ETHARP*/ | NETIF_FLAG_LINK_UP; 
	/* hardware address length */
	netif->hwaddr_len = 6;

	mmnif->ethaddr = (struct eth_addr *)netif->hwaddr;

	active = TRUE;

#ifdef MMNIF_DEBUG
	DEBUGPRINTF("mmnif init complete\n");
#endif

	return NULL;
}

/*
 * Receive a packet : recieve, pack it up and pass over to higher levels
 */
static void mmnif_rx(struct netif* netif)
{
	mmnif_t*		mmnif = netif->state;
	uint16_t		length,i;
	struct pbuf*	p = NULL;
	struct pbuf*	q;
	char*			data;
	uint32_t		pos;
	uint8_t			queued;

	/* retrieve pointer to actual data array */
	data = (char*) mmnif->rx_buff + sizeof(mm_rx_buffer_t);
	/* retrice position wich is needed to be worked on */
	pos = (mmnif->rx_buff->pos % MMNIF_RX_QUEUELEN) * MMNIF_RX_BUFFERLEN;

	/* The packet length is stored in the first 2 bytes but does not include
     * the header. Check for reasonable sizes before processing the data to
	 * prevent nasty memory overflow errors.
     */
	length = *((uint16_t*) (data + pos));

	/* If length is zero return silently */
	if (length == 0)
	{
//		mmnif->rx_buff->pos++;
//		mmnif->rx_buff->queued--;
		return;
	}
	if (length < sizeof(struct ip_hdr) ||length > netif->mtu)
	{
		DEBUGPRINTF("mmnif_rx(): illegal packet length %d => drop the packet\n",length);
		goto drop_packet;
	}

	/* From now on there is a real packet and it
	 * has to be worked on
	 */


#ifdef DEBUG_MMNIF
	DEBUGPRINTF("\n RECIEVED - 0x%.8X with legth: %d\n",data + pos,length+2);
	hex_dump(length+2,data + pos);
#endif
	
	/* drop the length word of the packet data since it's no longer needed*/
	pos += 2;

	
	/* Build the pbuf for the packet so the lwip 
	 * and other higher layer can handle it
	 */
	p = pbuf_alloc(PBUF_RAW, length, PBUF_POOL);
	if (!p) 
	{
		DEBUGPRINTF("mmnif_rx(): low on mem - packet dropped\n");
		goto drop_packet;
	}
	
	/* copy packet to pbuf structure going through linked list */
	for (q=p, i = 0; q!=NULL; q=q->next) 
	{
		memcpy((uint8_t*)q->payload + i,&data[pos+i],q->len);
		i +=q->len;
	}

	/* everything is copied to a new buffer so it's save to release 
	 * the old one for new incoming packets
	 */

	mmnif_lock_rx_hdr(own_ip_address & 0xFF);

	mmnif->rx_buff->pos++;
	mmnif->rx_buff->queued--;

	mmnif_unlock_rx_hdr(own_ip_address & 0xFF);

	/* using the mailbox to hand the buffer to the incoming packet thread
	 * so the "interrupt" itself is not taking to long
	 */
	mailbox_ptr_post(&mmnif->mbox, (void*)p);

	/* gather some stats and leave the rx handler */
	LINK_STATS_INC(link.xmit);
	mmnif->stats.rx++;

	mmnif->stats.rx_bytes += p->tot_len;

	if (mmnif->rx_buff->iv_intr)
		mmnif->stats.rx_intr++;
	else
		mmnif->stats.rx_poll++;

	return;

drop_packet:
	/* packet is lost so gather stats and leave the rx handler*/
	mmnif->rx_buff->pos++;
	mmnif->rx_buff->queued--;
	LINK_STATS_INC(link.drop);
	mmnif->stats.rx_err++;
	return;
}

/*
 * The wait implementation which is processing all incoming packets 
 */
static int mmnif_wait(struct netif* netif, uint32_t poll, int budget)
{	
	mmnif_t*			mmnif = netif->state;
	struct eth_hdr *	ethhdr;
	struct pbuf*		p = NULL;
	int					err = ERR_OK;
	unsigned int		npackets = 0;
	unsigned int		quota = 0;

	if (budget > mmnif->rx_buff->queued)
	{
		quota = mmnif->rx_buff->queued;
	}
	else
	{
		quota = budget;

		mmnif->stats.bdg_overflow++;
		if (mmnif->stats.bdg_overflow >= 0x10)
		 {
		 /*   enable polling and disable interrupts
		 * 

		 */
				mmnif_lock_rx_hdr(own_ip_address && 0xff);
				mmnif->rx_buff->iv_intr = FALSE;
				mmnif_unlock_rx_hdr(own_ip_address && 0xff);
#ifdef DEBUG_MMNIF
				DEBUGPRINTF("mmnif_wait(): heuristical polling enables\n");
#endif
				sem_post(&mmnif->com_poll);
				mmnif->stats.bdg_overflow = 0;
		 }

	}

	/* process up to quota packets from the receive queue */
	while (npackets <= quota)
	{
		/* fetch new data from mmnif_rx() if there is any */
		if (poll) 
		{
			/* if there is no data return immeadieatly*/
			if (mailbox_ptr_tryfetch(&(mmnif->mbox), (void**) &p))
				return err;
		}
		else 
		{
			mailbox_ptr_fetch(&(mmnif->mbox), (void**) &p);
		}

		/* if there is data, pass it up to the lwip
		 * so he can handle it properly
		 */

		/* full packet send to tcpip_thread to process */
		if ((err = mmnif_dev->input(p, mmnif_dev)) != ERR_OK) 
		{
			DEBUGPRINTF("mmnif_poll: IP input error\n");
			pbuf_free(p);
		}

		npackets++;
	}

	/* Note : i will add an return error wich indicates that 
	 * there is no budget left but messages in the queue
	 */
	return err;

}

/* 
 * worker thread	
 */
int mmnif_worker(void* e)
{
#ifdef DEBUG_MMNIF
	DEBUGPRINTF("Waiting for work to do!!!\n");
#endif
	while (active)
		mmnif_wait(mmnif_dev,0,MMNIF_WORKER_BUDGET);

	return NULL;
}

/*
 * the poll function wich is used if no interrupt wake up our mmnif_rx functions
 */
int mmnif_poll(void* e)
{
	mmnif_t*	mmnif;
	unsigned int diff = mmnif_timestamp();
	unsigned int tmp32 = 0;

	if (!mmnif_dev)
	{
		DEBUGPRINTF("mmnif_poll(): the driver is not initialized yet\n");
		return -1;
	}
	
	mmnif = (mmnif_t*) mmnif_dev->state;

#ifdef DEBUG_MMNIF
	DEBUGPRINTF("Polling for work to do!!!! ONBBBB  0x%.8X  BBBB\n\n",mmnif->rx_buff);
#endif

	if (!no_irq)
	{
		sem_wait(&mmnif->com_poll);
	}

	/*run while driver is up*/
	while (active)
	{
		while (!mmnif->rx_buff->queued)
		{
			tmp32 = mmnif_timestamp();
			diff = diff - tmp32 > 0 ? diff - tmp32 : tmp32 - diff;
			mmnif->stats.pll_empty++;
			if (mmnif->stats.pll_empty >= 0x100000)
			{
			/*   enable interrupts and suspend polling
			 * 
			 */
				mmnif_lock_rx_hdr(own_ip_address && 0xff);
				mmnif->rx_buff->iv_intr = TRUE;
				mmnif_unlock_rx_hdr(own_ip_address && 0xff);
#ifdef DEBUG_MMNIF
				DEBUGPRINTF("mmnif_poll(): heuristical interrupts enabled\n");
#endif
				sem_wait(&mmnif->com_poll);
				mmnif->stats.pll_empty = 1;
			}
		}
		mmnif->stats.pll_empty--;
//		udelay(30000);
		mmnif_rx(mmnif_dev);
//		mmnif_wait(mmnif_dev,0,MMNIF_WORKER_BUDGET);
	}

	return NULL;
}

/* mmnif_irqhandler(): handles incoming interrupts
 * its just a local wrapper for mmnif_rx()
 */
void mmnif_irqhandler(void)
{
	mmnif_t*	mmnif;
	/* return if mmnif_dev is not yet initialized*/
	if (!mmnif_dev)
	{
		DEBUGPRINTF("mmnif_irqhandler(): the driver is not initialized yet\n");
		return;
	}
	
	mmnif = (mmnif_t*) mmnif_dev->state;

	while (mmnif->rx_buff->queued)
		mmnif_rx(mmnif_dev);
}

/*
 * Open the interface should be called by kernel to use this network interface
 */
int mmnif_open(void)
{
	struct ip_addr	ipaddr;
	struct ip_addr	netmask;
	struct ip_addr	gw;

	/* calculate my own ip address from core number
	 * Note: core 1 is the router core
	 */
	IP4_ADDR(&gw, 0,0,0,0);
	IP4_ADDR(&ipaddr, 192,168,0,RCCE_ue()+1);
	IP4_ADDR(&netmask, 255,255,255,0);

	own_ip_address += RCCE_ue()+1;

#ifdef WIN32
	mmnif_dev = malloc(sizeof(struct netif));
#else
	mmnif_dev = kmalloc(sizeof(struct netif));
#endif
	/* register our Memory Mapped Virtual IP interface in the lwip stack
	 * and tell him how to use the interface:
	 * - mmnif_dev : the device data storage
	 * - ipaddr : the ip address wich should be used
	 * - gw : the gateway wicht should be used
	 * - mmnif_init : the initialization which has to be done in order to use our interface 
	 * - ethernet_input : tells him that he should get ethernet input (inclusice ARP)
	 *
	 * Note: Ethernet Input will be removed because its NOT needed and will  
	 * be replaced with ip_input
	 */
	if (!netif_add(mmnif_dev, &ipaddr, &netmask, &gw, NULL,(netif_init_fn)mmnif_init, tcpip_input/*ethernet_input*/)) 
	{
		DEBUGPRINTF("mmnif_open() : unable to add network interface\n");
		return -1;
	}

	/* set our network interface to the default interface for lwip*/
	//netif_set_default(mmnif_dev);
	/* tell lwip all initialization is done and we want to set it ab*/
	netif_set_up(mmnif_dev);

	/* test if interface is really up */
	if (!netif_is_up(mmnif_dev)) 
	{
		DEBUGPRINTF("mmnif_open(): network interface is not up\n");
		return -2;
	}

	/* indicate that the driver is active now*/
	active = TRUE;

	/* If interrupts are not used we immediately add the polling function
     * to the queue which would otherwise be done through the IRQ handler.
     */
//	if (no_irq) 
		mmnif_device_schedule();

	/* Start the device worker thread wich actually processes the incoming
	 * packet's this is not done in the "interrupt handler" to shorten them up
	 */
	mmnif_worker_schedule();

#ifdef DEBUG_MMNIF
	DEBUGPRINTF("mmnif_dev is open\n");
#endif

	return 0;

}

/*
 * close the interface should be called by kernel to close this interface and release resources
 * Note: it's temporarly empty. Support will be added.
 */
int mmnif_close(void)
{
	mmnif_t* mmnif;

	if (!mmnif_dev)
	{
		DEBUGPRINTF("mmnif_close(): you closed the device before it was properly opened -.-* \n");
	}

	mmnif = (mmnif_t*)mmnif_dev->state;
	/* indicate that the driver is not active anymore 
	 * - this will stop the polling thread i.e.
	 */

	/* resources has to be freed here
	 * will be added soon ;)
	 */

	active = FALSE;
#ifdef WIN32
	free(mmnif->tx_buff);
	free(mmnif_dev);
#else
//	kfree(mmnif->tx_buff);
//	kfree(mmnif_dev);
#endif
	return NULL;
}

#endif