299 lines
6.7 KiB
C
299 lines
6.7 KiB
C
/*
|
|
* Copyright 2010 Stefan Lankes, Chair for Operating Systems,
|
|
* RWTH Aachen University
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR COND */
|
|
|
|
#include <metalsvm/stdio.h>
|
|
#include <metalsvm/errno.h>
|
|
#include <metalsvm/processor.h>
|
|
#include <metalsvm/errno.h>
|
|
#include <asm/io.h>
|
|
#include <asm/irqflags.h>
|
|
#include <asm/irq.h>
|
|
#ifdef CONFIG_ROCKCREEK
|
|
#include <asm/RCCE.h>
|
|
#include <asm/RCCE_lib.h>
|
|
#include <asm/iRCCE.h>
|
|
#include <asm/SCC_API.h>
|
|
#include <asm/icc.h>
|
|
|
|
#include <net/mmnif.h>
|
|
|
|
|
|
bootinfo_t* bootinfo = (bootinfo_t*) SCC_BOOTINFO;
|
|
static int num_ues, my_ue;
|
|
|
|
/* PSE bit for Pentium+ equals MPE (message buffer enable) flag in RCK! So, use it to create _PAGE_MPB symbol... */
|
|
#define _CR4_MPE 0x00000800
|
|
|
|
/* maximal number of SCC's cores */
|
|
#define MAX_SCC_CORES (NUM_ROWS*NUM_COLS*NUM_CORES)
|
|
|
|
/*
|
|
* This is the modified MPB program, which is part of the RCCE distribution (src/mpb.c).
|
|
*
|
|
* This function clears the local MPB and resets the test&set register.
|
|
*/
|
|
static int scc_clear(void)
|
|
{
|
|
int tmp, x, y, z, offset;
|
|
|
|
// Initialize API
|
|
InitAPI(0);
|
|
|
|
// Find out who I am...
|
|
tmp=ReadConfigReg(CRB_OWN+MYTILEID);
|
|
x=(tmp>>3) & 0x0f; // bits 06:03
|
|
y=(tmp>>7) & 0x0f; // bits 10:07
|
|
z=(tmp ) & 0x07; // bits 02:00
|
|
|
|
// Allocate Message Passing Buffer
|
|
t_vcharp MPB;
|
|
MPBalloc(&MPB, x, y, z, 1);
|
|
if (!MPB) {
|
|
kprintf("Unable to allocate MPB for core %d of Tile x=%d, y= %d! Exiting.\n", z, x, y);
|
|
return 255;
|
|
}
|
|
|
|
// zap own MPB
|
|
for (offset=0; offset < 0x2000; offset+=8)
|
|
*(volatile unsigned long long int*)(MPB+offset) = 0;
|
|
|
|
// Clear test&set register write. Next read-access will read "1" (lock granted).
|
|
SetConfigReg(CRB_ADDR(x,y)+((z)?LOCK1:LOCK0), 1);
|
|
|
|
// frees Message Passing Buffer
|
|
MPBunalloc(&MPB);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void intr_handler(struct state *s)
|
|
{
|
|
// reset appropriate bit in the core configuration register
|
|
int tmp, z;
|
|
|
|
#ifdef CONFIG_LWIP
|
|
mmnif_irqhandler();
|
|
#endif
|
|
|
|
z = Z_PID(RC_COREID[my_ue]);
|
|
tmp=ReadConfigReg(CRB_OWN + (z==0 ? GLCFG0 : GLCFG1));
|
|
tmp &= ~2;
|
|
SetConfigReg(CRB_OWN + (z==0 ? GLCFG0 : GLCFG1), tmp);
|
|
}
|
|
|
|
int icc_init(void)
|
|
{
|
|
int i, z, tmp;
|
|
uint64_t start, end, ticks, freq = 533;
|
|
uint32_t cr4, msg = 0;
|
|
|
|
kputs("Initialize Rock Creek!\n");
|
|
|
|
/* Enable Messagepassing in CR4 */
|
|
cr4 = read_cr4();
|
|
cr4 = cr4 | _CR4_MPE;
|
|
write_cr4(cr4);
|
|
|
|
kprintf("address of the initrd: 0x%x\n", bootinfo->addr);
|
|
kprintf("size of the initrd: %d\n", bootinfo->size);
|
|
kprintf("rcce argc = %d\n", bootinfo->argc);
|
|
for(i=0; i<bootinfo->argc; i++)
|
|
kprintf("rcce argv[%d] = %s\n", i, bootinfo->argv[i]);
|
|
|
|
if (bootinfo->argc >= 3)
|
|
freq = atoi(bootinfo->argv[2]);
|
|
|
|
kputs("Reset SCC!\n");
|
|
scc_clear();
|
|
kputs("Wait some time...\n");
|
|
mb();
|
|
start = rdtsc();
|
|
do {
|
|
mb();
|
|
end = rdtsc();
|
|
ticks = end > start ? end - start : start - end;
|
|
} while(ticks*TIMER_FREQ < 300ULL*freq*1000000ULL);
|
|
|
|
if (RCCE_init(&bootinfo->argc, &bootinfo->argv) != RCCE_SUCCESS)
|
|
return -ENODEV;
|
|
if (iRCCE_init() != iRCCE_SUCCESS)
|
|
return -ENODEV;
|
|
|
|
// enable additional outputs
|
|
RCCE_debug_set(RCCE_DEBUG_ALL);
|
|
|
|
my_ue = RCCE_ue();
|
|
num_ues = RCCE_num_ues();
|
|
kprintf("Got rank %d of %d ranks\n", my_ue, num_ues);
|
|
|
|
RCCE_barrier(&RCCE_COMM_WORLD);
|
|
|
|
#if 0
|
|
kputs("RCCE test...\t");
|
|
if (my_ue == 0)
|
|
msg = 0x4711;
|
|
if (RCCE_bcast((char*) &msg, sizeof(msg), 0, RCCE_COMM_WORLD) == RCCE_SUCCESS)
|
|
kprintf("successfull! (0x%x)\n", msg);
|
|
else
|
|
kprintf("failed! (0x%x)\n", msg);
|
|
#endif
|
|
|
|
#if 0
|
|
char* str = RCCE_shmalloc(128);
|
|
if (my_ue == 1) {
|
|
memset(str, 0x00, 128);
|
|
strcpy(str, "Hello RCCE_shmalloc\n");
|
|
}
|
|
RCCE_barrier(&RCCE_COMM_WORLD);
|
|
kprintf("RCCE_shmalloc test: %s\n", str);
|
|
RCCE_shfree(str);
|
|
#endif
|
|
|
|
// reset INTR/LINT0 flag
|
|
z = Z_PID(RC_COREID[my_ue]);
|
|
tmp=ReadConfigReg(CRB_OWN + (z==0 ? GLCFG0 : GLCFG1));
|
|
tmp &= ~2;
|
|
SetConfigReg(CRB_OWN + (z==0 ? GLCFG0 : GLCFG1), tmp);
|
|
|
|
// set interrupt handler (INTR/LINT0)
|
|
irq_install_handler(124, intr_handler);
|
|
|
|
kputs("Now, the SCC is initialized!\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int icc_send_irq(int ue)
|
|
{
|
|
int tmp, x, y, z, addr;
|
|
|
|
z = Z_PID(RC_COREID[ue]);
|
|
x = X_PID(RC_COREID[ue]);
|
|
y = Y_PID(RC_COREID[ue]);
|
|
addr = CRB_ADDR(x,y) + (z==0 ? GLCFG0 : GLCFG1);
|
|
|
|
// send interrupt to ue
|
|
do {
|
|
NOP1;
|
|
tmp=ReadConfigReg(addr);
|
|
} while(tmp & 2);
|
|
tmp |= 2;
|
|
SetConfigReg(addr, tmp);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int icc_halt(void)
|
|
{
|
|
uint32_t flags;
|
|
#if 0
|
|
uint32_t do_send = 1;
|
|
|
|
do {
|
|
// iRCCE is not thread save => disable interrupts
|
|
flags = irq_nested_disable();
|
|
|
|
if (do_send) {
|
|
do_send = (iRCCE_isend_push() == iRCCE_PENDING);
|
|
iRCCE_irecv_push();
|
|
}
|
|
icc_mail_check();
|
|
|
|
irq_nested_enable(flags);
|
|
|
|
NOP1;
|
|
} while(do_send);
|
|
#else
|
|
// iRCCE is not thread save => disable interrupts
|
|
flags = irq_nested_disable();
|
|
icc_mail_check();
|
|
irq_nested_enable(flags);
|
|
NOP1;
|
|
#endif
|
|
//HALT;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int icc_mail_ping( void )
|
|
{
|
|
uint32_t flags;
|
|
int remote_rank = (my_ue+1)%2;
|
|
uint8_t payload[iRCCE_MAIL_HEADER_PAYLOAD];
|
|
uint64_t* timer = (uint64_t*) payload;
|
|
|
|
if (my_ue)
|
|
return -1;
|
|
|
|
kprintf( "Hello from mail_ping ... \n" );
|
|
|
|
// disable interrupts
|
|
flags = irq_nested_disable();
|
|
|
|
// start timer
|
|
*timer = rdtsc();
|
|
|
|
/* send ping request */
|
|
iRCCE_mail_send(sizeof(uint64_t), ICC_TAG_PINGREQUEST, 0, payload, remote_rank);
|
|
|
|
NOP8;
|
|
icc_send_irq(remote_rank);
|
|
|
|
/* check for incoming messages */
|
|
icc_mail_check();
|
|
|
|
// enable interrupts
|
|
irq_nested_enable(flags);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void icc_mail_check(void)
|
|
{
|
|
iRCCE_MAIL_HEADER* header = NULL;
|
|
int res;
|
|
uint64_t timer;
|
|
//char* recv_buffer;
|
|
|
|
// empty mailbox and interpret headers
|
|
while( (res = iRCCE_mail_recv( &header )) == iRCCE_SUCCESS ) {
|
|
switch(header->tag)
|
|
{
|
|
case ICC_TAG_PINGREQUEST:
|
|
iRCCE_mail_send( header->size, ICC_TAG_PINGRESPONSE, 0, header->payload, header->source );
|
|
NOP8;
|
|
icc_send_irq( header->source );
|
|
break;
|
|
case ICC_TAG_PINGRESPONSE:
|
|
timer = rdtsc() - *((uint64_t*) header->payload);
|
|
kprintf( "Response received in %d ticks!\n", timer );
|
|
break;
|
|
default:
|
|
kprintf("Invalid mail: tag = %d\n", header->tag);
|
|
break;
|
|
}
|
|
/*else if( header->tag == iRCCE_ANYLENGTH ) {
|
|
recv_buffer = (char*)kmalloc( header->size );
|
|
iRCCE_irecv( recv_buffer, header->size, header->source, NULL );
|
|
iRCCE_mail_send( 0, 2, 0, NULL, header->source );
|
|
}
|
|
else if( header->tag == iRCCE_ANYLENGTH_PIGGYBACK ) {
|
|
iRCCE_mail_send( 0, 2, 0, NULL, header->source );
|
|
}*/
|
|
|
|
iRCCE_mail_release( &header );
|
|
}
|
|
}
|
|
|
|
#endif
|