metalsvm/arch/x86/kernel/apic.c

/* 
 * Copyright 2010 Stefan Lankes, Chair for Operating Systems,
 *                               RWTH Aachen University
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * This file is part of MetalSVM.
 */

#include <metalsvm/stddef.h>
#include <metalsvm/stdio.h>
#include <metalsvm/stdlib.h>
#include <metalsvm/string.h>
#include <metalsvm/errno.h>
#include <metalsvm/processor.h>
#include <metalsvm/time.h>
#include <metalsvm/init.h>
#include <metalsvm/page.h>
#include <metalsvm/spinlock.h>
#include <metalsvm/mmu.h>
#include <asm/irq.h>
#include <asm/irqflags.h>
#include <asm/apic.h>
#include <asm/multiboot.h>
#ifdef CONFIG_ROCKCREEK
#include <asm/RCCE_lib.h>
#endif

/* disable optimization for the following functions */
//static int apic_send_ipi(uint32_t id, uint32_t mode, uint32_t vector) __attribute__((optimize(0)));
static int wakeup_all_aps(uint32_t start_eip)  __attribute__((optimize(0)));
//int apic_calibration(void) __attribute__((optimize(0)));
//int ioapic_intoff(uint8_t irq, uint8_t apicid) __attribute__((optimize(0)));
//int ioapic_inton(uint8_t irq, uint8_t apicid)  __attribute__((optimize(0)));

// IO APIC MMIO structure: write reg, then read or write data.
typedef struct {
	uint32_t reg;
	uint32_t pad[3];
	uint32_t data;
} ioapic_t;

static const apic_processor_entry_t* apic_processors[MAX_CORES] = {[0 ... MAX_CORES-1] = NULL};
static uint32_t boot_processor = MAX_CORES;
static apic_mp_t* apic_mp = NULL;
static apic_config_table_t* apic_config = NULL;
static uint32_t lapic = 0;
static volatile ioapic_t* ioapic = NULL;
static uint32_t ncores = 1;
static uint8_t irq_redirect[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF};
#if MAX_CORES > 1
static uint8_t boot_code[] = {0xE9, 0x1E, 0x00, 0x17, 0x00, 0x09, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x9A, 0xCF, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x92, 0xCF, 0x00, 0x0F, 0x01, 0x16, 0x03, 0x00, 0x0F, 0x20, 0xC0, 0x0C, 0x01, 0x0F, 0x22, 0xC0, 0x66, 0xEA, 0x36, 0x00, 0x01, 0x00, 0x08, 0x00, 0xFA, 0x31, 0xC0, 0x66, 0xB8, 0x10, 0x00, 0x8E, 0xD8, 0x8E, 0xC0, 0x8E, 0xE0, 0x8E, 0xE8, 0x8E, 0xD0, 0x0F, 0x20, 0xC0, 0x25, 0xFF, 0xFF, 0xFF, 0x9F, 0x0D, 0x20, 0x00, 0x00, 0x00, 0x0F, 0x22, 0xC0, 0x31, 0xC0, 0x0F, 0x22, 0xD8, 0xBC, 0xEF, 0xBE, 0xAD, 0xDE, 0x31, 0xC0, 0x31, 0xDB, 0xEA, 0xDE, 0xC0, 0xAD, 0xDE, 0x08, 0x00};
#endif
static uint8_t initialized = 0;
static atomic_int32_t cpu_online = ATOMIC_INIT(1);
spinlock_t bootlock = SPINLOCK_INIT;

static inline uint32_t lapic_read(uint32_t addr)
{
	return *((volatile uint32_t*) (lapic+addr));
}

static inline void lapic_write(uint32_t addr, uint32_t value)
{
	/* 
	 * to avoid a pentium bug, we have to read a apic register 
	 * before we write a value to this register
	 */
	asm volatile ("cmpl (%0), %%eax; movl %1, (%0)" :: "r"(lapic+addr), "r"(value));
	//asm volatile ("movl (%%eax), %%edx; movl %%ebx, (%%eax)" :: "a"(lapic+addr), "b"(value) : "%edx");
	//*((volatile uint32_t*) (lapic+addr)) = value;
}

static inline uint32_t ioapic_read(uint32_t reg)
{
	ioapic->reg = reg;

	return ioapic->data;
}

static inline void ioapic_write(uint32_t reg, uint32_t value)
{
	ioapic->reg = reg;
	ioapic->data = value;
}

/*
 * Send a 'End of Interrupt' command to the APIC
 */
void apic_eoi(void)
{
	if (BUILTIN_EXPECT(lapic, 1))
		lapic_write(APIC_EOI, 0);
}

uint32_t apic_cpu_id(void)
{
	if (lapic && initialized)
		return ((lapic_read(APIC_ID)) >> 24);
	return 0;
}

static inline uint32_t apic_version(void)
{
	if (lapic)
		return lapic_read(APIC_VERSION) & 0xFF;

	return 0;
}

static inline uint32_t apic_lvt_entries(void)
{
	if (lapic)
		return (lapic_read(APIC_VERSION) >> 16) & 0xFF;

	return 0;
}

int has_apic(void)
{
	return (lapic != 0);
}

int apic_is_enabled(void)
{
	return (lapic && initialized);
}

#if MAX_CORES > 1
#if 0
static int apic_send_ipi(uint32_t id, uint32_t mode, uint32_t vector)
{
        uint32_t i = 0;

	if(lapic_read(APIC_ICR1) & APIC_ICR_BUSY) {
		kprintf("ERROR: previous send not complete");
		return -EIO;
	}

	/* set destination and data */
	lapic_write(APIC_ICR2, (id << 24));
	lapic_write(APIC_ICR1, APIC_INT_ASSERT|mode|vector);

        while((lapic_read(APIC_ICR1) & APIC_ICR_BUSY) && (i < 1000))
		i++; // wait for it to finish, give up eventualy tho

	return ((lapic_read(APIC_ICR1) & APIC_ICR_BUSY) ? -EIO : 0); // did it fail (still delivering) or succeed ?
}
#endif

static int wakeup_all_aps(uint32_t start_eip)
{
	uint32_t i;

        kputs("Wakeup all application processors via IPI\n");

	if(lapic_read(APIC_ICR1) & APIC_ICR_BUSY) {
                kprintf("ERROR: previous send not complete");
                return -EIO;
        }

	vga_puts("Send IPI\n");
	// send out INIT to all aps
	lapic_write(APIC_ICR1, APIC_INT_ASSERT|APIC_DEST_ALLBUT|APIC_DM_INIT);
	udelay(10000);
	// send out the startup
	lapic_write(APIC_ICR1, APIC_INT_ASSERT|APIC_DEST_ALLBUT|APIC_DM_STARTUP|(start_eip >> 12));
	udelay(200);
	// do it again
	lapic_write(APIC_ICR1, APIC_INT_ASSERT|APIC_DEST_ALLBUT|APIC_DM_STARTUP|(start_eip >> 12));
	udelay(200);
	vga_puts("IPI done...\n");

	i = 0;
	while((lapic_read(APIC_ICR1) & APIC_ICR_BUSY) && (i < 1000))
		i++; // wait for it to finish, give up eventualy tho

	return ((lapic_read(APIC_ICR1) & APIC_ICR_BUSY) ? -EIO : 0); // did it fail (still delivering) or succeed ?
}
#endif

#if MAX_CORES > 1
static void smp_main(void)
{
	vga_puts("JJAJAJAJAJAJA\n");
	lowlevel_init();
	atomic_int32_inc(&cpu_online);
	kputs("JAJAJAJ\n");
	while(1) ;
}
#endif

#ifndef CONFIG_MULTIBOOT
static unsigned int* search_apic(unsigned int base, unsigned int limit) {
	uint32_t* ptr;

	for (ptr = (uint32_t*) base; (uint32_t) ptr < limit; ptr++) {
		if (*ptr == MP_FLT_SIGNATURE) {
			if (!(((apic_mp_t*)ptr)->version > 4) && ((apic_mp_t*)ptr)->features[0])
				return ptr;
		}
        }

	return NULL;
}
#endif

#if MAX_CORES > 1
int smp_init(void)
{
	uint32_t i;
	size_t bootaddr;
	int err;

	if (ncores <= 1)
		return -EINVAL;

	/*
	 * dirty hack: Copy 16bit startup code (see tools/smp_setup.asm)
	 * to a 16bit address. Wakeup the other cores via IPI. They start
         * at this address in real mode, switch to protected and finally
	 * they jump to smp_main.
	 */
	bootaddr = 0x10000;
	map_region(bootaddr, get_pages(1), 1, MAP_KERNEL_SPACE);
	for(i=0; i<sizeof(boot_code); i+=sizeof(size_t))
	{
		// replace 0xDEADC0DE with the address of the smp entry code
		if (*((size_t*) (bootaddr+i)) == 0xDEADC0DE)
			*((size_t*) (bootaddr+i)) = (size_t) smp_main;
	}

	kprintf("size of the boot_code %d\n", sizeof(boot_code));
	err = wakeup_all_aps(bootaddr);
	if (err)
		kprintf("Unable to wakeup application processors: %d\n", err);

	i = 0;
	while((ncores != atomic_int32_read(&cpu_online)) && (i < 1000))
		i++;

	kprintf("%d cores online\n", atomic_int32_read(&cpu_online));

	return 0;
}
#endif

/* 
 * detects the timer frequency of the APIC and restart
 * the APIC timer with the correct period
 */
int apic_calibration(void)
{
	uint8_t i;
	uint32_t flags;

#ifndef CONFIG_ROCKCREEK
	uint64_t ticks, old;
	uint32_t diff;

	if (!has_apic())
		return -ENXIO;

	lapic = map_region(0 /*lapic*/, lapic, 1, MAP_KERNEL_SPACE|MAP_NO_CACHE);
	if (BUILTIN_EXPECT(!lapic, 0))
		return -ENXIO;

	if (ioapic)
		ioapic = (ioapic_t*) map_region(0 /*(size_t)ioapic*/, (size_t) ioapic, 1, MAP_KERNEL_SPACE|MAP_NO_CACHE);

	old = get_clock_tick();

	/* wait for the next time slice */
	while((ticks = get_clock_tick()) - old == 0)
		;
	
	lapic_write(APIC_DCR, 0xB);		// set it to 1 clock increments
	lapic_write(APIC_LVT_T, 0x2007B);	// connects the timer to 123 and enables it
	lapic_write(APIC_ICR, 0xFFFFFFFFUL);

	/* wait 3 time slices to determine a ICR */
	while(get_clock_tick() - ticks < 3)
		;

	diff = 0xFFFFFFFFUL - lapic_read(APIC_CCR);
	diff = diff / 3;

	lapic_write(APIC_DCR, 0xB);		// set it to 1 clock increments
	lapic_write(APIC_LVT_T, 0x2007B);	// connects the timer to 123 and enables it
	lapic_write(APIC_ICR, diff);

	// Now, MetalSVM is able to use the APIC => Therefore, we disable the PIC
	outportb(0xA1, 0xFF);
	outportb(0x21, 0xFF);
#else
	/* 
	 * On the SCC, we already know the processor frequency
	 * and possess no PIC timer. Therfore, we use the rdtsc to
	 * to calibrate the APIC timer.
	 */
	uint64_t start, end, ticks;
	uint32_t diff;

	if (!has_apic())
		return -ENXIO;

	lapic = map_region(0 /*lapic*/, lapic, 1, MAP_KERNEL_SPACE|MAP_NO_CACHE);
	if (BUILTIN_EXPECT(!lapic, 0))
		return -ENXIO;

	if (ioapic) 
		ioapic = (ioapic_t*) map_region(0 /*(size_t)ioapic*/, (size_t)ioapic, 1, MAP_KERNEL_SPACE|MAP_NO_CACHE);

	lapic_write(APIC_DCR, 0xB);		// set it to 1 clock increments
	lapic_write(APIC_LVT_T, 0x2007B);	// connects the timer to 123 and enables it
	lapic_write(APIC_ICR, 0xFFFFFFFFUL);
	
	/* wait 3 time slices to determine a ICR */
	start = rdtsc();
	do {
		flush_pipeline();
		end = rdtsc();
		ticks = end > start ? end - start : start - end;
	} while(ticks*TIMER_FREQ < 3*RC_REFCLOCKMHZ*1000000UL);

	diff = 0xFFFFFFFFUL - lapic_read(APIC_CCR);
	diff = diff / 3;

	lapic_write(APIC_DCR, 0xB);		// set it to 1 clock increments
	lapic_write(APIC_LVT_T, 0x2007B);	// connects the timer to 123 and enables it
	lapic_write(APIC_ICR, diff);
#endif

	kprintf("APIC calibration determines an ICR of 0x%x\n", diff);

	flags = irq_nested_disable();
#if MAX_CORES > 1
	//smp_init();
#endif

	if (ioapic) {
		// now, we don't longer need the IOAPIC timer and turn it off
		ioapic_intoff(0, apic_processors[boot_processor]->id);
		// now lets turn everything else on
		for(i=1; i<24; i++) 
			ioapic_inton(i, apic_processors[boot_processor]->id);
	}
	initialized = 1;
	irq_nested_enable(flags);

	return 0;
}

static int apic_probe(void)
{
	size_t addr;
	uint32_t i, count;
	int isa_bus = -1;

	// searching MP signature in the reserved memory areas
#ifdef CONFIG_MULTIBOOT
	if (mb_info && (mb_info->flags & (1 << 6))) {
 		multiboot_memory_map_t* mmap = (multiboot_memory_map_t*) mb_info->mmap_addr;
		multiboot_memory_map_t* mmap_end = (void*) ((size_t) mb_info->mmap_addr + mb_info->mmap_length);

		while (mmap < mmap_end) {
			if (mmap->type == MULTIBOOT_MEMORY_RESERVED) {
				addr = mmap->addr;

				for(i=0; i<mmap->len; i++, addr++) {
					if (*((uint32_t*) addr) == MP_FLT_SIGNATURE) {
						apic_mp = (apic_mp_t*) addr;
						if (!(apic_mp->version > 4) && apic_mp->features[0])
							goto found_mp;
					}
				}
				
			}
			mmap++;
		}
	}
found_mp:
#else
	apic_mp = (apic_mp_t*) search_apic(0xF0000, 0x100000);
	if (!apic_mp)
		apic_mp = (apic_mp_t*) search_apic(0x9F000, 0xA0000);
#endif

	if (!apic_mp) 
		goto no_mp;

	kprintf("System uses Multiprocessing Specification 1.%u\n", apic_mp->version);	
	kprintf("MP features 1: %u\n", apic_mp->features[0]);

	if (apic_mp->features[0]) {
		kputs("Currently, MetalSVM supports only multiprocessing via the MP config tables!\n");
		goto no_mp;
	}

	apic_config = (apic_config_table_t*) apic_mp->mp_config;
	if (!apic_config || strncmp((void*) &apic_config->signature, "PCMP", 4) !=0) {
		kputs("Invalid MP config table\n");
		goto no_mp;
	}

	addr = (size_t) apic_config;
	addr += sizeof(apic_config_table_t);
	if (addr % 4)
		addr += 4 - addr % 4;

	// search the ISA bus => required to redirect the IRQs
	for(i=0; i<apic_config->entry_count; i++) {
		switch(*((uint8_t*) addr)) {
		case 0: 
			addr += 20;
			break;
		case 1: {
				apic_bus_entry_t* mp_bus;

				mp_bus = (apic_bus_entry_t*) addr;
				if (mp_bus->name[0] == 'I' && mp_bus->name[1] == 'S' &&
				    mp_bus->name[2] == 'A')
					isa_bus = i;
			}
		default:
			addr += 8;
		}
	}

	addr = (size_t) apic_config;
	addr += sizeof(apic_config_table_t);
	if (addr % 4)
		addr += 4 - addr % 4;

	for(i=0, count=0; i<apic_config->entry_count; i++) {
		if (*((uint8_t*) addr) == 0) { // cpu entry
			if (i < MAX_CORES) {
				apic_processors[i] = (apic_processor_entry_t*) addr;
				if (!(apic_processors[i]->cpu_flags & 0x01)) // is the processor usable?
					apic_processors[i] = NULL;
				else if (apic_processors[i]->cpu_flags & 0x02)
					boot_processor = i;
			}
			count++;
			addr += 20;
		} else if (*((uint8_t*) addr) == 2) { // IO_APIC
			apic_io_entry_t* io_entry = (apic_io_entry_t*) addr;
			ioapic = (ioapic_t*) io_entry->addr;
			addr += 8;
			kprintf("Found IOAPIC at 0x%x (ver. 0x%x)\n", ioapic,
				ioapic_read(IOAPIC_REG_VER)); 
		} else if (*((uint8_t*) addr) == 3) { // IO_INT
			apic_ioirq_entry_t* extint = (apic_ioirq_entry_t*) addr;
                        if (extint->src_bus == isa_bus) {
                                irq_redirect[extint->src_irq] = extint->dest_intin;
				kprintf("Redirect irq %u -> %u\n", extint->src_irq,  extint->dest_intin);
			}
			addr += 8;
		} else addr += 8;
	}
	kprintf("Found %u cores\n", count);

	if (count > MAX_CORES) {
		kputs("Found too many cores! Increase the macro MAX_CORES!\n");
		goto no_mp;
	}
	ncores = count;

check_lapic:
	if (apic_config) {
		lapic = apic_config->lapic;
	} else {
		uint32_t edx, dummy;

		cpuid(0x1, &dummy, &dummy, &dummy, &edx);
		if (edx & (1 << 9))
			lapic = 0xFEE00000;
	}

	if (!lapic)
		goto out;

	kprintf("Found APIC at 0x%x\n", lapic);
	kprintf("Maximum LVT Entry: 0x%x\n", apic_lvt_entries());
	kprintf("APIC Version: 0x%x\n", apic_version());

	if (!((apic_version() >> 4))) {
		kprintf("Currently, MetalSVM didn't supports extern APICs!\n");
		goto out;
	}

	if (apic_lvt_entries() < 3) {
		kprintf("LVT is too small\n");
		goto out;
	}

	return 0;

out:
	apic_mp = NULL;
	apic_config = NULL;
	lapic = 0;
	ncores = 1;
	return -ENXIO;

no_mp:
	apic_mp = NULL;
	apic_config = NULL;
	ncores = 1;
	goto check_lapic;
}

int apic_init(void)
{ 
	int ret;
	uint8_t i;
	uint32_t v;
	uint32_t max_lvt;

	ret = apic_probe();
	if (!ret)
		return ret;

	max_lvt = apic_lvt_entries();

	lapic_write(APIC_SVR, 0x17F);   // enable the apic and connect to the idt entry 127
	lapic_write(APIC_TPR, 0x00);	// allow all interrupts
	lapic_write(APIC_LVT_T, 0x10000);  // disable timer interrupt
	if (max_lvt >= 4)
		lapic_write(APIC_LVT_TSR, 0x10000); // disable thermal sensor interrupt
	if (max_lvt >= 5)
		lapic_write(APIC_LVT_PMC, 0x10000); // disable performance counter interrupt
	lapic_write(APIC_LINT0, 0x7C);	// connect LINT0 to idt entry 124
	lapic_write(APIC_LINT1, 0x7D);	// connect LINT1 to idt entry 125
	lapic_write(APIC_LVT_ER, 0x7E);	// connect error to idt entry 126

	if (0) { //ioapic) {
		// enable timer interrupt
		ioapic_inton(0, apic_processors[boot_processor]->id);
		// now lets turn everything else off
		for(i=1; i<24; i++)
			ioapic_intoff(i, apic_processors[boot_processor]->id);
	}

	return 0;
}

int ioapic_inton(uint8_t irq, uint8_t apicid)
{
	ioapic_route_t route;
	uint32_t off;

	if (BUILTIN_EXPECT(irq > 24, 0)){
		kprintf("IOAPIC: trying to turn on irq %i which is too high\n", irq);
		return -EINVAL;
	}

	if (irq < 16)
		off = irq_redirect[irq]*2;
	else
		off = irq*2;
#if 0
	route.lower.whole = ioapic_read(IOAPIC_REG_TABLE+1+off);
	route.dest.upper = ioapic_read(IOAPIC_REG_TABLE+off);
	route.lower.bitfield.mask = 0; // turn it on (stop masking)
#else
	route.lower.bitfield.dest_mode = 0;
	route.lower.bitfield.mask = 0;
	route.dest.physical.physical_dest = apicid; // send to the boot processor
	route.lower.bitfield.delivery_mode = 0;
	route.lower.bitfield.polarity = 0;
	route.lower.bitfield.trigger = 0;
	route.lower.bitfield.vector = 0x20+irq;
	route.lower.bitfield.mask = 0; // turn it on (stop masking)
#endif

	ioapic_write(IOAPIC_REG_TABLE+off, route.lower.whole);
	ioapic_write(IOAPIC_REG_TABLE+1+off, route.dest.upper);

	route.dest.upper = ioapic_read(IOAPIC_REG_TABLE+1+off);
        route.lower.whole = ioapic_read(IOAPIC_REG_TABLE+off);

	return 0;
}

int ioapic_intoff(uint8_t irq, uint8_t apicid)
{
	ioapic_route_t route;
	uint32_t off;

	if (BUILTIN_EXPECT(irq > 24, 0)){
		kprintf("IOAPIC: trying to turn on irq %i which is too high\n", irq);
		return -EINVAL;
	}

	if (irq < 16) 
		off = irq_redirect[irq]*2;
	else
		off = irq*2;

#if 0
	route.lower.whole = ioapic_read(IOAPIC_REG_TABLE+1+off);
	route.dest.upper = ioapic_read(IOAPIC_REG_TABLE+off);
	route.lower.bitfield.mask = 1; // turn it off (start masking)
#else
	route.lower.bitfield.dest_mode = 0;
	route.lower.bitfield.mask = 0;
	route.dest.physical.physical_dest = apicid;
	route.lower.bitfield.delivery_mode = 0;
	route.lower.bitfield.polarity = 0;
	route.lower.bitfield.trigger = 0;
	route.lower.bitfield.vector = 0x20+irq;
	route.lower.bitfield.mask = 1; // turn it off (start masking)
#endif

	ioapic_write(IOAPIC_REG_TABLE+off, route.lower.whole);
	ioapic_write(IOAPIC_REG_TABLE+1+off, route.dest.upper);

	return 0;
}