diff --git a/fpga/CMakeLists.txt b/fpga/CMakeLists.txt index 917ce45b2..58481fbf2 100644 --- a/fpga/CMakeLists.txt +++ b/fpga/CMakeLists.txt @@ -12,6 +12,7 @@ include_directories(thirdparty/CLI11) include_directories(thirdparty/rang) add_subdirectory(lib) +add_subdirectory(src) add_subdirectory(tests) # Project settings @@ -56,4 +57,4 @@ set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") set(CPACK_PACKAGE_FILE_NAME "${CPACK_SOURCE_PACKAGE_FILE_NAME}-${CPACK_RPM_PACKAGE_RELEASE}.${CPACK_RPM_PACKAGE_ARCHITECTURE}") set(CPACK_GENERATOR "RPM") -include(CPack) \ No newline at end of file +include(CPack) diff --git a/fpga/src/CMakeLists.txt b/fpga/src/CMakeLists.txt index fbca7c72a..0f6488b7f 100644 --- a/fpga/src/CMakeLists.txt +++ b/fpga/src/CMakeLists.txt @@ -1,5 +1,7 @@ add_executable(streamer streamer.cpp) +add_executable(pcimem pcimem.c) + target_link_libraries(streamer PUBLIC villas-fpga ) diff --git a/fpga/src/README.pcimem.md b/fpga/src/README.pcimem.md new file mode 100644 index 000000000..2be13d22d --- /dev/null +++ b/fpga/src/README.pcimem.md @@ -0,0 +1,133 @@ +== Overview == + +The pcimem application provides a simple method of reading and writing +to memory registers on a PCI card. + +Usage: ./pcimem { sys file } { offset } [ type [ data ] ] + sys file: sysfs file for the pci resource to act on + offset : offset into pci memory region to act upon + type : access operation type : [b]yte, [h]alfword, [w]ord + data : data to be written + +== Platform Support == + +WARNING !! This method is platform dependent and may not work on your +particular target architecture. Refer to the PowerPC section below. + +== Example == + +bash# ./pcimem /sys/devices/pci0001\:00/0001\:00\:07.0/resource0 0 w + /sys/devices/pci0001:00/0001:00:07.0/resource0 opened. + Target offset is 0x0, page size is 4096 + mmap(0, 4096, 0x3, 0x1, 3, 0x0) + PCI Memory mapped to address 0x4801f000. + Value at offset 0x0 (0x4801f000): 0xC0BE0100 + + +== Why do this at all ? == + +When I start working on a new PCI device driver I generally go through a +discovery phase of reading and writing to certain registers on the PCI card. +Over the years I have written lots of small kernel modules to probe addresses +within the PCI memory space, constantly iterating: modify code, recompile, scp +to target, load module, unload module, dmesg. + +Urk! There has to be a better way - sysfs and mmap() to the rescue. + + +== Sysfs == + +Let's start at with the PCI files under sysfs: + +bash# ls -l /sys/devices/pci0001\:00/0001\:00\:07.0/ +total 0 +-rw-r--r-- 1 root root     4096 Jul  2 20:13 broken_parity_status +lrwxrwxrwx 1 root root        0 Jul  2 20:13 bus -> ../../../bus/pci +-r--r--r-- 1 root root     4096 Jul  2 20:13 class +-rw-r--r-- 1 root root      256 Jul  2 20:13 config +-r--r--r-- 1 root root     4096 Jul  2 20:13 device +-r--r--r-- 1 root root     4096 Jul  2 20:13 devspec +-rw------- 1 root root     4096 Jul  2 20:13 enable +-r--r--r-- 1 root root     4096 Jul  2 20:13 irq +-r--r--r-- 1 root root     4096 Jul  2 20:13 local_cpus +-r--r--r-- 1 root root     4096 Jul  2 20:13 modalias +-rw-r--r-- 1 root root     4096 Jul  2 20:13 msi_bus +-r--r--r-- 1 root root     4096 Jul  2 20:13 resource +-rw------- 1 root root     4096 Jul  2 20:13 resource0 +-rw------- 1 root root    65536 Jul  2 20:13 resource1 +-rw------- 1 root root 16777216 Jul  2 20:13 resource2 +lrwxrwxrwx 1 root root        0 Jul  2 20:13 subsystem -> ../../../bus/pci +-r--r--r-- 1 root root     4096 Jul  2 20:13 subsystem_device +-r--r--r-- 1 root root     4096 Jul  2 20:13 subsystem_vendor +-rw-r--r-- 1 root root     4096 Jul  2 20:13 uevent +-r--r--r-- 1 root root     4096 Jul  2 20:13 vendor + +The vendor and device files report the PCI vendor ID and device ID: + +bash# cat device +0x0001 + +This info is also available from lspci + +bash# lspci -v +0001:00:07.0 Class 0680: Unknown device bec0:0001 (rev 01) +    Flags: bus master, 66MHz, medium devsel, latency 128, IRQ 31 +    Memory at 8d010000 (32-bit, non-prefetchable) [size=4K] +    Memory at 8d000000 (32-bit, non-prefetchable) [size=64K] +    Memory at 8c000000 (32-bit, non-prefetchable) [size=16M] + +This PCI card makes 3 seperate regions of memory available to the host +computer. The sysfs resource0 file corresponds to the first memory region. The +PCI card lets the host computer know about these memory regions using the BAR +registers in the PCI config. + +== mmap() == + +These sysfs resource can be used with mmap() to map the PCI memory into a +userspace applications memory space.  The application then has a pointer to the +start of the PCI memory region and can read and write values directly. (There +is a bit more going on here with respect to memory pointers, but that is all +taken care of by the kernel). + +fd = open("/sys/devices/pci0001\:00/0001\:00\:07.0/resource0", O_RDWR | O_SYNC); +ptr = mmap(0, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); +printf("PCI BAR0 0x0000 = 0x%4x\n",  *((unsigned short *) ptr); + +== PowerPC == + +To make this work on a PowerPC architecture you also need to make a small +change to the pci core. My example is from kernel 2.6.34, and hopefully this +will be fixed for us in a later kernel version. + +bash# vi arch/powerpc/kernel/pci-common.c +    /* If memory, add on the PCI bridge address offset */ +     if (mmap_state == pci_mmap_mem) { +-#if 0 /* See comment in pci_resource_to_user() for why this is disabled */ ++#if 1 /* See comment in pci_resource_to_user() for why this is disabled */ +         *offset += hose->pci_mem_offset; + #endif +         res_bit = IORESOURCE_MEM; +  +         /* We pass a fully fixed up address to userland for MMIO instead of +         * a BAR value because X is lame and expects to be able to use that +         * to pass to /dev/mem ! +         * +         * That means that we'll have potentially 64 bits values where some +         * userland apps only expect 32 (like X itself since it thinks only +         * Sparc has 64 bits MMIO) but if we don't do that, we break it on +         * 32 bits CHRPs :-( +         * +         * Hopefully, the sysfs insterface is immune to that gunk. Once X +         * has been fixed (and the fix spread enough), we can re-enable the +         * 2 lines below and pass down a BAR value to userland. In that case +         * we'll also have to re-enable the matching code in +         * __pci_mmap_make_offset(). +         * +         * BenH. +         */ +-#if 0 ++#if 1 +        else if (rsrc->flags & IORESOURCE_MEM) +                offset = hose->pci_mem_offset; +#endif + diff --git a/fpga/src/pcimem.c b/fpga/src/pcimem.c new file mode 100644 index 000000000..ca51a21ae --- /dev/null +++ b/fpga/src/pcimem.c @@ -0,0 +1,141 @@ +/* + * pcimem.c: Simple program to read/write from/to a pci device from userspace. + * + * Copyright (C) 2010, Bill Farrow (bfarrow@beyondelectronics.us) + * + * Based on the devmem2.c code + * Copyright (C) 2000, Jan-Derk Bakker (J.D.Bakker@its.tudelft.nl) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PRINT_ERROR \ + do { \ + fprintf(stderr, "Error at line %d, file %s (%d) [%s]\n", \ + __LINE__, __FILE__, errno, strerror(errno)); exit(1); \ + } while(0) + +#define MAP_SIZE 4096UL +#define MAP_MASK (MAP_SIZE - 1) + +int main(int argc, char **argv) { + int fd; + void *map_base, *virt_addr; + unsigned read_result, writeval; + char *filename; + off_t target; + int access_type = 'w'; + + if (argc < 3) { + // pcimem /sys/bus/pci/devices/0001\:00\:07.0/resource0 0x100 w 0x00 + // argv[0] [1] [2] [3] [4] + fprintf(stderr, "\nUsage:\t%s { sys file } { offset } [ type [ data ] ]\n" + "\tsys file: sysfs file for the pci resource to act on\n" + "\toffset : offset into pci memory region to act upon\n" + "\ttype : access operation type : [b]yte, [h]alfword, [w]ord\n" + "\tdata : data to be written\n\n", + argv[0]); + exit(1); + } + + filename = argv[1]; + target = strtoul(argv[2], 0, 0); + + if(argc > 3) + access_type = tolower(argv[3][0]); + + fd = open(filename, O_RDWR | O_SYNC); + if (fd < 0) + PRINT_ERROR; + + printf("%s opened.\n", filename); + printf("Target offset is %#lx, page size is %lu\n", target, sysconf(_SC_PAGE_SIZE)); + + fflush(stdout); + + /* Map one page */ + printf("mmap(%d, %lu, %#x, %#x, %d, %#lx)\n", 0, + MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, target); + + map_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, target & ~MAP_MASK); + + if(map_base == (void *) -1) + PRINT_ERROR; + + printf("PCI Memory mapped to address %p.\n", map_base); + fflush(stdout); + + virt_addr = map_base + (target & MAP_MASK); + + switch(access_type) { + case 'b': + read_result = *((unsigned char *) virt_addr); + break; + case 'h': + read_result = *((unsigned short *) virt_addr); + break; + case 'w': + read_result = *((unsigned int *) virt_addr); + break; + default: + fprintf(stderr, "Illegal data type '%c'.\n", access_type); + exit(2); + } + + printf("Value at offset %#lx (%p): %#x\n", target, virt_addr, read_result); + fflush(stdout); + + if(argc > 4) { + writeval = strtoul(argv[4], 0, 0); + switch(access_type) { + case 'b': + *((unsigned char *) virt_addr) = writeval; + read_result = *((unsigned char *) virt_addr); + break; + case 'h': + *((unsigned short *) virt_addr) = writeval; + read_result = *((unsigned short *) virt_addr); + break; + case 'w': + *((unsigned int *) virt_addr) = writeval; + read_result = *((unsigned int *) virt_addr); + break; + } + printf("Written %#x; readback %#x\n", writeval, read_result); + fflush(stdout); + } + + if(munmap(map_base, MAP_SIZE) == -1) + PRINT_ERROR; + + close(fd); + + return 0; +} +