From 6cb3b77c7a990fcf215af6bd0f2fb954dbcfdbc5 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Wed, 14 Feb 2018 14:32:07 +0100 Subject: [PATCH 01/21] ips/intc: don't fail if setting IRQ affinity is not possible This is the case when the application is not executed as root which is now possible, with the drawback that we cannot set the IRQ affinity anymore. --- fpga/lib/ips/intc.cpp | 16 ++++++++++++++-- fpga/lib/kernel/kernel.c | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index 6c841e4da..56d8f7877 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -21,6 +21,7 @@ *********************************************************************************/ #include +#include #include "config.h" #include "log.h" @@ -60,8 +61,19 @@ InterruptController::init() /* For each IRQ */ for (int i = 0; i < num_irqs; i++) { - /* Pin to core */ - if(kernel_irq_setaffinity(nos[i], card->affinity, nullptr) != 0) { + + /* Try pinning to core */ + int ret = kernel_irq_setaffinity(nos[i], card->affinity, nullptr); + + switch(ret) { + case 0: + // everything is fine + break; + case EACCES: + logger->warn("No permission to change affinity of VFIO-MSI interrupt. " + "This may degrade performance (increasing jitter)"); + break; + default: logger->error("Failed to change affinity of VFIO-MSI interrupt"); return false; } diff --git a/fpga/lib/kernel/kernel.c b/fpga/lib/kernel/kernel.c index dcac932db..a795c9f0f 100644 --- a/fpga/lib/kernel/kernel.c +++ b/fpga/lib/kernel/kernel.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -268,7 +269,7 @@ int kernel_irq_setaffinity(unsigned irq, uintmax_t affinity, uintmax_t *old) f = fopen(fn, "w+"); if (!f) - return -1; /* IRQ does not exist */ + return errno; if (old) ret = fscanf(f, "%jx", old); From 73c6ae1f71d11b56bd7c62c8702bce13637a5567 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 18:56:42 +0100 Subject: [PATCH 02/21] hwdef-parse: follow OR-gate merging DMA interrupts Also update JSON config with the new output. --- fpga/etc/fpga.json | 12 ++++++++++-- fpga/scripts/hwdef-parse.py | 33 +++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/fpga/etc/fpga.json b/fpga/etc/fpga.json index 2abcdc136..fe29ef418 100644 --- a/fpga/etc/fpga.json +++ b/fpga/etc/fpga.json @@ -147,7 +147,11 @@ "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:1", "name": "S2MM" } - ] + ], + "irqs": { + "mm2s_introut": "pcie_0_axi_pcie_intc_0:3", + "s2mm_introut": "pcie_0_axi_pcie_intc_0:4" + } }, "hier_0_axi_dma_axi_dma_1": { "vlnv": "xilinx.com:ip:axi_dma:7.1", @@ -182,7 +186,11 @@ "target": "hier_0_axis_interconnect_0_axis_interconnect_0_xbar:6", "name": "S2MM" } - ] + ], + "irqs": { + "mm2s_introut": "pcie_0_axi_pcie_intc_0:3", + "s2mm_introut": "pcie_0_axi_pcie_intc_0:4" + } }, "hier_0_axi_fifo_mm_s_0": { "vlnv": "xilinx.com:ip:axi_fifo_mm_s:4.1", diff --git a/fpga/scripts/hwdef-parse.py b/fpga/scripts/hwdef-parse.py index 7f0c57863..12d96e4f2 100755 --- a/fpga/scripts/hwdef-parse.py +++ b/fpga/scripts/hwdef-parse.py @@ -200,7 +200,7 @@ ports = concat.xpath('.//PORT[@DIR="I"]') for port in ports: name = port.get('NAME') signame = port.get('SIGNAME') - + # Skip unconnected IRQs if not signame: continue @@ -213,13 +213,34 @@ for port in ports: instance = ip.get('INSTANCE') vlnv = ip.get('VLNV') + modtype = ip.get('MODTYPE') - port = ip.xpath('.//PORT[@SIGNAME="{}" and @DIR="O"]'.format(signame))[0] - irqname = port.get('NAME') + originators = [] - if instance in ips: - irqs = ips[instance].setdefault('irqs', {}) - irqs[irqname] = '{}:{}'.format(intc.get('INSTANCE'), irq) + # follow one level of OR gates merging interrupts (may be generalized later) + if modtype == 'util_vector_logic': + logic_op = ip.xpath('.//PARAMETER[@NAME="C_OPERATION"]')[0] + if logic_op.get('VALUE') == 'or': + # hardware interrupts sharing the same IRQ at the controller + ports = ip.xpath('.//PORT[@DIR="I"]') + for port in ports: + signame = port.get('SIGNAME') + ip = root.xpath('.//MODULE[.//PORT[@SIGNAME="{}" and @DIR="O"]]'.format(signame))[0] + instance = ip.get('INSTANCE') + originators.append((instance, signame)) + else: + # consider this instance as originator + originators.append((instance, signame)) + + + for instance, signame in originators: + ip = root.xpath('.//MODULE[.//PORT[@SIGNAME="{}" and @DIR="O"]]'.format(signame))[0] + port = ip.xpath('.//PORT[@SIGNAME="{}" and @DIR="O"]'.format(signame))[0] + irqname = port.get('NAME') + + if instance in ips: + irqs = ips[instance].setdefault('irqs', {}) + irqs[irqname] = '{}:{}'.format(intc.get('INSTANCE'), irq) # Find BRAM storage depths (size) brams = root.xpath('.//MODULE[@MODTYPE="axi_bram_ctrl"]') From e8ef3e43808533322a7a23147461ce9702a953a8 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 19:54:18 +0100 Subject: [PATCH 03/21] lib/memory-manager: pass strings as const reference --- fpga/include/villas/memory_manager.hpp | 11 ++++++++--- fpga/lib/memory_manager.cpp | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fpga/include/villas/memory_manager.hpp b/fpga/include/villas/memory_manager.hpp index 941fd43c0..04bf60bf6 100644 --- a/fpga/include/villas/memory_manager.hpp +++ b/fpga/include/villas/memory_manager.hpp @@ -152,7 +152,8 @@ public: /// Create a default mapping MappingId - createMapping(uintptr_t src, uintptr_t dest, size_t size, std::string name, + createMapping(uintptr_t src, uintptr_t dest, size_t size, + const std::string& name, AddressSpaceId fromAddrSpace, AddressSpaceId toAddrSpace); @@ -167,7 +168,7 @@ public: AddressSpaceId - findAddressSpace(std::string name); + findAddressSpace(const std::string& name); MemoryTranslation getTranslation(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId); @@ -177,9 +178,13 @@ public: { return getTranslation(getProcessAddressSpace(), foreignAddrSpaceId); } static std::string - getSlaveAddrSpaceName(std::string ipInstance, std::string memoryBlock) + getSlaveAddrSpaceName(const std::string& ipInstance, const std::string& memoryBlock) { return ipInstance + "/" + memoryBlock; } + static std::string + getMasterAddrSpaceName(const std::string& ipInstance, const std::string& busInterface) + { return ipInstance + ":" + busInterface; } + void dump() { memoryGraph.dump(); } diff --git a/fpga/lib/memory_manager.cpp b/fpga/lib/memory_manager.cpp index 55f7858b1..f8e01e9b0 100644 --- a/fpga/lib/memory_manager.cpp +++ b/fpga/lib/memory_manager.cpp @@ -39,7 +39,7 @@ MemoryManager::getOrCreateAddressSpace(std::string name) MemoryManager::MappingId MemoryManager::createMapping(uintptr_t src, uintptr_t dest, size_t size, - std::string name, + const std::string& name, MemoryManager::AddressSpaceId fromAddrSpace, MemoryManager::AddressSpaceId toAddrSpace) { @@ -62,7 +62,7 @@ MemoryManager::addMapping(std::shared_ptr mapping, } MemoryManager::AddressSpaceId -MemoryManager::findAddressSpace(std::string name) +MemoryManager::findAddressSpace(const std::string& name) { return memoryGraph.findVertex( [&](const std::shared_ptr& v) { From 5d99f11a3480573765ba7e3e76e03371dcef7dfc Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 19:56:29 +0100 Subject: [PATCH 04/21] lib/ip: move definition of getBaseAddr() back to header This is a one-liner, so IMO increases readability. --- fpga/include/villas/fpga/ip.hpp | 3 ++- fpga/lib/ip.cpp | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp index b708ed1e4..e41cc1652 100644 --- a/fpga/include/villas/fpga/ip.hpp +++ b/fpga/include/villas/fpga/ip.hpp @@ -177,7 +177,8 @@ public: protected: uintptr_t - getBaseAddr(const std::string& block) const; + getBaseAddr(const std::string& block) const + { return getLocalAddr(block, 0); } uintptr_t getLocalAddr(const std::string& block, uintptr_t address) const; diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index 7b93320a8..375739a6e 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -328,13 +328,6 @@ IpCoreFactory::lookup(const Vlnv &vlnv) } -uintptr_t -IpCore::getBaseAddr(const std::string& block) const -{ - return getLocalAddr(block, 0); -} - - uintptr_t IpCore::getLocalAddr(const std::string& block, uintptr_t address) const { From 36259d00e0f70086c2db2667f1450834056d4fd9 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 19:57:47 +0100 Subject: [PATCH 05/21] lib/ip: save address space ids for each bus master interface --- fpga/include/villas/fpga/ip.hpp | 3 +++ fpga/lib/ip.cpp | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp index e41cc1652..b35048e3f 100644 --- a/fpga/include/villas/fpga/ip.hpp +++ b/fpga/include/villas/fpga/ip.hpp @@ -208,6 +208,9 @@ protected: /// Cached translations from the process address space to each memory block std::map addressTranslations; + + /// AXI bus master interfaces to access memory somewhere + std::map busMasterInterfaces; }; diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index 375739a6e..b7950c7da 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -195,15 +195,25 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) if(json_is_object(json_memory_view)) { logger->debug("Parse memory view of {}", *ip); - // create a master address space because this IP has a memory view - const MemoryManager::AddressSpaceId myAddrSpaceId = - MemoryManager::get().getOrCreateAddressSpace(id.getName()); // now find all slave address spaces this master can access const char* bus_name; json_t* json_bus; json_object_foreach(json_memory_view, bus_name, json_bus) { + // this IP has a memory view => it is a bus master somewhere + + // assemble name for master address space + const std::string myAddrSpaceName = + MemoryManager::getMasterAddrSpaceName(ip->getInstanceName(), + bus_name); + // create a master address space + const MemoryManager::AddressSpaceId myAddrSpaceId = + MemoryManager::get().getOrCreateAddressSpace(myAddrSpaceName); + + ip->busMasterInterfaces[bus_name] = myAddrSpaceId; + + const char* instance_name; json_t* json_instance; json_object_foreach(json_bus, instance_name, json_instance) { From aa2b0b324f6cc1501a3f2bef8f8f4e70f7d7755c Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 19:58:22 +0100 Subject: [PATCH 06/21] lib/ips/pcie: use cached address space id and supply interface to create mapping --- fpga/include/villas/fpga/ips/pcie.hpp | 3 +++ fpga/lib/ips/pcie.cpp | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fpga/include/villas/fpga/ips/pcie.hpp b/fpga/include/villas/fpga/ips/pcie.hpp index a63db2e20..290a8c577 100644 --- a/fpga/include/villas/fpga/ips/pcie.hpp +++ b/fpga/include/villas/fpga/ips/pcie.hpp @@ -48,6 +48,9 @@ public: friend class AxiPciExpressBridgeFactory; bool init(); + +private: + static constexpr char axiInterface[] = "M_AXI"; }; diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index df45d68cb..5b02f1261 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -38,12 +38,16 @@ static AxiPciExpressBridgeFactory factory; bool AxiPciExpressBridge::init() { + // Throw an exception if the is no bus master interface and thus no + // address space we can use for translation -> error + const MemoryManager::AddressSpaceId myAddrSpaceid = + busMasterInterfaces.at(axiInterface); + // Create an identity mapping from the FPGA card to this IP as an entry // point to all other IPs in the FPGA, because Vivado will generate a // memory view for this bridge that can see all others. - auto addrSpace = MemoryManager::get().findAddressSpace(getInstanceName()); MemoryManager::get().createMapping(0x00, 0x00, SIZE_MAX, "PCIeBridge", - card->addrSpaceId, addrSpace); + card->addrSpaceId, myAddrSpaceid); return true; } From 676fd9171ce893859e8b9bd396d6b8c9de762466 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Wed, 14 Feb 2018 16:03:06 +0100 Subject: [PATCH 07/21] lib/ip: make logger a class member of IpCore Logging is used everywhere and there's no justification of the clutter and runtime overhead of `aut logger = getLogger();` everywhere. --- fpga/include/villas/fpga/ip.hpp | 7 +++---- fpga/lib/ip.cpp | 6 +++--- fpga/lib/ip_node.cpp | 2 -- fpga/lib/ips/fifo.cpp | 2 -- fpga/lib/ips/intc.cpp | 2 -- fpga/lib/ips/switch.cpp | 8 -------- fpga/lib/ips/timer.cpp | 2 -- 7 files changed, 6 insertions(+), 23 deletions(-) diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp index b35048e3f..28f9e14f6 100644 --- a/fpga/include/villas/fpga/ip.hpp +++ b/fpga/include/villas/fpga/ip.hpp @@ -183,10 +183,6 @@ protected: uintptr_t getLocalAddr(const std::string& block, uintptr_t address) const; - SpdLogger - getLogger() const - { return loggerGetOrCreate(getInstanceName()); } - InterruptController* getInterruptController(const std::string& interruptName) const; @@ -197,6 +193,9 @@ protected: std::string description; }; + /// Specialized logger instance with the IPs name set as category + SpdLogger logger; + /// FPGA card this IP is instantiated on (populated by FpgaIpFactory) PCIeCard* card; diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index b7950c7da..dfbc60001 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -142,6 +142,7 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) // setup generic IP type properties ip->card = card; ip->id = id; + ip->logger = loggerGetOrCreate(id.getName()); json_t* json_ip = json_object_get(json_ips, id.getName().c_str()); @@ -313,9 +314,8 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) void -IpCore::dump() { - auto logger = getLogger(); - +IpCore::dump() +{ logger->info("IP: {}", *this); for(auto& [num, irq] : irqs) { logger->info("IRQ {}: {}:{}", diff --git a/fpga/lib/ip_node.cpp b/fpga/lib/ip_node.cpp index 6f25ac7ca..2ecb3f168 100644 --- a/fpga/lib/ip_node.cpp +++ b/fpga/lib/ip_node.cpp @@ -97,8 +97,6 @@ IpNode::loopbackPossible() const bool IpNode::connectLoopback() { - auto logger = getLogger(); - auto ports = getLoopbackPorts(); const auto& portMaster = portsMaster[ports.first]; const auto& portSlave = portsSlave[ports.second]; diff --git a/fpga/lib/ips/fifo.cpp b/fpga/lib/ips/fifo.cpp index 79e770b9c..2993d5fad 100644 --- a/fpga/lib/ips/fifo.cpp +++ b/fpga/lib/ips/fifo.cpp @@ -50,8 +50,6 @@ FifoFactory::FifoFactory() : bool Fifo::init() { - auto logger = getLogger(); - XLlFifo_Config fifo_cfg; try { diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index 56d8f7877..fa8524ed5 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -50,7 +50,6 @@ bool InterruptController::init() { const uintptr_t base = getBaseAddr(registerMemory); - auto logger = getLogger(); num_irqs = vfio_pci_msi_init(&card->vfio_device, efds); if (num_irqs < 0) @@ -97,7 +96,6 @@ InterruptController::init() bool InterruptController::enableInterrupt(InterruptController::IrqMaskType mask, bool polling) { - auto logger = getLogger(); const uintptr_t base = getBaseAddr(registerMemory); /* Current state of INTC */ diff --git a/fpga/lib/ips/switch.cpp b/fpga/lib/ips/switch.cpp index d9055c2f8..3439f9fbe 100644 --- a/fpga/lib/ips/switch.cpp +++ b/fpga/lib/ips/switch.cpp @@ -38,8 +38,6 @@ static AxiStreamSwitchFactory factory; bool AxiStreamSwitch::init() { - auto logger = getLogger(); - /* Setup AXI-stream switch */ XAxis_Switch_Config sw_cfg; sw_cfg.MaxNumMI = num_ports; @@ -66,8 +64,6 @@ AxiStreamSwitch::init() bool AxiStreamSwitch::connect(int portSlave, int portMaster) { - auto logger = getLogger(); - if(portMapping[portMaster] == portSlave) { logger->debug("Ports already connected"); return true; @@ -98,8 +94,6 @@ AxiStreamSwitch::connect(int portSlave, int portMaster) bool AxiStreamSwitch::disconnectMaster(int port) { - auto logger = getLogger(); - logger->debug("Disconnect slave {} from master {}", portMapping[port], port); @@ -111,8 +105,6 @@ AxiStreamSwitch::disconnectMaster(int port) bool AxiStreamSwitch::disconnectSlave(int port) { - auto logger = getLogger(); - for(auto [master, slave] : portMapping) { if(slave == port) { logger->debug("Disconnect slave {} from master {}", slave, master); diff --git a/fpga/lib/ips/timer.cpp b/fpga/lib/ips/timer.cpp index 1c7a023fe..91e36ade3 100644 --- a/fpga/lib/ips/timer.cpp +++ b/fpga/lib/ips/timer.cpp @@ -39,8 +39,6 @@ static TimerFactory factory; bool Timer::init() { - auto logger = getLogger(); - XTmrCtr_Config xtmr_cfg; xtmr_cfg.SysClockFreqHz = getFrequency(); From 64a89c2981161e553431915b468579751ff8fed5 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Wed, 14 Feb 2018 16:45:41 +0100 Subject: [PATCH 08/21] lib/ip: add debug output for 2-stage initialization --- fpga/lib/ip.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index dfbc60001..41ca7c52e 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -106,7 +106,7 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) // configure all IPs for(auto& id : orderedIps) { - loggerStatic->info("Initializing {}", id); + loggerStatic->info("Configuring {}", id); // find the appropriate factory that can create the specified VLNV // Note: @@ -289,6 +289,8 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) ip->addressTranslations.emplace(memoryBlock, translation); } + loggerStatic->info("Initializing {}", *ip); + if(not ip->init()) { loggerStatic->error("Cannot start IP {}", *ip); continue; From b21d6ddb9d1671153d7548d3417e3b1e0db4f7be Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 11:10:56 +0200 Subject: [PATCH 09/21] src: do not compile C benchmarks anymore --- fpga/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/fpga/CMakeLists.txt b/fpga/CMakeLists.txt index 4025ae5b9..ed8552fd2 100644 --- a/fpga/CMakeLists.txt +++ b/fpga/CMakeLists.txt @@ -11,4 +11,3 @@ include_directories(thirdparty/spdlog/include) add_subdirectory(lib) add_subdirectory(tests) -add_subdirectory(src) From ac483b21106a0d3a7537a1329d3f15d1564af0ab Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 14:31:51 +0200 Subject: [PATCH 10/21] scripts: fix non-root script * load IOMMU type 1 kernel module * determine IOMMU group dynamically * add user dkr to fpga group --- fpga/scripts/non_root.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) mode change 100644 => 100755 fpga/scripts/non_root.sh diff --git a/fpga/scripts/non_root.sh b/fpga/scripts/non_root.sh old mode 100644 new mode 100755 index 22e4f941d..a380ccf3a --- a/fpga/scripts/non_root.sh +++ b/fpga/scripts/non_root.sh @@ -1,16 +1,22 @@ #!/bin/bash -IOMMU_GROUP=24 +# PCI-e parameters of FPGA card PCI_BDF="0000:03:00.0" +PCI_VID="10ee" +PCI_PID="7022" modprobe vfio modprobe vfio_pci +modprobe vfio_iommu_type1 -echo "10ee 7022" > /sys/bus/pci/drivers/vfio-pci/new_id -echo ${PCI_BDF} > /sys/bus/pci/drivers/vfio-pci/bind +IOMMU_GROUP=`basename $(readlink /sys/bus/pci/devices/${PCI_BDF}/iommu_group)` + +# bind to vfio driver +echo "${PCI_VID} ${PCI_PID}" > /sys/bus/pci/drivers/vfio-pci/new_id groupadd -f fpga usermod -G fpga -a svg +usermod -G fpga -a dkr chgrp fpga /dev/vfio/${IOMMU_GROUP} chmod g+rw /dev/vfio/${IOMMU_GROUP} From b01a50184c3677a522e9b46505f1d04bf3ecf1bf Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:39:26 +0200 Subject: [PATCH 11/21] kernel/vfio: port to C++ This commit is 1/2 of a series of patches and not working on its own. --- fpga/include/villas/fpga/card.hpp | 11 +- fpga/include/villas/kernel/vfio.h | 123 ----- fpga/include/villas/kernel/vfio.hpp | 153 ++++++ fpga/lib/CMakeLists.txt | 17 +- fpga/lib/card.cpp | 34 +- fpga/lib/ips/intc.cpp | 13 +- fpga/lib/kernel/vfio.c | 652 ----------------------- fpga/lib/kernel/vfio.cpp | 766 ++++++++++++++++++++++++++++ fpga/tests/fpga.cpp | 12 +- 9 files changed, 954 insertions(+), 827 deletions(-) delete mode 100644 fpga/include/villas/kernel/vfio.h create mode 100644 fpga/include/villas/kernel/vfio.hpp delete mode 100644 fpga/lib/kernel/vfio.c create mode 100644 fpga/lib/kernel/vfio.cpp diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 4676e0b58..f8f9a0128 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -35,7 +35,7 @@ #include "common.h" #include "kernel/pci.h" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include #include @@ -91,8 +91,11 @@ public: struct pci *pci; struct pci_device filter; /**< Filter for PCI device. */ - ::vfio_container *vfio_container; - struct vfio_device vfio_device; /**< VFIO device handle. */ + /// The VFIO container that this card is part of + std::shared_ptr vfioContainer; + + /// The VFIO device that represents this card + VfioDevice* vfioDevice; /// Address space identifier of the master address space of this FPGA card. /// This will be used for address resolution of all IPs on this card. @@ -116,7 +119,7 @@ public: Plugin(Plugin::Type::FpgaCard, "FPGA Card plugin") {} static CardList - make(json_t *json, struct pci* pci, ::vfio_container* vc); + make(json_t *json, struct pci* pci, std::shared_ptr vc); static PCIeCard* create(); diff --git a/fpga/include/villas/kernel/vfio.h b/fpga/include/villas/kernel/vfio.h deleted file mode 100644 index 35a2465f9..000000000 --- a/fpga/include/villas/kernel/vfio.h +++ /dev/null @@ -1,123 +0,0 @@ -/** Virtual Function IO wrapper around kernel API - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - *********************************************************************************/ - -/** @addtogroup fpga Kernel @{ */ - -#pragma once - -#include -#include -#include - -#include -#include - -#include "list.h" - -#define VFIO_DEV(x) "/dev/vfio/" x - -#ifdef __cplusplus -extern "C" { -#endif - -/* Forward declarations */ -struct pci_device; - -struct vfio_group { - int fd; /**< VFIO group file descriptor */ - int index; /**< Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ */ - - struct vfio_group_status status; /**< Status of group */ - - struct list devices; - - struct vfio_container *container; /**< The VFIO container to which this group is belonging */ -}; - -struct vfio_device { - char *name; /**< Name of the device as listed under /sys/kernel/iommu_groups/[vfio_group::index]/devices/ */ - int fd; /**< VFIO device file descriptor */ - - struct vfio_device_info info; - struct vfio_irq_info *irqs; - struct vfio_region_info *regions; - - void **mappings; - - struct pci_device *pci_device; /**< libpci handle of the device */ - struct vfio_group *group; /**< The VFIO group this device belongs to */ -}; - -struct vfio_container { - int fd; - int version; - int extensions; - - uint64_t iova_next; /**< Next free IOVA address */ - - struct list groups; -}; - -/** Initialize a new VFIO container. */ -int vfio_init(struct vfio_container *c); - -/** Initialize a VFIO group and attach it to an existing VFIO container. */ -int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index); - -/** Initialize a VFIO device, lookup the VFIO group it belongs to, create the group if not already existing. */ -int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index); - -/** Initialie a VFIO-PCI device (uses vfio_device_attach() internally) */ -int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev); - -/** Hot resets a VFIO-PCI device */ -int vfio_pci_reset(struct vfio_device *d); - -int vfio_pci_msi_init(struct vfio_device *d, int efds[32]); - -int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]); - -int vfio_pci_msi_find(struct vfio_device *d, int nos[32]); - -/** Enable memory accesses and bus mastering for PCI device */ -int vfio_pci_enable(struct vfio_device *d); - -/** Reset a VFIO device */ -int vfio_device_reset(struct vfio_device *d); - -/** Release memory and close container */ -int vfio_destroy(struct vfio_container *c); - -/** Release memory of group */ -int vfio_group_destroy(struct vfio_group *g); - -/** Release memory of device */ -int vfio_device_destroy(struct vfio_device *g); - -/** Print a dump of all attached groups and their devices including regions and IRQs */ -void vfio_dump(struct vfio_container *c); - -/** Map a device memory region to the application address space (e.g. PCI BARs) */ -void * vfio_map_region(struct vfio_device *d, int idx); - -/** Get the size of a device memory region */ -size_t vfio_region_size(struct vfio_device *d, int idx); - -/** Map VM to an IOVA, which is accessible by devices in the container */ -int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len); - -/** Unmap DMA memory */ -int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len); - -/** munmap() a region which has been mapped by vfio_map_region() */ -int vfio_unmap_region(struct vfio_device *d, int idx); - -#ifdef __cplusplus -} -#endif - -/** @} */ diff --git a/fpga/include/villas/kernel/vfio.hpp b/fpga/include/villas/kernel/vfio.hpp new file mode 100644 index 000000000..9f794baf5 --- /dev/null +++ b/fpga/include/villas/kernel/vfio.hpp @@ -0,0 +1,153 @@ +/** Virtual Function IO wrapper around kernel API + * + * @file + * @author Steffen Vogel + * @author Daniel Krebs + * @copyright 2017, Steffen Vogel + * @copyright 2018, Daniel Krebs + *********************************************************************************/ + +/** @addtogroup fpga Kernel @{ */ + +#pragma once + +#include +#include +#include + +#include +#include + +#define VFIO_DEV(x) "/dev/vfio/" x + +/* Forward declarations */ +struct pci_device; + +namespace villas { + +class VfioContainer; +class VfioGroup; + + +class VfioDevice { + friend class VfioContainer; +public: + VfioDevice(const std::string& name, VfioGroup& group) : + name(name), group(group) {} + + ~VfioDevice(); + + bool reset(); + + /** Map a device memory region to the application address space (e.g. PCI BARs) */ + void* regionMap(size_t index); + + /** munmap() a region which has been mapped by vfio_map_region() */ + bool regionUnmap(size_t index); + + /** Get the size of a device memory region */ + size_t regionGetSize(size_t index); + + + /** Enable memory accesses and bus mastering for PCI device */ + bool pciEnable(); + + bool pciHotReset(); + int pciMsiInit(int efds[32]); + int pciMsiDeinit(int efds[32]); + bool pciMsiFind(int nos[32]); + + bool isVfioPciDevice() const; + +private: + /// Name of the device as listed under + /// /sys/kernel/iommu_groups/[vfio_group::index]/devices/ + std::string name; + + /// VFIO device file descriptor + int fd; + + struct vfio_device_info info; + + std::vector irqs; + std::vector regions; + std::vector mappings; + + /**< libpci handle of the device */ + const struct pci_device *pci_device; + + VfioGroup& group; /**< The VFIO group this device belongs to */ +}; + + + +class VfioGroup { + friend class VfioContainer; + friend VfioDevice; +private: + VfioGroup(int index) : fd(-1), index(index) {} +public: + ~VfioGroup(); + + static std::unique_ptr + attach(int containerFd, int groupIndex); + +private: + /// VFIO group file descriptor + int fd; + + /// Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ + int index; + + /// Status of group + struct vfio_group_status status; + + /// All devices owned by this group + std::list> devices; + + VfioContainer* container; /**< The VFIO container to which this group is belonging */ +}; + + +class VfioContainer { +private: + VfioContainer(); +public: + ~VfioContainer(); + + static std::shared_ptr + create(); + + void dump(); + + VfioDevice& attachDevice(const char *name, int groupIndex); + VfioDevice& attachDevice(const struct pci_device *pdev); + + /** + * @brief Map VM to an IOVA, which is accessible by devices in the container + * @param virt virtual address of memory + * @param phys IOVA where to map @p virt, -1 to use VFIO internal allocator + * @param length size of memory region in bytes + * @return IOVA address, UINTPTR_MAX on failure + */ + uintptr_t memoryMap(uintptr_t virt, uintptr_t phys, size_t length); + + /** munmap() a region which has been mapped by vfio_map_region() */ + bool memoryUnmap(uintptr_t phys, size_t length); + +private: + VfioGroup& getOrAttachGroup(int index); + +private: + int fd; + int version; + int extensions; + uint64_t iova_next; /**< Next free IOVA address */ + + /// All groups bound to this container + std::list> groups; +}; + +/** @} */ + +} // namespace villas diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 50d16ecf5..7150b61eb 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -1,23 +1,9 @@ set(SOURCES - ip.c - vlnv.c - card.c - vlnv.cpp card.cpp ip.cpp ip_node.cpp - ips/timer.c - ips/model.c - ips/switch.c - ips/dft.c - ips/fifo.c - ips/dma.c - ips/intc.c - ips/gpio.c - ips/rtds_axis.c - ips/timer.cpp ips/switch.cpp ips/fifo.cpp @@ -26,9 +12,8 @@ set(SOURCES kernel/kernel.c kernel/pci.c - kernel/vfio.c + kernel/vfio.cpp - plugin.c utils.c list.c log.c diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index c4a184fc9..e0e2ba456 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -27,7 +27,7 @@ #include "log.hpp" #include "kernel/pci.h" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include "fpga/ip.hpp" #include "fpga/card.hpp" @@ -38,9 +38,8 @@ namespace fpga { // instantiate factory to register static PCIeCardFactory PCIeCardFactory; - CardList -fpga::PCIeCardFactory::make(json_t *json, struct pci* pci, ::vfio_container* vc) +PCIeCardFactory::make(json_t *json, struct pci* pci, std::shared_ptr vc) { CardList cards; auto logger = getStaticLogger(); @@ -73,7 +72,7 @@ fpga::PCIeCardFactory::make(json_t *json, struct pci* pci, ::vfio_container* vc) // populate generic properties card->name = std::string(card_name); card->pci = pci; - card->vfio_container = vc; + card->vfioContainer = std::move(vc); card->affinity = affinity; card->do_reset = do_reset != 0; @@ -158,19 +157,26 @@ fpga::PCIeCard::init() } /* Attach PCIe card to VFIO container */ - ret = ::vfio_pci_attach(&vfio_device, vfio_container, pdev); - if (ret) { - logger->error("Failed to attach VFIO device"); + VfioDevice& device = vfioContainer->attachDevice(pdev); + this->vfioDevice = &device; + + + /* Enable memory access and PCI bus mastering for DMA */ + if (not device.pciEnable()) { + logger->error("Failed to enable PCI device"); return false; } /* Map PCIe BAR */ - const void* bar0_mapped = vfio_map_region(&vfio_device, VFIO_PCI_BAR0_REGION_INDEX); + const void* bar0_mapped = vfioDevice->regionMap(VFIO_PCI_BAR0_REGION_INDEX); if (bar0_mapped == MAP_FAILED) { logger->error("Failed to mmap() BAR0"); return false; } + // determine size of BAR0 region + const size_t bar0_size = vfioDevice->regionGetSize(VFIO_PCI_BAR0_REGION_INDEX); + /* Link mapped BAR0 to global memory graph */ @@ -180,9 +186,6 @@ fpga::PCIeCard::init() // create a new address space for this FPGA card this->addrSpaceId = MemoryManager::get().getOrCreateAddressSpace(name); - // determine size of BAR0 region - const size_t bar0_size = vfio_region_size(&vfio_device, - VFIO_PCI_BAR0_REGION_INDEX); // create a mapping from our address space to the FPGA card via vfio MemoryManager::get().createMapping(reinterpret_cast(bar0_mapped), @@ -190,18 +193,11 @@ fpga::PCIeCard::init() villasAddrSpace, this->addrSpaceId); - /* Enable memory access and PCI bus mastering for DMA */ - ret = vfio_pci_enable(&vfio_device); - if (ret) { - logger->error("Failed to enable PCI device"); - return false; - } /* Reset system? */ if (do_reset) { /* Reset / detect PCI device */ - ret = vfio_pci_reset(&vfio_device); - if (ret) { + if(not vfioDevice->pciHotReset()) { logger->error("Failed to reset PCI device"); return false; } diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index fa8524ed5..3fbad436d 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -27,7 +27,7 @@ #include "log.h" #include "plugin.hpp" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include "kernel/kernel.h" #include "fpga/card.hpp" @@ -43,7 +43,7 @@ static InterruptControllerFactory factory; InterruptController::~InterruptController() { - vfio_pci_msi_deinit(&card->vfio_device , this->efds); + card->vfioDevice->pciMsiDeinit(this->efds); } bool @@ -51,12 +51,13 @@ InterruptController::init() { const uintptr_t base = getBaseAddr(registerMemory); - num_irqs = vfio_pci_msi_init(&card->vfio_device, efds); + num_irqs = card->vfioDevice->pciMsiInit(efds); if (num_irqs < 0) return false; - if(vfio_pci_msi_find(&card->vfio_device, nos) != 0) + if(not card->vfioDevice->pciMsiFind(nos)) { return false; + } /* For each IRQ */ for (int i = 0; i < num_irqs; i++) { @@ -69,8 +70,8 @@ InterruptController::init() // everything is fine break; case EACCES: - logger->warn("No permission to change affinity of VFIO-MSI interrupt. " - "This may degrade performance (increasing jitter)"); + logger->warn("No permission to change affinity of VFIO-MSI interrupt, " + "performance may be degraded!"); break; default: logger->error("Failed to change affinity of VFIO-MSI interrupt"); diff --git a/fpga/lib/kernel/vfio.c b/fpga/lib/kernel/vfio.c deleted file mode 100644 index 7103cc519..000000000 --- a/fpga/lib/kernel/vfio.c +++ /dev/null @@ -1,652 +0,0 @@ -/** Virtual Function IO wrapper around kernel API - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#define _DEFAULT_SOURCE - -#include -#include -#include -#include -#include - -#include -#include - -#include "utils.h" -#include "log.h" - -#include "config.h" - -#include "kernel/kernel.h" -#include "kernel/vfio.h" -#include "kernel/pci.h" - -static const char *vfio_pci_region_names[] = { - "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX, - "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX, - "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX, - "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX, - "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX, - "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX, - "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX, - "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX, - "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX, -}; - -static const char *vfio_pci_irq_names[] = { - "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX, - "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX, - "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX, - "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX, - "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX, -}; - -/* Helpers */ -int vfio_get_iommu_name(int index, char *buf, size_t len) -{ - FILE *f; - char fn[256]; - - snprintf(fn, sizeof(fn), "/sys/kernel/iommu_groups/%d/name", index); - - f = fopen(fn, "r"); - if (!f) - return -1; - - int ret = fgets(buf, len, f) == buf ? 0 : -1; - - /* Remove trailing newline */ - char *c = strrchr(buf, '\n'); - if (c) - *c = 0; - - fclose(f); - - return ret; -} - -/* Destructors */ -int vfio_destroy(struct vfio_container *v) -{ - int ret; - - /* Release memory and close fds */ - list_destroy(&v->groups, (dtor_cb_t) vfio_group_destroy, true); - - /* Close container */ - ret = close(v->fd); - if (ret < 0) - return -1; - - debug(5, "VFIO: closed container: fd=%d", v->fd); - - return 0; -} - -int vfio_group_destroy(struct vfio_group *g) -{ - int ret; - - list_destroy(&g->devices, (dtor_cb_t) vfio_device_destroy, false); - - ret = ioctl(g->fd, VFIO_GROUP_UNSET_CONTAINER); - if (ret) - return ret; - - debug(5, "VFIO: released group from container: group=%u", g->index); - - ret = close(g->fd); - if (ret) - return ret; - - debug(5, "VFIO: closed group: group=%u, fd=%d", g->index, g->fd); - - return 0; -} - -int vfio_device_destroy(struct vfio_device *d) -{ - int ret; - - for (int i = 0; i < d->info.num_regions; i++) - vfio_unmap_region(d, i); - - ret = close(d->fd); - if (ret) - return ret; - - debug(5, "VFIO: closed device: name=%s, fd=%d", d->name, d->fd); - - free(d->mappings); - free(d->name); - - return 0; -} - -/* Constructors */ -int vfio_init(struct vfio_container *v) -{ - int ret; - - /* Initialize datastructures */ - memset(v, 0, sizeof(*v)); - - list_init(&v->groups); - - /* Load VFIO kernel module */ - if (kernel_module_load("vfio")) - error("Failed to load kernel module: %s", "vfio"); - - /* Open VFIO API */ - v->fd = open(VFIO_DEV("vfio"), O_RDWR); - if (v->fd < 0) - error("Failed to open VFIO container"); - - /* Check VFIO API version */ - v->version = ioctl(v->fd, VFIO_GET_API_VERSION); - if (v->version < 0 || v->version != VFIO_API_VERSION) - error("Failed to get VFIO version"); - - /* Check available VFIO extensions (IOMMU types) */ - v->extensions = 0; - for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { - ret = ioctl(v->fd, VFIO_CHECK_EXTENSION, i); - if (ret < 0) - error("Failed to get VFIO extensions"); - else if (ret > 0) - v->extensions |= (1 << i); - } - - return 0; -} - -int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index) -{ - int ret; - char buf[128]; - - g->index = index; - g->container = c; - - list_init(&g->devices); - - /* Open group fd */ - snprintf(buf, sizeof(buf), VFIO_DEV("%u"), g->index); - g->fd = open(buf, O_RDWR); - if (g->fd < 0) - serror("Failed to open VFIO group: %u", g->index); - - /* Claim group ownership */ - ret = ioctl(g->fd, VFIO_GROUP_SET_CONTAINER, &c->fd); - if (ret < 0) - serror("Failed to attach VFIO group to container"); - - /* Set IOMMU type */ - ret = ioctl(c->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); - if (ret < 0) - serror("Failed to set IOMMU type of container"); - - /* Check group viability and features */ - g->status.argsz = sizeof(g->status); - - ret = ioctl(g->fd, VFIO_GROUP_GET_STATUS, &g->status); - if (ret < 0) - serror("Failed to get VFIO group status"); - - if (!(g->status.flags & VFIO_GROUP_FLAGS_VIABLE)) - error("VFIO group is not available: bind all devices to the VFIO driver!"); - - list_push(&c->groups, g); - - return 0; -} - -int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev) -{ - char name[32]; - int ret; - - /* Load PCI bus driver for VFIO */ - if (kernel_module_load("vfio_pci")) - error("Failed to load kernel driver: %s", "vfio_pci"); - - /* Bind PCI card to vfio-pci driver if not already bound */ - ret = pci_get_driver(pdev, name, sizeof(name)); - if (ret || strcmp(name, "vfio-pci")) { - ret = pci_attach_driver(pdev, "vfio-pci"); - if (ret) - error("Failed to attach device to driver"); - } - - /* Get IOMMU group of device */ - int index = pci_get_iommu_group(pdev); - if (index < 0) - error("Failed to get IOMMU group of device"); - - /* VFIO device name consists of PCI BDF */ - snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, pdev->slot.bus, pdev->slot.device, pdev->slot.function); - - ret = vfio_device_attach(d, c, name, index); - if (ret < 0) - return ret; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) { - vfio_device_destroy(d); - return -1; - } - - d->pci_device = pdev; - - return 0; -} - -int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index) -{ - int ret; - struct vfio_group *g = NULL; - - /* Check if group already exists */ - for (size_t i = 0; i < list_length(&c->groups); i++) { - struct vfio_group *h = (struct vfio_group *) list_at(&c->groups, i); - - if (h->index == index) - g = h; - } - - if (!g) { - g = alloc(sizeof(struct vfio_group)); - - /* Aquire group ownership */ - ret = vfio_group_attach(g, c, index); - if (ret) - error("Failed to attach to IOMMU group: %u", index); - - info("Attached new group %u to VFIO container", g->index); - } - - d->group = g; - d->name = strdup(name); - - /* Open device fd */ - d->fd = ioctl(g->fd, VFIO_GROUP_GET_DEVICE_FD, d->name); - if (d->fd < 0) - serror("Failed to open VFIO device: %s", d->name); - - /* Get device info */ - d->info.argsz = sizeof(d->info); - - ret = ioctl(d->fd, VFIO_DEVICE_GET_INFO, &d->info); - if (ret < 0) - serror("Failed to get VFIO device info for: %s", d->name); - - d->irqs = alloc(d->info.num_irqs * sizeof(struct vfio_irq_info)); - d->regions = alloc(d->info.num_regions * sizeof(struct vfio_region_info)); - d->mappings = alloc(d->info.num_regions * sizeof(void *)); - - /* Get device regions */ - for (int i = 0; i < d->info.num_regions && i < 8; i++) { - struct vfio_region_info *region = &d->regions[i]; - - region->argsz = sizeof(*region); - region->index = i; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_REGION_INFO, region); - if (ret < 0) - serror("Failed to get regions of VFIO device: %s", d->name); - } - - /* Get device irqs */ - for (int i = 0; i < d->info.num_irqs; i++) { - struct vfio_irq_info *irq = &d->irqs[i]; - - irq->argsz = sizeof(*irq); - irq->index = i; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_IRQ_INFO, irq); - if (ret < 0) - serror("Failed to get IRQs of VFIO device: %s", d->name); - } - - list_push(&d->group->devices, d); - - return 0; -} - -int vfio_pci_reset(struct vfio_device *d) -{ - int ret; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - size_t reset_infolen = sizeof(struct vfio_pci_hot_reset_info) + sizeof(struct vfio_pci_dependent_device) * 64; - size_t resetlen = sizeof(struct vfio_pci_hot_reset) + sizeof(int32_t) * 1; - - struct vfio_pci_hot_reset_info *reset_info = (struct vfio_pci_hot_reset_info *) alloc(reset_infolen); - struct vfio_pci_hot_reset *reset = (struct vfio_pci_hot_reset *) alloc(resetlen); - - reset_info->argsz = reset_infolen; - reset->argsz = resetlen; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info); - if (ret) - return ret; - - debug(5, "VFIO: dependent devices for hot-reset:"); - for (int i = 0; i < reset_info->count; i++) { INDENT - struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; - debug(5, "%04x:%02x:%02x.%01x: iommu_group=%u", dd->segment, dd->bus, PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); - - if (dd->group_id != d->group->index) - return -3; - } - - reset->count = 1; - reset->group_fds[0] = d->group->fd; - - ret = ioctl(d->fd, VFIO_DEVICE_PCI_HOT_RESET, reset); - - free(reset_info); - - return ret; -} - -int vfio_pci_msi_find(struct vfio_device *d, int nos[32]) -{ - int ret, idx, irq; - char *end, *col, *last, line[1024], name[13]; - FILE *f; - - f = fopen("/proc/interrupts", "r"); - if (!f) - return -1; - - for (int i = 0; i < 32; i++) - nos[i] = -1; - - /* For each line in /proc/interruipts */ - while (fgets(line, sizeof(line), f)) { - col = strtok(line, " "); - - /* IRQ number is in first column */ - irq = strtol(col, &end, 10); - if (col == end) - continue; - - /* Find last column of line */ - do { - last = col; - } while ((col = strtok(NULL, " "))); - - - ret = sscanf(last, "vfio-msi[%u](%12[0-9:])", &idx, name); - if (ret == 2) { - if (strstr(d->name, name) == d->name) - nos[idx] = irq; - } - } - - fclose(f); - - return 0; -} - -int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]) -{ - int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; - struct vfio_irq_set *irq_set; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; - irq_set = alloc(irq_setlen); - - irq_set->argsz = irq_setlen; - irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; - irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; - irq_set->count = irq_count; - irq_set->start = 0; - - for (int i = 0; i < irq_count; i++) { - close(efds[i]); - efds[i] = -1; - } - - memcpy(irq_set->data, efds, sizeof(int) * irq_count); - - ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); - if (ret) - return -4; - - free(irq_set); - - return irq_count; -} - -int vfio_pci_msi_init(struct vfio_device *d, int efds[32]) -{ - int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; - struct vfio_irq_set *irq_set; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; - irq_set = alloc(irq_setlen); - - irq_set->argsz = irq_setlen; - irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; - irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; - irq_set->start = 0; - irq_set->count = irq_count; - - /* Now set the new eventfds */ - for (int i = 0; i < irq_count; i++) { - efds[i] = eventfd(0, 0); - if (efds[i] < 0) - return -3; - } - memcpy(irq_set->data, efds, sizeof(int) * irq_count); - - ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); - if (ret) - return -4; - - free(irq_set); - - return irq_count; -} - -int vfio_pci_enable(struct vfio_device *d) -{ - int ret; - uint32_t reg; - off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - ret = pread(d->fd, ®, sizeof(reg), offset); - if (ret != sizeof(reg)) - return -1; - - /* Enable memory access and PCI bus mastering which is required for DMA */ - reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; - - ret = pwrite(d->fd, ®, sizeof(reg), offset); - if (ret != sizeof(reg)) - return -1; - - return 0; -} - -int vfio_device_reset(struct vfio_device *d) -{ - if (d->info.flags & VFIO_DEVICE_FLAGS_RESET) - return ioctl(d->fd, VFIO_DEVICE_RESET); - else - return -1; /* not supported by this device */ -} - -void vfio_dump(struct vfio_container *v) -{ - info("VFIO Version: %u", v->version); - info("VFIO Extensions: %#x", v->extensions); - - for (size_t i = 0; i < list_length(&v->groups); i++) { - struct vfio_group *g = (struct vfio_group *) list_at(&v->groups, i); - - info("VFIO Group %u, viable=%u, container=%d", g->index, - (g->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0, - (g->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0 - ); - - - for (size_t i = 0; i < list_length(&g->devices); i++) { INDENT - struct vfio_device *d = (struct vfio_device *) list_at(&g->devices, i); - - info("Device %s: regions=%u, irqs=%u, flags=%#x", d->name, - d->info.num_regions, - d->info.num_irqs, - d->info.flags - ); - - for (int i = 0; i < d->info.num_regions && i < 8; i++) { INDENT - struct vfio_region_info *region = &d->regions[i]; - - if (region->size > 0) - info("Region %u %s: size=%#llx, offset=%#llx, flags=%u", - region->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI) ? vfio_pci_region_names[i] : "", - region->size, - region->offset, - region->flags - ); - } - - for (int i = 0; i < d->info.num_irqs; i++) { INDENT - struct vfio_irq_info *irq = &d->irqs[i]; - - if (irq->count > 0) - info("IRQ %u %s: count=%u, flags=%u", - irq->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI ) ? vfio_pci_irq_names[i] : "", - irq->count, - irq->flags - ); - } - } - } -} - -void * vfio_map_region(struct vfio_device *d, int idx) -{ - struct vfio_region_info *r = &d->regions[idx]; - - if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP)) - return MAP_FAILED; - - d->mappings[idx] = mmap(NULL, r->size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_32BIT, d->fd, r->offset); - - return d->mappings[idx]; -} - -size_t vfio_region_size(struct vfio_device *d, int idx) -{ - assert(d != NULL); - assert((size_t)idx < d->info.num_regions); - - return d->regions[idx].size; -} - -int vfio_unmap_region(struct vfio_device *d, int idx) -{ - int ret; - struct vfio_region_info *r = &d->regions[idx]; - - if (!d->mappings[idx]) - return -1; /* was not mapped */ - - debug(3, "VFIO: unmap region %u from device", idx); - - ret = munmap(d->mappings[idx], r->size); - if (ret) - return -2; - - d->mappings[idx] = NULL; - - return 0; -} - -int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) -{ - int ret; - - if (len & 0xFFF) { - len += 0x1000; - len &= ~0xFFF; - } - - /* Super stupid allocator */ - if (phys == -1) { - phys = c->iova_next; - c->iova_next += len; - } - - struct vfio_iommu_type1_dma_map dma_map = { - .argsz = sizeof(struct vfio_iommu_type1_dma_map), - .vaddr = virt, - .iova = phys, - .size = len, - .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE - }; - - ret = ioctl(c->fd, VFIO_IOMMU_MAP_DMA, &dma_map); - if (ret) - serror("Failed to create DMA mapping"); - - info("DMA map size=%#llx, iova=%#llx, vaddr=%#llx", dma_map.size, dma_map.iova, dma_map.vaddr); - - return 0; -} - -int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) -{ - int ret; - - struct vfio_iommu_type1_dma_unmap dma_unmap = { - .argsz = sizeof(struct vfio_iommu_type1_dma_unmap), - .flags = 0, - .iova = phys, - .size = len, - }; - - ret = ioctl(c->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); - if (ret) - serror("Failed to unmap DMA mapping"); - - return 0; -} diff --git a/fpga/lib/kernel/vfio.cpp b/fpga/lib/kernel/vfio.cpp new file mode 100644 index 000000000..09464b1f8 --- /dev/null +++ b/fpga/lib/kernel/vfio.cpp @@ -0,0 +1,766 @@ +/** Virtual Function IO wrapper around kernel API + * + * @author Steffen Vogel + * @author Daniel Krebs + * @copyright 2017, Steffen Vogel + * @copyright 2018, Daniel Krebs + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#define _DEFAULT_SOURCE + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "kernel/pci.h" +#include "kernel/kernel.h" + +#include "kernel/vfio.hpp" +#include "log.hpp" + +static auto logger = loggerGetOrCreate("Vfio"); + +static const char *vfio_pci_region_names[] = { + "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX, + "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX, + "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX, + "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX, + "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX, + "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX, + "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX, + "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX, + "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX, +}; + +static const char *vfio_pci_irq_names[] = { + "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX, + "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX, + "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX, + "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX, + "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX, +}; + +namespace villas { + + +VfioContainer::VfioContainer() + : iova_next(0) +{ + /* Load VFIO kernel module */ + if (kernel_module_load("vfio") != 0) { + logger->error("Failed to load kernel module: vfio"); + throw std::exception(); + } + + /* Open VFIO API */ + fd = open(VFIO_DEV("vfio"), O_RDWR); + if (fd < 0) { + logger->error("Failed to open VFIO container"); + throw std::exception(); + } + + /* Check VFIO API version */ + version = ioctl(fd, VFIO_GET_API_VERSION); + if (version < 0 || version != VFIO_API_VERSION) { + logger->error("Failed to get VFIO version"); + throw std::exception(); + } + + /* Check available VFIO extensions (IOMMU types) */ + extensions = 0; + for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { + int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i); + if (ret < 0) { + logger->error("Failed to get VFIO extensions"); + throw std::exception(); + } + else if (ret > 0) + extensions |= (1 << i); + } + + logger->debug("Version: {:#x}", version); + logger->debug("Extensions: {:#x}", extensions); +} + + +VfioContainer::~VfioContainer() +{ + logger->debug("Clean up container with fd {}", this->fd); + + /* Release memory and close fds */ + groups.clear(); + + /* Close container */ + int ret = close(fd); + if (ret < 0) { + logger->error("Cannot close vfio container"); + } + + logger->debug("VFIO: closed container: fd={}", fd); +} + + +std::shared_ptr +VfioContainer::create() +{ + std::shared_ptr container { new VfioContainer }; + return container; +} + + +void +VfioContainer::dump() +{ + logger->info("File descriptor: {}", fd); + logger->info("Version: {}", version); + logger->info("Extensions: 0x{:x}", extensions); + + for(auto& group : groups) { + logger->info("VFIO Group {}, viable={}, container={}", + group->index, + (group->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0, + (group->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0 + ); + + for(auto& device : group->devices) { + logger->info("Device {}: regions={}, irqs={}, flags={}", + device->name, + device->info.num_regions, + device->info.num_irqs, + device->info.flags + ); + + for (size_t i = 0; i < device->info.num_regions && i < 8; i++) { + struct vfio_region_info *region = &device->regions[i]; + + if (region->size > 0) + logger->info("Region {} {}: size={}, offset={}, flags={}", + region->index, + (device->info.flags & VFIO_DEVICE_FLAGS_PCI) ? + vfio_pci_region_names[i] : "", + region->size, + region->offset, + region->flags + ); + } + + for (size_t i = 0; i < device->info.num_irqs; i++) { + struct vfio_irq_info *irq = &device->irqs[i]; + + if (irq->count > 0) + logger->info("IRQ {} {}: count={}, flags={}", + irq->index, + (device->info.flags & VFIO_DEVICE_FLAGS_PCI ) ? + vfio_pci_irq_names[i] : "", + irq->count, + irq->flags + ); + } + } + } +} + + +VfioDevice& +VfioContainer::attachDevice(const char* name, int index) +{ + VfioGroup& group = getOrAttachGroup(index); + auto device = std::make_unique(name, group); + + /* Open device fd */ + device->fd = ioctl(group.fd, VFIO_GROUP_GET_DEVICE_FD, name); + if (device->fd < 0) { + logger->error("Failed to open VFIO device: {}", device->name); + throw std::exception(); + } + + /* Get device info */ + device->info.argsz = sizeof(device->info); + + int ret = ioctl(device->fd, VFIO_DEVICE_GET_INFO, &device->info); + if (ret < 0) { + logger->error("Failed to get VFIO device info for: {}", device->name); + throw std::exception(); + } + + logger->debug("Device has {} regions", device->info.num_regions); + logger->debug("Device has {} IRQs", device->info.num_irqs); + + // reserve slots already so that we can use the []-operator for access + device->irqs.resize(device->info.num_irqs); + device->regions.resize(device->info.num_regions); + device->mappings.resize(device->info.num_regions); + + /* Get device regions */ + for (size_t i = 0; i < device->info.num_regions && i < 8; i++) { + struct vfio_region_info region; + memset(®ion, 0, sizeof (region)); + + region.argsz = sizeof(region); + region.index = i; + + ret = ioctl(device->fd, VFIO_DEVICE_GET_REGION_INFO, ®ion); + if (ret < 0) { + logger->error("Failed to get region of VFIO device: {}", device->name); + throw std::exception(); + } + + device->regions[i] = region; + } + + + /* Get device irqs */ + for (size_t i = 0; i < device->info.num_irqs; i++) { + struct vfio_irq_info irq; + memset(&irq, 0, sizeof (irq)); + + irq.argsz = sizeof(irq); + irq.index = i; + + ret = ioctl(device->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq); + if (ret < 0) { + logger->error("Failed to get IRQs of VFIO device: {}", device->name); + throw std::exception(); + } + + device->irqs[i] = irq; + } + + group.devices.push_back(std::move(device)); + + return *group.devices.back().get(); +} + + +VfioDevice& +VfioContainer::attachDevice(const pci_device* pdev) +{ + int ret; + char name[32]; + static constexpr char kernelDriver[] = "vfio-pci"; + + /* Load PCI bus driver for VFIO */ + if (kernel_module_load("vfio_pci")) { + logger->error("Failed to load kernel driver: vfio_pci"); + throw std::exception(); + } + + /* Bind PCI card to vfio-pci driver if not already bound */ + ret = pci_get_driver(pdev, name, sizeof(name)); + if (ret || strcmp(name, kernelDriver)) { + logger->debug("Bind PCI card to kernel driver '{}'", kernelDriver); + ret = pci_attach_driver(pdev, kernelDriver); + if (ret) { + logger->error("Failed to attach device to driver"); + throw std::exception(); + } + } + + /* Get IOMMU group of device */ + int index = pci_get_iommu_group(pdev); + if (index < 0) { + logger->error("Failed to get IOMMU group of device"); + throw std::exception(); + } + + /* VFIO device name consists of PCI BDF */ + snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, + pdev->slot.bus, pdev->slot.device, pdev->slot.function); + + logger->info("Attach to device {} with index {}", std::string(name), index); + auto& device = attachDevice(name, index); + + device.pci_device = pdev; + + /* Check if this is really a vfio-pci device */ + if(not device.isVfioPciDevice()) { + logger->error("Device is not a vfio-pci device"); + throw std::exception(); + } + + return device; +} + + +uintptr_t +VfioContainer::memoryMap(uintptr_t virt, uintptr_t phys, size_t length) +{ + int ret; + + if (length & 0xFFF) { + length += 0x1000; + length &= ~0xFFF; + } + + /* Super stupid allocator */ + size_t iovaIncrement = 0; + if (phys == UINTPTR_MAX) { + phys = this->iova_next; + iovaIncrement = length; + } + + struct vfio_iommu_type1_dma_map dmaMap; + memset(&dmaMap, 0, sizeof(dmaMap)); + + dmaMap.argsz = sizeof(dmaMap); + dmaMap.vaddr = virt; + dmaMap.iova = phys; + dmaMap.size = length; + dmaMap.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + ret = ioctl(this->fd, VFIO_IOMMU_MAP_DMA, &dmaMap); + if (ret) { + logger->error("Failed to create DMA mapping: {}", ret); + return UINTPTR_MAX; + } + + logger->info("DMA map size={:#x}, iova={:#x}, vaddr={:#x}", dmaMap.size, dmaMap.iova, dmaMap.vaddr); + + // mapping successful, advance IOVA allocator + this->iova_next += iovaIncrement; + + // we intentionally don't return the actual mapped length, the users are only + // guaranteed to have their demanded memory mapped correctly + return dmaMap.iova; +} + + +bool +VfioContainer::memoryUnmap(uintptr_t phys, size_t length) +{ + int ret; + + struct vfio_iommu_type1_dma_unmap dmaUnmap; + dmaUnmap.argsz = sizeof(struct vfio_iommu_type1_dma_unmap); + dmaUnmap.flags = 0; + dmaUnmap.iova = phys; + dmaUnmap.size = length; + + ret = ioctl(this->fd, VFIO_IOMMU_UNMAP_DMA, &dmaUnmap); + if (ret) { + logger->error("Failed to unmap DMA mapping"); + return false; + } + + return true; +} + + +VfioGroup& +VfioContainer::getOrAttachGroup(int index) +{ + // search if group with index already exists + for(auto& group : groups) { + if(group->index == index) { + return *group; + } + } + + // group not yet part of this container, so acquire ownership + auto group = VfioGroup::attach(fd, index); + if(not group) { + logger->error("Failed to attach to IOMMU group: {}", index); + throw std::exception(); + } else { + logger->info("Attached new group {} to VFIO container", index); + } + + // push to our list + groups.push_back(std::move(group)); + + return *groups.back(); +} + + +VfioDevice::~VfioDevice() +{ + logger->debug("clean up device {} with fd {}", this->name, this->fd); + + for(auto& region : regions) { + regionUnmap(region.index); + } + + int ret = close(fd); + if (ret != 0) { + logger->error("Closing device fd {} failed", fd); + } + + logger->debug("VFIO: closed device: name={}, fd={}", name, fd); +} + + +bool +VfioDevice::reset() +{ + if (this->info.flags & VFIO_DEVICE_FLAGS_RESET) + return ioctl(this->fd, VFIO_DEVICE_RESET) == 0; + else + return false; /* not supported by this device */ +} + + +void* +VfioDevice::regionMap(size_t index) +{ + struct vfio_region_info *r = ®ions[index]; + + if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP)) + return MAP_FAILED; + + mappings[index] = mmap(nullptr, r->size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_32BIT, + fd, r->offset); + + return mappings[index]; +} + + +bool +VfioDevice::regionUnmap(size_t index) +{ + int ret; + struct vfio_region_info *r = ®ions[index]; + + if (!mappings[index]) + return false; /* was not mapped */ + + logger->debug("VFIO: unmap region {} from device", index); + + ret = munmap(mappings[index], r->size); + if (ret) + return false; + + mappings[index] = nullptr; + + return true; +} + + +size_t +VfioDevice::regionGetSize(size_t index) +{ + if(index >= regions.size()) { + logger->error("Index out of range: {} >= {}", index, regions.size()); + throw std::out_of_range("Index out of range"); + } + + return regions[index].size; +} + + +bool +VfioDevice::pciEnable() +{ + int ret; + uint32_t reg; + off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; + + /* Check if this is really a vfio-pci device */ + if (!(this->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return false; + + ret = pread(this->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return false; + + /* Enable memory access and PCI bus mastering which is required for DMA */ + reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + + ret = pwrite(this->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return false; + + return true; +} + + +bool +VfioDevice::pciHotReset() +{ + /* Check if this is really a vfio-pci device */ + if (not isVfioPciDevice()) + return false; + + const size_t reset_infolen = sizeof(struct vfio_pci_hot_reset_info) + + sizeof(struct vfio_pci_dependent_device) * 64; + auto reset_info = reinterpret_cast + (calloc(1, reset_infolen)); + + reset_info->argsz = reset_infolen; + + + if (ioctl(this->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info) != 0) { + free(reset_info); + return false; + } + + logger->debug("VFIO: dependent devices for hot-reset:"); + for (size_t i = 0; i < reset_info->count; i++) { + struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; + logger->debug("{:04x}:{:02x}:{:02x}.{:01x}: iommu_group={}", + dd->segment, dd->bus, + PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); + + if (static_cast(dd->group_id) != this->group.index) { + free(reset_info); + return false; + } + } + + const size_t resetlen = sizeof(struct vfio_pci_hot_reset) + + sizeof(int32_t) * 1; + auto reset = reinterpret_cast + (calloc(1, resetlen)); + + reset->argsz = resetlen; + reset->count = 1; + reset->group_fds[0] = this->group.fd; + + const bool success = ioctl(this->fd, VFIO_DEVICE_PCI_HOT_RESET, reset) == 0; + + free(reset); + free(reset_info); + + return success; +} + + +int +VfioDevice::pciMsiInit(int efds[]) +{ + /* Check if this is really a vfio-pci device */ + if(not isVfioPciDevice()) + return -1; + + const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + const size_t irqSetSize = sizeof(struct vfio_irq_set) + + sizeof(int) * irqCount; + + auto irqSet = reinterpret_cast(calloc(1, irqSetSize)); + if(irqSet == nullptr) + return -1; + + irqSet->argsz = irqSetSize; + irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irqSet->index = VFIO_PCI_MSI_IRQ_INDEX; + irqSet->start = 0; + irqSet->count = irqCount; + + /* Now set the new eventfds */ + for (size_t i = 0; i < irqCount; i++) { + efds[i] = eventfd(0, 0); + if (efds[i] < 0) { + free(irqSet); + return -1; + } + } + + memcpy(irqSet->data, efds, sizeof(int) * irqCount); + + if(ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) { + free(irqSet); + return -1; + } + + free(irqSet); + + return irqCount; +} + + +int +VfioDevice::pciMsiDeinit(int efds[]) +{ + /* Check if this is really a vfio-pci device */ + if(not isVfioPciDevice()) + return -1; + + const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + const size_t irqSetSize = sizeof(struct vfio_irq_set) + + sizeof(int) * irqCount; + + auto irqSet = reinterpret_cast(calloc(1, irqSetSize)); + if(irqSet == nullptr) + return -1; + + irqSet->argsz = irqSetSize; + irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irqSet->index = VFIO_PCI_MSI_IRQ_INDEX; + irqSet->count = irqCount; + irqSet->start = 0; + + for (size_t i = 0; i < irqCount; i++) { + close(efds[i]); + efds[i] = -1; + } + + memcpy(irqSet->data, efds, sizeof(int) * irqCount); + + if (ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) { + free(irqSet); + return -1; + } + + free(irqSet); + + return irqCount; +} + + +bool +VfioDevice::pciMsiFind(int nos[]) +{ + int ret, idx, irq; + char *end, *col, *last, line[1024], name[13]; + FILE *f; + + f = fopen("/proc/interrupts", "r"); + if (!f) + return false; + + for (int i = 0; i < 32; i++) + nos[i] = -1; + + /* For each line in /proc/interrupts */ + while (fgets(line, sizeof(line), f)) { + col = strtok(line, " "); + + /* IRQ number is in first column */ + irq = strtol(col, &end, 10); + if (col == end) + continue; + + /* Find last column of line */ + do { + last = col; + } while ((col = strtok(nullptr, " "))); + + + ret = sscanf(last, "vfio-msi[%u](%12[0-9:])", &idx, name); + if (ret == 2) { + if (strstr(this->name.c_str(), name) == this->name.c_str()) + nos[idx] = irq; + } + } + + fclose(f); + + return true; +} + + +bool +VfioDevice::isVfioPciDevice() const +{ + return info.flags & VFIO_DEVICE_FLAGS_PCI; +} + + +VfioGroup::~VfioGroup() +{ + logger->debug("clean up group {} with fd {}", this->index, this->fd); + int ret; + + /* Release memory and close fds */ + devices.clear(); + + if(fd < 0) { + logger->debug("Destructing group that has not been attached"); + } else { + ret = ioctl(fd, VFIO_GROUP_UNSET_CONTAINER); + if (ret != 0) { + logger->error("Cannot unset container for group fd {}", fd); + } + + logger->debug("Released group from container: group={}", index); + + ret = close(fd); + if (ret != 0) { + logger->error("Cannot close group fd {}", fd); + } + + logger->debug("Closed group: group={}, fd={}", index, fd); + } +} + + +std::unique_ptr +VfioGroup::attach(int containerFd, int groupIndex) +{ + std::unique_ptr group { new VfioGroup(groupIndex) }; + + /* Open group fd */ + std::stringstream groupPath; + groupPath << VFIO_DEV("") << groupIndex; + group->fd = open(groupPath.str().c_str(), O_RDWR); + if (group->fd < 0) { + logger->error("Failed to open VFIO group: {}", group->index); + return nullptr; + } + + logger->debug("VFIO group {} (fd {}) has path {}", + groupIndex, group->fd, groupPath.str()); + + int ret; + + /* Claim group ownership */ + ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &containerFd); + if (ret < 0) { + logger->error("Failed to attach VFIO group {} to container fd {} (error {})", + group->index, containerFd, ret); + return nullptr; + } + + /* Set IOMMU type */ + ret = ioctl(containerFd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret < 0) { + logger->error("Failed to set IOMMU type of container: {}", ret); + return nullptr; + } + + /* Check group viability and features */ + group->status.argsz = sizeof(group->status); + + ret = ioctl(group->fd, VFIO_GROUP_GET_STATUS, &group->status); + if (ret < 0) { + logger->error("Failed to get VFIO group status"); + return nullptr; + } + + if (!(group->status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + logger->error("VFIO group is not available: bind all devices to the VFIO driver!"); + return nullptr; + } + + return group; +} + +} // namespace villas + diff --git a/fpga/tests/fpga.cpp b/fpga/tests/fpga.cpp index 1f718ff51..fe9e79847 100644 --- a/fpga/tests/fpga.cpp +++ b/fpga/tests/fpga.cpp @@ -24,9 +24,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include "global.hpp" @@ -40,7 +40,6 @@ #define FPGA_AXI_HZ 125000000 static struct pci pci; -static struct vfio_container vc; FpgaState state; @@ -59,8 +58,7 @@ static void init() ret = pci_init(&pci); cr_assert_eq(ret, 0, "Failed to initialize PCI sub-system"); - ret = vfio_init(&vc); - cr_assert_eq(ret, 0, "Failed to initiliaze VFIO sub-system"); + auto vfioContainer = villas::VfioContainer::create(); /* Parse FPGA configuration */ f = fopen(TEST_CONFIG, "r"); @@ -81,7 +79,7 @@ static void init() villas::fpga::PCIeCardFactory* fpgaCardPlugin = dynamic_cast(plugin); // create all FPGA card instances using the corresponding plugin - state.cards = fpgaCardPlugin->make(fpgas, &pci, &vc); + state.cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer); cr_assert(state.cards.size() != 0, "No FPGA cards found!"); From 60882f1086190c3ed54f097b4bbbce2a48252b79 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:48:57 +0200 Subject: [PATCH 12/21] lib/memory: implement memory handling with allocators and blocks This commit is 2/2 of a series of patches and not working on its own. --- fpga/include/villas/fpga/card.hpp | 22 ++++- fpga/include/villas/fpga/ips/pcie.hpp | 1 + fpga/include/villas/memory.hpp | 122 +++++++++++++++++++++++++ fpga/include/villas/memory_manager.hpp | 16 +++- fpga/lib/CMakeLists.txt | 1 + fpga/lib/card.cpp | 84 +++++++++++++++-- fpga/lib/ips/pcie.cpp | 16 +++- fpga/lib/memory.cpp | 24 +++++ 8 files changed, 267 insertions(+), 19 deletions(-) create mode 100644 fpga/include/villas/memory.hpp create mode 100644 fpga/lib/memory.cpp diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index f8f9a0128..f9d78151a 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -38,6 +38,7 @@ #include "kernel/vfio.hpp" #include +#include #include #include "plugin.hpp" @@ -46,6 +47,7 @@ #include "config.h" #include "memory_manager.hpp" +#include "memory.hpp" #define PCI_FILTER_DEFAULT_FPGA { \ .id = { \ @@ -70,6 +72,7 @@ public: friend PCIeCardFactory; PCIeCard() : filter(PCI_FILTER_DEFAULT_FPGA) {} + ~PCIeCard(); bool init(); bool stop() { return true; } @@ -80,12 +83,19 @@ public: ip::IpCore* lookupIp(const std::string& name) const; ip::IpCore* lookupIp(const Vlnv& vlnv) const; + bool + mapMemoryBlock(const MemoryBlock& block); + +private: + /// Cache a set of already mapped memory blocks + std::set memoryBlocksMapped; + +public: // TODO: make this private ip::IpCoreList ips; ///< IPs located on this FPGA card bool do_reset; /**< Reset VILLASfpga during startup? */ int affinity; /**< Affinity for MSI interrupts */ - std::string name; /**< The name of the FPGA card */ struct pci *pci; @@ -97,17 +107,19 @@ public: /// The VFIO device that represents this card VfioDevice* vfioDevice; + /// Slave address space ID to access the PCIe address space from the FPGA + MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; + /// Address space identifier of the master address space of this FPGA card. /// This will be used for address resolution of all IPs on this card. - MemoryManager::AddressSpaceId addrSpaceId; - - size_t maplen; - size_t dmalen; + MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; protected: SpdLogger getLogger() const { return loggerGetOrCreate(name); } + + SpdLogger logger; }; using CardList = std::list>; diff --git a/fpga/include/villas/fpga/ips/pcie.hpp b/fpga/include/villas/fpga/ips/pcie.hpp index 290a8c577..28b01c5aa 100644 --- a/fpga/include/villas/fpga/ips/pcie.hpp +++ b/fpga/include/villas/fpga/ips/pcie.hpp @@ -51,6 +51,7 @@ public: private: static constexpr char axiInterface[] = "M_AXI"; + static constexpr char pcieMemory[] = "BAR0"; }; diff --git a/fpga/include/villas/memory.hpp b/fpga/include/villas/memory.hpp new file mode 100644 index 000000000..85dae02ea --- /dev/null +++ b/fpga/include/villas/memory.hpp @@ -0,0 +1,122 @@ +#pragma once + +#include +#include + +#include "log.hpp" +#include "memory_manager.hpp" + +namespace villas { + +class MemoryBlock { +protected: + MemoryBlock(MemoryManager::AddressSpaceId addrSpaceId, size_t size) : + addrSpaceId(addrSpaceId), size(size) {} + +public: + MemoryManager::AddressSpaceId getAddrSpaceId() const + { return addrSpaceId; } + + size_t getSize() const + { return size; } + +private: + MemoryManager::AddressSpaceId addrSpaceId; + size_t size; +}; + + +class MemoryAllocator { +}; + + +class HostRam : public MemoryAllocator { +public: + + template + class MemoryBlockHostRam : public MemoryBlock { + friend class HostRam; + private: + MemoryBlockHostRam(void* addr, size_t size, MemoryManager::AddressSpaceId foreignAddrSpaceId) : + MemoryBlock(foreignAddrSpaceId, size), + addr(addr), + translation(MemoryManager::get().getTranslationFromProcess(foreignAddrSpaceId)) + {} + public: + using Type = T; + + MemoryBlockHostRam() = delete; + + T& operator*() { + return *reinterpret_cast(translation.getLocalAddr(0)); + } + + T& operator [](int idx) { + const size_t offset = sizeof(T) * idx; + return *reinterpret_cast(translation.getLocalAddr(offset)); + } + + T* operator &() const { + return reinterpret_cast(translation.getLocalAddr(0)); + } + + private: + // addr needed for freeing later + void* addr; + + // cached memory translation for fast access + MemoryTranslation translation; + }; + + template + static MemoryBlockHostRam + allocate(size_t num) + { + /* Align to next bigger page size chunk */ + size_t length = num * sizeof(T); + if (length & size_t(0xFFF)) { + length += size_t(0x1000); + length &= size_t(~0xFFF); + } + + void* const addr = HostRam::allocate(length); + if(addr == nullptr) { + throw std::bad_alloc(); + } + + auto& mm = MemoryManager::get(); + + // assemble name for this block + std::stringstream name; + name << std::showbase << std::hex << reinterpret_cast(addr); + + auto blockAddrSpaceId = mm.getProcessAddressSpaceMemoryBlock(name.str()); + + // create mapping from VA space of process to this new block + mm.createMapping(reinterpret_cast(addr), 0, length, + "VA", + mm.getProcessAddressSpace(), + blockAddrSpaceId); + + // create object and corresponding address space in memory manager + return MemoryBlockHostRam(addr, length, blockAddrSpaceId); + } + + template + static inline bool + free(const MemoryBlockHostRam& block) + { + // TODO: remove address space from memory manager + // TODO: how to prevent use after free? + return HostRam::free(block.addr, block.size); + } + +private: + static void* + allocate(size_t length, int flags = 0); + + static bool + free(void*, size_t length); +}; + +} // namespace villas diff --git a/fpga/include/villas/memory_manager.hpp b/fpga/include/villas/memory_manager.hpp index 04bf60bf6..fb3db33c2 100644 --- a/fpga/include/villas/memory_manager.hpp +++ b/fpga/include/villas/memory_manager.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "log.hpp" #include "directed_graph.hpp" @@ -34,6 +35,10 @@ public: uintptr_t getForeignAddr(uintptr_t addrInLocalAddrSpace) const; + size_t + getSize() const + { return size; } + friend std::ostream& operator<< (std::ostream& stream, const MemoryTranslation& translation) { @@ -87,9 +92,9 @@ private: * the destination address space, where the mapping points to. Often, #dest * will be zero for mappings to hardware, but consider the example when * mapping FPGA to application memory: - * The application allocates a block 1kB at address - * 0x843001000 in its address space. The mapping would then have a #dest - * address of 0x843001000 and a #size of 1024. + * The application allocates a block 1kB at address 0x843001000 in its + * address space. The mapping would then have a #dest address of 0x843001000 + * and a #size of 1024. */ class Mapping : public graph::Edge { public: @@ -147,6 +152,11 @@ public: getProcessAddressSpace() { return getOrCreateAddressSpace("villas-fpga"); } + AddressSpaceId + getProcessAddressSpaceMemoryBlock(const std::string& memoryBlock) + { return getOrCreateAddressSpace(getSlaveAddrSpaceName("villas-fpga", memoryBlock)); } + + AddressSpaceId getOrCreateAddressSpace(std::string name); diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 7150b61eb..34901a74e 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -23,6 +23,7 @@ set(SOURCES plugin.cpp utils.cpp memory_manager.cpp + memory.cpp ) include(FindPkgConfig) diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index e0e2ba456..fb7369823 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -109,13 +109,33 @@ PCIeCardFactory::make(json_t *json, struct pci* pci, std::shared_ptrdebug("Unmap block {} at IOVA {:#x} of size {:#x}", + mappedMemoryBlock, iova, size); + vfioContainer->memoryUnmap(iova, size); + } +} + + ip::IpCore* PCIeCard::lookupIp(const std::string& name) const { @@ -124,9 +144,11 @@ PCIeCard::lookupIp(const std::string& name) const return ip.get(); } } + return nullptr; } + ip::IpCore* PCIeCard::lookupIp(const Vlnv& vlnv) const { @@ -135,17 +157,58 @@ PCIeCard::lookupIp(const Vlnv& vlnv) const return ip.get(); } } + return nullptr; } +bool +PCIeCard::mapMemoryBlock(const MemoryBlock& block) +{ + auto& mm = MemoryManager::get(); + const auto& addrSpaceId = block.getAddrSpaceId(); + + if(memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) { + // block already mapped + return true; + } else { + logger->debug("Create VFIO mapping for {}", addrSpaceId); + } + + + auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); + uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); + uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, + UINTPTR_MAX, + block.getSize()); + + if(iovaAddr == UINTPTR_MAX) { + logger->error("Cannot map memory at {:#x} of size {:#x}", + processBaseAddr, block.getSize()); + return false; + } + + + + mm.createMapping(iovaAddr, 0, block.getSize(), + "vfio", + this->addrSpaceIdDeviceToHost, + addrSpaceId); + + // remember that this block has already been mapped for later + memoryBlocksMapped.insert(addrSpaceId); + + return true; +} + + bool fpga::PCIeCard::init() { - int ret; struct pci_device *pdev; - auto logger = getLogger(); + auto& mm = MemoryManager::get(); + logger = getLogger(); logger->info("Initializing FPGA card {}", name); @@ -181,17 +244,18 @@ fpga::PCIeCard::init() /* Link mapped BAR0 to global memory graph */ // get the address space of the current application - auto villasAddrSpace = MemoryManager::get().getProcessAddressSpace(); + const auto villasAddrSpace = mm.getProcessAddressSpace(); + + // get the address space for the PCIe proxy we use with VFIO + const auto cardPCIeAddrSpaceName = mm.getMasterAddrSpaceName(name, "PCIe"); // create a new address space for this FPGA card - this->addrSpaceId = MemoryManager::get().getOrCreateAddressSpace(name); - + addrSpaceIdHostToDevice = mm.getOrCreateAddressSpace(cardPCIeAddrSpaceName); // create a mapping from our address space to the FPGA card via vfio - MemoryManager::get().createMapping(reinterpret_cast(bar0_mapped), + mm.createMapping(reinterpret_cast(bar0_mapped), 0, bar0_size, "VFIO_map", - villasAddrSpace, this->addrSpaceId); - + villasAddrSpace, addrSpaceIdHostToDevice); /* Reset system? */ diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index 5b02f1261..59174318b 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -38,6 +38,8 @@ static AxiPciExpressBridgeFactory factory; bool AxiPciExpressBridge::init() { + auto& mm = MemoryManager::get(); + // Throw an exception if the is no bus master interface and thus no // address space we can use for translation -> error const MemoryManager::AddressSpaceId myAddrSpaceid = @@ -47,7 +49,19 @@ AxiPciExpressBridge::init() // point to all other IPs in the FPGA, because Vivado will generate a // memory view for this bridge that can see all others. MemoryManager::get().createMapping(0x00, 0x00, SIZE_MAX, "PCIeBridge", - card->addrSpaceId, myAddrSpaceid); + card->addrSpaceIdHostToDevice, myAddrSpaceid); + + + /* Make PCIe (IOVA) address space available to FPGA via BAR0 */ + + // IPs that can access this address space will know it via their memory view + const auto addrSpaceNameDeviceToHost = + mm.getSlaveAddrSpaceName(getInstanceName(), pcieMemory); + + // save ID in card so we can create mappings later when needed (e.g. when + // allocating DMA memory in host RAM) + card->addrSpaceIdDeviceToHost = + mm.getOrCreateAddressSpace(addrSpaceNameDeviceToHost); return true; } diff --git a/fpga/lib/memory.cpp b/fpga/lib/memory.cpp new file mode 100644 index 000000000..f3d2802b4 --- /dev/null +++ b/fpga/lib/memory.cpp @@ -0,0 +1,24 @@ +#include +#include + +#include "memory.hpp" + +namespace villas { + +bool +HostRam::free(void* addr, size_t length) +{ + return munmap(addr, length) == 0; +} + + +void* +HostRam::allocate(size_t length, int flags) +{ + const int mmap_flags = flags | MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT; + const int mmap_protection = PROT_READ | PROT_WRITE; + + return mmap(nullptr, length, mmap_protection, mmap_flags, 0, 0); +} + +} // namespace villas From 272462278777431cefe7aa70931c5f274599e02f Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:40:58 +0200 Subject: [PATCH 13/21] kernel/pci: make some arguments const and fix debug output --- fpga/include/villas/kernel/pci.h | 6 +++--- fpga/lib/kernel/pci.c | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fpga/include/villas/kernel/pci.h b/fpga/include/villas/kernel/pci.h index db6beb0e6..10fd86ef6 100644 --- a/fpga/include/villas/kernel/pci.h +++ b/fpga/include/villas/kernel/pci.h @@ -58,13 +58,13 @@ int pci_device_compare(const struct pci_device *d, const struct pci_device *f); struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *filter); /** Get currently loaded driver for device */ -int pci_get_driver(struct pci_device *d, char *buf, size_t buflen); +int pci_get_driver(const struct pci_device *d, char *buf, size_t buflen); /** Bind a new LKM to the PCI device */ -int pci_attach_driver(struct pci_device *d, const char *driver); +int pci_attach_driver(const struct pci_device *d, const char *driver); /** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */ -int pci_get_iommu_group(struct pci_device *d); +int pci_get_iommu_group(const struct pci_device *d); #ifdef __cplusplus } diff --git a/fpga/lib/kernel/pci.c b/fpga/lib/kernel/pci.c index e4d6793ba..1f7336742 100644 --- a/fpga/lib/kernel/pci.c +++ b/fpga/lib/kernel/pci.c @@ -254,7 +254,7 @@ struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *f) return list_search(&p->devices, (cmp_cb_t) pci_device_compare, (void *) f); } -int pci_get_driver(struct pci_device *d, char *buf, size_t buflen) +int pci_get_driver(const struct pci_device *d, char *buf, size_t buflen) { int ret; char sysfs[1024], syml[1024]; @@ -273,7 +273,7 @@ int pci_get_driver(struct pci_device *d, char *buf, size_t buflen) return 0; } -int pci_attach_driver(struct pci_device *d, const char *driver) +int pci_attach_driver(const struct pci_device *d, const char *driver) { FILE *f; char fn[1024]; @@ -284,7 +284,7 @@ int pci_attach_driver(struct pci_device *d, const char *driver) if (!f) serror("Failed to add PCI id to %s driver (%s)", driver, fn); - debug(5, "Adding ID to %s module: %04x %04x", driver, d->id.vendor, d->id.device); + info("Adding ID to %s module: %04x %04x", driver, d->id.vendor, d->id.device); fprintf(f, "%04x %04x", d->id.vendor, d->id.device); fclose(f); @@ -294,14 +294,14 @@ int pci_attach_driver(struct pci_device *d, const char *driver) if (!f) serror("Failed to bind PCI device to %s driver (%s)", driver, fn); - debug(5, "Bind device to %s driver", driver); + info("Bind device to %s driver", driver); fprintf(f, "%04x:%02x:%02x.%x\n", d->slot.domain, d->slot.bus, d->slot.device, d->slot.function); fclose(f); return 0; } -int pci_get_iommu_group(struct pci_device *d) +int pci_get_iommu_group(const struct pci_device *d) { int ret; char *group, link[1024], sysfs[1024]; From aa1592ae2be143e1d14682e189637541eac7d9b1 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:41:40 +0200 Subject: [PATCH 14/21] utils: read_random() now returns the number of bytes written --- fpga/include/villas/utils.h | 2 +- fpga/lib/utils.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fpga/include/villas/utils.h b/fpga/include/villas/utils.h index 272afd6eb..828796fb6 100644 --- a/fpga/include/villas/utils.h +++ b/fpga/include/villas/utils.h @@ -240,7 +240,7 @@ int version_parse(const char *s, struct version *v); #endif /** Fill buffer with random data */ -ssize_t read_random(char *buf, size_t len); +size_t read_random(char *buf, size_t len); /** Get CPU timestep counter */ __attribute__((always_inline)) static inline uint64_t rdtsc() diff --git a/fpga/lib/utils.c b/fpga/lib/utils.c index 2e295191d..10fc8348f 100644 --- a/fpga/lib/utils.c +++ b/fpga/lib/utils.c @@ -260,14 +260,14 @@ void * memdup(const void *src, size_t bytes) return dst; } -ssize_t read_random(char *buf, size_t len) +size_t read_random(char *buf, size_t len) { int fd; ssize_t bytes, total; fd = open("/dev/urandom", O_RDONLY); if (fd < 0) - return -1; + return 0; bytes = 0; total = 0; From f025f5dcc7d5faeb7f6a1c72c50f9329e32f3c25 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:51:41 +0200 Subject: [PATCH 15/21] lib/ip: read base addresses from JSON into unsigned integer This caused sign extension for addresses with the MSB set which is obviously wrong since this is an address and not a number. With Jansson, there seems to be now other way since it only supports reading (signed) integers. --- fpga/lib/ip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index 41ca7c52e..d1d2d33a6 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -223,7 +223,7 @@ IpCoreFactory::make(PCIeCard* card, json_t *json_ips) json_t* json_block; json_object_foreach(json_instance, block_name, json_block) { - int base, high, size; + unsigned int base, high, size; int ret = json_unpack(json_block, "{ s: i, s: i, s: i }", "baseaddr", &base, "highaddr", &high, From 4f6694420faef4ab212fec800df46bf7f2bd4e79 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:56:00 +0200 Subject: [PATCH 16/21] lib: remove old and unused C files --- fpga/include/villas/fpga/card.h | 103 ---- fpga/include/villas/fpga/ip.h | 126 ----- fpga/include/villas/fpga/ips/dma.h | 88 ---- fpga/include/villas/fpga/ips/fifo.h | 52 -- fpga/include/villas/fpga/ips/intc.h | 56 --- fpga/include/villas/fpga/ips/switch.h | 67 --- fpga/include/villas/fpga/ips/timer.h | 43 -- fpga/include/villas/fpga/vlnv.h | 61 --- fpga/include/villas/plugin.h | 82 ---- fpga/lib/card.c | 315 ------------ fpga/lib/ip.c | 167 ------- fpga/lib/ips/dma.c | 657 -------------------------- fpga/lib/ips/fifo.c | 153 ------ fpga/lib/ips/intc.c | 180 ------- fpga/lib/ips/switch.c | 221 --------- fpga/lib/ips/timer.c | 61 --- fpga/lib/plugin.c | 111 ----- fpga/lib/vlnv.c | 64 --- 18 files changed, 2607 deletions(-) delete mode 100644 fpga/include/villas/fpga/card.h delete mode 100644 fpga/include/villas/fpga/ip.h delete mode 100644 fpga/include/villas/fpga/ips/dma.h delete mode 100644 fpga/include/villas/fpga/ips/fifo.h delete mode 100644 fpga/include/villas/fpga/ips/intc.h delete mode 100644 fpga/include/villas/fpga/ips/switch.h delete mode 100644 fpga/include/villas/fpga/ips/timer.h delete mode 100644 fpga/include/villas/fpga/vlnv.h delete mode 100644 fpga/include/villas/plugin.h delete mode 100644 fpga/lib/card.c delete mode 100644 fpga/lib/ip.c delete mode 100644 fpga/lib/ips/dma.c delete mode 100644 fpga/lib/ips/fifo.c delete mode 100644 fpga/lib/ips/intc.c delete mode 100644 fpga/lib/ips/switch.c delete mode 100644 fpga/lib/ips/timer.c delete mode 100644 fpga/lib/plugin.c delete mode 100644 fpga/lib/vlnv.c diff --git a/fpga/include/villas/fpga/card.h b/fpga/include/villas/fpga/card.h deleted file mode 100644 index f352fb081..000000000 --- a/fpga/include/villas/fpga/card.h +++ /dev/null @@ -1,103 +0,0 @@ -/** FPGA card - * - * This class represents a FPGA device. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include - -#include "common.h" -#include "kernel/pci.h" -#include "kernel/vfio.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* Forward declarations */ -struct fpga_ip; -struct vfio_container; - -struct fpga_card { - char *name; /**< The name of the FPGA card */ - - enum state state; /**< The state of this FPGA card. */ - - struct pci *pci; - struct pci_device filter; /**< Filter for PCI device. */ - - struct vfio_container *vfio_container; - struct vfio_device vfio_device; /**< VFIO device handle. */ - - int do_reset; /**< Reset VILLASfpga during startup? */ - int affinity; /**< Affinity for MSI interrupts */ - - struct list ips; /**< List of IP components on FPGA. */ - - char *map; /**< PCI BAR0 mapping for register access */ - - size_t maplen; - size_t dmalen; - - /* Some IP cores are special and referenced here */ - struct fpga_ip *intc; - struct fpga_ip *reset; - struct fpga_ip *sw; -}; - -/** Initialize FPGA card and its IP components. */ -int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc); - -/** Parse configuration of FPGA card including IP cores from config. */ -int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name); - -int fpga_card_parse_list(struct list *l, json_t *cfg); - -/** Check if the FPGA card configuration is plausible. */ -int fpga_card_check(struct fpga_card *c); - -/** Start FPGA card. */ -int fpga_card_start(struct fpga_card *c); - -/** Stop FPGA card. */ -int fpga_card_stop(struct fpga_card *c); - -/** Destroy FPGA card. */ -int fpga_card_destroy(struct fpga_card *c); - -/** Dump details of FPGA card to stdout. */ -void fpga_card_dump(struct fpga_card *c); - -/** Reset the FPGA to a known state */ -int fpga_card_reset(struct fpga_card *c); - -#ifdef __cplusplus -} -#endif - -/** @} */ diff --git a/fpga/include/villas/fpga/ip.h b/fpga/include/villas/fpga/ip.h deleted file mode 100644 index b6723043d..000000000 --- a/fpga/include/villas/fpga/ip.h +++ /dev/null @@ -1,126 +0,0 @@ -/** Interlectual Property component. - * - * This class represents a module within the FPGA. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include - -#include "common.h" - -#include "fpga/vlnv.h" - -#include "fpga/ips/dma.h" -#include "fpga/ips/switch.h" -#include "fpga/ips/fifo.h" -#include "fpga/ips/rtds_axis.h" -#include "fpga/ips/timer.h" -#include "fpga/ips/model.h" -#include "fpga/ips/dft.h" -#include "fpga/ips/intc.h" - -#ifdef __cplusplus -extern "C" { -#endif - -enum fpga_ip_types { - FPGA_IP_TYPE_DM_DMA, /**< A datamover IP exchanges streaming data between the FPGA and the CPU. */ - FPGA_IP_TYPE_DM_FIFO, /**< A datamover IP exchanges streaming data between the FPGA and the CPU. */ - FPGA_IP_TYPE_MODEL, /**< A model IP simulates a system on the FPGA. */ - FPGA_IP_TYPE_MATH, /**< A math IP performs some kind of mathematical operation on the streaming data */ - FPGA_IP_TYPE_MISC, /**< Other IP components like timer, counters, interrupt conctrollers or routing. */ - FPGA_IP_TYPE_INTERFACE /**< A interface IP connects the FPGA to another system or controller. */ -}; - -struct fpga_ip_type { - struct fpga_vlnv vlnv; - - enum fpga_ip_types type; - - int (*init)(struct fpga_ip *c); - int (*parse)(struct fpga_ip *c, json_t *cfg); - int (*check)(struct fpga_ip *c); - int (*start)(struct fpga_ip *c); - int (*stop)(struct fpga_ip *c); - int (*destroy)(struct fpga_ip *c); - int (*reset)(struct fpga_ip *c); - void (*dump)(struct fpga_ip *c); - - size_t size; /**< Amount of memory which should be reserved for struct fpga_ip::_vd */ -}; - -struct fpga_ip { - char *name; /**< Name of the FPGA IP component. */ - struct fpga_vlnv vlnv; /**< The Vendor, Library, Name, Version tag of the FPGA IP component. */ - - enum state state; /**< The current state of the FPGA IP component. */ - - struct fpga_ip_type *_vt; /**< Vtable containing FPGA IP type function pointers. */ - void *_vd; /**< Virtual data (used by struct fpga_ip::_vt functions) */ - - uintptr_t baseaddr; /**< The baseadress of this FPGA IP component */ - uintptr_t baseaddr_axi4; /**< Used by AXI4 FIFO DM */ - - int port; /**< The port of the AXI4-Stream switch to which this FPGA IP component is connected. */ - int irq; /**< The interrupt number of the FPGA IP component. */ - - struct fpga_card *card; /**< The FPGA to which this IP instance belongs to. */ -}; - -/** Initialize IP core. */ -int fpga_ip_init(struct fpga_ip *c, struct fpga_ip_type *vt); - -/** Parse IP core configuration from configuration file */ -int fpga_ip_parse(struct fpga_ip *c, json_t *cfg, const char *name); - -/** Check configuration of IP core. */ -int fpga_ip_check(struct fpga_ip *c); - -/** Start IP core. */ -int fpga_ip_start(struct fpga_ip *c); - -/** Stop IP core. */ -int fpga_ip_stop(struct fpga_ip *c); - -/** Release dynamic memory allocated by this IP core. */ -int fpga_ip_destroy(struct fpga_ip *c); - -/** Dump details about this IP core to stdout. */ -void fpga_ip_dump(struct fpga_ip *c); - -/** Reset IP component to its initial state. */ -int fpga_ip_reset(struct fpga_ip *c); - -/** Find a registered FPGA IP core type with the given VLNV identifier. */ -struct fpga_ip_type * fpga_ip_type_lookup(const char *vstr); - -#ifdef __cplusplus -} -#endif - -/** @} */ diff --git a/fpga/include/villas/fpga/ips/dma.h b/fpga/include/villas/fpga/ips/dma.h deleted file mode 100644 index 7a13fb903..000000000 --- a/fpga/include/villas/fpga/ips/dma.h +++ /dev/null @@ -1,88 +0,0 @@ -/** DMA related helper functions. - * - * These functions present a simpler interface to Xilinx' DMA driver (XAxiDma_*). - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include -#include -#include - -#include - -/* Forward declarations */ -struct fpga_ip; - -#define FPGA_DMA_BASEADDR 0x00000000 -#define FPGA_DMA_BOUNDARY 0x1000 -#define FPGA_DMA_BD_OFFSET 0xC0000000 -#define FPGA_DMA_BD_SIZE (32 << 20) // 32 MB - -#define XAXIDMA_SR_SGINCL_MASK 0x00000008 - -struct dma_mem { - char *base_virt; - char *base_phys; - size_t len; -}; - -struct dma { - XAxiDma inst; - - struct dma_mem bd; -}; - -struct ip; - -int dma_mem_split(struct dma_mem *o, struct dma_mem *a, struct dma_mem *b); - -int dma_alloc(struct fpga_ip *c, struct dma_mem *mem, size_t len, int flags); -int dma_free(struct fpga_ip *c, struct dma_mem *mem); - -int dma_write(struct fpga_ip *c, char *buf, size_t len); -int dma_read(struct fpga_ip *c, char *buf, size_t len); -int dma_read_complete(struct fpga_ip *c, char **buf, size_t *len); -int dma_write_complete(struct fpga_ip *c, char **buf, size_t *len); - -int dma_sg_write(struct fpga_ip *c, char *buf, size_t len); -int dma_sg_read(struct fpga_ip *c, char *buf, size_t len); - -int dma_sg_write_complete(struct fpga_ip *c, char **buf, size_t *len); -int dma_sg_read_complete(struct fpga_ip *c, char **buf, size_t *len); - -int dma_simple_read(struct fpga_ip *c, char *buf, size_t len); -int dma_simple_write(struct fpga_ip *c, char *buf, size_t len); - -int dma_simple_read_complete(struct fpga_ip *c, char **buf, size_t *len); -int dma_simple_write_complete(struct fpga_ip *c, char **buf, size_t *len); - -int dma_ping_pong(struct fpga_ip *c, char *src, char *dst, size_t len); - -int dma_start(struct fpga_ip *c); - -/** @} */ diff --git a/fpga/include/villas/fpga/ips/fifo.h b/fpga/include/villas/fpga/ips/fifo.h deleted file mode 100644 index 4724c1ac5..000000000 --- a/fpga/include/villas/fpga/ips/fifo.h +++ /dev/null @@ -1,52 +0,0 @@ -/** FIFO related helper functions - * - * These functions present a simpler interface to Xilinx' FIFO driver (XLlFifo_*) - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include - -#include -#include - -struct fifo { - XLlFifo inst; - - uint32_t baseaddr_axi4; -}; - -/* Forward declarations */ -struct ip; - -int fifo_start(struct fpga_ip *c); - -ssize_t fifo_write(struct fpga_ip *c, char *buf, size_t len); - -ssize_t fifo_read(struct fpga_ip *c, char *buf, size_t len); - -/** @} */ diff --git a/fpga/include/villas/fpga/ips/intc.h b/fpga/include/villas/fpga/ips/intc.h deleted file mode 100644 index bb7d44739..000000000 --- a/fpga/include/villas/fpga/ips/intc.h +++ /dev/null @@ -1,56 +0,0 @@ -/** AXI-PCIe Interrupt controller - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include - -enum intc_flags { - INTC_ENABLED = (1 << 0), - INTC_POLLING = (1 << 1) -}; - -struct intc { - int num_irqs; /**< Number of available MSI vectors */ - - int efds[32]; /**< Event FDs */ - int nos[32]; /**< Interrupt numbers from /proc/interrupts */ - - int flags[32]; /**< Mask of intc_flags */ -}; - -int intc_init(struct fpga_ip *c); - -int intc_destroy(struct fpga_ip *c); - -int intc_enable(struct fpga_ip *c, uint32_t mask, int poll); - -int intc_disable(struct fpga_ip *c, uint32_t mask); - -uint64_t intc_wait(struct fpga_ip *c, int irq); - -/** @} */ diff --git a/fpga/include/villas/fpga/ips/switch.h b/fpga/include/villas/fpga/ips/switch.h deleted file mode 100644 index 7fcf08fa6..000000000 --- a/fpga/include/villas/fpga/ips/switch.h +++ /dev/null @@ -1,67 +0,0 @@ -/** AXI Stream interconnect related helper functions - * - * These functions present a simpler interface to Xilinx' AXI Stream switch driver (XAxis_Switch_*) - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include -#include - -#include "list.h" - -/* Forward declarations */ -struct ip; - -struct sw_path { - const char *in; - const char *out; -}; - -struct sw { - XAxis_Switch inst; - - int num_ports; - struct list paths; -}; - -struct ip; - -int switch_start(struct fpga_ip *c); - -/** Initialize paths which have been parsed by switch_parse() */ -int switch_init_paths(struct fpga_ip *c); - -int switch_destroy(struct fpga_ip *c); - -int switch_parse(struct fpga_ip *c, json_t *cfg); - -int switch_connect(struct fpga_ip *c, struct fpga_ip *mi, struct fpga_ip *si); - -int switch_disconnect(struct fpga_ip *c, struct fpga_ip *mi, struct fpga_ip *si); - -/** @} */ diff --git a/fpga/include/villas/fpga/ips/timer.h b/fpga/include/villas/fpga/ips/timer.h deleted file mode 100644 index 31fac741e..000000000 --- a/fpga/include/villas/fpga/ips/timer.h +++ /dev/null @@ -1,43 +0,0 @@ -/** Timer related helper functions - * - * These functions present a simpler interface to Xilinx' Timer Counter driver (XTmrCtr_*) - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#pragma once - -#include - -/* Forward declarations */ -struct fpga_ip; - -struct timer { - XTmrCtr inst; -}; - -int timer_start(struct fpga_ip *c); - -/** @} */ diff --git a/fpga/include/villas/fpga/vlnv.h b/fpga/include/villas/fpga/vlnv.h deleted file mode 100644 index 2de8bbbcb..000000000 --- a/fpga/include/villas/fpga/vlnv.h +++ /dev/null @@ -1,61 +0,0 @@ -/** Vendor, Library, Name, Version (VLNV) tag. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -/** @addtogroup fpga VILLASfpga - * @{ - */ - -#ifndef _FPGA_VLNV_H_ -#define _FPGA_VLNV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Forward declarations */ -struct list; - -struct fpga_vlnv { - char *vendor; - char *library; - char *name; - char *version; -}; - -/** Return the first IP block in list \p l which matches the VLNV */ -struct fpga_ip * fpga_vlnv_lookup(struct list *l, struct fpga_vlnv *v); - -/** Check if IP block \p c matched VLNV. */ -int fpga_vlnv_cmp(struct fpga_vlnv *a, struct fpga_vlnv *b); - -/** Tokenizes VLNV \p vlnv and stores it into \p c */ -int fpga_vlnv_parse(struct fpga_vlnv *c, const char *vlnv); - -/** Release memory allocated by fpga_vlnv_parse(). */ -int fpga_vlnv_destroy(struct fpga_vlnv *v); - -#ifdef __cplusplus -} -#endif - -#endif /** _FPGA_VLNV_H_ @} */ diff --git a/fpga/include/villas/plugin.h b/fpga/include/villas/plugin.h deleted file mode 100644 index 8c915b573..000000000 --- a/fpga/include/villas/plugin.h +++ /dev/null @@ -1,82 +0,0 @@ -/** Loadable / plugin support. - * - * @file - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#pragma once - -#include "utils.h" -#include "fpga/ip.h" - -/** @todo This is ugly as hell and broken on OS X / Clang anyway. */ -#define REGISTER_PLUGIN(p) \ -__attribute__((constructor(110))) static void UNIQUE(__ctor)() {\ - if (plugins.state == STATE_DESTROYED) \ - list_init(&plugins); \ - list_push(&plugins, p); \ -} \ -__attribute__((destructor(110))) static void UNIQUE(__dtor)() { \ - if (plugins.state != STATE_DESTROYED) \ - list_remove(&plugins, p); \ -} - -extern struct list plugins; - -enum plugin_type { - PLUGIN_TYPE_FPGA_IP, -}; - -struct plugin { - const char *name; - const char *description; - void *handle; - char *path; - - enum plugin_type type; - - enum state state; - - int (*load)(struct plugin *p); - int (*unload)(struct plugin *p); - - struct fpga_ip_type ip; -}; - -/** Return a pointer to the plugin structure */ -#define plugin(vt) ((struct plugin *) ((char *) (vt) - offsetof(struct plugin, api))) - -#define plugin_name(vt) plugin(vt)->name -#define plugin_description(vt) plugin(vt)->description - -int plugin_init(struct plugin *p); - -int plugin_destroy(struct plugin *p); - -int plugin_parse(struct plugin *p, json_t *cfg); - -int plugin_load(struct plugin *p); - -int plugin_unload(struct plugin *p); - -void plugin_dump(enum plugin_type type); - -/** Find registered and loaded plugin with given name and type. */ -struct plugin * plugin_lookup(enum plugin_type type, const char *name); diff --git a/fpga/lib/card.c b/fpga/lib/card.c deleted file mode 100644 index 651ab76ac..000000000 --- a/fpga/lib/card.c +++ /dev/null @@ -1,315 +0,0 @@ -/** FPGA card. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include - -#include "config.h" -#include "log.h" -#include "log_config.h" -#include "list.h" -#include "utils.h" - -#include "kernel/pci.h" -#include "kernel/vfio.h" - -#include "fpga/ip.h" -#include "fpga/card.h" - -int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc) -{ - assert(c->state == STATE_DESTROYED); - - c->vfio_container = vc; - c->pci = pci; - - list_init(&c->ips); - - /* Default values */ - c->filter.id.vendor = FPGA_PCI_VID_XILINX; - c->filter.id.device = FPGA_PCI_PID_VFPGA; - - c->affinity = 0; - c->do_reset = 0; - - c->state = STATE_INITIALIZED; - - return 0; -} - -int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name) -{ - int ret; - - json_t *json_ips; - json_t *json_slot = NULL; - json_t *json_id = NULL; - json_error_t err; - - c->name = strdup(name); - - ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: b, s?: o, s?: o, s: o }", - "affinity", &c->affinity, - "do_reset", &c->do_reset, - "slot", &json_slot, - "id", &json_id, - "ips", &json_ips - ); - if (ret) - jerror(&err, "Failed to parse FPGA vard configuration"); - - if (json_slot) { - const char *err, *slot; - - slot = json_string_value(json_slot); - if (slot) { - ret = pci_device_parse_slot(&c->filter, slot, &err); - if (ret) - error("Failed to parse PCI slot: %s", err); - } - else - error("PCI slot must be a string"); - } - - if (json_id) { - const char *err, *id; - - id = json_string_value(json_id); - if (id) { - ret = pci_device_parse_id(&c->filter, (char*) id, &err); - if (ret) - error("Failed to parse PCI id: %s", err); - } - else - error("PCI ID must be a string"); - } - - if (!json_is_object(json_ips)) - error("FPGA card IPs section must be an object"); - - const char *name_ip; - json_t *json_ip; - json_object_foreach(json_ips, name_ip, json_ip) { - const char *vlnv; - - struct fpga_ip_type *vt; - struct fpga_ip *ip = (struct fpga_ip *) alloc(sizeof(struct fpga_ip)); - - ip->card = c; - - ret = json_unpack_ex(json_ip, &err, 0, "{ s: s }", "vlnv", &vlnv); - if (ret) - error("Failed to parse FPGA IP '%s' of card '%s'", name_ip, name); - - vt = fpga_ip_type_lookup(vlnv); - if (!vt) - error("FPGA IP core VLNV identifier '%s' is invalid", vlnv); - - ret = fpga_ip_init(ip, vt); - if (ret) - error("Failed to initalize FPGA IP core"); - - ret = fpga_ip_parse(ip, json_ip, name_ip); - if (ret) - error("Failed to parse FPGA IP core"); - - list_push(&c->ips, ip); - } - - c->state = STATE_PARSED; - - return 0; -} - -int fpga_card_parse_list(struct list *cards, json_t *cfg) -{ - int ret; - - if (!json_is_object(cfg)) - error("FPGA card configuration section must be a JSON object"); - - const char *name; - json_t *json_fpga; - json_object_foreach(cfg, name, json_fpga) { - struct fpga_card *c = (struct fpga_card *) alloc(sizeof(struct fpga_card)); - - ret = fpga_card_parse(c, json_fpga, name); - if (ret) - error("Failed to parse FPGA card configuration"); - - list_push(cards, c); - } - - return 0; -} - -int fpga_card_start(struct fpga_card *c) -{ - int ret; - - struct pci_device *pdev; - - assert(c->state == STATE_INITIALIZED); - - /* Search for FPGA card */ - pdev = pci_lookup_device(c->pci, &c->filter); - if (!pdev) - error("Failed to find PCI device"); - - /* Attach PCIe card to VFIO container */ - ret = vfio_pci_attach(&c->vfio_device, c->vfio_container, pdev); - if (ret) - error("Failed to attach VFIO device"); - - /* Map PCIe BAR */ - c->map = vfio_map_region(&c->vfio_device, VFIO_PCI_BAR0_REGION_INDEX); - if (c->map == MAP_FAILED) - serror("Failed to mmap() BAR0"); - - /* Enable memory access and PCI bus mastering for DMA */ - ret = vfio_pci_enable(&c->vfio_device); - if (ret) - serror("Failed to enable PCI device"); - - /* Reset system? */ - if (c->do_reset) { - /* Reset / detect PCI device */ - ret = vfio_pci_reset(&c->vfio_device); - if (ret) - serror("Failed to reset PCI device"); - - ret = fpga_card_reset(c); - if (ret) - error("Failed to reset FGPA card"); - } - - /* Initialize IP cores */ - for (size_t j = 0; j < list_length(&c->ips); j++) { - struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); - - ret = fpga_ip_start(i); - if (ret) - error("Failed to initalize FPGA IP core: %s (%u)", i->name, ret); - } - - c->state = STATE_STARTED; - - return 0; -} - -int fpga_card_stop(struct fpga_card *c) -{ - int ret; - - assert(c->state == STATE_STOPPED); - - for (size_t j = 0; j < list_length(&c->ips); j++) { - struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); - - ret = fpga_ip_stop(i); - if (ret) - error("Failed to stop FPGA IP core: %s (%u)", i->name, ret); - } - - c->state = STATE_STOPPED; - - return 0; -} - -void fpga_card_dump(struct fpga_card *c) -{ - info("VILLASfpga card:"); - { INDENT - info("Slot: %04x:%02x:%02x.%d", c->vfio_device.pci_device->slot.domain, c->vfio_device.pci_device->slot.bus, c->vfio_device.pci_device->slot.device, c->vfio_device.pci_device->slot.function); - info("Vendor ID: %04x", c->vfio_device.pci_device->id.vendor); - info("Device ID: %04x", c->vfio_device.pci_device->id.device); - info("Class ID: %04x", c->vfio_device.pci_device->id.class_code); - - info("BAR0 mapped at %p", c->map); - - info("IP blocks:"); - for (size_t j = 0; j < list_length(&c->ips); j++) { INDENT - struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); - - fpga_ip_dump(i); - } - } - - vfio_dump(c->vfio_device.group->container); -} - -int fpga_card_check(struct fpga_card *c) -{ - assert(c->state == STATE_PARSED); - - /* Check FPGA configuration */ - c->reset = fpga_vlnv_lookup(&c->ips, &(struct fpga_vlnv) { "xilinx.com", "ip", "axi_gpio", NULL }); - if (!c->reset) - error("FPGA is missing a reset controller"); - - c->intc = fpga_vlnv_lookup(&c->ips, &(struct fpga_vlnv) { "acs.eonerc.rwth-aachen.de", "user", "axi_pcie_intc", NULL }); - if (!c->intc) - error("FPGA is missing a interrupt controller"); - - c->sw = fpga_vlnv_lookup(&c->ips, &(struct fpga_vlnv) { "xilinx.com", "ip", "axis_interconnect", NULL }); - if (!c->sw) - warn("FPGA is missing an AXI4-Stream switch"); - - return 0; -} - -int fpga_card_destroy(struct fpga_card *c) -{ - list_destroy(&c->ips, (dtor_cb_t) fpga_ip_destroy, true); - - return 0; -} - -int fpga_card_reset(struct fpga_card *c) -{ - int ret; - char state[4096]; - - /* Save current state of PCI configuration space */ - ret = pread(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); - if (ret != sizeof(state)) - return -1; - - uint32_t *rst_reg = (uint32_t *) (c->map + c->reset->baseaddr); - - debug(3, "FPGA: reset"); - rst_reg[0] = 1; - - usleep(100000); - - /* Restore previous state of PCI configuration space */ - ret = pwrite(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); - if (ret != sizeof(state)) - return -1; - - /* After reset the value should be zero again */ - if (rst_reg[0]) - return -2; - - c->state = STATE_INITIALIZED; - - return 0; -} diff --git a/fpga/lib/ip.c b/fpga/lib/ip.c deleted file mode 100644 index 816bf2e7f..000000000 --- a/fpga/lib/ip.c +++ /dev/null @@ -1,167 +0,0 @@ -/** FPGA IP component. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include "log_config.h" -#include "log.h" -#include "plugin.h" - -int fpga_ip_init(struct fpga_ip *c, struct fpga_ip_type *vt) -{ - int ret; - - assert(c->state == STATE_DESTROYED); - - c->_vt = vt; - c->_vd = alloc(vt->size); - - ret = c->_vt->init ? c->_vt->init(c) : 0; - if (ret) - return ret; - - c->state = STATE_INITIALIZED; - - debug(8, "IP Core %s initalized (%u)", c->name, ret); - - return ret; -} - -int fpga_ip_parse(struct fpga_ip *c, json_t *cfg, const char *name) -{ - int ret, baseaddr = -1; - - assert(c->state != STATE_STARTED && c->state != STATE_DESTROYED); - - c->name = strdup(name); - c->baseaddr = -1; - c->irq = -1; - c->port = -1; - - json_error_t err; - - ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: i, s?: i }", - "baseaddr", &baseaddr, - "irq", &c->irq, - "port", &c->port - ); - if (ret) - jerror(&err, "Failed to parse configuration for FPGA IP '%s'", name); - - c->baseaddr = baseaddr; - - /* Type sepecific settings */ - ret = c->_vt && c->_vt->parse ? c->_vt->parse(c, cfg) : 0; - if (ret) - error("Failed to parse settings for IP core '%s'", name); - - c->state = STATE_PARSED; - - return 0; -} - -int fpga_ip_start(struct fpga_ip *c) -{ - int ret; - - assert(c->state == STATE_CHECKED); - - ret = c->_vt->start ? c->_vt->start(c) : 0; - if (ret) - return ret; - - c->state = STATE_STARTED; - - return 0; -} - -int fpga_ip_stop(struct fpga_ip *c) -{ - int ret; - - assert(c->state == STATE_STARTED); - - ret = c->_vt->stop ? c->_vt->stop(c) : 0; - if (ret) - return ret; - - c->state = STATE_STOPPED; - - return 0; -} - -int fpga_ip_destroy(struct fpga_ip *c) -{ - int ret; - - assert(c->state != STATE_DESTROYED); - - fpga_vlnv_destroy(&c->vlnv); - - ret = c->_vt->destroy ? c->_vt->destroy(c) : 0; - if (ret) - return ret; - - c->state = STATE_DESTROYED; - - free(c->_vd); - - return 0; -} - -int fpga_ip_reset(struct fpga_ip *c) -{ - debug(3, "Reset IP core: %s", c->name); - - return c->_vt->reset ? c->_vt->reset(c) : 0; -} - -void fpga_ip_dump(struct fpga_ip *c) -{ - assert(c->state != STATE_DESTROYED); - - info("IP %s: vlnv=%s:%s:%s:%s baseaddr=%#jx, irq=%d, port=%d", - c->name, c->vlnv.vendor, c->vlnv.library, c->vlnv.name, c->vlnv.version, - c->baseaddr, c->irq, c->port); - - if (c->_vt->dump) - c->_vt->dump(c); -} - -struct fpga_ip_type * fpga_ip_type_lookup(const char *vstr) -{ - int ret; - - struct fpga_vlnv vlnv; - - ret = fpga_vlnv_parse(&vlnv, vstr); - if (ret) - return NULL; - - /* Try to find matching IP type */ - for (size_t i = 0; i < list_length(&plugins); i++) { - struct plugin *p = (struct plugin *) list_at(&plugins, i); - - if (p->type == PLUGIN_TYPE_FPGA_IP && !fpga_vlnv_cmp(&vlnv, &p->ip.vlnv)) - return &p->ip; - } - - return NULL; -} diff --git a/fpga/lib/ips/dma.c b/fpga/lib/ips/dma.c deleted file mode 100644 index c02175709..000000000 --- a/fpga/lib/ips/dma.c +++ /dev/null @@ -1,657 +0,0 @@ -/** DMA related helper functions - * - * These functions present a simpler interface to Xilinx' DMA driver (XAxiDma_*) - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include -#include -#include - -#include "log.h" -#include "plugin.h" -#include "utils.h" - -#include "fpga/ip.h" -#include "fpga/card.h" -#include "fpga/ips/dma.h" - -int dma_mem_split(struct dma_mem *o, struct dma_mem *a, struct dma_mem *b) -{ - int split = o->len / 2; - - a->base_virt = o->base_virt; - a->base_phys = o->base_phys; - - b->base_virt = a->base_virt + split; - b->base_phys = a->base_phys + split; - - a->len = split; - b->len = o->len - split; - - return 0; -} - -int dma_alloc(struct fpga_ip *c, struct dma_mem *mem, size_t len, int flags) -{ - int ret; - - struct fpga_card *f = c->card; - - /* Align to next bigger page size chunk */ - if (len & 0xFFF) { - len += 0x1000; - len &= ~0xFFF; - } - - mem->len = len; - mem->base_phys = (void *) -1; /* find free */ - mem->base_virt = mmap(0, mem->len, PROT_READ | PROT_WRITE, flags | MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT, 0, 0); - if (mem->base_virt == MAP_FAILED) - return -1; - - ret = vfio_map_dma(f->vfio_device.group->container, (uint64_t) mem->base_virt, (uint64_t) mem->base_phys, mem->len); - if (ret) - return -2; - - return 0; -} - -int dma_free(struct fpga_ip *c, struct dma_mem *mem) -{ - int ret; - - ret = vfio_unmap_dma(c->card->vfio_device.group->container, (uint64_t) mem->base_virt, (uint64_t) mem->base_phys, mem->len); - if (ret) - return ret; - - ret = munmap(mem->base_virt, mem->len); - if (ret) - return ret; - - return 0; -} - -int dma_ping_pong(struct fpga_ip *c, char *src, char *dst, size_t len) -{ - int ret; - - ret = dma_read(c, dst, len); - if (ret) - return ret; - - ret = dma_write(c, src, len); - if (ret) - return ret; - - ret = dma_write_complete(c, NULL, NULL); - if (ret) - return ret; - - ret = dma_read_complete(c, NULL, NULL); - if (ret) - return ret; - - return 0; -} - -int dma_write(struct fpga_ip *c, char *buf, size_t len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - - debug(25, "DMA write: dmac=%s buf=%p len=%#zx", c->name, buf, len); - - return xdma->HasSg - ? dma_sg_write(c, buf, len) - : dma_simple_write(c, buf, len); -} - -int dma_read(struct fpga_ip *c, char *buf, size_t len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - - debug(25, "DMA read: dmac=%s buf=%p len=%#zx", c->name, buf, len); - - return xdma->HasSg - ? dma_sg_read(c, buf, len) - : dma_simple_read(c, buf, len); -} - -int dma_read_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - - debug(25, "DMA read complete: dmac=%s", c->name); - - return xdma->HasSg - ? dma_sg_read_complete(c, buf, len) - : dma_simple_read_complete(c, buf, len); -} - -int dma_write_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - - debug(25, "DMA write complete: dmac=%s", c->name); - - return xdma->HasSg - ? dma_sg_write_complete(c, buf, len) - : dma_simple_write_complete(c, buf, len); -} - -int dma_sg_write(struct fpga_ip *c, char *buf, size_t len) -{ - int ret, bdcnt; - - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetTxRing(xdma); - XAxiDma_Bd *bds, *bd; - - uint32_t remaining, bdlen, bdbuf, cr; - - /* Checks */ - if (!xdma->HasSg) - return -1; - - if (len < 1) - return -2; - - if (!xdma->HasMm2S) - return -3; - - if (!ring->HasDRE) { - uint32_t mask = xdma->MicroDmaMode ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN : ring->DataWidth - 1; - if ((uintptr_t) buf & mask) - return -4; - } - - bdcnt = CEIL(len, FPGA_DMA_BOUNDARY); - ret = XAxiDma_BdRingAlloc(ring, bdcnt, &bds); - if (ret != XST_SUCCESS) - return -5; - - remaining = len; - bdbuf = (uintptr_t) buf; - bd = bds; - for (int i = 0; i < bdcnt; i++) { - bdlen = MIN(remaining, FPGA_DMA_BOUNDARY); - - ret = XAxiDma_BdSetBufAddr(bd, bdbuf); - if (ret != XST_SUCCESS) - goto out; - - ret = XAxiDma_BdSetLength(bd, bdlen, ring->MaxTransferLen); - if (ret != XST_SUCCESS) - goto out; - - /* Set SOF / EOF / ID */ - cr = 0; - if (i == 0) - cr |= XAXIDMA_BD_CTRL_TXSOF_MASK; - if (i == bdcnt - 1) - cr |= XAXIDMA_BD_CTRL_TXEOF_MASK; - - XAxiDma_BdSetCtrl(bd, cr); - XAxiDma_BdSetId(bd, (uintptr_t) buf); - - remaining -= bdlen; - bdbuf += bdlen; - bd = (XAxiDma_Bd *) XAxiDma_BdRingNext(ring, bd); - } - - /* Give the BD to DMA to kick off the transmission. */ - ret = XAxiDma_BdRingToHw(ring, bdcnt, bds); - if (ret != XST_SUCCESS) - return -8; - - return 0; - -out: - ret = XAxiDma_BdRingUnAlloc(ring, bdcnt, bds); - if (ret != XST_SUCCESS) - return -6; - - return -5; -} - -int dma_sg_read(struct fpga_ip *c, char *buf, size_t len) -{ - int ret, bdcnt; - - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetRxRing(xdma); - XAxiDma_Bd *bds, *bd; - - uint32_t remaining, bdlen, bdbuf; - - /* Checks */ - if (!xdma->HasSg) - return -1; - - if (len < 1) - return -2; - - if (!xdma->HasS2Mm) - return -3; - - if (!ring->HasDRE) { - uint32_t mask = xdma->MicroDmaMode ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN : ring->DataWidth - 1; - if ((uintptr_t) buf & mask) - return -4; - } - - bdcnt = CEIL(len, FPGA_DMA_BOUNDARY); - ret = XAxiDma_BdRingAlloc(ring, bdcnt, &bds); - if (ret != XST_SUCCESS) - return -5; - - bdbuf = (uintptr_t) buf; - remaining = len; - bd = bds; - for (int i = 0; i < bdcnt; i++) { - bdlen = MIN(remaining, FPGA_DMA_BOUNDARY); - ret = XAxiDma_BdSetLength(bd, bdlen, ring->MaxTransferLen); - if (ret != XST_SUCCESS) - goto out; - - ret = XAxiDma_BdSetBufAddr(bd, bdbuf); - if (ret != XST_SUCCESS) - goto out; - - /* Receive BDs do not need to set anything for the control - * The hardware will set the SOF/EOF bits per stream ret */ - XAxiDma_BdSetCtrl(bd, 0); - XAxiDma_BdSetId(bd, (uintptr_t) buf); - - remaining -= bdlen; - bdbuf += bdlen; - bd = (XAxiDma_Bd *) XAxiDma_BdRingNext(ring, bd); - } - - ret = XAxiDma_BdRingToHw(ring, bdcnt, bds); - if (ret != XST_SUCCESS) - return -8; - - return 0; - -out: - ret = XAxiDma_BdRingUnAlloc(ring, bdcnt, bds); - if (ret != XST_SUCCESS) - return -6; - - return -5; -} - -int dma_sg_write_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetTxRing(xdma); - XAxiDma_Bd *bds; - - int processed, ret; - - /* Wait until the one BD TX transaction is done */ - while (!(XAxiDma_IntrGetIrq(xdma, XAXIDMA_DMA_TO_DEVICE) & XAXIDMA_IRQ_IOC_MASK)) - intc_wait(c->card->intc, c->irq); - XAxiDma_IntrAckIrq(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); - - processed = XAxiDma_BdRingFromHw(ring, XAXIDMA_ALL_BDS, &bds); - - if (len != NULL) - *len = XAxiDma_BdGetActualLength(bds, XAXIDMA_MAX_TRANSFER_LEN); - - if (buf != NULL) - *buf = (char *) (uintptr_t) XAxiDma_BdGetId(bds); - - /* Free all processed TX BDs for future transmission */ - ret = XAxiDma_BdRingFree(ring, processed, bds); - if (ret != XST_SUCCESS) - return -1; - - return 0; -} - -int dma_sg_read_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetRxRing(xdma); - XAxiDma_Bd *bds, *bd; - - int ret, bdcnt; - uint32_t recvlen, sr; - uintptr_t recvbuf = 0; - - if (!xdma->HasSg) - return -1; - - while (!(XAxiDma_IntrGetIrq(xdma, XAXIDMA_DEVICE_TO_DMA) & XAXIDMA_IRQ_IOC_MASK)) - intc_wait(c->card->intc, c->irq + 1); - XAxiDma_IntrAckIrq(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); - - bdcnt = XAxiDma_BdRingFromHw(ring, XAXIDMA_ALL_BDS, &bds); - - recvlen = 0; - - bd = bds; - for (int i = 0; i < bdcnt; i++) { - recvlen += XAxiDma_BdGetActualLength(bd, ring->MaxTransferLen); - - sr = XAxiDma_BdGetSts(bd); - if (sr & XAXIDMA_BD_STS_RXSOF_MASK) - if (i != 0) - warn("sof not first"); - - if (sr & XAXIDMA_BD_STS_RXEOF_MASK) - if (i != bdcnt - 1) - warn("eof not last"); - - recvbuf = XAxiDma_BdGetId(bd); - - bd = (XAxiDma_Bd *) XAxiDma_BdRingNext(ring, bd); - } - - if (len != NULL) - *len = recvlen; - if (buf != NULL) - *buf = (char *) recvbuf; - - /* Free all processed RX BDs for future transmission */ - ret = XAxiDma_BdRingFree(ring, bdcnt, bds); - if (ret != XST_SUCCESS) - return -3; - - return 0; -} - -int dma_simple_read(struct fpga_ip *c, char *buf, size_t len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetRxRing(xdma); - - /* Checks */ - if (xdma->HasSg) - return -1; - - if ((len < 1) || (len > FPGA_DMA_BOUNDARY)) - return -2; - - if (!xdma->HasS2Mm) - return -3; - - if (!ring->HasDRE) { - uint32_t mask = xdma->MicroDmaMode ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN : ring->DataWidth - 1; - if ((uintptr_t) buf & mask) - return -4; - } - - if(!(XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK)) { - if (XAxiDma_Busy(xdma, XAXIDMA_DEVICE_TO_DMA)) - return -5; - } - - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_OFFSET, LOWER_32_BITS((uintptr_t) buf)); - if (xdma->AddrWidth > 32) - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_MSB_OFFSET, UPPER_32_BITS((uintptr_t) buf)); - - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_CR_OFFSET, XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_CR_OFFSET) | XAXIDMA_CR_RUNSTOP_MASK); - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET, len); - - return XST_SUCCESS; -} - -int dma_simple_write(struct fpga_ip *c, char *buf, size_t len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetTxRing(xdma); - - /* Checks */ - if (xdma->HasSg) - return -1; - - if ((len < 1) || (len > FPGA_DMA_BOUNDARY)) - return -2; - - if (!xdma->HasMm2S) - return -3; - - if (!ring->HasDRE) { - uint32_t mask = xdma->MicroDmaMode ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN : ring->DataWidth - 1; - if ((uintptr_t) buf & mask) - return -4; - } - - /* If the engine is doing transfer, cannot submit */ - if(!(XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK)) { - if (XAxiDma_Busy(xdma, XAXIDMA_DMA_TO_DEVICE)) - return -5; - } - - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_SRCADDR_OFFSET, LOWER_32_BITS((uintptr_t) buf)); - if (xdma->AddrWidth > 32) - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_SRCADDR_MSB_OFFSET, UPPER_32_BITS((uintptr_t) buf)); - - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_CR_OFFSET, XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_CR_OFFSET) | XAXIDMA_CR_RUNSTOP_MASK); - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET, len); - - return XST_SUCCESS; -} - -int dma_simple_read_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetRxRing(xdma); - - while (!(XAxiDma_IntrGetIrq(xdma, XAXIDMA_DEVICE_TO_DMA) & XAXIDMA_IRQ_IOC_MASK)) - intc_wait(c->card->intc, c->irq + 1); - XAxiDma_IntrAckIrq(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); - - if (len) - *len = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET); - - if (buf) { - *buf = (char *) (uintptr_t) XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_DESTADDR_OFFSET); - if (xdma->AddrWidth > 32) - *buf += XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_DESTADDR_MSB_OFFSET); - } - - return 0; -} - -int dma_simple_write_complete(struct fpga_ip *c, char **buf, size_t *len) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - XAxiDma_BdRing *ring = XAxiDma_GetTxRing(xdma); - - while (!(XAxiDma_IntrGetIrq(xdma, XAXIDMA_DMA_TO_DEVICE) & XAXIDMA_IRQ_IOC_MASK)) - intc_wait(c->card->intc, c->irq); - XAxiDma_IntrAckIrq(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); - - if (len) - *len = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET); - - if (buf) { - *buf = (char *) (uintptr_t) XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SRCADDR_OFFSET); - if (xdma->AddrWidth > 32) - *buf += XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SRCADDR_MSB_OFFSET); - } - - return 0; -} - -static int dma_setup_ring(XAxiDma_BdRing *ring, struct dma_mem *bdbuf) -{ - int delay = 0; - int coalesce = 1; - int ret, cnt; - - XAxiDma_Bd clearbd; - - /* Disable all RX interrupts before RxBD space setup */ - XAxiDma_BdRingIntDisable(ring, XAXIDMA_IRQ_ALL_MASK); - - /* Set delay and coalescing */ - XAxiDma_BdRingSetCoalesce(ring, coalesce, delay); - - /* Setup Rx BD space */ - cnt = XAxiDma_BdRingCntCalc(XAXIDMA_BD_MINIMUM_ALIGNMENT, bdbuf->len); - - ret = XAxiDma_BdRingCreate(ring, (uintptr_t) bdbuf->base_phys, (uintptr_t) bdbuf->base_virt, XAXIDMA_BD_MINIMUM_ALIGNMENT, cnt); - if (ret != XST_SUCCESS) - return -1; - - XAxiDma_BdClear(&clearbd); - ret = XAxiDma_BdRingClone(ring, &clearbd); - if (ret != XST_SUCCESS) - return -2; - - /* Start the channel */ - ret = XAxiDma_BdRingStart(ring); - if (ret != XST_SUCCESS) - return -3; - - return XST_SUCCESS; -} - -static int dma_init_rings(XAxiDma *xdma, struct dma_mem *bd) -{ - int ret; - - struct dma_mem bd_rx, bd_tx; - - ret = dma_mem_split(bd, &bd_rx, &bd_tx); - if (ret) - return -1; - - ret = dma_setup_ring(XAxiDma_GetRxRing(xdma), &bd_rx); - if (ret != XST_SUCCESS) - return -2; - - ret = dma_setup_ring(XAxiDma_GetTxRing(xdma), &bd_tx); - if (ret != XST_SUCCESS) - return -3; - - return 0; -} - -int dma_start(struct fpga_ip *c) -{ - int ret, sg; - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma *xdma = &dma->inst; - - /* Guess DMA type */ - sg = (XAxiDma_In32((uintptr_t) c->card->map + c->baseaddr + XAXIDMA_TX_OFFSET+ XAXIDMA_SR_OFFSET) & - XAxiDma_In32((uintptr_t) c->card->map + c->baseaddr + XAXIDMA_RX_OFFSET+ XAXIDMA_SR_OFFSET) & XAXIDMA_SR_SGINCL_MASK) ? 1 : 0; - - XAxiDma_Config xdma_cfg = { - .BaseAddr = (uintptr_t) c->card->map + c->baseaddr, - .HasStsCntrlStrm = 0, - .HasMm2S = 1, - .HasMm2SDRE = 1, - .Mm2SDataWidth = 128, - .HasS2Mm = 1, - .HasS2MmDRE = 1, /* Data Realignment Engine */ - .HasSg = sg, - .S2MmDataWidth = 128, - .Mm2sNumChannels = 1, - .S2MmNumChannels = 1, - .Mm2SBurstSize = 64, - .S2MmBurstSize = 64, - .MicroDmaMode = 0, - .AddrWidth = 32 - }; - - ret = XAxiDma_CfgInitialize(xdma, &xdma_cfg); - if (ret != XST_SUCCESS) - return -1; - - /* Perform selftest */ - ret = XAxiDma_Selftest(xdma); - if (ret != XST_SUCCESS) - return -2; - - /* Map buffer descriptors */ - if (xdma->HasSg) { - ret = dma_alloc(c, &dma->bd, FPGA_DMA_BD_SIZE, 0); - if (ret) - return -3; - - ret = dma_init_rings(xdma, &dma->bd); - if (ret) - return -4; - } - - /* Enable completion interrupts for both channels */ - XAxiDma_IntrEnable(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); - XAxiDma_IntrEnable(xdma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); - - return 0; -} - -int dma_reset(struct fpga_ip *c) -{ - struct dma *dma = (struct dma *) c->_vd; - - XAxiDma_Reset(&dma->inst); - - return 0; -} - -static struct plugin p = { - .name = "Xilinx's AXI4 Direct Memory Access Controller", - .description = "Transfer data streams between VILLASnode and VILLASfpga", - .type = PLUGIN_TYPE_FPGA_IP, - .ip = { - .vlnv = { "xilinx.com", "ip", "axi_dma", NULL }, - .type = FPGA_IP_TYPE_DM_DMA, - .init = dma_start, - .reset = dma_reset, - .size = sizeof(struct dma) - } -}; - -REGISTER_PLUGIN(&p) diff --git a/fpga/lib/ips/fifo.c b/fpga/lib/ips/fifo.c deleted file mode 100644 index 1f4c058f3..000000000 --- a/fpga/lib/ips/fifo.c +++ /dev/null @@ -1,153 +0,0 @@ -/** FIFO related helper functions - * - * These functions present a simpler interface to Xilinx' FIFO driver (XLlFifo_*) - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include - -#include "utils.h" -#include "plugin.h" - -#include "fpga/ip.h" -#include "fpga/card.h" -#include "fpga/ips/fifo.h" -#include "fpga/ips/intc.h" - -int fifo_start(struct fpga_ip *c) -{ - int ret; - - struct fpga_card *f = c->card; - struct fifo *fifo = (struct fifo *) c->_vd; - - XLlFifo *xfifo = &fifo->inst; - XLlFifo_Config fifo_cfg = { - .BaseAddress = (uintptr_t) f->map + c->baseaddr, - .Axi4BaseAddress = (uintptr_t) c->card->map + fifo->baseaddr_axi4, - .Datainterface = (fifo->baseaddr_axi4 != -1) ? 1 : 0 /* use AXI4 for Data, AXI4-Lite for control */ - }; - - ret = XLlFifo_CfgInitialize(xfifo, &fifo_cfg, (uintptr_t) c->card->map + c->baseaddr); - if (ret != XST_SUCCESS) - return -1; - - XLlFifo_IntEnable(xfifo, XLLF_INT_RC_MASK); /* Receive complete IRQ */ - - return 0; -} - -int fifo_stop(struct fpga_ip *c) -{ - struct fifo *fifo = (struct fifo *) c->_vd; - - XLlFifo *xfifo = &fifo->inst; - - XLlFifo_IntDisable(xfifo, XLLF_INT_RC_MASK); /* Receive complete IRQ */ - - return 0; -} - -ssize_t fifo_write(struct fpga_ip *c, char *buf, size_t len) -{ - struct fifo *fifo = (struct fifo *) c->_vd; - - XLlFifo *xllfifo = &fifo->inst; - - uint32_t tdfv; - - tdfv = XLlFifo_TxVacancy(xllfifo); - if (tdfv < len) - return -1; - - XLlFifo_Write(xllfifo, buf, len); - XLlFifo_TxSetLen(xllfifo, len); - - return len; -} - -ssize_t fifo_read(struct fpga_ip *c, char *buf, size_t len) -{ - struct fifo *fifo = (struct fifo *) c->_vd; - - XLlFifo *xllfifo = &fifo->inst; - - size_t nextlen = 0; - uint32_t rxlen; - - while (!XLlFifo_IsRxDone(xllfifo)) - intc_wait(c->card->intc, c->irq); - XLlFifo_IntClear(xllfifo, XLLF_INT_RC_MASK); - - /* Get length of next frame */ - rxlen = XLlFifo_RxGetLen(xllfifo); - nextlen = MIN(rxlen, len); - - /* Read from FIFO */ - XLlFifo_Read(xllfifo, buf, nextlen); - - return nextlen; -} - -int fifo_parse(struct fpga_ip *c, json_t *cfg) -{ - struct fifo *fifo = (struct fifo *) c->_vd; - - int baseaddr_axi4 = -1, ret; - - json_error_t err; - - fifo->baseaddr_axi4 = -1; - - ret = json_unpack_ex(cfg, &err, 0, "{ s?: i }", "baseaddr_axi4", &baseaddr_axi4); - if (ret) - jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); - - fifo->baseaddr_axi4 = baseaddr_axi4; - - return 0; -} - -int fifo_reset(struct fpga_ip *c) -{ - struct fifo *fifo = (struct fifo *) c->_vd; - - XLlFifo_Reset(&fifo->inst); - - return 0; -} - -static struct plugin p = { - .name = "Xilinx's AXI4 FIFO data mover", - .description = "", - .type = PLUGIN_TYPE_FPGA_IP, - .ip = { - .vlnv = { "xilinx.com", "ip", "axi_fifo_mm_s", NULL }, - .type = FPGA_IP_TYPE_DM_FIFO, - .start = fifo_start, - .stop = fifo_stop, - .parse = fifo_parse, - .reset = fifo_reset, - .size = sizeof(struct fifo) - } -}; - -REGISTER_PLUGIN(&p) diff --git a/fpga/lib/ips/intc.c b/fpga/lib/ips/intc.c deleted file mode 100644 index 7fcf109d6..000000000 --- a/fpga/lib/ips/intc.c +++ /dev/null @@ -1,180 +0,0 @@ -/** AXI-PCIe Interrupt controller - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include - -#include "config.h" -#include "log.h" -#include "plugin.h" - -#include "kernel/vfio.h" -#include "kernel/kernel.h" - -#include "fpga/ip.h" -#include "fpga/card.h" -#include "fpga/ips/intc.h" - -int intc_start(struct fpga_ip *c) -{ - int ret; - - struct fpga_card *f = c->card; - struct intc *intc = (struct intc *) c->_vd; - - uintptr_t base = (uintptr_t) f->map + c->baseaddr; - - if (c != f->intc) - error("There can be only one interrupt controller per FPGA"); - - intc->num_irqs = vfio_pci_msi_init(&f->vfio_device, intc->efds); - if (intc->num_irqs < 0) - return -1; - - ret = vfio_pci_msi_find(&f->vfio_device, intc->nos); - if (ret) - return -2; - - /* For each IRQ */ - for (int i = 0; i < intc->num_irqs; i++) { - /* Pin to core */ - ret = kernel_irq_setaffinity(intc->nos[i], f->affinity, NULL); - if (ret) - serror("Failed to change affinity of VFIO-MSI interrupt"); - - /* Setup vector */ - XIntc_Out32(base + XIN_IVAR_OFFSET + i * 4, i); - } - - XIntc_Out32(base + XIN_IMR_OFFSET, 0); /* Use manual acknowlegement for all IRQs */ - XIntc_Out32(base + XIN_IAR_OFFSET, 0xFFFFFFFF); /* Acknowlege all pending IRQs manually */ - XIntc_Out32(base + XIN_IMR_OFFSET, 0xFFFFFFFF); /* Use fast acknowlegement for all IRQs */ - XIntc_Out32(base + XIN_IER_OFFSET, 0x00000000); /* Disable all IRQs by default */ - XIntc_Out32(base + XIN_MER_OFFSET, XIN_INT_HARDWARE_ENABLE_MASK | XIN_INT_MASTER_ENABLE_MASK); - - debug(4, "FPGA: enabled interrupts"); - - return 0; -} - -int intc_destroy(struct fpga_ip *c) -{ - struct fpga_card *f = c->card; - struct intc *intc = (struct intc *) c->_vd; - - vfio_pci_msi_deinit(&f->vfio_device, intc->efds); - - return 0; -} - -int intc_enable(struct fpga_ip *c, uint32_t mask, int flags) -{ - struct fpga_card *f = c->card; - struct intc *intc = (struct intc *) c->_vd; - - uint32_t ier, imr; - uintptr_t base = (uintptr_t) f->map + c->baseaddr; - - /* Current state of INTC */ - ier = XIntc_In32(base + XIN_IER_OFFSET); - imr = XIntc_In32(base + XIN_IMR_OFFSET); - - /* Clear pending IRQs */ - XIntc_Out32(base + XIN_IAR_OFFSET, mask); - - for (int i = 0; i < intc->num_irqs; i++) { - if (mask & (1 << i)) - intc->flags[i] = flags; - } - - if (flags & INTC_POLLING) { - XIntc_Out32(base + XIN_IMR_OFFSET, imr & ~mask); - XIntc_Out32(base + XIN_IER_OFFSET, ier & ~mask); - } - else { - XIntc_Out32(base + XIN_IER_OFFSET, ier | mask); - XIntc_Out32(base + XIN_IMR_OFFSET, imr | mask); - } - - debug(3, "New ier = %#x", XIntc_In32(base + XIN_IER_OFFSET)); - debug(3, "New imr = %#x", XIntc_In32(base + XIN_IMR_OFFSET)); - debug(3, "New isr = %#x", XIntc_In32(base + XIN_ISR_OFFSET)); - - debug(8, "FPGA: Interupt enabled: mask=%#x flags=%#x", mask, flags); - - return 0; -} - -int intc_disable(struct fpga_ip *c, uint32_t mask) -{ - struct fpga_card *f = c->card; - - uintptr_t base = (uintptr_t) f->map + c->baseaddr; - uint32_t ier = XIntc_In32(base + XIN_IER_OFFSET); - - XIntc_Out32(base + XIN_IER_OFFSET, ier & ~mask); - - return 0; -} - -uint64_t intc_wait(struct fpga_ip *c, int irq) -{ - struct fpga_card *f = c->card; - struct intc *intc = (struct intc *) c->_vd; - - uintptr_t base = (uintptr_t) f->map + c->baseaddr; - - if (intc->flags[irq] & INTC_POLLING) { - uint32_t isr, mask = 1 << irq; - - do { - isr = XIntc_In32(base + XIN_ISR_OFFSET); - pthread_testcancel(); - } while ((isr & mask) != mask); - - XIntc_Out32(base + XIN_IAR_OFFSET, mask); - - return 1; - } - else { - uint64_t cnt; - ssize_t ret = read(intc->efds[irq], &cnt, sizeof(cnt)); - if (ret != sizeof(cnt)) - return 0; - - return cnt; - } -} - -static struct plugin p = { - .name = "Xilinx's programmable interrupt controller", - .description = "", - .type = PLUGIN_TYPE_FPGA_IP, - .ip = { - .vlnv = { "acs.eonerc.rwth-aachen.de", "user", "axi_pcie_intc", NULL }, - .type = FPGA_IP_TYPE_MISC, - .start = intc_start, - .destroy = intc_destroy, - .size = sizeof(struct intc) - } -}; - -REGISTER_PLUGIN(&p) diff --git a/fpga/lib/ips/switch.c b/fpga/lib/ips/switch.c deleted file mode 100644 index 73c647be1..000000000 --- a/fpga/lib/ips/switch.c +++ /dev/null @@ -1,221 +0,0 @@ -/** AXI Stream interconnect related helper functions - * - * These functions present a simpler interface to Xilinx' AXI Stream switch driver (XAxis_Switch_*) - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include "list.h" -#include "log.h" -#include "log_config.h" -#include "plugin.h" - -#include "fpga/ip.h" -#include "fpga/card.h" -#include "fpga/ips/switch.h" - -int switch_start(struct fpga_ip *c) -{ - int ret; - - struct fpga_card *f = c->card; - struct sw *sw = (struct sw *) c->_vd; - - XAxis_Switch *xsw = &sw->inst; - - if (c != f->sw) - error("There can be only one AXI4-Stream interconnect per FPGA"); - - - /* Setup AXI-stream switch */ - XAxis_Switch_Config sw_cfg = { - .BaseAddress = (uintptr_t) f->map + c->baseaddr, - .MaxNumMI = sw->num_ports, - .MaxNumSI = sw->num_ports - }; - - ret = XAxisScr_CfgInitialize(xsw, &sw_cfg, (uintptr_t) c->card->map + c->baseaddr); - if (ret != XST_SUCCESS) - return -1; - - /* Disable all masters */ - XAxisScr_RegUpdateDisable(xsw); - XAxisScr_MiPortDisableAll(xsw); - XAxisScr_RegUpdateEnable(xsw); - - switch_init_paths(c); - - return 0; -} - -int switch_init_paths(struct fpga_ip *c) -{ - int ret; - struct sw *sw = (struct sw *) c->_vd; - - XAxis_Switch *xsw = &sw->inst; - - XAxisScr_RegUpdateDisable(xsw); - XAxisScr_MiPortDisableAll(xsw); - - for (size_t i = 0; i < list_length(&sw->paths); i++) { - struct sw_path *p = (struct sw_path *) list_at(&sw->paths, i); - struct fpga_ip *mi, *si; - - mi = list_lookup(&c->card->ips, p->out); - si = list_lookup(&c->card->ips, p->in); - - if (!mi || !si || mi->port == -1 || si->port == -1) - error("Invalid path configuration for FPGA"); - - ret = switch_connect(c, mi, si); - if (ret) - error("Failed to configure switch"); - } - - XAxisScr_RegUpdateEnable(xsw); - - return 0; -} - -int switch_destroy(struct fpga_ip *c) -{ - struct sw *sw = (struct sw *) c->_vd; - - list_destroy(&sw->paths, NULL, true); - - return 0; -} - -int switch_parse(struct fpga_ip *c, json_t *cfg) -{ - struct sw *sw = (struct sw *) c->_vd; - - int ret; - size_t index; - json_error_t err; - json_t *json_path, *json_paths = NULL; - - list_init(&sw->paths); - - ret = json_unpack_ex(cfg, &err, 0, "{ s: i, s?: o }", - "num_ports", &sw->num_ports, - "paths", &json_paths - ); - if (ret) - jerror(&err, "Failed to parse configuration of FPGA IP '%s'", c->name); - - if (!json_paths) - return 0; /* no switch config available */ - - if (!json_is_array(json_paths)) - error("Setting 'paths' of FPGA IP '%s' should be an array of JSON objects", c->name); - - json_array_foreach(json_paths, index, json_path) { - struct sw_path *p = (struct sw_path *) alloc(sizeof(struct sw_path)); - int reverse = 0; - - ret = json_unpack_ex(json_path, &err, 0, "{ s?: b, s: s, s: s }", - "reverse", &reverse, - "in", &p->in, - "out", &p->out - ); - if (ret) - jerror(&err, "Failed to parse path %zu of FPGA IP '%s'", index, c->name); - - list_push(&sw->paths, p); - - if (reverse) { - struct sw_path *r = memdup(p, sizeof(struct sw_path)); - - r->in = p->out; - r->out = p->in; - - list_push(&sw->paths, r); - } - } - - return 0; -} - -int switch_connect(struct fpga_ip *c, struct fpga_ip *mi, struct fpga_ip *si) -{ - struct sw *sw = (struct sw *) c->_vd; - XAxis_Switch *xsw = &sw->inst; - - uint32_t mux, port; - - /* Check if theres already something connected */ - for (int i = 0; i < sw->num_ports; i++) { - mux = XAxisScr_ReadReg(xsw->Config.BaseAddress, XAXIS_SCR_MI_MUX_START_OFFSET + i * 4); - if (!(mux & XAXIS_SCR_MI_X_DISABLE_MASK)) { - port = mux & ~XAXIS_SCR_MI_X_DISABLE_MASK; - - if (port == si->port) { - warn("Switch: Slave port %s (%u) has been connected already to port %u. Disconnecting...", si->name, si->port, i); - XAxisScr_RegUpdateDisable(xsw); - XAxisScr_MiPortDisable(xsw, i); - XAxisScr_RegUpdateEnable(xsw); - } - } - } - - /* Reconfigure switch */ - XAxisScr_RegUpdateDisable(xsw); - XAxisScr_MiPortEnable(xsw, mi->port, si->port); - XAxisScr_RegUpdateEnable(xsw); - - /* Reset IPs */ - /*ip_reset(mi); - ip_reset(si);*/ - - debug(8, "FPGA: Switch connected %s (%u) to %s (%u)", mi->name, mi->port, si->name, si->port); - - return 0; -} - -int switch_disconnect(struct fpga_ip *c, struct fpga_ip *mi, struct fpga_ip *si) -{ - struct sw *sw = (struct sw *) c->_vd; - XAxis_Switch *xsw = &sw->inst; - - if (!XAxisScr_IsMiPortEnabled(xsw, mi->port, si->port)) - return -1; - - XAxisScr_MiPortDisable(xsw, mi->port); - - return 0; -} - -static struct plugin p = { - .name = "Xilinx's AXI4-Stream switch", - .description = "", - .type = PLUGIN_TYPE_FPGA_IP, - .ip = { - .vlnv = { "xilinx.com", "ip", "axis_interconnect", NULL }, - .type = FPGA_IP_TYPE_MISC, - .start = switch_start, - .destroy = switch_destroy, - .parse = switch_parse, - .size = sizeof(struct sw) - } -}; - -REGISTER_PLUGIN(&p) diff --git a/fpga/lib/ips/timer.c b/fpga/lib/ips/timer.c deleted file mode 100644 index 57eb1a67a..000000000 --- a/fpga/lib/ips/timer.c +++ /dev/null @@ -1,61 +0,0 @@ -/** Timer related helper functions - * - * These functions present a simpler interface to Xilinx' Timer Counter driver (XTmrCtr_*) - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include "config.h" -#include "plugin.h" - -#include "fpga/ip.h" -#include "fpga/card.h" -#include "fpga/ips/timer.h" - -int timer_start(struct fpga_ip *c) -{ - struct fpga_card *f = c->card; - struct timer *tmr = (struct timer *) c->_vd; - - XTmrCtr *xtmr = &tmr->inst; - XTmrCtr_Config xtmr_cfg = { - .BaseAddress = (uintptr_t) f->map + c->baseaddr, - .SysClockFreqHz = FPGA_AXI_HZ - }; - - XTmrCtr_CfgInitialize(xtmr, &xtmr_cfg, (uintptr_t) f->map + c->baseaddr); - XTmrCtr_InitHw(xtmr); - - return 0; -} - -static struct plugin p = { - .name = "Xilinx's programmable timer / counter", - .description = "", - .type = PLUGIN_TYPE_FPGA_IP, - .ip = { - .vlnv = { "xilinx.com", "ip", "axi_timer", NULL }, - .type = FPGA_IP_TYPE_MISC, - .start = timer_start, - .size = sizeof(struct timer) - } -}; - -REGISTER_PLUGIN(&p) diff --git a/fpga/lib/plugin.c b/fpga/lib/plugin.c deleted file mode 100644 index 908d8eea4..000000000 --- a/fpga/lib/plugin.c +++ /dev/null @@ -1,111 +0,0 @@ -/** Loadable / plugin support. - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -#include "plugin.h" - -/** Global list of all known plugins */ -struct list plugins = { .state = STATE_DESTROYED }; - -LIST_INIT_STATIC(&plugins) - -int plugin_init(struct plugin *p) -{ - assert(p->state == STATE_DESTROYED); - - p->state = STATE_INITIALIZED; - - return 0; -} - -int plugin_parse(struct plugin *p, json_t *cfg) -{ - const char *path; - - path = json_string_value(cfg); - if (!path) - return -1; - - p->path = strdup(path); - - return 0; -} - -int plugin_load(struct plugin *p) -{ - p->handle = dlopen(p->path, RTLD_NOW); - if (!p->path) - return -1; - - p->state = STATE_LOADED; - - return 0; -} - -int plugin_unload(struct plugin *p) -{ - int ret; - - assert(p->state == STATE_LOADED); - - ret = dlclose(p->handle); - if (ret) - return -1; - - p->state = STATE_UNLOADED; - - return 0; -} - -int plugin_destroy(struct plugin *p) -{ - assert(p->state != STATE_DESTROYED && p->state != STATE_LOADED); - - if (p->path) - free(p->path); - - return 0; -} - -struct plugin * plugin_lookup(enum plugin_type type, const char *name) -{ - for (size_t i = 0; i < list_length(&plugins); i++) { - struct plugin *p = (struct plugin *) list_at(&plugins, i); - - if (p->type == type && strcmp(p->name, name) == 0) - return p; - } - - return NULL; -} - -void plugin_dump(enum plugin_type type) -{ - for (size_t i = 0; i < list_length(&plugins); i++) { - struct plugin *p = (struct plugin *) list_at(&plugins, i); - - if (p->type == type) - printf(" - %-13s: %s\n", p->name, p->description); - } -} diff --git a/fpga/lib/vlnv.c b/fpga/lib/vlnv.c deleted file mode 100644 index e6c693302..000000000 --- a/fpga/lib/vlnv.c +++ /dev/null @@ -1,64 +0,0 @@ -/** Vendor, Library, Name, Version (VLNV) tag - * - * @author Steffen Vogel - * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -#include "fpga/vlnv.h" -#include "fpga/ip.h" - -struct fpga_ip * fpga_vlnv_lookup(struct list *l, struct fpga_vlnv *v) -{ - return (struct fpga_ip *) list_search(l, (cmp_cb_t) fpga_vlnv_cmp, v); -} - -int fpga_vlnv_cmp(struct fpga_vlnv *a, struct fpga_vlnv *b) -{ - return ((!a->vendor || !b->vendor || !strcmp(a->vendor, b->vendor )) && - (!a->library || !b->library || !strcmp(a->library, b->library)) && - (!a->name || !b->name || !strcmp(a->name, b->name )) && - (!a->version || !b->version || !strcmp(a->version, b->version))) ? 0 : 1; -} - -int fpga_vlnv_parse(struct fpga_vlnv *c, const char *vlnv) -{ - char *tmp = strdup(vlnv); - - c->vendor = strdup(strtok(tmp, ":")); - c->library = strdup(strtok(NULL, ":")); - c->name = strdup(strtok(NULL, ":")); - c->version = strdup(strtok(NULL, ":")); - - free(tmp); - - return 0; -} - -int fpga_vlnv_destroy(struct fpga_vlnv *v) -{ - free(v->vendor); - free(v->library); - free(v->name); - free(v->version); - - return 0; -} From 507ea77ad6727d0a251e144c207d43739c919e68 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 16:14:37 +0200 Subject: [PATCH 17/21] ips/dma: add (simple) DMA driver --- fpga/include/villas/fpga/ips/dma.hpp | 119 +++++++++ fpga/lib/CMakeLists.txt | 1 + fpga/lib/ips/dma.cpp | 355 +++++++++++++++++++++++++++ 3 files changed, 475 insertions(+) create mode 100644 fpga/include/villas/fpga/ips/dma.hpp create mode 100644 fpga/lib/ips/dma.cpp diff --git a/fpga/include/villas/fpga/ips/dma.hpp b/fpga/include/villas/fpga/ips/dma.hpp new file mode 100644 index 000000000..b5cd34f51 --- /dev/null +++ b/fpga/include/villas/fpga/ips/dma.hpp @@ -0,0 +1,119 @@ +/** DMA driver + * + * @author Daniel Krebs + * @copyright 2018, RWTH Institute for Automation of Complex Power Systems (ACS) + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + ******************************************************************************/ + +/** @addtogroup fpga VILLASfpga + * @{ + */ + +#pragma once + +#include +#include + +#include + +#include "fpga/ip_node.hpp" +#include "memory.hpp" + +namespace villas { +namespace fpga { +namespace ip { + +class Dma : public IpNode +{ +public: + friend class DmaFactory; + + bool init(); + bool reset(); + + size_t write(const MemoryBlock& mem, size_t len); + size_t read(const MemoryBlock& mem, size_t len); + + bool writeComplete() + { return hasScatterGather() ? writeCompleteSG() : writeCompleteSimple(); } + + bool readComplete() + { return hasScatterGather() ? readCompleteSG() : readCompleteSimple(); } + + bool pingPong(const MemoryBlock& src, const MemoryBlock& dst, size_t len); + + inline bool + hasScatterGather() const + { return hasSG; } + +private: + size_t writeSG(const void* buf, size_t len); + size_t readSG(void* buf, size_t len); + bool writeCompleteSG(); + bool readCompleteSG(); + + size_t writeSimple(const void* buf, size_t len); + size_t readSimple(void* buf, size_t len); + bool writeCompleteSimple(); + bool readCompleteSimple(); + +private: + static constexpr char registerMemory[] = "Reg"; + + static constexpr char mm2sInterrupt[] = "mm2s_introut"; + static constexpr char mm2sInterface[] = "M_AXI_MM2S"; + + static constexpr char s2mmInterrupt[] = "s2mm_introut"; + static constexpr char s2mmInterface[] = "M_AXI_S2MM"; + + // optional Scatter-Gather interface to access descriptors + static constexpr char sgInterface[] = "M_AXI_SG"; + + std::list getMemoryBlocks() const + { return { registerMemory }; } + + XAxiDma xDma; + bool hasSG; +}; + + + +class DmaFactory : public IpNodeFactory { +public: + DmaFactory(); + + IpCore* create() + { return new Dma; } + + std::string + getName() const + { return "Dma"; } + + std::string + getDescription() const + { return "Xilinx's AXI4 Direct Memory Access Controller"; } + + Vlnv getCompatibleVlnv() const + { return {"xilinx.com:ip:axi_dma:"}; } +}; + +} // namespace ip +} // namespace fpga +} // namespace villas + +/** @} */ diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 34901a74e..f16350efe 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -9,6 +9,7 @@ set(SOURCES ips/fifo.cpp ips/intc.cpp ips/pcie.cpp + ips/dma.cpp kernel/kernel.c kernel/pci.c diff --git a/fpga/lib/ips/dma.cpp b/fpga/lib/ips/dma.cpp new file mode 100644 index 000000000..3a5878441 --- /dev/null +++ b/fpga/lib/ips/dma.cpp @@ -0,0 +1,355 @@ +/** DMA driver + * + * @author Daniel Krebs + * @copyright 2018, RWTH Institute for Automation of Complex Power Systems (ACS) + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + ******************************************************************************/ + +#include +#include + +#include + +#include "fpga/card.hpp" +#include "fpga/ips/dma.hpp" +#include "fpga/ips/intc.hpp" + +#include "log.hpp" +#include "memory_manager.hpp" + +// max. size of a DMA transfer in simple mode +#define FPGA_DMA_BOUNDARY 0x1000 + + +namespace villas { +namespace fpga { +namespace ip { + +// instantiate factory to make available to plugin infrastructure +static DmaFactory factory; + +DmaFactory::DmaFactory() : + IpNodeFactory(getName()) +{ + // nothing to do +} + + +bool +Dma::init() +{ + // if there is a scatter-gather interface, then this instance has it + hasSG = busMasterInterfaces.count(sgInterface) == 1; + logger->info("Scatter-Gather support: {}", hasScatterGather()); + + XAxiDma_Config xdma_cfg; + + xdma_cfg.BaseAddr = getBaseAddr(registerMemory); + xdma_cfg.HasStsCntrlStrm = 0; + xdma_cfg.HasMm2S = 1; + xdma_cfg.HasMm2SDRE = 1; + xdma_cfg.Mm2SDataWidth = 128; + xdma_cfg.HasS2Mm = 1; + xdma_cfg.HasS2MmDRE = 1; /* Data Realignment Engine */ + xdma_cfg.HasSg = hasScatterGather(); + xdma_cfg.S2MmDataWidth = 128; + xdma_cfg.Mm2sNumChannels = 1; + xdma_cfg.S2MmNumChannels = 1; + xdma_cfg.Mm2SBurstSize = 64; + xdma_cfg.S2MmBurstSize = 64; + xdma_cfg.MicroDmaMode = 0; + xdma_cfg.AddrWidth = 32; + + if (XAxiDma_CfgInitialize(&xDma, &xdma_cfg) != XST_SUCCESS) { + logger->error("Cannot initialize Xilinx DMA driver"); + return false; + } + + if (XAxiDma_Selftest(&xDma) != XST_SUCCESS) { + logger->error("DMA selftest failed"); + return false; + } else { + logger->debug("DMA selftest passed"); + } + + /* Map buffer descriptors */ + if (hasScatterGather()) { + logger->warn("Scatter Gather not yet implemented"); + return false; + +// ret = dma_alloc(c, &dma->bd, FPGA_DMA_BD_SIZE, 0); +// if (ret) +// return -3; + +// ret = dma_init_rings(&xDma, &dma->bd); +// if (ret) +// return -4; + } + + /* Enable completion interrupts for both channels */ + XAxiDma_IntrEnable(&xDma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); + XAxiDma_IntrEnable(&xDma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); + + irqs[mm2sInterrupt].irqController->enableInterrupt(irqs[mm2sInterrupt], false); + irqs[s2mmInterrupt].irqController->enableInterrupt(irqs[s2mmInterrupt], false); + + return true; +} + + +bool +Dma::reset() +{ + XAxiDma_Reset(&xDma); + + // value taken from libxil implementation + int timeout = 500; + + while(timeout > 0) { + if(XAxiDma_ResetIsDone(&xDma)) + return true; + + timeout--; + } + + return false; +} + + +bool +Dma::pingPong(const MemoryBlock& src, const MemoryBlock& dst, size_t len) +{ + if(this->read(dst, len) == 0) + return false; + + if(this->write(src, len) == 0) + return false; + + if(not this->writeComplete()) + return false; + + if(not this->readComplete()) + return false; + + return true; +} + + +size_t +Dma::write(const MemoryBlock& mem, size_t len) +{ + // make sure memory is reachable + if(not card->mapMemoryBlock(mem)) { + logger->error("Memory not accessible by DMA"); + return 0; + } + + auto& mm = MemoryManager::get(); + auto translation = mm.getTranslation(busMasterInterfaces[mm2sInterface], + mem.getAddrSpaceId()); + const void* buf = reinterpret_cast(translation.getLocalAddr(0)); + + return hasScatterGather() ? writeSG(buf, len) : writeSimple(buf, len); +} + + +size_t +Dma::read(const MemoryBlock& mem, size_t len) +{ + // make sure memory is reachable + if(not card->mapMemoryBlock(mem)) { + logger->error("Memory not accessible by DMA"); + return 0; + } + + auto& mm = MemoryManager::get(); + auto translation = mm.getTranslation(busMasterInterfaces[s2mmInterface], + mem.getAddrSpaceId()); + void* buf = reinterpret_cast(translation.getLocalAddr(0)); + + return hasScatterGather() ? readSG(buf, len) : readSimple(buf, len); +} + + +size_t +Dma::writeSG(const void* buf, size_t len) +{ + (void) buf; + (void) len; + logger->error("DMA Scatter Gather write not implemented"); + + return 0; +} + + +size_t +Dma::readSG(void* buf, size_t len) +{ + (void) buf; + (void) len; + logger->error("DMA Scatter Gather read not implemented"); + + return 0; +} + + +bool +Dma::writeCompleteSG() +{ + logger->error("DMA Scatter Gather write not implemented"); + + return false; +} + + +bool +Dma::readCompleteSG() +{ + logger->error("DMA Scatter Gather read not implemented"); + + return false; +} + + +size_t +Dma::writeSimple(const void *buf, size_t len) +{ + XAxiDma_BdRing *ring = XAxiDma_GetTxRing(&xDma); + + if ((len == 0) || (len > FPGA_DMA_BOUNDARY)) + return 0; + + if (not ring->HasDRE) { + const uint32_t mask = xDma.MicroDmaMode + ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN + : ring->DataWidth - 1; + + if (reinterpret_cast(buf) & mask) { + return 0; + } + } + + const bool dmaChannelHalted = + XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK; + + const bool dmaToDeviceBusy = XAxiDma_Busy(&xDma, XAXIDMA_DMA_TO_DEVICE); + + /* If the engine is doing a transfer, cannot submit */ + if (not dmaChannelHalted and dmaToDeviceBusy) { + return 0; + } + + // set lower 32 bit of source address + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_SRCADDR_OFFSET, + LOWER_32_BITS(reinterpret_cast(buf))); + + // if neccessary, set upper 32 bit of source address + if (xDma.AddrWidth > 32) { + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_SRCADDR_MSB_OFFSET, + UPPER_32_BITS(reinterpret_cast(buf))); + } + + // start DMA channel + auto channelControl = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_CR_OFFSET); + channelControl |= XAXIDMA_CR_RUNSTOP_MASK; + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_CR_OFFSET, channelControl); + + // set tail descriptor pointer + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET, len); + + + return len; +} + + +size_t +Dma::readSimple(void *buf, size_t len) +{ + XAxiDma_BdRing *ring = XAxiDma_GetRxRing(&xDma); + + if ((len == 0) || (len > FPGA_DMA_BOUNDARY)) + return 0; + + if (not ring->HasDRE) { + const uint32_t mask = xDma.MicroDmaMode + ? XAXIDMA_MICROMODE_MIN_BUF_ALIGN + : ring->DataWidth - 1; + + if (reinterpret_cast(buf) & mask) { + return 0; + } + } + + const bool dmaChannelHalted = + XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK; + + const bool deviceToDmaBusy = XAxiDma_Busy(&xDma, XAXIDMA_DEVICE_TO_DMA); + + /* If the engine is doing a transfer, cannot submit */ + if (not dmaChannelHalted and deviceToDmaBusy) { + return 0; + } + + // set lower 32 bit of destination address + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_OFFSET, + LOWER_32_BITS(reinterpret_cast(buf))); + + // if neccessary, set upper 32 bit of destination address + if (xDma.AddrWidth > 32) + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_MSB_OFFSET, + UPPER_32_BITS(reinterpret_cast(buf))); + + // start DMA channel + auto channelControl = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_CR_OFFSET); + channelControl |= XAXIDMA_CR_RUNSTOP_MASK; + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_CR_OFFSET, channelControl); + + // set tail descriptor pointer + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_BUFFLEN_OFFSET, len); + + return len; +} + + +bool +Dma::writeCompleteSimple() +{ + while (!(XAxiDma_IntrGetIrq(&xDma, XAXIDMA_DMA_TO_DEVICE) & XAXIDMA_IRQ_IOC_MASK)) + irqs[mm2sInterrupt].irqController->waitForInterrupt(irqs[mm2sInterrupt]); + + XAxiDma_IntrAckIrq(&xDma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); + + return true; +} + + +bool +Dma::readCompleteSimple() +{ + while (!(XAxiDma_IntrGetIrq(&xDma, XAXIDMA_DEVICE_TO_DMA) & XAXIDMA_IRQ_IOC_MASK)) + irqs[s2mmInterrupt].irqController->waitForInterrupt(irqs[s2mmInterrupt]); + + XAxiDma_IntrAckIrq(&xDma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); + + return true; +} + + +} // namespace ip +} // namespace fpga +} // namespace villas From e28345b992e10a328d92f91cb16c378da585c78d Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 16:15:31 +0200 Subject: [PATCH 18/21] tests/dma: add test for DMA driver --- fpga/tests/CMakeLists.txt | 2 +- fpga/tests/dma.cpp | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 fpga/tests/dma.cpp diff --git a/fpga/tests/CMakeLists.txt b/fpga/tests/CMakeLists.txt index 843c1b6a6..2e4cfe041 100644 --- a/fpga/tests/CMakeLists.txt +++ b/fpga/tests/CMakeLists.txt @@ -2,7 +2,7 @@ set(SOURCES main.cpp fpga.cpp logging.cpp -# dma.c + dma.cpp fifo.cpp # hls.c # intc.c diff --git a/fpga/tests/dma.cpp b/fpga/tests/dma.cpp new file mode 100644 index 000000000..fd69cfc94 --- /dev/null +++ b/fpga/tests/dma.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +#include + +#include "global.hpp" + +#include + + +Test(fpga, dma, .description = "DMA") +{ + auto logger = loggerGetOrCreate("unittest:dma"); + + size_t count = 0; + for(auto& ip : state.cards.front()->ips) { + // skip non-dma IPs + if(*ip != villas::fpga::Vlnv("xilinx.com:ip:axi_dma:")) + continue; + + logger->info("Testing {}", *ip); + + auto dma = reinterpret_cast(*ip); + + if(not dma.connectLoopback()) { + continue; + } + + count++; + + if(not dma.loopbackPossible()) { + logger->info("Loopback test not possible for {}", *ip); + continue; + } + + // Simple DMA can only transfer up to 4 kb due to PCIe page size burst + // limitation + size_t len = 4 * (1 << 10); + + /* Allocate memory to use with DMA */ + auto src = villas::HostRam::allocate(len); + auto dst = villas::HostRam::allocate(len); + + /* Get new random data */ + const size_t lenRandom = read_random(&src, len); + cr_assert(len == lenRandom, "Failed to get random data"); + + /* Start transfer */ + cr_assert(dma.pingPong(src, dst, len), "DMA ping pong failed"); + + /* Compare data */ + cr_assert(memcmp(&src, &dst, len) == 0, "Data not equal"); + + logger->info(TXT_GREEN("Passed")); + } + + villas::MemoryManager::get().dump(); + + cr_assert(count > 0, "No Dma found"); +} From ce7e6b36d5bbf9573a3d3ab0328b4be06b10afbd Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 14:59:05 +0200 Subject: [PATCH 19/21] kernel/vfio: check if all required kernel modules are loaded --- fpga/lib/kernel/vfio.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fpga/lib/kernel/vfio.cpp b/fpga/lib/kernel/vfio.cpp index 09464b1f8..9fae25ca8 100644 --- a/fpga/lib/kernel/vfio.cpp +++ b/fpga/lib/kernel/vfio.cpp @@ -72,10 +72,17 @@ namespace villas { VfioContainer::VfioContainer() : iova_next(0) { - /* Load VFIO kernel module */ - if (kernel_module_load("vfio") != 0) { - logger->error("Failed to load kernel module: vfio"); - throw std::exception(); + + static constexpr const char* requiredKernelModules[] = { + "vfio", "vfio_pci", "vfio_iommu_type1" + }; + + for(const char* module : requiredKernelModules) { + if(kernel_module_loaded(module) != 0) { + logger->error("Kernel module '{}' required but not loaded. " + "Please load manually!", module); + throw std::exception(); + } } /* Open VFIO API */ From 192aa106272dce34ad892a45a22b926e39be1b57 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 14:59:52 +0200 Subject: [PATCH 20/21] kernel/vfio: fix check for vfio extensions --- fpga/lib/kernel/vfio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/lib/kernel/vfio.cpp b/fpga/lib/kernel/vfio.cpp index 9fae25ca8..b52d0e1a3 100644 --- a/fpga/lib/kernel/vfio.cpp +++ b/fpga/lib/kernel/vfio.cpp @@ -101,7 +101,7 @@ VfioContainer::VfioContainer() /* Check available VFIO extensions (IOMMU types) */ extensions = 0; - for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { + for (unsigned int i = VFIO_TYPE1_IOMMU; i <= VFIO_NOIOMMU_IOMMU; i++) { int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i); if (ret < 0) { logger->error("Failed to get VFIO extensions"); From 8954e65b4fd06e78946cd585705e8dea25bccd09 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 16:45:54 +0200 Subject: [PATCH 21/21] kernel/vfio: some more cleanup --- fpga/lib/kernel/vfio.cpp | 44 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/fpga/lib/kernel/vfio.cpp b/fpga/lib/kernel/vfio.cpp index b52d0e1a3..dcc29c2e8 100644 --- a/fpga/lib/kernel/vfio.cpp +++ b/fpga/lib/kernel/vfio.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -118,7 +119,7 @@ VfioContainer::VfioContainer() VfioContainer::~VfioContainer() { - logger->debug("Clean up container with fd {}", this->fd); + logger->debug("Clean up container with fd {}", fd); /* Release memory and close fds */ groups.clear(); @@ -126,10 +127,8 @@ VfioContainer::~VfioContainer() /* Close container */ int ret = close(fd); if (ret < 0) { - logger->error("Cannot close vfio container"); + logger->error("Cannot close vfio container fd {}", fd); } - - logger->debug("VFIO: closed container: fd={}", fd); } @@ -270,7 +269,7 @@ VfioContainer::attachDevice(const pci_device* pdev) { int ret; char name[32]; - static constexpr char kernelDriver[] = "vfio-pci"; + static constexpr const char* kernelDriver = "vfio-pci"; /* Load PCI bus driver for VFIO */ if (kernel_module_load("vfio_pci")) { @@ -347,13 +346,14 @@ VfioContainer::memoryMap(uintptr_t virt, uintptr_t phys, size_t length) return UINTPTR_MAX; } - logger->info("DMA map size={:#x}, iova={:#x}, vaddr={:#x}", dmaMap.size, dmaMap.iova, dmaMap.vaddr); + logger->debug("DMA map size={:#x}, iova={:#x}, vaddr={:#x}", + dmaMap.size, dmaMap.iova, dmaMap.vaddr); // mapping successful, advance IOVA allocator this->iova_next += iovaIncrement; - // we intentionally don't return the actual mapped length, the users are only - // guaranteed to have their demanded memory mapped correctly + // we intentionally don't return the actual mapped length, the users are + // only guaranteed to have their demanded memory mapped correctly return dmaMap.iova; } @@ -395,7 +395,7 @@ VfioContainer::getOrAttachGroup(int index) logger->error("Failed to attach to IOMMU group: {}", index); throw std::exception(); } else { - logger->info("Attached new group {} to VFIO container", index); + logger->debug("Attached new group {} to VFIO container", index); } // push to our list @@ -407,7 +407,7 @@ VfioContainer::getOrAttachGroup(int index) VfioDevice::~VfioDevice() { - logger->debug("clean up device {} with fd {}", this->name, this->fd); + logger->debug("Clean up device {} with fd {}", this->name, this->fd); for(auto& region : regions) { regionUnmap(region.index); @@ -417,8 +417,6 @@ VfioDevice::~VfioDevice() if (ret != 0) { logger->error("Closing device fd {} failed", fd); } - - logger->debug("VFIO: closed device: name={}, fd={}", name, fd); } @@ -458,7 +456,7 @@ VfioDevice::regionUnmap(size_t index) if (!mappings[index]) return false; /* was not mapped */ - logger->debug("VFIO: unmap region {} from device", index); + logger->debug("Unmap region {} from device {}", index, name); ret = munmap(mappings[index], r->size); if (ret) @@ -487,7 +485,8 @@ VfioDevice::pciEnable() { int ret; uint32_t reg; - off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; + const off_t offset = PCI_COMMAND + + (static_cast(VFIO_PCI_CONFIG_REGION_INDEX) << 40); /* Check if this is really a vfio-pci device */ if (!(this->info.flags & VFIO_DEVICE_FLAGS_PCI)) @@ -528,10 +527,10 @@ VfioDevice::pciHotReset() return false; } - logger->debug("VFIO: dependent devices for hot-reset:"); + logger->debug("Dependent devices for hot-reset:"); for (size_t i = 0; i < reset_info->count; i++) { struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; - logger->debug("{:04x}:{:02x}:{:02x}.{:01x}: iommu_group={}", + logger->debug(" {:04x}:{:02x}:{:02x}.{:01x}: iommu_group={}", dd->segment, dd->bus, PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); @@ -568,7 +567,7 @@ VfioDevice::pciMsiInit(int efds[]) const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count; const size_t irqSetSize = sizeof(struct vfio_irq_set) + - sizeof(int) * irqCount; + sizeof(int) * irqCount; auto irqSet = reinterpret_cast(calloc(1, irqSetSize)); if(irqSet == nullptr) @@ -692,8 +691,7 @@ VfioDevice::isVfioPciDevice() const VfioGroup::~VfioGroup() { - logger->debug("clean up group {} with fd {}", this->index, this->fd); - int ret; + logger->debug("Clean up group {} with fd {}", this->index, this->fd); /* Release memory and close fds */ devices.clear(); @@ -701,19 +699,15 @@ VfioGroup::~VfioGroup() if(fd < 0) { logger->debug("Destructing group that has not been attached"); } else { - ret = ioctl(fd, VFIO_GROUP_UNSET_CONTAINER); + int ret = ioctl(fd, VFIO_GROUP_UNSET_CONTAINER); if (ret != 0) { logger->error("Cannot unset container for group fd {}", fd); } - logger->debug("Released group from container: group={}", index); - ret = close(fd); if (ret != 0) { logger->error("Cannot close group fd {}", fd); } - - logger->debug("Closed group: group={}, fd={}", index, fd); } } @@ -728,7 +722,7 @@ VfioGroup::attach(int containerFd, int groupIndex) groupPath << VFIO_DEV("") << groupIndex; group->fd = open(groupPath.str().c_str(), O_RDWR); if (group->fd < 0) { - logger->error("Failed to open VFIO group: {}", group->index); + logger->error("Failed to open VFIO group {}", group->index); return nullptr; }