From e254e7cfe6751b752b52d41a06488450d92afe20 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:42:17 +0100 Subject: [PATCH 01/16] move card class to own file Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 153 +-------------- fpga/lib/card.cpp | 297 ------------------------------ 2 files changed, 2 insertions(+), 448 deletions(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 6e47227a6..61a6548f6 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -1,154 +1,5 @@ -/** FPGA card - * - * This class represents a FPGA device. - * - * Author: Steffen Vogel - * Author: Daniel Krebs - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 - *********************************************************************************/ - -#pragma once - -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include - -namespace villas { -namespace fpga { - -// Forward declarations -struct vfio_container; -class PCIeCardFactory; class Card { public: - friend PCIeCardFactory; -}; - -class PCIeCard : public Card { -public: - - ~PCIeCard(); - - bool init(); - - bool stop() - { - return true; - } - - bool check() - { - return true; - } - - bool reset() - { - // TODO: Try via sysfs? - // echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset - return true; - } - - void dump() - { } - - std::shared_ptr - lookupIp(const std::string &name) const; - - std::shared_ptr - lookupIp(const Vlnv &vlnv) const; - - std::shared_ptr - lookupIp(const ip::IpIdentifier &id) const; - - bool mapMemoryBlock(const MemoryBlock &block); - bool unmapMemoryBlock(const MemoryBlock &block); - -private: - // Cache a set of already mapped memory blocks - std::set memoryBlocksMapped; - -public: // TODO: make this private - std::list> ips; // IPs located on this FPGA card - - bool doReset; // Reset VILLASfpga during startup? - int affinity; // Affinity for MSI interrupts - bool polling; // Poll on interrupts? - - std::string name; // The name of the FPGA card - - std::shared_ptr pdev; // PCI device handle - - // The VFIO container that this card is part of - std::shared_ptr vfioContainer; - - // The VFIO device that represents this card - std::shared_ptr vfioDevice; - - // Slave address space ID to access the PCIe address space from the FPGA - MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; - - // Address space identifier of the master address space of this FPGA card. - // This will be used for address resolution of all IPs on this card. - MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; - -protected: - Logger - getLogger() const - { - return villas::logging.get(name); - } - - Logger logger; -}; - -class PCIeCardFactory : public plugin::Plugin { -public: - - static - std::list> make(json_t *json, std::shared_ptr pci, std::shared_ptr vc); - - static - PCIeCard* make() - { - return new PCIeCard(); - } - - static Logger - getStaticLogger() - { - return villas::logging.get("pcie:card:factory"); - } - - virtual std::string - getName() const - { - return "pcie"; - } - - virtual std::string - getDescription() const - { - return "Xilinx PCIe FPGA cards"; - } - - virtual - std::string getType() const - { - return "card"; - } -}; - -} /* namespace fpga */ -} /* namespace villas */ + +}; \ No newline at end of file diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index da4825e83..e69de29bb 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -1,297 +0,0 @@ -/** FPGA card. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 - *********************************************************************************/ - -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include - -using namespace villas; -using namespace villas::fpga; - -// Instantiate factory to register -static PCIeCardFactory PCIeCardFactoryInstance; - -static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA))); - -std::list> PCIeCardFactory::make(json_t *json, std::shared_ptr pci, std::shared_ptr vc) -{ - std::list> cards; - auto logger = getStaticLogger(); - - const char *card_name; - json_t *json_card; - json_object_foreach(json, card_name, json_card) { - logger->info("Found config for FPGA card {}", card_name); - - json_t* json_ips = nullptr; - json_t* json_paths = nullptr; - const char* pci_slot = nullptr; - const char* pci_id = nullptr; - int do_reset = 0; - int affinity = 0; - int polling = 0; - - json_error_t err; - int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }", - "ips", &json_ips, - "affinity", &affinity, - "do_reset", &do_reset, - "slot", &pci_slot, - "id", &pci_id, - "polling", &polling, - "paths", &json_paths); - - if (ret != 0) - throw ConfigError(json_card, err, "", "Failed to parse card"); - - auto card = std::unique_ptr(make()); - - // Populate generic properties - card->name = std::string(card_name); - card->vfioContainer = vc; - card->affinity = affinity; - card->doReset = do_reset != 0; - card->polling = (polling != 0); - - kernel::pci::Device filter = defaultFilter; - - if (pci_id) - filter.id = kernel::pci::Id(pci_id); - if (pci_slot) - filter.slot = kernel::pci::Slot(pci_slot); - - // Search for FPGA card - card->pdev = pci->lookupDevice(filter); - if (!card->pdev) { - logger->warn("Failed to find PCI device"); - continue; - } - - if (not card->init()) { - logger->warn("Cannot start FPGA card {}", card_name); - continue; - } - - // Load IPs from a separate json file - if (json_is_string(json_ips)) { - auto json_ips_fn = json_string_value(json_ips); - json_ips = json_load_file(json_ips_fn, 0, nullptr); - if (json_ips == nullptr) - throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn); - } - - if (not json_is_object(json_ips)) - throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!"); - - card->ips = ip::CoreFactory::make(card.get(), json_ips); - if (card->ips.empty()) - throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name); - - if (not card->check()) - throw RuntimeError("Checking of FPGA card {} failed", card_name); - - // Additional static paths for AXI-Steram switch - if (json_paths != nullptr) { - if (not json_is_array(json_paths)) - throw ConfigError(json_paths, err, "", "Switch path configuration must be an array"); - - size_t i; - json_t *json_path; - json_array_foreach(json_paths, i, json_path) { - const char *from, *to; - int reverse = 0; - - ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }", - "from", &from, - "to", &to, - "reverse", &reverse - ); - if (ret != 0) - throw ConfigError(json_path, err, "", "Cannot parse switch path config"); - - auto masterIpCore = card->lookupIp(from); - if (!masterIpCore) - throw ConfigError(json_path, "", "Unknown IP {}", from); - - auto slaveIpCore = card->lookupIp(to); - if (!slaveIpCore) - throw ConfigError(json_path, "", "Unknown IP {}", to); - - auto masterIpNode = std::dynamic_pointer_cast(masterIpCore); - if (!masterIpNode) - throw ConfigError(json_path, "", "IP {} is not a streaming node", from); - - auto slaveIpNode = std::dynamic_pointer_cast(slaveIpCore); - if (!slaveIpNode) - throw ConfigError(json_path, "", "IP {} is not a streaming node", to); - - if (not masterIpNode->connect(*slaveIpNode, reverse != 0)) - throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to); - } - } - - cards.push_back(std::move(card)); - } - - return cards; -} - -PCIeCard::~PCIeCard() -{ - // Ensure IP destructors are called before memory is unmapped - ips.clear(); - - auto &mm = MemoryManager::get(); - - // Unmap all memory blocks - for (auto &mappedMemoryBlock : memoryBlocksMapped) { - auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock); - - const uintptr_t iova = translation.getLocalAddr(0); - const size_t size = translation.getSize(); - - logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", - mappedMemoryBlock, iova, size); - vfioContainer->memoryUnmap(iova, size); - } -} - -std::shared_ptr PCIeCard::lookupIp(const std::string &name) const -{ - for (auto &ip : ips) { - if (*ip == name) { - return ip; - } - } - - return nullptr; -} - -std::shared_ptr PCIeCard::lookupIp(const Vlnv &vlnv) const -{ - for (auto &ip : ips) { - if (*ip == vlnv) { - return ip; - } - } - - return nullptr; -} - -std::shared_ptr PCIeCard::lookupIp(const ip::IpIdentifier &id) const -{ - for (auto &ip : ips) { - if (*ip == id) { - return ip; - } - } - - return nullptr; -} - -bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block) -{ - if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) { - throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped."); - } - - auto &mm = MemoryManager::get(); - - auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId()); - - const uintptr_t iova = translation.getLocalAddr(0); - const size_t size = translation.getSize(); - - logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", - block.getAddrSpaceId(), iova, size); - vfioContainer->memoryUnmap(iova, size); - - memoryBlocksMapped.erase(block.getAddrSpaceId()); - - return true; -} - -bool PCIeCard::mapMemoryBlock(const MemoryBlock &block) -{ - if (not vfioContainer->isIommuEnabled()) { - logger->warn("VFIO mapping not supported without IOMMU"); - return false; - } - - auto &mm = MemoryManager::get(); - const auto &addrSpaceId = block.getAddrSpaceId(); - - if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) - // Block already mapped - return true; - else - logger->debug("Create VFIO mapping for {}", addrSpaceId); - - auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); - uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); - uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, - UINTPTR_MAX, - block.getSize()); - - if (iovaAddr == UINTPTR_MAX) { - logger->error("Cannot map memory at {:#x} of size {:#x}", - processBaseAddr, block.getSize()); - return false; - } - - mm.createMapping(iovaAddr, 0, block.getSize(), - "VFIO-D2H", - this->addrSpaceIdDeviceToHost, - addrSpaceId); - - // Remember that this block has already been mapped for later - memoryBlocksMapped.insert(addrSpaceId); - - return true; -} - -bool PCIeCard::init() -{ - logger = getLogger(); - - logger->info("Initializing FPGA card {}", name); - - // Attach PCIe card to VFIO container - vfioDevice = vfioContainer->attachDevice(*pdev); - - // Enable memory access and PCI bus mastering for DMA - if (not vfioDevice->pciEnable()) { - logger->error("Failed to enable PCI device"); - return false; - } - - // Reset system? - if (doReset) { - // Reset / detect PCI device - if (not vfioDevice->pciHotReset()) { - logger->error("Failed to reset PCI device"); - return false; - } - - if (not reset()) { - logger->error("Failed to reset FGPA card"); - return false; - } - } - - return true; -} From d2d7f9430d9e3a09ea95eb305653cb000fd35699 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:43:02 +0100 Subject: [PATCH 02/16] moved pciecard to own file Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/pciecard.hpp | 150 +++++++++++++ fpga/lib/pciecard.cpp | 297 ++++++++++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 fpga/include/villas/fpga/pciecard.hpp create mode 100644 fpga/lib/pciecard.cpp diff --git a/fpga/include/villas/fpga/pciecard.hpp b/fpga/include/villas/fpga/pciecard.hpp new file mode 100644 index 000000000..6af352501 --- /dev/null +++ b/fpga/include/villas/fpga/pciecard.hpp @@ -0,0 +1,150 @@ +/** FPGA card + * + * This class represents a FPGA device. + * + * Author: Steffen Vogel + * Author: Daniel Krebs + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +namespace villas { +namespace fpga { + +// Forward declarations +struct vfio_container; +class PCIeCardFactory; + +class PCIeCard : public Card { +public: + + ~PCIeCard(); + + bool init(); + + bool stop() + { + return true; + } + + bool check() + { + return true; + } + + bool reset() + { + // TODO: Try via sysfs? + // echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset + return true; + } + + void dump() + { } + + std::shared_ptr + lookupIp(const std::string &name) const; + + std::shared_ptr + lookupIp(const Vlnv &vlnv) const; + + std::shared_ptr + lookupIp(const ip::IpIdentifier &id) const; + + bool mapMemoryBlock(const MemoryBlock &block); + bool unmapMemoryBlock(const MemoryBlock &block); + +private: + // Cache a set of already mapped memory blocks + std::set memoryBlocksMapped; + +public: // TODO: make this private + std::list> ips; // IPs located on this FPGA card + + bool doReset; // Reset VILLASfpga during startup? + int affinity; // Affinity for MSI interrupts + bool polling; // Poll on interrupts? + + std::string name; // The name of the FPGA card + + std::shared_ptr pdev; // PCI device handle + + // The VFIO container that this card is part of + std::shared_ptr vfioContainer; + + // The VFIO device that represents this card + std::shared_ptr vfioDevice; + + // Slave address space ID to access the PCIe address space from the FPGA + MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; + + // Address space identifier of the master address space of this FPGA card. + // This will be used for address resolution of all IPs on this card. + MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; + +protected: + Logger + getLogger() const + { + return villas::logging.get(name); + } + + Logger logger; +}; + +class PCIeCardFactory : public plugin::Plugin { +public: + + static + std::list> make(json_t *json, std::shared_ptr pci, std::shared_ptr vc); + + static + PCIeCard* make() + { + return new PCIeCard(); + } + + static Logger + getStaticLogger() + { + return villas::logging.get("pcie:card:factory"); + } + + virtual std::string + getName() const + { + return "pcie"; + } + + virtual std::string + getDescription() const + { + return "Xilinx PCIe FPGA cards"; + } + + virtual + std::string getType() const + { + return "card"; + } +}; + +} /* namespace fpga */ +} /* namespace villas */ diff --git a/fpga/lib/pciecard.cpp b/fpga/lib/pciecard.cpp new file mode 100644 index 000000000..4d9e680d1 --- /dev/null +++ b/fpga/lib/pciecard.cpp @@ -0,0 +1,297 @@ +/** FPGA card. + * + * Author: Steffen Vogel + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +using namespace villas; +using namespace villas::fpga; + +// Instantiate factory to register +static PCIeCardFactory PCIeCardFactoryInstance; + +static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA))); + +std::list> PCIeCardFactory::make(json_t *json, std::shared_ptr pci, std::shared_ptr vc) +{ + std::list> cards; + auto logger = getStaticLogger(); + + const char *card_name; + json_t *json_card; + json_object_foreach(json, card_name, json_card) { + logger->info("Found config for FPGA card {}", card_name); + + json_t* json_ips = nullptr; + json_t* json_paths = nullptr; + const char* pci_slot = nullptr; + const char* pci_id = nullptr; + int do_reset = 0; + int affinity = 0; + int polling = 0; + + json_error_t err; + int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }", + "ips", &json_ips, + "affinity", &affinity, + "do_reset", &do_reset, + "slot", &pci_slot, + "id", &pci_id, + "polling", &polling, + "paths", &json_paths); + + if (ret != 0) + throw ConfigError(json_card, err, "", "Failed to parse card"); + + auto card = std::unique_ptr(make()); + + // Populate generic properties + card->name = std::string(card_name); + card->vfioContainer = vc; + card->affinity = affinity; + card->doReset = do_reset != 0; + card->polling = (polling != 0); + + kernel::pci::Device filter = defaultFilter; + + if (pci_id) + filter.id = kernel::pci::Id(pci_id); + if (pci_slot) + filter.slot = kernel::pci::Slot(pci_slot); + + // Search for FPGA card + card->pdev = pci->lookupDevice(filter); + if (!card->pdev) { + logger->warn("Failed to find PCI device"); + continue; + } + + if (not card->init()) { + logger->warn("Cannot start FPGA card {}", card_name); + continue; + } + + // Load IPs from a separate json file + if (json_is_string(json_ips)) { + auto json_ips_fn = json_string_value(json_ips); + json_ips = json_load_file(json_ips_fn, 0, nullptr); + if (json_ips == nullptr) + throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn); + } + + if (not json_is_object(json_ips)) + throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!"); + + card->ips = ip::CoreFactory::make(card.get(), json_ips); + if (card->ips.empty()) + throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name); + + if (not card->check()) + throw RuntimeError("Checking of FPGA card {} failed", card_name); + + // Additional static paths for AXI-Steram switch + if (json_paths != nullptr) { + if (not json_is_array(json_paths)) + throw ConfigError(json_paths, err, "", "Switch path configuration must be an array"); + + size_t i; + json_t *json_path; + json_array_foreach(json_paths, i, json_path) { + const char *from, *to; + int reverse = 0; + + ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }", + "from", &from, + "to", &to, + "reverse", &reverse + ); + if (ret != 0) + throw ConfigError(json_path, err, "", "Cannot parse switch path config"); + + auto masterIpCore = card->lookupIp(from); + if (!masterIpCore) + throw ConfigError(json_path, "", "Unknown IP {}", from); + + auto slaveIpCore = card->lookupIp(to); + if (!slaveIpCore) + throw ConfigError(json_path, "", "Unknown IP {}", to); + + auto masterIpNode = std::dynamic_pointer_cast(masterIpCore); + if (!masterIpNode) + throw ConfigError(json_path, "", "IP {} is not a streaming node", from); + + auto slaveIpNode = std::dynamic_pointer_cast(slaveIpCore); + if (!slaveIpNode) + throw ConfigError(json_path, "", "IP {} is not a streaming node", to); + + if (not masterIpNode->connect(*slaveIpNode, reverse != 0)) + throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to); + } + } + + cards.push_back(std::move(card)); + } + + return cards; +} + +PCIeCard::~PCIeCard() +{ + // Ensure IP destructors are called before memory is unmapped + ips.clear(); + + auto &mm = MemoryManager::get(); + + // Unmap all memory blocks + for (auto &mappedMemoryBlock : memoryBlocksMapped) { + auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock); + + const uintptr_t iova = translation.getLocalAddr(0); + const size_t size = translation.getSize(); + + logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", + mappedMemoryBlock, iova, size); + vfioContainer->memoryUnmap(iova, size); + } +} + +std::shared_ptr PCIeCard::lookupIp(const std::string &name) const +{ + for (auto &ip : ips) { + if (*ip == name) { + return ip; + } + } + + return nullptr; +} + +std::shared_ptr PCIeCard::lookupIp(const Vlnv &vlnv) const +{ + for (auto &ip : ips) { + if (*ip == vlnv) { + return ip; + } + } + + return nullptr; +} + +std::shared_ptr PCIeCard::lookupIp(const ip::IpIdentifier &id) const +{ + for (auto &ip : ips) { + if (*ip == id) { + return ip; + } + } + + return nullptr; +} + +bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block) +{ + if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) { + throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped."); + } + + auto &mm = MemoryManager::get(); + + auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId()); + + const uintptr_t iova = translation.getLocalAddr(0); + const size_t size = translation.getSize(); + + logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", + block.getAddrSpaceId(), iova, size); + vfioContainer->memoryUnmap(iova, size); + + memoryBlocksMapped.erase(block.getAddrSpaceId()); + + return true; +} + +bool PCIeCard::mapMemoryBlock(const MemoryBlock &block) +{ + if (not vfioContainer->isIommuEnabled()) { + logger->warn("VFIO mapping not supported without IOMMU"); + return false; + } + + auto &mm = MemoryManager::get(); + const auto &addrSpaceId = block.getAddrSpaceId(); + + if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) + // Block already mapped + return true; + else + logger->debug("Create VFIO mapping for {}", addrSpaceId); + + auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); + uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); + uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, + UINTPTR_MAX, + block.getSize()); + + if (iovaAddr == UINTPTR_MAX) { + logger->error("Cannot map memory at {:#x} of size {:#x}", + processBaseAddr, block.getSize()); + return false; + } + + mm.createMapping(iovaAddr, 0, block.getSize(), + "VFIO-D2H", + this->addrSpaceIdDeviceToHost, + addrSpaceId); + + // Remember that this block has already been mapped for later + memoryBlocksMapped.insert(addrSpaceId); + + return true; +} + +bool PCIeCard::init() +{ + logger = getLogger(); + + logger->info("Initializing FPGA card {}", name); + + // Attach PCIe card to VFIO container + vfioDevice = vfioContainer->attachDevice(*pdev); + + // Enable memory access and PCI bus mastering for DMA + if (not vfioDevice->pciEnable()) { + logger->error("Failed to enable PCI device"); + return false; + } + + // Reset system? + if (doReset) { + // Reset / detect PCI device + if (not vfioDevice->pciHotReset()) { + logger->error("Failed to reset PCI device"); + return false; + } + + if (not reset()) { + logger->error("Failed to reset FGPA card"); + return false; + } + } + + return true; +} From a10e568777e5675e67b57355114fa0835443471d Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:43:29 +0100 Subject: [PATCH 03/16] added pcieclass to buildsystem Signed-off-by: Pascal Henry Bauer --- fpga/lib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index be90c1ef5..571de91d4 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES vlnv.cpp card.cpp + pciecard.cpp core.cpp node.cpp utils.cpp From ae944b6ce392812cc4867fae3aca9a0303383780 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:48:06 +0100 Subject: [PATCH 04/16] added copyright information Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 11 ++++++++++- fpga/include/villas/fpga/pciecard.hpp | 2 +- fpga/lib/card.cpp | 9 +++++++++ fpga/lib/pciecard.cpp | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 61a6548f6..3ec43384a 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -1,4 +1,13 @@ - +/** FPGA card + * + * This class represents a FPGA device. + * + * Author: Steffen Vogel + * Author: Daniel Krebs + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + class Card { public: diff --git a/fpga/include/villas/fpga/pciecard.hpp b/fpga/include/villas/fpga/pciecard.hpp index 6af352501..7c784da63 100644 --- a/fpga/include/villas/fpga/pciecard.hpp +++ b/fpga/include/villas/fpga/pciecard.hpp @@ -1,4 +1,4 @@ -/** FPGA card +/** FPGA pciecard * * This class represents a FPGA device. * diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index e69de29bb..45ad4e4ab 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -0,0 +1,9 @@ +/** FPGA card + * + * This class represents a FPGA device. + * + * Author: Steffen Vogel + * Author: Daniel Krebs + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ \ No newline at end of file diff --git a/fpga/lib/pciecard.cpp b/fpga/lib/pciecard.cpp index 4d9e680d1..557698f12 100644 --- a/fpga/lib/pciecard.cpp +++ b/fpga/lib/pciecard.cpp @@ -1,4 +1,4 @@ -/** FPGA card. +/** FPGA pciecard. * * Author: Steffen Vogel * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC From 7534086e08261b972404cc0aad86f5c166c92972 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:55:35 +0100 Subject: [PATCH 05/16] change core to use base class card over pcieclass Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/core.hpp | 6 +++--- fpga/lib/core.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fpga/include/villas/fpga/core.hpp b/fpga/include/villas/fpga/core.hpp index 65dd1d1a3..7d193c38f 100644 --- a/fpga/include/villas/fpga/core.hpp +++ b/fpga/include/villas/fpga/core.hpp @@ -26,7 +26,7 @@ namespace villas { namespace fpga { // Forward declarations -class PCIeCard; +class Card; namespace ip { @@ -242,7 +242,7 @@ protected: Logger logger; // FPGA card this IP is instantiated on (populated by FpgaIpFactory) - PCIeCard* card; + Card* card; // Identifier of this IP with its instance name and VLNV IpIdentifier id; @@ -266,7 +266,7 @@ public: // Returns a running and checked FPGA IP static - std::list> make(PCIeCard* card, json_t *json_ips); + std::list> make(Card* card, json_t *json_ips); virtual std::string getType() const diff --git a/fpga/lib/core.cpp b/fpga/lib/core.cpp index 8549153a4..11e71aa3e 100644 --- a/fpga/lib/core.cpp +++ b/fpga/lib/core.cpp @@ -35,7 +35,7 @@ vlnvInitializationOrder = { Vlnv("xilinx.com:ip:axis_switch:"), }; -std::list> CoreFactory::make(PCIeCard* card, json_t *json_ips) +std::list> CoreFactory::make(Card* card, json_t *json_ips) { // We only have this logger until we know the factory to build an IP with auto loggerStatic = getStaticLogger(); From dfb4d30b3f7791edfbc67b97a17086919d63b47a Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 23:04:15 +0100 Subject: [PATCH 06/16] added namespace Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 3ec43384a..05da013a1 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -8,7 +8,13 @@ * SPDX-License-Identifier: Apache-2.0 *********************************************************************************/ +namespace villas { +namespace fpga { + class Card { public: -}; \ No newline at end of file +}; + +} /* namespace fpga */ +} /* namespace villas */ \ No newline at end of file From 22244fed04a39af2a06ae60fa43068aa212f31b0 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 23:26:35 +0100 Subject: [PATCH 07/16] add include guard Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 05da013a1..8528ace92 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -7,6 +7,7 @@ * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC * SPDX-License-Identifier: Apache-2.0 *********************************************************************************/ +#pragma once namespace villas { namespace fpga { From 0dd52db44d3647da22f668f475219b358bc60b64 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Thu, 26 Jan 2023 16:58:28 +0100 Subject: [PATCH 08/16] add card header declarations Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 59 ++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 8528ace92..e3839aeaf 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -4,17 +4,60 @@ * * Author: Steffen Vogel * Author: Daniel Krebs - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power + *Systems, EONERC SPDX-License-Identifier: Apache-2.0 *********************************************************************************/ #pragma once - -namespace villas { -namespace fpga { -class Card { -public: - +#include + +#include +#include + +namespace villas +{ +namespace fpga +{ + +class Card +{ + public: + bool polling; + std::shared_ptr vfioDevice; + + // Slave address space ID to access the PCIe address space from the + // FPGA + MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; + + // Address space identifier of the master address space of this FPGA + // card. This will be used for address resolution of all IPs on this + // card. + MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; + + std::list > ips; + + virtual ~Card(); + + std::shared_ptr lookupIp(const std::string &name) const; + std::shared_ptr lookupIp(const Vlnv &vlnv) const; + std::shared_ptr lookupIp(const ip::IpIdentifier &id) const; + + bool mapMemoryBlock(const MemoryBlock &block); + bool unmapMemoryBlock(const MemoryBlock &block); + + private: + // Cache a set of already mapped memory blocks + std::set memoryBlocksMapped; + + std::shared_ptr vfioContainer; + + protected: + // Logger getLogger() const + // { + // return villas::logging.get(name); + // } + + Logger logger; }; } /* namespace fpga */ From e8b593cf1febc4ad7307e28060e8a79198572f69 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Thu, 26 Jan 2023 17:01:08 +0100 Subject: [PATCH 09/16] added card definitions Signed-off-by: Pascal Henry Bauer --- fpga/lib/card.cpp | 119 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 3 deletions(-) diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index 45ad4e4ab..f46b19528 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -4,6 +4,119 @@ * * Author: Steffen Vogel * Author: Daniel Krebs - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 - *********************************************************************************/ \ No newline at end of file + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power + *Systems, EONERC SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#include + +using namespace villas; +using namespace villas::fpga; + +Card::~Card() +{ +} + +std::shared_ptr Card::lookupIp(const std::string &name) const +{ + for(auto &ip : ips) { + if(*ip == name) { + return ip; + } + } + + return nullptr; +} + +std::shared_ptr Card::lookupIp(const Vlnv &vlnv) const +{ + for(auto &ip : ips) { + if(*ip == vlnv) { + return ip; + } + } + + return nullptr; +} + +std::shared_ptr Card::lookupIp(const ip::IpIdentifier &id) const +{ + for(auto &ip : ips) { + if(*ip == id) { + return ip; + } + } + + return nullptr; +} + +bool Card::unmapMemoryBlock(const MemoryBlock &block) +{ + if(memoryBlocksMapped.find(block.getAddrSpaceId()) + == memoryBlocksMapped.end()) { + throw std::runtime_error( + "Block " + std::to_string(block.getAddrSpaceId()) + + " is not mapped but was requested to be unmapped."); + } + + auto &mm = MemoryManager::get(); + + auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, + block.getAddrSpaceId()); + + const uintptr_t iova = translation.getLocalAddr(0); + const size_t size = translation.getSize(); + + logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", + block.getAddrSpaceId(), + iova, + size); + vfioContainer->memoryUnmap(iova, size); + + memoryBlocksMapped.erase(block.getAddrSpaceId()); + + return true; +} + +bool Card::mapMemoryBlock(const MemoryBlock &block) +{ + if(not vfioContainer->isIommuEnabled()) { + logger->warn("VFIO mapping not supported without IOMMU"); + return false; + } + + auto &mm = MemoryManager::get(); + const auto &addrSpaceId = block.getAddrSpaceId(); + + if(memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) + // Block already mapped + return true; + else + logger->debug("Create VFIO mapping for {}", addrSpaceId); + + auto translationFromProcess + = mm.getTranslationFromProcess(addrSpaceId); + uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); + uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, + UINTPTR_MAX, + block.getSize()); + + if(iovaAddr == UINTPTR_MAX) { + logger->error("Cannot map memory at {:#x} of size {:#x}", + processBaseAddr, + block.getSize()); + return false; + } + + mm.createMapping(iovaAddr, + 0, + block.getSize(), + "VFIO-D2H", + this->addrSpaceIdDeviceToHost, + addrSpaceId); + + // Remember that this block has already been mapped for later + memoryBlocksMapped.insert(addrSpaceId); + + return true; +} From 6b87c9bc30c09c40e828c9e7a715ca38ac421f5d Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Thu, 26 Jan 2023 17:02:25 +0100 Subject: [PATCH 10/16] refactor to use pcie card (Legacy) Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/utils.hpp | 2 +- fpga/lib/ips/intc.cpp | 4 +++- fpga/lib/ips/pcie.cpp | 3 ++- fpga/tests/unit/fpga.cpp | 2 +- fpga/tests/unit/global.hpp | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fpga/include/villas/fpga/utils.hpp b/fpga/include/villas/fpga/utils.hpp index b1e9fbcf7..b4c52729d 100644 --- a/fpga/include/villas/fpga/utils.hpp +++ b/fpga/include/villas/fpga/utils.hpp @@ -9,7 +9,7 @@ #pragma once #include -#include +#include namespace villas { namespace fpga { diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index a314591c7..b6876fb55 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -14,6 +14,7 @@ #include #include +#include #include using namespace villas::fpga::ip; @@ -40,7 +41,8 @@ InterruptController::init() for (int i = 0; i < num_irqs; i++) { // Try pinning to core - int ret = kernel::setIRQAffinity(nos[i], card->affinity, nullptr); + PCIeCard* pciecard = dynamic_cast(card); + int ret = kernel::setIRQAffinity(nos[i], pciecard->affinity, nullptr); switch(ret) { case 0: diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index 7f9c840c0..c5072a625 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -12,6 +12,7 @@ #include #include +#include #include using namespace villas::fpga::ip; @@ -54,7 +55,7 @@ AxiPciExpressBridge::init() auto pciAddrSpaceId = mm.getPciAddressSpace(); - auto regions = card->pdev->getRegions(); + auto regions = dynamic_cast(card)->pdev->getRegions(); int i = 0; for (auto region : regions) { diff --git a/fpga/tests/unit/fpga.cpp b/fpga/tests/unit/fpga.cpp index ab80c2b25..b52c6e257 100644 --- a/fpga/tests/unit/fpga.cpp +++ b/fpga/tests/unit/fpga.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include "global.hpp" diff --git a/fpga/tests/unit/global.hpp b/fpga/tests/unit/global.hpp index 731a8cc13..d944363c9 100644 --- a/fpga/tests/unit/global.hpp +++ b/fpga/tests/unit/global.hpp @@ -9,7 +9,7 @@ #include -#include +#include class FpgaState { public: From 9be5c3c274fc292c1488e41e1c4cbd276701fa5b Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Thu, 26 Jan 2023 17:15:51 +0100 Subject: [PATCH 11/16] removed duplicate inherited members Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/pciecard.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fpga/include/villas/fpga/pciecard.hpp b/fpga/include/villas/fpga/pciecard.hpp index 7c784da63..6e3da0b85 100644 --- a/fpga/include/villas/fpga/pciecard.hpp +++ b/fpga/include/villas/fpga/pciecard.hpp @@ -76,11 +76,8 @@ private: std::set memoryBlocksMapped; public: // TODO: make this private - std::list> ips; // IPs located on this FPGA card - bool doReset; // Reset VILLASfpga during startup? int affinity; // Affinity for MSI interrupts - bool polling; // Poll on interrupts? std::string name; // The name of the FPGA card @@ -89,9 +86,6 @@ public: // TODO: make this private // The VFIO container that this card is part of std::shared_ptr vfioContainer; - // The VFIO device that represents this card - std::shared_ptr vfioDevice; - // Slave address space ID to access the PCIe address space from the FPGA MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; @@ -105,8 +99,6 @@ protected: { return villas::logging.get(name); } - - Logger logger; }; class PCIeCardFactory : public plugin::Plugin { From 3587ccc0faa6754b6e7ed82bdcded89afc14b5c4 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Thu, 26 Jan 2023 18:11:25 +0100 Subject: [PATCH 12/16] change pciecard name to pcie_card Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/{pciecard.hpp => pcie_card.hpp} | 0 fpga/include/villas/fpga/utils.hpp | 2 +- fpga/lib/CMakeLists.txt | 2 +- fpga/lib/ips/intc.cpp | 2 +- fpga/lib/ips/pcie.cpp | 2 +- fpga/lib/{pciecard.cpp => pcie_card.cpp} | 2 +- fpga/tests/unit/fpga.cpp | 4 ++-- fpga/tests/unit/global.hpp | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename fpga/include/villas/fpga/{pciecard.hpp => pcie_card.hpp} (100%) rename fpga/lib/{pciecard.cpp => pcie_card.cpp} (99%) diff --git a/fpga/include/villas/fpga/pciecard.hpp b/fpga/include/villas/fpga/pcie_card.hpp similarity index 100% rename from fpga/include/villas/fpga/pciecard.hpp rename to fpga/include/villas/fpga/pcie_card.hpp diff --git a/fpga/include/villas/fpga/utils.hpp b/fpga/include/villas/fpga/utils.hpp index b4c52729d..d21ef0ebc 100644 --- a/fpga/include/villas/fpga/utils.hpp +++ b/fpga/include/villas/fpga/utils.hpp @@ -9,7 +9,7 @@ #pragma once #include -#include +#include namespace villas { namespace fpga { diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 571de91d4..a89150c07 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -8,7 +8,7 @@ set(SOURCES vlnv.cpp card.cpp - pciecard.cpp + pcie_card.cpp core.cpp node.cpp utils.cpp diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index b6876fb55..22bb56267 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -14,8 +14,8 @@ #include #include -#include #include +#include using namespace villas::fpga::ip; diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index c5072a625..2fbe1e9d2 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -12,8 +12,8 @@ #include #include -#include #include +#include using namespace villas::fpga::ip; diff --git a/fpga/lib/pciecard.cpp b/fpga/lib/pcie_card.cpp similarity index 99% rename from fpga/lib/pciecard.cpp rename to fpga/lib/pcie_card.cpp index 557698f12..0ff96e1eb 100644 --- a/fpga/lib/pciecard.cpp +++ b/fpga/lib/pcie_card.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include using namespace villas; using namespace villas::fpga; diff --git a/fpga/tests/unit/fpga.cpp b/fpga/tests/unit/fpga.cpp index b52c6e257..e41dc3623 100644 --- a/fpga/tests/unit/fpga.cpp +++ b/fpga/tests/unit/fpga.cpp @@ -7,10 +7,10 @@ #include -#include #include -#include +#include #include +#include #include "global.hpp" diff --git a/fpga/tests/unit/global.hpp b/fpga/tests/unit/global.hpp index d944363c9..2ba503c95 100644 --- a/fpga/tests/unit/global.hpp +++ b/fpga/tests/unit/global.hpp @@ -9,7 +9,7 @@ #include -#include +#include class FpgaState { public: From 6de526c89408f1dfe51860f565309c4252d6f4cf Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 27 Jan 2023 13:59:40 +0100 Subject: [PATCH 13/16] fixed formatting Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index e3839aeaf..a5da5c220 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -1,11 +1,11 @@ -/** FPGA card +/** FPGA pciecard * * This class represents a FPGA device. * * Author: Steffen Vogel * Author: Daniel Krebs - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power - *Systems, EONERC SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 *********************************************************************************/ #pragma once @@ -14,14 +14,12 @@ #include #include -namespace villas -{ -namespace fpga -{ +namespace villas { +namespace fpga { class Card { - public: +public: bool polling; std::shared_ptr vfioDevice; @@ -45,13 +43,13 @@ class Card bool mapMemoryBlock(const MemoryBlock &block); bool unmapMemoryBlock(const MemoryBlock &block); - private: +private: // Cache a set of already mapped memory blocks std::set memoryBlocksMapped; std::shared_ptr vfioContainer; - protected: +protected: // Logger getLogger() const // { // return villas::logging.get(name); From 7f2ed2180dcac2ee8e77183015c136088288a31e Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 27 Jan 2023 14:04:12 +0100 Subject: [PATCH 14/16] fixed formatting Signed-off-by: Pascal Henry Bauer --- fpga/lib/card.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index f46b19528..a03820e81 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -5,7 +5,7 @@ * Author: Steffen Vogel * Author: Daniel Krebs * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power - *Systems, EONERC SPDX-License-Identifier: Apache-2.0 + * Systems, EONERC SPDX-License-Identifier: Apache-2.0 *********************************************************************************/ #include From 2a9db48888bcc9a77c3d24b91e968072836d813e Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 27 Jan 2023 15:52:22 +0100 Subject: [PATCH 15/16] fixed wrong name in comment Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index a5da5c220..eb903e765 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -1,4 +1,4 @@ -/** FPGA pciecard +/** FPGA card * * This class represents a FPGA device. * From f81a1ddc6dcf95097aa02dd9cbc5b38355111149 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 27 Jan 2023 16:57:07 +0100 Subject: [PATCH 16/16] moved destructor to base class Signed-off-by: Pascal Henry Bauer --- fpga/lib/card.cpp | 16 ++++++++++++++++ fpga/lib/pcie_card.cpp | 17 +---------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index a03820e81..3d8dda65c 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -15,6 +15,22 @@ using namespace villas::fpga; Card::~Card() { + // Ensure IP destructors are called before memory is unmapped + ips.clear(); + + auto &mm = MemoryManager::get(); + + // Unmap all memory blocks + for (auto &mappedMemoryBlock : memoryBlocksMapped) { + auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock); + + const uintptr_t iova = translation.getLocalAddr(0); + const size_t size = translation.getSize(); + + logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", + mappedMemoryBlock, iova, size); + vfioContainer->memoryUnmap(iova, size); + } } std::shared_ptr Card::lookupIp(const std::string &name) const diff --git a/fpga/lib/pcie_card.cpp b/fpga/lib/pcie_card.cpp index 0ff96e1eb..c75cc38f6 100644 --- a/fpga/lib/pcie_card.cpp +++ b/fpga/lib/pcie_card.cpp @@ -152,22 +152,7 @@ std::list> PCIeCardFactory::make(json_t *json, std::sh PCIeCard::~PCIeCard() { - // Ensure IP destructors are called before memory is unmapped - ips.clear(); - - auto &mm = MemoryManager::get(); - - // Unmap all memory blocks - for (auto &mappedMemoryBlock : memoryBlocksMapped) { - auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock); - - const uintptr_t iova = translation.getLocalAddr(0); - const size_t size = translation.getSize(); - - logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", - mappedMemoryBlock, iova, size); - vfioContainer->memoryUnmap(iova, size); - } + } std::shared_ptr PCIeCard::lookupIp(const std::string &name) const