From e254e7cfe6751b752b52d41a06488450d92afe20 Mon Sep 17 00:00:00 2001 From: Pascal Henry Bauer Date: Fri, 20 Jan 2023 21:42:17 +0100 Subject: [PATCH] move card class to own file Signed-off-by: Pascal Henry Bauer --- fpga/include/villas/fpga/card.hpp | 153 +-------------- fpga/lib/card.cpp | 297 ------------------------------ 2 files changed, 2 insertions(+), 448 deletions(-) diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 6e47227a6..61a6548f6 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -1,154 +1,5 @@ -/** FPGA card - * - * This class represents a FPGA device. - * - * Author: Steffen Vogel - * Author: Daniel Krebs - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 - *********************************************************************************/ - -#pragma once - -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include - -namespace villas { -namespace fpga { - -// Forward declarations -struct vfio_container; -class PCIeCardFactory; class Card { public: - friend PCIeCardFactory; -}; - -class PCIeCard : public Card { -public: - - ~PCIeCard(); - - bool init(); - - bool stop() - { - return true; - } - - bool check() - { - return true; - } - - bool reset() - { - // TODO: Try via sysfs? - // echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset - return true; - } - - void dump() - { } - - std::shared_ptr - lookupIp(const std::string &name) const; - - std::shared_ptr - lookupIp(const Vlnv &vlnv) const; - - std::shared_ptr - lookupIp(const ip::IpIdentifier &id) const; - - bool mapMemoryBlock(const MemoryBlock &block); - bool unmapMemoryBlock(const MemoryBlock &block); - -private: - // Cache a set of already mapped memory blocks - std::set memoryBlocksMapped; - -public: // TODO: make this private - std::list> ips; // IPs located on this FPGA card - - bool doReset; // Reset VILLASfpga during startup? - int affinity; // Affinity for MSI interrupts - bool polling; // Poll on interrupts? - - std::string name; // The name of the FPGA card - - std::shared_ptr pdev; // PCI device handle - - // The VFIO container that this card is part of - std::shared_ptr vfioContainer; - - // The VFIO device that represents this card - std::shared_ptr vfioDevice; - - // Slave address space ID to access the PCIe address space from the FPGA - MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; - - // Address space identifier of the master address space of this FPGA card. - // This will be used for address resolution of all IPs on this card. - MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; - -protected: - Logger - getLogger() const - { - return villas::logging.get(name); - } - - Logger logger; -}; - -class PCIeCardFactory : public plugin::Plugin { -public: - - static - std::list> make(json_t *json, std::shared_ptr pci, std::shared_ptr vc); - - static - PCIeCard* make() - { - return new PCIeCard(); - } - - static Logger - getStaticLogger() - { - return villas::logging.get("pcie:card:factory"); - } - - virtual std::string - getName() const - { - return "pcie"; - } - - virtual std::string - getDescription() const - { - return "Xilinx PCIe FPGA cards"; - } - - virtual - std::string getType() const - { - return "card"; - } -}; - -} /* namespace fpga */ -} /* namespace villas */ + +}; \ No newline at end of file diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index da4825e83..e69de29bb 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -1,297 +0,0 @@ -/** FPGA card. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC - * SPDX-License-Identifier: Apache-2.0 - *********************************************************************************/ - -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include - -using namespace villas; -using namespace villas::fpga; - -// Instantiate factory to register -static PCIeCardFactory PCIeCardFactoryInstance; - -static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA))); - -std::list> PCIeCardFactory::make(json_t *json, std::shared_ptr pci, std::shared_ptr vc) -{ - std::list> cards; - auto logger = getStaticLogger(); - - const char *card_name; - json_t *json_card; - json_object_foreach(json, card_name, json_card) { - logger->info("Found config for FPGA card {}", card_name); - - json_t* json_ips = nullptr; - json_t* json_paths = nullptr; - const char* pci_slot = nullptr; - const char* pci_id = nullptr; - int do_reset = 0; - int affinity = 0; - int polling = 0; - - json_error_t err; - int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }", - "ips", &json_ips, - "affinity", &affinity, - "do_reset", &do_reset, - "slot", &pci_slot, - "id", &pci_id, - "polling", &polling, - "paths", &json_paths); - - if (ret != 0) - throw ConfigError(json_card, err, "", "Failed to parse card"); - - auto card = std::unique_ptr(make()); - - // Populate generic properties - card->name = std::string(card_name); - card->vfioContainer = vc; - card->affinity = affinity; - card->doReset = do_reset != 0; - card->polling = (polling != 0); - - kernel::pci::Device filter = defaultFilter; - - if (pci_id) - filter.id = kernel::pci::Id(pci_id); - if (pci_slot) - filter.slot = kernel::pci::Slot(pci_slot); - - // Search for FPGA card - card->pdev = pci->lookupDevice(filter); - if (!card->pdev) { - logger->warn("Failed to find PCI device"); - continue; - } - - if (not card->init()) { - logger->warn("Cannot start FPGA card {}", card_name); - continue; - } - - // Load IPs from a separate json file - if (json_is_string(json_ips)) { - auto json_ips_fn = json_string_value(json_ips); - json_ips = json_load_file(json_ips_fn, 0, nullptr); - if (json_ips == nullptr) - throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn); - } - - if (not json_is_object(json_ips)) - throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!"); - - card->ips = ip::CoreFactory::make(card.get(), json_ips); - if (card->ips.empty()) - throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name); - - if (not card->check()) - throw RuntimeError("Checking of FPGA card {} failed", card_name); - - // Additional static paths for AXI-Steram switch - if (json_paths != nullptr) { - if (not json_is_array(json_paths)) - throw ConfigError(json_paths, err, "", "Switch path configuration must be an array"); - - size_t i; - json_t *json_path; - json_array_foreach(json_paths, i, json_path) { - const char *from, *to; - int reverse = 0; - - ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }", - "from", &from, - "to", &to, - "reverse", &reverse - ); - if (ret != 0) - throw ConfigError(json_path, err, "", "Cannot parse switch path config"); - - auto masterIpCore = card->lookupIp(from); - if (!masterIpCore) - throw ConfigError(json_path, "", "Unknown IP {}", from); - - auto slaveIpCore = card->lookupIp(to); - if (!slaveIpCore) - throw ConfigError(json_path, "", "Unknown IP {}", to); - - auto masterIpNode = std::dynamic_pointer_cast(masterIpCore); - if (!masterIpNode) - throw ConfigError(json_path, "", "IP {} is not a streaming node", from); - - auto slaveIpNode = std::dynamic_pointer_cast(slaveIpCore); - if (!slaveIpNode) - throw ConfigError(json_path, "", "IP {} is not a streaming node", to); - - if (not masterIpNode->connect(*slaveIpNode, reverse != 0)) - throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to); - } - } - - cards.push_back(std::move(card)); - } - - return cards; -} - -PCIeCard::~PCIeCard() -{ - // Ensure IP destructors are called before memory is unmapped - ips.clear(); - - auto &mm = MemoryManager::get(); - - // Unmap all memory blocks - for (auto &mappedMemoryBlock : memoryBlocksMapped) { - auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock); - - const uintptr_t iova = translation.getLocalAddr(0); - const size_t size = translation.getSize(); - - logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", - mappedMemoryBlock, iova, size); - vfioContainer->memoryUnmap(iova, size); - } -} - -std::shared_ptr PCIeCard::lookupIp(const std::string &name) const -{ - for (auto &ip : ips) { - if (*ip == name) { - return ip; - } - } - - return nullptr; -} - -std::shared_ptr PCIeCard::lookupIp(const Vlnv &vlnv) const -{ - for (auto &ip : ips) { - if (*ip == vlnv) { - return ip; - } - } - - return nullptr; -} - -std::shared_ptr PCIeCard::lookupIp(const ip::IpIdentifier &id) const -{ - for (auto &ip : ips) { - if (*ip == id) { - return ip; - } - } - - return nullptr; -} - -bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block) -{ - if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) { - throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped."); - } - - auto &mm = MemoryManager::get(); - - auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId()); - - const uintptr_t iova = translation.getLocalAddr(0); - const size_t size = translation.getSize(); - - logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}", - block.getAddrSpaceId(), iova, size); - vfioContainer->memoryUnmap(iova, size); - - memoryBlocksMapped.erase(block.getAddrSpaceId()); - - return true; -} - -bool PCIeCard::mapMemoryBlock(const MemoryBlock &block) -{ - if (not vfioContainer->isIommuEnabled()) { - logger->warn("VFIO mapping not supported without IOMMU"); - return false; - } - - auto &mm = MemoryManager::get(); - const auto &addrSpaceId = block.getAddrSpaceId(); - - if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) - // Block already mapped - return true; - else - logger->debug("Create VFIO mapping for {}", addrSpaceId); - - auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); - uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); - uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, - UINTPTR_MAX, - block.getSize()); - - if (iovaAddr == UINTPTR_MAX) { - logger->error("Cannot map memory at {:#x} of size {:#x}", - processBaseAddr, block.getSize()); - return false; - } - - mm.createMapping(iovaAddr, 0, block.getSize(), - "VFIO-D2H", - this->addrSpaceIdDeviceToHost, - addrSpaceId); - - // Remember that this block has already been mapped for later - memoryBlocksMapped.insert(addrSpaceId); - - return true; -} - -bool PCIeCard::init() -{ - logger = getLogger(); - - logger->info("Initializing FPGA card {}", name); - - // Attach PCIe card to VFIO container - vfioDevice = vfioContainer->attachDevice(*pdev); - - // Enable memory access and PCI bus mastering for DMA - if (not vfioDevice->pciEnable()) { - logger->error("Failed to enable PCI device"); - return false; - } - - // Reset system? - if (doReset) { - // Reset / detect PCI device - if (not vfioDevice->pciHotReset()) { - logger->error("Failed to reset PCI device"); - return false; - } - - if (not reset()) { - logger->error("Failed to reset FGPA card"); - return false; - } - } - - return true; -}