Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

Merge pull request #78 from VILLASframework/refactorPcieClass

Browse source 
Refactor pcie class
This commit is contained in:
Niklas Eiling 2023-01-30 11:34:14 +01:00 • committed by GitHub
commit 02ce5b15dc
12 changed files with 547 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

148
fpga/include/villas/fpga/card.hpp
View file

@ -7,148 +7,56 @@
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
* SPDX-License-Identifier: Apache-2.0
*********************************************************************************/
#pragma once
#include <list>
#include <set>
#include <string>
#include <jansson.h>
#include <villas/plugin.hpp>
#include <villas/memory.hpp>
#include <villas/kernel/pci.hpp>
#include <villas/kernel/vfio_container.hpp>
#include <villas/fpga/config.h>
#include <villas/fpga/core.hpp>
#include <villas/kernel/vfio_container.hpp>
namespace villas {
namespace fpga {
// Forward declarations
struct vfio_container;
class PCIeCardFactory;
class Card {
class Card
{
public:
friend PCIeCardFactory;
};
bool polling;
std::shared_ptr<kernel::vfio::Device> vfioDevice;
class PCIeCard : public Card {
public:
// Slave address space ID to access the PCIe address space from the
// FPGA
MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost;
~PCIeCard();
// Address space identifier of the master address space of this FPGA
// card. This will be used for address resolution of all IPs on this
// card.
MemoryManager::AddressSpaceId addrSpaceIdHostToDevice;
bool init();
std::list<std::shared_ptr<ip::Core> > ips;
bool stop()
{
return true;
}
virtual ~Card();
bool check()
{
return true;
}
std::shared_ptr<ip::Core> lookupIp(const std::string &name) const;
std::shared_ptr<ip::Core> lookupIp(const Vlnv &vlnv) const;
std::shared_ptr<ip::Core> lookupIp(const ip::IpIdentifier &id) const;
bool reset()
{
// TODO: Try via sysfs?
// echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset
return true;
}
void dump()
{ }
std::shared_ptr<ip::Core>
lookupIp(const std::string &name) const;
std::shared_ptr<ip::Core>
lookupIp(const Vlnv &vlnv) const;
std::shared_ptr<ip::Core>
lookupIp(const ip::IpIdentifier &id) const;
bool mapMemoryBlock(const MemoryBlock &block);
bool unmapMemoryBlock(const MemoryBlock &block);
bool mapMemoryBlock(const MemoryBlock &block);
bool unmapMemoryBlock(const MemoryBlock &block);
private:
// Cache a set of already mapped memory blocks
std::set<MemoryManager::AddressSpaceId> memoryBlocksMapped;
// Cache a set of already mapped memory blocks
std::set<MemoryManager::AddressSpaceId> memoryBlocksMapped;
public: // TODO: make this private
std::list<std::shared_ptr<ip::Core>> ips; // IPs located on this FPGA card
bool doReset; // Reset VILLASfpga during startup?
int affinity; // Affinity for MSI interrupts
bool polling; // Poll on interrupts?
std::string name; // The name of the FPGA card
std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
// The VFIO container that this card is part of
std::shared_ptr<kernel::vfio::Container> vfioContainer;
// The VFIO device that represents this card
std::shared_ptr<kernel::vfio::Device> vfioDevice;
// Slave address space ID to access the PCIe address space from the FPGA
MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost;
// Address space identifier of the master address space of this FPGA card.
// This will be used for address resolution of all IPs on this card.
MemoryManager::AddressSpaceId addrSpaceIdHostToDevice;
std::shared_ptr<kernel::vfio::Container> vfioContainer;
protected:
Logger
getLogger() const
{
return villas::logging.get(name);
}
// Logger getLogger() const
// {
// return villas::logging.get(name);
// }
Logger logger;
};
class PCIeCardFactory : public plugin::Plugin {
public:
static
std::list<std::shared_ptr<PCIeCard>> make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc);
static
PCIeCard* make()
{
return new PCIeCard();
}
static Logger
getStaticLogger()
{
return villas::logging.get("pcie:card:factory");
}
virtual std::string
getName() const
{
return "pcie";
}
virtual std::string
getDescription() const
{
return "Xilinx PCIe FPGA cards";
}
virtual
std::string getType() const
{
return "card";
}
Logger logger;
};
} /* namespace fpga */
} /* namespace villas */
} /* namespace villas */

6
fpga/include/villas/fpga/core.hpp
View file

@ -26,7 +26,7 @@ namespace villas {
namespace fpga {
// Forward declarations
class PCIeCard;
class Card;
namespace ip {
@ -242,7 +242,7 @@ protected:
Logger logger;
// FPGA card this IP is instantiated on (populated by FpgaIpFactory)
PCIeCard* card;
Card* card;
// Identifier of this IP with its instance name and VLNV
IpIdentifier id;
@ -266,7 +266,7 @@ public:
// Returns a running and checked FPGA IP
static
std::list<std::shared_ptr<Core>> make(PCIeCard* card, json_t *json_ips);
std::list<std::shared_ptr<Core>> make(Card* card, json_t *json_ips);
virtual
std::string getType() const

142
fpga/include/villas/fpga/pcie_card.hpp Normal file
View file

@ -0,0 +1,142 @@
/** FPGA pciecard
*
* This class represents a FPGA device.
*
* Author: Steffen Vogel <post@steffenvogel.de>
* Author: Daniel Krebs <github@daniel-krebs.net>
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
* SPDX-License-Identifier: Apache-2.0
*********************************************************************************/
#pragma once
#include <list>
#include <set>
#include <string>
#include <jansson.h>
#include <villas/plugin.hpp>
#include <villas/memory.hpp>
#include <villas/kernel/pci.hpp>
#include <villas/kernel/vfio_container.hpp>
#include <villas/fpga/card.hpp>
#include <villas/fpga/config.h>
#include <villas/fpga/core.hpp>
namespace villas {
namespace fpga {
// Forward declarations
struct vfio_container;
class PCIeCardFactory;
class PCIeCard : public Card {
public:
~PCIeCard();
bool init();
bool stop()
{
return true;
}
bool check()
{
return true;
}
bool reset()
{
// TODO: Try via sysfs?
// echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset
return true;
}
void dump()
{ }
std::shared_ptr<ip::Core>
lookupIp(const std::string &name) const;
std::shared_ptr<ip::Core>
lookupIp(const Vlnv &vlnv) const;
std::shared_ptr<ip::Core>
lookupIp(const ip::IpIdentifier &id) const;
bool mapMemoryBlock(const MemoryBlock &block);
bool unmapMemoryBlock(const MemoryBlock &block);
private:
// Cache a set of already mapped memory blocks
std::set<MemoryManager::AddressSpaceId> memoryBlocksMapped;
public: // TODO: make this private
bool doReset; // Reset VILLASfpga during startup?
int affinity; // Affinity for MSI interrupts
std::string name; // The name of the FPGA card
std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
// The VFIO container that this card is part of
std::shared_ptr<kernel::vfio::Container> vfioContainer;
// Slave address space ID to access the PCIe address space from the FPGA
MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost;
// Address space identifier of the master address space of this FPGA card.
// This will be used for address resolution of all IPs on this card.
MemoryManager::AddressSpaceId addrSpaceIdHostToDevice;
protected:
Logger
getLogger() const
{
return villas::logging.get(name);
}
};
class PCIeCardFactory : public plugin::Plugin {
public:
static
std::list<std::shared_ptr<PCIeCard>> make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc);
static
PCIeCard* make()
{
return new PCIeCard();
}
static Logger
getStaticLogger()
{
return villas::logging.get("pcie:card:factory");
}
virtual std::string
getName() const
{
return "pcie";
}
virtual std::string
getDescription() const
{
return "Xilinx PCIe FPGA cards";
}
virtual
std::string getType() const
{
return "card";
}
};
} /* namespace fpga */
} /* namespace villas */

2
fpga/include/villas/fpga/utils.hpp
View file

@ -9,7 +9,7 @@
#pragma once
#include <string>
#include <villas/fpga/card.hpp>
#include <villas/fpga/pcie_card.hpp>
namespace villas {
namespace fpga {

1
fpga/lib/CMakeLists.txt
View file

@ -8,6 +8,7 @@
set(SOURCES
vlnv.cpp
card.cpp
pcie_card.cpp
core.cpp
node.cpp
utils.cpp

321
fpga/lib/card.cpp
View file

@ -1,158 +1,21 @@
/** FPGA card.
/** FPGA card
*
* This class represents a FPGA device.
*
* Author: Steffen Vogel <post@steffenvogel.de>
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
* SPDX-License-Identifier: Apache-2.0
* Author: Daniel Krebs <github@daniel-krebs.net>
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power
* Systems, EONERC SPDX-License-Identifier: Apache-2.0
*********************************************************************************/
#include <string>
#include <memory>
#include <utility>
#include <villas/exceptions.hpp>
#include <villas/memory.hpp>
#include <villas/kernel/pci.hpp>
#include <villas/kernel/vfio_container.hpp>
#include <villas/fpga/core.hpp>
#include <villas/fpga/node.hpp>
#include <villas/fpga/card.hpp>
using namespace villas;
using namespace villas::fpga;
// Instantiate factory to register
static PCIeCardFactory PCIeCardFactoryInstance;
static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA)));
std::list<std::shared_ptr<PCIeCard>> PCIeCardFactory::make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc)
Card::~Card()
{
std::list<std::shared_ptr<PCIeCard>> cards;
auto logger = getStaticLogger();
const char *card_name;
json_t *json_card;
json_object_foreach(json, card_name, json_card) {
logger->info("Found config for FPGA card {}", card_name);
json_t* json_ips = nullptr;
json_t* json_paths = nullptr;
const char* pci_slot = nullptr;
const char* pci_id = nullptr;
int do_reset = 0;
int affinity = 0;
int polling = 0;
json_error_t err;
int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }",
"ips", &json_ips,
"affinity", &affinity,
"do_reset", &do_reset,
"slot", &pci_slot,
"id", &pci_id,
"polling", &polling,
"paths", &json_paths);
if (ret != 0)
throw ConfigError(json_card, err, "", "Failed to parse card");
auto card = std::unique_ptr<PCIeCard>(make());
// Populate generic properties
card->name = std::string(card_name);
card->vfioContainer = vc;
card->affinity = affinity;
card->doReset = do_reset != 0;
card->polling = (polling != 0);
kernel::pci::Device filter = defaultFilter;
if (pci_id)
filter.id = kernel::pci::Id(pci_id);
if (pci_slot)
filter.slot = kernel::pci::Slot(pci_slot);
// Search for FPGA card
card->pdev = pci->lookupDevice(filter);
if (!card->pdev) {
logger->warn("Failed to find PCI device");
continue;
}
if (not card->init()) {
logger->warn("Cannot start FPGA card {}", card_name);
continue;
}
// Load IPs from a separate json file
if (json_is_string(json_ips)) {
auto json_ips_fn = json_string_value(json_ips);
json_ips = json_load_file(json_ips_fn, 0, nullptr);
if (json_ips == nullptr)
throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn);
}
if (not json_is_object(json_ips))
throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!");
card->ips = ip::CoreFactory::make(card.get(), json_ips);
if (card->ips.empty())
throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name);
if (not card->check())
throw RuntimeError("Checking of FPGA card {} failed", card_name);
// Additional static paths for AXI-Steram switch
if (json_paths != nullptr) {
if (not json_is_array(json_paths))
throw ConfigError(json_paths, err, "", "Switch path configuration must be an array");
size_t i;
json_t *json_path;
json_array_foreach(json_paths, i, json_path) {
const char *from, *to;
int reverse = 0;
ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }",
"from", &from,
"to", &to,
"reverse", &reverse
);
if (ret != 0)
throw ConfigError(json_path, err, "", "Cannot parse switch path config");
auto masterIpCore = card->lookupIp(from);
if (!masterIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", from);
auto slaveIpCore = card->lookupIp(to);
if (!slaveIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", to);
auto masterIpNode = std::dynamic_pointer_cast<ip::Node>(masterIpCore);
if (!masterIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", from);
auto slaveIpNode = std::dynamic_pointer_cast<ip::Node>(slaveIpCore);
if (!slaveIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", to);
if (not masterIpNode->connect(*slaveIpNode, reverse != 0))
throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to);
}
}
cards.push_back(std::move(card));
}
return cards;
}
PCIeCard::~PCIeCard()
{
// Ensure IP destructors are called before memory is unmapped
// Ensure IP destructors are called before memory is unmapped
ips.clear();
auto &mm = MemoryManager::get();
@ -170,128 +33,106 @@ PCIeCard::~PCIeCard()
}
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const std::string &name) const
std::shared_ptr<ip::Core> Card::lookupIp(const std::string &name) const
{
for (auto &ip : ips) {
if (*ip == name) {
return ip;
}
}
for(auto &ip : ips) {
if(*ip == name) {
return ip;
}
}
return nullptr;
return nullptr;
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const Vlnv &vlnv) const
std::shared_ptr<ip::Core> Card::lookupIp(const Vlnv &vlnv) const
{
for (auto &ip : ips) {
if (*ip == vlnv) {
return ip;
}
}
for(auto &ip : ips) {
if(*ip == vlnv) {
return ip;
}
}
return nullptr;
return nullptr;
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const ip::IpIdentifier &id) const
std::shared_ptr<ip::Core> Card::lookupIp(const ip::IpIdentifier &id) const
{
for (auto &ip : ips) {
if (*ip == id) {
return ip;
}
}
for(auto &ip : ips) {
if(*ip == id) {
return ip;
}
}
return nullptr;
return nullptr;
}
bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block)
bool Card::unmapMemoryBlock(const MemoryBlock &block)
{
if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) {
throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped.");
}
if(memoryBlocksMapped.find(block.getAddrSpaceId())
== memoryBlocksMapped.end()) {
throw std::runtime_error(
"Block " + std::to_string(block.getAddrSpaceId())
+ " is not mapped but was requested to be unmapped.");
}
auto &mm = MemoryManager::get();
auto &mm = MemoryManager::get();
auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId());
auto translation = mm.getTranslation(addrSpaceIdDeviceToHost,
block.getAddrSpaceId());
const uintptr_t iova = translation.getLocalAddr(0);
const size_t size = translation.getSize();
const uintptr_t iova = translation.getLocalAddr(0);
const size_t size = translation.getSize();
logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}",
block.getAddrSpaceId(), iova, size);
vfioContainer->memoryUnmap(iova, size);
logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}",
block.getAddrSpaceId(),
iova,
size);
vfioContainer->memoryUnmap(iova, size);
memoryBlocksMapped.erase(block.getAddrSpaceId());
memoryBlocksMapped.erase(block.getAddrSpaceId());
return true;
return true;
}
bool PCIeCard::mapMemoryBlock(const MemoryBlock &block)
bool Card::mapMemoryBlock(const MemoryBlock &block)
{
if (not vfioContainer->isIommuEnabled()) {
logger->warn("VFIO mapping not supported without IOMMU");
return false;
}
if(not vfioContainer->isIommuEnabled()) {
logger->warn("VFIO mapping not supported without IOMMU");
return false;
}
auto &mm = MemoryManager::get();
const auto &addrSpaceId = block.getAddrSpaceId();
auto &mm = MemoryManager::get();
const auto &addrSpaceId = block.getAddrSpaceId();
if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end())
// Block already mapped
return true;
else
logger->debug("Create VFIO mapping for {}", addrSpaceId);
if(memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end())
// Block already mapped
return true;
else
logger->debug("Create VFIO mapping for {}", addrSpaceId);
auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId);
uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0);
uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr,
UINTPTR_MAX,
block.getSize());
auto translationFromProcess
= mm.getTranslationFromProcess(addrSpaceId);
uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0);
uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr,
UINTPTR_MAX,
block.getSize());
if (iovaAddr == UINTPTR_MAX) {
logger->error("Cannot map memory at {:#x} of size {:#x}",
processBaseAddr, block.getSize());
return false;
}
if(iovaAddr == UINTPTR_MAX) {
logger->error("Cannot map memory at {:#x} of size {:#x}",
processBaseAddr,
block.getSize());
return false;
}
mm.createMapping(iovaAddr, 0, block.getSize(),
"VFIO-D2H",
this->addrSpaceIdDeviceToHost,
addrSpaceId);
mm.createMapping(iovaAddr,
0,
block.getSize(),
"VFIO-D2H",
this->addrSpaceIdDeviceToHost,
addrSpaceId);
// Remember that this block has already been mapped for later
memoryBlocksMapped.insert(addrSpaceId);
// Remember that this block has already been mapped for later
memoryBlocksMapped.insert(addrSpaceId);
return true;
}
bool PCIeCard::init()
{
logger = getLogger();
logger->info("Initializing FPGA card {}", name);
// Attach PCIe card to VFIO container
vfioDevice = vfioContainer->attachDevice(*pdev);
// Enable memory access and PCI bus mastering for DMA
if (not vfioDevice->pciEnable()) {
logger->error("Failed to enable PCI device");
return false;
}
// Reset system?
if (doReset) {
// Reset / detect PCI device
if (not vfioDevice->pciHotReset()) {
logger->error("Failed to reset PCI device");
return false;
}
if (not reset()) {
logger->error("Failed to reset FGPA card");
return false;
}
}
return true;
return true;
}

2
fpga/lib/core.cpp
View file

@ -35,7 +35,7 @@ vlnvInitializationOrder = {
Vlnv("xilinx.com:ip:axis_switch:"),
};
std::list<std::shared_ptr<Core>> CoreFactory::make(PCIeCard* card, json_t *json_ips)
std::list<std::shared_ptr<Core>> CoreFactory::make(Card* card, json_t *json_ips)
{
// We only have this logger until we know the factory to build an IP with
auto loggerStatic = getStaticLogger();

4
fpga/lib/ips/intc.cpp
View file

@ -15,6 +15,7 @@
#include <villas/fpga/card.hpp>
#include <villas/fpga/ips/intc.hpp>
#include <villas/fpga/pcie_card.hpp>
using namespace villas::fpga::ip;
@ -40,7 +41,8 @@ InterruptController::init()
for (int i = 0; i < num_irqs; i++) {
// Try pinning to core
int ret = kernel::setIRQAffinity(nos[i], card->affinity, nullptr);
PCIeCard* pciecard = dynamic_cast<PCIeCard*>(card);
int ret = kernel::setIRQAffinity(nos[i], pciecard->affinity, nullptr);
switch(ret) {
case 0:

3
fpga/lib/ips/pcie.cpp
View file

@ -13,6 +13,7 @@
#include <villas/fpga/card.hpp>
#include <villas/fpga/ips/pcie.hpp>
#include <villas/fpga/pcie_card.hpp>
using namespace villas::fpga::ip;
@ -54,7 +55,7 @@ AxiPciExpressBridge::init()
auto pciAddrSpaceId = mm.getPciAddressSpace();
auto regions = card->pdev->getRegions();
auto regions = dynamic_cast<PCIeCard*>(card)->pdev->getRegions();
int i = 0;
for (auto region : regions) {

282
fpga/lib/pcie_card.cpp Normal file
View file

@ -0,0 +1,282 @@
/** FPGA pciecard.
*
* Author: Steffen Vogel <post@steffenvogel.de>
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
* SPDX-License-Identifier: Apache-2.0
*********************************************************************************/
#include <string>
#include <memory>
#include <utility>
#include <villas/exceptions.hpp>
#include <villas/memory.hpp>
#include <villas/kernel/pci.hpp>
#include <villas/kernel/vfio_container.hpp>
#include <villas/fpga/core.hpp>
#include <villas/fpga/node.hpp>
#include <villas/fpga/pcie_card.hpp>
using namespace villas;
using namespace villas::fpga;
// Instantiate factory to register
static PCIeCardFactory PCIeCardFactoryInstance;
static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA)));
std::list<std::shared_ptr<PCIeCard>> PCIeCardFactory::make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc)
{
std::list<std::shared_ptr<PCIeCard>> cards;
auto logger = getStaticLogger();
const char *card_name;
json_t *json_card;
json_object_foreach(json, card_name, json_card) {
logger->info("Found config for FPGA card {}", card_name);
json_t* json_ips = nullptr;
json_t* json_paths = nullptr;
const char* pci_slot = nullptr;
const char* pci_id = nullptr;
int do_reset = 0;
int affinity = 0;
int polling = 0;
json_error_t err;
int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }",
"ips", &json_ips,
"affinity", &affinity,
"do_reset", &do_reset,
"slot", &pci_slot,
"id", &pci_id,
"polling", &polling,
"paths", &json_paths);
if (ret != 0)
throw ConfigError(json_card, err, "", "Failed to parse card");
auto card = std::unique_ptr<PCIeCard>(make());
// Populate generic properties
card->name = std::string(card_name);
card->vfioContainer = vc;
card->affinity = affinity;
card->doReset = do_reset != 0;
card->polling = (polling != 0);
kernel::pci::Device filter = defaultFilter;
if (pci_id)
filter.id = kernel::pci::Id(pci_id);
if (pci_slot)
filter.slot = kernel::pci::Slot(pci_slot);
// Search for FPGA card
card->pdev = pci->lookupDevice(filter);
if (!card->pdev) {
logger->warn("Failed to find PCI device");
continue;
}
if (not card->init()) {
logger->warn("Cannot start FPGA card {}", card_name);
continue;
}
// Load IPs from a separate json file
if (json_is_string(json_ips)) {
auto json_ips_fn = json_string_value(json_ips);
json_ips = json_load_file(json_ips_fn, 0, nullptr);
if (json_ips == nullptr)
throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn);
}
if (not json_is_object(json_ips))
throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!");
card->ips = ip::CoreFactory::make(card.get(), json_ips);
if (card->ips.empty())
throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name);
if (not card->check())
throw RuntimeError("Checking of FPGA card {} failed", card_name);
// Additional static paths for AXI-Steram switch
if (json_paths != nullptr) {
if (not json_is_array(json_paths))
throw ConfigError(json_paths, err, "", "Switch path configuration must be an array");
size_t i;
json_t *json_path;
json_array_foreach(json_paths, i, json_path) {
const char *from, *to;
int reverse = 0;
ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }",
"from", &from,
"to", &to,
"reverse", &reverse
);
if (ret != 0)
throw ConfigError(json_path, err, "", "Cannot parse switch path config");
auto masterIpCore = card->lookupIp(from);
if (!masterIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", from);
auto slaveIpCore = card->lookupIp(to);
if (!slaveIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", to);
auto masterIpNode = std::dynamic_pointer_cast<ip::Node>(masterIpCore);
if (!masterIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", from);
auto slaveIpNode = std::dynamic_pointer_cast<ip::Node>(slaveIpCore);
if (!slaveIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", to);
if (not masterIpNode->connect(*slaveIpNode, reverse != 0))
throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to);
}
}
cards.push_back(std::move(card));
}
return cards;
}
PCIeCard::~PCIeCard()
{
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const std::string &name) const
{
for (auto &ip : ips) {
if (*ip == name) {
return ip;
}
}
return nullptr;
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const Vlnv &vlnv) const
{
for (auto &ip : ips) {
if (*ip == vlnv) {
return ip;
}
}
return nullptr;
}
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const ip::IpIdentifier &id) const
{
for (auto &ip : ips) {
if (*ip == id) {
return ip;
}
}
return nullptr;
}
bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block)
{
if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) {
throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped.");
}
auto &mm = MemoryManager::get();
auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId());
const uintptr_t iova = translation.getLocalAddr(0);
const size_t size = translation.getSize();
logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}",
block.getAddrSpaceId(), iova, size);
vfioContainer->memoryUnmap(iova, size);
memoryBlocksMapped.erase(block.getAddrSpaceId());
return true;
}
bool PCIeCard::mapMemoryBlock(const MemoryBlock &block)
{
if (not vfioContainer->isIommuEnabled()) {
logger->warn("VFIO mapping not supported without IOMMU");
return false;
}
auto &mm = MemoryManager::get();
const auto &addrSpaceId = block.getAddrSpaceId();
if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end())
// Block already mapped
return true;
else
logger->debug("Create VFIO mapping for {}", addrSpaceId);
auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId);
uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0);
uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr,
UINTPTR_MAX,
block.getSize());
if (iovaAddr == UINTPTR_MAX) {
logger->error("Cannot map memory at {:#x} of size {:#x}",
processBaseAddr, block.getSize());
return false;
}
mm.createMapping(iovaAddr, 0, block.getSize(),
"VFIO-D2H",
this->addrSpaceIdDeviceToHost,
addrSpaceId);
// Remember that this block has already been mapped for later
memoryBlocksMapped.insert(addrSpaceId);
return true;
}
bool PCIeCard::init()
{
logger = getLogger();
logger->info("Initializing FPGA card {}", name);
// Attach PCIe card to VFIO container
vfioDevice = vfioContainer->attachDevice(*pdev);
// Enable memory access and PCI bus mastering for DMA
if (not vfioDevice->pciEnable()) {
logger->error("Failed to enable PCI device");
return false;
}
// Reset system?
if (doReset) {
// Reset / detect PCI device
if (not vfioDevice->pciHotReset()) {
logger->error("Failed to reset PCI device");
return false;
}
if (not reset()) {
logger->error("Failed to reset FGPA card");
return false;
}
}
return true;
}

4
fpga/tests/unit/fpga.cpp
View file

@ -7,10 +7,10 @@
#include <criterion/criterion.h>
#include <villas/utils.hpp>
#include <villas/fpga/core.hpp>
#include <villas/fpga/card.hpp>
#include <villas/fpga/pcie_card.hpp>
#include <villas/fpga/vlnv.hpp>
#include <villas/utils.hpp>
#include "global.hpp"

2
fpga/tests/unit/global.hpp
View file

@ -9,7 +9,7 @@
#include <cstdlib>
#include <villas/fpga/card.hpp>
#include <villas/fpga/pcie_card.hpp>
class FpgaState {
public: