mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
moved pciecard to own file
Signed-off-by: Pascal Henry Bauer <pascal.bauer@rwth-aachen.de>
This commit is contained in:
Pascal Henry Bauer
parent
e254e7cfe6
commit
d2d7f9430d
2 changed files with 447 additions and 0 deletions
150
fpga/include/villas/fpga/pciecard.hpp
Normal file
150
fpga/include/villas/fpga/pciecard.hpp
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/** FPGA card
|
||||
*
|
||||
* This class represents a FPGA device.
|
||||
*
|
||||
* Author: Steffen Vogel <post@steffenvogel.de>
|
||||
* Author: Daniel Krebs <github@daniel-krebs.net>
|
||||
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*********************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <list>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <jansson.h>
|
||||
|
||||
#include <villas/plugin.hpp>
|
||||
#include <villas/memory.hpp>
|
||||
|
||||
#include <villas/kernel/pci.hpp>
|
||||
#include <villas/kernel/vfio_container.hpp>
|
||||
|
||||
#include <villas/fpga/card.hpp>
|
||||
#include <villas/fpga/config.h>
|
||||
#include <villas/fpga/core.hpp>
|
||||
|
||||
namespace villas {
|
||||
namespace fpga {
|
||||
|
||||
// Forward declarations
|
||||
struct vfio_container;
|
||||
class PCIeCardFactory;
|
||||
|
||||
class PCIeCard : public Card {
|
||||
public:
|
||||
|
||||
~PCIeCard();
|
||||
|
||||
bool init();
|
||||
|
||||
bool stop()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
bool check()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
bool reset()
|
||||
{
|
||||
// TODO: Try via sysfs?
|
||||
// echo 1 > /sys/bus/pci/devices/0000\:88\:00.0/reset
|
||||
return true;
|
||||
}
|
||||
|
||||
void dump()
|
||||
{ }
|
||||
|
||||
std::shared_ptr<ip::Core>
|
||||
lookupIp(const std::string &name) const;
|
||||
|
||||
std::shared_ptr<ip::Core>
|
||||
lookupIp(const Vlnv &vlnv) const;
|
||||
|
||||
std::shared_ptr<ip::Core>
|
||||
lookupIp(const ip::IpIdentifier &id) const;
|
||||
|
||||
bool mapMemoryBlock(const MemoryBlock &block);
|
||||
bool unmapMemoryBlock(const MemoryBlock &block);
|
||||
|
||||
private:
|
||||
// Cache a set of already mapped memory blocks
|
||||
std::set<MemoryManager::AddressSpaceId> memoryBlocksMapped;
|
||||
|
||||
public: // TODO: make this private
|
||||
std::list<std::shared_ptr<ip::Core>> ips; // IPs located on this FPGA card
|
||||
|
||||
bool doReset; // Reset VILLASfpga during startup?
|
||||
int affinity; // Affinity for MSI interrupts
|
||||
bool polling; // Poll on interrupts?
|
||||
|
||||
std::string name; // The name of the FPGA card
|
||||
|
||||
std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
|
||||
|
||||
// The VFIO container that this card is part of
|
||||
std::shared_ptr<kernel::vfio::Container> vfioContainer;
|
||||
|
||||
// The VFIO device that represents this card
|
||||
std::shared_ptr<kernel::vfio::Device> vfioDevice;
|
||||
|
||||
// Slave address space ID to access the PCIe address space from the FPGA
|
||||
MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost;
|
||||
|
||||
// Address space identifier of the master address space of this FPGA card.
|
||||
// This will be used for address resolution of all IPs on this card.
|
||||
MemoryManager::AddressSpaceId addrSpaceIdHostToDevice;
|
||||
|
||||
protected:
|
||||
Logger
|
||||
getLogger() const
|
||||
{
|
||||
return villas::logging.get(name);
|
||||
}
|
||||
|
||||
Logger logger;
|
||||
};
|
||||
|
||||
class PCIeCardFactory : public plugin::Plugin {
|
||||
public:
|
||||
|
||||
static
|
||||
std::list<std::shared_ptr<PCIeCard>> make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc);
|
||||
|
||||
static
|
||||
PCIeCard* make()
|
||||
{
|
||||
return new PCIeCard();
|
||||
}
|
||||
|
||||
static Logger
|
||||
getStaticLogger()
|
||||
{
|
||||
return villas::logging.get("pcie:card:factory");
|
||||
}
|
||||
|
||||
virtual std::string
|
||||
getName() const
|
||||
{
|
||||
return "pcie";
|
||||
}
|
||||
|
||||
virtual std::string
|
||||
getDescription() const
|
||||
{
|
||||
return "Xilinx PCIe FPGA cards";
|
||||
}
|
||||
|
||||
virtual
|
||||
std::string getType() const
|
||||
{
|
||||
return "card";
|
||||
}
|
||||
};
|
||||
|
||||
} /* namespace fpga */
|
||||
} /* namespace villas */
|
||||
297
fpga/lib/pciecard.cpp
Normal file
297
fpga/lib/pciecard.cpp
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
/** FPGA card.
|
||||
*
|
||||
* Author: Steffen Vogel <post@steffenvogel.de>
|
||||
* SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*********************************************************************************/
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#include <villas/exceptions.hpp>
|
||||
#include <villas/memory.hpp>
|
||||
|
||||
#include <villas/kernel/pci.hpp>
|
||||
#include <villas/kernel/vfio_container.hpp>
|
||||
|
||||
#include <villas/fpga/core.hpp>
|
||||
#include <villas/fpga/node.hpp>
|
||||
#include <villas/fpga/pciecard.hpp>
|
||||
|
||||
using namespace villas;
|
||||
using namespace villas::fpga;
|
||||
|
||||
// Instantiate factory to register
|
||||
static PCIeCardFactory PCIeCardFactoryInstance;
|
||||
|
||||
static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA)));
|
||||
|
||||
std::list<std::shared_ptr<PCIeCard>> PCIeCardFactory::make(json_t *json, std::shared_ptr<kernel::pci::DeviceList> pci, std::shared_ptr<kernel::vfio::Container> vc)
|
||||
{
|
||||
std::list<std::shared_ptr<PCIeCard>> cards;
|
||||
auto logger = getStaticLogger();
|
||||
|
||||
const char *card_name;
|
||||
json_t *json_card;
|
||||
json_object_foreach(json, card_name, json_card) {
|
||||
logger->info("Found config for FPGA card {}", card_name);
|
||||
|
||||
json_t* json_ips = nullptr;
|
||||
json_t* json_paths = nullptr;
|
||||
const char* pci_slot = nullptr;
|
||||
const char* pci_id = nullptr;
|
||||
int do_reset = 0;
|
||||
int affinity = 0;
|
||||
int polling = 0;
|
||||
|
||||
json_error_t err;
|
||||
int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }",
|
||||
"ips", &json_ips,
|
||||
"affinity", &affinity,
|
||||
"do_reset", &do_reset,
|
||||
"slot", &pci_slot,
|
||||
"id", &pci_id,
|
||||
"polling", &polling,
|
||||
"paths", &json_paths);
|
||||
|
||||
if (ret != 0)
|
||||
throw ConfigError(json_card, err, "", "Failed to parse card");
|
||||
|
||||
auto card = std::unique_ptr<PCIeCard>(make());
|
||||
|
||||
// Populate generic properties
|
||||
card->name = std::string(card_name);
|
||||
card->vfioContainer = vc;
|
||||
card->affinity = affinity;
|
||||
card->doReset = do_reset != 0;
|
||||
card->polling = (polling != 0);
|
||||
|
||||
kernel::pci::Device filter = defaultFilter;
|
||||
|
||||
if (pci_id)
|
||||
filter.id = kernel::pci::Id(pci_id);
|
||||
if (pci_slot)
|
||||
filter.slot = kernel::pci::Slot(pci_slot);
|
||||
|
||||
// Search for FPGA card
|
||||
card->pdev = pci->lookupDevice(filter);
|
||||
if (!card->pdev) {
|
||||
logger->warn("Failed to find PCI device");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (not card->init()) {
|
||||
logger->warn("Cannot start FPGA card {}", card_name);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Load IPs from a separate json file
|
||||
if (json_is_string(json_ips)) {
|
||||
auto json_ips_fn = json_string_value(json_ips);
|
||||
json_ips = json_load_file(json_ips_fn, 0, nullptr);
|
||||
if (json_ips == nullptr)
|
||||
throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to load FPGA IP cores from {}", json_ips_fn);
|
||||
}
|
||||
|
||||
if (not json_is_object(json_ips))
|
||||
throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!");
|
||||
|
||||
card->ips = ip::CoreFactory::make(card.get(), json_ips);
|
||||
if (card->ips.empty())
|
||||
throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name);
|
||||
|
||||
if (not card->check())
|
||||
throw RuntimeError("Checking of FPGA card {} failed", card_name);
|
||||
|
||||
// Additional static paths for AXI-Steram switch
|
||||
if (json_paths != nullptr) {
|
||||
if (not json_is_array(json_paths))
|
||||
throw ConfigError(json_paths, err, "", "Switch path configuration must be an array");
|
||||
|
||||
size_t i;
|
||||
json_t *json_path;
|
||||
json_array_foreach(json_paths, i, json_path) {
|
||||
const char *from, *to;
|
||||
int reverse = 0;
|
||||
|
||||
ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }",
|
||||
"from", &from,
|
||||
"to", &to,
|
||||
"reverse", &reverse
|
||||
);
|
||||
if (ret != 0)
|
||||
throw ConfigError(json_path, err, "", "Cannot parse switch path config");
|
||||
|
||||
auto masterIpCore = card->lookupIp(from);
|
||||
if (!masterIpCore)
|
||||
throw ConfigError(json_path, "", "Unknown IP {}", from);
|
||||
|
||||
auto slaveIpCore = card->lookupIp(to);
|
||||
if (!slaveIpCore)
|
||||
throw ConfigError(json_path, "", "Unknown IP {}", to);
|
||||
|
||||
auto masterIpNode = std::dynamic_pointer_cast<ip::Node>(masterIpCore);
|
||||
if (!masterIpNode)
|
||||
throw ConfigError(json_path, "", "IP {} is not a streaming node", from);
|
||||
|
||||
auto slaveIpNode = std::dynamic_pointer_cast<ip::Node>(slaveIpCore);
|
||||
if (!slaveIpNode)
|
||||
throw ConfigError(json_path, "", "IP {} is not a streaming node", to);
|
||||
|
||||
if (not masterIpNode->connect(*slaveIpNode, reverse != 0))
|
||||
throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to);
|
||||
}
|
||||
}
|
||||
|
||||
cards.push_back(std::move(card));
|
||||
}
|
||||
|
||||
return cards;
|
||||
}
|
||||
|
||||
PCIeCard::~PCIeCard()
|
||||
{
|
||||
// Ensure IP destructors are called before memory is unmapped
|
||||
ips.clear();
|
||||
|
||||
auto &mm = MemoryManager::get();
|
||||
|
||||
// Unmap all memory blocks
|
||||
for (auto &mappedMemoryBlock : memoryBlocksMapped) {
|
||||
auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, mappedMemoryBlock);
|
||||
|
||||
const uintptr_t iova = translation.getLocalAddr(0);
|
||||
const size_t size = translation.getSize();
|
||||
|
||||
logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}",
|
||||
mappedMemoryBlock, iova, size);
|
||||
vfioContainer->memoryUnmap(iova, size);
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const std::string &name) const
|
||||
{
|
||||
for (auto &ip : ips) {
|
||||
if (*ip == name) {
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const Vlnv &vlnv) const
|
||||
{
|
||||
for (auto &ip : ips) {
|
||||
if (*ip == vlnv) {
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::shared_ptr<ip::Core> PCIeCard::lookupIp(const ip::IpIdentifier &id) const
|
||||
{
|
||||
for (auto &ip : ips) {
|
||||
if (*ip == id) {
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool PCIeCard::unmapMemoryBlock(const MemoryBlock &block)
|
||||
{
|
||||
if (memoryBlocksMapped.find(block.getAddrSpaceId()) == memoryBlocksMapped.end()) {
|
||||
throw std::runtime_error("Block " + std::to_string(block.getAddrSpaceId()) + " is not mapped but was requested to be unmapped.");
|
||||
}
|
||||
|
||||
auto &mm = MemoryManager::get();
|
||||
|
||||
auto translation = mm.getTranslation(addrSpaceIdDeviceToHost, block.getAddrSpaceId());
|
||||
|
||||
const uintptr_t iova = translation.getLocalAddr(0);
|
||||
const size_t size = translation.getSize();
|
||||
|
||||
logger->debug("Unmap block {} at IOVA {:#x} of size {:#x}",
|
||||
block.getAddrSpaceId(), iova, size);
|
||||
vfioContainer->memoryUnmap(iova, size);
|
||||
|
||||
memoryBlocksMapped.erase(block.getAddrSpaceId());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PCIeCard::mapMemoryBlock(const MemoryBlock &block)
|
||||
{
|
||||
if (not vfioContainer->isIommuEnabled()) {
|
||||
logger->warn("VFIO mapping not supported without IOMMU");
|
||||
return false;
|
||||
}
|
||||
|
||||
auto &mm = MemoryManager::get();
|
||||
const auto &addrSpaceId = block.getAddrSpaceId();
|
||||
|
||||
if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end())
|
||||
// Block already mapped
|
||||
return true;
|
||||
else
|
||||
logger->debug("Create VFIO mapping for {}", addrSpaceId);
|
||||
|
||||
auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId);
|
||||
uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0);
|
||||
uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr,
|
||||
UINTPTR_MAX,
|
||||
block.getSize());
|
||||
|
||||
if (iovaAddr == UINTPTR_MAX) {
|
||||
logger->error("Cannot map memory at {:#x} of size {:#x}",
|
||||
processBaseAddr, block.getSize());
|
||||
return false;
|
||||
}
|
||||
|
||||
mm.createMapping(iovaAddr, 0, block.getSize(),
|
||||
"VFIO-D2H",
|
||||
this->addrSpaceIdDeviceToHost,
|
||||
addrSpaceId);
|
||||
|
||||
// Remember that this block has already been mapped for later
|
||||
memoryBlocksMapped.insert(addrSpaceId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PCIeCard::init()
|
||||
{
|
||||
logger = getLogger();
|
||||
|
||||
logger->info("Initializing FPGA card {}", name);
|
||||
|
||||
// Attach PCIe card to VFIO container
|
||||
vfioDevice = vfioContainer->attachDevice(*pdev);
|
||||
|
||||
// Enable memory access and PCI bus mastering for DMA
|
||||
if (not vfioDevice->pciEnable()) {
|
||||
logger->error("Failed to enable PCI device");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reset system?
|
||||
if (doReset) {
|
||||
// Reset / detect PCI device
|
||||
if (not vfioDevice->pciHotReset()) {
|
||||
logger->error("Failed to reset PCI device");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (not reset()) {
|
||||
logger->error("Failed to reset FGPA card");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue