Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

make FPGA device interface agnostic

Browse source 
remove explicit mentioning of PCIe in the use of Device as a preparation
for integrating platform devices. auto formatted some files.

Signed-off-by: Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
This commit is contained in:
Niklas Eiling 2023-12-12 10:59:51 +01:00 • committed by Niklas Eiling
parent 7ce1ee5a6c
commit 654ee84e9e
8 changed files with 228 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
fpga/common

@ -1 +1 @@
Subproject commit 7bef5e02015805a5f8988be3da78f4df844a7bff
Subproject commit 0cb6cd23caacdd504753fd7f32e0a606d1b964f3

3
fpga/etc/fpgas.json
View file

@ -5,7 +5,8 @@
"slot": "0000:88:00.0",
"do_reset": true,
"ips": "vc707-xbar-pcie/vc707-xbar-pcie.json",
"polling": false
"polling": false,
"interface": "pcie"
}
}
}

1
fpga/include/villas/fpga/card.hpp
View file

@ -23,6 +23,7 @@ class Card
public:
bool polling;
std::string name; // The name of the FPGA card
std::shared_ptr<kernel::vfio::Container> vfioContainer;
std::shared_ptr<kernel::vfio::Device> vfioDevice;

55
fpga/include/villas/fpga/pcie_card.hpp
View file

@ -64,55 +64,32 @@ public: // TODO: make this private
bool doReset; // Reset VILLASfpga during startup?
int affinity; // Affinity for MSI interrupts
std::string name; // The name of the FPGA card
std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
protected:
Logger
getLogger() const
{
return villas::logging.get(name);
}
protected:
Logger getLogger() const { return villas::logging.get(name); }
};
class PCIeCardFactory : public plugin::Plugin {
public:
static std::shared_ptr<PCIeCard>
make(json_t *json, std::string card_name,
std::shared_ptr<kernel::vfio::Container> vc,
const std::filesystem::path &searchPath);
static std::list<std::shared_ptr<PCIeCard>> make(json_t *json,
std::shared_ptr<kernel::pci::DeviceList> pci,
std::shared_ptr<kernel::vfio::Container> vc,
const std::filesystem::path& searchPath = std::filesystem::path());
static PCIeCard *make() { return new PCIeCard(); }
static
PCIeCard* make()
{
return new PCIeCard();
}
static Logger getStaticLogger() {
return villas::logging.get("pcie:card:factory");
}
static Logger
getStaticLogger()
{
return villas::logging.get("pcie:card:factory");
}
virtual std::string getName() const { return "pcie"; }
virtual std::string
getName() const
{
return "pcie";
}
virtual std::string getDescription() const {
return "Xilinx PCIe FPGA cards";
}
virtual std::string
getDescription() const
{
return "Xilinx PCIe FPGA cards";
}
virtual
std::string getType() const
{
return "card";
}
virtual std::string getType() const { return "card"; }
};
} // namespace fpga

5
fpga/include/villas/fpga/utils.hpp
View file

@ -13,9 +13,12 @@
namespace villas {
namespace fpga {
std::shared_ptr<fpga::PCIeCard>
std::shared_ptr<fpga::Card>
setupFpgaCard(const std::string &configFile, const std::string &fpgaName);
int createCards(json_t *config, std::list<std::shared_ptr<fpga::Card>> &cards,
std::filesystem::path &searchPath);
void setupColorHandling();
class ConnectString {

4
fpga/lib/ips/dma.cpp
View file

@ -284,8 +284,8 @@ bool Dma::writeScatterGather(const void *buf, size_t len)
ret = XAxiDma_BdRingAlloc(txRing, 1, &bd);
if (ret != XST_SUCCESS) {
hwLock.unlock();
throw RuntimeError("BdRingAlloc returned {}.", ret);
}
throw RuntimeError("Write: BdRingAlloc returned {}.", ret);
}
ret = XAxiDma_BdSetBufAddr(bd, (uintptr_t) buf);
if (ret != XST_SUCCESS) {

220
fpga/lib/pcie_card.cpp
View file

@ -25,140 +25,134 @@ static PCIeCardFactory PCIeCardFactoryInstance;
static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA)));
std::list<std::shared_ptr<PCIeCard>> PCIeCardFactory::make(json_t *json,
std::shared_ptr<kernel::pci::DeviceList> pci,
std::shared_ptr<kernel::vfio::Container> vc,
const std::filesystem::path& searchPath)
{
std::list<std::shared_ptr<PCIeCard>> cards;
auto logger = getStaticLogger();
std::shared_ptr<PCIeCard>
PCIeCardFactory::make(json_t *json_card, std::string card_name,
std::shared_ptr<kernel::vfio::Container> vc,
const std::filesystem::path &searchPath) {
auto logger = getStaticLogger();
const char *card_name;
json_t *json_card;
json_object_foreach(json, card_name, json_card) {
logger->info("Found config for FPGA card {}", card_name);
json_t *json_ips = nullptr;
json_t *json_paths = nullptr;
const char *pci_slot = nullptr;
const char *pci_id = nullptr;
int do_reset = 0;
int affinity = 0;
int polling = 0;
json_t* json_ips = nullptr;
json_t* json_paths = nullptr;
const char* pci_slot = nullptr;
const char* pci_id = nullptr;
int do_reset = 0;
int affinity = 0;
int polling = 0;
json_error_t err;
int ret = json_unpack_ex(
json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }",
"ips", &json_ips, "affinity", &affinity, "do_reset", &do_reset, "slot",
&pci_slot, "id", &pci_id, "polling", &polling, "paths", &json_paths);
json_error_t err;
int ret = json_unpack_ex(json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }",
"ips", &json_ips,
"affinity", &affinity,
"do_reset", &do_reset,
"slot", &pci_slot,
"id", &pci_id,
"polling", &polling,
"paths", &json_paths);
if (ret != 0)
throw ConfigError(json_card, err, "", "Failed to parse card");
if (ret != 0)
throw ConfigError(json_card, err, "", "Failed to parse card");
auto card = std::unique_ptr<PCIeCard>(make());
auto card = std::unique_ptr<PCIeCard>(make());
// Populate generic properties
card->name = std::string(card_name);
card->vfioContainer = vc;
card->affinity = affinity;
card->doReset = do_reset != 0;
card->polling = (polling != 0);
// Populate generic properties
card->name = std::string(card_name);
card->vfioContainer = vc;
card->affinity = affinity;
card->doReset = do_reset != 0;
card->polling = (polling != 0);
kernel::pci::Device filter = defaultFilter;
kernel::pci::Device filter = defaultFilter;
if (pci_id)
filter.id = kernel::pci::Id(pci_id);
if (pci_slot)
filter.slot = kernel::pci::Slot(pci_slot);
if (pci_id)
filter.id = kernel::pci::Id(pci_id);
if (pci_slot)
filter.slot = kernel::pci::Slot(pci_slot);
// Search for FPGA card
card->pdev = kernel::pci::DeviceList::getInstance()->lookupDevice(filter);
if (!card->pdev) {
logger->warn("Failed to find PCI device");
return nullptr;
}
// Search for FPGA card
card->pdev = pci->lookupDevice(filter);
if (!card->pdev) {
logger->warn("Failed to find PCI device");
continue;
}
if (not card->init()) {
logger->warn("Cannot start FPGA card {}", card_name);
return nullptr;
}
if (not card->init()) {
logger->warn("Cannot start FPGA card {}", card_name);
continue;
}
// Load IPs from a separate json file
if (!json_is_string(json_ips)) {
logger->debug("FPGA IP cores config item is not a string.");
throw ConfigError(json_ips, "node-config-fpga-ips",
"FPGA IP cores config item is not a string.");
}
if (!searchPath.empty()) {
std::filesystem::path json_ips_path =
searchPath / json_string_value(json_ips);
logger->debug("searching for FPGA IP cors config at {}",
json_ips_path.string());
json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr);
}
if (json_ips == nullptr) {
json_ips = json_load_file(json_string_value(json_ips), 0, nullptr);
logger->debug("searching for FPGA IP cors config at {}",
json_string_value(json_ips));
if (json_ips == nullptr) {
throw ConfigError(json_ips, "node-config-fpga-ips",
"Failed to find FPGA IP cores config");
}
}
// Load IPs from a separate json file
if (!json_is_string(json_ips)) {
logger->debug("FPGA IP cores config item is not a string.");
throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP cores config item is not a string.");
}
if (!searchPath.empty()) {
std::filesystem::path json_ips_path = searchPath / json_string_value(json_ips);
logger->debug("searching for FPGA IP cors config at {}", json_ips_path.string());
json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr);
}
if (json_ips == nullptr) {
json_ips = json_load_file(json_string_value(json_ips), 0, nullptr);
logger->debug("searching for FPGA IP cors config at {}", json_string_value(json_ips));
if (json_ips == nullptr) {
throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to find FPGA IP cores config");
}
}
if (not json_is_object(json_ips))
throw ConfigError(json_ips, "node-config-fpga-ips",
"FPGA IP core list must be an object!");
if (not json_is_object(json_ips))
throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!");
card->ips = ip::CoreFactory::make(card.get(), json_ips);
if (card->ips.empty())
throw ConfigError(json_ips, "node-config-fpga-ips",
"Cannot initialize IPs of FPGA card {}", card_name);
card->ips = ip::CoreFactory::make(card.get(), json_ips);
if (card->ips.empty())
throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name);
if (not card->check())
throw RuntimeError("Checking of FPGA card {} failed", card_name);
if (not card->check())
throw RuntimeError("Checking of FPGA card {} failed", card_name);
// Additional static paths for AXI-Steram switch
if (json_paths != nullptr) {
if (not json_is_array(json_paths))
throw ConfigError(json_paths, err, "",
"Switch path configuration must be an array");
// Additional static paths for AXI-Steram switch
if (json_paths != nullptr) {
if (not json_is_array(json_paths))
throw ConfigError(json_paths, err, "", "Switch path configuration must be an array");
size_t i;
json_t *json_path;
json_array_foreach(json_paths, i, json_path) {
const char *from, *to;
int reverse = 0;
size_t i;
json_t *json_path;
json_array_foreach(json_paths, i, json_path) {
const char *from, *to;
int reverse = 0;
ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }", "from",
&from, "to", &to, "reverse", &reverse);
if (ret != 0)
throw ConfigError(json_path, err, "",
"Cannot parse switch path config");
ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }",
"from", &from,
"to", &to,
"reverse", &reverse
);
if (ret != 0)
throw ConfigError(json_path, err, "", "Cannot parse switch path config");
auto masterIpCore = card->lookupIp(from);
if (!masterIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", from);
auto masterIpCore = card->lookupIp(from);
if (!masterIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", from);
auto slaveIpCore = card->lookupIp(to);
if (!slaveIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", to);
auto slaveIpCore = card->lookupIp(to);
if (!slaveIpCore)
throw ConfigError(json_path, "", "Unknown IP {}", to);
auto masterIpNode = std::dynamic_pointer_cast<ip::Node>(masterIpCore);
if (!masterIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", from);
auto masterIpNode = std::dynamic_pointer_cast<ip::Node>(masterIpCore);
if (!masterIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", from);
auto slaveIpNode = std::dynamic_pointer_cast<ip::Node>(slaveIpCore);
if (!slaveIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", to);
auto slaveIpNode = std::dynamic_pointer_cast<ip::Node>(slaveIpCore);
if (!slaveIpNode)
throw ConfigError(json_path, "", "IP {} is not a streaming node", to);
if (not masterIpNode->connect(*slaveIpNode, reverse != 0))
throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to);
}
}
cards.push_back(std::move(card));
}
return cards;
if (not masterIpNode->connect(*slaveIpNode, reverse != 0))
throw ConfigError(json_path, "", "Failed to connect node {} to {}",
from, to);
}
}
// Deallocate JSON config
json_decref(json_ips);
return card;
}
PCIeCard::~PCIeCard()

160
fpga/lib/utils.cpp
View file

@ -33,7 +33,6 @@
using namespace villas;
static std::shared_ptr<kernel::pci::DeviceList> pciDevices;
static auto logger = villas::logging.get("streamer");
fpga::ConnectString::ConnectString(std::string& connectString, int maxPortNum) :
@ -64,24 +63,26 @@ void fpga::ConnectString::parseString(std::string& connectString)
return;
}
static const std::regex regex("([0-9]+)([<\\->]+)([0-9]+|stdin|stdout|pipe)");
std::smatch match;
static const std::regex regex(
"([0-9]+)([<\\->]+)([0-9]+|stdin|stdout|pipe|dma)");
std::smatch match;
if (!std::regex_match(connectString, match, regex) || match.size() != 4) {
logger->error("Invalid connect string: {}", connectString);
throw std::runtime_error("Invalid connect string");
}
if (!std::regex_match(connectString, match, regex) ||
match.size() != 4) {
logger->error("Invalid connect string: {}", connectString);
throw std::runtime_error("Invalid connect string");
}
if (match[2] == "<->") {
bidirectional = true;
} else if(match[2] == "<-") {
invert = true;
}
if (match[2] == "<->") {
bidirectional = true;
} else if (match[2] == "<-") {
invert = true;
}
std::string srcStr = (invert ? match[3] : match[1]);
std::string dstStr = (invert ? match[1] : match[3]);
std::string srcStr = (invert ? match[3] : match[1]);
std::string dstStr = (invert ? match[1] : match[3]);
srcAsInt = portStringToInt(srcStr);
srcAsInt = portStringToInt(srcStr);
dstAsInt = portStringToInt(dstStr);
if (srcAsInt == -1) {
srcIsStdin = true;
@ -95,16 +96,16 @@ void fpga::ConnectString::parseString(std::string& connectString)
int fpga::ConnectString::portStringToInt(std::string &str) const
{
if (str == "stdin" || str == "stdout" || str == "pipe") {
return -1;
} else {
const int port = std::stoi(str);
if (str == "stdin" || str == "stdout" || str == "pipe" || str == "dma") {
return -1;
} else {
const int port = std::stoi(str);
if (port > maxPortNum || port < 0)
throw std::runtime_error("Invalid port number");
if (port > maxPortNum || port < 0)
throw std::runtime_error("Invalid port number");
return port;
}
return port;
}
}
@ -163,58 +164,87 @@ void fpga::setupColorHandling()
});
}
std::shared_ptr<fpga::PCIeCard>
fpga::setupFpgaCard(const std::string &configFile, const std::string &fpgaName)
{
pciDevices = std::make_shared<kernel::pci::DeviceList>();
int fpga::createCards(json_t *config,
std::list<std::shared_ptr<fpga::Card>> &cards,
std::filesystem::path &searchPath) {
int numFpgas = 0;
auto vfioContainer = std::make_shared<kernel::vfio::Container>();
auto configDir = std::filesystem::path().parent_path();
auto vfioContainer = std::make_shared<kernel::vfio::Container>();
auto configDir = std::filesystem::path(configFile).parent_path();
json_t *fpgas = json_object_get(config, "fpgas");
if (fpgas == nullptr) {
logger->error("No section 'fpgas' found in config");
exit(1);
}
// Parse FPGA configuration
FILE* f = fopen(configFile.c_str(), "r");
if (!f)
throw RuntimeError("Cannot open config file: {}", configFile);
const char *card_name;
json_t *json_card;
json_object_foreach(fpgas, card_name, json_card) {
const char *interfaceName;
json_error_t err;
logger->info("Found config for FPGA card {}", card_name);
int ret = json_unpack_ex(json_card, &err, 0, "{s: s}", "interface",
&interfaceName);
if (ret) {
throw ConfigError(json_card, err, "interface",
"Failed to parse interface name for card {}",
card_name);
}
std::string interfaceNameStr(interfaceName);
if (interfaceNameStr == "pcie") {
auto card = fpga::PCIeCardFactory::make(json_card, std::string(card_name),
vfioContainer, searchPath);
if (card) {
cards.push_back(std::move(card));
numFpgas++;
}
} else if (interfaceNameStr == "platform") {
throw RuntimeError("Platform interface not implemented yet");
} else {
throw RuntimeError("Unknown interface type {}", interfaceNameStr);
}
}
return numFpgas;
}
json_t* json = json_loadf(f, 0, nullptr);
if (!json) {
logger->error("Cannot parse JSON config");
fclose(f);
throw RuntimeError("Cannot parse JSON config");
}
std::shared_ptr<fpga::Card> fpga::setupFpgaCard(const std::string &configFile,
const std::string &fpgaName) {
auto vfioContainer = std::make_shared<kernel::vfio::Container>();
auto configDir = std::filesystem::path(configFile).parent_path();
fclose(f);
// Parse FPGA configuration
FILE *f = fopen(configFile.c_str(), "r");
if (!f)
throw RuntimeError("Cannot open config file: {}", configFile);
json_t* fpgas = json_object_get(json, "fpgas");
if (fpgas == nullptr) {
logger->error("No section 'fpgas' found in config");
exit(1);
}
json_t *json = json_loadf(f, 0, nullptr);
if (!json) {
logger->error("Cannot parse JSON config");
fclose(f);
throw RuntimeError("Cannot parse JSON config");
}
// Get the FPGA card plugin
auto fpgaCardFactory = plugin::registry->lookup<fpga::PCIeCardFactory>("pcie");
if (fpgaCardFactory == nullptr) {
logger->error("No FPGA plugin found");
exit(1);
}
fclose(f);
// Create all FPGA card instances using the corresponding plugin
auto cards = fpgaCardFactory->make(fpgas, pciDevices, vfioContainer, configDir);
// Create all FPGA card instances using the corresponding plugin
auto cards = std::list<std::shared_ptr<fpga::Card>>();
createCards(json, cards, configDir);
std::shared_ptr<fpga::PCIeCard> card;
for (auto &fpgaCard : cards) {
if (fpgaCard->name == fpgaName) {
return fpgaCard;
}
}
std::shared_ptr<fpga::Card> card = nullptr;
for (auto &fpgaCard : cards) {
if (fpgaCard->name == fpgaName) {
card = fpgaCard;
break;
}
}
// Deallocate JSON config
json_decref(json);
// Deallocate JSON config
json_decref(json);
if (!card)
throw RuntimeError("FPGA card {} not found in config or not working",
fpgaName);
if (!card)
throw RuntimeError("FPGA card {} not found in config or not working", fpgaName);
return card;
return card;
}
std::unique_ptr<fpga::BufferedSampleFormatter> fpga::getBufferedSampleFormatter(