Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

rework fpga node type

Browse source 
The various changes in fpga require a rewrite of the fpga node type.
To allow relative paths for the fpga config file, Config and SuperNode
had to be modified so they store the path of the main config file.
The syntax of the fpga node type configuration has changed - the example
config in etc has been modified accordingly.

Signed-off-by: Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
This commit is contained in:
Niklas Eiling 2023-12-12 14:15:55 +01:00 • committed by Niklas Eiling
parent f09e6e909b
commit 18aa0c8862
8 changed files with 176 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common

@ -1 +1 @@
Subproject commit cd74015a05d51a0a198dae0f33e3b441d0f77acf
Subproject commit 0cb6cd23caacdd504753fd7f32e0a606d1b964f3

41
etc/examples/nodes/fpga.conf
View file

@ -7,44 +7,27 @@ logging = {
fpgas = {
vc707 = {
# Card identification
id = "10ee:7021"
# slot = "88:00.0"
do_reset = true
ips = "/global/projects/villas/fpga/software/etc/vc707-xbar-pcie/vc707-xbar-pcie.json"
paths = (
{
from = "aurora_8b10b_ch2"
to = "aurora_8b10b_ch3"
reverse = true
}
)
interface = "pcie",
id = "10ee:7021",
slot = "0000:88:00.0",
do_reset = true,
ips = "../../../fpga/etc/vc707-xbar-pcie/vc707-xbar-pcie.json",
polling = false,
}
}
nodes = {
dma_0 = {
fpga_0 = {
type = "fpga",
fpga = "vc707"
target = ""
datamover = "dma_0"
use_irqs = false
card = "vc707"
connect = ["0->3", "3->dma", "0<-dma"]
}
}
paths = (
{
in = "dma_0"
hooks = (
{
type = "print"
}
)
in = "fpga_0"
out = "fpga_0"
hooks = ({ type = "print"})
}
)

2
fpga

@ -1 +1 @@
Subproject commit 662b8d626ca963996c1179283f9ff1acfbb14710
Subproject commit ac959229978fa3d433bd982a81c279d35e310245

4
include/villas/config_class.hpp
View file

@ -11,6 +11,7 @@
#include <jansson.h>
#include <unistd.h>
#include <filesystem>
#include <functional>
#include <regex>
@ -32,6 +33,7 @@ protected:
Logger logger;
std::list<std::string> includeDirectories;
std::filesystem::path configPath;
// Check if file exists on local system.
static bool isLocalFile(const std::string &uri) {
@ -87,6 +89,8 @@ public:
json_t *load(const std::string &u, bool resolveIncludes = true,
bool resolveEnvVars = true);
std::filesystem::path &getConfigPath() { return configPath; }
};
} // namespace node

59
include/villas/nodes/fpga.hpp
View file

@ -1,7 +1,9 @@
/* Communicate with VILLASfpga Xilinx FPGA boards.
*
* Author: Steffen Vogel <post@steffenvogel.de>
* Author: Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
* SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
* SPDX-FileCopyrightText: 2023 Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
* SPDX-License-Identifier: Apache-2.0
*/
@ -20,48 +22,42 @@
namespace villas {
namespace node {
#define FPGA_DMA_VLNV
#define FPGA_AURORA_VLNV "acs.eonerc.rwth-aachen.de:user:aurora_axis:"
class FpgaNode : public Node {
enum InterfaceType { PCIE, PLATFORM };
protected:
int irqFd;
int coalesce;
bool polling;
std::shared_ptr<fpga::PCIeCard> card;
std::shared_ptr<fpga::ip::Dma> dma;
std::shared_ptr<fpga::ip::Node> intf;
std::unique_ptr<const MemoryBlock> blockRx;
std::unique_ptr<const MemoryBlock> blockTx;
// Config only
// Settings
std::string cardName;
std::string intfName;
std::string dmaName;
std::list<std::string> connectStrings;
protected:
virtual int _read(Sample *smps[], unsigned cnt);
// State
std::shared_ptr<fpga::Card> card;
std::shared_ptr<villas::fpga::ip::Dma> dma;
std::shared_ptr<villas::MemoryBlock> blockRx[2];
std::shared_ptr<villas::MemoryBlock> blockTx;
virtual int _write(Sample *smps[], unsigned cnt);
// Non-public methods
virtual int _read(Sample *smps[], unsigned cnt) override;
virtual int _write(Sample *smps[], unsigned cnt) override;
public:
FpgaNode(const uuid_t &id = {}, const std::string &name = "");
virtual ~FpgaNode();
virtual int parse(json_t *json);
virtual int prepare() override;
virtual const std::string &getDetails();
virtual int parse(json_t *json) override;
virtual int check();
virtual int check() override;
virtual int prepare();
virtual int start() override;
virtual std::vector<int> getPollFDs();
virtual std::vector<int> getPollFDs() override;
virtual const std::string &getDetails() override;
};
class FpgaNodeFactory : public NodeFactory {
@ -69,7 +65,8 @@ class FpgaNodeFactory : public NodeFactory {
public:
using NodeFactory::NodeFactory;
virtual Node *make(const uuid_t &id = {}, const std::string &nme = "") {
virtual Node *make(const uuid_t &id = {},
const std::string &nme = "") override {
auto *n = new FpgaNode(id, nme);
init(n);
@ -77,17 +74,17 @@ public:
return n;
}
virtual int getFlags() const {
virtual int getFlags() const override {
return (int)NodeFactory::Flags::SUPPORTS_READ |
(int)NodeFactory::Flags::SUPPORTS_WRITE |
(int)NodeFactory::Flags::SUPPORTS_POLL;
}
virtual std::string getName() const { return "fpga"; }
virtual std::string getName() const override { return "fpga"; }
virtual std::string getDescription() const { return "VILLASfpga"; }
virtual std::string getDescription() const override { return "VILLASfpga"; }
virtual int start(SuperNode *sn);
virtual int start(SuperNode *sn) override;
};
} // namespace node

2
include/villas/super_node.hpp
View file

@ -141,6 +141,8 @@ public:
json_t *getConfig() { return config.root; }
std::filesystem::path &getConfigPath() { return config.getConfigPath(); }
std::string getConfigUri() const { return uri; }
int getAffinity() const { return affinity; }

3
lib/config.cpp
View file

@ -72,13 +72,14 @@ json_t *Config::load(const std::string &u, bool resolveInc,
FILE *Config::loadFromStdio() {
logger->info("Reading configuration from standard input");
configPath = std::filesystem::current_path();
return stdin;
}
FILE *Config::loadFromLocalFile(const std::string &u) {
logger->info("Reading configuration from local file: {}", u);
configPath = u;
FILE *f = fopen(u.c_str(), "r");
if (!f)
throw RuntimeError("Failed to open configuration from: {}", u);

261
lib/nodes/fpga.cpp
View file

@ -1,12 +1,10 @@
/* Communicate with VILLASfpga Xilinx FPGA boards.
*
* Author: Steffen Vogel <post@steffenvogel.de>
* SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
* Author: Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
* SPDX-FileCopyrightText: 2023 Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
* SPDX-License-Identifier: Apache-2.0
*/
#include <algorithm>
#include <csignal>
#include <iostream>
#include <memory>
#include <string>
@ -16,16 +14,12 @@
#include <villas/exceptions.hpp>
#include <villas/log.hpp>
#include <villas/node_compat.hpp>
#include <villas/nodes/fpga.hpp>
#include <villas/sample.hpp>
#include <villas/super_node.hpp>
#include <villas/utils.hpp>
#include <villas/fpga/card.hpp>
#include <villas/fpga/core.hpp>
#include <villas/fpga/ips/dma.hpp>
#include <villas/fpga/vlnv.hpp>
#include <villas/fpga/utils.hpp>
using namespace villas;
using namespace villas::node;
@ -33,20 +27,69 @@ using namespace villas::fpga;
using namespace villas::utils;
// Global state
static std::list<std::shared_ptr<fpga::PCIeCard>> cards;
static std::map<fpga::ip::Dma, FpgaNode *> dmaMap;
static std::list<std::shared_ptr<fpga::Card>> cards;
static std::shared_ptr<kernel::pci::DeviceList> pciDevices;
static std::shared_ptr<kernel::vfio::Container> vfioContainer;
using namespace villas;
using namespace villas::node;
FpgaNode::FpgaNode(const uuid_t &id, const std::string &name)
: Node(id, name), irqFd(-1), coalesce(0), polling(true) {}
: Node(id, name), cardName(""), card(), dma(), blockRx(), blockTx() {}
FpgaNode::~FpgaNode() {}
int FpgaNode::prepare() {
for (auto &fpgaCard : cards) {
if (fpgaCard->name == cardName) {
card = fpgaCard;
break;
}
}
if (card == nullptr) {
throw ConfigError(config, "node-config-fpga",
"There is no FPGA card with the name: {}", cardName);
}
std::vector<std::shared_ptr<fpga::ip::AuroraXilinx>> aurora_channels;
for (int i = 0; i < 4; i++) {
auto name = fmt::format("aurora_8b10b_ch{}", i);
auto id = fpga::ip::IpIdentifier("xilinx.com:ip:aurora_8b10b:", name);
auto aurora =
std::dynamic_pointer_cast<fpga::ip::AuroraXilinx>(card->lookupIp(id));
if (aurora == nullptr) {
logger->error("No Aurora interface found on FPGA");
return 1;
}
aurora_channels.push_back(aurora);
}
dma = std::dynamic_pointer_cast<fpga::ip::Dma>(
card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:")));
if (dma == nullptr) {
logger->error("No DMA found on FPGA ");
return 1;
}
// Configure Crossbar switch
for (std::string str : connectStrings) {
const fpga::ConnectString parsedConnectString(str);
parsedConnectString.configCrossBar(dma, aurora_channels);
}
auto &alloc = HostRam::getAllocator();
blockRx[0] = alloc.allocateBlock(0x200 * sizeof(float));
blockRx[1] = alloc.allocateBlock(0x200 * sizeof(float));
blockTx = alloc.allocateBlock(0x200 * sizeof(float));
villas::MemoryAccessor<float> memRx[] = {*(blockRx[0]), *(blockRx[1])};
villas::MemoryAccessor<float> memTx = *blockTx;
dma->makeAccesibleFromVA(blockRx[0]);
dma->makeAccesibleFromVA(blockRx[1]);
dma->makeAccesibleFromVA(blockTx);
return Node::prepare();
}
int FpgaNode::parse(json_t *json) {
int ret = Node::parse(json);
if (ret)
@ -54,29 +97,28 @@ int FpgaNode::parse(json_t *json) {
json_error_t err;
const char *card = nullptr;
const char *intf = nullptr;
const char *dma = nullptr;
int poll = polling;
const char *jsonCardName = nullptr;
json_t *jsonConnectStrings = nullptr;
ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }",
"card", &card, "interface", &intf, "dma", &dma,
"coalesce", &coalesce, "polling", &polling);
if (ret)
ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: o}", "card", &jsonCardName,
"connect", &jsonConnectStrings);
if (ret) {
throw ConfigError(json, err, "node-config-fpga",
"Failed to parse configuration of node {}",
this->getName());
if (card)
cardName = card;
if (intf)
intfName = intf;
if (dma)
dmaName = dma;
polling = poll; // cast int to bool
}
cardName = std::string(jsonCardName);
if (jsonConnectStrings != nullptr && json_is_array(jsonConnectStrings)) {
for (size_t i = 0; i < json_array_size(jsonConnectStrings); i++) {
json_t *jsonConnectString = json_array_get(jsonConnectStrings, i);
if (jsonConnectString == nullptr || !json_is_string(jsonConnectString)) {
throw ConfigError(jsonConnectString, "node-config-fpga",
"Failed to parse connect string");
}
connectStrings.push_back(
std::string(json_string_value(jsonConnectString)));
}
}
return 0;
}
@ -85,9 +127,13 @@ const std::string &FpgaNode::getDetails() {
if (details.empty()) {
auto &name = card ? card->name : cardName;
details =
fmt::format("fpga={}, dma={}, if={}, polling={}, coalesce={}", name,
dma->getInstanceName(), polling ? "yes" : "no", coalesce);
const char *const delim = ", ";
std::ostringstream imploded;
std::copy(connectStrings.begin(), connectStrings.end(),
std::ostream_iterator<std::string>(imploded, delim));
details = fmt::format("fpga={}, connect={}", name, imploded.str());
}
return details;
@ -95,136 +141,81 @@ const std::string &FpgaNode::getDetails() {
int FpgaNode::check() { return 0; }
int FpgaNode::prepare() {
auto it = cardName.empty()
? cards.begin()
: std::find_if(cards.begin(), cards.end(),
[this](std::shared_ptr<fpga::PCIeCard> c) {
return c->name == cardName;
});
if (it == cards.end())
throw ConfigError(json_object_get(config, "fpga"), "node-config-fpga-card",
"Invalid FPGA card name: {}", cardName);
card = *it;
auto intfCore = intfName.empty()
? card->lookupIp(fpga::Vlnv(FPGA_AURORA_VLNV))
: card->lookupIp(intfName);
if (!intfCore)
throw ConfigError(config, "node-config-fpga-interface",
"There is no interface IP with the name: {}", intfName);
intf = std::dynamic_pointer_cast<fpga::ip::Node>(intfCore);
if (!intf)
throw RuntimeError("The IP {} is not a interface", *intfCore);
auto dmaCore = dmaName.empty() ? card->lookupIp(fpga::Vlnv(FPGA_DMA_VLNV))
: card->lookupIp(dmaName);
if (!dmaCore)
throw ConfigError(config, "node-config-fpga-dma",
"There is no DMA IP with the name: {}", dmaName);
dma = std::dynamic_pointer_cast<fpga::ip::Dma>(dmaCore);
if (!dma)
throw RuntimeError("The IP {} is not a DMA controller", *dmaCore);
int ret = intf->connect(*(dma), true);
if (ret)
throw RuntimeError("Failed to connect: {} -> {}", *(intf), *(dma));
auto &alloc = HostDmaRam::getAllocator();
const std::shared_ptr<villas::MemoryBlock> blockRx =
alloc.allocateBlock(0x100 * sizeof(float));
const std::shared_ptr<villas::MemoryBlock> blockTx =
alloc.allocateBlock(0x100 * sizeof(float));
villas::MemoryAccessor<float> memRx = *blockRx;
villas::MemoryAccessor<float> memTx = *blockTx;
dma->makeAccesibleFromVA(blockRx);
dma->makeAccesibleFromVA(blockTx);
// Show some debugging infos
auto &mm = MemoryManager::get();
dma->dump();
intf->dump();
mm.getGraph().dump();
return Node::prepare();
int FpgaNode::start() {
// enque first read
dma->read(*(blockRx[0]), blockRx[0]->getSize());
return Node::start();
}
int FpgaNode::_read(Sample *smps[], unsigned cnt) {
static size_t cur = 0, next = 1;
unsigned read;
Sample *smp = smps[0];
assert(cnt == 1);
dma->read(*blockRx, blockRx->getSize()); // TODO: calc size
const size_t bytesRead = dma->readComplete().bytes;
read = bytesRead / sizeof(int32_t);
dma->read(*(blockRx[next]), blockRx[next]->getSize()); // TODO: calc size
auto c = dma->readComplete();
read = c.bytes / sizeof(float);
auto mem = MemoryAccessor<uint32_t>(*blockRx);
if (c.interrupts > 1) {
logger->warn("Missed {} interrupts", c.interrupts - 1);
}
for (unsigned i = 0; i < MIN(read, smp->capacity); i++)
smp->data[i].i = mem[i];
auto mem = MemoryAccessor<float>(*(blockRx[cur]));
smp->length = 0;
for (unsigned i = 0; i < MIN(read, smp->capacity); i++) {
smp->data[i].f = static_cast<double>(mem[i]);
smp->length++;
}
smp->flags = (int)SampleFlags::HAS_DATA;
smp->signals = in.signals;
cur = next;
next = (next + 1) % (sizeof(blockRx) / sizeof(blockRx[0]));
return read;
return 1;
}
int FpgaNode::_write(Sample *smps[], unsigned cnt) {
int written;
unsigned int written;
Sample *smp = smps[0];
assert(cnt == 1);
assert(cnt == 1 && smps != nullptr && smps[0] != nullptr);
auto mem = MemoryAccessor<uint32_t>(*blockTx);
auto mem = MemoryAccessor<float>(*blockTx);
for (unsigned i = 0; i < smps[0]->length; i++)
mem[i] = smps[0]->data[i].i;
for (unsigned i = 0; i < smps[0]->length; i++) {
mem[i] = static_cast<float>(smps[0]->data[i].f);
}
bool state = dma->write(*blockTx, smp->length * sizeof(int32_t));
bool state = dma->write(*blockTx, smp->length * sizeof(float));
if (!state)
return -1;
written = 0; // The number of samples written
written = dma->writeComplete().bytes /
sizeof(float); // The number of samples written
return written;
if (written != smp->length) {
logger->warn("Wrote {} samples, but {} were expected", written,
smp->length);
}
return 1;
}
std::vector<int> FpgaNode::getPollFDs() {
std::vector<int> fds;
if (!polling)
fds.push_back(irqFd);
return fds;
return card->vfioDevice->getEventfdList();
}
int FpgaNodeFactory::start(SuperNode *sn) {
vfioContainer = std::make_shared<kernel::vfio::Container>();
pciDevices = std::make_shared<kernel::pci::DeviceList>();
// Get the FPGA card plugin
auto pcieCardPlugin = plugin::registry->lookup<fpga::PCIeCardFactory>("pcie");
if (!pcieCardPlugin)
throw RuntimeError("No FPGA PCIe plugin found");
json_t *json_cfg = sn->getConfig();
json_t *json_fpgas = json_object_get(json_cfg, "fpgas");
if (!json_fpgas)
throw ConfigError(json_cfg, "node-config-fpgas",
"No section 'fpgas' found in config");
// Create all FPGA card instances using the corresponding plugin
auto piceCards =
fpga::PCIeCardFactory::make(json_fpgas, pciDevices, vfioContainer);
cards.splice(cards.end(), piceCards);
if (cards.empty()) {
std::filesystem::path searchPath = sn->getConfigPath().parent_path();
createCards(sn->getConfig(), cards, searchPath);
}
return NodeFactory::start(sn);
}