From 18aa0c8862b60b7f9f1552d14e9733c6e18a429c Mon Sep 17 00:00:00 2001 From: Niklas Eiling Date: Tue, 12 Dec 2023 14:15:55 +0100 Subject: [PATCH] rework fpga node type The various changes in fpga require a rewrite of the fpga node type. To allow relative paths for the fpga config file, Config and SuperNode had to be modified so they store the path of the main config file. The syntax of the fpga node type configuration has changed - the example config in etc has been modified accordingly. Signed-off-by: Niklas Eiling --- common | 2 +- etc/examples/nodes/fpga.conf | 41 ++--- fpga | 2 +- include/villas/config_class.hpp | 4 + include/villas/nodes/fpga.hpp | 59 ++++---- include/villas/super_node.hpp | 2 + lib/config.cpp | 3 +- lib/nodes/fpga.cpp | 261 +++++++++++++++----------------- 8 files changed, 176 insertions(+), 198 deletions(-) diff --git a/common b/common index cd74015a0..0cb6cd23c 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit cd74015a05d51a0a198dae0f33e3b441d0f77acf +Subproject commit 0cb6cd23caacdd504753fd7f32e0a606d1b964f3 diff --git a/etc/examples/nodes/fpga.conf b/etc/examples/nodes/fpga.conf index 56a1e1ac3..e7ba7b1a3 100644 --- a/etc/examples/nodes/fpga.conf +++ b/etc/examples/nodes/fpga.conf @@ -7,44 +7,27 @@ logging = { fpgas = { vc707 = { - # Card identification - id = "10ee:7021" - # slot = "88:00.0" - - do_reset = true - - ips = "/global/projects/villas/fpga/software/etc/vc707-xbar-pcie/vc707-xbar-pcie.json" - - paths = ( - { - from = "aurora_8b10b_ch2" - to = "aurora_8b10b_ch3" - - reverse = true - } - ) + interface = "pcie", + id = "10ee:7021", + slot = "0000:88:00.0", + do_reset = true, + ips = "../../../fpga/etc/vc707-xbar-pcie/vc707-xbar-pcie.json", + polling = false, } } nodes = { - dma_0 = { + fpga_0 = { type = "fpga", - fpga = "vc707" - target = "" - - datamover = "dma_0" - use_irqs = false + card = "vc707" + connect = ["0->3", "3->dma", "0<-dma"] } } paths = ( { - in = "dma_0" - - hooks = ( - { - type = "print" - } - ) + in = "fpga_0" + out = "fpga_0" + hooks = ({ type = "print"}) } ) diff --git a/fpga b/fpga index 662b8d626..ac9592299 160000 --- a/fpga +++ b/fpga @@ -1 +1 @@ -Subproject commit 662b8d626ca963996c1179283f9ff1acfbb14710 +Subproject commit ac959229978fa3d433bd982a81c279d35e310245 diff --git a/include/villas/config_class.hpp b/include/villas/config_class.hpp index 60acdc251..a499fcab7 100644 --- a/include/villas/config_class.hpp +++ b/include/villas/config_class.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -32,6 +33,7 @@ protected: Logger logger; std::list includeDirectories; + std::filesystem::path configPath; // Check if file exists on local system. static bool isLocalFile(const std::string &uri) { @@ -87,6 +89,8 @@ public: json_t *load(const std::string &u, bool resolveIncludes = true, bool resolveEnvVars = true); + + std::filesystem::path &getConfigPath() { return configPath; } }; } // namespace node diff --git a/include/villas/nodes/fpga.hpp b/include/villas/nodes/fpga.hpp index 7a25dbf7f..da9bdebd2 100644 --- a/include/villas/nodes/fpga.hpp +++ b/include/villas/nodes/fpga.hpp @@ -1,7 +1,9 @@ /* Communicate with VILLASfpga Xilinx FPGA boards. * * Author: Steffen Vogel + * Author: Niklas Eiling * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-FileCopyrightText: 2023 Niklas Eiling * SPDX-License-Identifier: Apache-2.0 */ @@ -20,48 +22,42 @@ namespace villas { namespace node { -#define FPGA_DMA_VLNV -#define FPGA_AURORA_VLNV "acs.eonerc.rwth-aachen.de:user:aurora_axis:" - class FpgaNode : public Node { + enum InterfaceType { PCIE, PLATFORM }; + protected: - int irqFd; - int coalesce; - bool polling; - - std::shared_ptr card; - - std::shared_ptr dma; - std::shared_ptr intf; - - std::unique_ptr blockRx; - std::unique_ptr blockTx; - - // Config only + // Settings std::string cardName; - std::string intfName; - std::string dmaName; + std::list connectStrings; -protected: - virtual int _read(Sample *smps[], unsigned cnt); + // State + std::shared_ptr card; + std::shared_ptr dma; + std::shared_ptr blockRx[2]; + std::shared_ptr blockTx; - virtual int _write(Sample *smps[], unsigned cnt); + // Non-public methods + virtual int _read(Sample *smps[], unsigned cnt) override; + + virtual int _write(Sample *smps[], unsigned cnt) override; public: FpgaNode(const uuid_t &id = {}, const std::string &name = ""); virtual ~FpgaNode(); - virtual int parse(json_t *json); + virtual int prepare() override; - virtual const std::string &getDetails(); + virtual int parse(json_t *json) override; - virtual int check(); + virtual int check() override; - virtual int prepare(); + virtual int start() override; - virtual std::vector getPollFDs(); + virtual std::vector getPollFDs() override; + + virtual const std::string &getDetails() override; }; class FpgaNodeFactory : public NodeFactory { @@ -69,7 +65,8 @@ class FpgaNodeFactory : public NodeFactory { public: using NodeFactory::NodeFactory; - virtual Node *make(const uuid_t &id = {}, const std::string &nme = "") { + virtual Node *make(const uuid_t &id = {}, + const std::string &nme = "") override { auto *n = new FpgaNode(id, nme); init(n); @@ -77,17 +74,17 @@ public: return n; } - virtual int getFlags() const { + virtual int getFlags() const override { return (int)NodeFactory::Flags::SUPPORTS_READ | (int)NodeFactory::Flags::SUPPORTS_WRITE | (int)NodeFactory::Flags::SUPPORTS_POLL; } - virtual std::string getName() const { return "fpga"; } + virtual std::string getName() const override { return "fpga"; } - virtual std::string getDescription() const { return "VILLASfpga"; } + virtual std::string getDescription() const override { return "VILLASfpga"; } - virtual int start(SuperNode *sn); + virtual int start(SuperNode *sn) override; }; } // namespace node diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index 0bf2d4b60..861825d75 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -141,6 +141,8 @@ public: json_t *getConfig() { return config.root; } + std::filesystem::path &getConfigPath() { return config.getConfigPath(); } + std::string getConfigUri() const { return uri; } int getAffinity() const { return affinity; } diff --git a/lib/config.cpp b/lib/config.cpp index 66acc5205..45de082d2 100644 --- a/lib/config.cpp +++ b/lib/config.cpp @@ -72,13 +72,14 @@ json_t *Config::load(const std::string &u, bool resolveInc, FILE *Config::loadFromStdio() { logger->info("Reading configuration from standard input"); - + configPath = std::filesystem::current_path(); return stdin; } FILE *Config::loadFromLocalFile(const std::string &u) { logger->info("Reading configuration from local file: {}", u); + configPath = u; FILE *f = fopen(u.c_str(), "r"); if (!f) throw RuntimeError("Failed to open configuration from: {}", u); diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp index 7f12d1382..bb7e12986 100644 --- a/lib/nodes/fpga.cpp +++ b/lib/nodes/fpga.cpp @@ -1,12 +1,10 @@ /* Communicate with VILLASfpga Xilinx FPGA boards. * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * Author: Niklas Eiling + * SPDX-FileCopyrightText: 2023 Niklas Eiling * SPDX-License-Identifier: Apache-2.0 */ -#include -#include #include #include #include @@ -16,16 +14,12 @@ #include #include -#include #include #include #include #include -#include -#include -#include -#include +#include using namespace villas; using namespace villas::node; @@ -33,20 +27,69 @@ using namespace villas::fpga; using namespace villas::utils; // Global state -static std::list> cards; -static std::map dmaMap; +static std::list> cards; -static std::shared_ptr pciDevices; static std::shared_ptr vfioContainer; -using namespace villas; -using namespace villas::node; - FpgaNode::FpgaNode(const uuid_t &id, const std::string &name) - : Node(id, name), irqFd(-1), coalesce(0), polling(true) {} + : Node(id, name), cardName(""), card(), dma(), blockRx(), blockTx() {} FpgaNode::~FpgaNode() {} +int FpgaNode::prepare() { + for (auto &fpgaCard : cards) { + if (fpgaCard->name == cardName) { + card = fpgaCard; + break; + } + } + if (card == nullptr) { + throw ConfigError(config, "node-config-fpga", + "There is no FPGA card with the name: {}", cardName); + } + + std::vector> aurora_channels; + for (int i = 0; i < 4; i++) { + auto name = fmt::format("aurora_8b10b_ch{}", i); + auto id = fpga::ip::IpIdentifier("xilinx.com:ip:aurora_8b10b:", name); + auto aurora = + std::dynamic_pointer_cast(card->lookupIp(id)); + if (aurora == nullptr) { + logger->error("No Aurora interface found on FPGA"); + return 1; + } + + aurora_channels.push_back(aurora); + } + + dma = std::dynamic_pointer_cast( + card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:"))); + if (dma == nullptr) { + logger->error("No DMA found on FPGA "); + return 1; + } + + // Configure Crossbar switch + for (std::string str : connectStrings) { + const fpga::ConnectString parsedConnectString(str); + parsedConnectString.configCrossBar(dma, aurora_channels); + } + + auto &alloc = HostRam::getAllocator(); + + blockRx[0] = alloc.allocateBlock(0x200 * sizeof(float)); + blockRx[1] = alloc.allocateBlock(0x200 * sizeof(float)); + blockTx = alloc.allocateBlock(0x200 * sizeof(float)); + villas::MemoryAccessor memRx[] = {*(blockRx[0]), *(blockRx[1])}; + villas::MemoryAccessor memTx = *blockTx; + + dma->makeAccesibleFromVA(blockRx[0]); + dma->makeAccesibleFromVA(blockRx[1]); + dma->makeAccesibleFromVA(blockTx); + + return Node::prepare(); +} + int FpgaNode::parse(json_t *json) { int ret = Node::parse(json); if (ret) @@ -54,29 +97,28 @@ int FpgaNode::parse(json_t *json) { json_error_t err; - const char *card = nullptr; - const char *intf = nullptr; - const char *dma = nullptr; - int poll = polling; + const char *jsonCardName = nullptr; + json_t *jsonConnectStrings = nullptr; - ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }", - "card", &card, "interface", &intf, "dma", &dma, - "coalesce", &coalesce, "polling", &polling); - if (ret) + ret = json_unpack_ex(json, &err, 0, "{ s: s, s?: o}", "card", &jsonCardName, + "connect", &jsonConnectStrings); + if (ret) { throw ConfigError(json, err, "node-config-fpga", "Failed to parse configuration of node {}", this->getName()); - - if (card) - cardName = card; - - if (intf) - intfName = intf; - - if (dma) - dmaName = dma; - - polling = poll; // cast int to bool + } + cardName = std::string(jsonCardName); + if (jsonConnectStrings != nullptr && json_is_array(jsonConnectStrings)) { + for (size_t i = 0; i < json_array_size(jsonConnectStrings); i++) { + json_t *jsonConnectString = json_array_get(jsonConnectStrings, i); + if (jsonConnectString == nullptr || !json_is_string(jsonConnectString)) { + throw ConfigError(jsonConnectString, "node-config-fpga", + "Failed to parse connect string"); + } + connectStrings.push_back( + std::string(json_string_value(jsonConnectString))); + } + } return 0; } @@ -85,9 +127,13 @@ const std::string &FpgaNode::getDetails() { if (details.empty()) { auto &name = card ? card->name : cardName; - details = - fmt::format("fpga={}, dma={}, if={}, polling={}, coalesce={}", name, - dma->getInstanceName(), polling ? "yes" : "no", coalesce); + const char *const delim = ", "; + + std::ostringstream imploded; + std::copy(connectStrings.begin(), connectStrings.end(), + std::ostream_iterator(imploded, delim)); + + details = fmt::format("fpga={}, connect={}", name, imploded.str()); } return details; @@ -95,136 +141,81 @@ const std::string &FpgaNode::getDetails() { int FpgaNode::check() { return 0; } -int FpgaNode::prepare() { - auto it = cardName.empty() - ? cards.begin() - : std::find_if(cards.begin(), cards.end(), - [this](std::shared_ptr c) { - return c->name == cardName; - }); - - if (it == cards.end()) - throw ConfigError(json_object_get(config, "fpga"), "node-config-fpga-card", - "Invalid FPGA card name: {}", cardName); - - card = *it; - - auto intfCore = intfName.empty() - ? card->lookupIp(fpga::Vlnv(FPGA_AURORA_VLNV)) - : card->lookupIp(intfName); - if (!intfCore) - throw ConfigError(config, "node-config-fpga-interface", - "There is no interface IP with the name: {}", intfName); - - intf = std::dynamic_pointer_cast(intfCore); - if (!intf) - throw RuntimeError("The IP {} is not a interface", *intfCore); - - auto dmaCore = dmaName.empty() ? card->lookupIp(fpga::Vlnv(FPGA_DMA_VLNV)) - : card->lookupIp(dmaName); - if (!dmaCore) - throw ConfigError(config, "node-config-fpga-dma", - "There is no DMA IP with the name: {}", dmaName); - - dma = std::dynamic_pointer_cast(dmaCore); - if (!dma) - throw RuntimeError("The IP {} is not a DMA controller", *dmaCore); - - int ret = intf->connect(*(dma), true); - if (ret) - throw RuntimeError("Failed to connect: {} -> {}", *(intf), *(dma)); - - auto &alloc = HostDmaRam::getAllocator(); - - const std::shared_ptr blockRx = - alloc.allocateBlock(0x100 * sizeof(float)); - const std::shared_ptr blockTx = - alloc.allocateBlock(0x100 * sizeof(float)); - villas::MemoryAccessor memRx = *blockRx; - villas::MemoryAccessor memTx = *blockTx; - - dma->makeAccesibleFromVA(blockRx); - dma->makeAccesibleFromVA(blockTx); - - // Show some debugging infos - auto &mm = MemoryManager::get(); - - dma->dump(); - intf->dump(); - mm.getGraph().dump(); - - return Node::prepare(); +int FpgaNode::start() { + // enque first read + dma->read(*(blockRx[0]), blockRx[0]->getSize()); + return Node::start(); } int FpgaNode::_read(Sample *smps[], unsigned cnt) { + static size_t cur = 0, next = 1; unsigned read; Sample *smp = smps[0]; assert(cnt == 1); - dma->read(*blockRx, blockRx->getSize()); // TODO: calc size - const size_t bytesRead = dma->readComplete().bytes; - read = bytesRead / sizeof(int32_t); + dma->read(*(blockRx[next]), blockRx[next]->getSize()); // TODO: calc size + auto c = dma->readComplete(); + read = c.bytes / sizeof(float); - auto mem = MemoryAccessor(*blockRx); + if (c.interrupts > 1) { + logger->warn("Missed {} interrupts", c.interrupts - 1); + } - for (unsigned i = 0; i < MIN(read, smp->capacity); i++) - smp->data[i].i = mem[i]; + auto mem = MemoryAccessor(*(blockRx[cur])); + + smp->length = 0; + for (unsigned i = 0; i < MIN(read, smp->capacity); i++) { + smp->data[i].f = static_cast(mem[i]); + smp->length++; + } + smp->flags = (int)SampleFlags::HAS_DATA; smp->signals = in.signals; + cur = next; + next = (next + 1) % (sizeof(blockRx) / sizeof(blockRx[0])); - return read; + return 1; } int FpgaNode::_write(Sample *smps[], unsigned cnt) { - int written; + unsigned int written; Sample *smp = smps[0]; - assert(cnt == 1); + assert(cnt == 1 && smps != nullptr && smps[0] != nullptr); - auto mem = MemoryAccessor(*blockTx); + auto mem = MemoryAccessor(*blockTx); - for (unsigned i = 0; i < smps[0]->length; i++) - mem[i] = smps[0]->data[i].i; + for (unsigned i = 0; i < smps[0]->length; i++) { + mem[i] = static_cast(smps[0]->data[i].f); + } - bool state = dma->write(*blockTx, smp->length * sizeof(int32_t)); + bool state = dma->write(*blockTx, smp->length * sizeof(float)); if (!state) return -1; - written = 0; // The number of samples written + written = dma->writeComplete().bytes / + sizeof(float); // The number of samples written - return written; + if (written != smp->length) { + logger->warn("Wrote {} samples, but {} were expected", written, + smp->length); + } + + return 1; } std::vector FpgaNode::getPollFDs() { - std::vector fds; - - if (!polling) - fds.push_back(irqFd); - - return fds; + return card->vfioDevice->getEventfdList(); } int FpgaNodeFactory::start(SuperNode *sn) { vfioContainer = std::make_shared(); - pciDevices = std::make_shared(); - // Get the FPGA card plugin - auto pcieCardPlugin = plugin::registry->lookup("pcie"); - if (!pcieCardPlugin) - throw RuntimeError("No FPGA PCIe plugin found"); - - json_t *json_cfg = sn->getConfig(); - json_t *json_fpgas = json_object_get(json_cfg, "fpgas"); - if (!json_fpgas) - throw ConfigError(json_cfg, "node-config-fpgas", - "No section 'fpgas' found in config"); - - // Create all FPGA card instances using the corresponding plugin - auto piceCards = - fpga::PCIeCardFactory::make(json_fpgas, pciDevices, vfioContainer); - - cards.splice(cards.end(), piceCards); + if (cards.empty()) { + std::filesystem::path searchPath = sn->getConfigPath().parent_path(); + createCards(sn->getConfig(), cards, searchPath); + } return NodeFactory::start(sn); }