Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

fpga: first compiling code of fpga support

Browse source
This commit is contained in:
Steffen Vogel 2020-06-15 22:21:56 +02:00
parent f8850b4110
commit a5bc8eb90f
6 changed files with 399 additions and 400 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common

@ -1 +1 @@
Subproject commit 952945fc4bcdcdca0dfbe1389f811ceb7b5c5744
Subproject commit 3b5952a413ba8f8c7731c6a0c8336e1f523884b8

2
fpga

@ -1 +1 @@
Subproject commit 1f6a181af64d806df12611ab0c5d0db33aa53cc8
Subproject commit 21340e7bc56417b9967f5a066f93ac2f99777ab1

70
include/villas/nodes/villas_fpga.hpp → include/villas/nodes/fpga.hpp
View file

@ -22,7 +22,7 @@
*********************************************************************************/
/**
* @addtogroup villas_fpga BSD villas_fpga Node Type
* @addtogroup fpga BSD fpga Node Type
* @ingroup node
* @{
*/
@ -34,67 +34,73 @@
#include <villas/io.h>
#include <villas/timing.h>
struct villas_fpga {
std::string fpga_name;
#include <villas/fpga/card.hpp>
#include <villas/fpga/node.hpp>
#include <villas/fpga/ips/dma.hpp>
using namespace villas;
#define FPGA_DMA_VLNV
#define FPGA_AURORA_VLNV "acs.eonerc.rwth-aachen.de:user:aurora_axis:"
struct fpga {
int irqFd;
int coalesce;
bool polling;
std::shared_ptr<villas::fpga::PCIeCard> card;
fpga::IpNode *dma;
fpga::IpNode *intf;
std::shared_ptr<villas::fpga::ip::Dma> dma;
std::shared_ptr<villas::fpga::ip::Node> intf;
fpga::Mem mem;
fpga::Block block;
struct {
villas::MemoryAccessor<uint32_t> mem;
villas::MemoryBlock block;
} in, out;
// Config only
std::string cardName;
std::string intfName;
std::string dmaName;
};
/** @see node_vtable::type_start */
int villas_fpga_type_start(villas::node::SuperNode *sn);
int fpga_type_start(villas::node::SuperNode *sn);
/** @see node_type::type_stop */
int villas_fpga_type_stop();
int fpga_type_stop();
/** @see node_type::init */
int villas_fpga_init(struct node *n);
int fpga_init(struct node *n);
/** @see node_type::destroy */
int villas_fpga_destroy(struct node *n);
int fpga_destroy(struct node *n);
/** @see node_type::parse */
int villas_fpga_parse(struct node *n, json_t *cfg);
int fpga_parse(struct node *n, json_t *cfg);
/** @see node_type::print */
char * villas_fpga_print(struct node *n);
char * fpga_print(struct node *n);
/** @see node_type::check */
int villas_fpga_check();
int fpga_check();
/** @see node_type::prepare */
int villas_fpga_prepare();
int fpga_prepare();
/** @see node_type::start */
int villas_fpga_start(struct node *n);
int fpga_start(struct node *n);
/** @see node_type::stop */
int villas_fpga_stop(struct node *n);
/** @see node_type::pause */
int villas_fpga_pause(struct node *n);
/** @see node_type::resume */
int villas_fpga_resume(struct node *n);
int fpga_stop(struct node *n);
/** @see node_type::write */
int villas_fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release);
int fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release);
/** @see node_type::read */
int villas_fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release);
/** @see node_type::reverse */
int villas_fpga_reverse(struct node *n);
int fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release);
/** @see node_type::poll_fds */
int villas_fpga_poll_fds(struct node *n, int fds[]);
/** @see node_type::netem_fds */
int villas_fpga_netem_fds(struct node *n, int fds[]);
int fpga_poll_fds(struct node *n, int fds[]);
/** @} */

2
lib/nodes/CMakeLists.txt
View file

@ -166,7 +166,7 @@ endif()
# Enable VILLASfpga support
if(WITH_NODE_FPGA)
list(APPEND NODE_SRC villas_fpga.cpp)
list(APPEND NODE_SRC fpga.cpp)
list(APPEND LIBRARIES villas-fpga)
endif()

358
lib/nodes/fpga.cpp Normal file
View file

@ -0,0 +1,358 @@
/** Communicate with VILLASfpga Xilinx FPGA boards
*
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
* @license GNU General Public License (version 3)
*
* VILLASnode
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
#include <csignal>
#include <iostream>
#include <vector>
#include <string>
#include <algorithm>
#include <jansson.h>
#include <villas/nodes/fpga.hpp>
#include <villas/log.hpp>
#include <villas/utils.hpp>
#include <villas/sample.h>
#include <villas/plugin.h>
#include <villas/super_node.hpp>
#include <villas/fpga/core.hpp>
#include <villas/fpga/vlnv.hpp>
#include <villas/fpga/ips/dma.hpp>
/* Forward declartions */
static struct plugin p;
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
/* Global state */
static fpga::PCIeCard::List cards;
static std::map<fpga::ip::Dma, struct fpga *> dmaMap;
static std::shared_ptr<kernel::pci::DeviceList> pciDevices;
static std::shared_ptr<kernel::vfio::Container> vfioContainer;
using namespace villas;
// static std::shared_ptr<fpga::ip::Dma>
// fpga_find_dma(const fpga::PCIeCard &card)
// {
// // for (auto &ip : card.ips) {
// // }
// return nullptr;
// }
int fpga_type_start(node::SuperNode *sn)
{
vfioContainer = kernel::vfio::Container::create();
pciDevices = std::make_shared<kernel::pci::DeviceList>();
// get the FPGA card plugin
auto pcieCardPlugin = plugin::Registry::lookup<fpga::PCIeCardFactory>("pcie");
if (!pcieCardPlugin)
throw RuntimeError("No FPGA PCIe plugin found");
json_t *cfg = sn->getConfig();
json_t *fpgas = json_object_get(cfg, "fpgas");
if (!fpgas)
throw ConfigError(cfg, "node-config-fpgas", "No section 'fpgas' found in config");
// create all FPGA card instances using the corresponding plugin
auto pcieCards = pcieCardPlugin->make(fpgas, pciDevices, vfioContainer);
cards.splice(cards.end(), pcieCards);
return 0;
}
int fpga_type_stop()
{
vfioContainer.reset(); // TODO: is this the proper way?
return 0;
}
int fpga_init(struct node *n)
{
struct fpga *f = (struct fpga *) n->_vd;
f->coalesce = 0;
f->irqFd = -1;
f->polling = true;
new (&f->cardName) std::string();
new (&f->dmaName) std::string();
new (&f->intfName) std::string();
new (&f->card) std::shared_ptr<fpga::PCIeCard>();
new (&f->dma) std::shared_ptr<fpga::ip::Node>();
new (&f->intf) std::shared_ptr<fpga::ip::Node>();
new (&f->in.mem) std::shared_ptr<MemoryBlock>();
new (&f->out.mem) std::shared_ptr<MemoryBlock>();
return 0;
}
int fpga_destroy(struct node *n)
{
struct fpga *f = (struct fpga *) n->_vd;
using mbptr = MemoryAccessor<uint32_t>;
using cptr = std::shared_ptr<fpga::PCIeCard>;
using nptr = std::shared_ptr<fpga::ip::Node>;
using dptr = std::shared_ptr<fpga::ip::Dma>;
using sptr = std::string;
f->cardName.~sptr();
f->dmaName.~sptr();
f->intfName.~sptr();
f->card.~cptr();
f->dma.~dptr();
f->intf.~nptr();
f->in.mem.~mbptr();
f->out.mem.~mbptr();
return 0;
}
int fpga_parse(struct node *n, json_t *cfg)
{
int ret;
struct fpga *f = (struct fpga *) n->_vd;
json_error_t err;
const char *card = nullptr;
const char *intf = nullptr;
const char *dma = nullptr;
int polling = f->polling;
ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }",
"card", &card,
"interface", &intf,
"dma", &dma,
"coalesce", &f->coalesce,
"polling", &polling
);
if (ret)
throw ConfigError(cfg, err, "node-config-fpga", "Failed to parse configuration of node {}", node_name(n));
if (card)
f->cardName = card;
if (intf)
f->intfName = intf;
if (dma)
f->dmaName = dma;
f->polling = polling; // cast int to bool
return 0;
}
char * fpga_print(struct node *n)
{
struct fpga *f = (struct fpga *) n->_vd;
return strf("fpga=%s, dma=%s, if=%s, polling=%s, coalesce=%d",
f->card->name.c_str(),
f->dma->getInstanceName().c_str(),
f->polling ? "yes" : "no",
f->coalesce
);
}
int fpga_check(struct node *n)
{
// struct fpga *f = (struct fpga *) n->_vd;
return 0;
}
int fpga_prepare(struct node *n)
{
int ret;
struct fpga *f = (struct fpga *) n->_vd;
auto it = f->cardName.empty()
? cards.begin()
: std::find_if(cards.begin(), cards.end(), [f](const fpga::PCIeCard::Ptr &c) {
return c->name == f->cardName;
});
if (it == cards.end())
throw ConfigError(json_object_get(n->cfg, "fpga"), "node-config-fpga-card", "Invalid FPGA card name: {}", f->cardName);
f->card = *it;
auto intf = f->intfName.empty()
? f->card->lookupIp(fpga::Vlnv(FPGA_AURORA_VLNV))
: f->card->lookupIp(f->intfName);
if (!intf)
throw ConfigError(n->cfg, "node-config-fpga-interface", "There is no interface IP with the name: {}", f->intfName);
f->intf = std::dynamic_pointer_cast<fpga::ip::Node>(intf);
if (!f->intf)
throw RuntimeError("The IP {} is not a interface", *intf);
auto dma = f->dmaName.empty()
? f->card->lookupIp(fpga::Vlnv(FPGA_DMA_VLNV))
: f->card->lookupIp(f->dmaName);
if (!dma)
throw ConfigError(n->cfg, "node-config-fpga-dma", "There is no DMA IP with the name: {}", f->dmaName);
f->dma = std::dynamic_pointer_cast<fpga::ip::Dma>(dma);
if (!f->dma)
throw RuntimeError("The IP {} is not a DMA controller", *dma);
ret = f->intf->connect(*(f->dma), true);
if (ret)
throw RuntimeError("Failed to connect: {} -> {}",
*(f->intf), *(f->dma)
);
auto &alloc = HostRam::getAllocator();
f->in.mem = alloc.allocate<uint32_t>(0x100 / sizeof(int32_t));
f->out.mem = alloc.allocate<uint32_t>(0x100 / sizeof(int32_t));
f->in.block = f->in.mem.getMemoryBlock();
f->out.block = f->out.mem.getMemoryBlock();
f->dma->makeAccesibleFromVA(f->in.block);
f->dma->makeAccesibleFromVA(f->out.block);
f->dma->dump();
f->intf->dump();
MemoryManager::get().getGraph().dump();
return 0;
}
int fpga_start(struct node *n)
{
// struct fpga *f = (struct fpga *) n->_vd;
return 0;
}
int fpga_stop(struct node *n)
{
//struct fpga *f = (struct fpga *) n->_vd;
return 0;
}
int fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release)
{
int read;
struct fpga *f = (struct fpga *) n->_vd;
struct sample *smp = smps[0];
assert(cnt == 1);
f->dma->read(f->in.block, f->in.block.getSize()); // TODO: calc size
const size_t bytesRead = f->dma->readComplete();
read = bytesRead / sizeof(int32_t);
for (unsigned i = 0; i < MIN(read, smp->capacity); i++)
smp->data[i].i = f->in.mem[i];
return read;
}
int fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release)
{
int written;
struct fpga *f = (struct fpga *) n->_vd;
struct sample *smp = smps[0];
assert(cnt == 1);
for (unsigned i = 0; i < smps[0]->length; i++)
f->out.mem[i] = smps[0]->data[i].i;
bool state = f->dma->write(f->out.block, smp->length * sizeof(int32_t));
if (!state)
return -1;
written = 0; /* The number of samples written */
return written;
}
int fpga_poll_fds(struct node *n, int fds[])
{
struct fpga *f = (struct fpga *) n->_vd;
if (f->polling)
return 0;
else {
fds[0] = f->irqFd;
return 1; /* The number of file descriptors which have been set in fds */
}
}
__attribute__((constructor(110)))
static void register_plugin() {
if (plugins.state == State::DESTROYED)
vlist_init(&plugins);
p.name = "fpga";
p.description = "Communicate with VILLASfpga Xilinx FPGA boards";
p.type = PluginType::NODE;
p.node.instances.state = State::DESTROYED;
p.node.vectorize = 1;
p.node.size = sizeof(struct fpga);
p.node.type.start = fpga_type_start;
p.node.type.stop = fpga_type_stop;
p.node.init = fpga_init;
p.node.destroy = fpga_destroy;
p.node.prepare = fpga_prepare;
p.node.parse = fpga_parse;
p.node.print = fpga_print;
p.node.check = fpga_check;
p.node.start = fpga_start;
p.node.stop = fpga_stop;
p.node.read = fpga_read;
p.node.write = fpga_write;
p.node.poll_fds = fpga_poll_fds;
vlist_init(&p.node.instances);
vlist_push(&plugins, &p);
}
__attribute__((destructor(110)))
static void deregister_plugin() {
if (plugins.state != State::DESTROYED)
vlist_remove_all(&plugins, &p);
}

365
lib/nodes/villas_fpga.cpp
View file

@ -1,365 +0,0 @@
/** Communicate with VILLASfpga Xilinx FPGA boards
*
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC
* @license GNU General Public License (version 3)
*
* VILLASnode
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
#include <csignal>
#include <iostream>
#include <vector>
#include <string>
#include <algorithm>
#include <jansson.h>
#include <villas/nodes/villas_fpga.hpp>
#include <villas/log.hpp>
#include <villas/utils.h>
#include <villas/utils.hpp>
#include <villas/sample.h>
#include <villas/plugin.h>
#include <villas/super_node.hpp>
#include <villas/fpga/ip.hpp>
#include <villas/fpga/card.hpp>
#include <villas/fpga/vlnv.hpp>
#include <villas/fpga/ips/dma.hpp>
#include <villas/fpga/ips/rtds.hpp>
/* Forward declartions */
static struct plugin p;
using namespace villas;
using namespace villas::node;
using namespace villas::utils;
static struct pci pci;
int villas_fpga_parse_config()
{
/* Parse FPGA configuration */
FILE* f = fopen(configFile.c_str(), "r");
if (f == nullptr) {
logger->error("Cannot open config file: {}", configFile);
}
json_t* json = json_loadf(f, 0, nullptr);
if (json == nullptr) {
logger->error("Cannot parse JSON config");
fclose(f);
return -1;
}
fclose(f);
json_t* fpgas = json_object_get(json, "fpgas");
if (fpgas == nullptr) {
logger->error("No section 'fpgas' found in config");
exit(1);
}
return 0;
}
int villas_fpga_type_start(villas::node::SuperNode *sn)
{
int ret;
ret = pci_init(&pci);
if (ret) {
logger->error("Cannot initialize PCI subsystem");
return ret;
}
auto vfioContainer = villas::VfioContainer::create();
ret = villas_fpga_parse_config();
if (ret)
return ret;
// get the FPGA card plugin
villas::Plugin* plugin = villas::Plugin::lookup(villas::Plugin::Type::FpgaCard, "");
if (plugin == nullptr) {
logger->error("No FPGA plugin found");
exit(1);
}
villas::fpga::PCIeCardFactory* fpgaCardPlugin =
dynamic_cast<villas::fpga::PCIeCardFactory*>(plugin);
// create all FPGA card instances using the corresponding plugin
auto cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer);
return 0;
}
int villas_fpga_type_stop()
{
return 0;
}
int villas_fpga_init(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
return 0;
}
int villas_fpga_destroy(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
return 0;
}
int villas_fpga_parse(struct node *n, json_t *cfg)
{
int ret;
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
json_error_t err;
const char *fpga_name;
const char *dma_vlnv = "";
const char *if_vlnv = "";
ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: s, s?: s }",
"fpga", &s->setting1,
"", &s->setting2
);
if (ret)
jerror(&err, "Failed to parse configuration of node %s", node_name(n));
return 0;
}
char * villas_fpga_print(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
return strf("fpga=%s, dma=%s, if=%s", f->fpga_name, f->dma_vlnv, f->);
}
int villas_fpga_check(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
if (s->setting1 > 100 || s->setting1 < 0)
return -1;
if (!s->setting2 || strlen(s->setting2) > 10)
return -1;
return 0;
}
int villas_fpga_prepare(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
for (auto& fpgaCard : cards) {
if (fpgaCard->name == fpgaName) {
f->card = fpgaCard;
break;
}
}
if (!card)
logger->error("FPGA card {} not found in config or not working", fpgaName);
// deallocate JSON config
//json_decref(json);
auto rtds = dynamic_cast<fpga::ip::Rtds*>
(card->lookupIp(fpga::Vlnv("acs.eonerc.rwth-aachen.de:user:rtds_axis:")));
//auto dma = dynamic_cast<fpga::ip::Dma*>
// (card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:")));
auto dma = dynamic_cast<fpga::ip::Dma*>
(card->lookupIp("hier_0_axi_dma_axi_dma_1"));
if (!rtds) {
logger->error("No RTDS interface found on FPGA");
return 1;
}
if (!dma) {
logger->error("No DMA found on FPGA ");
return 1;
}
rtds->dump();
rtds->connect(rtds->getMasterPort(rtds->masterPort),
dma->getSlavePort(dma->s2mmPort));
dma->connect(dma->getMasterPort(dma->mm2sPort),
rtds->getSlavePort(rtds->slavePort));
auto &alloc = villas::HostRam::getAllocator();
auto mem = alloc.allocate<int32_t>(0x100 / sizeof(int32_t));
auto block = mem.getMemoryBlock();
dma->makeAccesibleFromVA(block);
// auto &mm = MemoryManager::get();
// mm.getMemoryGraph().dump("graph.dot");
return 0;
}
int villas_fpga_start(struct node *n)
{
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
s->start_time = time_now();
return 0;
}
int villas_fpga_stop(struct node *n)
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0;
}
int villas_fpga_pause(struct node *n)
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0;
}
int villas_fpga_resume(struct node *n)
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0;
}
int villas_fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release)
{
int read;
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
struct timespec now;
dma->read(block, block.getSize());
const size_t bytesRead = dma->readComplete();
const size_t valuesRead = bytesRead / sizeof(int32_t);
for (size_t i = 0; i < valuesRead; i++) {
std::cerr << mem[i] << ";";
}
std::cerr << std::endl;
return read;
}
int villas_fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release)
{
int written;
struct villas_fpga *f = (struct villas_fpga *) n->_vd;
size_t memIdx = 0;
for (unsigned i = 0; i < smps[0]->length; i++)
f->tx_mem[memIdx++] = smps[0]->data[i].i;
bool state = dma->write(block, memIdx * sizeof(int32_t));
if (!state)
logger->error("Failed to write to device");
written = 0; /* The number of samples written */
return written;
}
int villas_fpga_reverse(struct node *n)
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0;
}
int villas_fpga_poll_fds(struct node *n, int fds[])
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0; /* The number of file descriptors which have been set in fds */
}
int villas_fpga_netem_fds(struct node *n, int fds[])
{
//struct villas_fpga *f = (struct villas_fpga *) n->_vd;
/* TODO: Add implementation here. */
return 0; /* The number of file descriptors which have been set in fds */
}
__attribute__((constructor(110)))
static void register_plugin() {
if (plugins.state == State::DESTROYED)
vlist_init(&plugins);
p.name = "fpga";
p.description = "Communicate with VILLASfpga Xilinx FPGA boards";
p.type = PluginType::NODE;
p.node.instances.state = State::DESTROYED;
p.node.vectorize = 0;
p.node.size = sizeof(struct villas_fpga);
p.node.type.start = villas_fpga_type_start;
p.node.type.stop = villas_fpga_type_stop;
p.node.init = villas_fpga_init;
p.node.destroy = villas_fpga_destroy;
p.node.prepare = villas_fpga_prepare;
p.node.parse = villas_fpga_parse;
p.node.print = villas_fpga_print;
p.node.check = villas_fpga_check;
p.node.start = villas_fpga_start;
p.node.stop = villas_fpga_stop;
p.node.pause = villas_fpga_pause;
p.node.resume = villas_fpga_resume;
p.node.read = villas_fpga_read;
p.node.write = villas_fpga_write;
p.node.reverse = villas_fpga_reverse;
p.node.poll_fds = villas_fpga_poll_fds;
p.node.netem_fds = villas_fpga_netem_fds;
vlist_init(&p.node.instances);
vlist_push(&plugins, &p);
}
__attribute__((destructor(110)))
static void deregister_plugin() {
if (plugins.state != State::DESTROYED)
vlist_remove_all(&plugins, &p);
}