diff --git a/common b/common index 952945fc4..3b5952a41 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 952945fc4bcdcdca0dfbe1389f811ceb7b5c5744 +Subproject commit 3b5952a413ba8f8c7731c6a0c8336e1f523884b8 diff --git a/fpga b/fpga index 1f6a181af..21340e7bc 160000 --- a/fpga +++ b/fpga @@ -1 +1 @@ -Subproject commit 1f6a181af64d806df12611ab0c5d0db33aa53cc8 +Subproject commit 21340e7bc56417b9967f5a066f93ac2f99777ab1 diff --git a/include/villas/nodes/villas_fpga.hpp b/include/villas/nodes/fpga.hpp similarity index 59% rename from include/villas/nodes/villas_fpga.hpp rename to include/villas/nodes/fpga.hpp index 9616b3f76..dfb8db14d 100644 --- a/include/villas/nodes/villas_fpga.hpp +++ b/include/villas/nodes/fpga.hpp @@ -22,7 +22,7 @@ *********************************************************************************/ /** - * @addtogroup villas_fpga BSD villas_fpga Node Type + * @addtogroup fpga BSD fpga Node Type * @ingroup node * @{ */ @@ -34,67 +34,73 @@ #include #include -struct villas_fpga { - std::string fpga_name; +#include +#include +#include + +using namespace villas; + +#define FPGA_DMA_VLNV +#define FPGA_AURORA_VLNV "acs.eonerc.rwth-aachen.de:user:aurora_axis:" + +struct fpga { + int irqFd; + int coalesce; + bool polling; std::shared_ptr card; - fpga::IpNode *dma; - fpga::IpNode *intf; + std::shared_ptr dma; + std::shared_ptr intf; - fpga::Mem mem; - fpga::Block block; + struct { + villas::MemoryAccessor mem; + villas::MemoryBlock block; + } in, out; + + // Config only + std::string cardName; + std::string intfName; + std::string dmaName; }; /** @see node_vtable::type_start */ -int villas_fpga_type_start(villas::node::SuperNode *sn); +int fpga_type_start(villas::node::SuperNode *sn); /** @see node_type::type_stop */ -int villas_fpga_type_stop(); +int fpga_type_stop(); /** @see node_type::init */ -int villas_fpga_init(struct node *n); +int fpga_init(struct node *n); /** @see node_type::destroy */ -int villas_fpga_destroy(struct node *n); +int fpga_destroy(struct node *n); /** @see node_type::parse */ -int villas_fpga_parse(struct node *n, json_t *cfg); +int fpga_parse(struct node *n, json_t *cfg); /** @see node_type::print */ -char * villas_fpga_print(struct node *n); +char * fpga_print(struct node *n); /** @see node_type::check */ -int villas_fpga_check(); +int fpga_check(); /** @see node_type::prepare */ -int villas_fpga_prepare(); +int fpga_prepare(); /** @see node_type::start */ -int villas_fpga_start(struct node *n); +int fpga_start(struct node *n); /** @see node_type::stop */ -int villas_fpga_stop(struct node *n); - -/** @see node_type::pause */ -int villas_fpga_pause(struct node *n); - -/** @see node_type::resume */ -int villas_fpga_resume(struct node *n); +int fpga_stop(struct node *n); /** @see node_type::write */ -int villas_fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release); +int fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release); /** @see node_type::read */ -int villas_fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release); - -/** @see node_type::reverse */ -int villas_fpga_reverse(struct node *n); +int fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release); /** @see node_type::poll_fds */ -int villas_fpga_poll_fds(struct node *n, int fds[]); - -/** @see node_type::netem_fds */ -int villas_fpga_netem_fds(struct node *n, int fds[]); +int fpga_poll_fds(struct node *n, int fds[]); /** @} */ diff --git a/lib/nodes/CMakeLists.txt b/lib/nodes/CMakeLists.txt index 93475f650..209a707e0 100644 --- a/lib/nodes/CMakeLists.txt +++ b/lib/nodes/CMakeLists.txt @@ -166,7 +166,7 @@ endif() # Enable VILLASfpga support if(WITH_NODE_FPGA) - list(APPEND NODE_SRC villas_fpga.cpp) + list(APPEND NODE_SRC fpga.cpp) list(APPEND LIBRARIES villas-fpga) endif() diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp new file mode 100644 index 000000000..a8401a547 --- /dev/null +++ b/lib/nodes/fpga.cpp @@ -0,0 +1,358 @@ +/** Communicate with VILLASfpga Xilinx FPGA boards + * + * @author Steffen Vogel + * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* Forward declartions */ +static struct plugin p; + +using namespace villas; +using namespace villas::node; +using namespace villas::utils; + +/* Global state */ +static fpga::PCIeCard::List cards; +static std::map dmaMap; + +static std::shared_ptr pciDevices; +static std::shared_ptr vfioContainer; + +using namespace villas; + +// static std::shared_ptr +// fpga_find_dma(const fpga::PCIeCard &card) +// { +// // for (auto &ip : card.ips) { + +// // } + +// return nullptr; +// } + +int fpga_type_start(node::SuperNode *sn) +{ + vfioContainer = kernel::vfio::Container::create(); + pciDevices = std::make_shared(); + + // get the FPGA card plugin + auto pcieCardPlugin = plugin::Registry::lookup("pcie"); + if (!pcieCardPlugin) + throw RuntimeError("No FPGA PCIe plugin found"); + + json_t *cfg = sn->getConfig(); + json_t *fpgas = json_object_get(cfg, "fpgas"); + if (!fpgas) + throw ConfigError(cfg, "node-config-fpgas", "No section 'fpgas' found in config"); + + // create all FPGA card instances using the corresponding plugin + auto pcieCards = pcieCardPlugin->make(fpgas, pciDevices, vfioContainer); + + cards.splice(cards.end(), pcieCards); + + return 0; +} + +int fpga_type_stop() +{ + vfioContainer.reset(); // TODO: is this the proper way? + + return 0; +} + +int fpga_init(struct node *n) +{ + struct fpga *f = (struct fpga *) n->_vd; + + f->coalesce = 0; + f->irqFd = -1; + f->polling = true; + + new (&f->cardName) std::string(); + new (&f->dmaName) std::string(); + new (&f->intfName) std::string(); + + new (&f->card) std::shared_ptr(); + + new (&f->dma) std::shared_ptr(); + new (&f->intf) std::shared_ptr(); + + new (&f->in.mem) std::shared_ptr(); + new (&f->out.mem) std::shared_ptr(); + + return 0; +} + +int fpga_destroy(struct node *n) +{ + struct fpga *f = (struct fpga *) n->_vd; + + using mbptr = MemoryAccessor; + using cptr = std::shared_ptr; + using nptr = std::shared_ptr; + using dptr = std::shared_ptr; + using sptr = std::string; + + f->cardName.~sptr(); + f->dmaName.~sptr(); + f->intfName.~sptr(); + + f->card.~cptr(); + + f->dma.~dptr(); + f->intf.~nptr(); + + f->in.mem.~mbptr(); + f->out.mem.~mbptr(); + + return 0; +} + +int fpga_parse(struct node *n, json_t *cfg) +{ + int ret; + struct fpga *f = (struct fpga *) n->_vd; + + json_error_t err; + + const char *card = nullptr; + const char *intf = nullptr; + const char *dma = nullptr; + int polling = f->polling; + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }", + "card", &card, + "interface", &intf, + "dma", &dma, + "coalesce", &f->coalesce, + "polling", &polling + ); + if (ret) + throw ConfigError(cfg, err, "node-config-fpga", "Failed to parse configuration of node {}", node_name(n)); + + if (card) + f->cardName = card; + + if (intf) + f->intfName = intf; + + if (dma) + f->dmaName = dma; + + f->polling = polling; // cast int to bool + + return 0; +} + +char * fpga_print(struct node *n) +{ + struct fpga *f = (struct fpga *) n->_vd; + + return strf("fpga=%s, dma=%s, if=%s, polling=%s, coalesce=%d", + f->card->name.c_str(), + f->dma->getInstanceName().c_str(), + f->polling ? "yes" : "no", + f->coalesce + ); +} + +int fpga_check(struct node *n) +{ + // struct fpga *f = (struct fpga *) n->_vd; + + return 0; +} + +int fpga_prepare(struct node *n) +{ + int ret; + struct fpga *f = (struct fpga *) n->_vd; + + auto it = f->cardName.empty() + ? cards.begin() + : std::find_if(cards.begin(), cards.end(), [f](const fpga::PCIeCard::Ptr &c) { + return c->name == f->cardName; + }); + + if (it == cards.end()) + throw ConfigError(json_object_get(n->cfg, "fpga"), "node-config-fpga-card", "Invalid FPGA card name: {}", f->cardName); + + f->card = *it; + + auto intf = f->intfName.empty() + ? f->card->lookupIp(fpga::Vlnv(FPGA_AURORA_VLNV)) + : f->card->lookupIp(f->intfName); + if (!intf) + throw ConfigError(n->cfg, "node-config-fpga-interface", "There is no interface IP with the name: {}", f->intfName); + + f->intf = std::dynamic_pointer_cast(intf); + if (!f->intf) + throw RuntimeError("The IP {} is not a interface", *intf); + + auto dma = f->dmaName.empty() + ? f->card->lookupIp(fpga::Vlnv(FPGA_DMA_VLNV)) + : f->card->lookupIp(f->dmaName); + if (!dma) + throw ConfigError(n->cfg, "node-config-fpga-dma", "There is no DMA IP with the name: {}", f->dmaName); + + f->dma = std::dynamic_pointer_cast(dma); + if (!f->dma) + throw RuntimeError("The IP {} is not a DMA controller", *dma); + + ret = f->intf->connect(*(f->dma), true); + if (ret) + throw RuntimeError("Failed to connect: {} -> {}", + *(f->intf), *(f->dma) + ); + + auto &alloc = HostRam::getAllocator(); + + f->in.mem = alloc.allocate(0x100 / sizeof(int32_t)); + f->out.mem = alloc.allocate(0x100 / sizeof(int32_t)); + + f->in.block = f->in.mem.getMemoryBlock(); + f->out.block = f->out.mem.getMemoryBlock(); + + f->dma->makeAccesibleFromVA(f->in.block); + f->dma->makeAccesibleFromVA(f->out.block); + + f->dma->dump(); + f->intf->dump(); + MemoryManager::get().getGraph().dump(); + + return 0; +} + +int fpga_start(struct node *n) +{ + // struct fpga *f = (struct fpga *) n->_vd; + + return 0; +} + +int fpga_stop(struct node *n) +{ + //struct fpga *f = (struct fpga *) n->_vd; + + return 0; +} + +int fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release) +{ + int read; + struct fpga *f = (struct fpga *) n->_vd; + struct sample *smp = smps[0]; + + assert(cnt == 1); + + f->dma->read(f->in.block, f->in.block.getSize()); // TODO: calc size + const size_t bytesRead = f->dma->readComplete(); + read = bytesRead / sizeof(int32_t); + + for (unsigned i = 0; i < MIN(read, smp->capacity); i++) + smp->data[i].i = f->in.mem[i]; + + return read; +} + +int fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release) +{ + int written; + struct fpga *f = (struct fpga *) n->_vd; + struct sample *smp = smps[0]; + + assert(cnt == 1); + + for (unsigned i = 0; i < smps[0]->length; i++) + f->out.mem[i] = smps[0]->data[i].i; + + bool state = f->dma->write(f->out.block, smp->length * sizeof(int32_t)); + if (!state) + return -1; + + written = 0; /* The number of samples written */ + + return written; +} + +int fpga_poll_fds(struct node *n, int fds[]) +{ + struct fpga *f = (struct fpga *) n->_vd; + + if (f->polling) + return 0; + else { + fds[0] = f->irqFd; + + return 1; /* The number of file descriptors which have been set in fds */ + } +} + +__attribute__((constructor(110))) +static void register_plugin() { + if (plugins.state == State::DESTROYED) + vlist_init(&plugins); + + p.name = "fpga"; + p.description = "Communicate with VILLASfpga Xilinx FPGA boards"; + p.type = PluginType::NODE; + p.node.instances.state = State::DESTROYED; + p.node.vectorize = 1; + p.node.size = sizeof(struct fpga); + p.node.type.start = fpga_type_start; + p.node.type.stop = fpga_type_stop; + p.node.init = fpga_init; + p.node.destroy = fpga_destroy; + p.node.prepare = fpga_prepare; + p.node.parse = fpga_parse; + p.node.print = fpga_print; + p.node.check = fpga_check; + p.node.start = fpga_start; + p.node.stop = fpga_stop; + p.node.read = fpga_read; + p.node.write = fpga_write; + p.node.poll_fds = fpga_poll_fds; + + vlist_init(&p.node.instances); + vlist_push(&plugins, &p); +} + +__attribute__((destructor(110))) +static void deregister_plugin() { + if (plugins.state != State::DESTROYED) + vlist_remove_all(&plugins, &p); +} diff --git a/lib/nodes/villas_fpga.cpp b/lib/nodes/villas_fpga.cpp deleted file mode 100644 index a6993d477..000000000 --- a/lib/nodes/villas_fpga.cpp +++ /dev/null @@ -1,365 +0,0 @@ -/** Communicate with VILLASfpga Xilinx FPGA boards - * - * @author Steffen Vogel - * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -/* Forward declartions */ -static struct plugin p; - -using namespace villas; -using namespace villas::node; -using namespace villas::utils; - -static struct pci pci; - -int villas_fpga_parse_config() -{ - /* Parse FPGA configuration */ - FILE* f = fopen(configFile.c_str(), "r"); - if (f == nullptr) { - logger->error("Cannot open config file: {}", configFile); - } - - json_t* json = json_loadf(f, 0, nullptr); - if (json == nullptr) { - logger->error("Cannot parse JSON config"); - fclose(f); - return -1; - } - - fclose(f); - - json_t* fpgas = json_object_get(json, "fpgas"); - if (fpgas == nullptr) { - logger->error("No section 'fpgas' found in config"); - exit(1); - } - - return 0; -} - -int villas_fpga_type_start(villas::node::SuperNode *sn) -{ - int ret; - - ret = pci_init(&pci); - if (ret) { - logger->error("Cannot initialize PCI subsystem"); - return ret; - } - - auto vfioContainer = villas::VfioContainer::create(); - - ret = villas_fpga_parse_config(); - if (ret) - return ret; - - // get the FPGA card plugin - villas::Plugin* plugin = villas::Plugin::lookup(villas::Plugin::Type::FpgaCard, ""); - if (plugin == nullptr) { - logger->error("No FPGA plugin found"); - exit(1); - } - - villas::fpga::PCIeCardFactory* fpgaCardPlugin = - dynamic_cast(plugin); - - // create all FPGA card instances using the corresponding plugin - auto cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer); - - return 0; -} - -int villas_fpga_type_stop() -{ - - return 0; -} - -int villas_fpga_init(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - return 0; -} - -int villas_fpga_destroy(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - return 0; -} - -int villas_fpga_parse(struct node *n, json_t *cfg) -{ - int ret; - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - json_error_t err; - - const char *fpga_name; - const char *dma_vlnv = ""; - const char *if_vlnv = ""; - - ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: s, s?: s }", - "fpga", &s->setting1, - "", &s->setting2 - ); - if (ret) - jerror(&err, "Failed to parse configuration of node %s", node_name(n)); - - return 0; -} - -char * villas_fpga_print(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - return strf("fpga=%s, dma=%s, if=%s", f->fpga_name, f->dma_vlnv, f->); -} - -int villas_fpga_check(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - if (s->setting1 > 100 || s->setting1 < 0) - return -1; - - if (!s->setting2 || strlen(s->setting2) > 10) - return -1; - - return 0; -} - -int villas_fpga_prepare(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - for (auto& fpgaCard : cards) { - if (fpgaCard->name == fpgaName) { - f->card = fpgaCard; - break; - } - } - - if (!card) - logger->error("FPGA card {} not found in config or not working", fpgaName); - - // deallocate JSON config - //json_decref(json); - - auto rtds = dynamic_cast - (card->lookupIp(fpga::Vlnv("acs.eonerc.rwth-aachen.de:user:rtds_axis:"))); - - //auto dma = dynamic_cast - // (card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:"))); - auto dma = dynamic_cast - (card->lookupIp("hier_0_axi_dma_axi_dma_1")); - - if (!rtds) { - logger->error("No RTDS interface found on FPGA"); - return 1; - } - - if (!dma) { - logger->error("No DMA found on FPGA "); - return 1; - } - - rtds->dump(); - - rtds->connect(rtds->getMasterPort(rtds->masterPort), - dma->getSlavePort(dma->s2mmPort)); - - dma->connect(dma->getMasterPort(dma->mm2sPort), - rtds->getSlavePort(rtds->slavePort)); - - auto &alloc = villas::HostRam::getAllocator(); - - auto mem = alloc.allocate(0x100 / sizeof(int32_t)); - auto block = mem.getMemoryBlock(); - - dma->makeAccesibleFromVA(block); - - // auto &mm = MemoryManager::get(); - // mm.getMemoryGraph().dump("graph.dot"); - - return 0; -} - -int villas_fpga_start(struct node *n) -{ - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - s->start_time = time_now(); - - return 0; -} - -int villas_fpga_stop(struct node *n) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; -} - -int villas_fpga_pause(struct node *n) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; -} - -int villas_fpga_resume(struct node *n) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; -} - -int villas_fpga_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release) -{ - int read; - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - struct timespec now; - - dma->read(block, block.getSize()); - const size_t bytesRead = dma->readComplete(); - const size_t valuesRead = bytesRead / sizeof(int32_t); - - for (size_t i = 0; i < valuesRead; i++) { - std::cerr << mem[i] << ";"; - } - std::cerr << std::endl; - - return read; -} - -int villas_fpga_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *release) -{ - int written; - struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - size_t memIdx = 0; - - for (unsigned i = 0; i < smps[0]->length; i++) - f->tx_mem[memIdx++] = smps[0]->data[i].i; - - bool state = dma->write(block, memIdx * sizeof(int32_t)); - if (!state) - logger->error("Failed to write to device"); - - written = 0; /* The number of samples written */ - - return written; -} - -int villas_fpga_reverse(struct node *n) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; -} - -int villas_fpga_poll_fds(struct node *n, int fds[]) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; /* The number of file descriptors which have been set in fds */ -} - -int villas_fpga_netem_fds(struct node *n, int fds[]) -{ - //struct villas_fpga *f = (struct villas_fpga *) n->_vd; - - /* TODO: Add implementation here. */ - - return 0; /* The number of file descriptors which have been set in fds */ -} - -__attribute__((constructor(110))) -static void register_plugin() { - if (plugins.state == State::DESTROYED) - vlist_init(&plugins); - - p.name = "fpga"; - p.description = "Communicate with VILLASfpga Xilinx FPGA boards"; - p.type = PluginType::NODE; - p.node.instances.state = State::DESTROYED; - p.node.vectorize = 0; - p.node.size = sizeof(struct villas_fpga); - p.node.type.start = villas_fpga_type_start; - p.node.type.stop = villas_fpga_type_stop; - p.node.init = villas_fpga_init; - p.node.destroy = villas_fpga_destroy; - p.node.prepare = villas_fpga_prepare; - p.node.parse = villas_fpga_parse; - p.node.print = villas_fpga_print; - p.node.check = villas_fpga_check; - p.node.start = villas_fpga_start; - p.node.stop = villas_fpga_stop; - p.node.pause = villas_fpga_pause; - p.node.resume = villas_fpga_resume; - p.node.read = villas_fpga_read; - p.node.write = villas_fpga_write; - p.node.reverse = villas_fpga_reverse; - p.node.poll_fds = villas_fpga_poll_fds; - p.node.netem_fds = villas_fpga_netem_fds; - - vlist_init(&p.node.instances); - vlist_push(&plugins, &p); -} - -__attribute__((destructor(110))) -static void deregister_plugin() { - if (plugins.state != State::DESTROYED) - vlist_remove_all(&plugins, &p); -}