/** Communicate with VILLASfpga Xilinx FPGA boards * * @author Steffen Vogel * @copyright 2014-2020, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * * VILLASnode * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . *********************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Forward declartions */ static struct vnode_type p; using namespace villas; using namespace villas::node; using namespace villas::utils; /* Global state */ static fpga::PCIeCard::List cards; static std::map dmaMap; static std::shared_ptr pciDevices; static std::shared_ptr vfioContainer; using namespace villas; // static std::shared_ptr // fpga_find_dma(const fpga::PCIeCard &card) // { // // for (auto &ip : card.ips) { // // } // return nullptr; // } int fpga_type_start(node::SuperNode *sn) { vfioContainer = kernel::vfio::Container::create(); pciDevices = std::make_shared(); // get the FPGA card plugin auto pcieCardPlugin = plugin::Registry::lookup("pcie"); if (!pcieCardPlugin) throw RuntimeError("No FPGA PCIe plugin found"); json_t *json = sn->getConfig(); json_t *fpgas = json_object_get(json, "fpgas"); if (!fpgas) throw ConfigError(json, "node-config-fpgas", "No section 'fpgas' found in config"); // create all FPGA card instances using the corresponding plugin auto pcieCards = pcieCardPlugin->make(fpgas, pciDevices, vfioContainer); cards.splice(cards.end(), pcieCards); return 0; } int fpga_type_stop() { vfioContainer.reset(); // TODO: is this the proper way? return 0; } int fpga_init(struct vnode *n) { struct fpga *f = (struct fpga *) n->_vd; f->coalesce = 0; f->irqFd = -1; f->polling = true; new (&f->cardName) std::string(); new (&f->dmaName) std::string(); new (&f->intfName) std::string(); new (&f->card) std::shared_ptr(); new (&f->dma) std::shared_ptr(); new (&f->intf) std::shared_ptr(); // TODO: fixme // new (&f->in.mem) std::shared_ptr(); // new (&f->out.mem) std::shared_ptr(); return 0; } int fpga_destroy(struct vnode *n) { struct fpga *f = (struct fpga *) n->_vd; using mbptr = MemoryAccessor; using cptr = std::shared_ptr; using nptr = std::shared_ptr; using dptr = std::shared_ptr; using sptr = std::string; f->cardName.~sptr(); f->dmaName.~sptr(); f->intfName.~sptr(); f->card.~cptr(); f->dma.~dptr(); f->intf.~nptr(); f->in.mem.~mbptr(); f->out.mem.~mbptr(); return 0; } int fpga_parse(struct vnode *n, json_t *json) { int ret; struct fpga *f = (struct fpga *) n->_vd; json_error_t err; const char *card = nullptr; const char *intf = nullptr; const char *dma = nullptr; int polling = f->polling; ret = json_unpack_ex(json, &err, 0, "{ s?: s, s?: s, s?: s, s?: i, s?: b }", "card", &card, "interface", &intf, "dma", &dma, "coalesce", &f->coalesce, "polling", &polling ); if (ret) throw ConfigError(json, err, "node-config-node-fpga"); if (card) f->cardName = card; if (intf) f->intfName = intf; if (dma) f->dmaName = dma; f->polling = polling; // cast int to bool return 0; } char * fpga_print(struct vnode *n) { struct fpga *f = (struct fpga *) n->_vd; return strf("fpga=%s, dma=%s, if=%s, polling=%s, coalesce=%d", f->card->name.c_str(), f->dma->getInstanceName().c_str(), f->polling ? "yes" : "no", f->coalesce ); } int fpga_check(struct vnode *n) { // struct fpga *f = (struct fpga *) n->_vd; return 0; } int fpga_prepare(struct vnode *n) { int ret; struct fpga *f = (struct fpga *) n->_vd; auto it = f->cardName.empty() ? cards.begin() : std::find_if(cards.begin(), cards.end(), [f](const fpga::PCIeCard::Ptr &c) { return c->name == f->cardName; }); if (it == cards.end()) throw ConfigError(json_object_get(n->config, "fpga"), "node-config-node-fpga-card", "Invalid FPGA card name: {}", f->cardName); f->card = *it; auto intf = f->intfName.empty() ? f->card->lookupIp(fpga::Vlnv(FPGA_AURORA_VLNV)) : f->card->lookupIp(f->intfName); if (!intf) throw ConfigError(n->config, "node-config-node-fpga-interface", "There is no interface IP with the name: {}", f->intfName); f->intf = std::dynamic_pointer_cast(intf); if (!f->intf) throw RuntimeError("The IP {} is not a interface", *intf); auto dma = f->dmaName.empty() ? f->card->lookupIp(fpga::Vlnv(FPGA_DMA_VLNV)) : f->card->lookupIp(f->dmaName); if (!dma) throw ConfigError(n->config, "node-config-node-fpga-dma", "There is no DMA IP with the name: {}", f->dmaName); f->dma = std::dynamic_pointer_cast(dma); if (!f->dma) throw RuntimeError("The IP {} is not a DMA controller", *dma); ret = f->intf->connect(*(f->dma), true); if (ret) throw RuntimeError("Failed to connect: {} -> {}", *(f->intf), *(f->dma) ); auto &alloc = HostRam::getAllocator(); f->in.mem = alloc.allocate(0x100 / sizeof(int32_t)); f->out.mem = alloc.allocate(0x100 / sizeof(int32_t)); f->in.block = f->in.mem.getMemoryBlock(); f->out.block = f->out.mem.getMemoryBlock(); f->dma->makeAccesibleFromVA(f->in.block); f->dma->makeAccesibleFromVA(f->out.block); f->dma->dump(); f->intf->dump(); MemoryManager::get().getGraph().dump(); return 0; } int fpga_start(struct vnode *n) { // struct fpga *f = (struct fpga *) n->_vd; return 0; } int fpga_stop(struct vnode *n) { //struct fpga *f = (struct fpga *) n->_vd; return 0; } int fpga_read(struct vnode *n, struct sample * const smps[], unsigned cnt) { unsigned read; struct fpga *f = (struct fpga *) n->_vd; struct sample *smp = smps[0]; assert(cnt == 1); f->dma->read(f->in.block, f->in.block.getSize()); // TODO: calc size const size_t bytesRead = f->dma->readComplete(); read = bytesRead / sizeof(int32_t); for (unsigned i = 0; i < MIN(read, smp->capacity); i++) smp->data[i].i = f->in.mem[i]; smp->signals = &n->in.signals; return read; } int fpga_write(struct vnode *n, struct sample * const smps[], unsigned cnt) { int written; struct fpga *f = (struct fpga *) n->_vd; struct sample *smp = smps[0]; assert(cnt == 1); for (unsigned i = 0; i < smps[0]->length; i++) f->out.mem[i] = smps[0]->data[i].i; bool state = f->dma->write(f->out.block, smp->length * sizeof(int32_t)); if (!state) return -1; written = 0; /* The number of samples written */ return written; } int fpga_poll_fds(struct vnode *n, int fds[]) { struct fpga *f = (struct fpga *) n->_vd; if (f->polling) return 0; else { fds[0] = f->irqFd; return 1; /* The number of file descriptors which have been set in fds */ } } __attribute__((constructor(110))) static void register_plugin() { p.name = "fpga"; p.description = "Communicate with VILLASfpga Xilinx FPGA boards"; p.vectorize = 1; p.size = sizeof(struct fpga); p.start = fpga_type_start; p.stop = fpga_type_stop; p.init = fpga_init; p.destroy = fpga_destroy; p.prepare = fpga_prepare; p.parse = fpga_parse; p.print = fpga_print; p.check = fpga_check; p.start = fpga_start; p.stop = fpga_stop; p.read = fpga_read; p.write = fpga_write; p.poll_fds = fpga_poll_fds; if (!node_types) node_types = new NodeTypeList(); node_types->push_back(&p); }