From 0d0aae090d5b668f1d3292d3ae063cb5c729adcf Mon Sep 17 00:00:00 2001 From: Niklas Eiling Date: Mon, 28 Nov 2022 11:05:46 +0100 Subject: [PATCH] add villas-fpga-xbar-select; improve DMA parameters in ips/dma --- fpga/include/villas/fpga/ips/dma.hpp | 4 +- fpga/lib/ips/dma.cpp | 3 +- fpga/src/CMakeLists.txt | 4 + fpga/src/villas-fpga-cat.cpp | 25 ++++-- fpga/src/villas-fpga-xbar-select.cpp | 118 +++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 fpga/src/villas-fpga-xbar-select.cpp diff --git a/fpga/include/villas/fpga/ips/dma.hpp b/fpga/include/villas/fpga/ips/dma.hpp index 8ee11247b..52acf4cbd 100644 --- a/fpga/include/villas/fpga/ips/dma.hpp +++ b/fpga/include/villas/fpga/ips/dma.hpp @@ -135,7 +135,7 @@ private: int delay = 0; // Coalesce is the number of messages/BDs to wait for before issuing an interrupt uint32_t writeCoalesce = 1; - uint32_t readCoalesce = 4; + uint32_t readCoalesce = 16; // (maximum) size of a single message on the read channel in bytes. // The message buffer/BD should have enough room for this many bytes. @@ -143,7 +143,7 @@ private: // When using SG: ringBdSize is the maximum number of BDs usable in the ring // Depending on alignment, the actual number of BDs usable can be smaller - static constexpr size_t requestedRingBdSize = 1024; + static constexpr size_t requestedRingBdSize = 2048; uint32_t actualRingBdSize = XAxiDma_BdRingCntCalc(XAXIDMA_BD_MINIMUM_ALIGNMENT, requestedRingBdSize); MemoryBlock::UniquePtr sgRingTx; MemoryBlock::UniquePtr sgRingRx; diff --git a/fpga/lib/ips/dma.cpp b/fpga/lib/ips/dma.cpp index c7bb73bbc..70e793189 100644 --- a/fpga/lib/ips/dma.cpp +++ b/fpga/lib/ips/dma.cpp @@ -66,7 +66,8 @@ bool Dma::init() // Map buffer descriptors if (hasScatterGather()) { if (actualRingBdSize < 2*readCoalesce || actualRingBdSize < 2*writeCoalesce) { - throw RuntimeError("Ring buffer size is too small for coalesce value"); + throw RuntimeError("Ring buffer size is too small for coalesce value {} < 2*{}", + actualRingBdSize, std::max(readCoalesce, writeCoalesce)); } setupScatterGather(); } diff --git a/fpga/src/CMakeLists.txt b/fpga/src/CMakeLists.txt index 8721a1d4f..bded7cb3b 100644 --- a/fpga/src/CMakeLists.txt +++ b/fpga/src/CMakeLists.txt @@ -22,6 +22,7 @@ add_executable(villas-fpga-pipe villas-fpga-pipe.cpp) add_executable(villas-fpga-cat villas-fpga-cat.cpp) +add_executable(villas-fpga-xbar-select villas-fpga-xbar-select.cpp) target_link_libraries(villas-fpga-pipe PUBLIC villas-fpga @@ -29,5 +30,8 @@ target_link_libraries(villas-fpga-pipe PUBLIC target_link_libraries(villas-fpga-cat PUBLIC villas-fpga ) +target_link_libraries(villas-fpga-xbar-select PUBLIC + villas-fpga +) add_executable(pcimem pcimem.c) diff --git a/fpga/src/villas-fpga-cat.cpp b/fpga/src/villas-fpga-cat.cpp index 5fa8ca2de..6b5917f12 100644 --- a/fpga/src/villas-fpga-cat.cpp +++ b/fpga/src/villas-fpga-cat.cpp @@ -104,30 +104,35 @@ int main(int argc, char* argv[]) dma->connectLoopback(); #endif auto &alloc = villas::HostRam::getAllocator(); - auto mem = alloc.allocate(0x100); - auto block = mem.getMemoryBlock(); + villas::MemoryAccessor mem[] = {alloc.allocate(0x200), alloc.allocate(0x200)}; + const villas::MemoryBlock block[] = {mem[0].getMemoryBlock(), mem[1].getMemoryBlock()}; - dma->makeAccesibleFromVA(block); + for (auto b : block) { + dma->makeAccesibleFromVA(b); + } auto &mm = MemoryManager::get(); mm.getGraph().dump("graph.dot"); + // Setup read transfer + dma->read(block[0], block[0].getSize()); + size_t cur = 0, next = 1; while (true) { - // Setup read transfer - dma->read(block, block.getSize()); - + dma->read(block[next], block[next].getSize()); auto bytesRead = dma->readComplete(); + // Setup read transfer + //auto valuesRead = bytesRead / sizeof(int32_t); - logger->info("Read {} bytes", bytesRead); + //logger->info("Read {} bytes", bytesRead); //for (size_t i = 0; i < valuesRead; i++) // std::cerr << std::hex << mem[i] << ";"; //std::cerr << std::endl; for (size_t i = 0; i*4 < bytesRead; i++) { - int32_t ival = mem[i]; + int32_t ival = mem[cur][i]; float fval = *((float*)(&ival)); - std::cerr << std::hex << ival << ","; + //std::cerr << std::hex << ival << ","; std::cerr << fval << std::endl; /*int64_t ival = (int64_t)(mem[1] & 0xFFFF) << 48 | (int64_t)(mem[1] & 0xFFFF0000) << 16 | @@ -138,6 +143,8 @@ int main(int argc, char* argv[]) bytesRead -= 8;*/ //logger->info("Read value: {}", dval); } + cur = next; + next = (next + 1) % (sizeof(mem)/sizeof(mem[0])); } } catch (const RuntimeError &e) { logger->error("Error: {}", e.what()); diff --git a/fpga/src/villas-fpga-xbar-select.cpp b/fpga/src/villas-fpga-xbar-select.cpp new file mode 100644 index 000000000..b5da3bb80 --- /dev/null +++ b/fpga/src/villas-fpga-xbar-select.cpp @@ -0,0 +1,118 @@ +/** Streaming data from STDIN/OUT to FPGA. + * + * @author Daniel Krebs + * @copyright 2017-2022, Steffen Vogel + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace villas; + +static std::shared_ptr pciDevices; +static auto logger = villas::logging.get("cat"); + +int main(int argc, char* argv[]) +{ + // Command Line Parser + CLI::App app{"VILLASfpga xbar select"}; + + try { + std::string configFile; + app.add_option("-c,--config", configFile, "Configuration file") + ->check(CLI::ExistingFile); + + std::string fpgaName = "vc707"; + app.add_option("--fpga", fpgaName, "Which FPGA to use"); + app.parse(argc, argv); + + // Logging setup + spdlog::set_level(spdlog::level::debug); + fpga::setupColorHandling(); + + if (configFile.empty()) { + logger->error("No configuration file provided/ Please use -c/--config argument"); + return 1; + } + + auto card = fpga::setupFpgaCard(configFile, fpgaName); + + std::vector aurora_channels; + for (int i = 0; i < 4; i++) { + auto name = fmt::format("aurora_8b10b_ch{}", i); + auto id = fpga::ip::IpIdentifier("xilinx.com:ip:aurora_8b10b:", name); + auto aurora = std::dynamic_pointer_cast(card->lookupIp(id)); + if (aurora == nullptr) { + logger->error("No Aurora interface found on FPGA"); + return 1; + } + + aurora_channels.push_back(aurora); + } + + auto dma = std::dynamic_pointer_cast + (card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:"))); + if (dma == nullptr) { + logger->error("No DMA found on FPGA "); + return 1; + } + + for (auto aurora : aurora_channels) + aurora->dump(); + + // Configure Crossbar switch + // connect DINO to RTDS + //aurora_channels[0]->connect(aurora_channels[0]->getDefaultMasterPort(), aurora_channels[2]->getDefaultSlavePort()); + //aurora_channels[2]->connect(aurora_channels[2]->getDefaultMasterPort(), aurora_channels[0]->getDefaultSlavePort()); + // connect DINO to OPAL-RT + //aurora_channels[1]->connect(aurora_channels[1]->getDefaultMasterPort(), aurora_channels[2]->getDefaultSlavePort()); + //aurora_channels[2]->connect(aurora_channels[2]->getDefaultMasterPort(), aurora_channels[1]->getDefaultSlavePort()); + // connect OPAL-RT to RTDS + aurora_channels[0]->connect(aurora_channels[0]->getDefaultMasterPort(), aurora_channels[1]->getDefaultSlavePort()); + aurora_channels[1]->connect(aurora_channels[1]->getDefaultMasterPort(), aurora_channels[0]->getDefaultSlavePort()); + + } catch (const RuntimeError &e) { + logger->error("Error: {}", e.what()); + return -1; + } catch (const CLI::ParseError &e) { + return app.exit(e); + } + + return 0; +}