From 3f8a38adce3fe5e67923fe40f3e67f40f5aad874 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 28 Oct 2022 11:32:01 -0400 Subject: [PATCH] dma: first successful test with scatter gather (Aurora IPs still broken?) --- fpga/include/villas/fpga/ips/dma.hpp | 6 ++++ fpga/lib/ips/dma.cpp | 28 ++++++++++--------- fpga/src/villas-fpga-pipe.cpp | 42 ++++++++++++++++++++-------- 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/fpga/include/villas/fpga/ips/dma.hpp b/fpga/include/villas/fpga/ips/dma.hpp index d34a851c5..85f3ca1ed 100644 --- a/fpga/include/villas/fpga/ips/dma.hpp +++ b/fpga/include/villas/fpga/ips/dma.hpp @@ -1,6 +1,8 @@ /** DMA driver * * @author Daniel Krebs + * @author Steffen Vogel + * @author Niklas Eiling * @copyright 2018-2022, Institute for Automation of Complex Power Systems, EONERC * @license GNU General Public License (version 3) * @@ -123,6 +125,7 @@ private: } XAxiDma xDma; + XAxiDma_Config xConfig; bool hasSG; int delay; @@ -157,6 +160,9 @@ public: { return Vlnv("xilinx.com:ip:axi_dma:"); } + + virtual bool + configureJson(Core& ip, json_t* json) override; }; } /* namespace ip */ diff --git a/fpga/lib/ips/dma.cpp b/fpga/lib/ips/dma.cpp index 928e80fec..be5063de5 100644 --- a/fpga/lib/ips/dma.cpp +++ b/fpga/lib/ips/dma.cpp @@ -361,17 +361,20 @@ Dma::writeCompleteScatterGather() int ret = XST_FAILURE; // Poll until the one BD TX transaction is done. - while ((processedBds = XAxiDma_BdRingFromHw(txRing, XAXIDMA_ALL_BDS, &bd)) == 0) {} + // TODO: Use IRQs + while ((processedBds = XAxiDma_BdRingFromHw(txRing, 1, &bd)) == 0) {} if (bd == nullptr) throw RuntimeError("BD was null"); + auto bytesWritten = XAxiDma_BdGetLength(bd, txRing->MaxTransferLen); + // Free all processed TX BDs for future transmission. ret = XAxiDma_BdRingFree(txRing, processedBds, bd); if (ret != XST_SUCCESS) throw RuntimeError("Failed to free {} TX BDs {}", processedBds, ret); - return processedBds; + return bytesWritten; } size_t @@ -382,18 +385,21 @@ Dma::readCompleteScatterGather() auto rxRing = XAxiDma_GetRxRing(&xDma); int ret = XST_FAILURE; - // Wait until the data has been received by the Rx channel. - while ((processedBds = XAxiDma_BdRingFromHw(rxRing, XAXIDMA_ALL_BDS, &bd)) == 0) { } + // Wait until the data has been received by the RX channel. + // TODO: Use IRQs + while ((processedBds = XAxiDma_BdRingFromHw(rxRing, 1, &bd)) == 0) { } + + auto bytesRead = XAxiDma_BdGetActualLength(bd, rxRing->MaxTransferLen); if (bd == nullptr) - throw RuntimeError("BdPtr was null."); + throw RuntimeError("Bd was null."); // Free all processed RX BDs for future transmission. ret = XAxiDma_BdRingFree(rxRing, processedBds, bd); if (ret != XST_SUCCESS) throw RuntimeError("Failed to free {} TX BDs {}.", processedBds, ret); - return 0; + return bytesRead; } bool @@ -453,9 +459,7 @@ Dma::readSimple(void *buf, size_t len) return false; } - const bool dmaChannelHalted = - XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK; - + const bool dmaChannelHalted = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_SR_OFFSET) & XAXIDMA_HALTED_MASK; const bool deviceToDmaBusy = XAxiDma_Busy(&xDma, XAXIDMA_DEVICE_TO_DMA); // If the engine is doing a transfer, cannot submit @@ -463,13 +467,11 @@ Dma::readSimple(void *buf, size_t len) return false; // Set lower 32 bit of destination address - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_OFFSET, - LOWER_32_BITS(reinterpret_cast(buf))); + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_OFFSET, LOWER_32_BITS(reinterpret_cast(buf))); // If neccessary, set upper 32 bit of destination address if (xDma.AddrWidth > 32) - XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_MSB_OFFSET, - UPPER_32_BITS(reinterpret_cast(buf))); + XAxiDma_WriteReg(ring->ChanBase, XAXIDMA_DESTADDR_MSB_OFFSET, UPPER_32_BITS(reinterpret_cast(buf))); // Start DMA channel auto channelControl = XAxiDma_ReadReg(ring->ChanBase, XAXIDMA_CR_OFFSET); diff --git a/fpga/src/villas-fpga-pipe.cpp b/fpga/src/villas-fpga-pipe.cpp index 8096440b2..341180fa5 100644 --- a/fpga/src/villas-fpga-pipe.cpp +++ b/fpga/src/villas-fpga-pipe.cpp @@ -145,11 +145,17 @@ int main(int argc, char* argv[]) auto card = setupFpgaCard(configFile, fpgaName); - auto aurora = std::dynamic_pointer_cast - (card->lookupIp(fpga::Vlnv("xilinx.com:ip:aurora_8b10b:"))); - if (aurora == nullptr) { - logger->error("No Aurora interface found on FPGA"); - return 1; + std::vector aurora_channels; + for (int i = 0; i < 4; i++) { + auto name = fmt::format("aurora_8b10b_ch{}", i); + auto id = fpga::ip::IpIdentifier("xilinx.com:ip:aurora_8b10b:", name); + auto aurora = std::dynamic_pointer_cast(card->lookupIp(id)); + if (aurora == nullptr) { + logger->error("No Aurora interface found on FPGA"); + return 1; + } + + aurora_channels.push_back(aurora); } auto dma = std::dynamic_pointer_cast @@ -159,12 +165,16 @@ int main(int argc, char* argv[]) return 1; } - aurora->dump(); + for (auto aurora : aurora_channels) + aurora->dump(); // Configure Crossbar switch - aurora->connect(aurora->getDefaultMasterPort(), dma->getSlavePort(dma->s2mmPort)); - dma->connect(dma->getMasterPort(dma->mm2sPort), aurora->getDefaultSlavePort()); - +#if 1 + aurora_channels[2]->connect(aurora_channels[2]->getDefaultMasterPort(), dma->getDefaultSlavePort()); + dma->connect(dma->getDefaultMasterPort(), aurora_channels[3]->getDefaultSlavePort()); +#else + dma->connectLoopback(); +#endif auto &alloc = villas::HostRam::getAllocator(); auto mem = alloc.allocate(0x100); auto block = mem.getMemoryBlock(); @@ -175,6 +185,10 @@ int main(int argc, char* argv[]) mm.getGraph().dump("graph.dot"); while (true) { + // Setup read transfer + dma->read(block, block.getSize()); + + // Read values from stdin std::string line; std::getline(std::cin, line); auto values = villas::utils::tokenize(line, ";"); @@ -187,13 +201,17 @@ int main(int argc, char* argv[]) mem[i++] = number; } + // Initiate write transfer bool state = dma->write(block, i * sizeof(int32_t)); if (!state) logger->error("Failed to write to device"); - dma->read(block, block.getSize()); - const size_t bytesRead = dma->readComplete(); - const size_t valuesRead = bytesRead / sizeof(int32_t); + auto bytesWritten = dma->writeComplete(); + logger->info("Wrote {} bytes", bytesWritten); + + auto bytesRead = dma->readComplete(); + auto valuesRead = bytesRead / sizeof(int32_t); + logger->info("Read {} bytes", bytesRead); for (size_t i = 0; i < valuesRead; i++) std::cerr << mem[i] << ";";