From 7dcdfaccd9793cad630390fbdd6d442869f0702a Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 15 May 2018 17:53:06 +0200 Subject: [PATCH] ips/dma: let user deal with making memory accessible to DMA It is probably too costly to do (and verify) it on every read or write. Furthermore, the user knows better how to make a certain memory available to the DMA. --- fpga/include/villas/fpga/ip.hpp | 4 ++ fpga/include/villas/fpga/ips/dma.hpp | 4 ++ fpga/lib/ips/dma.cpp | 60 ++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp index f565aa4a2..1950a8ed2 100644 --- a/fpga/include/villas/fpga/ip.hpp +++ b/fpga/include/villas/fpga/ip.hpp @@ -193,6 +193,10 @@ protected: InterruptController* getInterruptController(const std::string& interruptName) const; + MemoryManager::AddressSpaceId + getMasterAddrSpaceByInterface(const std::string& masterInterfaceName) const + { return busMasterInterfaces.at(masterInterfaceName); } + protected: struct IrqPort { int num; diff --git a/fpga/include/villas/fpga/ips/dma.hpp b/fpga/include/villas/fpga/ips/dma.hpp index d8b609794..88530dbe3 100644 --- a/fpga/include/villas/fpga/ips/dma.hpp +++ b/fpga/include/villas/fpga/ips/dma.hpp @@ -57,6 +57,8 @@ public: bool memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len); + bool makeAccesibleFromVA(const MemoryBlock& mem); + inline bool hasScatterGather() const { return hasSG; } @@ -72,6 +74,8 @@ private: bool writeCompleteSimple(); bool readCompleteSimple(); + bool isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface); + private: static constexpr char registerMemory[] = "Reg"; diff --git a/fpga/lib/ips/dma.cpp b/fpga/lib/ips/dma.cpp index ed9f0a888..3d3bf0e70 100644 --- a/fpga/lib/ips/dma.cpp +++ b/fpga/lib/ips/dma.cpp @@ -159,17 +159,14 @@ Dma::memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len) size_t Dma::write(const MemoryBlock& mem, size_t len) { - // make sure memory is reachable - if(not card->mapMemoryBlock(mem)) { - logger->error("Memory not accessible by DMA"); - return 0; - } - auto& mm = MemoryManager::get(); + + // user has to make sure that memory is accessible, otherwise this will throw auto translation = mm.getTranslation(busMasterInterfaces[mm2sInterface], mem.getAddrSpaceId()); const void* buf = reinterpret_cast(translation.getLocalAddr(0)); + logger->debug("Write to address: {:p}", buf); return hasScatterGather() ? writeSG(buf, len) : writeSimple(buf, len); } @@ -177,17 +174,14 @@ Dma::write(const MemoryBlock& mem, size_t len) size_t Dma::read(const MemoryBlock& mem, size_t len) { - // make sure memory is reachable - if(not card->mapMemoryBlock(mem)) { - logger->error("Memory not accessible by DMA"); - return 0; - } - auto& mm = MemoryManager::get(); + + // user has to make sure that memory is accessible, otherwise this will throw auto translation = mm.getTranslation(busMasterInterfaces[s2mmInterface], mem.getAddrSpaceId()); void* buf = reinterpret_cast(translation.getLocalAddr(0)); + logger->debug("Read from address: {:p}", buf); return hasScatterGather() ? readSG(buf, len) : readSimple(buf, len); } @@ -356,6 +350,48 @@ Dma::readCompleteSimple() } +bool +Dma::makeAccesibleFromVA(const MemoryBlock& mem) +{ + // only symmetric mapping supported currently + if(isMemoryBlockAccesible(mem, s2mmInterface) and + isMemoryBlockAccesible(mem, mm2sInterface)) { + return true; + } + + // try mapping via FPGA-card (VFIO) + if(not card->mapMemoryBlock(mem)) { + logger->error("Memory not accessible by DMA"); + return false; + } + + // sanity-check if mapping worked, this shouldn't be neccessary + if(not isMemoryBlockAccesible(mem, s2mmInterface) or + not isMemoryBlockAccesible(mem, mm2sInterface)) { + logger->error("Mapping memory via card didn't work, but reported success?!"); + return false; + } + + return true; +} + + +bool +Dma::isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface) +{ + auto& mm = MemoryManager::get(); + + try { + mm.findPath(getMasterAddrSpaceByInterface(interface), mem.getAddrSpaceId()); + } catch(const std::out_of_range&) { + // not (yet) accessible + return false; + } + + return true; +} + + } // namespace ip } // namespace fpga } // namespace villas