From 7dcdfaccd9793cad630390fbdd6d442869f0702a Mon Sep 17 00:00:00 2001
From: Daniel Krebs <github@daniel-krebs.net>
Date: Tue, 15 May 2018 17:53:06 +0200
Subject: [PATCH] ips/dma: let user deal with making memory accessible to DMA

It is probably too costly to do (and verify) it on every read
or write. Furthermore, the user knows better how to make a certain
memory available to the DMA.
---
 fpga/include/villas/fpga/ip.hpp      |  4 ++
 fpga/include/villas/fpga/ips/dma.hpp |  4 ++
 fpga/lib/ips/dma.cpp                 | 60 ++++++++++++++++++++++------
 3 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp
index f565aa4a2..1950a8ed2 100644
--- a/fpga/include/villas/fpga/ip.hpp
+++ b/fpga/include/villas/fpga/ip.hpp
@@ -193,6 +193,10 @@ protected:
 	InterruptController*
 	getInterruptController(const std::string& interruptName) const;
 
+	MemoryManager::AddressSpaceId
+	getMasterAddrSpaceByInterface(const std::string& masterInterfaceName) const
+	{ return busMasterInterfaces.at(masterInterfaceName); }
+
 protected:
 	struct IrqPort {
 		int num;
diff --git a/fpga/include/villas/fpga/ips/dma.hpp b/fpga/include/villas/fpga/ips/dma.hpp
index d8b609794..88530dbe3 100644
--- a/fpga/include/villas/fpga/ips/dma.hpp
+++ b/fpga/include/villas/fpga/ips/dma.hpp
@@ -57,6 +57,8 @@ public:
 
 	bool memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len);
 
+	bool makeAccesibleFromVA(const MemoryBlock& mem);
+
 	inline bool
 	hasScatterGather() const
 	{ return hasSG; }
@@ -72,6 +74,8 @@ private:
 	bool writeCompleteSimple();
 	bool readCompleteSimple();
 
+	bool isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface);
+
 private:
 	static constexpr char registerMemory[] = "Reg";
 
diff --git a/fpga/lib/ips/dma.cpp b/fpga/lib/ips/dma.cpp
index ed9f0a888..3d3bf0e70 100644
--- a/fpga/lib/ips/dma.cpp
+++ b/fpga/lib/ips/dma.cpp
@@ -159,17 +159,14 @@ Dma::memcpy(const MemoryBlock& src, const MemoryBlock& dst, size_t len)
 size_t
 Dma::write(const MemoryBlock& mem, size_t len)
 {
-	// make sure memory is reachable
-	if(not card->mapMemoryBlock(mem)) {
-		logger->error("Memory not accessible by DMA");
-		return 0;
-	}
-
 	auto& mm = MemoryManager::get();
+
+	// user has to make sure that memory is accessible, otherwise this will throw
 	auto translation = mm.getTranslation(busMasterInterfaces[mm2sInterface],
 	                                     mem.getAddrSpaceId());
 	const void* buf = reinterpret_cast<void*>(translation.getLocalAddr(0));
 
+	logger->debug("Write to address: {:p}", buf);
 	return hasScatterGather() ? writeSG(buf, len) : writeSimple(buf, len);
 }
 
@@ -177,17 +174,14 @@ Dma::write(const MemoryBlock& mem, size_t len)
 size_t
 Dma::read(const MemoryBlock& mem, size_t len)
 {
-	// make sure memory is reachable
-	if(not card->mapMemoryBlock(mem)) {
-		logger->error("Memory not accessible by DMA");
-		return 0;
-	}
-
 	auto& mm = MemoryManager::get();
+
+	// user has to make sure that memory is accessible, otherwise this will throw
 	auto translation = mm.getTranslation(busMasterInterfaces[s2mmInterface],
 	                                     mem.getAddrSpaceId());
 	void* buf = reinterpret_cast<void*>(translation.getLocalAddr(0));
 
+	logger->debug("Read from address: {:p}", buf);
 	return hasScatterGather() ? readSG(buf, len) : readSimple(buf, len);
 }
 
@@ -356,6 +350,48 @@ Dma::readCompleteSimple()
 }
 
 
+bool
+Dma::makeAccesibleFromVA(const MemoryBlock& mem)
+{
+	// only symmetric mapping supported currently
+	if(isMemoryBlockAccesible(mem, s2mmInterface) and
+	   isMemoryBlockAccesible(mem, mm2sInterface)) {
+		return true;
+	}
+
+	// try mapping via FPGA-card (VFIO)
+	if(not card->mapMemoryBlock(mem)) {
+		logger->error("Memory not accessible by DMA");
+		return false;
+	}
+
+	// sanity-check if mapping worked, this shouldn't be neccessary
+	if(not isMemoryBlockAccesible(mem, s2mmInterface) or
+	   not isMemoryBlockAccesible(mem, mm2sInterface)) {
+		logger->error("Mapping memory via card didn't work, but reported success?!");
+		return false;
+	}
+
+	return true;
+}
+
+
+bool
+Dma::isMemoryBlockAccesible(const MemoryBlock& mem, const std::string& interface)
+{
+	auto& mm = MemoryManager::get();
+
+	try {
+		mm.findPath(getMasterAddrSpaceByInterface(interface), mem.getAddrSpaceId());
+	} catch(const std::out_of_range&) {
+		// not (yet) accessible
+		return false;
+	}
+
+	return true;
+}
+
+
 } // namespace ip
 } // namespace fpga
 } // namespace villas