From 95e29f27060276d06d37ec9fb96333bf37a3414d Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Tue, 13 Feb 2018 12:22:50 +0100 Subject: [PATCH] memory-manager: allow for traversing address spaces Major rework of the memory manager. Adds a memory translation class to resolve addresses across address spaces and extents the memory manager in order to do so. --- fpga/include/villas/memory_manager.hpp | 199 +++++++++++++++++++------ fpga/lib/memory_manager.cpp | 126 +++++++++++++++- fpga/tests/graph.cpp | 15 +- 3 files changed, 286 insertions(+), 54 deletions(-) diff --git a/fpga/include/villas/memory_manager.hpp b/fpga/include/villas/memory_manager.hpp index 935f262b3..941fd43c0 100644 --- a/fpga/include/villas/memory_manager.hpp +++ b/fpga/include/villas/memory_manager.hpp @@ -2,96 +2,207 @@ #include #include +#include #include "log.hpp" #include "directed_graph.hpp" namespace villas { - -class Mapping : public graph::Edge { - friend class MemoryManager; - +/** + * @brief Translation between a local (master) to a foreign (slave) address space + * + * Memory translations can be chained together using the `+=` operator which is + * used internally by the MemoryManager to compute a translation through + * multiple hops (memory mappings). + */ +class MemoryTranslation { public: - // create mapping here (if needed) - Mapping() {} - // destroy mapping here (if needed) - virtual ~Mapping(); + /** + * @brief MemoryTranslation + * @param src Base address of local address space + * @param dst Base address of foreign address space + * @param size Size of "memory window" + */ + MemoryTranslation(uintptr_t src, uintptr_t dst, size_t size) : + src(src), dst(dst), size(size) {} + + uintptr_t + getLocalAddr(uintptr_t addrInForeignAddrSpace) const; + + uintptr_t + getForeignAddr(uintptr_t addrInLocalAddrSpace) const; friend std::ostream& - operator<< (std::ostream& stream, const Mapping& mapping) + operator<< (std::ostream& stream, const MemoryTranslation& translation) { - return stream << static_cast(mapping) << " = " - << std::hex - << "(src=0x" << mapping.src - << ", dest=0x" << mapping.dest - << ", size=0x" << mapping.size + return stream << std::hex + << "(src=0x" << translation.src + << ", dst=0x" << translation.dst + << ", size=0x" << translation.size << ")"; } -private: - uintptr_t src; - uintptr_t dest; - size_t size; -}; - -class AddressSpace : public graph::Vertex { - friend class MemoryManager; - -public: - friend std::ostream& - operator<< (std::ostream& stream, const AddressSpace& addrSpace) - { - return stream << static_cast(addrSpace) << " = " - << addrSpace.name; - } + /// Merge two MemoryTranslations together + MemoryTranslation& operator+=(const MemoryTranslation& other); private: - std::string name; + uintptr_t src; ///< Base address of local address space + uintptr_t dst; ///< Base address of foreign address space + size_t size; ///< Size of "memory window" }; -// is or has a graph +/** + * @brief Global memory manager to resolve addresses across address spaces + * + * Every entity in the system has to register its (master) address space and + * create mappings to other (slave) address spaces that it can access. A + * directed graph is then constructed which allows to traverse addresses spaces + * through multiple mappings and resolve addresses through this "tunnel" of + * memory mappings. + */ class MemoryManager { private: - // This is a singleton, so private constructor + // This is a singleton, so private constructor ... MemoryManager() : - memoryGraph("MemoryGraph") {} + memoryGraph("MemoryGraph"), + logger(loggerGetOrCreate("MemoryManager")) {} - // no copying or assigning + // ... and no copying or assigning MemoryManager(const MemoryManager&) = delete; MemoryManager& operator=(const MemoryManager&) = delete ; + /** + * @brief Custom edge in memory graph representing a memory mapping + * + * A memory mapping maps from one address space into another and can only be + * traversed in the forward direction which reflects the nature of real + * memory mappings. + * + * Implementation Notes: + * The member #src is the address in the "from" address space, where the + * destination address space is mapped. The member #dest is the address in + * the destination address space, where the mapping points to. Often, #dest + * will be zero for mappings to hardware, but consider the example when + * mapping FPGA to application memory: + * The application allocates a block 1kB at address + * 0x843001000 in its address space. The mapping would then have a #dest + * address of 0x843001000 and a #size of 1024. + */ + class Mapping : public graph::Edge { + public: + std::string name; ///< Human-readable name + uintptr_t src; ///< Base address in "from" address space + uintptr_t dest; ///< Base address in "to" address space + size_t size; ///< Size of the mapping + + friend std::ostream& + operator<< (std::ostream& stream, const Mapping& mapping) + { + return stream << static_cast(mapping) << " = " + << mapping.name + << std::hex + << "(src=0x" << mapping.src + << ", dest=0x" << mapping.dest + << ", size=0x" << mapping.size + << ")"; + } + + }; + + + /** + * @brief Custom vertex in memory graph representing an address space + * + * Since most information in the memory graph is stored in the edges (memory + * mappings), this is just a small extension to the default vertex. It only + * associates an additional string #name for human-readability. + */ + class AddressSpace : public graph::Vertex { + public: + std::string name; ///< Human-readable name + + friend std::ostream& + operator<< (std::ostream& stream, const AddressSpace& addrSpace) + { + return stream << static_cast(addrSpace) << " = " + << addrSpace.name; + } + }; + + /// Memory graph with custom edges and vertices for address resolution using MemoryGraph = graph::DirectedGraph; public: using AddressSpaceId = MemoryGraph::VertexIdentifier; using MappingId = MemoryGraph::EdgeIdentifier; - static MemoryManager& get(); + /// Get singleton instance + static MemoryManager& + get(); + AddressSpaceId + getProcessAddressSpace() + { return getOrCreateAddressSpace("villas-fpga"); } - AddressSpaceId createAddressSpace(std::string name); + AddressSpaceId + getOrCreateAddressSpace(std::string name); /// Create a default mapping - MappingId createMapping(uintptr_t src, uintptr_t dest, size_t size, - AddressSpaceId fromAddrSpace, - AddressSpaceId toAddrSpace); + MappingId + createMapping(uintptr_t src, uintptr_t dest, size_t size, std::string name, + AddressSpaceId fromAddrSpace, + AddressSpaceId toAddrSpace); /// Add a mapping /// /// Can be used to derive from Mapping in order to implement custom /// constructor/destructor. - MappingId addMapping(std::shared_ptr mapping, - AddressSpaceId fromAddrSpace, - AddressSpaceId toAddrSpace); + MappingId + addMapping(std::shared_ptr mapping, + AddressSpaceId fromAddrSpace, + AddressSpaceId toAddrSpace); - void dump() + + AddressSpaceId + findAddressSpace(std::string name); + + MemoryTranslation + getTranslation(AddressSpaceId fromAddrSpaceId, AddressSpaceId toAddrSpaceId); + + MemoryTranslation + getTranslationFromProcess(AddressSpaceId foreignAddrSpaceId) + { return getTranslation(getProcessAddressSpace(), foreignAddrSpaceId); } + + static std::string + getSlaveAddrSpaceName(std::string ipInstance, std::string memoryBlock) + { return ipInstance + "/" + memoryBlock; } + + void + dump() { memoryGraph.dump(); } + private: + /// Convert a Mapping to MemoryTranslation for calculations + static MemoryTranslation + getTranslationFromMapping(const Mapping& mapping) + { return MemoryTranslation(mapping.src, mapping.dest, mapping.size); } + + +private: + /// Directed graph that stores address spaces and memory mappings MemoryGraph memoryGraph; + + /// Cache mapping of names to address space ids for fast lookup + std::map addrSpaceLookup; + + /// Logger for universal access in this class + SpdLogger logger; + + /// Static pointer to global instance, because this is a singleton static MemoryManager* instance; }; diff --git a/fpga/lib/memory_manager.cpp b/fpga/lib/memory_manager.cpp index 59269fe04..55f7858b1 100644 --- a/fpga/lib/memory_manager.cpp +++ b/fpga/lib/memory_manager.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include "memory_manager.hpp" @@ -18,20 +20,32 @@ MemoryManager::get() } MemoryManager::AddressSpaceId -MemoryManager::createAddressSpace(std::string name) +MemoryManager::getOrCreateAddressSpace(std::string name) { - std::shared_ptr addrSpace(new AddressSpace); - addrSpace->name = name; + try { + // try fast lookup + return addrSpaceLookup.at(name); + } catch (const std::out_of_range&) { + // does not yet exist, create + std::shared_ptr addrSpace(new AddressSpace); + addrSpace->name = name; - return memoryGraph.addVertex(addrSpace); + // cache it for the next access + addrSpaceLookup[name] = memoryGraph.addVertex(addrSpace); + + return addrSpaceLookup[name]; + } } MemoryManager::MappingId MemoryManager::createMapping(uintptr_t src, uintptr_t dest, size_t size, + std::string name, MemoryManager::AddressSpaceId fromAddrSpace, MemoryManager::AddressSpaceId toAddrSpace) { std::shared_ptr mapping(new Mapping); + + mapping->name = name; mapping->src = src; mapping->dest = dest; mapping->size = size; @@ -47,11 +61,109 @@ MemoryManager::addMapping(std::shared_ptr mapping, return memoryGraph.addEdge(mapping, fromAddrSpace, toAddrSpace); } - -Mapping::~Mapping() +MemoryManager::AddressSpaceId +MemoryManager::findAddressSpace(std::string name) { - + return memoryGraph.findVertex( + [&](const std::shared_ptr& v) { + return v->name == name; + }); } +MemoryTranslation +MemoryManager::getTranslation(MemoryManager::AddressSpaceId fromAddrSpaceId, + MemoryManager::AddressSpaceId toAddrSpaceId) +{ + // find a path through the memory graph + MemoryGraph::Path path; + if(not memoryGraph.getPath(fromAddrSpaceId, toAddrSpaceId, path)) { + auto fromAddrSpace = memoryGraph.getVertex(fromAddrSpaceId); + auto toAddrSpace = memoryGraph.getVertex(toAddrSpaceId); + + logger->error("No translation found from ({}) to ({})", + *fromAddrSpace, *toAddrSpace); + + throw std::out_of_range("no translation found"); + } + + // start with an identity mapping + MemoryTranslation translation(0, 0, SIZE_MAX); + + // iterate through path and merge all mappings into a single translation + for(auto& mappingId : path) { + auto mapping = memoryGraph.getEdge(mappingId); + translation += getTranslationFromMapping(*mapping); + } + + return translation; +} + +uintptr_t +MemoryTranslation::getLocalAddr(uintptr_t addrInForeignAddrSpace) const +{ + assert(addrInForeignAddrSpace >= dst); + assert(addrInForeignAddrSpace < (dst + size)); + return src + addrInForeignAddrSpace - dst; +} + +uintptr_t +MemoryTranslation::getForeignAddr(uintptr_t addrInLocalAddrSpace) const +{ + assert(addrInLocalAddrSpace >= src); + assert(addrInLocalAddrSpace < (src + size)); + return dst + addrInLocalAddrSpace - src; +} + +MemoryTranslation& +MemoryTranslation::operator+=(const MemoryTranslation& other) +{ + auto logger = loggerGetOrCreate("MemoryTranslation"); + // set level to debug to enable debug output + logger->set_level(spdlog::level::info); + + const uintptr_t this_dst_high = this->dst + this->size; + const uintptr_t other_src_high = other.src + other.size; + + // make sure there is a common memory area + assert(other.src < this_dst_high); + assert(this->dst < other_src_high); + + const uintptr_t hi = std::max(this_dst_high, other_src_high); + const uintptr_t lo = std::min(this->dst, other.src); + + const uintptr_t diff_hi = (this_dst_high > other_src_high) + ? (this_dst_high - other_src_high) + : (other_src_high - this_dst_high); + + const uintptr_t diff_lo = (this->dst > other.src) + ? (this->dst - other.src) + : (other.src - this->dst); + + const size_t size = (hi - lo) - diff_hi - diff_lo; + + logger->debug("this->src: 0x{:x}", this->src); + logger->debug("this->dst: 0x{:x}", this->dst); + logger->debug("this->size: 0x{:x}", this->size); + logger->debug("other.src: 0x{:x}", other.src); + logger->debug("other.dst: 0x{:x}", other.dst); + logger->debug("other.size: 0x{:x}", other.size); + logger->debug("this_dst_high: 0x{:x}", this_dst_high); + logger->debug("other_src_high: 0x{:x}", other_src_high); + logger->debug("hi: 0x{:x}", hi); + logger->debug("lo: 0x{:x}", lo); + logger->debug("diff_hi: 0x{:x}", diff_hi); + logger->debug("diff_hi: 0x{:x}", diff_lo); + logger->debug("size: 0x{:x}", size); + + this->src += other.src; + this->dst += other.dst; + this->size = size; + + logger->debug("result src: 0x{:x}", this->src); + logger->debug("result dst: 0x{:x}", this->dst); + logger->debug("result size: 0x{:x}", this->size); + + return *this; +} } // namespace villas diff --git a/fpga/tests/graph.cpp b/fpga/tests/graph.cpp index 409d26537..4be542a5c 100644 --- a/fpga/tests/graph.cpp +++ b/fpga/tests/graph.cpp @@ -112,12 +112,21 @@ Test(graph, path, .description = "Find path") Test(graph, memory_manager, .description = "Global Memory Manager") { + auto logger = loggerGetOrCreate("unittest:mm"); auto& mm = villas::MemoryManager::get(); - auto dmaRegs = mm.createAddressSpace("DMA Registers"); - auto pcieBridge = mm.createAddressSpace("PCIe Bridge"); + logger->info("Create address spaces"); + auto dmaRegs = mm.getOrCreateAddressSpace("DMA Registers"); + auto pcieBridge = mm.getOrCreateAddressSpace("PCIe Bridge"); - mm.createMapping(0x1000, 0, 0x1000, dmaRegs, pcieBridge); + logger->info("Create a mapping"); + mm.createMapping(0x1000, 0, 0x1000, "Testmapping", dmaRegs, pcieBridge); + + logger->info("Find address space by name"); + auto vertex = mm.findAddressSpace("PCIe Bridge"); + logger->info(" found: {}", vertex); mm.dump(); + + logger->info(TXT_GREEN("Passed")); }