diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index f8f9a0128..f9d78151a 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -38,6 +38,7 @@ #include "kernel/vfio.hpp" #include +#include #include #include "plugin.hpp" @@ -46,6 +47,7 @@ #include "config.h" #include "memory_manager.hpp" +#include "memory.hpp" #define PCI_FILTER_DEFAULT_FPGA { \ .id = { \ @@ -70,6 +72,7 @@ public: friend PCIeCardFactory; PCIeCard() : filter(PCI_FILTER_DEFAULT_FPGA) {} + ~PCIeCard(); bool init(); bool stop() { return true; } @@ -80,12 +83,19 @@ public: ip::IpCore* lookupIp(const std::string& name) const; ip::IpCore* lookupIp(const Vlnv& vlnv) const; + bool + mapMemoryBlock(const MemoryBlock& block); + +private: + /// Cache a set of already mapped memory blocks + std::set memoryBlocksMapped; + +public: // TODO: make this private ip::IpCoreList ips; ///< IPs located on this FPGA card bool do_reset; /**< Reset VILLASfpga during startup? */ int affinity; /**< Affinity for MSI interrupts */ - std::string name; /**< The name of the FPGA card */ struct pci *pci; @@ -97,17 +107,19 @@ public: /// The VFIO device that represents this card VfioDevice* vfioDevice; + /// Slave address space ID to access the PCIe address space from the FPGA + MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; + /// Address space identifier of the master address space of this FPGA card. /// This will be used for address resolution of all IPs on this card. - MemoryManager::AddressSpaceId addrSpaceId; - - size_t maplen; - size_t dmalen; + MemoryManager::AddressSpaceId addrSpaceIdHostToDevice; protected: SpdLogger getLogger() const { return loggerGetOrCreate(name); } + + SpdLogger logger; }; using CardList = std::list>; diff --git a/fpga/include/villas/fpga/ips/pcie.hpp b/fpga/include/villas/fpga/ips/pcie.hpp index 290a8c577..28b01c5aa 100644 --- a/fpga/include/villas/fpga/ips/pcie.hpp +++ b/fpga/include/villas/fpga/ips/pcie.hpp @@ -51,6 +51,7 @@ public: private: static constexpr char axiInterface[] = "M_AXI"; + static constexpr char pcieMemory[] = "BAR0"; }; diff --git a/fpga/include/villas/memory.hpp b/fpga/include/villas/memory.hpp new file mode 100644 index 000000000..85dae02ea --- /dev/null +++ b/fpga/include/villas/memory.hpp @@ -0,0 +1,122 @@ +#pragma once + +#include +#include + +#include "log.hpp" +#include "memory_manager.hpp" + +namespace villas { + +class MemoryBlock { +protected: + MemoryBlock(MemoryManager::AddressSpaceId addrSpaceId, size_t size) : + addrSpaceId(addrSpaceId), size(size) {} + +public: + MemoryManager::AddressSpaceId getAddrSpaceId() const + { return addrSpaceId; } + + size_t getSize() const + { return size; } + +private: + MemoryManager::AddressSpaceId addrSpaceId; + size_t size; +}; + + +class MemoryAllocator { +}; + + +class HostRam : public MemoryAllocator { +public: + + template + class MemoryBlockHostRam : public MemoryBlock { + friend class HostRam; + private: + MemoryBlockHostRam(void* addr, size_t size, MemoryManager::AddressSpaceId foreignAddrSpaceId) : + MemoryBlock(foreignAddrSpaceId, size), + addr(addr), + translation(MemoryManager::get().getTranslationFromProcess(foreignAddrSpaceId)) + {} + public: + using Type = T; + + MemoryBlockHostRam() = delete; + + T& operator*() { + return *reinterpret_cast(translation.getLocalAddr(0)); + } + + T& operator [](int idx) { + const size_t offset = sizeof(T) * idx; + return *reinterpret_cast(translation.getLocalAddr(offset)); + } + + T* operator &() const { + return reinterpret_cast(translation.getLocalAddr(0)); + } + + private: + // addr needed for freeing later + void* addr; + + // cached memory translation for fast access + MemoryTranslation translation; + }; + + template + static MemoryBlockHostRam + allocate(size_t num) + { + /* Align to next bigger page size chunk */ + size_t length = num * sizeof(T); + if (length & size_t(0xFFF)) { + length += size_t(0x1000); + length &= size_t(~0xFFF); + } + + void* const addr = HostRam::allocate(length); + if(addr == nullptr) { + throw std::bad_alloc(); + } + + auto& mm = MemoryManager::get(); + + // assemble name for this block + std::stringstream name; + name << std::showbase << std::hex << reinterpret_cast(addr); + + auto blockAddrSpaceId = mm.getProcessAddressSpaceMemoryBlock(name.str()); + + // create mapping from VA space of process to this new block + mm.createMapping(reinterpret_cast(addr), 0, length, + "VA", + mm.getProcessAddressSpace(), + blockAddrSpaceId); + + // create object and corresponding address space in memory manager + return MemoryBlockHostRam(addr, length, blockAddrSpaceId); + } + + template + static inline bool + free(const MemoryBlockHostRam& block) + { + // TODO: remove address space from memory manager + // TODO: how to prevent use after free? + return HostRam::free(block.addr, block.size); + } + +private: + static void* + allocate(size_t length, int flags = 0); + + static bool + free(void*, size_t length); +}; + +} // namespace villas diff --git a/fpga/include/villas/memory_manager.hpp b/fpga/include/villas/memory_manager.hpp index 04bf60bf6..fb3db33c2 100644 --- a/fpga/include/villas/memory_manager.hpp +++ b/fpga/include/villas/memory_manager.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "log.hpp" #include "directed_graph.hpp" @@ -34,6 +35,10 @@ public: uintptr_t getForeignAddr(uintptr_t addrInLocalAddrSpace) const; + size_t + getSize() const + { return size; } + friend std::ostream& operator<< (std::ostream& stream, const MemoryTranslation& translation) { @@ -87,9 +92,9 @@ private: * the destination address space, where the mapping points to. Often, #dest * will be zero for mappings to hardware, but consider the example when * mapping FPGA to application memory: - * The application allocates a block 1kB at address - * 0x843001000 in its address space. The mapping would then have a #dest - * address of 0x843001000 and a #size of 1024. + * The application allocates a block 1kB at address 0x843001000 in its + * address space. The mapping would then have a #dest address of 0x843001000 + * and a #size of 1024. */ class Mapping : public graph::Edge { public: @@ -147,6 +152,11 @@ public: getProcessAddressSpace() { return getOrCreateAddressSpace("villas-fpga"); } + AddressSpaceId + getProcessAddressSpaceMemoryBlock(const std::string& memoryBlock) + { return getOrCreateAddressSpace(getSlaveAddrSpaceName("villas-fpga", memoryBlock)); } + + AddressSpaceId getOrCreateAddressSpace(std::string name); diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 7150b61eb..34901a74e 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -23,6 +23,7 @@ set(SOURCES plugin.cpp utils.cpp memory_manager.cpp + memory.cpp ) include(FindPkgConfig) diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index e0e2ba456..fb7369823 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -109,13 +109,33 @@ PCIeCardFactory::make(json_t *json, struct pci* pci, std::shared_ptrdebug("Unmap block {} at IOVA {:#x} of size {:#x}", + mappedMemoryBlock, iova, size); + vfioContainer->memoryUnmap(iova, size); + } +} + + ip::IpCore* PCIeCard::lookupIp(const std::string& name) const { @@ -124,9 +144,11 @@ PCIeCard::lookupIp(const std::string& name) const return ip.get(); } } + return nullptr; } + ip::IpCore* PCIeCard::lookupIp(const Vlnv& vlnv) const { @@ -135,17 +157,58 @@ PCIeCard::lookupIp(const Vlnv& vlnv) const return ip.get(); } } + return nullptr; } +bool +PCIeCard::mapMemoryBlock(const MemoryBlock& block) +{ + auto& mm = MemoryManager::get(); + const auto& addrSpaceId = block.getAddrSpaceId(); + + if(memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) { + // block already mapped + return true; + } else { + logger->debug("Create VFIO mapping for {}", addrSpaceId); + } + + + auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); + uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); + uintptr_t iovaAddr = vfioContainer->memoryMap(processBaseAddr, + UINTPTR_MAX, + block.getSize()); + + if(iovaAddr == UINTPTR_MAX) { + logger->error("Cannot map memory at {:#x} of size {:#x}", + processBaseAddr, block.getSize()); + return false; + } + + + + mm.createMapping(iovaAddr, 0, block.getSize(), + "vfio", + this->addrSpaceIdDeviceToHost, + addrSpaceId); + + // remember that this block has already been mapped for later + memoryBlocksMapped.insert(addrSpaceId); + + return true; +} + + bool fpga::PCIeCard::init() { - int ret; struct pci_device *pdev; - auto logger = getLogger(); + auto& mm = MemoryManager::get(); + logger = getLogger(); logger->info("Initializing FPGA card {}", name); @@ -181,17 +244,18 @@ fpga::PCIeCard::init() /* Link mapped BAR0 to global memory graph */ // get the address space of the current application - auto villasAddrSpace = MemoryManager::get().getProcessAddressSpace(); + const auto villasAddrSpace = mm.getProcessAddressSpace(); + + // get the address space for the PCIe proxy we use with VFIO + const auto cardPCIeAddrSpaceName = mm.getMasterAddrSpaceName(name, "PCIe"); // create a new address space for this FPGA card - this->addrSpaceId = MemoryManager::get().getOrCreateAddressSpace(name); - + addrSpaceIdHostToDevice = mm.getOrCreateAddressSpace(cardPCIeAddrSpaceName); // create a mapping from our address space to the FPGA card via vfio - MemoryManager::get().createMapping(reinterpret_cast(bar0_mapped), + mm.createMapping(reinterpret_cast(bar0_mapped), 0, bar0_size, "VFIO_map", - villasAddrSpace, this->addrSpaceId); - + villasAddrSpace, addrSpaceIdHostToDevice); /* Reset system? */ diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index 5b02f1261..59174318b 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -38,6 +38,8 @@ static AxiPciExpressBridgeFactory factory; bool AxiPciExpressBridge::init() { + auto& mm = MemoryManager::get(); + // Throw an exception if the is no bus master interface and thus no // address space we can use for translation -> error const MemoryManager::AddressSpaceId myAddrSpaceid = @@ -47,7 +49,19 @@ AxiPciExpressBridge::init() // point to all other IPs in the FPGA, because Vivado will generate a // memory view for this bridge that can see all others. MemoryManager::get().createMapping(0x00, 0x00, SIZE_MAX, "PCIeBridge", - card->addrSpaceId, myAddrSpaceid); + card->addrSpaceIdHostToDevice, myAddrSpaceid); + + + /* Make PCIe (IOVA) address space available to FPGA via BAR0 */ + + // IPs that can access this address space will know it via their memory view + const auto addrSpaceNameDeviceToHost = + mm.getSlaveAddrSpaceName(getInstanceName(), pcieMemory); + + // save ID in card so we can create mappings later when needed (e.g. when + // allocating DMA memory in host RAM) + card->addrSpaceIdDeviceToHost = + mm.getOrCreateAddressSpace(addrSpaceNameDeviceToHost); return true; } diff --git a/fpga/lib/memory.cpp b/fpga/lib/memory.cpp new file mode 100644 index 000000000..f3d2802b4 --- /dev/null +++ b/fpga/lib/memory.cpp @@ -0,0 +1,24 @@ +#include +#include + +#include "memory.hpp" + +namespace villas { + +bool +HostRam::free(void* addr, size_t length) +{ + return munmap(addr, length) == 0; +} + + +void* +HostRam::allocate(size_t length, int flags) +{ + const int mmap_flags = flags | MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT; + const int mmap_protection = PROT_READ | PROT_WRITE; + + return mmap(nullptr, length, mmap_protection, mmap_flags, 0, 0); +} + +} // namespace villas