From 89b5169a6ec4846f20a8794ac05db86ac4537c63 Mon Sep 17 00:00:00 2001
From: Daniel Krebs <github@daniel-krebs.net>
Date: Tue, 15 May 2018 17:41:40 +0200
Subject: [PATCH] ips/pcie: parse AXI/PCI BARs and create mappings to/from PCIe
 address space

This is used for translations that don't use VFIO which used to bridge
the PCIe address space by creating direct mappings from process VA to
the FPGA. When we want to communicate directly via PCIe without the
involvment of the CPU/VFIO, we need the proper translations that are
configured in the FPGA hardware.
---
 fpga/include/villas/fpga/ips/pcie.hpp | 15 +++++
 fpga/lib/ips/pcie.cpp                 | 96 +++++++++++++++++++++++++++
 2 files changed, 111 insertions(+)

diff --git a/fpga/include/villas/fpga/ips/pcie.hpp b/fpga/include/villas/fpga/ips/pcie.hpp
index 28b01c5aa..ebce36178 100644
--- a/fpga/include/villas/fpga/ips/pcie.hpp
+++ b/fpga/include/villas/fpga/ips/pcie.hpp
@@ -52,6 +52,19 @@ public:
 private:
 	static constexpr char axiInterface[] = "M_AXI";
 	static constexpr char pcieMemory[] = "BAR0";
+
+	struct AxiBar {
+		uintptr_t base;
+		size_t size;
+		uintptr_t translation;
+	};
+
+	struct PciBar {
+		uintptr_t translation;
+	};
+
+	std::map<std::string, AxiBar> axiToPcieTranslations;
+	std::map<std::string, PciBar> pcieToAxiTranslations;
 };
 
 
@@ -64,6 +77,8 @@ public:
 	getCompatibleVlnvString()
 	{ return "xilinx.com:ip:axi_pcie:"; }
 
+	bool configureJson(IpCore& ip, json_t *json_ip);
+
 	IpCore* create()
 	{ return new AxiPciExpressBridge; }
 
diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp
index 59174318b..3b273bbd8 100644
--- a/fpga/lib/ips/pcie.cpp
+++ b/fpga/lib/ips/pcie.cpp
@@ -63,6 +63,102 @@ AxiPciExpressBridge::init()
 	card->addrSpaceIdDeviceToHost =
 	        mm.getOrCreateAddressSpace(addrSpaceNameDeviceToHost);
 
+	auto pciAddrSpaceId = mm.getPciAddressSpace();
+
+	struct pci_region* pci_regions = nullptr;
+	size_t num_regions = pci_get_regions(card->pdev, &pci_regions);
+
+	for(size_t i = 0; i < num_regions; i++) {
+		const size_t region_size = pci_regions[i].end - pci_regions[i].start + 1;
+
+		char barName[] = "BARx";
+		barName[3] = '0' + pci_regions[i].num;
+		auto pciBar = pcieToAxiTranslations.at(barName);
+
+
+		logger->info("PCI-BAR{}: bus addr={:#x} size={:#x}",
+		             pci_regions[i].num, pci_regions[i].start, region_size);
+		logger->info("PCI-BAR{}: AXI translation offset {:#x}",
+		             i, pciBar.translation);
+
+		mm.createMapping(pci_regions[i].start, pciBar.translation, region_size,
+		                 std::string("PCI-") + barName,
+		                 pciAddrSpaceId, card->addrSpaceIdHostToDevice);
+
+	}
+
+	if(pci_regions != nullptr) {
+		logger->debug("freeing pci regions");
+		free(pci_regions);
+	}
+
+
+	for(auto& [barName, axiBar] : axiToPcieTranslations) {
+		logger->info("AXI-{}: bus addr={:#x} size={:#x}",
+		             barName, axiBar.base, axiBar.size);
+		logger->info("AXI-{}: PCI translation offset: {:#x}",
+		             barName, axiBar.translation);
+
+		auto barXAddrSpaceName = mm.getSlaveAddrSpaceName(getInstanceName(), barName);
+		auto barXAddrSpaceId = mm.getOrCreateAddressSpace(barXAddrSpaceName);
+
+		// base is already incorporated into mapping of each IP by Vivado, so
+		// the mapping src has to be 0
+		mm.createMapping(0, axiBar.translation, axiBar.size,
+		                 std::string("AXI-") + barName,
+		                 barXAddrSpaceId, pciAddrSpaceId);
+	}
+
+	return true;
+}
+
+bool
+AxiPciExpressBridgeFactory::configureJson(IpCore& ip, json_t* json_ip)
+{
+	auto logger = getLogger();
+	auto& pcie = reinterpret_cast<AxiPciExpressBridge&>(ip);
+
+	for(auto barType : std::list<std::string>{"axi_bars", "pcie_bars"}) {
+		json_t* json_bars = json_object_get(json_ip, barType.c_str());
+		if(not json_is_object(json_bars)) {
+			return false;
+		}
+
+		json_t* json_bar;
+		const char* bar_name;
+		json_object_foreach(json_bars, bar_name, json_bar) {
+			unsigned int translation;
+			int ret = json_unpack(json_bar, "{ s: i }", "translation", &translation);
+			if(ret != 0) {
+				logger->error("Cannot parse {}/{}", barType, bar_name);
+				return false;
+			}
+
+			if(barType == "axi_bars") {
+				json_int_t base, high, size;
+				int ret = json_unpack(json_bar, "{ s: I, s: I, s: I }",
+				                      "baseaddr", &base,
+				                      "highaddr", &high,
+				                      "size", &size);
+				if(ret != 0) {
+					logger->error("Cannot parse {}/{}", barType, bar_name);
+					return false;
+				}
+
+				pcie.axiToPcieTranslations[bar_name] = {
+				    .base = static_cast<uintptr_t>(base),
+				    .size = static_cast<size_t>(size),
+				    .translation = translation
+				};
+
+			} else {
+				pcie.pcieToAxiTranslations[bar_name] = {
+				    .translation = translation
+				};
+			}
+		}
+	}
+
 	return true;
 }