From 6c225c8faed532c006bbff33524c88af47815fd4 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 15 Jun 2020 21:21:05 +0200 Subject: [PATCH] update VILLAScommon submodule --- fpga/common | 2 +- fpga/include/villas/fpga/card.hpp | 42 +++++++++++-------------- fpga/include/villas/fpga/node.hpp | 2 ++ fpga/lib/card.cpp | 52 +++++++++++++------------------ fpga/lib/ips/intc.cpp | 6 ++-- fpga/lib/ips/pcie.cpp | 26 +++++++--------- fpga/src/fpga.c | 5 +-- fpga/src/villas-fpga-pipe.cpp | 11 +++---- fpga/tests/fpga.cpp | 9 ++---- 9 files changed, 64 insertions(+), 91 deletions(-) diff --git a/fpga/common b/fpga/common index 952945fc4..3b5952a41 160000 --- a/fpga/common +++ b/fpga/common @@ -1 +1 @@ -Subproject commit 952945fc4bcdcdca0dfbe1389f811ceb7b5c5744 +Subproject commit 3b5952a413ba8f8c7731c6a0c8336e1f523884b8 diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index b555bc6e3..7efd21988 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -38,35 +38,32 @@ #include #include -#include +#include #include #include #include -#define PCI_FILTER_DEFAULT_FPGA { \ - .id = { \ - .vendor = FPGA_PCI_VID_XILINX, \ - .device = FPGA_PCI_PID_VFPGA, \ - .class_code = 0 \ - }, \ - .slot = { } \ -} - namespace villas { namespace fpga { - /* Forward declarations */ struct vfio_container; class PCIeCardFactory; -class PCIeCard { +class Card { public: + using Ptr = std::shared_ptr; + using List = std::list; + friend PCIeCardFactory; - PCIeCard() : filter(PCI_FILTER_DEFAULT_FPGA) {} +}; + +class PCIeCard : public Card { +public: + ~PCIeCard(); bool init(); @@ -95,20 +92,18 @@ private: public: // TODO: make this private ip::Core::List ips; ///< IPs located on this FPGA card - bool do_reset; /**< Reset VILLASfpga during startup? */ + bool doReset; /**< Reset VILLASfpga during startup? */ int affinity; /**< Affinity for MSI interrupts */ std::string name; /**< The name of the FPGA card */ - struct pci* pci; - struct pci_device filter; /**< Filter for PCI device. */ - struct pci_device* pdev; /**< PCI device handle */ + std::shared_ptr pdev; /**< PCI device handle */ /// The VFIO container that this card is part of - std::shared_ptr vfioContainer; + std::shared_ptr vfioContainer; /// The VFIO device that represents this card - VfioDevice* vfioDevice; + kernel::vfio::Device* vfioDevice; /// Slave address space ID to access the PCIe address space from the FPGA MemoryManager::AddressSpaceId addrSpaceIdDeviceToHost; @@ -125,16 +120,15 @@ protected: Logger logger; }; -using CardList = std::list>; - class PCIeCardFactory : public plugin::Plugin { public: - static CardList - make(json_t *json, struct pci* pci, std::shared_ptr vc); + static Card::List + make(json_t *json, std::shared_ptr pci, std::shared_ptr vc); static PCIeCard* - create(); + create() + { return new PCIeCard(); } static Logger getStaticLogger() diff --git a/fpga/include/villas/fpga/node.hpp b/fpga/include/villas/fpga/node.hpp index 6108e7034..70e83f304 100644 --- a/fpga/include/villas/fpga/node.hpp +++ b/fpga/include/villas/fpga/node.hpp @@ -88,6 +88,8 @@ public: class Node : public virtual Core { public: + using Ptr = std::shared_ptr; + friend class NodeFactory; struct StreamPort { diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index 4e3fbfceb..6bd6187d7 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -33,13 +33,16 @@ #include #include +using namespace villas; using namespace villas::fpga; // instantiate factory to register -static PCIeCardFactory PCIeCardFactory; +static PCIeCardFactory villas::fpga::PCIeCardFactory; + +static const kernel::pci::Device defaultFilter((kernel::pci::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA))); PCIeCard::List -PCIeCardFactory::make(json_t *json, kernel::pci::DeviceList *pci, std::shared_ptr vc) +PCIeCardFactory::make(json_t *json, std::shared_ptr pci, std::shared_ptr vc) { PCIeCard::List cards; auto logger = getStaticLogger(); @@ -71,19 +74,22 @@ PCIeCardFactory::make(json_t *json, kernel::pci::DeviceList *pci, std::shared_pt // populate generic properties card->name = std::string(card_name); - card->pci = pci; card->vfioContainer = std::move(vc); card->affinity = affinity; card->doReset = do_reset != 0; - const char* error; + kernel::pci::Device filter = defaultFilter; + + if (pci_id) + filter.id = kernel::pci::Id(pci_id); + if (pci_slot) + filter.slot = kernel::pci::Slot(pci_slot); - if (pci_slot != nullptr and pci_device_parse_slot(&card->filter, pci_slot, &error) != 0) { - logger->warn("Failed to parse PCI slot: {}", error); - } - - if (pci_id != nullptr and pci_device_parse_id(&card->filter, pci_id, &error) != 0) { - logger->warn("Failed to parse PCI ID: {}", error); + /* Search for FPGA card */ + card->pdev = pci->lookupDevice(filter); + if (!card->pdev) { + logger->warn("Failed to find PCI device"); + continue; } if (not card->init()) { @@ -107,14 +113,6 @@ PCIeCardFactory::make(json_t *json, kernel::pci::DeviceList *pci, std::shared_pt return cards; } - -PCIeCard* -PCIeCardFactory::create() -{ - return new fpga::PCIeCard; -} - - PCIeCard::~PCIeCard() { auto &mm = MemoryManager::get(); @@ -183,12 +181,11 @@ PCIeCard::mapMemoryBlock(const MemoryBlock &block) auto &mm = MemoryManager::get(); const auto &addrSpaceId = block.getAddrSpaceId(); - if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) { + if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) // block already mapped return true; - } else { + else logger->debug("Create VFIO mapping for {}", addrSpaceId); - } auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); @@ -221,16 +218,9 @@ PCIeCard::init() logger->info("Initializing FPGA card {}", name); - /* Search for FPGA card */ - pdev = pci_lookup_device(pci, &filter); - if (!pdev) { - logger->error("Failed to find PCI device"); - return false; - } - /* Attach PCIe card to VFIO container */ - kernel::vfio::Device &device = vfioContainer->attachDevice(pdev); - this->kernel::vfio::Device = &device; + kernel::vfio::Device &device = vfioContainer->attachDevice(*pdev); + this->vfioDevice = &device; /* Enable memory access and PCI bus mastering for DMA */ if (not device.pciEnable()) { @@ -241,7 +231,7 @@ PCIeCard::init() /* Reset system? */ if (doReset) { /* Reset / detect PCI device */ - if (not kernel::vfio::Device->pciHotReset()) { + if (not vfioDevice->pciHotReset()) { logger->error("Failed to reset PCI device"); return false; } diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index 098886e3e..ee877119c 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -38,7 +38,7 @@ static InterruptControllerFactory factory; InterruptController::~InterruptController() { - card->kernel::vfio::Device->pciMsiDeinit(this->efds); + card->vfioDevice->pciMsiDeinit(this->efds); } bool @@ -46,11 +46,11 @@ InterruptController::init() { const uintptr_t base = getBaseAddr(registerMemory); - num_irqs = card->kernel::vfio::Device->pciMsiInit(efds); + num_irqs = card->vfioDevice->pciMsiInit(efds); if (num_irqs < 0) return false; - if (not card->kernel::vfio::Device->pciMsiFind(nos)) { + if (not card->vfioDevice->pciMsiFind(nos)) { return false; } diff --git a/fpga/lib/ips/pcie.cpp b/fpga/lib/ips/pcie.cpp index cac9f5471..4e71d258f 100644 --- a/fpga/lib/ips/pcie.cpp +++ b/fpga/lib/ips/pcie.cpp @@ -43,14 +43,14 @@ AxiPciExpressBridge::init() card->addrSpaceIdHostToDevice = busMasterInterfaces.at(axiInterface); /* Map PCIe BAR0 via VFIO */ - const void* bar0_mapped = card->kernel::vfio::Device->regionMap(VFIO_PCI_BAR0_REGION_INDEX); + const void* bar0_mapped = card->vfioDevice->regionMap(VFIO_PCI_BAR0_REGION_INDEX); if (bar0_mapped == MAP_FAILED) { logger->error("Failed to mmap() BAR0"); return false; } // determine size of BAR0 region - const size_t bar0_size = card->kernel::vfio::Device->regionGetSize(VFIO_PCI_BAR0_REGION_INDEX); + const size_t bar0_size = card->vfioDevice->regionGetSize(VFIO_PCI_BAR0_REGION_INDEX); // create a mapping from process address space to the FPGA card via vfio mm.createMapping(reinterpret_cast(bar0_mapped), @@ -72,34 +72,28 @@ AxiPciExpressBridge::init() auto pciAddrSpaceId = mm.getPciAddressSpace(); - struct pci_region* pci_regions = nullptr; - size_t num_regions = pci_get_regions(card->pdev, &pci_regions); + auto regions = card->pdev->getRegions(); - for (size_t i = 0; i < num_regions; i++) { - const size_t region_size = pci_regions[i].end - pci_regions[i].start + 1; + int i = 0; + for (auto region : regions) { + const size_t region_size = region.end - region.start + 1; char barName[] = "BARx"; - barName[3] = '0' + pci_regions[i].num; + barName[3] = '0' + region.num; auto pciBar = pcieToAxiTranslations.at(barName); logger->info("PCI-BAR{}: bus addr={:#x} size={:#x}", - pci_regions[i].num, pci_regions[i].start, region_size); + region.num, region.start, region_size); logger->info("PCI-BAR{}: AXI translation offset {:#x}", i, pciBar.translation); - mm.createMapping(pci_regions[i].start, pciBar.translation, region_size, + mm.createMapping(region.start, pciBar.translation, region_size, std::string("PCI-") + barName, pciAddrSpaceId, card->addrSpaceIdHostToDevice); } - if (pci_regions != nullptr) { - logger->debug("freeing pci regions"); - free(pci_regions); - } - - for (auto& [barName, axiBar] : axiToPcieTranslations) { logger->info("AXI-{}: bus addr={:#x} size={:#x}", barName, axiBar.base, axiBar.size); @@ -114,6 +108,8 @@ AxiPciExpressBridge::init() mm.createMapping(0, axiBar.translation, axiBar.size, std::string("AXI-") + barName, barXAddrSpaceId, pciAddrSpaceId); + + i++; } return true; diff --git a/fpga/src/fpga.c b/fpga/src/fpga.c index 80e40ce5c..08e130352 100644 --- a/fpga/src/fpga.c +++ b/fpga/src/fpga.c @@ -54,7 +54,6 @@ int main(int argc, char *argv[]) struct list cards; struct vfio_container vc; - struct pci pci; struct fpga_card *card; /* Parse arguments */ @@ -88,9 +87,7 @@ check: if (optarg == endptr) json_error_t err; json_t *json; - ret = pci_init(&pci); - if (ret) - return -1; + auto pciDevices = std::make_shared(); ret = vfio_init(&vc); if (ret) diff --git a/fpga/src/villas-fpga-pipe.cpp b/fpga/src/villas-fpga-pipe.cpp index 418e47832..7a33135cf 100644 --- a/fpga/src/villas-fpga-pipe.cpp +++ b/fpga/src/villas-fpga-pipe.cpp @@ -43,7 +43,7 @@ using namespace villas; -static struct pci pci; +static std::shared_ptr pciDevices; static auto logger = villas::logging.get("streamer"); void setupColorHandling() @@ -66,12 +66,9 @@ void setupColorHandling() std::shared_ptr setupFpgaCard(const std::string &configFile, const std::string &fpgaName) { - if (pci_init(&pci) != 0) { - logger->error("Cannot initialize PCI"); - exit(1); - } + pciDevices = std::make_shared(); - auto vfioContainer = villas::VfioContainer::create(); + auto vfioContainer = kernel::vfio::Container::create(); /* Parse FPGA configuration */ FILE* f = fopen(configFile.c_str(), "r"); @@ -102,7 +99,7 @@ setupFpgaCard(const std::string &configFile, const std::string &fpgaName) } // create all FPGA card instances using the corresponding plugin - auto cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer); + auto cards = fpgaCardPlugin->make(fpgas, pciDevices, vfioContainer); for (auto &fpgaCard : cards) { if (fpgaCard->name == fpgaName) { diff --git a/fpga/tests/fpga.cpp b/fpga/tests/fpga.cpp index 97f2b6eb0..b71569056 100644 --- a/fpga/tests/fpga.cpp +++ b/fpga/tests/fpga.cpp @@ -40,14 +40,12 @@ using namespace villas; -static struct pci pci; +static std::shared_ptr pciDevices; FpgaState state; static void init() { - int ret; - FILE *f; json_error_t err; @@ -56,8 +54,7 @@ static void init() plugin::Registry::dumpList(); - ret = pci_init(&pci); - cr_assert_eq(ret, 0, "Failed to initialize PCI sub-system"); + pciDevices = std::make_shared(); auto vfioContainer = kernel::vfio::Container::create(); @@ -80,7 +77,7 @@ static void init() cr_assert_not_null(fpgaCardPlugin, "No plugin for FPGA card found"); // create all FPGA card instances using the corresponding plugin - state.cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer); + state.cards = fpgaCardPlugin->make(fpgas, pciDevices, vfioContainer); cr_assert(state.cards.size() != 0, "No FPGA cards found!");