/* FPGA pciecard. * * Author: Steffen Vogel * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, RWTH Aachen University * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include #include #include #include #include #include #include #include using namespace villas; using namespace villas::fpga; // Instantiate factory to register static PCIeCardFactory PCIeCardFactoryInstance; static const kernel::devices::PciDevice defaultFilter( (kernel::devices::Id(FPGA_PCI_VID_XILINX, FPGA_PCI_PID_VFPGA))); std::shared_ptr PCIeCardFactory::make(json_t *json_card, std::string card_name, std::shared_ptr vc, const std::filesystem::path &searchPath) { auto logger = getStaticLogger(); // make sure the vfio container has the required modules kernel::loadModule("vfio_pci"); kernel::loadModule("vfio_iommu_type1"); json_t *json_ips = nullptr; json_t *json_paths = nullptr; const char *pci_slot = nullptr; const char *pci_id = nullptr; int do_reset = 0; int affinity = 0; int polling = 0; json_error_t err; int ret = json_unpack_ex( json_card, &err, 0, "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o }", "ips", &json_ips, "affinity", &affinity, "do_reset", &do_reset, "slot", &pci_slot, "id", &pci_id, "polling", &polling, "paths", &json_paths); if (ret != 0) throw ConfigError(json_card, err, "", "Failed to parse card"); auto card = std::shared_ptr(make()); // Populate generic properties card->name = std::string(card_name); card->vfioContainer = vc; card->affinity = affinity; card->doReset = do_reset != 0; card->polling = (polling != 0); kernel::devices::PciDevice filter = defaultFilter; if (pci_id) filter.id = kernel::devices::Id(pci_id); if (pci_slot) filter.slot = kernel::devices::Slot(pci_slot); // Search for FPGA card card->pdev = kernel::devices::PciDeviceList::getInstance()->lookupDevice(filter); if (!card->pdev) { logger->warn("Failed to find PCI device"); return nullptr; } if (not card->init()) { logger->warn("Cannot start FPGA card {}", card_name); return nullptr; } // Load IPs from a separate json file if (!json_is_string(json_ips)) { logger->debug("FPGA IP cores config item is not a string."); throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP cores config item is not a string."); } if (!searchPath.empty()) { std::filesystem::path json_ips_path = searchPath / json_string_value(json_ips); logger->debug("searching for FPGA IP cors config at {}", json_ips_path.string()); json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr); } else { json_ips = json_load_file(json_string_value(json_ips), 0, nullptr); } if (json_ips == nullptr) { json_ips = json_load_file(json_string_value(json_ips), 0, nullptr); logger->debug("searching for FPGA IP cors config at {}", json_string_value(json_ips)); if (json_ips == nullptr) { throw ConfigError(json_ips, "node-config-fpga-ips", "Failed to find FPGA IP cores config"); } } if (not json_is_object(json_ips)) throw ConfigError(json_ips, "node-config-fpga-ips", "FPGA IP core list must be an object!"); ip::CoreFactory::make(card.get(), json_ips); if (card->ips.empty()) throw ConfigError(json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name); if (not card->check()) throw RuntimeError("Checking of FPGA card {} failed", card_name); // Additional static paths for AXI-Steram switch if (json_paths != nullptr) { if (not json_is_array(json_paths)) throw ConfigError(json_paths, err, "", "Switch path configuration must be an array"); size_t i; json_t *json_path; json_array_foreach(json_paths, i, json_path) { const char *from, *to; int reverse = 0; ret = json_unpack_ex(json_path, &err, 0, "{ s: s, s: s, s?: b }", "from", &from, "to", &to, "reverse", &reverse); if (ret != 0) throw ConfigError(json_path, err, "", "Cannot parse switch path config"); auto masterIpCore = card->lookupIp(from); if (!masterIpCore) throw ConfigError(json_path, "", "Unknown IP {}", from); auto slaveIpCore = card->lookupIp(to); if (!slaveIpCore) throw ConfigError(json_path, "", "Unknown IP {}", to); auto masterIpNode = std::dynamic_pointer_cast(masterIpCore); if (!masterIpNode) throw ConfigError(json_path, "", "IP {} is not a streaming node", from); auto slaveIpNode = std::dynamic_pointer_cast(slaveIpCore); if (!slaveIpNode) throw ConfigError(json_path, "", "IP {} is not a streaming node", to); if (not masterIpNode->connect(*slaveIpNode, reverse != 0)) throw ConfigError(json_path, "", "Failed to connect node {} to {}", from, to); } } // Deallocate JSON config json_decref(json_ips); return card; } PCIeCard::~PCIeCard() {} void PCIeCard::connectVFIOtoIps( std::list> configuredIps) { // TODO: implement } bool PCIeCard::init() { logger = getLogger(); logger->info("Initializing FPGA card {}", name); // Attach PCIe card to VFIO container vfioDevice = vfioContainer->attachDevice(*pdev); // Enable memory access and PCI bus mastering for DMA if (not vfioDevice->pciEnable()) { logger->error("Failed to enable PCI device"); return false; } // Reset system? if (doReset) { // Reset / detect PCI device if (not vfioDevice->pciHotReset()) { logger->error("Failed to reset PCI device"); return false; } if (not reset()) { logger->error("Failed to reset FGPA card"); return false; } } return true; }