diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index a1111d804..0f4c002f1 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -30,35 +30,67 @@ #pragma once + #include #include "common.h" #include "kernel/pci.h" #include "kernel/vfio.h" + +#include +#include + #include "plugin.hpp" +#include "config.h" + +#define PCI_FILTER_DEFAULT_FPGA { \ + .id = { \ + .vendor = FPGA_PCI_VID_XILINX, \ + .device = FPGA_PCI_PID_VFPGA \ + } \ +} + namespace villas { /* Forward declarations */ struct fpga_ip; struct vfio_container; +class FpgaCardPlugin; +class FpgaIp; -struct fpga_card { - char *name; /**< The name of the FPGA card */ +class FpgaCard { +public: - enum state state; /**< The state of this FPGA card. */ + friend FpgaCardPlugin; + + FpgaCard() : filter(PCI_FILTER_DEFAULT_FPGA) {} + + bool start(); + bool stop() { return true; } + bool check() { return true; } + bool reset() { return true; } + void dump() { } + + + using IpList = std::list; + IpList ips; ///< IPs located on this FPGA card + + bool do_reset; /**< Reset VILLASfpga during startup? */ + int affinity; /**< Affinity for MSI interrupts */ + + + std::string name; /**< The name of the FPGA card */ struct pci *pci; struct pci_device filter; /**< Filter for PCI device. */ - struct vfio_container *vfio_container; + ::vfio_container *vfio_container; struct vfio_device vfio_device; /**< VFIO device handle. */ - int do_reset; /**< Reset VILLASfpga during startup? */ - int affinity; /**< Affinity for MSI interrupts */ - struct list ips; /**< List of IP components on FPGA. */ +// struct list ips; /**< List of IP components on FPGA. */ char *map; /**< PCI BAR0 mapping for register access */ @@ -66,9 +98,25 @@ struct fpga_card { size_t dmalen; /* Some IP cores are special and referenced here */ - struct fpga_ip *intc; - struct fpga_ip *reset; - struct fpga_ip *sw; +// struct fpga_ip *intc; +// struct fpga_ip *reset; +// struct fpga_ip *sw; +}; + + + +class FpgaCardPlugin : public Plugin { +public: + + FpgaCardPlugin() : + Plugin("FPGA Card plugin") + { pluginType = Plugin::Type::FpgaCard; } + + static std::list + make(json_t *json, struct pci* pci, ::vfio_container* vc); + + static FpgaCard* + create(); }; /** Initialize FPGA card and its IP components. */ diff --git a/fpga/include/villas/fpga/ip.hpp b/fpga/include/villas/fpga/ip.hpp index 2f31562da..9b61414d9 100644 --- a/fpga/include/villas/fpga/ip.hpp +++ b/fpga/include/villas/fpga/ip.hpp @@ -41,7 +41,7 @@ #include "fpga/vlnv.hpp" #include "plugin.hpp" -#include "card.h" +#include "card.hpp" #include @@ -74,11 +74,7 @@ public: virtual bool start() { return true; } virtual bool stop() { return true; } virtual bool reset() { return true; } - virtual void dump() - { - info("IP %s: vlnv=%s baseaddr=%#jx, irq=%d, port=%d", - name.c_str(), vlnv.toString().c_str(), baseaddr, irq, port); - } + virtual void dump(); protected: uintptr_t @@ -90,12 +86,12 @@ protected: protected: // populated by FpgaIpFactory - struct fpga_card *card; /**< FPGA card this IP is instantiated on */ - std::string name; /**< Name defined in JSON config */ - FpgaVlnv vlnv; /**< VLNV defined in JSON config */ - uintptr_t baseaddr; /**< The baseadress of this FPGA IP component */ - int irq; /**< The interrupt number of the FPGA IP component. */ - int port; /**< The port of the AXI4-Stream switch to which this FPGA IP component is connected. */ + FpgaCard* card; /**< FPGA card this IP is instantiated on */ + std::string name; /**< Name defined in JSON config */ + FpgaVlnv vlnv; /**< VLNV defined in JSON config */ + uintptr_t baseaddr; /**< The baseadress of this FPGA IP component */ + int irq; /**< The interrupt number of the FPGA IP component. */ + int port; /**< The port of the AXI4-Stream switch to which this FPGA IP component is connected. */ }; @@ -106,14 +102,16 @@ public: { pluginType = Plugin::Type::FpgaIp; } /// Returns a running and checked FPGA IP - static FpgaIp* make(struct fpga_card* card, json_t *json, std::string name); + static FpgaIp* + make(FpgaCard* card, json_t *json, std::string name); private: /// Create a concrete IP instance virtual FpgaIp* create() = 0; /// Configure IP instance from JSON config - virtual bool configureJson(FpgaIp* ip, json_t *json) = 0; + virtual bool configureJson(FpgaIp* ip, json_t *json) + { return true; } virtual FpgaVlnv getCompatibleVlnv() const = 0; virtual std::string getName() const = 0; diff --git a/fpga/include/villas/plugin.hpp b/fpga/include/villas/plugin.hpp index 7ee980df8..26a220fd2 100644 --- a/fpga/include/villas/plugin.hpp +++ b/fpga/include/villas/plugin.hpp @@ -38,12 +38,13 @@ public: enum class Type { Unknown, FpgaIp, + FpgaCard, }; Plugin(std::string name); virtual ~Plugin(); - // each plugin is a singleton, so copying is not allowed + // copying a plugin doesn't make sense, so explicitly deny it Plugin(Plugin const&) = delete; void operator=(Plugin const&) = delete; @@ -61,20 +62,19 @@ public: static Plugin * lookup(Type type, std::string name); + /// Get all plugins of a given type. static std::list lookup(Type type); - // check if this makes sense! (no intermediate plugins) + // TODO: check if this makes sense! (no intermediate plugins) bool operator==(const Plugin& other) const; Type pluginType; - std::string name; std::string description; std::string path; void *handle; - enum state state; private: diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index f225cbd98..a2f69ab83 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -32,11 +32,157 @@ #include "kernel/pci.h" #include "kernel/vfio.h" -#include "fpga/ip.h" -#include "fpga/card.h" +#include + +#include "fpga/ip.hpp" +#include "fpga/card.hpp" namespace villas { +static FpgaCardPlugin +fpgaCardPlugin; + +std::list +FpgaCardPlugin::make(json_t *json, struct pci* pci, ::vfio_container* vc) +{ + std::list cards; + + const char *card_name; + json_t *json_card; + json_object_foreach(json, card_name, json_card) { + std::cout << "Found config for FPGA card " << card_name << std::endl; + + json_t* json_ips = nullptr; + const char* pci_slot = nullptr; + const char* pci_id = nullptr; + int do_reset = 0; + int affinity = 0; + + int ret = json_unpack(json_card, "{ s: o, s?: i, s?: b, s?: s, s?: s }", + "ips", &json_ips, + "affinity", &affinity, + "do_reset", &do_reset, + "slot", &pci_slot, + "id", &pci_id); + + if(ret != 0) { + std::cout << " Cannot parse JSON config" << std::endl; + continue; + } + + FpgaCard* card = create(); + + // populate generic properties + card->name = std::string(card_name); + card->pci = pci; + card->vfio_container = vc; + card->affinity = affinity; + card->do_reset = do_reset != 0; + + const char* error; + + if (pci_slot != nullptr and pci_device_parse_slot(&card->filter, pci_slot, &error) != 0) + std::cout << " Failed to parse PCI slot: " << error << std::endl + << " -> ignoring" << std::endl; + + if (pci_id != nullptr and pci_device_parse_id(&card->filter, pci_id, &error) != 0) + std::cout << " Failed to parse PCI ID: " << error << std::endl + << " -> ignoring" << std::endl;; + + + // TODO: currently fails, fix and remove comment +// if(not card->start()) { +// std::cout << " cannot start, destroying ..." << std::endl; +// delete card; +// continue; +// } + + const char *ip_name; + json_t *json_ip; + json_object_foreach(json_ips, ip_name, json_ip) { + std::cout << " Found IP " << ip_name << std::endl; + + FpgaIp* ip = FpgaIpFactory::make(card, json_ip, ip_name); + if(ip == nullptr) { + std::cout << " -> cannot initialize" << std::endl; + continue; + } + + card->ips.push_back(ip); + } + + if(not card->check()) { + std::cout << " checking failed, destroying ..." << std::endl; + delete card; + continue; + } + + cards.push_back(card); + } + + return cards; +} + +FpgaCard* +FpgaCardPlugin::create() +{ + return new FpgaCard; +} + + +bool FpgaCard::start() +{ + int ret; + struct pci_device *pdev; + + /* Search for FPGA card */ + pdev = pci_lookup_device(pci, &filter); + if (!pdev) + error("Failed to find PCI device"); + + /* Attach PCIe card to VFIO container */ + ret = ::vfio_pci_attach(&vfio_device, vfio_container, pdev); + if (ret) + error("Failed to attach VFIO device"); + + /* Map PCIe BAR */ + map = (char*) vfio_map_region(&vfio_device, VFIO_PCI_BAR0_REGION_INDEX); + if (map == MAP_FAILED) + serror("Failed to mmap() BAR0"); + + /* Enable memory access and PCI bus mastering for DMA */ + ret = vfio_pci_enable(&vfio_device); + if (ret) + serror("Failed to enable PCI device"); + + /* Reset system? */ + if (do_reset) { + /* Reset / detect PCI device */ + ret = vfio_pci_reset(&vfio_device); + if (ret) + serror("Failed to reset PCI device"); + + if(not reset()) { + std::cout << "Failed to reset FGPA card" << std::endl; + return false; + } + } + + /* Initialize IP cores */ +// for (size_t j = 0; j < list_length(&ips); j++) { +// struct fpga_ip *i = (struct fpga_ip *) list_at(&ips, j); + +// ret = fpga_ip_start(i); +// if (ret) +// error("Failed to initalize FPGA IP core: %s (%u)", i->name, ret); +// } + + return 0; +} + + +#if 0 + int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc) { assert(c->state == STATE_DESTROYED); @@ -321,4 +467,6 @@ int fpga_card_reset(struct fpga_card *c) return 0; } +#endif + } // namespace villas diff --git a/fpga/lib/ip.cpp b/fpga/lib/ip.cpp index d573ef03d..4e96228ae 100644 --- a/fpga/lib/ip.cpp +++ b/fpga/lib/ip.cpp @@ -27,9 +27,15 @@ #include "fpga/ip.hpp" #include +#include namespace villas { +void FpgaIp::dump() { + info("IP %s: vlnv=%s baseaddr=%#jx, irq=%d, port=%d", + name.c_str(), vlnv.toString().c_str(), baseaddr, irq, port); +} + FpgaIpFactory* FpgaIpFactory::lookup(const FpgaVlnv &vlnv) { @@ -43,29 +49,45 @@ FpgaIpFactory* FpgaIpFactory::lookup(const FpgaVlnv &vlnv) return nullptr; } -FpgaIp *FpgaIpFactory::make(fpga_card *card, json_t *json, std::string name) +FpgaIp *FpgaIpFactory::make(FpgaCard* card, json_t *json, std::string name) { - // extract VLNV from JSON + int ret; const char* vlnv_raw; - if(json_unpack(json, "{ s: s }", "vlnv", &vlnv_raw) != 0) - error("IP '%s' has no entry 'vlnv'", name.c_str()); + + // extract VLNV from JSON + ret = json_unpack(json, "{ s: s }", + "vlnv", &vlnv_raw); + if(ret != 0) { + std::cout << "IP " << name << " has no entry 'vlnv'" << std::endl; + return nullptr; + } + + // parse VLNV + FpgaVlnv vlnv(vlnv_raw); // find the appropriate factory that can create the specified VLNV // Note: // This is the magic part! Factories automatically register as a plugin // as soon as they are instantiated. If there are multiple candidates, // the first suitable factory will be used. - FpgaVlnv vlnv(vlnv_raw); FpgaIpFactory* fpgaIpFactory = lookup(vlnv); if(fpgaIpFactory == nullptr) { - error("No ip factory registered to handle VLNV '%s'", vlnv.toString().c_str()); - } else { - info("Using %s for IP %s", fpgaIpFactory->getName().c_str(), vlnv.toString().c_str()); + std::cout << "No IP plugin registered to handle VLNV " << vlnv << std::endl; + return nullptr; } - // create new IP instance + std::cout << "Using " << fpgaIpFactory->getName() << " for IP " << vlnv << std::endl; + + + // Create new IP instance. Since this function is virtual, it will construct + // the right, specialized type without knowing it here because we have + // already picked the right factory. FpgaIp* ip = fpgaIpFactory->create(); + if(ip == nullptr) { + std::cout << "Cannot create an instance of " << fpgaIpFactory->getName() << std::endl; + goto fail; + } // setup generic IP type properties ip->card = card; @@ -73,23 +95,34 @@ FpgaIp *FpgaIpFactory::make(fpga_card *card, json_t *json, std::string name) ip->vlnv = vlnv; // extract some optional properties - int ret = json_unpack(json, "{ s?: i, s?: i, s?: i }", - "baseaddr", &ip->baseaddr, - "irq", &ip->irq, - "port", &ip->port); - if(ret != 0) - error("Problem while parsing JSON"); + ret = json_unpack(json, "{ s?: i, s?: i, s?: i }", + "baseaddr", &ip->baseaddr, + "irq", &ip->irq, + "port", &ip->port); + if(ret != 0) { + std::cout << "Problem while parsing JSON" << std::endl; + goto fail; + } // IP-specific setup via JSON config fpgaIpFactory->configureJson(ip, json); - if(not ip->start()) - error("Cannot start IP"); + // TODO: currently fails, fix and remove comment +// if(not ip->start()) { +// std::cout << "Cannot start IP" << ip->name << std::endl; +// goto fail; +// } - if(not ip->check()) - error("Checking IP failed"); + if(not ip->check()) { + std::cout << "Checking IP " << ip->name << " failed" << std::endl; + goto fail; + } return ip; + +fail: + delete ip; + return nullptr; } } // namespace villas diff --git a/fpga/tests/main.cpp b/fpga/tests/main.cpp index 41549943f..900caee81 100644 --- a/fpga/tests/main.cpp +++ b/fpga/tests/main.cpp @@ -29,8 +29,11 @@ #include #include +#include +#include + #define FPGA_CARD "vc707" -#define TEST_CONFIG "/villas/etc/fpga.json" +#define TEST_CONFIG "../etc/fpga.json" #define TEST_LEN 0x1000 #define CPU_HZ 3392389000 @@ -48,6 +51,8 @@ static void init() FILE *f; json_error_t err; + villas::Plugin::dumpList(); + ret = pci_init(&pci); cr_assert_eq(ret, 0, "Failed to initialize PCI sub-system"); @@ -67,6 +72,16 @@ static void init() cr_assert_not_null(fpgas, "No section 'fpgas' found in config"); cr_assert(json_object_size(json) > 0, "No FPGAs defined in config"); + // get the FPGA card plugin + villas::Plugin* plugin = villas::Plugin::lookup(villas::Plugin::Type::FpgaCard, ""); + cr_assert_not_null(plugin, "No plugin for FPGA card found"); + villas::FpgaCardPlugin* fpgaCardPlugin = dynamic_cast(plugin); + + // create an FPGA card instance using the corresponding plugin +// villas::FpgaCard* fpgaCard = fpgaCardPlugin->make(json_); + + std::list fpgaCards = fpgaCardPlugin->make(fpgas, &pci, &vc); + json_t *json_card = json_object_get(fpgas, FPGA_CARD); cr_assert_not_null(json_card, "FPGA card " FPGA_CARD " not found");