From 7d883089c2136ed573f89dfc17c474387dbe4c32 Mon Sep 17 00:00:00 2001 From: daniel-k Date: Wed, 6 Dec 2017 12:37:24 +0100 Subject: [PATCH] lib/card: copy C->C++ and just make it compile --- fpga/include/villas/fpga/card.h | 8 + fpga/include/villas/fpga/card.hpp | 102 ++++++++++ fpga/include/villas/kernel/pci.h | 8 + fpga/include/villas/list.h | 8 + fpga/include/villas/utils.h | 8 + fpga/lib/CMakeLists.txt | 1 + fpga/lib/card.cpp | 324 ++++++++++++++++++++++++++++++ 7 files changed, 459 insertions(+) create mode 100644 fpga/include/villas/fpga/card.hpp create mode 100644 fpga/lib/card.cpp diff --git a/fpga/include/villas/fpga/card.h b/fpga/include/villas/fpga/card.h index 89d19c0c1..f352fb081 100644 --- a/fpga/include/villas/fpga/card.h +++ b/fpga/include/villas/fpga/card.h @@ -35,6 +35,10 @@ #include "kernel/pci.h" #include "kernel/vfio.h" +#ifdef __cplusplus +extern "C" { +#endif + /* Forward declarations */ struct fpga_ip; struct vfio_container; @@ -92,4 +96,8 @@ void fpga_card_dump(struct fpga_card *c); /** Reset the FPGA to a known state */ int fpga_card_reset(struct fpga_card *c); +#ifdef __cplusplus +} +#endif + /** @} */ diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp new file mode 100644 index 000000000..a1111d804 --- /dev/null +++ b/fpga/include/villas/fpga/card.hpp @@ -0,0 +1,102 @@ +/** FPGA card + * + * This class represents a FPGA device. + * + * @file + * @author Steffen Vogel + * @author Daniel Krebs + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +/** @addtogroup fpga VILLASfpga + * @{ + */ + +#pragma once + +#include + +#include "common.h" +#include "kernel/pci.h" +#include "kernel/vfio.h" + +#include "plugin.hpp" + +namespace villas { + +/* Forward declarations */ +struct fpga_ip; +struct vfio_container; + +struct fpga_card { + char *name; /**< The name of the FPGA card */ + + enum state state; /**< The state of this FPGA card. */ + + struct pci *pci; + struct pci_device filter; /**< Filter for PCI device. */ + + struct vfio_container *vfio_container; + struct vfio_device vfio_device; /**< VFIO device handle. */ + + int do_reset; /**< Reset VILLASfpga during startup? */ + int affinity; /**< Affinity for MSI interrupts */ + + struct list ips; /**< List of IP components on FPGA. */ + + char *map; /**< PCI BAR0 mapping for register access */ + + size_t maplen; + size_t dmalen; + + /* Some IP cores are special and referenced here */ + struct fpga_ip *intc; + struct fpga_ip *reset; + struct fpga_ip *sw; +}; + +/** Initialize FPGA card and its IP components. */ +int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc); + +/** Parse configuration of FPGA card including IP cores from config. */ +int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name); + +int fpga_card_parse_list(struct list *l, json_t *cfg); + +/** Check if the FPGA card configuration is plausible. */ +int fpga_card_check(struct fpga_card *c); + +/** Start FPGA card. */ +int fpga_card_start(struct fpga_card *c); + +/** Stop FPGA card. */ +int fpga_card_stop(struct fpga_card *c); + +/** Destroy FPGA card. */ +int fpga_card_destroy(struct fpga_card *c); + +/** Dump details of FPGA card to stdout. */ +void fpga_card_dump(struct fpga_card *c); + +/** Reset the FPGA to a known state */ +int fpga_card_reset(struct fpga_card *c); + +} // namespace villas + +/** @} */ diff --git a/fpga/include/villas/kernel/pci.h b/fpga/include/villas/kernel/pci.h index b88da6bf4..d784cedf8 100644 --- a/fpga/include/villas/kernel/pci.h +++ b/fpga/include/villas/kernel/pci.h @@ -14,6 +14,10 @@ #define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) #define PCI_FUNC(devfn) ((devfn) & 0x07) +#ifdef __cplusplus +extern "C" { +#endif + struct pci_device { struct { int vendor; @@ -59,4 +63,8 @@ int pci_attach_driver(struct pci_device *d, const char *driver); /** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */ int pci_get_iommu_group(struct pci_device *d); +#ifdef __cplusplus +} +#endif + /** @} */ diff --git a/fpga/include/villas/list.h b/fpga/include/villas/list.h index f6ba29dae..b6d8c56e2 100644 --- a/fpga/include/villas/list.h +++ b/fpga/include/villas/list.h @@ -45,6 +45,10 @@ __attribute__((destructor(105))) static void UNIQUE(__dtor)() { \ #define list_first(list) list_at(list, 0) #define list_last(list) list_at(list, (list)->length-1) +#ifdef __cplusplus +extern "C" { +#endif + /** Callback to destroy list elements. * * @param data A pointer to the data which should be freed. @@ -110,3 +114,7 @@ void list_sort(struct list *l, cmp_cb_t cmp); /** Set single element in list */ int list_set(struct list *l, int index, void *value); + +#ifdef __cplusplus +} +#endif diff --git a/fpga/include/villas/utils.h b/fpga/include/villas/utils.h index b552cc52b..272afd6eb 100644 --- a/fpga/include/villas/utils.h +++ b/fpga/include/villas/utils.h @@ -32,6 +32,10 @@ #include "log.h" +#ifdef __cplusplus +extern "C" { +#endif + #ifdef __GNUC__ #define LIKELY(x) __builtin_expect((x),1) #define UNLIKELY(x) __builtin_expect((x),0) @@ -274,3 +278,7 @@ pid_t spawn(const char *name, char *const argv[]); /** Determines the string length as printed on the screen (ignores escable sequences). */ size_t strlenp(const char *str); + +#ifdef __cplusplus +} +#endif diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 30a98fddc..f80b17e8a 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES vlnv.cpp vlnv.c card.c + card.cpp ips/timer.c ips/model.c diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp new file mode 100644 index 000000000..f225cbd98 --- /dev/null +++ b/fpga/lib/card.cpp @@ -0,0 +1,324 @@ +/** FPGA card. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include +#include + +#include "config.h" +#include "log.h" +#include "log_config.h" +#include "list.h" +#include "utils.h" + +#include "kernel/pci.h" +#include "kernel/vfio.h" + +#include "fpga/ip.h" +#include "fpga/card.h" + +namespace villas { + +int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc) +{ + assert(c->state == STATE_DESTROYED); + + c->vfio_container = vc; + c->pci = pci; + + list_init(&c->ips); + + /* Default values */ + c->filter.id.vendor = FPGA_PCI_VID_XILINX; + c->filter.id.device = FPGA_PCI_PID_VFPGA; + + c->affinity = 0; + c->do_reset = 0; + + c->state = STATE_INITIALIZED; + + return 0; +} + +int fpga_card_parse(struct fpga_card *c, json_t *cfg, const char *name) +{ + int ret; + + json_t *json_ips; + json_t *json_slot = NULL; + json_t *json_id = NULL; + json_error_t err; + + c->name = strdup(name); + + ret = json_unpack_ex(cfg, &err, 0, "{ s?: i, s?: b, s?: o, s?: o, s: o }", + "affinity", &c->affinity, + "do_reset", &c->do_reset, + "slot", &json_slot, + "id", &json_id, + "ips", &json_ips + ); + if (ret) + jerror(&err, "Failed to parse FPGA vard configuration"); + + if (json_slot) { + const char *err, *slot; + + slot = json_string_value(json_slot); + if (slot) { + ret = pci_device_parse_slot(&c->filter, slot, &err); + if (ret) + error("Failed to parse PCI slot: %s", err); + } + else + error("PCI slot must be a string"); + } + + if (json_id) { + const char *err, *id; + + id = json_string_value(json_id); + if (id) { + ret = pci_device_parse_id(&c->filter, (char*) id, &err); + if (ret) + error("Failed to parse PCI id: %s", err); + } + else + error("PCI ID must be a string"); + } + + if (!json_is_object(json_ips)) + error("FPGA card IPs section must be an object"); + + const char *name_ip; + json_t *json_ip; + json_object_foreach(json_ips, name_ip, json_ip) { + const char *vlnv; + + struct fpga_ip_type *vt; + struct fpga_ip *ip = (struct fpga_ip *) alloc(sizeof(struct fpga_ip)); + + ip->card = c; + + ret = json_unpack_ex(json_ip, &err, 0, "{ s: s }", "vlnv", &vlnv); + if (ret) + error("Failed to parse FPGA IP '%s' of card '%s'", name_ip, name); + + vt = fpga_ip_type_lookup(vlnv); + if (!vt) + error("FPGA IP core VLNV identifier '%s' is invalid", vlnv); + + ret = fpga_ip_init(ip, vt); + if (ret) + error("Failed to initalize FPGA IP core"); + + ret = fpga_ip_parse(ip, json_ip, name_ip); + if (ret) + error("Failed to parse FPGA IP core"); + + list_push(&c->ips, ip); + } + + c->state = STATE_PARSED; + + return 0; +} + +int fpga_card_parse_list(struct list *cards, json_t *cfg) +{ + int ret; + + if (!json_is_object(cfg)) + error("FPGA card configuration section must be a JSON object"); + + const char *name; + json_t *json_fpga; + json_object_foreach(cfg, name, json_fpga) { + struct fpga_card *c = (struct fpga_card *) alloc(sizeof(struct fpga_card)); + + ret = ::fpga_card_parse(c, json_fpga, name); + if (ret) + error("Failed to parse FPGA card configuration"); + + list_push(cards, c); + } + + return 0; +} + +int fpga_card_start(struct fpga_card *c) +{ + int ret; + + struct pci_device *pdev; + + assert(c->state == STATE_INITIALIZED); + + /* Search for FPGA card */ + pdev = pci_lookup_device(c->pci, &c->filter); + if (!pdev) + error("Failed to find PCI device"); + + /* Attach PCIe card to VFIO container */ + ret = vfio_pci_attach(&c->vfio_device, c->vfio_container, pdev); + if (ret) + error("Failed to attach VFIO device"); + + /* Map PCIe BAR */ + c->map = (char*) vfio_map_region(&c->vfio_device, VFIO_PCI_BAR0_REGION_INDEX); + if (c->map == MAP_FAILED) + serror("Failed to mmap() BAR0"); + + /* Enable memory access and PCI bus mastering for DMA */ + ret = vfio_pci_enable(&c->vfio_device); + if (ret) + serror("Failed to enable PCI device"); + + /* Reset system? */ + if (c->do_reset) { + /* Reset / detect PCI device */ + ret = vfio_pci_reset(&c->vfio_device); + if (ret) + serror("Failed to reset PCI device"); + + ret = fpga_card_reset(c); + if (ret) + error("Failed to reset FGPA card"); + } + + /* Initialize IP cores */ + for (size_t j = 0; j < list_length(&c->ips); j++) { + struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); + + ret = fpga_ip_start(i); + if (ret) + error("Failed to initalize FPGA IP core: %s (%u)", i->name, ret); + } + + c->state = STATE_STARTED; + + return 0; +} + +int fpga_card_stop(struct fpga_card *c) +{ + int ret; + + assert(c->state == STATE_STOPPED); + + for (size_t j = 0; j < list_length(&c->ips); j++) { + struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); + + ret = fpga_ip_stop(i); + if (ret) + error("Failed to stop FPGA IP core: %s (%u)", i->name, ret); + } + + c->state = STATE_STOPPED; + + return 0; +} + +void fpga_card_dump(struct fpga_card *c) +{ + info("VILLASfpga card:"); + { INDENT + info("Slot: %04x:%02x:%02x.%d", c->vfio_device.pci_device->slot.domain, c->vfio_device.pci_device->slot.bus, c->vfio_device.pci_device->slot.device, c->vfio_device.pci_device->slot.function); + info("Vendor ID: %04x", c->vfio_device.pci_device->id.vendor); + info("Device ID: %04x", c->vfio_device.pci_device->id.device); + info("Class ID: %04x", c->vfio_device.pci_device->id.class_code); + + info("BAR0 mapped at %p", c->map); + + info("IP blocks:"); + for (size_t j = 0; j < list_length(&c->ips); j++) { INDENT + struct fpga_ip *i = (struct fpga_ip *) list_at(&c->ips, j); + + fpga_ip_dump(i); + } + } + + vfio_dump(c->vfio_device.group->container); +} + +int fpga_card_check(struct fpga_card *c) +{ + assert(c->state == STATE_PARSED); + + + /* Check FPGA configuration */ + struct fpga_vlnv vlnv_reset = { "xilinx.com", "ip", "axi_gpio", NULL }; + c->reset = fpga_vlnv_lookup(&c->ips, &vlnv_reset); + if (!c->reset) + error("FPGA is missing a reset controller"); + + struct fpga_vlnv vlnv_intc = { "acs.eonerc.rwth-aachen.de", "user", "axi_pcie_intc", NULL }; + c->intc = fpga_vlnv_lookup(&c->ips, &vlnv_intc); + if (!c->intc) + error("FPGA is missing a interrupt controller"); + + struct fpga_vlnv vlnv_sw = { "xilinx.com", "ip", "axis_interconnect", NULL }; + c->sw = fpga_vlnv_lookup(&c->ips, &vlnv_sw); + if (!c->sw) + warn("FPGA is missing an AXI4-Stream switch"); + + return 0; +} + +int fpga_card_destroy(struct fpga_card *c) +{ + list_destroy(&c->ips, (dtor_cb_t) fpga_ip_destroy, true); + + return 0; +} + +int fpga_card_reset(struct fpga_card *c) +{ + int ret; + char state[4096]; + + /* Save current state of PCI configuration space */ + ret = pread(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); + if (ret != sizeof(state)) + return -1; + + uint32_t *rst_reg = (uint32_t *) (c->map + c->reset->baseaddr); + + debug(3, "FPGA: reset"); + rst_reg[0] = 1; + + usleep(100000); + + /* Restore previous state of PCI configuration space */ + ret = pwrite(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); + if (ret != sizeof(state)) + return -1; + + /* After reset the value should be zero again */ + if (rst_reg[0]) + return -2; + + c->state = STATE_INITIALIZED; + + return 0; +} + +} // namespace villas