From b01a50184c3677a522e9b46505f1d04bf3ecf1bf Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Mon, 26 Mar 2018 15:39:26 +0200 Subject: [PATCH] kernel/vfio: port to C++ This commit is 1/2 of a series of patches and not working on its own. --- fpga/include/villas/fpga/card.hpp | 11 +- fpga/include/villas/kernel/vfio.h | 123 ----- fpga/include/villas/kernel/vfio.hpp | 153 ++++++ fpga/lib/CMakeLists.txt | 17 +- fpga/lib/card.cpp | 34 +- fpga/lib/ips/intc.cpp | 13 +- fpga/lib/kernel/vfio.c | 652 ----------------------- fpga/lib/kernel/vfio.cpp | 766 ++++++++++++++++++++++++++++ fpga/tests/fpga.cpp | 12 +- 9 files changed, 954 insertions(+), 827 deletions(-) delete mode 100644 fpga/include/villas/kernel/vfio.h create mode 100644 fpga/include/villas/kernel/vfio.hpp delete mode 100644 fpga/lib/kernel/vfio.c create mode 100644 fpga/lib/kernel/vfio.cpp diff --git a/fpga/include/villas/fpga/card.hpp b/fpga/include/villas/fpga/card.hpp index 4676e0b58..f8f9a0128 100644 --- a/fpga/include/villas/fpga/card.hpp +++ b/fpga/include/villas/fpga/card.hpp @@ -35,7 +35,7 @@ #include "common.h" #include "kernel/pci.h" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include #include @@ -91,8 +91,11 @@ public: struct pci *pci; struct pci_device filter; /**< Filter for PCI device. */ - ::vfio_container *vfio_container; - struct vfio_device vfio_device; /**< VFIO device handle. */ + /// The VFIO container that this card is part of + std::shared_ptr vfioContainer; + + /// The VFIO device that represents this card + VfioDevice* vfioDevice; /// Address space identifier of the master address space of this FPGA card. /// This will be used for address resolution of all IPs on this card. @@ -116,7 +119,7 @@ public: Plugin(Plugin::Type::FpgaCard, "FPGA Card plugin") {} static CardList - make(json_t *json, struct pci* pci, ::vfio_container* vc); + make(json_t *json, struct pci* pci, std::shared_ptr vc); static PCIeCard* create(); diff --git a/fpga/include/villas/kernel/vfio.h b/fpga/include/villas/kernel/vfio.h deleted file mode 100644 index 35a2465f9..000000000 --- a/fpga/include/villas/kernel/vfio.h +++ /dev/null @@ -1,123 +0,0 @@ -/** Virtual Function IO wrapper around kernel API - * - * @file - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - *********************************************************************************/ - -/** @addtogroup fpga Kernel @{ */ - -#pragma once - -#include -#include -#include - -#include -#include - -#include "list.h" - -#define VFIO_DEV(x) "/dev/vfio/" x - -#ifdef __cplusplus -extern "C" { -#endif - -/* Forward declarations */ -struct pci_device; - -struct vfio_group { - int fd; /**< VFIO group file descriptor */ - int index; /**< Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ */ - - struct vfio_group_status status; /**< Status of group */ - - struct list devices; - - struct vfio_container *container; /**< The VFIO container to which this group is belonging */ -}; - -struct vfio_device { - char *name; /**< Name of the device as listed under /sys/kernel/iommu_groups/[vfio_group::index]/devices/ */ - int fd; /**< VFIO device file descriptor */ - - struct vfio_device_info info; - struct vfio_irq_info *irqs; - struct vfio_region_info *regions; - - void **mappings; - - struct pci_device *pci_device; /**< libpci handle of the device */ - struct vfio_group *group; /**< The VFIO group this device belongs to */ -}; - -struct vfio_container { - int fd; - int version; - int extensions; - - uint64_t iova_next; /**< Next free IOVA address */ - - struct list groups; -}; - -/** Initialize a new VFIO container. */ -int vfio_init(struct vfio_container *c); - -/** Initialize a VFIO group and attach it to an existing VFIO container. */ -int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index); - -/** Initialize a VFIO device, lookup the VFIO group it belongs to, create the group if not already existing. */ -int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index); - -/** Initialie a VFIO-PCI device (uses vfio_device_attach() internally) */ -int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev); - -/** Hot resets a VFIO-PCI device */ -int vfio_pci_reset(struct vfio_device *d); - -int vfio_pci_msi_init(struct vfio_device *d, int efds[32]); - -int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]); - -int vfio_pci_msi_find(struct vfio_device *d, int nos[32]); - -/** Enable memory accesses and bus mastering for PCI device */ -int vfio_pci_enable(struct vfio_device *d); - -/** Reset a VFIO device */ -int vfio_device_reset(struct vfio_device *d); - -/** Release memory and close container */ -int vfio_destroy(struct vfio_container *c); - -/** Release memory of group */ -int vfio_group_destroy(struct vfio_group *g); - -/** Release memory of device */ -int vfio_device_destroy(struct vfio_device *g); - -/** Print a dump of all attached groups and their devices including regions and IRQs */ -void vfio_dump(struct vfio_container *c); - -/** Map a device memory region to the application address space (e.g. PCI BARs) */ -void * vfio_map_region(struct vfio_device *d, int idx); - -/** Get the size of a device memory region */ -size_t vfio_region_size(struct vfio_device *d, int idx); - -/** Map VM to an IOVA, which is accessible by devices in the container */ -int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len); - -/** Unmap DMA memory */ -int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len); - -/** munmap() a region which has been mapped by vfio_map_region() */ -int vfio_unmap_region(struct vfio_device *d, int idx); - -#ifdef __cplusplus -} -#endif - -/** @} */ diff --git a/fpga/include/villas/kernel/vfio.hpp b/fpga/include/villas/kernel/vfio.hpp new file mode 100644 index 000000000..9f794baf5 --- /dev/null +++ b/fpga/include/villas/kernel/vfio.hpp @@ -0,0 +1,153 @@ +/** Virtual Function IO wrapper around kernel API + * + * @file + * @author Steffen Vogel + * @author Daniel Krebs + * @copyright 2017, Steffen Vogel + * @copyright 2018, Daniel Krebs + *********************************************************************************/ + +/** @addtogroup fpga Kernel @{ */ + +#pragma once + +#include +#include +#include + +#include +#include + +#define VFIO_DEV(x) "/dev/vfio/" x + +/* Forward declarations */ +struct pci_device; + +namespace villas { + +class VfioContainer; +class VfioGroup; + + +class VfioDevice { + friend class VfioContainer; +public: + VfioDevice(const std::string& name, VfioGroup& group) : + name(name), group(group) {} + + ~VfioDevice(); + + bool reset(); + + /** Map a device memory region to the application address space (e.g. PCI BARs) */ + void* regionMap(size_t index); + + /** munmap() a region which has been mapped by vfio_map_region() */ + bool regionUnmap(size_t index); + + /** Get the size of a device memory region */ + size_t regionGetSize(size_t index); + + + /** Enable memory accesses and bus mastering for PCI device */ + bool pciEnable(); + + bool pciHotReset(); + int pciMsiInit(int efds[32]); + int pciMsiDeinit(int efds[32]); + bool pciMsiFind(int nos[32]); + + bool isVfioPciDevice() const; + +private: + /// Name of the device as listed under + /// /sys/kernel/iommu_groups/[vfio_group::index]/devices/ + std::string name; + + /// VFIO device file descriptor + int fd; + + struct vfio_device_info info; + + std::vector irqs; + std::vector regions; + std::vector mappings; + + /**< libpci handle of the device */ + const struct pci_device *pci_device; + + VfioGroup& group; /**< The VFIO group this device belongs to */ +}; + + + +class VfioGroup { + friend class VfioContainer; + friend VfioDevice; +private: + VfioGroup(int index) : fd(-1), index(index) {} +public: + ~VfioGroup(); + + static std::unique_ptr + attach(int containerFd, int groupIndex); + +private: + /// VFIO group file descriptor + int fd; + + /// Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ + int index; + + /// Status of group + struct vfio_group_status status; + + /// All devices owned by this group + std::list> devices; + + VfioContainer* container; /**< The VFIO container to which this group is belonging */ +}; + + +class VfioContainer { +private: + VfioContainer(); +public: + ~VfioContainer(); + + static std::shared_ptr + create(); + + void dump(); + + VfioDevice& attachDevice(const char *name, int groupIndex); + VfioDevice& attachDevice(const struct pci_device *pdev); + + /** + * @brief Map VM to an IOVA, which is accessible by devices in the container + * @param virt virtual address of memory + * @param phys IOVA where to map @p virt, -1 to use VFIO internal allocator + * @param length size of memory region in bytes + * @return IOVA address, UINTPTR_MAX on failure + */ + uintptr_t memoryMap(uintptr_t virt, uintptr_t phys, size_t length); + + /** munmap() a region which has been mapped by vfio_map_region() */ + bool memoryUnmap(uintptr_t phys, size_t length); + +private: + VfioGroup& getOrAttachGroup(int index); + +private: + int fd; + int version; + int extensions; + uint64_t iova_next; /**< Next free IOVA address */ + + /// All groups bound to this container + std::list> groups; +}; + +/** @} */ + +} // namespace villas diff --git a/fpga/lib/CMakeLists.txt b/fpga/lib/CMakeLists.txt index 50d16ecf5..7150b61eb 100644 --- a/fpga/lib/CMakeLists.txt +++ b/fpga/lib/CMakeLists.txt @@ -1,23 +1,9 @@ set(SOURCES - ip.c - vlnv.c - card.c - vlnv.cpp card.cpp ip.cpp ip_node.cpp - ips/timer.c - ips/model.c - ips/switch.c - ips/dft.c - ips/fifo.c - ips/dma.c - ips/intc.c - ips/gpio.c - ips/rtds_axis.c - ips/timer.cpp ips/switch.cpp ips/fifo.cpp @@ -26,9 +12,8 @@ set(SOURCES kernel/kernel.c kernel/pci.c - kernel/vfio.c + kernel/vfio.cpp - plugin.c utils.c list.c log.c diff --git a/fpga/lib/card.cpp b/fpga/lib/card.cpp index c4a184fc9..e0e2ba456 100644 --- a/fpga/lib/card.cpp +++ b/fpga/lib/card.cpp @@ -27,7 +27,7 @@ #include "log.hpp" #include "kernel/pci.h" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include "fpga/ip.hpp" #include "fpga/card.hpp" @@ -38,9 +38,8 @@ namespace fpga { // instantiate factory to register static PCIeCardFactory PCIeCardFactory; - CardList -fpga::PCIeCardFactory::make(json_t *json, struct pci* pci, ::vfio_container* vc) +PCIeCardFactory::make(json_t *json, struct pci* pci, std::shared_ptr vc) { CardList cards; auto logger = getStaticLogger(); @@ -73,7 +72,7 @@ fpga::PCIeCardFactory::make(json_t *json, struct pci* pci, ::vfio_container* vc) // populate generic properties card->name = std::string(card_name); card->pci = pci; - card->vfio_container = vc; + card->vfioContainer = std::move(vc); card->affinity = affinity; card->do_reset = do_reset != 0; @@ -158,19 +157,26 @@ fpga::PCIeCard::init() } /* Attach PCIe card to VFIO container */ - ret = ::vfio_pci_attach(&vfio_device, vfio_container, pdev); - if (ret) { - logger->error("Failed to attach VFIO device"); + VfioDevice& device = vfioContainer->attachDevice(pdev); + this->vfioDevice = &device; + + + /* Enable memory access and PCI bus mastering for DMA */ + if (not device.pciEnable()) { + logger->error("Failed to enable PCI device"); return false; } /* Map PCIe BAR */ - const void* bar0_mapped = vfio_map_region(&vfio_device, VFIO_PCI_BAR0_REGION_INDEX); + const void* bar0_mapped = vfioDevice->regionMap(VFIO_PCI_BAR0_REGION_INDEX); if (bar0_mapped == MAP_FAILED) { logger->error("Failed to mmap() BAR0"); return false; } + // determine size of BAR0 region + const size_t bar0_size = vfioDevice->regionGetSize(VFIO_PCI_BAR0_REGION_INDEX); + /* Link mapped BAR0 to global memory graph */ @@ -180,9 +186,6 @@ fpga::PCIeCard::init() // create a new address space for this FPGA card this->addrSpaceId = MemoryManager::get().getOrCreateAddressSpace(name); - // determine size of BAR0 region - const size_t bar0_size = vfio_region_size(&vfio_device, - VFIO_PCI_BAR0_REGION_INDEX); // create a mapping from our address space to the FPGA card via vfio MemoryManager::get().createMapping(reinterpret_cast(bar0_mapped), @@ -190,18 +193,11 @@ fpga::PCIeCard::init() villasAddrSpace, this->addrSpaceId); - /* Enable memory access and PCI bus mastering for DMA */ - ret = vfio_pci_enable(&vfio_device); - if (ret) { - logger->error("Failed to enable PCI device"); - return false; - } /* Reset system? */ if (do_reset) { /* Reset / detect PCI device */ - ret = vfio_pci_reset(&vfio_device); - if (ret) { + if(not vfioDevice->pciHotReset()) { logger->error("Failed to reset PCI device"); return false; } diff --git a/fpga/lib/ips/intc.cpp b/fpga/lib/ips/intc.cpp index fa8524ed5..3fbad436d 100644 --- a/fpga/lib/ips/intc.cpp +++ b/fpga/lib/ips/intc.cpp @@ -27,7 +27,7 @@ #include "log.h" #include "plugin.hpp" -#include "kernel/vfio.h" +#include "kernel/vfio.hpp" #include "kernel/kernel.h" #include "fpga/card.hpp" @@ -43,7 +43,7 @@ static InterruptControllerFactory factory; InterruptController::~InterruptController() { - vfio_pci_msi_deinit(&card->vfio_device , this->efds); + card->vfioDevice->pciMsiDeinit(this->efds); } bool @@ -51,12 +51,13 @@ InterruptController::init() { const uintptr_t base = getBaseAddr(registerMemory); - num_irqs = vfio_pci_msi_init(&card->vfio_device, efds); + num_irqs = card->vfioDevice->pciMsiInit(efds); if (num_irqs < 0) return false; - if(vfio_pci_msi_find(&card->vfio_device, nos) != 0) + if(not card->vfioDevice->pciMsiFind(nos)) { return false; + } /* For each IRQ */ for (int i = 0; i < num_irqs; i++) { @@ -69,8 +70,8 @@ InterruptController::init() // everything is fine break; case EACCES: - logger->warn("No permission to change affinity of VFIO-MSI interrupt. " - "This may degrade performance (increasing jitter)"); + logger->warn("No permission to change affinity of VFIO-MSI interrupt, " + "performance may be degraded!"); break; default: logger->error("Failed to change affinity of VFIO-MSI interrupt"); diff --git a/fpga/lib/kernel/vfio.c b/fpga/lib/kernel/vfio.c deleted file mode 100644 index 7103cc519..000000000 --- a/fpga/lib/kernel/vfio.c +++ /dev/null @@ -1,652 +0,0 @@ -/** Virtual Function IO wrapper around kernel API - * - * @author Steffen Vogel - * @copyright 2017, Steffen Vogel - * @license GNU General Public License (version 3) - * - * VILLASfpga - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#define _DEFAULT_SOURCE - -#include -#include -#include -#include -#include - -#include -#include - -#include "utils.h" -#include "log.h" - -#include "config.h" - -#include "kernel/kernel.h" -#include "kernel/vfio.h" -#include "kernel/pci.h" - -static const char *vfio_pci_region_names[] = { - "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX, - "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX, - "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX, - "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX, - "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX, - "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX, - "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX, - "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX, - "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX, -}; - -static const char *vfio_pci_irq_names[] = { - "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX, - "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX, - "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX, - "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX, - "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX, -}; - -/* Helpers */ -int vfio_get_iommu_name(int index, char *buf, size_t len) -{ - FILE *f; - char fn[256]; - - snprintf(fn, sizeof(fn), "/sys/kernel/iommu_groups/%d/name", index); - - f = fopen(fn, "r"); - if (!f) - return -1; - - int ret = fgets(buf, len, f) == buf ? 0 : -1; - - /* Remove trailing newline */ - char *c = strrchr(buf, '\n'); - if (c) - *c = 0; - - fclose(f); - - return ret; -} - -/* Destructors */ -int vfio_destroy(struct vfio_container *v) -{ - int ret; - - /* Release memory and close fds */ - list_destroy(&v->groups, (dtor_cb_t) vfio_group_destroy, true); - - /* Close container */ - ret = close(v->fd); - if (ret < 0) - return -1; - - debug(5, "VFIO: closed container: fd=%d", v->fd); - - return 0; -} - -int vfio_group_destroy(struct vfio_group *g) -{ - int ret; - - list_destroy(&g->devices, (dtor_cb_t) vfio_device_destroy, false); - - ret = ioctl(g->fd, VFIO_GROUP_UNSET_CONTAINER); - if (ret) - return ret; - - debug(5, "VFIO: released group from container: group=%u", g->index); - - ret = close(g->fd); - if (ret) - return ret; - - debug(5, "VFIO: closed group: group=%u, fd=%d", g->index, g->fd); - - return 0; -} - -int vfio_device_destroy(struct vfio_device *d) -{ - int ret; - - for (int i = 0; i < d->info.num_regions; i++) - vfio_unmap_region(d, i); - - ret = close(d->fd); - if (ret) - return ret; - - debug(5, "VFIO: closed device: name=%s, fd=%d", d->name, d->fd); - - free(d->mappings); - free(d->name); - - return 0; -} - -/* Constructors */ -int vfio_init(struct vfio_container *v) -{ - int ret; - - /* Initialize datastructures */ - memset(v, 0, sizeof(*v)); - - list_init(&v->groups); - - /* Load VFIO kernel module */ - if (kernel_module_load("vfio")) - error("Failed to load kernel module: %s", "vfio"); - - /* Open VFIO API */ - v->fd = open(VFIO_DEV("vfio"), O_RDWR); - if (v->fd < 0) - error("Failed to open VFIO container"); - - /* Check VFIO API version */ - v->version = ioctl(v->fd, VFIO_GET_API_VERSION); - if (v->version < 0 || v->version != VFIO_API_VERSION) - error("Failed to get VFIO version"); - - /* Check available VFIO extensions (IOMMU types) */ - v->extensions = 0; - for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { - ret = ioctl(v->fd, VFIO_CHECK_EXTENSION, i); - if (ret < 0) - error("Failed to get VFIO extensions"); - else if (ret > 0) - v->extensions |= (1 << i); - } - - return 0; -} - -int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index) -{ - int ret; - char buf[128]; - - g->index = index; - g->container = c; - - list_init(&g->devices); - - /* Open group fd */ - snprintf(buf, sizeof(buf), VFIO_DEV("%u"), g->index); - g->fd = open(buf, O_RDWR); - if (g->fd < 0) - serror("Failed to open VFIO group: %u", g->index); - - /* Claim group ownership */ - ret = ioctl(g->fd, VFIO_GROUP_SET_CONTAINER, &c->fd); - if (ret < 0) - serror("Failed to attach VFIO group to container"); - - /* Set IOMMU type */ - ret = ioctl(c->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); - if (ret < 0) - serror("Failed to set IOMMU type of container"); - - /* Check group viability and features */ - g->status.argsz = sizeof(g->status); - - ret = ioctl(g->fd, VFIO_GROUP_GET_STATUS, &g->status); - if (ret < 0) - serror("Failed to get VFIO group status"); - - if (!(g->status.flags & VFIO_GROUP_FLAGS_VIABLE)) - error("VFIO group is not available: bind all devices to the VFIO driver!"); - - list_push(&c->groups, g); - - return 0; -} - -int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev) -{ - char name[32]; - int ret; - - /* Load PCI bus driver for VFIO */ - if (kernel_module_load("vfio_pci")) - error("Failed to load kernel driver: %s", "vfio_pci"); - - /* Bind PCI card to vfio-pci driver if not already bound */ - ret = pci_get_driver(pdev, name, sizeof(name)); - if (ret || strcmp(name, "vfio-pci")) { - ret = pci_attach_driver(pdev, "vfio-pci"); - if (ret) - error("Failed to attach device to driver"); - } - - /* Get IOMMU group of device */ - int index = pci_get_iommu_group(pdev); - if (index < 0) - error("Failed to get IOMMU group of device"); - - /* VFIO device name consists of PCI BDF */ - snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, pdev->slot.bus, pdev->slot.device, pdev->slot.function); - - ret = vfio_device_attach(d, c, name, index); - if (ret < 0) - return ret; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) { - vfio_device_destroy(d); - return -1; - } - - d->pci_device = pdev; - - return 0; -} - -int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index) -{ - int ret; - struct vfio_group *g = NULL; - - /* Check if group already exists */ - for (size_t i = 0; i < list_length(&c->groups); i++) { - struct vfio_group *h = (struct vfio_group *) list_at(&c->groups, i); - - if (h->index == index) - g = h; - } - - if (!g) { - g = alloc(sizeof(struct vfio_group)); - - /* Aquire group ownership */ - ret = vfio_group_attach(g, c, index); - if (ret) - error("Failed to attach to IOMMU group: %u", index); - - info("Attached new group %u to VFIO container", g->index); - } - - d->group = g; - d->name = strdup(name); - - /* Open device fd */ - d->fd = ioctl(g->fd, VFIO_GROUP_GET_DEVICE_FD, d->name); - if (d->fd < 0) - serror("Failed to open VFIO device: %s", d->name); - - /* Get device info */ - d->info.argsz = sizeof(d->info); - - ret = ioctl(d->fd, VFIO_DEVICE_GET_INFO, &d->info); - if (ret < 0) - serror("Failed to get VFIO device info for: %s", d->name); - - d->irqs = alloc(d->info.num_irqs * sizeof(struct vfio_irq_info)); - d->regions = alloc(d->info.num_regions * sizeof(struct vfio_region_info)); - d->mappings = alloc(d->info.num_regions * sizeof(void *)); - - /* Get device regions */ - for (int i = 0; i < d->info.num_regions && i < 8; i++) { - struct vfio_region_info *region = &d->regions[i]; - - region->argsz = sizeof(*region); - region->index = i; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_REGION_INFO, region); - if (ret < 0) - serror("Failed to get regions of VFIO device: %s", d->name); - } - - /* Get device irqs */ - for (int i = 0; i < d->info.num_irqs; i++) { - struct vfio_irq_info *irq = &d->irqs[i]; - - irq->argsz = sizeof(*irq); - irq->index = i; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_IRQ_INFO, irq); - if (ret < 0) - serror("Failed to get IRQs of VFIO device: %s", d->name); - } - - list_push(&d->group->devices, d); - - return 0; -} - -int vfio_pci_reset(struct vfio_device *d) -{ - int ret; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - size_t reset_infolen = sizeof(struct vfio_pci_hot_reset_info) + sizeof(struct vfio_pci_dependent_device) * 64; - size_t resetlen = sizeof(struct vfio_pci_hot_reset) + sizeof(int32_t) * 1; - - struct vfio_pci_hot_reset_info *reset_info = (struct vfio_pci_hot_reset_info *) alloc(reset_infolen); - struct vfio_pci_hot_reset *reset = (struct vfio_pci_hot_reset *) alloc(resetlen); - - reset_info->argsz = reset_infolen; - reset->argsz = resetlen; - - ret = ioctl(d->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info); - if (ret) - return ret; - - debug(5, "VFIO: dependent devices for hot-reset:"); - for (int i = 0; i < reset_info->count; i++) { INDENT - struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; - debug(5, "%04x:%02x:%02x.%01x: iommu_group=%u", dd->segment, dd->bus, PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); - - if (dd->group_id != d->group->index) - return -3; - } - - reset->count = 1; - reset->group_fds[0] = d->group->fd; - - ret = ioctl(d->fd, VFIO_DEVICE_PCI_HOT_RESET, reset); - - free(reset_info); - - return ret; -} - -int vfio_pci_msi_find(struct vfio_device *d, int nos[32]) -{ - int ret, idx, irq; - char *end, *col, *last, line[1024], name[13]; - FILE *f; - - f = fopen("/proc/interrupts", "r"); - if (!f) - return -1; - - for (int i = 0; i < 32; i++) - nos[i] = -1; - - /* For each line in /proc/interruipts */ - while (fgets(line, sizeof(line), f)) { - col = strtok(line, " "); - - /* IRQ number is in first column */ - irq = strtol(col, &end, 10); - if (col == end) - continue; - - /* Find last column of line */ - do { - last = col; - } while ((col = strtok(NULL, " "))); - - - ret = sscanf(last, "vfio-msi[%u](%12[0-9:])", &idx, name); - if (ret == 2) { - if (strstr(d->name, name) == d->name) - nos[idx] = irq; - } - } - - fclose(f); - - return 0; -} - -int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]) -{ - int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; - struct vfio_irq_set *irq_set; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; - irq_set = alloc(irq_setlen); - - irq_set->argsz = irq_setlen; - irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; - irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; - irq_set->count = irq_count; - irq_set->start = 0; - - for (int i = 0; i < irq_count; i++) { - close(efds[i]); - efds[i] = -1; - } - - memcpy(irq_set->data, efds, sizeof(int) * irq_count); - - ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); - if (ret) - return -4; - - free(irq_set); - - return irq_count; -} - -int vfio_pci_msi_init(struct vfio_device *d, int efds[32]) -{ - int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count; - struct vfio_irq_set *irq_set; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - irq_setlen = sizeof(struct vfio_irq_set) + sizeof(int) * irq_count; - irq_set = alloc(irq_setlen); - - irq_set->argsz = irq_setlen; - irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; - irq_set->index = VFIO_PCI_MSI_IRQ_INDEX; - irq_set->start = 0; - irq_set->count = irq_count; - - /* Now set the new eventfds */ - for (int i = 0; i < irq_count; i++) { - efds[i] = eventfd(0, 0); - if (efds[i] < 0) - return -3; - } - memcpy(irq_set->data, efds, sizeof(int) * irq_count); - - ret = ioctl(d->fd, VFIO_DEVICE_SET_IRQS, irq_set); - if (ret) - return -4; - - free(irq_set); - - return irq_count; -} - -int vfio_pci_enable(struct vfio_device *d) -{ - int ret; - uint32_t reg; - off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; - - /* Check if this is really a vfio-pci device */ - if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) - return -1; - - ret = pread(d->fd, ®, sizeof(reg), offset); - if (ret != sizeof(reg)) - return -1; - - /* Enable memory access and PCI bus mastering which is required for DMA */ - reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; - - ret = pwrite(d->fd, ®, sizeof(reg), offset); - if (ret != sizeof(reg)) - return -1; - - return 0; -} - -int vfio_device_reset(struct vfio_device *d) -{ - if (d->info.flags & VFIO_DEVICE_FLAGS_RESET) - return ioctl(d->fd, VFIO_DEVICE_RESET); - else - return -1; /* not supported by this device */ -} - -void vfio_dump(struct vfio_container *v) -{ - info("VFIO Version: %u", v->version); - info("VFIO Extensions: %#x", v->extensions); - - for (size_t i = 0; i < list_length(&v->groups); i++) { - struct vfio_group *g = (struct vfio_group *) list_at(&v->groups, i); - - info("VFIO Group %u, viable=%u, container=%d", g->index, - (g->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0, - (g->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0 - ); - - - for (size_t i = 0; i < list_length(&g->devices); i++) { INDENT - struct vfio_device *d = (struct vfio_device *) list_at(&g->devices, i); - - info("Device %s: regions=%u, irqs=%u, flags=%#x", d->name, - d->info.num_regions, - d->info.num_irqs, - d->info.flags - ); - - for (int i = 0; i < d->info.num_regions && i < 8; i++) { INDENT - struct vfio_region_info *region = &d->regions[i]; - - if (region->size > 0) - info("Region %u %s: size=%#llx, offset=%#llx, flags=%u", - region->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI) ? vfio_pci_region_names[i] : "", - region->size, - region->offset, - region->flags - ); - } - - for (int i = 0; i < d->info.num_irqs; i++) { INDENT - struct vfio_irq_info *irq = &d->irqs[i]; - - if (irq->count > 0) - info("IRQ %u %s: count=%u, flags=%u", - irq->index, (d->info.flags & VFIO_DEVICE_FLAGS_PCI ) ? vfio_pci_irq_names[i] : "", - irq->count, - irq->flags - ); - } - } - } -} - -void * vfio_map_region(struct vfio_device *d, int idx) -{ - struct vfio_region_info *r = &d->regions[idx]; - - if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP)) - return MAP_FAILED; - - d->mappings[idx] = mmap(NULL, r->size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_32BIT, d->fd, r->offset); - - return d->mappings[idx]; -} - -size_t vfio_region_size(struct vfio_device *d, int idx) -{ - assert(d != NULL); - assert((size_t)idx < d->info.num_regions); - - return d->regions[idx].size; -} - -int vfio_unmap_region(struct vfio_device *d, int idx) -{ - int ret; - struct vfio_region_info *r = &d->regions[idx]; - - if (!d->mappings[idx]) - return -1; /* was not mapped */ - - debug(3, "VFIO: unmap region %u from device", idx); - - ret = munmap(d->mappings[idx], r->size); - if (ret) - return -2; - - d->mappings[idx] = NULL; - - return 0; -} - -int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) -{ - int ret; - - if (len & 0xFFF) { - len += 0x1000; - len &= ~0xFFF; - } - - /* Super stupid allocator */ - if (phys == -1) { - phys = c->iova_next; - c->iova_next += len; - } - - struct vfio_iommu_type1_dma_map dma_map = { - .argsz = sizeof(struct vfio_iommu_type1_dma_map), - .vaddr = virt, - .iova = phys, - .size = len, - .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE - }; - - ret = ioctl(c->fd, VFIO_IOMMU_MAP_DMA, &dma_map); - if (ret) - serror("Failed to create DMA mapping"); - - info("DMA map size=%#llx, iova=%#llx, vaddr=%#llx", dma_map.size, dma_map.iova, dma_map.vaddr); - - return 0; -} - -int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len) -{ - int ret; - - struct vfio_iommu_type1_dma_unmap dma_unmap = { - .argsz = sizeof(struct vfio_iommu_type1_dma_unmap), - .flags = 0, - .iova = phys, - .size = len, - }; - - ret = ioctl(c->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); - if (ret) - serror("Failed to unmap DMA mapping"); - - return 0; -} diff --git a/fpga/lib/kernel/vfio.cpp b/fpga/lib/kernel/vfio.cpp new file mode 100644 index 000000000..09464b1f8 --- /dev/null +++ b/fpga/lib/kernel/vfio.cpp @@ -0,0 +1,766 @@ +/** Virtual Function IO wrapper around kernel API + * + * @author Steffen Vogel + * @author Daniel Krebs + * @copyright 2017, Steffen Vogel + * @copyright 2018, Daniel Krebs + * @license GNU General Public License (version 3) + * + * VILLASfpga + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#define _DEFAULT_SOURCE + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "kernel/pci.h" +#include "kernel/kernel.h" + +#include "kernel/vfio.hpp" +#include "log.hpp" + +static auto logger = loggerGetOrCreate("Vfio"); + +static const char *vfio_pci_region_names[] = { + "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX, + "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX, + "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX, + "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX, + "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX, + "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX, + "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX, + "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX, + "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX, +}; + +static const char *vfio_pci_irq_names[] = { + "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX, + "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX, + "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX, + "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX, + "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX, +}; + +namespace villas { + + +VfioContainer::VfioContainer() + : iova_next(0) +{ + /* Load VFIO kernel module */ + if (kernel_module_load("vfio") != 0) { + logger->error("Failed to load kernel module: vfio"); + throw std::exception(); + } + + /* Open VFIO API */ + fd = open(VFIO_DEV("vfio"), O_RDWR); + if (fd < 0) { + logger->error("Failed to open VFIO container"); + throw std::exception(); + } + + /* Check VFIO API version */ + version = ioctl(fd, VFIO_GET_API_VERSION); + if (version < 0 || version != VFIO_API_VERSION) { + logger->error("Failed to get VFIO version"); + throw std::exception(); + } + + /* Check available VFIO extensions (IOMMU types) */ + extensions = 0; + for (int i = 1; i < VFIO_DMA_CC_IOMMU; i++) { + int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i); + if (ret < 0) { + logger->error("Failed to get VFIO extensions"); + throw std::exception(); + } + else if (ret > 0) + extensions |= (1 << i); + } + + logger->debug("Version: {:#x}", version); + logger->debug("Extensions: {:#x}", extensions); +} + + +VfioContainer::~VfioContainer() +{ + logger->debug("Clean up container with fd {}", this->fd); + + /* Release memory and close fds */ + groups.clear(); + + /* Close container */ + int ret = close(fd); + if (ret < 0) { + logger->error("Cannot close vfio container"); + } + + logger->debug("VFIO: closed container: fd={}", fd); +} + + +std::shared_ptr +VfioContainer::create() +{ + std::shared_ptr container { new VfioContainer }; + return container; +} + + +void +VfioContainer::dump() +{ + logger->info("File descriptor: {}", fd); + logger->info("Version: {}", version); + logger->info("Extensions: 0x{:x}", extensions); + + for(auto& group : groups) { + logger->info("VFIO Group {}, viable={}, container={}", + group->index, + (group->status.flags & VFIO_GROUP_FLAGS_VIABLE) > 0, + (group->status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) > 0 + ); + + for(auto& device : group->devices) { + logger->info("Device {}: regions={}, irqs={}, flags={}", + device->name, + device->info.num_regions, + device->info.num_irqs, + device->info.flags + ); + + for (size_t i = 0; i < device->info.num_regions && i < 8; i++) { + struct vfio_region_info *region = &device->regions[i]; + + if (region->size > 0) + logger->info("Region {} {}: size={}, offset={}, flags={}", + region->index, + (device->info.flags & VFIO_DEVICE_FLAGS_PCI) ? + vfio_pci_region_names[i] : "", + region->size, + region->offset, + region->flags + ); + } + + for (size_t i = 0; i < device->info.num_irqs; i++) { + struct vfio_irq_info *irq = &device->irqs[i]; + + if (irq->count > 0) + logger->info("IRQ {} {}: count={}, flags={}", + irq->index, + (device->info.flags & VFIO_DEVICE_FLAGS_PCI ) ? + vfio_pci_irq_names[i] : "", + irq->count, + irq->flags + ); + } + } + } +} + + +VfioDevice& +VfioContainer::attachDevice(const char* name, int index) +{ + VfioGroup& group = getOrAttachGroup(index); + auto device = std::make_unique(name, group); + + /* Open device fd */ + device->fd = ioctl(group.fd, VFIO_GROUP_GET_DEVICE_FD, name); + if (device->fd < 0) { + logger->error("Failed to open VFIO device: {}", device->name); + throw std::exception(); + } + + /* Get device info */ + device->info.argsz = sizeof(device->info); + + int ret = ioctl(device->fd, VFIO_DEVICE_GET_INFO, &device->info); + if (ret < 0) { + logger->error("Failed to get VFIO device info for: {}", device->name); + throw std::exception(); + } + + logger->debug("Device has {} regions", device->info.num_regions); + logger->debug("Device has {} IRQs", device->info.num_irqs); + + // reserve slots already so that we can use the []-operator for access + device->irqs.resize(device->info.num_irqs); + device->regions.resize(device->info.num_regions); + device->mappings.resize(device->info.num_regions); + + /* Get device regions */ + for (size_t i = 0; i < device->info.num_regions && i < 8; i++) { + struct vfio_region_info region; + memset(®ion, 0, sizeof (region)); + + region.argsz = sizeof(region); + region.index = i; + + ret = ioctl(device->fd, VFIO_DEVICE_GET_REGION_INFO, ®ion); + if (ret < 0) { + logger->error("Failed to get region of VFIO device: {}", device->name); + throw std::exception(); + } + + device->regions[i] = region; + } + + + /* Get device irqs */ + for (size_t i = 0; i < device->info.num_irqs; i++) { + struct vfio_irq_info irq; + memset(&irq, 0, sizeof (irq)); + + irq.argsz = sizeof(irq); + irq.index = i; + + ret = ioctl(device->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq); + if (ret < 0) { + logger->error("Failed to get IRQs of VFIO device: {}", device->name); + throw std::exception(); + } + + device->irqs[i] = irq; + } + + group.devices.push_back(std::move(device)); + + return *group.devices.back().get(); +} + + +VfioDevice& +VfioContainer::attachDevice(const pci_device* pdev) +{ + int ret; + char name[32]; + static constexpr char kernelDriver[] = "vfio-pci"; + + /* Load PCI bus driver for VFIO */ + if (kernel_module_load("vfio_pci")) { + logger->error("Failed to load kernel driver: vfio_pci"); + throw std::exception(); + } + + /* Bind PCI card to vfio-pci driver if not already bound */ + ret = pci_get_driver(pdev, name, sizeof(name)); + if (ret || strcmp(name, kernelDriver)) { + logger->debug("Bind PCI card to kernel driver '{}'", kernelDriver); + ret = pci_attach_driver(pdev, kernelDriver); + if (ret) { + logger->error("Failed to attach device to driver"); + throw std::exception(); + } + } + + /* Get IOMMU group of device */ + int index = pci_get_iommu_group(pdev); + if (index < 0) { + logger->error("Failed to get IOMMU group of device"); + throw std::exception(); + } + + /* VFIO device name consists of PCI BDF */ + snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, + pdev->slot.bus, pdev->slot.device, pdev->slot.function); + + logger->info("Attach to device {} with index {}", std::string(name), index); + auto& device = attachDevice(name, index); + + device.pci_device = pdev; + + /* Check if this is really a vfio-pci device */ + if(not device.isVfioPciDevice()) { + logger->error("Device is not a vfio-pci device"); + throw std::exception(); + } + + return device; +} + + +uintptr_t +VfioContainer::memoryMap(uintptr_t virt, uintptr_t phys, size_t length) +{ + int ret; + + if (length & 0xFFF) { + length += 0x1000; + length &= ~0xFFF; + } + + /* Super stupid allocator */ + size_t iovaIncrement = 0; + if (phys == UINTPTR_MAX) { + phys = this->iova_next; + iovaIncrement = length; + } + + struct vfio_iommu_type1_dma_map dmaMap; + memset(&dmaMap, 0, sizeof(dmaMap)); + + dmaMap.argsz = sizeof(dmaMap); + dmaMap.vaddr = virt; + dmaMap.iova = phys; + dmaMap.size = length; + dmaMap.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + ret = ioctl(this->fd, VFIO_IOMMU_MAP_DMA, &dmaMap); + if (ret) { + logger->error("Failed to create DMA mapping: {}", ret); + return UINTPTR_MAX; + } + + logger->info("DMA map size={:#x}, iova={:#x}, vaddr={:#x}", dmaMap.size, dmaMap.iova, dmaMap.vaddr); + + // mapping successful, advance IOVA allocator + this->iova_next += iovaIncrement; + + // we intentionally don't return the actual mapped length, the users are only + // guaranteed to have their demanded memory mapped correctly + return dmaMap.iova; +} + + +bool +VfioContainer::memoryUnmap(uintptr_t phys, size_t length) +{ + int ret; + + struct vfio_iommu_type1_dma_unmap dmaUnmap; + dmaUnmap.argsz = sizeof(struct vfio_iommu_type1_dma_unmap); + dmaUnmap.flags = 0; + dmaUnmap.iova = phys; + dmaUnmap.size = length; + + ret = ioctl(this->fd, VFIO_IOMMU_UNMAP_DMA, &dmaUnmap); + if (ret) { + logger->error("Failed to unmap DMA mapping"); + return false; + } + + return true; +} + + +VfioGroup& +VfioContainer::getOrAttachGroup(int index) +{ + // search if group with index already exists + for(auto& group : groups) { + if(group->index == index) { + return *group; + } + } + + // group not yet part of this container, so acquire ownership + auto group = VfioGroup::attach(fd, index); + if(not group) { + logger->error("Failed to attach to IOMMU group: {}", index); + throw std::exception(); + } else { + logger->info("Attached new group {} to VFIO container", index); + } + + // push to our list + groups.push_back(std::move(group)); + + return *groups.back(); +} + + +VfioDevice::~VfioDevice() +{ + logger->debug("clean up device {} with fd {}", this->name, this->fd); + + for(auto& region : regions) { + regionUnmap(region.index); + } + + int ret = close(fd); + if (ret != 0) { + logger->error("Closing device fd {} failed", fd); + } + + logger->debug("VFIO: closed device: name={}, fd={}", name, fd); +} + + +bool +VfioDevice::reset() +{ + if (this->info.flags & VFIO_DEVICE_FLAGS_RESET) + return ioctl(this->fd, VFIO_DEVICE_RESET) == 0; + else + return false; /* not supported by this device */ +} + + +void* +VfioDevice::regionMap(size_t index) +{ + struct vfio_region_info *r = ®ions[index]; + + if (!(r->flags & VFIO_REGION_INFO_FLAG_MMAP)) + return MAP_FAILED; + + mappings[index] = mmap(nullptr, r->size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_32BIT, + fd, r->offset); + + return mappings[index]; +} + + +bool +VfioDevice::regionUnmap(size_t index) +{ + int ret; + struct vfio_region_info *r = ®ions[index]; + + if (!mappings[index]) + return false; /* was not mapped */ + + logger->debug("VFIO: unmap region {} from device", index); + + ret = munmap(mappings[index], r->size); + if (ret) + return false; + + mappings[index] = nullptr; + + return true; +} + + +size_t +VfioDevice::regionGetSize(size_t index) +{ + if(index >= regions.size()) { + logger->error("Index out of range: {} >= {}", index, regions.size()); + throw std::out_of_range("Index out of range"); + } + + return regions[index].size; +} + + +bool +VfioDevice::pciEnable() +{ + int ret; + uint32_t reg; + off_t offset = ((off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40) + PCI_COMMAND; + + /* Check if this is really a vfio-pci device */ + if (!(this->info.flags & VFIO_DEVICE_FLAGS_PCI)) + return false; + + ret = pread(this->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return false; + + /* Enable memory access and PCI bus mastering which is required for DMA */ + reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + + ret = pwrite(this->fd, ®, sizeof(reg), offset); + if (ret != sizeof(reg)) + return false; + + return true; +} + + +bool +VfioDevice::pciHotReset() +{ + /* Check if this is really a vfio-pci device */ + if (not isVfioPciDevice()) + return false; + + const size_t reset_infolen = sizeof(struct vfio_pci_hot_reset_info) + + sizeof(struct vfio_pci_dependent_device) * 64; + auto reset_info = reinterpret_cast + (calloc(1, reset_infolen)); + + reset_info->argsz = reset_infolen; + + + if (ioctl(this->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info) != 0) { + free(reset_info); + return false; + } + + logger->debug("VFIO: dependent devices for hot-reset:"); + for (size_t i = 0; i < reset_info->count; i++) { + struct vfio_pci_dependent_device *dd = &reset_info->devices[i]; + logger->debug("{:04x}:{:02x}:{:02x}.{:01x}: iommu_group={}", + dd->segment, dd->bus, + PCI_SLOT(dd->devfn), PCI_FUNC(dd->devfn), dd->group_id); + + if (static_cast(dd->group_id) != this->group.index) { + free(reset_info); + return false; + } + } + + const size_t resetlen = sizeof(struct vfio_pci_hot_reset) + + sizeof(int32_t) * 1; + auto reset = reinterpret_cast + (calloc(1, resetlen)); + + reset->argsz = resetlen; + reset->count = 1; + reset->group_fds[0] = this->group.fd; + + const bool success = ioctl(this->fd, VFIO_DEVICE_PCI_HOT_RESET, reset) == 0; + + free(reset); + free(reset_info); + + return success; +} + + +int +VfioDevice::pciMsiInit(int efds[]) +{ + /* Check if this is really a vfio-pci device */ + if(not isVfioPciDevice()) + return -1; + + const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + const size_t irqSetSize = sizeof(struct vfio_irq_set) + + sizeof(int) * irqCount; + + auto irqSet = reinterpret_cast(calloc(1, irqSetSize)); + if(irqSet == nullptr) + return -1; + + irqSet->argsz = irqSetSize; + irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irqSet->index = VFIO_PCI_MSI_IRQ_INDEX; + irqSet->start = 0; + irqSet->count = irqCount; + + /* Now set the new eventfds */ + for (size_t i = 0; i < irqCount; i++) { + efds[i] = eventfd(0, 0); + if (efds[i] < 0) { + free(irqSet); + return -1; + } + } + + memcpy(irqSet->data, efds, sizeof(int) * irqCount); + + if(ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) { + free(irqSet); + return -1; + } + + free(irqSet); + + return irqCount; +} + + +int +VfioDevice::pciMsiDeinit(int efds[]) +{ + /* Check if this is really a vfio-pci device */ + if(not isVfioPciDevice()) + return -1; + + const size_t irqCount = irqs[VFIO_PCI_MSI_IRQ_INDEX].count; + const size_t irqSetSize = sizeof(struct vfio_irq_set) + + sizeof(int) * irqCount; + + auto irqSet = reinterpret_cast(calloc(1, irqSetSize)); + if(irqSet == nullptr) + return -1; + + irqSet->argsz = irqSetSize; + irqSet->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irqSet->index = VFIO_PCI_MSI_IRQ_INDEX; + irqSet->count = irqCount; + irqSet->start = 0; + + for (size_t i = 0; i < irqCount; i++) { + close(efds[i]); + efds[i] = -1; + } + + memcpy(irqSet->data, efds, sizeof(int) * irqCount); + + if (ioctl(fd, VFIO_DEVICE_SET_IRQS, irqSet) != 0) { + free(irqSet); + return -1; + } + + free(irqSet); + + return irqCount; +} + + +bool +VfioDevice::pciMsiFind(int nos[]) +{ + int ret, idx, irq; + char *end, *col, *last, line[1024], name[13]; + FILE *f; + + f = fopen("/proc/interrupts", "r"); + if (!f) + return false; + + for (int i = 0; i < 32; i++) + nos[i] = -1; + + /* For each line in /proc/interrupts */ + while (fgets(line, sizeof(line), f)) { + col = strtok(line, " "); + + /* IRQ number is in first column */ + irq = strtol(col, &end, 10); + if (col == end) + continue; + + /* Find last column of line */ + do { + last = col; + } while ((col = strtok(nullptr, " "))); + + + ret = sscanf(last, "vfio-msi[%u](%12[0-9:])", &idx, name); + if (ret == 2) { + if (strstr(this->name.c_str(), name) == this->name.c_str()) + nos[idx] = irq; + } + } + + fclose(f); + + return true; +} + + +bool +VfioDevice::isVfioPciDevice() const +{ + return info.flags & VFIO_DEVICE_FLAGS_PCI; +} + + +VfioGroup::~VfioGroup() +{ + logger->debug("clean up group {} with fd {}", this->index, this->fd); + int ret; + + /* Release memory and close fds */ + devices.clear(); + + if(fd < 0) { + logger->debug("Destructing group that has not been attached"); + } else { + ret = ioctl(fd, VFIO_GROUP_UNSET_CONTAINER); + if (ret != 0) { + logger->error("Cannot unset container for group fd {}", fd); + } + + logger->debug("Released group from container: group={}", index); + + ret = close(fd); + if (ret != 0) { + logger->error("Cannot close group fd {}", fd); + } + + logger->debug("Closed group: group={}, fd={}", index, fd); + } +} + + +std::unique_ptr +VfioGroup::attach(int containerFd, int groupIndex) +{ + std::unique_ptr group { new VfioGroup(groupIndex) }; + + /* Open group fd */ + std::stringstream groupPath; + groupPath << VFIO_DEV("") << groupIndex; + group->fd = open(groupPath.str().c_str(), O_RDWR); + if (group->fd < 0) { + logger->error("Failed to open VFIO group: {}", group->index); + return nullptr; + } + + logger->debug("VFIO group {} (fd {}) has path {}", + groupIndex, group->fd, groupPath.str()); + + int ret; + + /* Claim group ownership */ + ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &containerFd); + if (ret < 0) { + logger->error("Failed to attach VFIO group {} to container fd {} (error {})", + group->index, containerFd, ret); + return nullptr; + } + + /* Set IOMMU type */ + ret = ioctl(containerFd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret < 0) { + logger->error("Failed to set IOMMU type of container: {}", ret); + return nullptr; + } + + /* Check group viability and features */ + group->status.argsz = sizeof(group->status); + + ret = ioctl(group->fd, VFIO_GROUP_GET_STATUS, &group->status); + if (ret < 0) { + logger->error("Failed to get VFIO group status"); + return nullptr; + } + + if (!(group->status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + logger->error("VFIO group is not available: bind all devices to the VFIO driver!"); + return nullptr; + } + + return group; +} + +} // namespace villas + diff --git a/fpga/tests/fpga.cpp b/fpga/tests/fpga.cpp index 1f718ff51..fe9e79847 100644 --- a/fpga/tests/fpga.cpp +++ b/fpga/tests/fpga.cpp @@ -24,9 +24,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include "global.hpp" @@ -40,7 +40,6 @@ #define FPGA_AXI_HZ 125000000 static struct pci pci; -static struct vfio_container vc; FpgaState state; @@ -59,8 +58,7 @@ static void init() ret = pci_init(&pci); cr_assert_eq(ret, 0, "Failed to initialize PCI sub-system"); - ret = vfio_init(&vc); - cr_assert_eq(ret, 0, "Failed to initiliaze VFIO sub-system"); + auto vfioContainer = villas::VfioContainer::create(); /* Parse FPGA configuration */ f = fopen(TEST_CONFIG, "r"); @@ -81,7 +79,7 @@ static void init() villas::fpga::PCIeCardFactory* fpgaCardPlugin = dynamic_cast(plugin); // create all FPGA card instances using the corresponding plugin - state.cards = fpgaCardPlugin->make(fpgas, &pci, &vc); + state.cards = fpgaCardPlugin->make(fpgas, &pci, vfioContainer); cr_assert(state.cards.size() != 0, "No FPGA cards found!");