diff --git a/common/include/villas/kernel/vfio_container.hpp b/common/include/villas/kernel/vfio_container.hpp index 06dfc333e..14d383fe8 100644 --- a/common/include/villas/kernel/vfio_container.hpp +++ b/common/include/villas/kernel/vfio_container.hpp @@ -26,16 +26,18 @@ namespace vfio { class Container { private: - //This is a singleton: There can only be one container to rule them all. + // This is a singleton: There can only be one container to rule them all. Container(); + public: - //The Container instance is lazily initialized and correctly destroyed. + // The Container instance is lazily initialized and correctly destroyed. static Container* getInstance() { static Container instance; return &instance; }; - //No copying allowed + + // No copying allowed Container(Container const&) = delete; void operator=(Container const&) = delete; @@ -71,7 +73,7 @@ private: uint64_t iova_next; /**< Next free IOVA address */ bool hasIommu; - /// All groups bound to this container + // All groups bound to this container std::list> groups; Logger log; diff --git a/common/include/villas/kernel/vfio_device.hpp b/common/include/villas/kernel/vfio_device.hpp index fcacf1576..dc1a87eac 100644 --- a/common/include/villas/kernel/vfio_device.hpp +++ b/common/include/villas/kernel/vfio_device.hpp @@ -41,17 +41,16 @@ public: bool reset(); - /** Map a device memory region to the application address space (e.g. PCI BARs) */ + // Map a device memory region to the application address space (e.g. PCI BARs) void* regionMap(size_t index); - /** munmap() a region which has been mapped by vfio_map_region() */ + // munmap() a region which has been mapped by vfio_map_region() bool regionUnmap(size_t index); - /** Get the size of a device memory region */ + // Get the size of a device memory region size_t regionGetSize(size_t index); - - /** Enable memory accesses and bus mastering for PCI device */ + // Enable memory accesses and bus mastering for PCI device bool pciEnable(); int pciMsiInit(int efds[32]); @@ -73,11 +72,11 @@ public: { this->attachedToGroup = true; } private: - /// Name of the device as listed under - /// /sys/kernel/iommu_groups/[vfio_group::index]/devices/ + // Name of the device as listed under + // /sys/kernel/iommu_groups/[vfio_group::index]/devices/ std::string name; - /// VFIO device file descriptor + // VFIO device file descriptor int fd; bool attachedToGroup; @@ -89,7 +88,7 @@ private: std::vector regions; std::vector mappings; - /**< libpci handle of the device */ + // libpci handle of the device const kernel::pci::Device *pci_device; Logger log; diff --git a/common/include/villas/kernel/vfio_group.hpp b/common/include/villas/kernel/vfio_group.hpp index 25d28fadd..0c991a890 100644 --- a/common/include/villas/kernel/vfio_group.hpp +++ b/common/include/villas/kernel/vfio_group.hpp @@ -48,19 +48,20 @@ public: bool checkStatus(); void dump(); + private: - /// VFIO group file descriptor + // VFIO group file descriptor int fd; - /// Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ + // Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ int index; bool attachedToContainer; - /// Status of group + // Status of group struct vfio_group_status status; - /// All devices owned by this group + // All devices owned by this group std::list> devices; Logger log; diff --git a/common/lib/kernel/pci.cpp b/common/lib/kernel/pci.cpp index fcdd34dab..e14edad54 100644 --- a/common/lib/kernel/pci.cpp +++ b/common/lib/kernel/pci.cpp @@ -377,7 +377,7 @@ Device::getIOMMUGroup() const { int ret; char *group; - //readlink does not add a null terminator! + // readlink does not add a null terminator! char link[1024] = {0}; char sysfs[1024]; diff --git a/common/lib/kernel/vfio_container.cpp b/common/lib/kernel/vfio_container.cpp index 6c301eb77..9320dbec6 100644 --- a/common/lib/kernel/vfio_container.cpp +++ b/common/lib/kernel/vfio_container.cpp @@ -56,17 +56,17 @@ Container::Container() : "Please load manually!", module); } - /* Open VFIO API */ + // Open VFIO API fd = open(VFIO_DEV, O_RDWR); if (fd < 0) throw RuntimeError("Failed to open VFIO container"); - /* Check VFIO API version */ + // Check VFIO API version version = ioctl(fd, VFIO_GET_API_VERSION); if (version < 0 || version != VFIO_API_VERSION) throw RuntimeError("Failed to get VFIO version"); - /* Check available VFIO extensions (IOMMU types) */ + // Check available VFIO extensions (IOMMU types) extensions = 0; for (unsigned int i = VFIO_TYPE1_IOMMU; i <= VFIO_NOIOMMU_IOMMU; i++) { int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i); @@ -93,11 +93,12 @@ Container::Container() : Container::~Container() { - /* Release memory and close fds */ + // Release memory and close fds groups.clear(); log->debug("Cleaning up container with fd {}", fd); - /* Close container */ + + // Close container int ret = close(fd); if (ret < 0) log->error("Error closing vfio container fd {}: {}", fd, ret); @@ -108,7 +109,7 @@ void Container::attachGroup(std::shared_ptr group) if (group->isAttachedToContainer()) throw RuntimeError("Group is already attached to a container"); - /* Claim group ownership */ + // Claim group ownership int ret = ioctl(group->getFileDescriptor(), VFIO_GROUP_SET_CONTAINER, &fd); if (ret < 0) { log->error("Failed to attach VFIO group {} to container fd {} (error {})", @@ -116,7 +117,7 @@ void Container::attachGroup(std::shared_ptr group) throw RuntimeError("Failed to attach VFIO group to container"); } - /* Set IOMMU type */ + // Set IOMMU type int iommu_type = isIommuEnabled() ? VFIO_TYPE1_IOMMU : VFIO_NOIOMMU_IOMMU; ret = ioctl(fd, VFIO_SET_IOMMU, iommu_type); @@ -132,20 +133,20 @@ void Container::attachGroup(std::shared_ptr group) else log->debug("Attached new group {} to VFIO container", group->getIndex()); - /* Push to our list */ + // Push to our list groups.push_back(std::move(group)); } std::shared_ptr Container::getOrAttachGroup(int index) { - /* Search if group with index already exists */ + // Search if group with index already exists for (auto &group : groups) { if (group->getIndex() == index) { return group; } } - /* Group not yet part of this container, so acquire ownership */ + // Group not yet part of this container, so acquire ownership auto group = std::make_shared(index, isIommuEnabled()); attachGroup(group); @@ -185,17 +186,17 @@ Container::attachDevice(const pci::Device &pdev) char name[32], iommu_state[4]; static constexpr const char* kernelDriver = "vfio-pci"; - /* Load PCI bus driver for VFIO */ + // Load PCI bus driver for VFIO if (kernel::loadModule("vfio_pci")) throw RuntimeError("Failed to load kernel driver: vfio_pci"); - /* Bind PCI card to vfio-pci driver if not already bound */ + // Bind PCI card to vfio-pci driver if not already bound if (pdev.getDriver() != kernelDriver) { log->debug("Bind PCI card to kernel driver '{}'", kernelDriver); pdev.attachDriver(kernelDriver); } - /* Get IOMMU group of device */ + // Get IOMMU group of device int index = isIommuEnabled() ? pdev.getIOMMUGroup() : 0; if (index < 0) { ret = kernel::getCmdlineParam("intel_iommu", iommu_state, sizeof(iommu_state)); @@ -208,7 +209,7 @@ Container::attachDevice(const pci::Device &pdev) throw RuntimeError("Failed to get IOMMU group of device"); } - /* VFIO device name consists of PCI BDF */ + // VFIO device name consists of PCI BDF snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev.slot.domain, pdev.slot.bus, pdev.slot.device, pdev.slot.function); @@ -217,7 +218,7 @@ Container::attachDevice(const pci::Device &pdev) auto device = group->attachDevice(name, &pdev); - /* Check if this is really a vfio-pci device */ + // Check if this is really a vfio-pci device if (!device->isVfioPciDevice()) throw RuntimeError("Device is not a vfio-pci device"); @@ -240,7 +241,7 @@ Container::memoryMap(uintptr_t virt, uintptr_t phys, size_t length) length &= ~0xFFF; } - /* Super stupid allocator */ + // Super stupid allocator size_t iovaIncrement = 0; if (phys == UINTPTR_MAX) { phys = this->iova_next; @@ -265,12 +266,11 @@ Container::memoryMap(uintptr_t virt, uintptr_t phys, size_t length) log->debug("DMA map size={:#x}, iova={:#x}, vaddr={:#x}", dmaMap.size, dmaMap.iova, dmaMap.vaddr); - /* Mapping successful, advance IOVA allocator */ + // Mapping successful, advance IOVA allocator this->iova_next += iovaIncrement; - /* We intentionally don't return the actual mapped length, the users are - * only guaranteed to have their demanded memory mapped correctly - */ + // We intentionally don't return the actual mapped length, the users are + // only guaranteed to have their demanded memory mapped correctly return dmaMap.iova; } diff --git a/common/lib/kernel/vfio_device.cpp b/common/lib/kernel/vfio_device.cpp index f4aa6a839..4f251c86e 100644 --- a/common/lib/kernel/vfio_device.cpp +++ b/common/lib/kernel/vfio_device.cpp @@ -33,23 +33,23 @@ using namespace villas::kernel::vfio; static const char *vfio_pci_region_names[] = { - "PCI_BAR0", /* VFIO_PCI_BAR0_REGION_INDEX */ - "PCI_BAR1", /* VFIO_PCI_BAR1_REGION_INDEX */ - "PCI_BAR2", /* VFIO_PCI_BAR2_REGION_INDEX */ - "PCI_BAR3", /* VFIO_PCI_BAR3_REGION_INDEX */ - "PCI_BAR4", /* VFIO_PCI_BAR4_REGION_INDEX */ - "PCI_BAR5", /* VFIO_PCI_BAR5_REGION_INDEX */ - "PCI_ROM", /* VFIO_PCI_ROM_REGION_INDEX */ - "PCI_CONFIG", /* VFIO_PCI_CONFIG_REGION_INDEX */ - "PCI_VGA" /* VFIO_PCI_INTX_IRQ_INDEX */ + "PCI_BAR0", // VFIO_PCI_BAR0_REGION_INDEX + "PCI_BAR1", // VFIO_PCI_BAR1_REGION_INDEX + "PCI_BAR2", // VFIO_PCI_BAR2_REGION_INDEX + "PCI_BAR3", // VFIO_PCI_BAR3_REGION_INDEX + "PCI_BAR4", // VFIO_PCI_BAR4_REGION_INDEX + "PCI_BAR5", // VFIO_PCI_BAR5_REGION_INDEX + "PCI_ROM", // VFIO_PCI_ROM_REGION_INDEX + "PCI_CONFIG", // VFIO_PCI_CONFIG_REGION_INDEX + "PCI_VGA" // VFIO_PCI_INTX_IRQ_INDEX }; static const char *vfio_pci_irq_names[] = { - "PCI_INTX", /* VFIO_PCI_INTX_IRQ_INDEX */ - "PCI_MSI", /* VFIO_PCI_MSI_IRQ_INDEX */ - "PCI_MSIX", /* VFIO_PCI_MSIX_IRQ_INDEX */ - "PCI_ERR", /* VFIO_PCI_ERR_IRQ_INDEX */ - "PCI_REQ" /* VFIO_PCI_REQ_IRQ_INDEX */ + "PCI_INTX", // VFIO_PCI_INTX_IRQ_INDEX + "PCI_MSI", // VFIO_PCI_MSI_IRQ_INDEX + "PCI_MSIX", // VFIO_PCI_MSIX_IRQ_INDEX + "PCI_ERR", // VFIO_PCI_ERR_IRQ_INDEX + "PCI_REQ" // VFIO_PCI_REQ_IRQ_INDEX }; Device::Device(const std::string &name, int groupFileDescriptor, const kernel::pci::Device *pci_device) : @@ -67,12 +67,12 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p if (groupFileDescriptor < 0) throw RuntimeError("Invalid group file descriptor"); - /* Open device fd */ + // Open device fd fd = ioctl(groupFileDescriptor, VFIO_GROUP_GET_DEVICE_FD, name.c_str()); if (fd < 0) throw RuntimeError("Failed to open VFIO device: {}", name.c_str()); - /* Get device info */ + // Get device info info.argsz = sizeof(info); int ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &info); @@ -82,12 +82,12 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p log->debug("Device has {} regions", info.num_regions); log->debug("Device has {} IRQs", info.num_irqs); - /* Reserve slots already so that we can use the []-operator for access */ + // Reserve slots already so that we can use the []-operator for access irqs.resize(info.num_irqs); regions.resize(info.num_regions); mappings.resize(info.num_regions); - /* Get device regions */ + // Get device regions for (size_t i = 0; i < info.num_regions && i < 8; i++) { struct vfio_region_info region; memset(®ion, 0, sizeof (region)); @@ -103,7 +103,7 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p } - /* Get device irqs */ + // Get device IRQs for (size_t i = 0; i < info.num_irqs; i++) { struct vfio_irq_info irq; memset(&irq, 0, sizeof (irq)); @@ -146,7 +146,7 @@ Device::reset() if (this->info.flags & VFIO_DEVICE_FLAGS_RESET) return ioctl(this->fd, VFIO_DEVICE_RESET) == 0; else - return false; /* not supported by this device */ + return false; // not supported by this device } @@ -179,7 +179,7 @@ Device::regionUnmap(size_t index) struct vfio_region_info *r = ®ions[index]; if (!mappings[index]) - return false; /* was not mapped */ + return false; // Was not mapped log->debug("Unmap region {} from device {}", index, name); @@ -251,7 +251,7 @@ Device::pciEnable() const off64_t offset = PCI_COMMAND + (static_cast(VFIO_PCI_CONFIG_REGION_INDEX) << 40); - /* Check if this is really a vfio-pci device */ + // Check if this is really a vfio-pci device if (!(this->info.flags & VFIO_DEVICE_FLAGS_PCI)) return false; @@ -259,7 +259,7 @@ Device::pciEnable() if (ret != sizeof(reg)) return false; - /* Enable memory access and PCI bus mastering which is required for DMA */ + // Enable memory access and PCI bus mastering which is required for DMA reg |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; ret = pwrite64(this->fd, ®, sizeof(reg), offset); @@ -273,7 +273,7 @@ Device::pciEnable() int Device::pciMsiInit(int efds[]) { - /* Check if this is really a vfio-pci device */ + // Check if this is really a vfio-pci device if (not isVfioPciDevice()) return -1; @@ -295,7 +295,7 @@ Device::pciMsiInit(int efds[]) irqSet->start = 0; irqSet->count = irqCount; - /* Now set the new eventfds */ + // Now set the new eventfds for (size_t i = 0; i < irqCount; i++) { efds[i] = eventfd(0, 0); if (efds[i] < 0) { @@ -320,7 +320,7 @@ Device::pciMsiInit(int efds[]) int Device::pciMsiDeinit(int efds[]) { - /* Check if this is really a vfio-pci device */ + // Check if this is really a vfio-pci device if (not isVfioPciDevice()) return -1; @@ -372,16 +372,16 @@ Device::pciMsiFind(int nos[]) for (int i = 0; i < 32; i++) nos[i] = -1; - /* For each line in /proc/interrupts */ + // For each line in /proc/interrupts while (fgets(line, sizeof(line), f)) { col = strtok(line, " "); - /* IRQ number is in first column */ + // IRQ number is in first column irq = strtol(col, &end, 10); if (col == end) continue; - /* Find last column of line */ + // Find last column of line do { last = col; } while ((col = strtok(nullptr, " "))); @@ -408,7 +408,7 @@ Device::isVfioPciDevice() const bool Device::pciHotReset() { - /* Check if this is really a vfio-pci device */ + // Check if this is really a vfio-pci device if (!isVfioPciDevice()) return false; diff --git a/common/lib/kernel/vfio_group.cpp b/common/lib/kernel/vfio_group.cpp index b17b4a25e..b51cd2ab8 100644 --- a/common/lib/kernel/vfio_group.cpp +++ b/common/lib/kernel/vfio_group.cpp @@ -42,7 +42,7 @@ Group::Group(int index, bool iommuEnabled) : devices(), log(logging.get("kernel:vfio::Group")) { - /* Open group fd */ + // Open group fd std::stringstream groupPath; groupPath << VFIO_PATH << (iommuEnabled ? "" : "noiommu-") @@ -87,7 +87,7 @@ bool Group::checkStatus() return false; } - /* Check group viability and features */ + // Check group viability and features status.argsz = sizeof(status); ret = ioctl(fd, VFIO_GROUP_GET_STATUS, &status); @@ -118,7 +118,7 @@ void Group::dump() Group::~Group() { - /* Release memory and close fds */ + // Release memory and close fds devices.clear(); log->debug("Cleaning up group {} with fd {}", index, fd);